turbine_rb 0.1.3 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -1
- data/Gemfile +2 -0
- data/Gemfile.lock +6 -4
- data/lib/turbine_pb.rb +3 -0
- data/lib/turbine_rb/client.rb +41 -21
- data/lib/turbine_rb/version.rb +1 -1
- data/lib/turbine_rb.rb +2 -2
- metadata +5 -10
- data/lib/templates/app/Gemfile +0 -5
- data/lib/templates/app/app.json +0 -8
- data/lib/templates/app/app.rb +0 -67
- data/lib/templates/app/fixtures/demo.json +0 -9
- data/lib/templates/app/ignoregit +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 62316927d652aef0b27ec9a324d2341d81d4a2181f6ddd496a5ffae24c13abdd
|
4
|
+
data.tar.gz: f875f7ebec9024cae2858c6d70bde6123070727d1176d85ba3e312956126c846
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 31005fa778f97bc5bcbd8560f418540f906f6a10d44a08121a64691e9c0e764cb369fcf5315f7222420f3fc0760e48a846c696f752befa493094793d5cd17bab
|
7
|
+
data.tar.gz: ead41b4f99a0fc52226568149dee48b338267b295a6e57ca9a89cbd123fb57da2069fa8fad6b84bb3a8fe1ce586954ae7812699612c86f677ea6a4780334f9f2
|
data/.rubocop.yml
CHANGED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
turbine_rb (0.1
|
4
|
+
turbine_rb (0.2.1)
|
5
5
|
grpc (~> 1.48)
|
6
6
|
hash_dot (~> 2.5.0)
|
7
7
|
|
@@ -9,13 +9,14 @@ GEM
|
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
11
|
ast (2.4.2)
|
12
|
+
bump (0.10.0)
|
12
13
|
coderay (1.1.3)
|
13
14
|
diff-lcs (1.5.0)
|
14
15
|
ffi (1.15.5)
|
15
16
|
formatador (1.1.0)
|
16
|
-
google-protobuf (3.21.
|
17
|
-
google-protobuf (3.21.
|
18
|
-
google-protobuf (3.21.
|
17
|
+
google-protobuf (3.21.11)
|
18
|
+
google-protobuf (3.21.11-x86_64-darwin)
|
19
|
+
google-protobuf (3.21.11-x86_64-linux)
|
19
20
|
googleapis-common-protos-types (1.4.0)
|
20
21
|
google-protobuf (~> 3.14)
|
21
22
|
grpc (1.50.0)
|
@@ -107,6 +108,7 @@ PLATFORMS
|
|
107
108
|
x86_64-linux
|
108
109
|
|
109
110
|
DEPENDENCIES
|
111
|
+
bump (~> 0.10.0)
|
110
112
|
guard-rspec
|
111
113
|
mocktail
|
112
114
|
pry
|
data/lib/turbine_pb.rb
CHANGED
@@ -22,6 +22,9 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
|
|
22
22
|
end
|
23
23
|
add_message "turbine_core.Resource" do
|
24
24
|
optional :name, :string, 1
|
25
|
+
optional :source, :bool, 2
|
26
|
+
optional :destination, :bool, 3
|
27
|
+
optional :collection, :string, 4
|
25
28
|
end
|
26
29
|
add_message "turbine_core.Collection" do
|
27
30
|
optional :name, :string, 1
|
data/lib/turbine_rb/client.rb
CHANGED
@@ -7,9 +7,13 @@ module TurbineRb
|
|
7
7
|
class App
|
8
8
|
attr_reader :core_server
|
9
9
|
|
10
|
-
def initialize(grpc_server,
|
10
|
+
def initialize(grpc_server, recording:)
|
11
11
|
@core_server = grpc_server
|
12
|
-
@
|
12
|
+
@recording = recording
|
13
|
+
end
|
14
|
+
|
15
|
+
def recording?
|
16
|
+
@recording
|
13
17
|
end
|
14
18
|
|
15
19
|
def resource(name:)
|
@@ -19,33 +23,38 @@ module TurbineRb
|
|
19
23
|
end
|
20
24
|
|
21
25
|
def process(records:, process:)
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
+
pb_collection = core_server.add_process_to_collection(
|
27
|
+
TurbineCore::ProcessCollectionRequest.new(
|
28
|
+
collection: Collection.unwrap(records),
|
29
|
+
process: TurbineCore::ProcessCollectionRequest::Process.new(name: process.class.name)
|
30
|
+
)
|
26
31
|
)
|
32
|
+
records.tap do |r|
|
33
|
+
r.pb_collection = process_call(process: process, pb_collection: pb_collection)
|
34
|
+
r.pb_stream = pb_collection.stream
|
35
|
+
end
|
36
|
+
end
|
27
37
|
|
28
|
-
|
29
|
-
|
30
|
-
records_interface = TurbineRb::Records.new(unwrapped_records.records)
|
31
|
-
processed_records = process.call(records: records_interface) unless @is_recording
|
32
|
-
records.pb_collection = processed_records.map(&:serialize_core_record) unless @is_recording
|
38
|
+
def process_call(process:, pb_collection:)
|
39
|
+
return pb_collection if recording?
|
33
40
|
|
34
|
-
|
41
|
+
process
|
42
|
+
.call(records: TurbineRb::Records.new(pb_collection.records))
|
43
|
+
.map(&:serialize_core_record)
|
35
44
|
end
|
36
45
|
|
37
46
|
# register_secrets accepts either a single string or an array of strings
|
38
47
|
def register_secrets(secrets)
|
39
|
-
[
|
48
|
+
[secrets].flatten.map do |secret|
|
40
49
|
raise MissingSecretError, "secret #{secret} is not an environment variable" unless ENV.key?(secret)
|
41
50
|
|
42
51
|
req = TurbineCore::Secret.new(name: secret, value: ENV[secret])
|
43
|
-
|
52
|
+
core_server.register_secret(req)
|
44
53
|
end
|
45
54
|
end
|
46
55
|
|
47
56
|
class Resource
|
48
|
-
attr_reader :pb_resource
|
57
|
+
attr_reader :pb_resource, :app
|
49
58
|
|
50
59
|
def initialize(res, app)
|
51
60
|
@pb_resource = res
|
@@ -59,7 +68,9 @@ module TurbineRb
|
|
59
68
|
req.configs = TurbineCore::Configs.new(config: pb_configs)
|
60
69
|
end
|
61
70
|
|
62
|
-
|
71
|
+
app.core_server
|
72
|
+
.read_collection(req)
|
73
|
+
.wrap(app) # wrap in Collection to enable chaining
|
63
74
|
end
|
64
75
|
|
65
76
|
def write(records:, collection:, configs: nil)
|
@@ -67,20 +78,29 @@ module TurbineRb
|
|
67
78
|
records = records.unwrap
|
68
79
|
end
|
69
80
|
|
70
|
-
req = TurbineCore::WriteCollectionRequest.new(
|
71
|
-
|
81
|
+
req = TurbineCore::WriteCollectionRequest.new(
|
82
|
+
resource: @pb_resource,
|
83
|
+
sourceCollection: records,
|
84
|
+
targetCollection: collection
|
85
|
+
)
|
72
86
|
|
73
87
|
if configs
|
74
88
|
pb_configs = configs.keys.map { |key| TurbineCore::Config.new(field: key, value: configs[key]) }
|
75
89
|
req.configs = TurbineCore::Configs.new(config: pb_configs)
|
76
90
|
end
|
77
91
|
|
78
|
-
|
92
|
+
app.core_server.write_collection_to_resource(req)
|
79
93
|
end
|
80
94
|
end
|
81
95
|
|
82
96
|
class Collection
|
83
|
-
attr_accessor :pb_collection, :pb_stream, :name
|
97
|
+
attr_accessor :pb_collection, :pb_stream, :name, :app
|
98
|
+
|
99
|
+
def self.unwrap(collection)
|
100
|
+
return collection.unwrap if collection.instance_of?(Collection)
|
101
|
+
|
102
|
+
collection
|
103
|
+
end
|
84
104
|
|
85
105
|
def initialize(name, collection, stream, app)
|
86
106
|
@name = name
|
@@ -94,7 +114,7 @@ module TurbineRb
|
|
94
114
|
end
|
95
115
|
|
96
116
|
def process_with(process:)
|
97
|
-
|
117
|
+
app.process(records: self, process: process)
|
98
118
|
end
|
99
119
|
|
100
120
|
def unwrap
|
data/lib/turbine_rb/version.rb
CHANGED
data/lib/turbine_rb.rb
CHANGED
@@ -43,12 +43,12 @@ module TurbineRb
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def run
|
46
|
-
app = TurbineRb::Client::App.new(init_core_server)
|
46
|
+
app = TurbineRb::Client::App.new(init_core_server, recording: false)
|
47
47
|
TurbineRb.app.call(app)
|
48
48
|
end
|
49
49
|
|
50
50
|
def record
|
51
|
-
app = TurbineRb::Client::App.new(init_core_server,
|
51
|
+
app = TurbineRb::Client::App.new(init_core_server, recording: true)
|
52
52
|
TurbineRb.app.call(app)
|
53
53
|
end
|
54
54
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: turbine_rb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Meroxa
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-12-
|
11
|
+
date: 2022-12-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: grpc
|
@@ -69,11 +69,6 @@ files:
|
|
69
69
|
- lib/service_pb.rb
|
70
70
|
- lib/service_services_pb.rb
|
71
71
|
- lib/templates/Dockerfile
|
72
|
-
- lib/templates/app/Gemfile
|
73
|
-
- lib/templates/app/app.json
|
74
|
-
- lib/templates/app/app.rb
|
75
|
-
- lib/templates/app/fixtures/demo.json
|
76
|
-
- lib/templates/app/ignoregit
|
77
72
|
- lib/turbine_pb.rb
|
78
73
|
- lib/turbine_rb.rb
|
79
74
|
- lib/turbine_rb/client.rb
|
@@ -90,7 +85,7 @@ metadata:
|
|
90
85
|
homepage_uri: https://github.com/meroxa/turbine-core/tree/main/lib/ruby/turbine_rb
|
91
86
|
source_code_uri: https://github.com/meroxa/turbine-core
|
92
87
|
changelog_uri: https://github.com/meroxa/turbine-core
|
93
|
-
post_install_message:
|
88
|
+
post_install_message:
|
94
89
|
rdoc_options: []
|
95
90
|
require_paths:
|
96
91
|
- lib
|
@@ -106,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
106
101
|
version: '0'
|
107
102
|
requirements: []
|
108
103
|
rubygems_version: 3.3.7
|
109
|
-
signing_key:
|
104
|
+
signing_key:
|
110
105
|
specification_version: 4
|
111
106
|
summary: Meroxa data application framework for Ruby
|
112
107
|
test_files: []
|
data/lib/templates/app/Gemfile
DELETED
data/lib/templates/app/app.json
DELETED
data/lib/templates/app/app.rb
DELETED
@@ -1,67 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "rubygems"
|
4
|
-
require "bundler/setup"
|
5
|
-
require "turbine_rb"
|
6
|
-
|
7
|
-
class MyApp
|
8
|
-
def call(app)
|
9
|
-
# To configure resources for your production datastores
|
10
|
-
# on Meroxa, use the Dashboard, CLI, or Terraform Provider
|
11
|
-
# For more details refer to: http://docs.meroxa.com/
|
12
|
-
#
|
13
|
-
# Identify the upstream datastore with the `resource` function
|
14
|
-
# Replace `demopg` with the resource name configured on Meroxa
|
15
|
-
database = app.resource(name: "demopg")
|
16
|
-
|
17
|
-
# Specify which upstream records to pull
|
18
|
-
# with the `records` function
|
19
|
-
# Replace `collection_name` with a table, collection,
|
20
|
-
# or bucket name in your data store.
|
21
|
-
# If a configuration is needed for your source,
|
22
|
-
# you can pass it as a second argument to the `records` function. For example:
|
23
|
-
# database.records(collection: "collection_name", configs: {"incrementing.column.name" => "id"})
|
24
|
-
records = database.records(collection: "collection_name")
|
25
|
-
|
26
|
-
# Register secrets to be available in the function:
|
27
|
-
# app.register_secrets("MY_ENV_TEST")
|
28
|
-
|
29
|
-
# Register several secrets at once:
|
30
|
-
# app.register_secrets(["MY_ENV_TEST", "MY_OTHER_ENV_TEST"])
|
31
|
-
|
32
|
-
# Specify the code to execute against `records` with the `process` function.
|
33
|
-
# Replace `Passthrough` with your desired function.
|
34
|
-
# Ensure desired function matches `Passthrough`'s' function signature.
|
35
|
-
processed_records = app.process(records: records, process: Passthrough.new)
|
36
|
-
|
37
|
-
# Specify where to write records using the `write` function.
|
38
|
-
# Replace `collection_archive` with whatever data organisation method
|
39
|
-
# is relevant to the datastore (e.g., table, bucket, collection, etc.)
|
40
|
-
# If additional connector configs are needed, provided another argument. For example:
|
41
|
-
# database.write(
|
42
|
-
# records: processed_records,
|
43
|
-
# collection: "collection_archive",
|
44
|
-
# configs: {"behavior.on.null.values": "ignore"})
|
45
|
-
database.write(records: processed_records, collection: "collection_archive")
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
class Passthrough < TurbineRb::Process
|
50
|
-
def call(records:)
|
51
|
-
puts "got records: #{records}"
|
52
|
-
# To get the value of unformatted records, use record .value getter method
|
53
|
-
# records.map { |r| puts r.value }
|
54
|
-
#
|
55
|
-
# To transform unformatted records, use record .value setter method
|
56
|
-
# records.map { |r| r.value = "newdata" }
|
57
|
-
#
|
58
|
-
# To get the value of json formatted records, use record .get method
|
59
|
-
# records.map { |r| puts r.get("message") }
|
60
|
-
#
|
61
|
-
# To transform json formatted records, use record .set methods
|
62
|
-
# records.map { |r| r.set('message', 'goodbye') }
|
63
|
-
records
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
TurbineRb.register(MyApp.new)
|
data/lib/templates/app/ignoregit
DELETED
File without changes
|