turbine_rb 0.1.2 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0adaff81024cca99dc5d4b043e98e3fe6a69f29d5d9ee2fd312c66ede6840f4c
-  data.tar.gz: c702105c5d3b5672bab440ac16beef7ff0c4497147b66cee1f9dad2cce31bbf6
+  metadata.gz: 62316927d652aef0b27ec9a324d2341d81d4a2181f6ddd496a5ffae24c13abdd
+  data.tar.gz: f875f7ebec9024cae2858c6d70bde6123070727d1176d85ba3e312956126c846
 SHA512:
-  metadata.gz: 5ea6a41d9686349f4f02d90ac74b653a08ddf693feb12453e7a056a077982ae8dbdec792209acc2e24263e5e576762266a8d37ecebc0329815e028b26fdd5cc1
-  data.tar.gz: 3d473b1e102a17b9b247977bbccb5617bb4a7d861e6584035e00ab406032b83e31363ea769d525c15faccd2ce573cabd82ed2473c267ee770c164905aa50a31b
+  metadata.gz: 31005fa778f97bc5bcbd8560f418540f906f6a10d44a08121a64691e9c0e764cb369fcf5315f7222420f3fc0760e48a846c696f752befa493094793d5cd17bab
+  data.tar.gz: ead41b4f99a0fc52226568149dee48b338267b295a6e57ca9a89cbd123fb57da2069fa8fad6b84bb3a8fe1ce586954ae7812699612c86f677ea6a4780334f9f2

data/.rubocop.yml

@@ -35,4 +35,7 @@ RSpec/ExampleLength:
 
 # Enable this after refactoring spec/
 RSpec/MultipleDescribes:
-  Enabled: false
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Max: 10

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## [0.1.3] - 2022-12-03
+
+- fix: set new keys on unformatted json
+
 ## [0.1.2] - 2022-12-02
 
 - fix: add record interface to local run

data/Gemfile

@@ -19,3 +19,5 @@ group :test do
   gem "mocktail"
   gem "rspec", "~> 3.0"
 end
+
+gem "bump", "~> 0.10.0"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    turbine_rb (0.1.0)
+    turbine_rb (0.2.1)
       grpc (~> 1.48)
       hash_dot (~> 2.5.0)
 
@@ -9,13 +9,14 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    bump (0.10.0)
     coderay (1.1.3)
     diff-lcs (1.5.0)
     ffi (1.15.5)
     formatador (1.1.0)
-    google-protobuf (3.21.9)
-    google-protobuf (3.21.9-x86_64-darwin)
-    google-protobuf (3.21.9-x86_64-linux)
+    google-protobuf (3.21.11)
+    google-protobuf (3.21.11-x86_64-darwin)
+    google-protobuf (3.21.11-x86_64-linux)
     googleapis-common-protos-types (1.4.0)
       google-protobuf (~> 3.14)
     grpc (1.50.0)
@@ -107,6 +108,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  bump (~> 0.10.0)
   guard-rspec
   mocktail
   pry

data/lib/turbine_pb.rb

@@ -22,6 +22,9 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
     end
     add_message "turbine_core.Resource" do
       optional :name, :string, 1
+      optional :source, :bool, 2
+      optional :destination, :bool, 3
+      optional :collection, :string, 4
     end
     add_message "turbine_core.Collection" do
       optional :name, :string, 1

data/lib/turbine_rb/client.rb

@@ -7,9 +7,13 @@ module TurbineRb
     class App
       attr_reader :core_server
 
-      def initialize(grpc_server, is_recording: false)
+      def initialize(grpc_server, recording:)
         @core_server = grpc_server
-        @is_recording = is_recording
+        @recording = recording
+      end
+
+      def recording?
+        @recording
       end
 
       def resource(name:)
@@ -19,33 +23,38 @@ module TurbineRb
       end
 
       def process(records:, process:)
-        unwrapped_records = records.unwrap if records.instance_of?(Collection)
-
-        pr = TurbineCore::ProcessCollectionRequest::Process.new(
-          name: process.class.name
+        pb_collection = core_server.add_process_to_collection(
+          TurbineCore::ProcessCollectionRequest.new(
+            collection: Collection.unwrap(records),
+            process: TurbineCore::ProcessCollectionRequest::Process.new(name: process.class.name)
+          )
         )
+        records.tap do |r|
+          r.pb_collection = process_call(process: process, pb_collection: pb_collection)
+          r.pb_stream = pb_collection.stream
+        end
+      end
 
-        req = TurbineCore::ProcessCollectionRequest.new(collection: unwrapped_records, process: pr)
-        @core_server.add_process_to_collection(req)
-        records_interface = TurbineRb::Records.new(unwrapped_records.records)
-        processed_records = process.call(records: records_interface) unless @is_recording
-        records.pb_collection = processed_records.map(&:serialize_core_record) unless @is_recording
+      def process_call(process:, pb_collection:)
+        return pb_collection if recording?
 
-        records
+        process
+          .call(records: TurbineRb::Records.new(pb_collection.records))
+          .map(&:serialize_core_record)
       end
 
       # register_secrets accepts either a single string or an array of strings
       def register_secrets(secrets)
-        [*secrets].map do |secret|
+        [secrets].flatten.map do |secret|
           raise MissingSecretError, "secret #{secret} is not an environment variable" unless ENV.key?(secret)
 
           req = TurbineCore::Secret.new(name: secret, value: ENV[secret])
-          @core_server.register_secret(req)
+          core_server.register_secret(req)
         end
       end
 
       class Resource
-        attr_reader :pb_resource
+        attr_reader :pb_resource, :app
 
         def initialize(res, app)
           @pb_resource = res
@@ -59,7 +68,9 @@ module TurbineRb
             req.configs = TurbineCore::Configs.new(config: pb_configs)
           end
 
-          @app.core_server.read_collection(req).wrap(@app) # wrap in Collection to enable chaining
+          app.core_server
+             .read_collection(req)
+             .wrap(app) # wrap in Collection to enable chaining
         end
 
         def write(records:, collection:, configs: nil)
@@ -67,20 +78,29 @@ module TurbineRb
             records = records.unwrap
           end
 
-          req = TurbineCore::WriteCollectionRequest.new(resource: @pb_resource, sourceCollection: records,
-                                                        targetCollection: collection)
+          req = TurbineCore::WriteCollectionRequest.new(
+            resource: @pb_resource,
+            sourceCollection: records,
+            targetCollection: collection
+          )
 
           if configs
             pb_configs = configs.keys.map { |key| TurbineCore::Config.new(field: key, value: configs[key]) }
             req.configs = TurbineCore::Configs.new(config: pb_configs)
           end
 
-          @app.core_server.write_collection_to_resource(req)
+          app.core_server.write_collection_to_resource(req)
         end
       end
 
       class Collection
-        attr_accessor :pb_collection, :pb_stream, :name
+        attr_accessor :pb_collection, :pb_stream, :name, :app
+
+        def self.unwrap(collection)
+          return collection.unwrap if collection.instance_of?(Collection)
+
+          collection
+        end
 
         def initialize(name, collection, stream, app)
           @name = name
@@ -94,7 +114,7 @@ module TurbineRb
         end
 
         def process_with(process:)
-          @app.process(records: self, process: process)
+          app.process(records: self, process: process)
         end
 
         def unwrap

data/lib/turbine_rb/records.rb

@@ -39,12 +39,14 @@ module TurbineRb
 
     def set(key, value)
       @value = value unless value_hash?
-      return unless value_hash?
-
-      begin
-        @value.send(payload_key(key))
-      rescue NoMethodError
-        set_schema_field(key, value)
+      return @value unless value_hash?
+
+      if json_schema?
+        begin
+          @value.send(payload_key(key))
+        rescue NoMethodError
+          set_schema_field(key, value)
+        end
       end
 
       @value.send("#{payload_key(key)}=", value)

data/lib/turbine_rb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TurbineRb
-  VERSION = "0.1.2"
+  VERSION = "0.2.1"
 end

data/lib/turbine_rb.rb

@@ -43,12 +43,12 @@ module TurbineRb
     end
 
     def run
-      app = TurbineRb::Client::App.new(init_core_server)
+      app = TurbineRb::Client::App.new(init_core_server, recording: false)
       TurbineRb.app.call(app)
     end
 
     def record
-      app = TurbineRb::Client::App.new(init_core_server, is_recording: true)
+      app = TurbineRb::Client::App.new(init_core_server, recording: true)
       TurbineRb.app.call(app)
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbine_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.1
 platform: ruby
 authors:
 - Meroxa
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-02 00:00:00.000000000 Z
+date: 2022-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -69,11 +69,6 @@ files:
 - lib/service_pb.rb
 - lib/service_services_pb.rb
 - lib/templates/Dockerfile
-- lib/templates/app/Gemfile
-- lib/templates/app/app.json
-- lib/templates/app/app.rb
-- lib/templates/app/fixtures/demo.json
-- lib/templates/app/ignoregit
 - lib/turbine_pb.rb
 - lib/turbine_rb.rb
 - lib/turbine_rb/client.rb
@@ -90,7 +85,7 @@ metadata:
   homepage_uri: https://github.com/meroxa/turbine-core/tree/main/lib/ruby/turbine_rb
   source_code_uri: https://github.com/meroxa/turbine-core
   changelog_uri: https://github.com/meroxa/turbine-core
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -106,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.3.7
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Meroxa data application framework for Ruby
 test_files: []

data/lib/templates/app/Gemfile

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gem "turbine_rb"

data/lib/templates/app/app.json

@@ -1,8 +0,0 @@
-{
-    "name": "ruby-example",
-    "language": "ruby",
-    "environment": "common",
-    "resources": {
-        "demopg": "fixtures/demo.json"
-    }
-}

data/lib/templates/app/app.rb

@@ -1,67 +0,0 @@
-# frozen_string_literal: true
-
-require "rubygems"
-require "bundler/setup"
-require "turbine_rb"
-
-class MyApp
-  def call(app)
-    # To configure resources for your production datastores
-    # on Meroxa, use the Dashboard, CLI, or Terraform Provider
-    # For more details refer to: http://docs.meroxa.com/
-    #
-    # Identify the upstream datastore with the `resource` function
-    # Replace `demopg` with the resource name configured on Meroxa
-    database = app.resource(name: "demopg")
-
-    # Specify which upstream records to pull
-    # with the `records` function
-    # Replace `collection_name` with a table, collection,
-    # or bucket name in your data store.
-    # If a configuration is needed for your source,
-    # you can pass it as a second argument to the `records` function. For example:
-    # database.records(collection: "collection_name", configs: {"incrementing.column.name" => "id"})
-    records = database.records(collection: "collection_name")
-
-    # Register secrets to be available in the function:
-    # app.register_secrets("MY_ENV_TEST")
-
-    # Register several secrets at once:
-    # app.register_secrets(["MY_ENV_TEST", "MY_OTHER_ENV_TEST"])
-
-    # Specify the code to execute against `records` with the `process` function.
-    # Replace `Passthrough` with your desired function.
-    # Ensure desired function matches `Passthrough`'s' function signature.
-    processed_records = app.process(records: records, process: Passthrough.new)
-
-    # Specify where to write records using the `write` function.
-    # Replace `collection_archive` with whatever data organisation method
-    # is relevant to the datastore (e.g., table, bucket, collection, etc.)
-    # If additional connector configs are needed, provided another argument. For example:
-    # database.write(
-    #   records: processed_records,
-    #   collection: "collection_archive",
-    #   configs: {"behavior.on.null.values": "ignore"})
-    database.write(records: processed_records, collection: "collection_archive")
-  end
-end
-
-class Passthrough < TurbineRb::Process
-  def call(records:)
-    puts "got records: #{records}"
-    # To get the value of unformatted records, use record .value getter method
-    # records.map { |r| puts r.value }
-    #
-    # To transform unformatted records, use record .value setter method
-    # records.map { |r| r.value = "newdata" }
-    #
-    # To get the value of json formatted records, use record .get method
-    # records.map { |r| puts r.get("message") }
-    #
-    # To transform json formatted records, use record .set methods
-    # records.map { |r| r.set('message', 'goodbye') }
-    records
-  end
-end
-
-TurbineRb.register(MyApp.new)

data/lib/templates/app/fixtures/demo.json

@@ -1,9 +0,0 @@
-{
-    "events": [
-        {
-            "key": "1",
-            "value": {"message":"hello"},
-            "timestamp": "1662758822"
-        }
-    ]
-}