turbine_rb 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da43639e3f731f3d00c53ed1e5dfd30b905d74549d3b5a8ce6c21604091c9dc2
-  data.tar.gz: 57fcd45315e4eaa26eb21bcc78478ff7a07afce5ac04ec6eafe351104fd82a37
+  metadata.gz: 0adaff81024cca99dc5d4b043e98e3fe6a69f29d5d9ee2fd312c66ede6840f4c
+  data.tar.gz: c702105c5d3b5672bab440ac16beef7ff0c4497147b66cee1f9dad2cce31bbf6
 SHA512:
-  metadata.gz: 397f59b0c96496fa4e2e84386f0d885040ba0878e68da94c8e871689753b004ca8618821d8c570a41bca61b5514d8509a8c185aab25fac89a9de267f39c6d904
-  data.tar.gz: 745c1cb5eaf39a45e5eb460b27f0211e1f4e813d71018b26c2751a26806be85ad3f602de3496f5bc5a32d7cf153829f9a8621ac648e05864b02ae61f33501b8a
+  metadata.gz: 5ea6a41d9686349f4f02d90ac74b653a08ddf693feb12453e7a056a077982ae8dbdec792209acc2e24263e5e576762266a8d37ecebc0329815e028b26fdd5cc1
+  data.tar.gz: 3d473b1e102a17b9b247977bbccb5617bb4a7d861e6584035e00ab406032b83e31363ea769d525c15faccd2ce573cabd82ed2473c267ee770c164905aa50a31b

data/CHANGELOG.md

@@ -1,4 +1,9 @@
+## [0.1.2] - 2022-12-02
+
+- fix: add record interface to local run
+
 ## [0.1.1] - 2022-12-01
+
 - fix: dockerfile entrypoint
 
 ## [0.1.0] - 2022-11-30

data/lib/templates/app/app.rb

@@ -6,48 +6,59 @@ require "turbine_rb"
 
 class MyApp
   def call(app)
+    # To configure resources for your production datastores
+    # on Meroxa, use the Dashboard, CLI, or Terraform Provider
+    # For more details refer to: http://docs.meroxa.com/
+    #
+    # Identify the upstream datastore with the `resource` function
+    # Replace `demopg` with the resource name configured on Meroxa
     database = app.resource(name: "demopg")
 
-    # ELT pipeline example
-    # records = database.records(collection: 'events')
-    # database.write(records: records, collection: 'events_copy')
-
-    # procedural API
-    records = database.records(collection: "events")
+    # Specify which upstream records to pull
+    # with the `records` function
+    # Replace `collection_name` with a table, collection,
+    # or bucket name in your data store.
+    # If a configuration is needed for your source,
+    # you can pass it as a second argument to the `records` function. For example:
+    # database.records(collection: "collection_name", configs: {"incrementing.column.name" => "id"})
+    records = database.records(collection: "collection_name")
 
-    # This register the secret to be available in the turbine application
-    app.register_secrets("MY_ENV_TEST")
+    # Register secrets to be available in the function:
+    # app.register_secrets("MY_ENV_TEST")
 
-    # you can also register several secrets at once
+    # Register several secrets at once:
     # app.register_secrets(["MY_ENV_TEST", "MY_OTHER_ENV_TEST"])
 
-    # Passthrough just has to match the signature
+    # Specify the code to execute against `records` with the `process` function.
+    # Replace `Passthrough` with your desired function.
+    # Ensure desired function matches `Passthrough`'s' function signature.
     processed_records = app.process(records: records, process: Passthrough.new)
-    database.write(records: processed_records, collection: "events_copy")
-
-    # out_records = processed_records.join(records, key: "user_id", window: 1.day) # stream joins
 
-    # chaining API
-    # database.records(collection: "events").
-    #   process_with(process: Passthrough.new).
-    #   write_to(resource: database, collection: "events_copy")
+    # Specify where to write records using the `write` function.
+    # Replace `collection_archive` with whatever data organisation method
+    # is relevant to the datastore (e.g., table, bucket, collection, etc.)
+    # If additional connector configs are needed, provided another argument. For example:
+    # database.write(
+    #   records: processed_records,
+    #   collection: "collection_archive",
+    #   configs: {"behavior.on.null.values": "ignore"})
+    database.write(records: processed_records, collection: "collection_archive")
   end
 end
 
-# might be useful to signal that this is a special Turbine call
 class Passthrough < TurbineRb::Process
   def call(records:)
     puts "got records: #{records}"
-    # to get the value of unformatted records, use record .value getter method
+    # To get the value of unformatted records, use record .value getter method
     # records.map { |r| puts r.value }
     #
-    # to transform unformatted records, use record .value setter method
+    # To transform unformatted records, use record .value setter method
     # records.map { |r| r.value = "newdata" }
     #
-    # to get the value of json formatted records, use record .get method
+    # To get the value of json formatted records, use record .get method
     # records.map { |r| puts r.get("message") }
     #
-    # to transform json formatted records, use record .set methods
+    # To transform json formatted records, use record .set methods
     # records.map { |r| r.set('message', 'goodbye') }
     records
   end

data/lib/turbine_rb/client.rb

@@ -27,7 +27,9 @@ module TurbineRb
 
         req = TurbineCore::ProcessCollectionRequest.new(collection: unwrapped_records, process: pr)
         @core_server.add_process_to_collection(req)
-        records.pb_collection = process.call(records: records.pb_collection) unless @is_recording
+        records_interface = TurbineRb::Records.new(unwrapped_records.records)
+        processed_records = process.call(records: records_interface) unless @is_recording
+        records.pb_collection = processed_records.map(&:serialize_core_record) unless @is_recording
 
         records
       end

data/lib/turbine_rb/records.rb

@@ -4,6 +4,7 @@ require "json"
 require "hash_dot"
 
 module TurbineRb
+  # rubocop:disable Metrics/ClassLength
   class Record
     attr_accessor :key, :value, :timestamp
 
@@ -21,41 +22,32 @@ module TurbineRb
     end
 
     def serialize
-      Io::Meroxa::Funtime::Record.new(key: @key, value: @value.to_json, timestamp: @timestamp)
+      Io::Meroxa::Funtime::Record.new(key: @key, value: serialize_value, timestamp: @timestamp)
+    end
+
+    def serialize_core_record
+      TurbineCore::Record.new(key: @key, value: serialize_value, timestamp: @timestamp)
     end
 
     def get(key)
       if value_string? || value_array?
         @value
-      elsif cdc_format?
-        @value.send("payload.after.#{key}")
       else
-        @value.send("payload.#{key}")
+        @value.send(payload_key(key))
       end
     end
 
     def set(key, value)
-      if !value_hash?
-        @value = value
-      else
-        payload_key = cdc_format? ? "payload.after" : "payload"
-
-        begin
-          @value.send("#{payload_key}.#{key}")
-        rescue NoMethodError
-          schema = @value.send("schema.fields")
-          new_schema_field = { field: key, optional: true, type: "string" }.to_dot
-
-          if cdc_format?
-            schema_fields = schema.find { |f| f.field == "after" }
-            schema_fields.fields.unshift(new_schema_field)
-          else
-            schema.unshift(new_schema_field)
-          end
-        end
-
-        @value.send("#{payload_key}.#{key}=", value)
+      @value = value unless value_hash?
+      return unless value_hash?
+
+      begin
+        @value.send(payload_key(key))
+      rescue NoMethodError
+        set_schema_field(key, value)
       end
+
+      @value.send("#{payload_key(key)}=", value)
     end
 
     def unwrap!
@@ -87,6 +79,16 @@ module TurbineRb
       @value.is_a?(Hash)
     end
 
+    def payload_key(prop_key)
+      if cdc_format?
+        "payload.after.#{prop_key}"
+      elsif json_schema?
+        "payload.#{prop_key}"
+      else
+        prop_key
+      end
+    end
+
     def json_schema?
       value_hash? &&
         @value.key?("payload") &&
@@ -112,7 +114,24 @@ module TurbineRb
         "unsupported"
       end
     end
+
+    def set_schema_field(key, value)
+      schema = @value.send("schema.fields")
+      new_schema_field = { field: key, optional: true, type: type_of_value(value) }.to_dot
+
+      schema.find { |f| f.field == "after" }.fields.unshift(new_schema_field) if cdc_format?
+      schema.unshift(new_schema_field) if json_schema?
+    end
+
+    def serialize_value
+      if value_string?
+        @value
+      else
+        @value.to_json
+      end
+    end
   end
+  # rubocop:enable Metrics/ClassLength
 
   class Records < SimpleDelegator
     def initialize(pb_records)

data/lib/turbine_rb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TurbineRb
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbine_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Meroxa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-01 00:00:00.000000000 Z
+date: 2022-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc