turbine_rb 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: da43639e3f731f3d00c53ed1e5dfd30b905d74549d3b5a8ce6c21604091c9dc2
-  data.tar.gz: 57fcd45315e4eaa26eb21bcc78478ff7a07afce5ac04ec6eafe351104fd82a37
+  metadata.gz: 400b81da43fa55a94e4d85544197b6cfca8eeeff18d7db8cfb351abacc07c518
+  data.tar.gz: 480a684b89b0a8ab8ec9cca5d1a314d144f6760b097a011db84aae006eb457b1
 SHA512:
-  metadata.gz: 397f59b0c96496fa4e2e84386f0d885040ba0878e68da94c8e871689753b004ca8618821d8c570a41bca61b5514d8509a8c185aab25fac89a9de267f39c6d904
-  data.tar.gz: 745c1cb5eaf39a45e5eb460b27f0211e1f4e813d71018b26c2751a26806be85ad3f602de3496f5bc5a32d7cf153829f9a8621ac648e05864b02ae61f33501b8a
+  metadata.gz: cc55f7f0ba4310bddd702d95d8feb9d7c5bf16fe7856eba58f6301af6d6a75504af8c615f53ac71edf7e30333594cdbc8a91707af1596c2508d7580c32ab0e8e
+  data.tar.gz: c59f54f06f4cc0538ff3c17607871f104fb1d43e9083cb4ec1da11b2cb2ebb4a7426c0b03135dc9dd0a1e4b2d3fda823307502dba1b08a1eba6e8049503bfbaa

data/CHANGELOG.md

@@ -1,4 +1,13 @@
+## [0.1.3] - 2022-12-03
+
+- fix: set new keys on unformatted json
+
+## [0.1.2] - 2022-12-02
+
+- fix: add record interface to local run
+
 ## [0.1.1] - 2022-12-01
+
 - fix: dockerfile entrypoint
 
 ## [0.1.0] - 2022-11-30

data/lib/templates/app/app.rb

@@ -6,48 +6,59 @@ require "turbine_rb"
 
 class MyApp
   def call(app)
+    # To configure resources for your production datastores
+    # on Meroxa, use the Dashboard, CLI, or Terraform Provider
+    # For more details refer to: http://docs.meroxa.com/
+    #
+    # Identify the upstream datastore with the `resource` function
+    # Replace `demopg` with the resource name configured on Meroxa
     database = app.resource(name: "demopg")
 
-    # ELT pipeline example
-    # records = database.records(collection: 'events')
-    # database.write(records: records, collection: 'events_copy')
-
-    # procedural API
-    records = database.records(collection: "events")
+    # Specify which upstream records to pull
+    # with the `records` function
+    # Replace `collection_name` with a table, collection,
+    # or bucket name in your data store.
+    # If a configuration is needed for your source,
+    # you can pass it as a second argument to the `records` function. For example:
+    # database.records(collection: "collection_name", configs: {"incrementing.column.name" => "id"})
+    records = database.records(collection: "collection_name")
 
-    # This register the secret to be available in the turbine application
-    app.register_secrets("MY_ENV_TEST")
+    # Register secrets to be available in the function:
+    # app.register_secrets("MY_ENV_TEST")
 
-    # you can also register several secrets at once
+    # Register several secrets at once:
     # app.register_secrets(["MY_ENV_TEST", "MY_OTHER_ENV_TEST"])
 
-    # Passthrough just has to match the signature
+    # Specify the code to execute against `records` with the `process` function.
+    # Replace `Passthrough` with your desired function.
+    # Ensure desired function matches `Passthrough`'s' function signature.
     processed_records = app.process(records: records, process: Passthrough.new)
-    database.write(records: processed_records, collection: "events_copy")
-
-    # out_records = processed_records.join(records, key: "user_id", window: 1.day) # stream joins
 
-    # chaining API
-    # database.records(collection: "events").
-    #   process_with(process: Passthrough.new).
-    #   write_to(resource: database, collection: "events_copy")
+    # Specify where to write records using the `write` function.
+    # Replace `collection_archive` with whatever data organisation method
+    # is relevant to the datastore (e.g., table, bucket, collection, etc.)
+    # If additional connector configs are needed, provided another argument. For example:
+    # database.write(
+    #   records: processed_records,
+    #   collection: "collection_archive",
+    #   configs: {"behavior.on.null.values": "ignore"})
+    database.write(records: processed_records, collection: "collection_archive")
   end
 end
 
-# might be useful to signal that this is a special Turbine call
 class Passthrough < TurbineRb::Process
   def call(records:)
     puts "got records: #{records}"
-    # to get the value of unformatted records, use record .value getter method
+    # To get the value of unformatted records, use record .value getter method
     # records.map { |r| puts r.value }
     #
-    # to transform unformatted records, use record .value setter method
+    # To transform unformatted records, use record .value setter method
     # records.map { |r| r.value = "newdata" }
     #
-    # to get the value of json formatted records, use record .get method
+    # To get the value of json formatted records, use record .get method
     # records.map { |r| puts r.get("message") }
     #
-    # to transform json formatted records, use record .set methods
+    # To transform json formatted records, use record .set methods
     # records.map { |r| r.set('message', 'goodbye') }
     records
   end

data/lib/turbine_rb/client.rb

@@ -27,7 +27,9 @@ module TurbineRb
 
         req = TurbineCore::ProcessCollectionRequest.new(collection: unwrapped_records, process: pr)
         @core_server.add_process_to_collection(req)
-        records.pb_collection = process.call(records: records.pb_collection) unless @is_recording
+        records_interface = TurbineRb::Records.new(unwrapped_records.records)
+        processed_records = process.call(records: records_interface) unless @is_recording
+        records.pb_collection = processed_records.map(&:serialize_core_record) unless @is_recording
 
         records
       end

data/lib/turbine_rb/records.rb

@@ -4,6 +4,7 @@ require "json"
 require "hash_dot"
 
 module TurbineRb
+  # rubocop:disable Metrics/ClassLength
   class Record
     attr_accessor :key, :value, :timestamp
 
@@ -21,41 +22,34 @@ module TurbineRb
     end
 
     def serialize
-      Io::Meroxa::Funtime::Record.new(key: @key, value: @value.to_json, timestamp: @timestamp)
+      Io::Meroxa::Funtime::Record.new(key: @key, value: serialize_value, timestamp: @timestamp)
+    end
+
+    def serialize_core_record
+      TurbineCore::Record.new(key: @key, value: serialize_value, timestamp: @timestamp)
     end
 
     def get(key)
       if value_string? || value_array?
         @value
-      elsif cdc_format?
-        @value.send("payload.after.#{key}")
       else
-        @value.send("payload.#{key}")
+        @value.send(payload_key(key))
       end
     end
 
     def set(key, value)
-      if !value_hash?
-        @value = value
-      else
-        payload_key = cdc_format? ? "payload.after" : "payload"
+      @value = value unless value_hash?
+      return @value unless value_hash?
 
+      if json_schema?
         begin
-          @value.send("#{payload_key}.#{key}")
+          @value.send(payload_key(key))
         rescue NoMethodError
-          schema = @value.send("schema.fields")
-          new_schema_field = { field: key, optional: true, type: "string" }.to_dot
-
-          if cdc_format?
-            schema_fields = schema.find { |f| f.field == "after" }
-            schema_fields.fields.unshift(new_schema_field)
-          else
-            schema.unshift(new_schema_field)
-          end
+          set_schema_field(key, value)
         end
-
-        @value.send("#{payload_key}.#{key}=", value)
       end
+
+      @value.send("#{payload_key(key)}=", value)
     end
 
     def unwrap!
@@ -87,6 +81,16 @@ module TurbineRb
       @value.is_a?(Hash)
     end
 
+    def payload_key(prop_key)
+      if cdc_format?
+        "payload.after.#{prop_key}"
+      elsif json_schema?
+        "payload.#{prop_key}"
+      else
+        prop_key
+      end
+    end
+
     def json_schema?
       value_hash? &&
         @value.key?("payload") &&
@@ -112,7 +116,24 @@ module TurbineRb
         "unsupported"
       end
     end
+
+    def set_schema_field(key, value)
+      schema = @value.send("schema.fields")
+      new_schema_field = { field: key, optional: true, type: type_of_value(value) }.to_dot
+
+      schema.find { |f| f.field == "after" }.fields.unshift(new_schema_field) if cdc_format?
+      schema.unshift(new_schema_field) if json_schema?
+    end
+
+    def serialize_value
+      if value_string?
+        @value
+      else
+        @value.to_json
+      end
+    end
   end
+  # rubocop:enable Metrics/ClassLength
 
   class Records < SimpleDelegator
     def initialize(pb_records)

data/lib/turbine_rb/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TurbineRb
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: turbine_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Meroxa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-01 00:00:00.000000000 Z
+date: 2022-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc