turbine_rb 0.1.0

data/lib/templates/app/app.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "rubygems"
+require "bundler/setup"
+require "turbine_rb"
+
+class MyApp
+  def call(app)
+    database = app.resource(name: "demopg")
+
+    # ELT pipeline example
+    # records = database.records(collection: 'events')
+    # database.write(records: records, collection: 'events_copy')
+
+    # procedural API
+    records = database.records(collection: "events")
+
+    # This register the secret to be available in the turbine application
+    app.register_secrets("MY_ENV_TEST")
+
+    # you can also register several secrets at once
+    # app.register_secrets(["MY_ENV_TEST", "MY_OTHER_ENV_TEST"])
+
+    # Passthrough just has to match the signature
+    processed_records = app.process(records: records, process: Passthrough.new)
+    database.write(records: processed_records, collection: "events_copy")
+
+    # out_records = processed_records.join(records, key: "user_id", window: 1.day) # stream joins
+
+    # chaining API
+    # database.records(collection: "events").
+    #   process_with(process: Passthrough.new).
+    #   write_to(resource: database, collection: "events_copy")
+  end
+end
+
+# might be useful to signal that this is a special Turbine call
+class Passthrough < TurbineRb::Process
+  def call(records:)
+    puts "got records: #{records}"
+    # to get the value of unformatted records, use record .value getter method
+    # records.map { |r| puts r.value }
+    #
+    # to transform unformatted records, use record .value setter method
+    # records.map { |r| r.value = "newdata" }
+    #
+    # to get the value of json formatted records, use record .get method
+    # records.map { |r| puts r.get("message") }
+    #
+    # to transform json formatted records, use record .set methods
+    # records.map { |r| r.set('message', 'goodbye') }
+    records
+  end
+end
+
+TurbineRb.register(MyApp.new)

data/lib/templates/app/fixtures/demo.json

@@ -0,0 +1,9 @@
+{
+    "events": [
+        {
+            "key": "1",
+            "value": {"message":"hello"},
+            "timestamp": "1662758822"
+        }
+    ]
+}

data/lib/turbine_pb.rb

@@ -0,0 +1,100 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: turbine.proto
+
+require 'google/protobuf'
+
+require 'google/protobuf/empty_pb'
+require 'google/protobuf/timestamp_pb'
+require 'google/protobuf/wrappers_pb'
+require 'validate/validate_pb'
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_file("turbine.proto", :syntax => :proto3) do
+    add_message "turbine_core.InitRequest" do
+      optional :appName, :string, 1
+      optional :configFilePath, :string, 2
+      optional :language, :enum, 3, "turbine_core.Language"
+      optional :gitSHA, :string, 4
+      optional :turbineVersion, :string, 5
+    end
+    add_message "turbine_core.GetResourceRequest" do
+      optional :name, :string, 1
+    end
+    add_message "turbine_core.Resource" do
+      optional :name, :string, 1
+    end
+    add_message "turbine_core.Collection" do
+      optional :name, :string, 1
+      optional :stream, :string, 2
+      repeated :records, :message, 3, "turbine_core.Record"
+    end
+    add_message "turbine_core.Record" do
+      optional :key, :string, 1
+      optional :value, :bytes, 2
+      optional :timestamp, :message, 3, "google.protobuf.Timestamp"
+    end
+    add_message "turbine_core.ReadCollectionRequest" do
+      optional :resource, :message, 1, "turbine_core.Resource"
+      optional :collection, :string, 2
+      optional :configs, :message, 3, "turbine_core.Configs"
+    end
+    add_message "turbine_core.WriteCollectionRequest" do
+      optional :resource, :message, 1, "turbine_core.Resource"
+      optional :sourceCollection, :message, 2, "turbine_core.Collection"
+      optional :targetCollection, :string, 3
+      optional :configs, :message, 4, "turbine_core.Configs"
+    end
+    add_message "turbine_core.Configs" do
+      repeated :config, :message, 1, "turbine_core.Config"
+    end
+    add_message "turbine_core.Config" do
+      optional :field, :string, 1
+      optional :value, :string, 2
+    end
+    add_message "turbine_core.ProcessCollectionRequest" do
+      optional :process, :message, 1, "turbine_core.ProcessCollectionRequest.Process"
+      optional :collection, :message, 2, "turbine_core.Collection"
+    end
+    add_message "turbine_core.ProcessCollectionRequest.Process" do
+      optional :name, :string, 1
+    end
+    add_message "turbine_core.Secret" do
+      optional :name, :string, 1
+      optional :value, :string, 2
+    end
+    add_message "turbine_core.ListResourcesResponse" do
+      repeated :resources, :message, 1, "turbine_core.Resource"
+    end
+    add_message "turbine_core.GetSpecRequest" do
+      optional :image, :string, 1
+    end
+    add_message "turbine_core.GetSpecResponse" do
+      optional :spec, :bytes, 1
+    end
+    add_enum "turbine_core.Language" do
+      value :GOLANG, 0
+      value :PYTHON, 1
+      value :JAVASCRIPT, 2
+      value :RUBY, 3
+    end
+  end
+end
+
+module TurbineCore
+  InitRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.InitRequest").msgclass
+  GetResourceRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.GetResourceRequest").msgclass
+  Resource = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Resource").msgclass
+  Collection = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Collection").msgclass
+  Record = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Record").msgclass
+  ReadCollectionRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.ReadCollectionRequest").msgclass
+  WriteCollectionRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.WriteCollectionRequest").msgclass
+  Configs = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Configs").msgclass
+  Config = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Config").msgclass
+  ProcessCollectionRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.ProcessCollectionRequest").msgclass
+  ProcessCollectionRequest::Process = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.ProcessCollectionRequest.Process").msgclass
+  Secret = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Secret").msgclass
+  ListResourcesResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.ListResourcesResponse").msgclass
+  GetSpecRequest = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.GetSpecRequest").msgclass
+  GetSpecResponse = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.GetSpecResponse").msgclass
+  Language = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("turbine_core.Language").enummodule
+end

data/lib/turbine_rb/client.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module TurbineRb
+  module Client
+    class MissingSecretError < StandardError; end
+
+    class App
+      attr_reader :core_server
+
+      def initialize(grpc_server, is_recording: false)
+        @core_server = grpc_server
+        @is_recording = is_recording
+      end
+
+      def resource(name:)
+        req = TurbineCore::GetResourceRequest.new(name: name)
+        res = @core_server.get_resource(req)
+        Resource.new(res, self)
+      end
+
+      def process(records:, process:)
+        unwrapped_records = records.unwrap if records.instance_of?(Collection)
+
+        pr = TurbineCore::ProcessCollectionRequest::Process.new(
+          name: process.class.name
+        )
+
+        req = TurbineCore::ProcessCollectionRequest.new(collection: unwrapped_records, process: pr)
+        @core_server.add_process_to_collection(req)
+        records.pb_collection = process.call(records: records.pb_collection) unless @is_recording
+
+        records
+      end
+
+      # register_secrets accepts either a single string or an array of strings
+      def register_secrets(secrets)
+        [*secrets].map do |secret|
+          raise MissingSecretError, "secret #{secret} is not an environment variable" unless ENV.key?(secret)
+
+          req = TurbineCore::Secret.new(name: secret, value: ENV[secret])
+          @core_server.register_secret(req)
+        end
+      end
+
+      class Resource
+        attr_reader :pb_resource
+
+        def initialize(res, app)
+          @pb_resource = res
+          @app = app
+        end
+
+        def records(collection:, configs: nil)
+          req = TurbineCore::ReadCollectionRequest.new(resource: @pb_resource, collection: collection)
+          if configs
+            pb_configs = configs.keys.map { |key| TurbineCore::Config.new(field: key, value: configs[key]) }
+            req.configs = TurbineCore::Configs.new(config: pb_configs)
+          end
+
+          @app.core_server.read_collection(req).wrap(@app) # wrap in Collection to enable chaining
+        end
+
+        def write(records:, collection:, configs: nil)
+          if records.instance_of?(Collection) # it has been processed by a function, so unwrap back to gRPC collection
+            records = records.unwrap
+          end
+
+          req = TurbineCore::WriteCollectionRequest.new(resource: @pb_resource, sourceCollection: records,
+                                                        targetCollection: collection)
+
+          if configs
+            pb_configs = configs.keys.map { |key| TurbineCore::Config.new(field: key, value: configs[key]) }
+            req.configs = TurbineCore::Configs.new(config: pb_configs)
+          end
+
+          @app.core_server.write_collection_to_resource(req)
+        end
+      end
+
+      class Collection
+        attr_accessor :pb_collection, :pb_stream, :name
+
+        def initialize(name, collection, stream, app)
+          @name = name
+          @pb_collection = collection
+          @pb_stream = stream
+          @app = app
+        end
+
+        def write_to(resource:, collection:, configs: nil)
+          resource.write(records: self, collection: collection, configs: configs)
+        end
+
+        def process_with(process:)
+          @app.process(records: self, process: process)
+        end
+
+        def unwrap
+          TurbineCore::Collection.new( # convert back to TurbineCore::Collection
+            name: name,
+            records: pb_collection.to_a,
+            stream: pb_stream
+          )
+        end
+      end
+    end
+  end
+end

data/lib/turbine_rb/collection_patch.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+TurbineCore::Collection.class_eval do
+  def wrap(app)
+    TurbineRb::Client::App::Collection.new(
+      name,
+      records,
+      stream,
+      app
+    )
+  end
+end

data/lib/turbine_rb/records.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "json"
+require "hash_dot"
+
+module TurbineRb
+  class Record
+    attr_accessor :key, :value, :timestamp
+
+    def initialize(pb_record)
+      @key = pb_record.key
+      @timestamp = pb_record.timestamp
+
+      begin
+        @value = JSON.parse(pb_record.value)
+      rescue JSON::ParserError
+        @value = pb_record.value
+      end
+
+      @value = @value.to_dot if value_hash?
+    end
+
+    def serialize
+      Io::Meroxa::Funtime::Record.new(key: @key, value: @value.to_json, timestamp: @timestamp)
+    end
+
+    def get(key)
+      if value_string? || value_array?
+        @value
+      elsif cdc_format?
+        @value.send("payload.after.#{key}")
+      else
+        @value.send("payload.#{key}")
+      end
+    end
+
+    def set(key, value)
+      if !value_hash?
+        @value = value
+      else
+        payload_key = cdc_format? ? "payload.after" : "payload"
+
+        begin
+          @value.send("#{payload_key}.#{key}")
+        rescue NoMethodError
+          schema = @value.send("schema.fields")
+          new_schema_field = { field: key, optional: true, type: "string" }.to_dot
+
+          if cdc_format?
+            schema_fields = schema.find { |f| f.field == "after" }
+            schema_fields.fields.unshift(new_schema_field)
+          else
+            schema.unshift(new_schema_field)
+          end
+        end
+
+        @value.send("#{payload_key}.#{key}=", value)
+      end
+    end
+
+    def unwrap!
+      return unless cdc_format?
+
+      payload = @value.send("payload")
+      schema = @value.send("schema.fields")
+      schema_fields = schema.find { |f| f.field == "after" }
+      unless schema_fields.nil?
+        schema_fields.delete("field")
+        schema_fields.name = @value.send("schema.name")
+        @value.send("schema=", schema_fields)
+      end
+
+      @value.send("payload=", payload.after)
+    end
+
+    private
+
+    def value_string?
+      @value.is_a?(String)
+    end
+
+    def value_array?
+      @value.is_a?(Array)
+    end
+
+    def value_hash?
+      @value.is_a?(Hash)
+    end
+
+    def json_schema?
+      value_hash? &&
+        @value.key?("payload") &&
+        @value.key?("schema")
+    end
+
+    def cdc_format?
+      json_schema? &&
+        @value.payload.key?("source")
+    end
+
+    def type_of_value(value)
+      case value
+      when String
+        "string"
+      when Integer
+        "int32"
+      when Float
+        "float32"
+      when true, false
+        "boolean"
+      else
+        "unsupported"
+      end
+    end
+  end
+
+  class Records < SimpleDelegator
+    def initialize(pb_records)
+      super
+      records = pb_records.map { |r| Record.new(r) }
+      __setobj__(records)
+    end
+
+    def unwrap!
+      records = __getobj__
+      records.each(&:unwrap!)
+    end
+  end
+end

data/lib/turbine_rb/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module TurbineRb
+  VERSION = "0.1.0"
+end

data/lib/turbine_rb.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "service_services_pb"
+require "turbine_services_pb"
+
+require "turbine_rb/collection_patch"
+require "turbine_rb/version"
+require "turbine_rb/client"
+require "turbine_rb/records"
+
+require "optparse"
+require "fileutils"
+
+require "grpc"
+require "grpc/health/v1/health_pb"
+require "grpc/health/checker"
+
+module TurbineRb
+  class Error < StandardError; end
+
+  class << self
+    attr_reader :app, :process_klass
+
+    def register(app)
+      @app = app
+    end
+
+    def register_fn(fn_klass)
+      @process_klass = fn_klass
+    end
+
+    def serve
+      process_function = @process_klass.new
+      process_function_impl = ProcessImpl.new(process_function)
+      function_addr = ENV["MEROXA_FUNCTION_ADDR"] ||= "0.0.0.0:50500"
+
+      @grpc_server = GRPC::RpcServer.new
+      @grpc_server.add_http2_port(function_addr, :this_port_is_insecure)
+      @grpc_server.handle(process_function_impl)
+      @grpc_server.handle(HealthCheck)
+      puts "serving function #{process_function.class.name} on #{function_addr}"
+      @grpc_server.run_till_terminated_or_interrupted([1, "int", "SIGQUIT"])
+    end
+
+    def run
+      app = TurbineRb::Client::App.new(init_core_server)
+      TurbineRb.app.call(app)
+    end
+
+    def record
+      app = TurbineRb::Client::App.new(init_core_server, is_recording: true)
+      TurbineRb.app.call(app)
+    end
+
+    def build
+      docker_file = File.join(__dir__, "templates", "Dockerfile")
+      dest_app = Dir.getwd
+      FileUtils.cp(docker_file, dest_app)
+    end
+
+    private
+
+    def init_core_server
+      # TODO: figure out what the deal is with :this_channel_is_insecure
+      core_server = TurbineCore::TurbineService::Stub.new(ENV["TURBINE_CORE_SERVER"], :this_channel_is_insecure)
+      git_sha = ARGV[0]
+
+      req = TurbineCore::InitRequest.new(
+        appName: app.class.name,
+        configFilePath: Dir.getwd,
+        language: :RUBY,
+        gitSHA: git_sha,
+        turbineVersion: Gem.loaded_specs["turbine_rb"].version.version
+      )
+
+      core_server.init(req)
+      core_server
+    end
+  end
+
+  class ProcessImpl < Io::Meroxa::Funtime::Function::Service
+    def initialize(process)
+      @process = process
+      super()
+    end
+
+    def process(request, _call)
+      records = TurbineRb::Records.new(request.records)
+
+      # records are processed but not in proto format
+      processed_records = @process.call(records: records)
+
+      # to proto
+      serialized_records = processed_records.map(&:serialize)
+
+      Io::Meroxa::Funtime::ProcessRecordResponse.new(records: serialized_records)
+    end
+  end
+
+  class Process
+    def self.inherited(subclass)
+      TurbineRb.register_fn(subclass)
+      super
+    end
+  end
+
+  class HealthCheck < Grpc::Health::V1::Health::Service
+    def check(req, req_view)
+      checker = Grpc::Health::Checker.new
+      checker.set_status_for_services(
+        Grpc::Health::V1::HealthCheckResponse::ServingStatus::SERVING,
+        "function"
+      )
+      checker.check(req, req_view)
+    end
+  end
+end

data/lib/turbine_services_pb.rb

@@ -0,0 +1,30 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# Source: turbine.proto for package 'turbine_core'
+
+require 'grpc'
+require 'turbine_pb'
+
+module TurbineCore
+  module TurbineService
+    class Service
+
+      include ::GRPC::GenericService
+
+      self.marshal_class_method = :encode
+      self.unmarshal_class_method = :decode
+      self.service_name = 'turbine_core.TurbineService'
+
+      rpc :Init, ::TurbineCore::InitRequest, ::Google::Protobuf::Empty
+      rpc :GetResource, ::TurbineCore::GetResourceRequest, ::TurbineCore::Resource
+      rpc :ReadCollection, ::TurbineCore::ReadCollectionRequest, ::TurbineCore::Collection
+      rpc :WriteCollectionToResource, ::TurbineCore::WriteCollectionRequest, ::Google::Protobuf::Empty
+      rpc :AddProcessToCollection, ::TurbineCore::ProcessCollectionRequest, ::TurbineCore::Collection
+      rpc :RegisterSecret, ::TurbineCore::Secret, ::Google::Protobuf::Empty
+      rpc :HasFunctions, ::Google::Protobuf::Empty, ::Google::Protobuf::BoolValue
+      rpc :ListResources, ::Google::Protobuf::Empty, ::TurbineCore::ListResourcesResponse
+      rpc :GetSpec, ::TurbineCore::GetSpecRequest, ::TurbineCore::GetSpecResponse
+    end
+
+    Stub = Service.rpc_stub_class
+  end
+end

data/lib/validate/validate_pb.rb

@@ -0,0 +1,3 @@
+# This file is a place holder, do not remove.
+# The plugin used for `validate/validate.proto` does not support ruby, however
+# the `require` file in the emitted code references it even though it doesn't exist.

data/sig/turbine_framework.rbs

@@ -0,0 +1,4 @@
+module TurbineRb
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: turbine_rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Meroxa
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-12-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: grpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.48'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.48'
+- !ruby/object:Gem::Dependency
+  name: hash_dot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.0
+description: |
+  Turbine is a data application framework for building
+  server-side applications that are event-driven,
+  respond to data in real-time, and scale using cloud-native best practices
+email:
+- production@meroxa.io
+executables:
+- turbine-function
+- turbine-build
+- turbine-record
+- turbine-run
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/turbine-build
+- bin/turbine-function
+- bin/turbine-record
+- bin/turbine-run
+- lib/service_pb.rb
+- lib/service_services_pb.rb
+- lib/templates/Dockerfile
+- lib/templates/app/Gemfile
+- lib/templates/app/app.json
+- lib/templates/app/app.rb
+- lib/templates/app/fixtures/demo.json
+- lib/templates/app/ignoregit
+- lib/turbine_pb.rb
+- lib/turbine_rb.rb
+- lib/turbine_rb/client.rb
+- lib/turbine_rb/collection_patch.rb
+- lib/turbine_rb/records.rb
+- lib/turbine_rb/version.rb
+- lib/turbine_services_pb.rb
+- lib/validate/validate_pb.rb
+- sig/turbine_framework.rbs
+homepage: https://github.com/meroxa/turbine-core/tree/main/lib/ruby/turbine_rb
+licenses:
+- LicenseRef-LICENSE.txt
+metadata:
+  homepage_uri: https://github.com/meroxa/turbine-core/tree/main/lib/ruby/turbine_rb
+  source_code_uri: https://github.com/meroxa/turbine-core
+  changelog_uri: https://github.com/meroxa/turbine-core
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Meroxa data application framework for Ruby
+test_files: []