streamdal 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdf556c16ea4239101cc21f803b78c458c5e2490a1c5df6b2211f5268cc94799
-  data.tar.gz: 1bca44db3048358a5bdf978504a8559c1a81ede836a11269c5ab8020698d71b6
+  metadata.gz: ff825b88dbb81240b3ea5c3fd2cf264a88ecdafbb197ba886ed11acfe098ab93
+  data.tar.gz: 11a9e57851f4296c715b393d8c3ca7f33506d0050e6a4bf74121f92c141288c4
 SHA512:
-  metadata.gz: 9f0fb7a83fc6e924f520d5402385bd857aff3a27884fd2a1aa9572a002723ad26d3849a2930829e6a05e450547ab1e80293992a693bbeb2bb9bacad8756011f9
-  data.tar.gz: 2176e44cd407be37c0137f48c77f7a56550cddacc36a5a8043643049b416138c1b96b35c850c630dfd1f9febdc4e72e9224fb055b8dfd871c54e9bef5fbdbe47
+  metadata.gz: e9a744de7a691f637c48289c4647a3b369e89659c02ba4140b57addd8dbf5c3b98e769016b8cdd9158563622570aa82e98bd36fba237f6ec8802590c6c63f943
+  data.tar.gz: 698cdc39714da510b577bd7bc98d8da174da6c13c32c32c47041e51c64d444e7d5368d3f090b3ec239092fa3db01c93248a7ef14512491c0d6e2122b2446d58c

data/lib/audiences.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Audiences
   def aud_to_str(aud)
     "#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
@@ -5,7 +7,7 @@ module Audiences
 
   def str_to_aud(str)
     # TODO: move to common package
-    parts = str.split(".")
+    parts = str.split('.')
     aud = Streamdal::Protos::Audience.new
     aud.service_name = parts[0]
     aud.component_name = parts[1]
@@ -20,9 +22,7 @@ module Audiences
 
   def _add_audience(aud)
     # Add an audience to the local cache map and send to server
-    if _seen_audience(aud)
-      return
-    end
+    return if _seen_audience(aud)
 
     @audiences[aud_to_str(aud)] = aud
 
@@ -42,4 +42,4 @@ module Audiences
       @stub.new_audience(req, metadata: _metadata)
     end
   end
-end
+end

data/lib/hostfunc.rb

@@ -1,4 +1,6 @@
-require "steps/sp_steps_kv_pb"
+require 'steps/sp_steps_kv_pb'
+require 'sp_wsm_pb'
+require 'steps/sp_steps_httprequest_pb'
 
 module Streamdal
   class HostFunc
@@ -14,7 +16,7 @@ module Streamdal
     # kv_exists is a host function that is used to check if a key exists in the KV store
     def kv_exists(caller, ptr, len)
 
-      data = caller.export("memory").to_memory.read(ptr, len)
+      data = caller.export('memory').to_memory.read(ptr, len)
 
       # Read request from memory and decode into HttpRequest
       req = Streamdal::Protos::KVStep.decode(data)
@@ -35,32 +37,47 @@ module Streamdal
     ##
     # http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
     def http_request(caller, ptr, len)
-      data = caller.export("memory").to_memory.read(ptr, len)
+      data = caller.export('memory').to_memory.read(ptr, len)
 
       # Read request from memory and decode into HttpRequest
-      req = Streamdal::Protos::HttpRequest.decode(data)
+      req = Streamdal::Protos::WASMRequest.decode(data)
+
+      begin
+        req_body = _get_request_body_for_mode(req)
+      rescue => e
+        return _http_request_response(caller, 400, e.to_s, {})
+      end
 
       # Attempt to make HTTP request
       # On error, return a mock 400 response with the error as the body
       begin
-        response = _make_http_request(req)
+        response = _make_http_request(req.step.http_request.request, req_body)
       rescue => e
-        wasm_resp = Streamdal::Protos::HttpResponse.new
-        wasm_resp.code = 400
-        wasm_resp.body = "Unable to execute HTTP request: #{e}"
-        return wasm_resp
+        return _http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
       end
 
-      # Successful request, build the proto from the httparty response
+      # Convert body to utf8
+      out = encode(response.body)
+
+      _http_request_response(caller, response.code, out, response.headers)
+    end
+
+    def encode(str)
+      str.force_encoding('ascii-8bit').encode('utf-8', invalid: :replace, undef: :replace, replace: '?')
+    end
+
+    private
+
+    def _http_request_response(caller, code, body, headers)
       wasm_resp = Streamdal::Protos::HttpResponse.new
-      wasm_resp.code = response.code
-      wasm_resp.body = response.body
+      wasm_resp.code = code
+      wasm_resp.body = body
       wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
 
       # Headers can have multiple values, but we just want a map[string]string here for simplicity
       # The client can pase by the delimiter ";" if needed.
-      response.headers.each do |k, values|
-        wasm_resp.headers[k] = values.kind_of?(Array) ? values.join("; ") : values
+      headers.each do |k, values|
+        wasm_resp.headers[k] = values.is_a?(Array) ? values.join('; ') : values
       end
 
       # Write the HttpResponse proto message to WASM memory
@@ -68,17 +85,35 @@ module Streamdal
       write_to_memory(caller, wasm_resp)
     end
 
-    private
+    def _get_request_body_for_mode(req)
+      http_req = req.step.http_request.request
+
+      case http_req.body_mode
+      when :HTTP_REQUEST_BODY_MODE_INTER_STEP_RESULT
+        raise 'Inter step result is empty' if req.inter_step_result.nil?
+
+        detective_res = req.inter_step_result.detective_result
+
+        raise 'Detective result is empty' if detective_res.nil?
+
+        # Wipe values to prevent PII from being leaked
+        detective_res.matches.each { |step_res|
+          step_res.value = ''
+        }
+
+        req.inter_step_result.to_json
+      else
+        http_req.body
+      end
+    end
 
     ##
     # Performs an http request
-    def _make_http_request(req)
-      if req.nil?
-        raise "req is required"
-      end
+    def _make_http_request(req, body)
+      raise 'req is required' if req.nil?
 
       options = {
-        headers: { "Content-Type": "application/json", },
+        headers: { "Content-Type": 'application/json' }
       }
 
       req.headers.each { |key, value| options.headers[key] = value }
@@ -87,15 +122,15 @@ module Streamdal
       when :HTTP_REQUEST_METHOD_GET
         return HTTParty.get(req.url)
       when :HTTP_REQUEST_METHOD_POST
-        options.body = req.body
+        options.body = body
         return HTTParty.post(req.url, options)
       when :HTTP_REQUEST_METHOD_PUT
-        options.body = req.body
+        options.body = body
         return HTTParty.put(req.url, options)
       when :HTTP_REQUEST_METHOD_DELETE
         return HTTParty.delete(req.url)
       when :HTTP_REQUEST_METHOD_PATCH
-        options.body = req.body
+        options.body = body
         return HTTParty.patch(req.url, options)
       when :HTTP_REQUEST_METHOD_HEAD
         return HTTParty.head(req.url)
@@ -105,5 +140,25 @@ module Streamdal
         raise ArgumentError, "Invalid http request method: #{req.method}"
       end
     end
+
+    # Called by host functions to write memory to wasm instance so that
+    # the wasm module can read the result of a host function call
+    def write_to_memory(caller, res)
+      alloc = caller.export('alloc').to_func
+      memory = caller.export('memory').to_memory
+
+      # Serialize protobuf message
+      resp = res.to_proto
+
+      # Allocate memory for response
+      resp_ptr = alloc.call(resp.length)
+
+      # Write response to memory
+      memory.write(resp_ptr, resp)
+
+      # return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
+      resp_ptr << 32 | resp.length
+    end
+
   end
 end

data/lib/kv.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Streamdal
   class KeyValue
     def initialize
@@ -49,4 +51,4 @@ module Streamdal
       end
     end
   end
-end
+end

data/lib/metrics.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Streamdal
   class Counter
     attr_accessor :last_updated, :name, :aud, :labels
@@ -6,15 +8,15 @@ module Streamdal
       @name = name
       @aud = aud
       @labels = labels
-      @value = 0
-      @last_updated = Time::now
+      @value = value
+      @last_updated = Time.now
       @value_mtx = Mutex.new
     end
 
     def incr(val)
       @value_mtx.synchronize do
-        @value = @value + val
-        @last_updated = Time::now
+        @value += val
+        @last_updated = Time.now
       end
     end
 
@@ -33,18 +35,18 @@ module Streamdal
 
   class Metrics
 
-    COUNTER_CONSUME_BYTES = "counter_consume_bytes"
-    COUNTER_CONSUME_PROCESSED = "counter_consume_processed"
-    COUNTER_CONSUME_ERRORS = "counter_consume_errors"
-    COUNTER_PRODUCE_BYTES = "counter_produce_bytes"
-    COUNTER_PRODUCE_PROCESSED = "counter_produce_processed"
-    COUNTER_PRODUCE_ERRORS = "counter_produce_errors"
-    COUNTER_NOTIFY = "counter_notify"
-    COUNTER_DROPPED_TAIL_MESSAGES = "counter_dropped_tail_messages"
-    COUNTER_CONSUME_BYTES_RATE = "counter_consume_bytes_rate"
-    COUNTER_PRODUCE_BYTES_RATE = "counter_produce_bytes_rate"
-    COUNTER_CONSUME_PROCESSED_RATE = "counter_consume_processed_rate"
-    COUNTER_PRODUCE_PROCESSED_RATE = "counter_produce_processed_rate"
+    COUNTER_CONSUME_BYTES = 'counter_consume_bytes'
+    COUNTER_CONSUME_PROCESSED = 'counter_consume_processed'
+    COUNTER_CONSUME_ERRORS = 'counter_consume_errors'
+    COUNTER_PRODUCE_BYTES = 'counter_produce_bytes'
+    COUNTER_PRODUCE_PROCESSED = 'counter_produce_processed'
+    COUNTER_PRODUCE_ERRORS = 'counter_produce_errors'
+    COUNTER_NOTIFY = 'counter_notify'
+    COUNTER_DROPPED_TAIL_MESSAGES = 'counter_dropped_tail_messages'
+    COUNTER_CONSUME_BYTES_RATE = 'counter_consume_bytes_rate'
+    COUNTER_PRODUCE_BYTES_RATE = 'counter_produce_bytes_rate'
+    COUNTER_CONSUME_PROCESSED_RATE = 'counter_consume_processed_rate'
+    COUNTER_PRODUCE_PROCESSED_RATE = 'counter_produce_processed_rate'
 
     WORKER_POOL_SIZE = 3
     DEFAULT_COUNTER_REAPER_INTERVAL = 10
@@ -54,9 +56,7 @@ module Streamdal
     CounterEntry = Struct.new(:name, :aud, :labels, :value)
 
     def initialize(cfg)
-      if cfg.nil?
-        raise ArgumentError, "cfg is nil"
-      end
+      raise ArgumentError, 'cfg is nil' if cfg.nil?
 
       @cfg = cfg
       @log = cfg[:log]
@@ -80,30 +80,22 @@ module Streamdal
 
       # Exit any remaining threads
       @workers.each do |w|
-        if w.running?
-          w.exit
-        end
+        w.exit if w.running?
       end
     end
 
     def self.composite_id(counter_name, labels = {})
-      if labels.nil?
-        labels = {}
-      end
-      "#{counter_name}-#{labels.values.join("-")}".freeze
+      labels = {} if labels.nil?
+      "#{counter_name}-#{labels.values.join('-')}"
     end
 
     def get_counter(ce)
-      if ce.nil?
-        raise ArgumentError, "ce is nil"
-      end
+      raise ArgumentError, 'ce is nil' if ce.nil?
 
-      k = Metrics::composite_id(ce.name, ce.labels)
+      k = Metrics.composite_id(ce.name, ce.labels)
 
       @counters_mtx.synchronize do
-        if @counters.key?(k)
-          @counters[k]
-        end
+        @counters[k] if @counters.key?(k)
       end
 
       # No counter exists, create a new one and return it
@@ -114,7 +106,7 @@ module Streamdal
       c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
 
       @counters_mtx.synchronize do
-        @counters[Metrics::composite_id(ce.name, ce.labels)] = c
+        @counters[Metrics.composite_id(ce.name, ce.labels)] = c
       end
 
       c
@@ -166,31 +158,29 @@ module Streamdal
     def _run_publisher
       # Background thread that reads values from counters, adds them to the publish queue, and then
       # resets the counter's value back to zero
-      unless @exit
-        @log.debug("Starting publisher")
+      return if @exit
 
-        # Sleep on startup and then and between each loop run
-        sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
+      @log.debug('Starting publisher')
 
-        # Get all counters
-        # Loop over each counter, get the value,
-        #   if value > 0, continue
-        #   if now() - last_updated > 10 seconds, remove counter
-        # Grab copy of counters
-        @counters_mtx.lock
-        new_counters = @counters.dup
-        @counters_mtx.unlock
+      # Sleep on startup and then and between each loop run
+      sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
 
-        new_counters.each do |_, counter|
-          if counter.val == 0
-            next
-          end
+      # Get all counters
+      # Loop over each counter, get the value,
+      #   if value > 0, continue
+      #   if now() - last_updated > 10 seconds, remove counter
+      # Grab copy of counters
+      @counters_mtx.lock
+      new_counters = @counters.dup
+      @counters_mtx.unlock
 
-          ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
-          counter.reset
+      new_counters.each_value do |counter|
+        next if counter.val.zero?
 
-          @publish_queue.push(ce)
-        end
+        ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
+        counter.reset
+
+        @publish_queue.push(ce)
       end
     end
 
@@ -199,9 +189,8 @@ module Streamdal
 
       until @exit
         ce = @incr_queue.pop
-        if ce.nil?
-          next
-        end
+        next if ce.nil?
+
         begin
           _publish_metrics(ce)
         rescue => e
@@ -213,7 +202,7 @@ module Streamdal
     end
 
     def _run_reaper
-      @log.debug("Starting reaper")
+      @log.debug('Starting reaper')
 
       until @exit
         # Sleep on startup and then and between each loop run
@@ -226,11 +215,9 @@ module Streamdal
         # Grab copy of counters
         @counters_mtx.synchronize do
           @counters.each do |name, counter|
-            if counter.val > 0
-              next
-            end
+            next if counter.val.positive?
 
-            if Time::now - counter.last_updated > DEFAULT_COUNTER_TTL
+            if Time.now - counter.last_updated > DEFAULT_COUNTER_TTL
               @log.debug("Reaping counter '#{name}'")
               @counters.delete(name)
             end
@@ -238,7 +225,7 @@ module Streamdal
         end
       end
 
-      @log.debug("Exiting reaper")
+      @log.debug('Exiting reaper')
     end
 
     def _run_incrementer_worker(worker_id)
@@ -259,7 +246,7 @@ module Streamdal
 
     # Returns metadata for gRPC requests to the internal gRPC API
     def _metadata
-      { "auth-token" => @cfg[:streamdal_token].to_s }
+      { 'auth-token' => @cfg[:streamdal_token].to_s }
     end
   end
-end
+end

data/lib/schema.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 include Streamdal::Protos
 
 module Schemas
@@ -8,29 +10,21 @@ module Schemas
   end
 
   def _get_schema(aud)
-    if @schemas.key?(aud_to_str(aud))
-      return @schemas[aud_to_str(aud)].json_schema
-    end
+    return @schemas[aud_to_str(aud)].json_schema if @schemas.key?(aud_to_str(aud))
 
-    ""
+    ''
   end
 
   def _handle_schema(aud, step, wasm_resp)
     # Only handle schema steps
-    if step.infer_schema.nil?
-      return nil
-    end
+    return nil if step.infer_schema.nil?
 
     # Only successful schema inferences
-    if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
-      return nil
-    end
+    return nil if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
 
     # If existing schema matches, do nothing
     existing_schema = _get_schema(aud)
-    if existing_schema == wasm_resp.output_step
-      return nil
-    end
+    return nil if existing_schema == wasm_resp.output_step
 
     _set_schema(aud, wasm_resp.output_step)
 
@@ -44,4 +38,4 @@ module Schemas
       @stub.send_schema(req, metadata: _metadata)
     end
   end
-end
+end

data/lib/streamdal.rb

@@ -8,11 +8,11 @@ require 'sp_sdk_pb'
 require 'sp_common_pb'
 require 'sp_info_pb'
 require 'sp_internal_pb'
-require "sp_internal_services_pb"
+require 'sp_internal_services_pb'
 require 'sp_pipeline_pb'
-require "sp_wsm_pb"
-require "steps/sp_steps_httprequest_pb"
-require "steps/sp_steps_kv_pb"
+require 'sp_wsm_pb'
+require 'steps/sp_steps_httprequest_pb'
+require 'steps/sp_steps_kv_pb'
 require 'timeout'
 require 'google/protobuf'
 require_relative 'audiences'
@@ -31,7 +31,6 @@ DEFAULT_HEARTBEAT_INTERVAL = 1 # 1 second
 MAX_PAYLOAD_SIZE = 1024 * 1024 # 1 megabyte
 
 module Streamdal
-
   OPERATION_TYPE_PRODUCER = 2
   OPERATION_TYPE_CONSUMER = 1
   CLIENT_TYPE_SDK = 1
@@ -39,7 +38,6 @@ module Streamdal
 
   # Data class to hold instantiated wasm functions
   class WasmFunction
-
     ##
     # Instance of an initialized wasm module and associated memory store
 
@@ -66,7 +64,6 @@ module Streamdal
     end
   end
 
-
   class Client
 
     ##
@@ -117,20 +114,14 @@ module Streamdal
 
       # Exit any remaining threads
       @workers.each do |w|
-        if w.running?
-          w.exit
-        end
+        w.exit if w.running?
       end
     end
 
     def process(data, audience)
-      if data.length == 0
-        raise "data is required"
-      end
+      raise 'data is required' if data.empty?
 
-      if audience.nil?
-        raise "audience is required"
-      end
+      raise 'audience is required' if audience.nil?
 
       resp = Streamdal::Protos::SDKResponse.new
       resp.status = :EXEC_STATUS_TRUE
@@ -144,8 +135,8 @@ module Streamdal
         "operation_type": aud.operation_type,
         "operation": aud.operation_name,
         "component": aud.component_name,
-        "pipeline_name": "",
-        "pipeline_id": "",
+        "pipeline_name": '',
+        "pipeline_id": '',
       }
 
       # TODO: metrics
@@ -166,7 +157,7 @@ module Streamdal
       if payload_size > MAX_PAYLOAD_SIZE
         # TODO: add metrics
         resp.status = :EXEC_STATUS_ERROR
-        resp.error = "payload size exceeds maximum allowed size"
+        resp.error = 'payload size exceeds maximum allowed size'
         resp
       end
 
@@ -174,8 +165,8 @@ module Streamdal
       original_data = data
 
       pipelines = _get_pipelines(aud)
-      if pipelines.length == 0
-        _send_tail(aud, "", original_data, original_data)
+      if pipelines.empty?
+        _send_tail(aud, '', original_data, original_data)
         return resp
       end
 
@@ -215,13 +206,9 @@ module Streamdal
               break
             end
 
-            if @cfg[:dry_run]
-              @log.debug "Running step '#{step.name}' in dry-run mode"
-            end
+            @log.debug "Running step '#{step.name}' in dry-run mode" if @cfg[:dry_run]
 
-            if wasm_resp.output_payload.length > 0
-              resp.data = wasm_resp.output_payload
-            end
+            resp.data = wasm_resp.output_payload if wasm_resp.output_payload.length.positive?
 
             _handle_schema(aud, step, wasm_resp)
 
@@ -296,10 +283,10 @@ module Streamdal
         end # pipelines.each
       end # timeout
 
-      _send_tail(aud, "", original_data, resp.data)
+      _send_tail(aud, '', original_data, resp.data)
 
       if @cfg[:dry_run]
-        @log.debug "Dry-run, setting response data to original data"
+        @log.debug 'Dry-run, setting response data to original data'
         resp.data = original_data
       end
 
@@ -309,42 +296,32 @@ module Streamdal
     private
 
     def _validate_cfg(cfg)
-      if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
-        raise "streamdal_url is required"
-      end
+      raise 'streamdal_url is required' if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
 
-      if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
-        raise "streamdal_token is required"
-      end
+      raise 'streamdal_token is required' if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
 
-      if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
-        raise "service_name is required"
-      end
+      raise 'service_name is required' if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
 
       if cfg[:log].nil? || cfg[:streamdal_token].empty?
-        logger = Logger.new(STDOUT)
+        logger = Logger.new($stdout)
         logger.level = Logger::ERROR
         cfg[:log] = logger
       end
 
-      if cfg[:pipeline_timeout].nil?
-        cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT
-      end
+      cfg[:pipeline_timeout] = DEFAULT_PIPELINE_TIMEOUT if cfg[:pipeline_timeout].nil?
 
-      if cfg[:step_timeout].nil?
-        cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT
-      end
+      cfg[:step_timeout] = DEFAULT_STEP_TIMEOUT if cfg[:step_timeout].nil?
     end
 
     def _handle_command(cmd)
       case cmd.command.to_s
-      when "kv"
+      when 'kv'
         _handle_kv(cmd)
-      when "tail"
+      when 'tail'
         _handle_tail_request(cmd)
-      when "set_pipelines"
+      when 'set_pipelines'
         _set_pipelines(cmd)
-      when "keep_alive"
+      when 'keep_alive'
         # Do nothing
       else
         @log.error "unknown command type #{cmd.command}"
@@ -378,13 +355,11 @@ module Streamdal
     end
 
     def _set_pipelines(cmd)
-      if cmd.nil?
-        raise "cmd is required"
-      end
+      raise 'cmd is required' if cmd.nil?
 
       cmd.set_pipelines.pipelines.each_with_index { |p, pIdx|
         p.steps.each_with_index { |step, idx|
-          if step._wasm_bytes == ""
+          if step._wasm_bytes == ''
             if cmd.set_pipelines.wasm_modules.has_key?(step._wasm_id)
               step._wasm_bytes = cmd.set_pipelines.wasm_modules[step._wasm_id].bytes
               cmd.set_pipelines.pipelines[pIdx].steps[idx] = step
@@ -416,19 +391,17 @@ module Streamdal
     def _get_function(step)
       # We cache functions so we can eliminate the wasm bytes from steps to save on memory
       # And also to avoid re-initializing the same function multiple times
-      if @functions.key?(step._wasm_id)
-        return @functions[step._wasm_id]
-      end
+      return @functions[step._wasm_id] if @functions.key?(step._wasm_id)
 
       engine = Wasmtime::Engine.new
       mod = Wasmtime::Module.new(engine, step._wasm_bytes)
       linker = Wasmtime::Linker.new(engine, wasi: true)
 
-      linker.func_new("env", "httpRequest", [:i32, :i32], [:i64]) do |caller, ptr, len|
+      linker.func_new('env', 'httpRequest', %i[i32 i32], [:i64]) do |caller, ptr, len|
         @hostfunc.http_request(caller, ptr, len)
       end
 
-      linker.func_new("env", "kvExists", [:i32, :i32], [:i64]) do |caller, ptr, len|
+      linker.func_new('env', 'kvExists', %i[i32 i32], [:i64]) do |caller, ptr, len|
         @hostfunc.kv_exists(caller, ptr, len)
       end
 
@@ -442,8 +415,6 @@ module Streamdal
 
       instance = linker.instantiate(store, mod)
 
-      # TODO: host funcs
-
       # Store in cache
       func = WasmFunction.new
       func.instance = instance
@@ -454,17 +425,11 @@ module Streamdal
     end
 
     def _call_wasm(step, data, isr)
-      if step.nil?
-        raise "step is required"
-      end
+      raise 'step is required' if step.nil?
 
-      if data.nil?
-        raise "data is required"
-      end
+      raise 'data is required' if data.nil?
 
-      if isr.nil?
-        isr = Streamdal::Protos::InterStepResult.new
-      end
+      isr = Streamdal::Protos::InterStepResult.new if isr.nil?
 
       req = Streamdal::Protos::WASMRequest.new
       req.step = step.clone
@@ -480,8 +445,8 @@ module Streamdal
         resp = Streamdal::Protos::WASMResponse.new
         resp.exit_code = :WASM_EXIT_CODE_ERROR
         resp.exit_msg = "Failed to execute WASM: #{e}"
-        resp.output_payload = ""
-        return resp
+        resp.output_payload = ''
+        resp
       end
     end
 
@@ -500,9 +465,9 @@ module Streamdal
 
       ci = Streamdal::Protos::ClientInfo.new
       ci.client_type = :CLIENT_TYPE_SDK
-      ci.library_name = "ruby-sdk"
-      ci.library_version = "0.0.1"
-      ci.language = "ruby"
+      ci.library_name = 'ruby-sdk'
+      ci.library_version = '0.0.1'
+      ci.language = 'ruby'
       ci.arch = arch
       ci.os = os
 
@@ -511,23 +476,21 @@ module Streamdal
 
     # Returns metadata for gRPC requests to the internal gRPC API
     def _metadata
-      { "auth-token" => @cfg[:streamdal_token].to_s }
+      { 'auth-token' => @cfg[:streamdal_token].to_s }
     end
 
     def _register
-      @log.info("register started")
+      @log.info('register started')
 
       # Register with Streamdal External gRPC API
       resps = @stub.register(_gen_register_request, metadata: _metadata)
       resps.each do |r|
-        if @exit
-          break
-        end
+        break if @exit
 
         _handle_command(r)
       end
 
-      @log.info("register exited")
+      @log.info('register exited')
     end
 
     def _exec_wasm(req)
@@ -535,14 +498,14 @@ module Streamdal
 
       # Empty out _wasm_bytes, we don't need it anymore
       # TODO: does this actually update the original object?
-      req.step._wasm_bytes = ""
+      req.step._wasm_bytes = ''
 
       data = req.to_proto
 
-      memory = wasm_func.instance.export("memory").to_memory
-      alloc = wasm_func.instance.export("alloc").to_func
-      dealloc = wasm_func.instance.export("dealloc").to_func
-      f = wasm_func.instance.export("f").to_func
+      memory = wasm_func.instance.export('memory').to_memory
+      alloc = wasm_func.instance.export('alloc').to_func
+      dealloc = wasm_func.instance.export('dealloc').to_func
+      f = wasm_func.instance.export('f').to_func
 
       start_ptr = alloc.call(data.length)
 
@@ -568,9 +531,7 @@ module Streamdal
 
       _add_audience(aud)
 
-      if @pipelines.key?(aud_str)
-        return @pipelines[aud_str]
-      end
+      return @pipelines[aud_str] if @pipelines.key?(aud_str)
 
       []
     end
@@ -581,7 +542,7 @@ module Streamdal
         req.session_id = @session_id
         req.audiences = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Audience, [])
 
-        @audiences.each do |_, aud|
+        @audiences.each_value do |aud|
           req.audiences.push(aud)
         end
 
@@ -616,18 +577,14 @@ module Streamdal
 
     def _get_active_tails_for_audience(aud)
       aud_str = aud_to_str(aud)
-      if @tails.key?(aud_str)
-        return @tails[aud_str].values
-      end
+      return @tails[aud_str].values if @tails.key?(aud_str)
 
       []
     end
 
     def _send_tail(aud, pipeline_id, original_data, new_data)
       tails = _get_active_tails_for_audience(aud)
-      if tails.length == 0
-        return nil
-      end
+      return nil if tails.empty?
 
       tails.each do |tail|
         req = Streamdal::Protos::TailResponse.new
@@ -644,19 +601,13 @@ module Streamdal
     end
 
     def _notify_condition(pipeline, step, aud, cond, data, cond_type)
-      if cond.nil?
-        return nil
-      end
+      return nil if cond.nil?
 
-      if cond.notification.nil?
-        return nil
-      end
+      return nil if cond.notification.nil?
 
-      @log.debug "Notifying"
+      @log.debug 'Notifying'
 
-      if @cfg[:dry_run]
-        return nil
-      end
+      return nil if @cfg[:dry_run]
 
       @metrics.incr(CounterEntry.new(Metrics::COUNTER_NOTIFY, aud, {
         "service": @cfg[:service_name],
@@ -706,15 +657,12 @@ module Streamdal
       t.start_tail_workers
 
       _set_active_tail(t)
-
     end
 
     def _set_active_tail(tail)
       key = aud_to_str(tail.request.audience)
 
-      unless @tails.key?(key)
-        @tails[key] = {}
-      end
+      @tails[key] = {} unless @tails.key?(key)
 
       @tails[key][tail.request.id] = tail
     end
@@ -722,9 +670,7 @@ module Streamdal
     def _set_paused_tail(tail)
       key = aud_to_str(tail.request.aud)
 
-      unless @paused_tails.key?(key)
-        @paused_tails[key] = {}
-      end
+      @paused_tails[key] = {} unless @paused_tails.key?(key)
 
       @paused_tails[key][tail.request.id] = tail
     end
@@ -739,18 +685,14 @@ module Streamdal
         # Remove from active tails
         @tails[key].delete(cmd.tail.request.id)
 
-        if @tails[key].length == 0
-          @tails.delete(key)
-        end
+        @tails.delete(key) if @tails[key].empty?
       end
 
-      if @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
-        @paused_tails[key].delete(cmd.tail.request.id)
+      return unless @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
 
-        if @paused_tails[key].length == 0
-          @paused_tails.delete(key)
-        end
-      end
+      @paused_tails[key].delete(cmd.tail.request.id)
+
+      @paused_tails.delete(key) if @paused_tails[key].empty?
     end
 
     def _stop_all_tails
@@ -766,9 +708,7 @@ module Streamdal
           t.stop_tail
           tails[aud].delete(tail.request.id)
 
-          if tails[aud].length == 0
-            tails.delete(aud)
-          end
+          tails.delete(aud) if tails[aud].empty?
         end
       end
     end
@@ -799,54 +739,30 @@ module Streamdal
     def _remove_active_tail(aud, tail_id)
       key = aud_to_str(aud)
 
-      if @tails.key?(key) && @tails[key].key?(tail_id)
-        t = @tails[key][tail_id]
-        t.stop_tail
+      return unless @tails.key?(key) && @tails[key].key?(tail_id)
 
-        @tails[key].delete(tail_id)
+      t = @tails[key][tail_id]
+      t.stop_tail
 
-        if @tails[key].length == 0
-          @tails.delete(key)
-        end
+      @tails[key].delete(tail_id)
 
-        t
-      end
+      @tails.delete(key) if @tails[key].empty?
+
+      t
     end
 
     def _remove_paused_tail(aud, tail_id)
       key = aud_to_str(aud)
 
-      if @paused_tails.key?(key) && @paused_tails[key].key?(tail_id)
-        t = @paused_tails[key][tail_id]
+      return unless @paused_tails.key?(key) && @paused_tails[key].key?(tail_id)
 
-        @paused_tails[key].delete(tail_id)
+      t = @paused_tails[key][tail_id]
 
-        if @paused_tails[key].length == 0
-          @paused_tails.delete(key)
-        end
+      @paused_tails[key].delete(tail_id)
 
-        t
-      end
-    end
-
-    # Called by host functions to write memory to wasm instance so that
-    # the wasm module can read the result of a host function call
-    def write_to_memory(caller, res)
-      alloc = caller.export("alloc").to_func
-      memory = caller.export("memory").to_memory
-
-      # Serialize protobuf message
-      resp = res.to_proto
+      @paused_tails.delete(key) if @paused_tails[key].empty?
 
-      # Allocate memory for response
-      resp_ptr = alloc.call(resp.length)
-
-      # Write response to memory
-      memory.write(resp_ptr, resp)
-
-      # return 64bit integer where first 32 bits is the pointer, and the last 32 is the length
-      resp_ptr << 32 | resp.length
+      t
     end
-
   end
-end
+end

data/lib/tail.rb

@@ -1,5 +1,4 @@
-# TODO: implement token bucket limiter
-require "bozos_buckets"
+require 'bozos_buckets'
 
 NUM_TAIL_WORKERS = 2
 MIN_TAIL_RESPONSE_INTERVAL_MS = 100
@@ -15,18 +14,19 @@ module Streamdal
       @logger = log
       @metrics = metrics
       @active = active
-      @last_msg = Time::at(0)
+      @last_msg = Time.at(0)
       @queue = Queue.new
       @workers = []
 
       # Only use rate limiting if sample_options is set
-      unless request.sample_options.nil?
-        @limiter = BozosBuckets::Bucket.new(
-          initial_token_count: request.sample_options.sample_rate,
-          refill_rate: request.sample_options.sample_interval_seconds,
-          max_token_count: request.sample_options.sample_rate
-        )
-      end
+      return if request.sample_options.nil?
+
+      @limiter = BozosBuckets::Bucket.new(
+        initial_token_count: request.sample_options.sample_rate,
+        refill_rate: request.sample_options.sample_interval_seconds,
+        max_token_count: request.sample_options.sample_rate
+      )
+
     end
 
     def start_tail_workers
@@ -43,11 +43,8 @@ module Streamdal
       sleep(1)
 
       @workers.each do |worker|
-        if worker.alive?
-          worker.exit
-        end
+        worker.exit if worker.alive?
       end
-
     end
 
     def start_tail_worker(worker_id)
@@ -63,35 +60,32 @@ module Streamdal
           next
         end
 
-        if Time::now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
+        if Time.now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
           sleep(MIN_TAIL_RESPONSE_INTERVAL_MS)
           @metrics.incr(Metrics::CounterEntry.new(COUNTER_DROPPED_TAIL_MESSAGES, nil, {}, 1))
           @logger.debug("Dropped tail message for '#{@request.id}' due to rate limiting")
           next
         end
 
-        unless stub.nil?
-          tail_response = @queue.pop(non_block = false)
-          @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
+        next if stub.nil?
+
+        tail_response = @queue.pop(false)
+        @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
 
-          begin
-            stub.send_tail([tail_response], metadata: { "auth-token" => @auth_token })
-          rescue => e
-            @logger.error("Error sending tail request: #{e}")
-          end
+        begin
+          stub.send_tail([tail_response], metadata: { 'auth-token' => @auth_token })
+        rescue Error => e
+          @logger.error("Error sending tail request: #{e}")
         end
       end
 
       @logger.debug "Tail worker #{worker_id} exited"
-
     end
 
     def should_send
-      if @limiter.nil?
-        true
-      end
+      true if @limiter.nil?
 
       @limiter.use_tokens(1)
     end
   end
-end
+end

data/lib/wasm_spec.rb

@@ -0,0 +1,259 @@
+require_relative 'spec_helper'
+require 'steps/sp_steps_httprequest_pb'
+require 'steps/sp_steps_detective_pb'
+require 'steps/sp_steps_transform_pb'
+require_relative 'streamdal'
+
+class TestClient < Streamdal::Client
+
+  attr_accessor :kv
+
+  # Ignore rubocop warning
+  # rubocop:disable Lint/MissingSuper
+  def initialize
+    @cfg = {
+      step_timeout: 2
+    }
+    @functions = {}
+
+    logger = Logger.new($stdout)
+    logger.level = Logger::ERROR
+
+    @log = logger
+    @kv = Streamdal::KeyValue.new
+    @hostfunc = Streamdal::HostFunc.new(@kv)
+  end
+end
+
+RSpec.describe 'WASM' do
+  let(:client) { TestClient.new }
+
+  context '_call_wasm' do
+    it 'raises an error if step is nil' do
+      expect { client.send(:_call_wasm, nil, nil, nil) }.to raise_error('step is required')
+    end
+
+    it 'raises an error if data is nil' do
+      step = Streamdal::Protos::HttpRequestStep.new
+      expect { client.send(:_call_wasm, step, nil, nil) }.to raise_error('data is required')
+    end
+  end
+
+  context 'detective.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'detective.wasm'))
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'detective'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+      @step.detective = Streamdal::Protos::DetectiveStep.new
+      @step.detective.path = 'object.field'
+      @step.detective.args = Google::Protobuf::RepeatedField.new(:string, ['streamdal'])
+      @step.detective.negate = false
+      @step.detective.type = :DETECTIVE_TYPE_STRING_CONTAINS_ANY
+    end
+
+    it 'detects email in JSON payload' do
+      data = '{"object":{"field":"streamdal@gmail.com"}}'
+
+      res = client.send(:_call_wasm, @step, data, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq(data)
+
+    end
+
+    it 'does not detect email in JSON payload' do
+      data = '{"object":{"field":"mark@gmail.com"}}'
+
+      res = client.send(:_call_wasm, @step, data, nil)
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
+    end
+  end
+
+  context 'httprequest.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'httprequest.wasm'))
+
+      http_req_step = Streamdal::Protos::HttpRequestStep.new
+      http_req_step.request = Streamdal::Protos::HttpRequest.new
+      http_req_step.request.url = 'https://www.google.com/404_me'
+      http_req_step.request.method = :HTTP_REQUEST_METHOD_GET
+      http_req_step.request.body = ''
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'http request'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+      @step.http_request = http_req_step
+    end
+
+    it 'returns false on 404' do
+      res = client.send(:_call_wasm, @step, '', nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_msg).to eq('Request returned non-200 response code: 404')
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
+    end
+  end
+
+  context 'inferschema.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'inferschema.wasm'))
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'schema inference'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+      @step.infer_schema = Streamdal::Protos::InferSchemaStep.new
+    end
+
+    it 'infers schema' do
+      payload = '{"object": {"payload": "test"}}'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expected_schema = '{"$schema":"http://json-schema.org/draft-07/schema#","properties":{"object":{"properties":{"payload":{"type":"string"}},"required":["payload"],"type":"object"}},"required":["object"],"type":"object"}'
+
+      expect(res).not_to be_nil
+      expect(res.exit_msg).to eq('inferred fresh schema')
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq(payload)
+      expect(res.output_step).to eq(expected_schema)
+    end
+  end
+
+  context 'transform.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'transform.wasm'))
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'transform'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+    end
+
+    it 'deletes a field from a payload' do
+      @step.transform = Streamdal::Protos::TransformStep.new
+      @step.transform.type = :TRANSFORM_TYPE_DELETE_FIELD
+      @step.transform.delete_field_options = Streamdal::Protos::TransformDeleteFieldOptions.new
+      @step.transform.delete_field_options.paths = Google::Protobuf::RepeatedField.new(:string, ['object.another'])
+
+      payload = '{"object": {"payload": "old val", "another": "field"}}'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_msg).to eq('Successfully transformed payload')
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq('{"object": {"payload": "old val"}}')
+    end
+
+    it 'replaces a fields value with a new one' do
+      @step.transform = Streamdal::Protos::TransformStep.new
+      @step.transform.type = :TRANSFORM_TYPE_REPLACE_VALUE
+      @step.transform.replace_value_options = Streamdal::Protos::TransformReplaceValueOptions.new
+      @step.transform.replace_value_options.path = 'object.payload'
+      @step.transform.replace_value_options.value = '"new val"'
+
+      payload = '{"object": {"payload": "old val"}}'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_msg).to eq('Successfully transformed payload')
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq('{"object": {"payload": "new val"}}')
+    end
+
+    it 'truncates the value of a field' do
+      @step.transform = Streamdal::Protos::TransformStep.new
+      @step.transform.type = :TRANSFORM_TYPE_TRUNCATE_VALUE
+      @step.transform.truncate_options = Streamdal::Protos::TransformTruncateOptions.new
+      @step.transform.truncate_options.type = :TRANSFORM_TRUNCATE_TYPE_LENGTH
+      @step.transform.truncate_options.path = 'object.payload'
+      @step.transform.truncate_options.value = 3
+
+      payload = '{"object": {"payload": "old val"}}'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_msg).to eq('Successfully transformed payload')
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq('{"object": {"payload": "old"}}')
+    end
+
+    it 'performs dynamic transformation' do
+      # TODO: add this test
+    end
+  end
+
+  context 'validjson.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'validjson.wasm'))
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'validate json'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+      @step.valid_json = Streamdal::Protos::ValidJSONStep.new
+
+    end
+
+    it 'validates a valid JSON payload' do
+
+      payload = '{"object": {"payload": "test"}}'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+      expect(res.output_payload).to eq(payload)
+    end
+
+    it 'returns false on invalid JSON payload' do
+      payload = '{"object": {"payload": "test"'
+      res = client.send(:_call_wasm, @step, payload, nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
+    end
+  end
+
+  context 'kv.wasm' do
+    before(:each) do
+      wasm_bytes = File.read(File.join(File.dirname(__FILE__), '..', 'test-assets', 'wasm', 'kv.wasm'))
+
+      client.kv.set('test', 'test')
+
+      @step = Streamdal::Protos::PipelineStep.new
+      @step.name = 'kv exists'
+      @step._wasm_bytes = wasm_bytes.b
+      @step._wasm_id = SecureRandom.uuid
+      @step._wasm_function = 'f'
+      @step.kv = Streamdal::Protos::KVStep.new
+      @step.kv.key = 'test'
+      @step.kv.mode = :KV_MODE_STATIC
+      @step.kv.action = :KV_ACTION_EXISTS
+    end
+
+    it 'returns true if a key exists' do
+      res = client.send(:_call_wasm, @step, '', nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_TRUE)
+    end
+
+    it 'returns false when a key doesnt exist' do
+      @step.kv.key = 'not_exists'
+      res = client.send(:_call_wasm, @step, '', nil)
+
+      expect(res).not_to be_nil
+      expect(res.exit_code).to eq(:WASM_EXIT_CODE_FALSE)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: streamdal
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Mark Gregan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-03 00:00:00.000000000 Z
+date: 2024-06-19 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: mark@streamdal.com
@@ -33,6 +33,7 @@ files:
 - lib/tail_spec.rb
 - lib/validation.rb
 - lib/validation_spec.rb
+- lib/wasm_spec.rb
 homepage: https://docs.streamdal.com
 licenses:
 - Apache-2.0
@@ -45,7 +46,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '='
     - !ruby/object:Gem::Version
-      version: 0.0.1
+      version: 0.0.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="