streamdal 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cdf556c16ea4239101cc21f803b78c458c5e2490a1c5df6b2211f5268cc94799
-  data.tar.gz: 1bca44db3048358a5bdf978504a8559c1a81ede836a11269c5ab8020698d71b6
+  metadata.gz: ca014e8334d38a589d1cd3aab58362ac3095fc077a93c076301f2e1552f9b375
+  data.tar.gz: 77110e0cc07e851e7b21ef831bed28495353bd70fce865159c41b18d4a40f7d0
 SHA512:
-  metadata.gz: 9f0fb7a83fc6e924f520d5402385bd857aff3a27884fd2a1aa9572a002723ad26d3849a2930829e6a05e450547ab1e80293992a693bbeb2bb9bacad8756011f9
-  data.tar.gz: 2176e44cd407be37c0137f48c77f7a56550cddacc36a5a8043643049b416138c1b96b35c850c630dfd1f9febdc4e72e9224fb055b8dfd871c54e9bef5fbdbe47
+  metadata.gz: 227c07d36fd54e8bf35a79b434417f59c80e45c72678ca82ada629b8268b744b614616d28b196e48b85c93f7a031330ebc8884a8ca826553eb647953069d7890
+  data.tar.gz: 9c83c8388b8e871d60b1b670b6f919b6a05878c731b6b8978545f9b782885c195cb30d02f1ec160cba58974ad53f99d9506410301185c5e036128eca4c56bb3b

data/lib/audiences.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Audiences
   def aud_to_str(aud)
     "#{aud.service_name}.#{aud.component_name}.#{aud.operation_type}.#{aud.operation_name}"
@@ -5,7 +7,7 @@ module Audiences
 
   def str_to_aud(str)
     # TODO: move to common package
-    parts = str.split(".")
+    parts = str.split('.')
     aud = Streamdal::Protos::Audience.new
     aud.service_name = parts[0]
     aud.component_name = parts[1]
@@ -20,9 +22,7 @@ module Audiences
 
   def _add_audience(aud)
     # Add an audience to the local cache map and send to server
-    if _seen_audience(aud)
-      return
-    end
+    return if _seen_audience(aud)
 
     @audiences[aud_to_str(aud)] = aud
 
@@ -42,4 +42,4 @@ module Audiences
       @stub.new_audience(req, metadata: _metadata)
     end
   end
-end
+end

data/lib/hostfunc.rb

@@ -1,4 +1,6 @@
-require "steps/sp_steps_kv_pb"
+require 'steps/sp_steps_kv_pb'
+require 'sp_wsm_pb'
+require 'steps/sp_steps_httprequest_pb'
 
 module Streamdal
   class HostFunc
@@ -14,7 +16,7 @@ module Streamdal
     # kv_exists is a host function that is used to check if a key exists in the KV store
     def kv_exists(caller, ptr, len)
 
-      data = caller.export("memory").to_memory.read(ptr, len)
+      data = caller.export('memory').to_memory.read(ptr, len)
 
       # Read request from memory and decode into HttpRequest
       req = Streamdal::Protos::KVStep.decode(data)
@@ -35,32 +37,40 @@ module Streamdal
     ##
     # http_request performs a http request on behalf of a wasm module since WASI cannot talk sockets
     def http_request(caller, ptr, len)
-      data = caller.export("memory").to_memory.read(ptr, len)
+      data = caller.export('memory').to_memory.read(ptr, len)
 
       # Read request from memory and decode into HttpRequest
-      req = Streamdal::Protos::HttpRequest.decode(data)
+      req = Streamdal::Protos::WASMRequest.decode(data)
+
+      begin
+        req_body = self._get_request_body_for_mode(req)
+      rescue => e
+        return self._http_request_response(caller, 400, e.to_s, {})
+      end
 
       # Attempt to make HTTP request
       # On error, return a mock 400 response with the error as the body
       begin
-        response = _make_http_request(req)
+        response = _make_http_request(req.step.http_request.request, req_body)
       rescue => e
-        wasm_resp = Streamdal::Protos::HttpResponse.new
-        wasm_resp.code = 400
-        wasm_resp.body = "Unable to execute HTTP request: #{e}"
-        return wasm_resp
+        return self._http_request_response(caller, 400, "Unable to execute HTTP request: #{e}", {})
       end
 
-      # Successful request, build the proto from the httparty response
+      self._http_request_response(caller, response.code, response.body, response.headers)
+    end
+
+    private
+
+    def _http_request_response(caller, code, body, headers)
       wasm_resp = Streamdal::Protos::HttpResponse.new
-      wasm_resp.code = response.code
-      wasm_resp.body = response.body
+      wasm_resp.code = code
+      wasm_resp.body = body
       wasm_resp.headers = Google::Protobuf::Map.new(:string, :string, {})
 
       # Headers can have multiple values, but we just want a map[string]string here for simplicity
       # The client can pase by the delimiter ";" if needed.
-      response.headers.each do |k, values|
-        wasm_resp.headers[k] = values.kind_of?(Array) ? values.join("; ") : values
+      headers.each do |k, values|
+        wasm_resp.headers[k] = values.is_a?(Array) ? values.join('; ') : values
       end
 
       # Write the HttpResponse proto message to WASM memory
@@ -68,17 +78,39 @@ module Streamdal
       write_to_memory(caller, wasm_resp)
     end
 
-    private
+    def _get_request_body_for_mode(req)
+      http_req = req.step.http_request.request
+
+      case http_req.body_mode
+      when :HTTP_REQUEST_BODY_MODE_INTER_STEP_RESULT
+        raise 'Inter step result is empty' if req.inter_step_result.nil?
+
+        detective_res = req.inter_step_result.detective_result
+
+        raise 'Detective result is empty' if detective_res.nil?
+
+        # Wipe values to prevent PII from being leaked
+        detective_res.matches.each { |step_res|
+          step_res.value = ''
+        }
+
+        req.inter_step_result.to_json
+      when :HTTP_REQUEST_BODY_MODE_STATIC
+        http_req.body
+      else
+        raise 'invalid http request body mode'
+      end
+    end
 
     ##
     # Performs an http request
-    def _make_http_request(req)
+    def _make_http_request(req, body)
       if req.nil?
-        raise "req is required"
+        raise 'req is required'
       end
 
       options = {
-        headers: { "Content-Type": "application/json", },
+        headers: { "Content-Type": 'application/json', },
       }
 
       req.headers.each { |key, value| options.headers[key] = value }
@@ -87,15 +119,15 @@ module Streamdal
       when :HTTP_REQUEST_METHOD_GET
         return HTTParty.get(req.url)
       when :HTTP_REQUEST_METHOD_POST
-        options.body = req.body
+        options.body = body
         return HTTParty.post(req.url, options)
       when :HTTP_REQUEST_METHOD_PUT
-        options.body = req.body
+        options.body = body
         return HTTParty.put(req.url, options)
       when :HTTP_REQUEST_METHOD_DELETE
         return HTTParty.delete(req.url)
       when :HTTP_REQUEST_METHOD_PATCH
-        options.body = req.body
+        options.body = body
         return HTTParty.patch(req.url, options)
       when :HTTP_REQUEST_METHOD_HEAD
         return HTTParty.head(req.url)

data/lib/kv.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Streamdal
   class KeyValue
     def initialize
@@ -49,4 +51,4 @@ module Streamdal
       end
     end
   end
-end
+end

data/lib/metrics.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Streamdal
   class Counter
     attr_accessor :last_updated, :name, :aud, :labels
@@ -6,15 +8,15 @@ module Streamdal
       @name = name
       @aud = aud
       @labels = labels
-      @value = 0
-      @last_updated = Time::now
+      @value = value
+      @last_updated = Time.now
       @value_mtx = Mutex.new
     end
 
     def incr(val)
       @value_mtx.synchronize do
-        @value = @value + val
-        @last_updated = Time::now
+        @value += val
+        @last_updated = Time.now
       end
     end
 
@@ -33,18 +35,18 @@ module Streamdal
 
   class Metrics
 
-    COUNTER_CONSUME_BYTES = "counter_consume_bytes"
-    COUNTER_CONSUME_PROCESSED = "counter_consume_processed"
-    COUNTER_CONSUME_ERRORS = "counter_consume_errors"
-    COUNTER_PRODUCE_BYTES = "counter_produce_bytes"
-    COUNTER_PRODUCE_PROCESSED = "counter_produce_processed"
-    COUNTER_PRODUCE_ERRORS = "counter_produce_errors"
-    COUNTER_NOTIFY = "counter_notify"
-    COUNTER_DROPPED_TAIL_MESSAGES = "counter_dropped_tail_messages"
-    COUNTER_CONSUME_BYTES_RATE = "counter_consume_bytes_rate"
-    COUNTER_PRODUCE_BYTES_RATE = "counter_produce_bytes_rate"
-    COUNTER_CONSUME_PROCESSED_RATE = "counter_consume_processed_rate"
-    COUNTER_PRODUCE_PROCESSED_RATE = "counter_produce_processed_rate"
+    COUNTER_CONSUME_BYTES = 'counter_consume_bytes'
+    COUNTER_CONSUME_PROCESSED = 'counter_consume_processed'
+    COUNTER_CONSUME_ERRORS = 'counter_consume_errors'
+    COUNTER_PRODUCE_BYTES = 'counter_produce_bytes'
+    COUNTER_PRODUCE_PROCESSED = 'counter_produce_processed'
+    COUNTER_PRODUCE_ERRORS = 'counter_produce_errors'
+    COUNTER_NOTIFY = 'counter_notify'
+    COUNTER_DROPPED_TAIL_MESSAGES = 'counter_dropped_tail_messages'
+    COUNTER_CONSUME_BYTES_RATE = 'counter_consume_bytes_rate'
+    COUNTER_PRODUCE_BYTES_RATE = 'counter_produce_bytes_rate'
+    COUNTER_CONSUME_PROCESSED_RATE = 'counter_consume_processed_rate'
+    COUNTER_PRODUCE_PROCESSED_RATE = 'counter_produce_processed_rate'
 
     WORKER_POOL_SIZE = 3
     DEFAULT_COUNTER_REAPER_INTERVAL = 10
@@ -54,9 +56,7 @@ module Streamdal
     CounterEntry = Struct.new(:name, :aud, :labels, :value)
 
     def initialize(cfg)
-      if cfg.nil?
-        raise ArgumentError, "cfg is nil"
-      end
+      raise ArgumentError, 'cfg is nil' if cfg.nil?
 
       @cfg = cfg
       @log = cfg[:log]
@@ -80,30 +80,22 @@ module Streamdal
 
       # Exit any remaining threads
       @workers.each do |w|
-        if w.running?
-          w.exit
-        end
+        w.exit if w.running?
       end
     end
 
     def self.composite_id(counter_name, labels = {})
-      if labels.nil?
-        labels = {}
-      end
-      "#{counter_name}-#{labels.values.join("-")}".freeze
+      labels = {} if labels.nil?
+      "#{counter_name}-#{labels.values.join('-')}"
     end
 
     def get_counter(ce)
-      if ce.nil?
-        raise ArgumentError, "ce is nil"
-      end
+      raise ArgumentError, 'ce is nil' if ce.nil?
 
-      k = Metrics::composite_id(ce.name, ce.labels)
+      k = Metrics.composite_id(ce.name, ce.labels)
 
       @counters_mtx.synchronize do
-        if @counters.key?(k)
-          @counters[k]
-        end
+        @counters[k] if @counters.key?(k)
       end
 
       # No counter exists, create a new one and return it
@@ -114,7 +106,7 @@ module Streamdal
       c = Counter.new(ce.name, ce.aud, ce.labels, ce.value)
 
       @counters_mtx.synchronize do
-        @counters[Metrics::composite_id(ce.name, ce.labels)] = c
+        @counters[Metrics.composite_id(ce.name, ce.labels)] = c
       end
 
       c
@@ -166,31 +158,29 @@ module Streamdal
     def _run_publisher
       # Background thread that reads values from counters, adds them to the publish queue, and then
       # resets the counter's value back to zero
-      unless @exit
-        @log.debug("Starting publisher")
+      return if @exit
 
-        # Sleep on startup and then and between each loop run
-        sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
+      @log.debug('Starting publisher')
 
-        # Get all counters
-        # Loop over each counter, get the value,
-        #   if value > 0, continue
-        #   if now() - last_updated > 10 seconds, remove counter
-        # Grab copy of counters
-        @counters_mtx.lock
-        new_counters = @counters.dup
-        @counters_mtx.unlock
+      # Sleep on startup and then and between each loop run
+      sleep(DEFAULT_COUNTER_PUBLISH_INTERVAL)
 
-        new_counters.each do |_, counter|
-          if counter.val == 0
-            next
-          end
+      # Get all counters
+      # Loop over each counter, get the value,
+      #   if value > 0, continue
+      #   if now() - last_updated > 10 seconds, remove counter
+      # Grab copy of counters
+      @counters_mtx.lock
+      new_counters = @counters.dup
+      @counters_mtx.unlock
 
-          ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
-          counter.reset
+      new_counters.each_value do |counter|
+        next if counter.val.zero?
 
-          @publish_queue.push(ce)
-        end
+        ce = CounterEntry.new(counter.name, counter.aud, counter.labels, counter.val)
+        counter.reset
+
+        @publish_queue.push(ce)
       end
     end
 
@@ -199,9 +189,8 @@ module Streamdal
 
       until @exit
         ce = @incr_queue.pop
-        if ce.nil?
-          next
-        end
+        next if ce.nil?
+
         begin
           _publish_metrics(ce)
         rescue => e
@@ -213,7 +202,7 @@ module Streamdal
     end
 
     def _run_reaper
-      @log.debug("Starting reaper")
+      @log.debug('Starting reaper')
 
       until @exit
         # Sleep on startup and then and between each loop run
@@ -226,11 +215,9 @@ module Streamdal
         # Grab copy of counters
         @counters_mtx.synchronize do
           @counters.each do |name, counter|
-            if counter.val > 0
-              next
-            end
+            next if counter.val.positive?
 
-            if Time::now - counter.last_updated > DEFAULT_COUNTER_TTL
+            if Time.now - counter.last_updated > DEFAULT_COUNTER_TTL
               @log.debug("Reaping counter '#{name}'")
               @counters.delete(name)
             end
@@ -238,7 +225,7 @@ module Streamdal
         end
       end
 
-      @log.debug("Exiting reaper")
+      @log.debug('Exiting reaper')
     end
 
     def _run_incrementer_worker(worker_id)
@@ -259,7 +246,7 @@ module Streamdal
 
     # Returns metadata for gRPC requests to the internal gRPC API
     def _metadata
-      { "auth-token" => @cfg[:streamdal_token].to_s }
+      { 'auth-token' => @cfg[:streamdal_token].to_s }
     end
   end
-end
+end

data/lib/schema.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 include Streamdal::Protos
 
 module Schemas
@@ -8,29 +10,21 @@ module Schemas
   end
 
   def _get_schema(aud)
-    if @schemas.key?(aud_to_str(aud))
-      return @schemas[aud_to_str(aud)].json_schema
-    end
+    return @schemas[aud_to_str(aud)].json_schema if @schemas.key?(aud_to_str(aud))
 
-    ""
+    ''
   end
 
   def _handle_schema(aud, step, wasm_resp)
     # Only handle schema steps
-    if step.infer_schema.nil?
-      return nil
-    end
+    return nil if step.infer_schema.nil?
 
     # Only successful schema inferences
-    if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
-      return nil
-    end
+    return nil if wasm_resp.exit_code != :WASM_EXIT_CODE_TRUE
 
     # If existing schema matches, do nothing
     existing_schema = _get_schema(aud)
-    if existing_schema == wasm_resp.output_step
-      return nil
-    end
+    return nil if existing_schema == wasm_resp.output_step
 
     _set_schema(aud, wasm_resp.output_step)
 
@@ -44,4 +38,4 @@ module Schemas
       @stub.send_schema(req, metadata: _metadata)
     end
   end
-end
+end

data/lib/streamdal.rb

@@ -8,11 +8,11 @@ require 'sp_sdk_pb'
 require 'sp_common_pb'
 require 'sp_info_pb'
 require 'sp_internal_pb'
-require "sp_internal_services_pb"
+require 'sp_internal_services_pb'
 require 'sp_pipeline_pb'
-require "sp_wsm_pb"
-require "steps/sp_steps_httprequest_pb"
-require "steps/sp_steps_kv_pb"
+require 'sp_wsm_pb'
+require 'steps/sp_steps_httprequest_pb'
+require 'steps/sp_steps_kv_pb'
 require 'timeout'
 require 'google/protobuf'
 require_relative 'audiences'
@@ -124,12 +124,12 @@ module Streamdal
     end
 
     def process(data, audience)
-      if data.length == 0
-        raise "data is required"
+      if data.empty?
+        raise 'data is required'
       end
 
       if audience.nil?
-        raise "audience is required"
+        raise 'audience is required'
       end
 
       resp = Streamdal::Protos::SDKResponse.new
@@ -144,8 +144,8 @@ module Streamdal
         "operation_type": aud.operation_type,
         "operation": aud.operation_name,
         "component": aud.component_name,
-        "pipeline_name": "",
-        "pipeline_id": "",
+        "pipeline_name": '',
+        "pipeline_id": '',
       }
 
       # TODO: metrics
@@ -166,7 +166,7 @@ module Streamdal
       if payload_size > MAX_PAYLOAD_SIZE
         # TODO: add metrics
         resp.status = :EXEC_STATUS_ERROR
-        resp.error = "payload size exceeds maximum allowed size"
+        resp.error = 'payload size exceeds maximum allowed size'
         resp
       end
 
@@ -174,8 +174,8 @@ module Streamdal
       original_data = data
 
       pipelines = _get_pipelines(aud)
-      if pipelines.length == 0
-        _send_tail(aud, "", original_data, original_data)
+      if pipelines.empty?
+        _send_tail(aud, '', original_data, original_data)
         return resp
       end
 
@@ -219,7 +219,7 @@ module Streamdal
               @log.debug "Running step '#{step.name}' in dry-run mode"
             end
 
-            if wasm_resp.output_payload.length > 0
+            if wasm_resp.output_payload.length.positive?
               resp.data = wasm_resp.output_payload
             end
 
@@ -296,10 +296,10 @@ module Streamdal
         end # pipelines.each
       end # timeout
 
-      _send_tail(aud, "", original_data, resp.data)
+      _send_tail(aud, '', original_data, resp.data)
 
       if @cfg[:dry_run]
-        @log.debug "Dry-run, setting response data to original data"
+        @log.debug 'Dry-run, setting response data to original data'
         resp.data = original_data
       end
 
@@ -310,19 +310,19 @@ module Streamdal
 
     def _validate_cfg(cfg)
       if cfg[:streamdal_url].nil? || cfg[:streamdal_url].empty?
-        raise "streamdal_url is required"
+        raise 'streamdal_url is required'
       end
 
       if cfg[:streamdal_token].nil? || cfg[:streamdal_token].empty?
-        raise "streamdal_token is required"
+        raise 'streamdal_token is required'
       end
 
       if cfg[:service_name].nil? || cfg[:streamdal_token].empty?
-        raise "service_name is required"
+        raise 'service_name is required'
       end
 
       if cfg[:log].nil? || cfg[:streamdal_token].empty?
-        logger = Logger.new(STDOUT)
+        logger = Logger.new($stdout)
         logger.level = Logger::ERROR
         cfg[:log] = logger
       end
@@ -338,13 +338,13 @@ module Streamdal
 
     def _handle_command(cmd)
       case cmd.command.to_s
-      when "kv"
+      when 'kv'
         _handle_kv(cmd)
-      when "tail"
+      when 'tail'
         _handle_tail_request(cmd)
-      when "set_pipelines"
+      when 'set_pipelines'
         _set_pipelines(cmd)
-      when "keep_alive"
+      when 'keep_alive'
         # Do nothing
       else
         @log.error "unknown command type #{cmd.command}"
@@ -378,13 +378,11 @@ module Streamdal
     end
 
     def _set_pipelines(cmd)
-      if cmd.nil?
-        raise "cmd is required"
-      end
+      raise 'cmd is required' if cmd.nil?
 
       cmd.set_pipelines.pipelines.each_with_index { |p, pIdx|
         p.steps.each_with_index { |step, idx|
-          if step._wasm_bytes == ""
+          if step._wasm_bytes == ''
             if cmd.set_pipelines.wasm_modules.has_key?(step._wasm_id)
               step._wasm_bytes = cmd.set_pipelines.wasm_modules[step._wasm_id].bytes
               cmd.set_pipelines.pipelines[pIdx].steps[idx] = step
@@ -424,11 +422,11 @@ module Streamdal
       mod = Wasmtime::Module.new(engine, step._wasm_bytes)
       linker = Wasmtime::Linker.new(engine, wasi: true)
 
-      linker.func_new("env", "httpRequest", [:i32, :i32], [:i64]) do |caller, ptr, len|
+      linker.func_new('env', 'httpRequest', %i[i32 i32], [:i64]) do |caller, ptr, len|
         @hostfunc.http_request(caller, ptr, len)
       end
 
-      linker.func_new("env", "kvExists", [:i32, :i32], [:i64]) do |caller, ptr, len|
+      linker.func_new('env', 'kvExists', %i[i32 i32], [:i64]) do |caller, ptr, len|
         @hostfunc.kv_exists(caller, ptr, len)
       end
 
@@ -442,8 +440,6 @@ module Streamdal
 
       instance = linker.instantiate(store, mod)
 
-      # TODO: host funcs
-
       # Store in cache
       func = WasmFunction.new
       func.instance = instance
@@ -455,11 +451,11 @@ module Streamdal
 
     def _call_wasm(step, data, isr)
       if step.nil?
-        raise "step is required"
+        raise 'step is required'
       end
 
       if data.nil?
-        raise "data is required"
+        raise 'data is required'
       end
 
       if isr.nil?
@@ -480,7 +476,7 @@ module Streamdal
         resp = Streamdal::Protos::WASMResponse.new
         resp.exit_code = :WASM_EXIT_CODE_ERROR
         resp.exit_msg = "Failed to execute WASM: #{e}"
-        resp.output_payload = ""
+        resp.output_payload = ''
         return resp
       end
     end
@@ -500,9 +496,9 @@ module Streamdal
 
       ci = Streamdal::Protos::ClientInfo.new
       ci.client_type = :CLIENT_TYPE_SDK
-      ci.library_name = "ruby-sdk"
-      ci.library_version = "0.0.1"
-      ci.language = "ruby"
+      ci.library_name = 'ruby-sdk'
+      ci.library_version = '0.0.1'
+      ci.language = 'ruby'
       ci.arch = arch
       ci.os = os
 
@@ -511,11 +507,11 @@ module Streamdal
 
     # Returns metadata for gRPC requests to the internal gRPC API
     def _metadata
-      { "auth-token" => @cfg[:streamdal_token].to_s }
+      { 'auth-token' => @cfg[:streamdal_token].to_s }
     end
 
     def _register
-      @log.info("register started")
+      @log.info('register started')
 
       # Register with Streamdal External gRPC API
       resps = @stub.register(_gen_register_request, metadata: _metadata)
@@ -527,7 +523,7 @@ module Streamdal
         _handle_command(r)
       end
 
-      @log.info("register exited")
+      @log.info('register exited')
     end
 
     def _exec_wasm(req)
@@ -535,14 +531,14 @@ module Streamdal
 
       # Empty out _wasm_bytes, we don't need it anymore
       # TODO: does this actually update the original object?
-      req.step._wasm_bytes = ""
+      req.step._wasm_bytes = ''
 
       data = req.to_proto
 
-      memory = wasm_func.instance.export("memory").to_memory
-      alloc = wasm_func.instance.export("alloc").to_func
-      dealloc = wasm_func.instance.export("dealloc").to_func
-      f = wasm_func.instance.export("f").to_func
+      memory = wasm_func.instance.export('memory').to_memory
+      alloc = wasm_func.instance.export('alloc').to_func
+      dealloc = wasm_func.instance.export('dealloc').to_func
+      f = wasm_func.instance.export('f').to_func
 
       start_ptr = alloc.call(data.length)
 
@@ -581,7 +577,7 @@ module Streamdal
         req.session_id = @session_id
         req.audiences = Google::Protobuf::RepeatedField.new(:message, Streamdal::Protos::Audience, [])
 
-        @audiences.each do |_, aud|
+        @audiences.each_value do |aud|
           req.audiences.push(aud)
         end
 
@@ -625,7 +621,7 @@ module Streamdal
 
     def _send_tail(aud, pipeline_id, original_data, new_data)
       tails = _get_active_tails_for_audience(aud)
-      if tails.length == 0
+      if tails.empty?
         return nil
       end
 
@@ -652,7 +648,7 @@ module Streamdal
         return nil
       end
 
-      @log.debug "Notifying"
+      @log.debug 'Notifying'
 
       if @cfg[:dry_run]
         return nil
@@ -739,7 +735,7 @@ module Streamdal
         # Remove from active tails
         @tails[key].delete(cmd.tail.request.id)
 
-        if @tails[key].length == 0
+        if @tails[key].empty?
           @tails.delete(key)
         end
       end
@@ -747,7 +743,7 @@ module Streamdal
       if @paused_tails.key?(key) && @paused_tails[key].key?(cmd.tail.request.id)
         @paused_tails[key].delete(cmd.tail.request.id)
 
-        if @paused_tails[key].length == 0
+        if @paused_tails[key].empty?
           @paused_tails.delete(key)
         end
       end
@@ -766,7 +762,7 @@ module Streamdal
           t.stop_tail
           tails[aud].delete(tail.request.id)
 
-          if tails[aud].length == 0
+          if tails[aud].empty?
             tails.delete(aud)
           end
         end
@@ -805,7 +801,7 @@ module Streamdal
 
         @tails[key].delete(tail_id)
 
-        if @tails[key].length == 0
+        if @tails[key].empty?
           @tails.delete(key)
         end
 
@@ -821,7 +817,7 @@ module Streamdal
 
         @paused_tails[key].delete(tail_id)
 
-        if @paused_tails[key].length == 0
+        if @paused_tails[key].empty?
           @paused_tails.delete(key)
         end
 
@@ -832,8 +828,8 @@ module Streamdal
     # Called by host functions to write memory to wasm instance so that
     # the wasm module can read the result of a host function call
     def write_to_memory(caller, res)
-      alloc = caller.export("alloc").to_func
-      memory = caller.export("memory").to_memory
+      alloc = caller.export('alloc').to_func
+      memory = caller.export('memory').to_memory
 
       # Serialize protobuf message
       resp = res.to_proto

data/lib/tail.rb

@@ -1,5 +1,4 @@
-# TODO: implement token bucket limiter
-require "bozos_buckets"
+require 'bozos_buckets'
 
 NUM_TAIL_WORKERS = 2
 MIN_TAIL_RESPONSE_INTERVAL_MS = 100
@@ -15,18 +14,19 @@ module Streamdal
       @logger = log
       @metrics = metrics
       @active = active
-      @last_msg = Time::at(0)
+      @last_msg = Time.at(0)
       @queue = Queue.new
       @workers = []
 
       # Only use rate limiting if sample_options is set
-      unless request.sample_options.nil?
-        @limiter = BozosBuckets::Bucket.new(
-          initial_token_count: request.sample_options.sample_rate,
-          refill_rate: request.sample_options.sample_interval_seconds,
-          max_token_count: request.sample_options.sample_rate
-        )
-      end
+      return if request.sample_options.nil?
+
+      @limiter = BozosBuckets::Bucket.new(
+        initial_token_count: request.sample_options.sample_rate,
+        refill_rate: request.sample_options.sample_interval_seconds,
+        max_token_count: request.sample_options.sample_rate
+      )
+
     end
 
     def start_tail_workers
@@ -43,11 +43,8 @@ module Streamdal
       sleep(1)
 
       @workers.each do |worker|
-        if worker.alive?
-          worker.exit
-        end
+        worker.exit if worker.alive?
       end
-
     end
 
     def start_tail_worker(worker_id)
@@ -63,35 +60,32 @@ module Streamdal
           next
         end
 
-        if Time::now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
+        if Time.now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
           sleep(MIN_TAIL_RESPONSE_INTERVAL_MS)
           @metrics.incr(Metrics::CounterEntry.new(COUNTER_DROPPED_TAIL_MESSAGES, nil, {}, 1))
           @logger.debug("Dropped tail message for '#{@request.id}' due to rate limiting")
           next
         end
 
-        unless stub.nil?
-          tail_response = @queue.pop(non_block = false)
-          @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
+        next if stub.nil?
+
+        tail_response = @queue.pop(false)
+        @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
 
-          begin
-            stub.send_tail([tail_response], metadata: { "auth-token" => @auth_token })
-          rescue => e
-            @logger.error("Error sending tail request: #{e}")
-          end
+        begin
+          stub.send_tail([tail_response], metadata: { 'auth-token' => @auth_token })
+        rescue Error => e
+          @logger.error("Error sending tail request: #{e}")
         end
       end
 
       @logger.debug "Tail worker #{worker_id} exited"
-
     end
 
     def should_send
-      if @limiter.nil?
-        true
-      end
+      true if @limiter.nil?
 
       @limiter.use_tokens(1)
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: streamdal
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Mark Gregan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-05-03 00:00:00.000000000 Z
+date: 2024-06-19 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: mark@streamdal.com
@@ -45,7 +45,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '='
     - !ruby/object:Gem::Version
-      version: 0.0.1
+      version: 0.0.2
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="