streamdal 0.0.1

data/lib/streamdal_spec.rb

@@ -0,0 +1,5 @@
+require_relative 'spec_helper'
+
+RSpec.describe "streamdal" do
+
+end

data/lib/tail.rb

@@ -0,0 +1,97 @@
+# TODO: implement token bucket limiter
+require "bozos_buckets"
+
+NUM_TAIL_WORKERS = 2
+MIN_TAIL_RESPONSE_INTERVAL_MS = 100
+
+module Streamdal
+  class Tail
+    attr_accessor :queue, :active, :request
+
+    def initialize(request, streamdal_url, auth_token, log, metrics, active = false)
+      @request = request
+      @streamdal_url = streamdal_url
+      @auth_token = auth_token
+      @logger = log
+      @metrics = metrics
+      @active = active
+      @last_msg = Time::at(0)
+      @queue = Queue.new
+      @workers = []
+
+      # Only use rate limiting if sample_options is set
+      unless request.sample_options.nil?
+        @limiter = BozosBuckets::Bucket.new(
+          initial_token_count: request.sample_options.sample_rate,
+          refill_rate: request.sample_options.sample_interval_seconds,
+          max_token_count: request.sample_options.sample_rate
+        )
+      end
+    end
+
+    def start_tail_workers
+      NUM_TAIL_WORKERS.times do |worker_id|
+        @workers << Thread.new { start_tail_worker(worker_id + 1) }
+      end
+
+      @active = true
+    end
+
+    def stop_tail
+      @active = false
+
+      sleep(1)
+
+      @workers.each do |worker|
+        if worker.alive?
+          worker.exit
+        end
+      end
+
+    end
+
+    def start_tail_worker(worker_id)
+      @logger.debug("Starting tail worker #{worker_id}")
+
+      # Each worker gets it's own gRPC connection
+      stub = Streamdal::Protos::Internal::Stub.new(@streamdal_url, :this_channel_is_insecure)
+
+      while @active
+        # If the queue is empty, sleep for a bit and loop again
+        if @queue.empty?
+          sleep(0.1)
+          next
+        end
+
+        if Time::now - @last_msg < MIN_TAIL_RESPONSE_INTERVAL_MS
+          sleep(MIN_TAIL_RESPONSE_INTERVAL_MS)
+          @metrics.incr(Metrics::CounterEntry.new(COUNTER_DROPPED_TAIL_MESSAGES, nil, {}, 1))
+          @logger.debug("Dropped tail message for '#{@request.id}' due to rate limiting")
+          next
+        end
+
+        unless stub.nil?
+          tail_response = @queue.pop(non_block = false)
+          @logger.debug("Sending tail request for '#{tail_response.tail_request_id}'")
+
+          begin
+            stub.send_tail([tail_response], metadata: { "auth-token" => @auth_token })
+          rescue => e
+            @logger.error("Error sending tail request: #{e}")
+          end
+        end
+      end
+
+      @logger.debug "Tail worker #{worker_id} exited"
+
+    end
+
+    def should_send
+      if @limiter.nil?
+        true
+      end
+
+      @limiter.use_tokens(1)
+    end
+  end
+end

data/lib/tail_spec.rb

@@ -0,0 +1,5 @@
+require_relative 'spec_helper'
+
+RSpec.describe "Streamdal::Tail" do
+
+end

data/lib/validation.rb

@@ -0,0 +1,88 @@
+##
+# Validation mix-in module
+require "steps/sp_steps_kv_pb"
+
+module Validation
+  def validate_set_pipelines(cmd)
+    if cmd.nil?
+      raise "cmd is required"
+    end
+
+    if cmd.audience.nil?
+      raise "audience is required"
+    end
+
+    if cmd.set_pipelines.nil?
+      raise "set_pipelines command is required"
+    end
+
+    cmd.set_pipelines.each do |pipeline|
+      if pipeline.id == ""
+        raise "pipeline id is required"
+      end
+    end
+  end
+
+  def validate_kv_command(cmd)
+    if cmd.nil?
+      raise "cmd is required"
+    end
+
+    if cmd.kv.nil?
+      raise "kv command is required"
+    end
+  end
+
+  def validate_kv_instruction(inst)
+    if inst.nil?
+      raise "instruction is required"
+    end
+
+    if inst.id == ""
+      raise "instruction id is required"
+    end
+
+    if inst.action.nil?
+      raise "instruction action is required"
+    end
+
+    if inst.action == :KV_ACTION_UNSET
+      raise "instruction action is required"
+    end
+
+    if inst.object.nil? and inst.action != :KV_ACTION_DELETE_ALL
+      raise "instruction object is required"
+    end
+
+    validate_kv_object(inst.object)
+  end
+
+  def validate_kv_object(obj)
+    if obj.nil?
+      raise "object is required"
+    end
+
+    if obj.key == ""
+      raise "kv object key is required"
+    end
+
+    if obj.value == ""
+      raise "kv object value is required"
+    end
+
+  end
+
+  def validate_tail_request(cmd)
+    if cmd.nil?
+      raise "cmd is required"
+    end
+
+    if cmd.audience.nil?
+      raise "audience is required"
+    end
+
+    if cmd.tail.nil?
+      raise "tail is required"
+    end
+  end
+end

data/lib/validation_spec.rb

@@ -0,0 +1,77 @@
+require 'rspec'
+require 'sp_command_pb'
+require 'sp_kv_pb'
+require_relative 'spec_helper'
+require_relative 'validation'
+
+module Streamdal
+  class TestClient
+    include Validation
+  end
+end
+
+RSpec.describe "Validation" do
+  let(:validation) { Streamdal::TestClient.new }
+
+  context "#validate_kv_command" do
+    it "raises an error if cmd is nil" do
+      expect { validation.validate_kv_command(nil) }.to raise_error("cmd is required")
+    end
+
+    it "raises an error if cmd.kv is nil" do
+      cmd = Streamdal::Protos::Command.new
+      expect { validation.validate_kv_command(cmd) }.to raise_error("kv command is required")
+    end
+  end
+
+  context "#validate_kv_instruction" do
+    it "raises an error if inst is nil" do
+      expect { validation.validate_kv_instruction(nil) }.to raise_error("instruction is required")
+    end
+
+    it "raises an error if inst.id is empty" do
+      inst = Streamdal::Protos::KVInstruction.new
+      expect { validation.validate_kv_instruction(inst) }.to raise_error("instruction id is required")
+    end
+
+    it "raises an error if inst.action is KV_ACTION_UNSET" do
+      inst = Streamdal::Protos::KVInstruction.new(id: "id", action: 0)
+      expect { validation.validate_kv_instruction(inst) }.to raise_error("instruction action is required")
+    end
+
+    it "raises an error if inst.object is nil" do
+      inst = Streamdal::Protos::KVInstruction.new(id: "id", action: 1)
+      expect { validation.validate_kv_instruction(inst) }.to raise_error("instruction object is required")
+    end
+  end
+
+  context '#validate_kv_object' do
+    it "raises an error if obj is nil" do
+      expect { validation.validate_kv_object(nil) }.to raise_error("object is required")
+    end
+
+    it "raises an error if obj.key is empty" do
+      obj = Streamdal::Protos::KVObject.new
+      expect { validation.validate_kv_object(obj) }.to raise_error("kv object key is required")
+    end
+
+    it "raises an error if obj.value is empty" do
+      obj = Streamdal::Protos::KVObject.new(key: "key")
+      expect { validation.validate_kv_object(obj) }.to raise_error("kv object value is required")
+    end
+  end
+
+  context "#validate_tail_request" do
+    it "raises an error if cmd is nil" do
+      expect { validation.validate_tail_request(nil) }.to raise_error("cmd is required")
+    end
+
+    it "raises an error if cmd.tail_request is nil" do
+      cmd = Streamdal::Protos::Command.new
+      cmd.audience = Streamdal::Protos::Audience.new
+      cmd.tail = nil
+      expect { validation.validate_tail_request(cmd) }.to raise_error("tail is required")
+    end
+  end
+
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: streamdal
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Mark Gregan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-05-03 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: mark@streamdal.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/audiences.rb
+- lib/audiences_spec.rb
+- lib/hostfunc.rb
+- lib/hostfunc_spec.rb
+- lib/kv.rb
+- lib/kv_spec.rb
+- lib/metrics.rb
+- lib/metrics_spec.rb
+- lib/schema.rb
+- lib/schema_spec.rb
+- lib/spec_helper.rb
+- lib/streamdal.rb
+- lib/streamdal_spec.rb
+- lib/tail.rb
+- lib/tail_spec.rb
+- lib/validation.rb
+- lib/validation_spec.rb
+homepage: https://docs.streamdal.com
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '='
+    - !ruby/object:Gem::Version
+      version: 0.0.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.5
+signing_key: 
+specification_version: 4
+summary: Streamdal SDK
+test_files: []