sqreen-kit 0.1.0

data/lib/sqreen/kit/signals/auth_signals_client.rb

@@ -0,0 +1,45 @@
+require 'sqreen/kit/signals/signals_client'
+
+module Sqreen
+  module Kit
+    module Signals
+      class AuthSignalsClient
+        attr_reader :headers # for testing only
+
+        # @param signals_client [SignalsClient]
+        # @param auth_data [Hash] :session_key, :api_key, :app_name
+        def initialize(signals_client, auth_data)
+          @signals_client = signals_client
+
+          @headers = {}
+          session_key = auth_data[:session_key]
+          api_key = auth_data[:api_key]
+          app_name = auth_data[:app_name]
+          if session_key
+            @headers['X-Session-Key'] = session_key
+          elsif api_key
+            @headers['X-Api-Key'] = api_key
+            @headers['X-App-Name'] = app_name if app_name
+          else
+            raise ArgumentError, 'Authentication data not provided'
+          end
+        end
+
+        # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] signals_and_traces
+        def report_batch(signals_and_traces)
+          @signals_client.report_batch(signals_and_traces, @headers)
+        end
+
+        # @param [Sqreen::Kit::Signals::Signal] signal
+        def report_signal(signal)
+          @signals_client.report_signal(signal, @headers)
+        end
+
+        # @param [Sqreen::Kit::Signals::Trace] trace
+        def report_trace(trace)
+          @signals_client.report_trace(trace, @headers)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/kit/signals/batch_collector.rb

@@ -0,0 +1,177 @@
+require 'sqreen/kit/loggable'
+
+module Sqreen
+  module Kit
+    module Signals
+      class BatchCollector
+        include Loggable
+
+        EXIT_SENTINEL = Object.new.freeze
+        DEFAULT_MAX_DELAY_S = 45
+        DEFAULT_FLUSH_SIZE = 30
+        DEFAULT_MAX_BATCH_SIZE = 100
+
+        attr_reader :auth_sig_client,
+                    :flush_size,
+                    :max_delay_s,
+                    :max_batch_size,
+                    :queue
+
+        # @param auth_sig_client [AuthSignalsClient]
+        def initialize(auth_sig_client, opts = {})
+          @auth_sig_client = auth_sig_client
+          @flush_size = opts[:flush_size] || DEFAULT_FLUSH_SIZE
+          @max_batch_size = opts[:max_batch_size] || DEFAULT_MAX_BATCH_SIZE
+          @max_delay_s = opts[:max_delay_s] || DEFAULT_MAX_DELAY_S
+          @queue = QueueWithTimeout.new
+          @thread = nil
+
+          if max_batch_size < flush_size # rubocop:disable Style/GuardClause
+            raise ArgumentError, 'max batch size < flush size'
+          end
+        end
+
+        def <<(signal_or_trace)
+          @queue << signal_or_trace
+        end
+
+        def start
+          @processing_loop = ProcessingLoop.new(self)
+          @thread = Thread.new do
+            @processing_loop.run
+          end
+        end
+
+        def running?
+          return false if thread.nil?
+          @thread.alive?
+        end
+
+        def close
+          return if @thread.nil?
+
+          @queue << EXIT_SENTINEL
+          @thread.join
+        end
+
+        class ProcessingLoop
+          include Loggable
+
+          # @param [BatchCollector] collector
+          def initialize(collector)
+            @collector = collector
+            @next_batch = []
+            @deadline = nil
+          end
+
+          def queue
+            @collector.queue
+          end
+
+          def max_batch_size
+            @collector.max_batch_size
+          end
+
+          def flush_size
+            @collector.flush_size
+          end
+
+          def run
+            while run_loop_once; end
+            logger.info 'Collector thread exiting'
+          end
+
+          private
+
+          def run_loop_once
+            el = queue.pop(@deadline)
+            if el.nil? # deadline passed
+              submit
+            elsif el.equal?(EXIT_SENTINEL)
+              return false
+            else
+              # a signal or a trace
+              if @next_batch.empty?
+                # first object, set a deadline
+                @deadline = Time.now.to_f + @collector.max_delay_s
+              end
+              @next_batch << el
+              # drain the queue completely
+              until @next_batch.size >= max_batch_size || (el = queue.pop_nb).nil?
+                if el.equal?(EXIT_SENTINEL)
+                  queue << EXIT_SENTINEL # push it back
+                  break
+                end
+                @next_batch << el
+              end
+
+              submit if @next_batch.size >= flush_size
+            end
+
+            true
+          end
+
+          def submit
+            logger.debug { "Batch submit. Batch size: #{@next_batch.size}" }
+            @deadline = nil
+            return if @next_batch.empty?
+            cur_batch = @next_batch
+            @next_batch = []
+            @collector.auth_sig_client.report_batch(cur_batch)
+          end
+        end
+
+        # Adapted from https://spin.atomicobject.com/2014/07/07/ruby-queue-pop-timeout/
+        class QueueWithTimeout
+          include Loggable
+
+          MAX_QUEUE_SIZE = 1000
+
+          def initialize
+            @mutex = Mutex.new
+            @queue = []
+            @received = ConditionVariable.new
+          end
+
+          def <<(x)
+            @mutex.synchronize do
+              if @queue.size >= MAX_QUEUE_SIZE
+                # processing loop is prob spending too much time on http requests
+                logger.warn "Queue is full! Dropping #{x}"
+                next
+              end
+              @queue << x
+              @received.signal
+            end
+          end
+
+          # non-blocking pop
+          def pop_nb
+            @mutex.synchronize do
+              return nil if @queue.empty?
+              @queue.shift
+            end
+          end
+
+          # @param deadline [Float]
+          def pop(deadline = nil)
+            @mutex.synchronize do
+              if deadline.nil?
+                # wait indefinitely until there is an element in the queue
+                @received.wait(@mutex) while @queue.empty?
+              elsif @queue.empty?
+                # wait for element or timeout
+                while @queue.empty? && (remaining_time = deadline - Time.now.to_f) > 0
+                  @received.wait(@mutex, remaining_time)
+                end
+              end
+
+              return nil if @queue.empty?
+              @queue.shift
+            end
+          end
+        end # end QueueWithTimeout
+      end
+    end
+  end
+end

data/lib/sqreen/kit/signals/context/http_context.rb

@@ -0,0 +1,139 @@
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/context/http/2020-01-01T00_00_00_000Z/schema.cue
+
+module Sqreen
+  module Kit
+    module Signals
+      module Context
+      end
+    end
+  end
+end
+
+class Sqreen::Kit::Signals::Context::HttpContext
+  include Sqreen::Kit::Signals::DtoHelper
+
+  PARAMS_ATTRS = [:params_query, :params_form, :params_cookies,
+                  :params_json, :params_other].freeze
+
+  SCHEMA_VERSION = 'http/2020-01-01T00:00:00.000Z'.freeze
+  add_mandatory_attrs :headers, :user_agent, :scheme, :verb,
+                      :host, :port, :remote_ip, :remote_port, :path
+
+  # @return [String]
+  attr_accessor :rid
+
+  # mandatory
+  # @return [Array<Array<String>>] inner arrays have two values: [name, value]
+  attr_accessor :headers
+
+  # mandatory
+  # @return [String]
+  attr_accessor :user_agent
+
+  # mandatory
+  # @return [String]
+  attr_accessor :scheme
+
+  # mandatory
+  # @param [String]
+  attr_writer :verb
+  def verb
+    raise 'verb not set' unless defined?(@verb) && @verb
+    @verb.upcase
+  end
+
+  # mandatory
+  # Host header; may include the port
+  # @return [String]
+  attr_accessor :host
+
+  # mandatory
+  # @return [Integer|String]
+  attr_accessor :port
+
+  # mandatory
+  # @return [String]
+  attr_accessor :remote_ip
+
+  # mandatory
+  # @return [Integer|String]
+  attr_accessor :remote_port
+
+  # mandatory
+  # XXX: Unclear is decoded or not
+  # @return [String]
+  attr_accessor :path
+
+  # @return [String]
+  attr_accessor :referer
+
+  # @return [Hash]
+  attr_accessor :params_query
+
+  # @return [Hash]
+  attr_accessor :params_form
+
+  # @return [Hash]
+  attr_accessor :params_cookies
+
+  # @return [Hash]
+  attr_accessor :params_json
+
+  # @return [Hash]
+  attr_accessor :params_other
+
+  # @return [String]
+  attr_accessor :endpoint
+
+  # @return [Boolean]
+  attr_accessor :reveal_replayed
+
+  # Response status
+  # @return [Integer]
+  attr_accessor :status
+
+  # Response content length
+  # @return [Integer]
+  attr_accessor :content_length
+
+  # Response content type
+  # @return [String]
+  attr_accessor :content_type
+
+  def to_h
+    check_mandatories
+
+    {
+      request: compact_hash({
+                              rid: rid,
+                              headers: headers,
+                              user_agent: user_agent,
+                              scheme: scheme,
+                              verb: verb,
+                              host: host,
+                              port: port,
+                              remote_ip: remote_ip,
+                              remote_port: remote_port,
+                              path: path,
+                              referer: referer,
+                              parameters: compact_hash({
+                                                         query: params_query,
+                                                         form: params_form,
+                                                         cookies: params_cookies,
+                                                         json: params_json,
+                                                         other: params_other,
+                                                       }),
+                              endpoint: endpoint,
+                              # yes, this one is in CamelCase
+                              isRevealReplayed: reveal_replayed,
+                            }),
+      response: compact_hash({
+                               status: status,
+                               content_type: content_type,
+                               content_length: content_length,
+                             }),
+    }
+  end
+end

data/lib/sqreen/kit/signals/dto_helper.rb

@@ -0,0 +1,147 @@
+require 'json'
+
+module Sqreen
+  module Kit
+    module Signals
+      # Provides a helper constructor, default to_h, and mandatory field checking
+      module DtoHelper
+        DO_NOT_CONVERT_TYPES = [
+          NilClass, Hash, Array, Numeric, TrueClass, FalseClass
+        ].freeze
+
+        module ClassMethods
+          # method should have been defined initially in an ancestor
+          def validate_str_attr(attr, regex)
+            define_method(:"#{attr}=") do |val|
+              unless val =~ regex
+                raise "Unexpected format for attribute #{attr}: " \
+                      "'#{val}' does not match #{regex}"
+              end
+              super(val)
+            end
+          end
+
+          # All the classes/modules in the ancestor chain
+          # including DtoHelper
+          def all_implementing_modules
+            ancestors
+              .select { |c| c != DtoHelper && c.ancestors.include?(DtoHelper) }
+          end
+
+          def attributes_for_to_h
+            @all_attributes ||= begin
+              all_implementing_modules
+                .map(&:attributes_for_to_h_self)
+                .reduce(:+)
+                .uniq
+            end
+          end
+
+          def attributes_for_to_h_self
+            methods = public_instance_methods(false)
+
+            methods.reject { |m| m.to_s.end_with? '=' }
+                   .select { |m| methods.include?(:"#{m}=") }
+          end
+
+          def add_mandatory_attrs(*args)
+            @self_mandatory_attrs ||= []
+            @self_mandatory_attrs += args
+          end
+
+          def self_mandatory_attrs
+            @self_mandatory_attrs || []
+          end
+
+          def mandatory_attrs
+            @mandatory_attrs ||= begin
+              all_implementing_modules
+                .map(&:self_mandatory_attrs)
+                .reduce(:+)
+                .uniq
+            end
+          end
+
+          def attr_accessor_time(attr)
+            define_method :"#{attr}=" do |value|
+              unless value.is_a?(Time)
+                unless value.is_a?(String)
+                  raise ArgumentError, 'expected Time or String object'
+                end
+                unless value =~ /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/
+                  raise ArgumentError, "Invalid time format for #{value}"
+                end
+              end
+              instance_variable_set("@#{attr}", value)
+            end
+
+            define_method attr do
+              cur_val = instance_variable_get("@#{attr}")
+
+              return nil if cur_val.nil?
+              return cur_val if cur_val.is_a?(String)
+              cur_val.strftime('%Y-%m-%dT%H:%M:%S.%L%z')
+            end
+          end
+
+          def included(mod)
+            # make sure that classes/modules indirectly including DtoHelper
+            # also have their singleton class including ClassMethods
+            return if mod.singleton_class.ancestors.include?(ClassMethods)
+            mod.extend(ClassMethods)
+          end
+        end
+
+        def self.included(mod)
+          mod.extend(ClassMethods)
+        end
+
+        def initialize(values = {})
+          values.each do |attr, val|
+            public_send("#{attr}=", val)
+          end
+        end
+
+        def compact_hash(h)
+          h.delete_if { |_k, v| v.nil? }
+        end
+
+        def to_h
+          check_mandatories
+
+          res = {}
+          self.class.attributes_for_to_h.each do |attr|
+            value = public_send(attr)
+            if (value.class.ancestors & DO_NOT_CONVERT_TYPES).empty? && \
+               value.respond_to?(:to_h)
+              value = value.to_h
+            end
+
+            res[attr] = value unless value.nil?
+          end
+          res
+        end
+
+        def append_to_h_filter(proc)
+          @filters ||= []
+          @filters << proc
+        end
+
+        def to_json
+          res = (@filters || []).reduce(to_h) { |accum, filter| filter[accum] }
+          res.to_json
+        end
+
+        private
+
+        def check_mandatories
+          self.class.mandatory_attrs.each do |attr|
+            if public_send(attr).nil?
+              raise "The attribute #{attr} is not set in #{inspect}"
+            end
+          end
+        end
+      end
+    end
+  end
+end