sqreen 1.20.1 → 1.21.0.beta1

data/lib/sqreen/ecosystem/module_registry.rb

@@ -0,0 +1,39 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    class ModuleRegistry
+      include Sqreen::Ecosystem::Loggable
+
+      def initialize
+        @mods = []
+      end
+
+      def register(mod)
+        @mods << mod
+      end
+
+      def init_all
+        logger.info { "Initializing #{@mods.size} ecosystem modules" }
+        each_module do |mod|
+          logger.debug { "Initializing module with type #{mod.class}" }
+          mod.setup
+        end
+      end
+
+      def destroy_all
+        # not implemented
+      end
+
+      # @param [Class] type
+      def each_module(type = nil, &block)
+        selected_mods = type ? (@mods.select { |mod| mod.is_a?(type) }) : @mods
+        if block_given?
+          selected_mods.each(&block)
+        else
+          selected_mods.each
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/redis/redis_connection.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/tracing/signals/tracing_client'
+require 'sqreen/ecosystem/module_api/signal_producer'
+
+module Sqreen
+  module Ecosystem
+    module Redis
+      class RedisConnection
+        include ModuleApi::SignalProducer
+        include ModuleApi::TracingPushDown
+        include ModuleApi::Instrumentation
+
+        def setup
+          instrument 'Redis::Connection::TCPSocket#connect',
+                     before: method(:before_advice)
+        end
+
+        private
+
+        def before_advice
+          return unless should_sample?('client')
+
+          host = call.args[0]
+          port = call.args[1]
+
+          signal = Sqreen::Kit::Signals::Specialized::TracingClient.new
+          signal.payload = Sqreen::Kit::Signals::Specialized::TracingClient::Payload.new(
+            transport: 'redis',
+            host: host, port: port
+          )
+          submit_signal signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/sampler.rb

@@ -0,0 +1,160 @@
+require 'thread' # for Mutex
+require 'singleton'
+require 'sqreen/ecosystem/loggable'
+
+# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000024-sampling.md
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      class Sampler
+        # @param [Array<Hash{String=>Object}>] definition
+        def initialize(definition)
+          @lines = definition.map { |h| Line.new(h) }
+        end
+
+        def should_sample?
+          line = @lines.find(&:triggers?)
+          line ? line.saved_definition : false
+        end
+
+        class Line
+          include Loggable
+
+          attr_reader :saved_definition
+
+          # @param [Hash{String=>Object}] definition
+          def initialize(definition)
+            @saved_definition = definition
+            @primitives = []
+
+            unknown = definition.keys - PRIMITIVES_MAP.keys
+            unless unknown.empty?
+              logger.warn "Unknown primitives: #{unknown}"
+              @primitives << AlwaysFalsePrimitive.instance
+              return
+            end
+
+            PRIMITIVES_MAP.each do |key, prim_class|
+              next unless definition[key]
+              @primitives << prim_class.new(definition[key])
+            end
+            # if @primitives is empty the line will always
+            # return true: [].all?(&:triggers?) is true
+          end
+
+          def triggers?
+            @primitives.all?(&:triggers?)
+          end
+        end
+
+        class AlwaysFalsePrimitive
+          include Singleton
+
+          def triggers?
+            false
+          end
+        end
+
+        class CallsPrimitive
+          def initialize(calls_period)
+            @calls_period = calls_period
+            @count = 0
+            @mutex = Mutex.new
+          end
+
+          def triggers?
+            prev_count = nil
+            @mutex.synchronize do
+              prev_count = @count
+              @count += 1
+            end
+
+            (prev_count % @calls_period).zero?
+          end
+        end
+
+        class RandomPrimitive
+          def initialize(probability)
+            @probability = probability
+          end
+
+          def triggers?
+            @probability >= rand
+          end
+        end
+
+        class MaxDurationMinutesPrimitive
+          def initialize(time_in_minutes)
+            @deadline = Sqreen.time + time_in_minutes * 60
+            @passed = false # no locking needed
+          end
+
+          def triggers?
+            return false if @passed
+            if Sqreen.time > @deadline
+              @passed = true
+              return false
+            end
+            true
+          end
+        end
+
+        class TargetPerMinutePrimitive
+          def initialize(max_calls)
+            @max_calls = max_calls
+            @minute_last_call = cur_minute
+            @calls_accumulated = 0
+            @mutex = Mutex.new
+          end
+
+          def triggers?
+            this_minute = cur_minute
+            calls_cur_minute = @mutex.synchronize do
+              if @minute_last_call == this_minute
+                @calls_accumulated += 1
+              else
+                @minute_last_call = this_minute
+                @calls_accumulated = 1
+              end
+            end
+
+            calls_cur_minute <= @max_calls
+          end
+
+          private
+
+          def cur_minute
+            (Sqreen.time / 60).floor
+          end
+        end
+
+        class MaxCallsPrimitive
+          def initialize(max_calls)
+            @max_calls = max_calls
+            @disabled = false # to avoid lock
+            @mutex = Mutex.new
+            @num_calls = 0
+          end
+
+          def triggers?
+            return false if @disabled
+            num_calls = @mutex.synchronize do
+              @num_calls += 1
+            end
+
+            num_calls <= @max_calls
+          end
+        end
+
+        PRIMITIVES_MAP = {
+          "calls"                => CallsPrimitive,
+          "random"               => RandomPrimitive,
+          "max_duration_minutes" => MaxDurationMinutesPrimitive,
+          "target_per_minute"    => TargetPerMinutePrimitive,
+          "max_calls"            => MaxCallsPrimitive,
+        }.freeze
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb

@@ -0,0 +1,150 @@
+require 'thread'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/tracing/sampler'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      # tracing sampling configuration, as specified by the 2nd argument of the
+      # tracing_enable command.
+      # See https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000025-enabling-tracing.md
+      class SamplingConfiguration
+        include Loggable
+
+        DEFAULT_SCOPE = '*'.freeze
+
+        def initialize(sampling_config)
+          @sampling_config = sampling_config
+          @samplers = {}
+          @samplers_virtual = build_virtual_holders(sampling_config)
+        end
+
+        # @param [String] scope either the scope or the part behind @ in a virtual scope
+        # @param [String] qualifier the part after @ in a virtual scope or nil
+        def should_sample?(scope, qualifier = nil)
+          if qualifier
+            fetch_sampler_virtual(scope, qualifier).should_sample?
+          else
+            fetch_sampler(scope).should_sample?
+          end
+        end
+
+        def forget_virtual_scope(scope, qualifier)
+          holder = @samplers_virtual[scope]
+          return unless holder.delete!(qualifier)
+        end
+
+        private
+
+        def fetch_sampler_virtual(scope, qualifier)
+          holder = @samplers_virtual[scope]
+
+          # no virtual scope configured, fallback to plain scope
+          return fetch_sampler(scope) unless holder
+
+          holder[qualifier]
+        end
+
+        def fetch_sampler(scope)
+          sampler = @samplers[scope]
+          return sampler if sampler
+
+          cfg = @sampling_config[scope] || @sampling_config[DEFAULT_SCOPE]
+          if cfg
+            @samplers[scope] = SamplingConfiguration.build_sampler(scope, cfg)
+          else
+            logger.info { "Disabling scope #{scope} due to its not being configured" }
+            @samplers[scope] = DisabledScopeSampler
+          end
+        end
+
+        def build_virtual_holders(sampling_config)
+          Hash[
+            sampling_config
+            .select { |scope, _cfg| scope.end_with?('@*') }
+            .map do |scope, cfg|
+              parent = scope[0...-2] # remove trailing @*
+              [parent, VirtualScopesHolder.new(parent, cfg)]
+            end
+          ]
+        end
+
+        class << self
+          include Loggable
+
+          def build_sampler(scope, cfg)
+            do_build_sampler(cfg['enabled'], cfg['sampling'] || [{}])
+          rescue StandardError => e
+            logger.warn "Invalid sampling configuration for #{scope}: #{e.inspect}"
+            logger.debug { e.backtrace }
+            DisabledScopeSampler
+          end
+
+          private
+
+          def do_build_sampler(enabled, sampling)
+            if enabled
+              Sampler.new(sampling)
+            else
+              logger.debug { "Disabling scope #{scope}" }
+              DisabledScopeSampler
+            end
+          end
+        end
+      end
+
+      class VirtualScopesHolder
+        include Loggable
+
+        MAX_VIRTUAL_SCOPES = 120
+        DISCARD_SIZE = 20
+
+        def initialize(parent, cfg)
+          @parent = parent
+          @cfg = cfg
+          @virtual_scopes = {}
+          @mutex = Mutex.new
+        end
+
+        def [](qualifier)
+          return false unless @cfg['enabled']
+
+          @mutex.synchronize do
+            sampler = @virtual_scopes[qualifier]
+            return sampler if sampler
+
+            @virtual_scopes[qualifier] = create_virtual(qualifier)
+          end
+        end
+
+        def delete!(qualifier)
+          @virtual_scopes.delete(qualifier)
+        end
+
+        private
+
+        def create_virtual(qualifier)
+          discard if @virtual_scopes.size >= MAX_VIRTUAL_SCOPES
+
+          @virtual_scopes[qualifier] =
+            SamplingConfiguration.build_sampler("#{@parent}@#{qualifier}", @cfg)
+        end
+
+        def discard
+          logger.info { "Discarding excess virtual scopes for scope '#{@parent}'" }
+          discard_keys = @virtual_scopes.keys.sample(DISCARD_SIZE)
+          @virtual_scopes.delete_if { |k, _v| discard_keys.include? k }
+        end
+      end
+
+      # fake sampler that always returns false
+      class DisabledScopeSampler
+        class << self
+          def should_sample?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/client-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingClient < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.client'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :host
+
+    SCHEMA_VERSION = 'tracing/client-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/server-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingServer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.server'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :client_ip
+
+    SCHEMA_VERSION = 'tracing/server-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :client_ip
+
+    # @return [Array<String>]
+    attr_accessor :previous_hops
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end