sqreen 1.20.1-java → 1.22.0-java

data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb

@@ -0,0 +1,150 @@
+require 'thread'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/tracing/sampler'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      # tracing sampling configuration, as specified by the 2nd argument of the
+      # tracing_enable command.
+      # See https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000025-enabling-tracing.md
+      class SamplingConfiguration
+        include Loggable
+
+        DEFAULT_SCOPE = '*'.freeze
+
+        def initialize(sampling_config)
+          @sampling_config = sampling_config
+          @samplers = {}
+          @samplers_virtual = build_virtual_holders(sampling_config)
+        end
+
+        # @param [String] scope either the scope or the part behind @ in a virtual scope
+        # @param [String] qualifier the part after @ in a virtual scope or nil
+        def should_sample?(scope, qualifier = nil)
+          if qualifier
+            fetch_sampler_virtual(scope, qualifier).should_sample?
+          else
+            fetch_sampler(scope).should_sample?
+          end
+        end
+
+        def forget_virtual_scope(scope, qualifier)
+          holder = @samplers_virtual[scope]
+          return unless holder.delete!(qualifier)
+        end
+
+        private
+
+        def fetch_sampler_virtual(scope, qualifier)
+          holder = @samplers_virtual[scope]
+
+          # no virtual scope configured, fallback to plain scope
+          return fetch_sampler(scope) unless holder
+
+          holder[qualifier]
+        end
+
+        def fetch_sampler(scope)
+          sampler = @samplers[scope]
+          return sampler if sampler
+
+          cfg = @sampling_config[scope] || @sampling_config[DEFAULT_SCOPE]
+          if cfg
+            @samplers[scope] = SamplingConfiguration.build_sampler(scope, cfg)
+          else
+            logger.info { "Disabling scope #{scope} due to its not being configured" }
+            @samplers[scope] = DisabledScopeSampler
+          end
+        end
+
+        def build_virtual_holders(sampling_config)
+          Hash[
+            sampling_config
+            .select { |scope, _cfg| scope.end_with?('@*') }
+            .map do |scope, cfg|
+              parent = scope[0...-2] # remove trailing @*
+              [parent, VirtualScopesHolder.new(parent, cfg)]
+            end
+          ]
+        end
+
+        class << self
+          include Loggable
+
+          def build_sampler(scope, cfg)
+            do_build_sampler(cfg['enabled'], cfg['sampling'] || [{}])
+          rescue StandardError => e
+            logger.warn "Invalid sampling configuration for #{scope}: #{e.inspect}"
+            logger.debug { e.backtrace }
+            DisabledScopeSampler
+          end
+
+          private
+
+          def do_build_sampler(enabled, sampling)
+            if enabled
+              Sampler.new(sampling)
+            else
+              logger.debug { "Disabling scope #{scope}" }
+              DisabledScopeSampler
+            end
+          end
+        end
+      end
+
+      class VirtualScopesHolder
+        include Loggable
+
+        MAX_VIRTUAL_SCOPES = 120
+        DISCARD_SIZE = 20
+
+        def initialize(parent, cfg)
+          @parent = parent
+          @cfg = cfg
+          @virtual_scopes = {}
+          @mutex = Mutex.new
+        end
+
+        def [](qualifier)
+          return false unless @cfg['enabled']
+
+          @mutex.synchronize do
+            sampler = @virtual_scopes[qualifier]
+            return sampler if sampler
+
+            @virtual_scopes[qualifier] = create_virtual(qualifier)
+          end
+        end
+
+        def delete!(qualifier)
+          @virtual_scopes.delete(qualifier)
+        end
+
+        private
+
+        def create_virtual(qualifier)
+          discard if @virtual_scopes.size >= MAX_VIRTUAL_SCOPES
+
+          @virtual_scopes[qualifier] =
+            SamplingConfiguration.build_sampler("#{@parent}@#{qualifier}", @cfg)
+        end
+
+        def discard
+          logger.info { "Discarding excess virtual scopes for scope '#{@parent}'" }
+          discard_keys = @virtual_scopes.keys.sample(DISCARD_SIZE)
+          @virtual_scopes.delete_if { |k, _v| discard_keys.include? k }
+        end
+      end
+
+      # fake sampler that always returns false
+      class DisabledScopeSampler
+        class << self
+          def should_sample?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/client-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingClient < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.client'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :host
+
+    SCHEMA_VERSION = 'tracing/client-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_consumer.rb

@@ -0,0 +1,56 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/consumer-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingConsumer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.consumer'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :message_type, :host, :tracing_identifier
+
+    SCHEMA_VERSION = 'tracing/consumer-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :message_type
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :topic
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_producer.rb

@@ -0,0 +1,56 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/producer-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingProducer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.producer'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :message_type, :host, :tracing_identifier
+
+    SCHEMA_VERSION = 'tracing/producer-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :message_type
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :topic
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/server-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingServer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.server'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :client_ip
+
+    SCHEMA_VERSION = 'tracing/server-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :client_ip
+
+    # @return [Array<String>]
+    attr_accessor :previous_hops
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing_broker.rb

@@ -0,0 +1,101 @@
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/exception_reporting'
+
+module Sqreen
+  module Ecosystem
+    class TracingBroker
+      include Loggable
+      include ExceptionReporting
+
+      # Stores a lookup resolution so that lookup (incl. sampling) is done
+      # only once, not in the beginning of the producer code (when it asks
+      # whether it should proceed) and again once it delivers the data
+      ObserverLookup = Struct.new(:modules)
+
+      # @return [Sqreen::Ecosystem::Tracing::SamplingConfiguration]
+      attr_writer :sampling_configuration
+
+      # @param [Array<Sqreen::Ecosystem::ModuleApi::Tracing>] tracing_modules
+      def initialize(tracing_modules)
+        @sampling_configuration = nil
+        @type_to_subscribers = {}
+        tracing_modules.each do |mod|
+          consumed_type = mod.consumed_type
+          @type_to_subscribers[consumed_type] ||= []
+          @type_to_subscribers[consumed_type] << mod
+        end
+      end
+
+      # @param [Object] data
+      # @param [Sqreen::Ecosystem::TracingBroker::ObserverLookup] prev_lookup
+      def publish(data, prev_lookup)
+        prev_lookup.modules.each do |mod|
+          mod_process_data mod, data
+        end
+      end
+
+      # @param [Module] data_type
+      # @param [Hash] _hints reserved for future use, e.g. virtual scopes
+      # @return [Sqreen::Ecosystem::TracingBroker::ObserverLookup]
+      def interested_consumers(data_type, _hints = {})
+        unless @sampling_configuration
+          logger.debug do
+            "Declaring no one is interested in #{data_type} " \
+            "because tracing hasn't been enabled yet"
+          end
+          return false
+        end
+
+        # if we have several modules with the same scope, we
+        # should ask whether we should sample only once
+        scope_to_should_sample = Hash.new do |hash, scope|
+          result = @sampling_configuration.should_sample?(scope)
+          if result
+            logger.debug { "Will sample scope #{scope}. Sampling line: #{result}" }
+          else
+            logger.debug { "Will NOT sample scope #{scope}" }
+          end
+
+          hash[scope] = result
+        end
+
+        res = subscribers(data_type).select do |mod|
+          scope_to_should_sample[mod.scope]
+        end
+
+        res.empty? ? false : ObserverLookup.new(res)
+      end
+
+      private
+
+      # @param [Sqreen::Ecosystem::ModuleApi::Tracing] mod
+      def mod_process_data(mod, data)
+        mod.receive(data)
+      rescue ::Exception => e # rubocop:disable Lint/RescueException
+        report_exception("Error invoking tracing module #{mod}", e)
+      end
+
+      # @param [Module] data_type
+      # @return Array<Sqreen::Ecosystem::ModuleApi::Tracing>
+      def subscribers(data_type)
+        subscribers = @type_to_subscribers[data_type]
+
+        # None of the modules subscribes to data_type directly,
+        # but maybe they subscribe to one of the ancestors
+        # Cache this lookup
+        unless subscribers
+          subscribers = parents(data_type).inject([]) do |accum, type|
+            accum + (@type_to_subscribers[type] || [])
+          end
+          @type_to_subscribers[data_type] = subscribers
+        end
+
+        subscribers
+      end
+
+      def parents(type)
+        type.ancestors - Object.ancestors - [type]
+      end
+    end
+  end
+end