sqreen 1.20.4-java → 1.21.0.beta3-java

data/lib/sqreen/ecosystem/module_registry.rb

@@ -0,0 +1,44 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    class ModuleRegistry
+      include Sqreen::Ecosystem::Loggable
+
+      def initialize
+        @mods = []
+      end
+
+      def register(mod)
+        @mods << mod
+      end
+
+      def init_all
+        logger.info { "Initializing #{@mods.size} ecosystem modules" }
+        each_module do |mod|
+          next unless mod.respond_to? :setup
+          logger.debug { "Initializing module with type #{mod.class}" }
+          mod.setup
+        end
+      end
+
+      def destroy_all
+        # not implemented
+      end
+
+      # @param [Class] type
+      def each_module(type = nil, &block)
+        selected_mods = type ? (@mods.select { |mod| mod.is_a?(type) }) : @mods
+        if block_given?
+          selected_mods.each(&block)
+        else
+          selected_mods.each
+        end
+      end
+
+      def module_subset(type)
+        each_module(type).to_a
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/redis/redis_connection.rb

@@ -0,0 +1,43 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Redis
+      class RedisConnection
+        class RedisConnectionData
+          include ModuleApi::Tracing::ClientData
+
+          attr_accessor :port
+        end
+
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+        include ModuleApi::TracingIdGeneration
+
+        def setup
+          advice = wrap_for_interest(ModuleApi::Tracing::ClientData, &method(:before_advice))
+          instrument 'Redis::Connection::TCPSocket#connect',
+                     before: advice
+        end
+
+        private
+
+        def before_advice
+          host = call.args[0]
+          port = call.args[1]
+
+          RedisConnectionData.new(
+            transport: 'redis',
+            host: host,
+            port: port,
+            tracing_identifier: create_tracing_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/modules/client.rb

@@ -0,0 +1,31 @@
+require 'sqreen/ecosystem/tracing/signals/tracing_client'
+require 'sqreen/ecosystem/module_api/tracing'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        class Client
+          include ModuleApi::Tracing
+
+          consumes    ModuleApi::Tracing::ClientData
+          fixed_scope 'client'
+
+          # @param [Sqreen::Ecosystem::ModuleApi::Tracing::ClientData] data
+          def receive(data)
+            signal = Tracing::Signals::TracingClient.new
+            signal.payload = Tracing::Signals::TracingClient::Payload.new(
+              transport: data.transport,
+              host: data.host,
+              ip: data.ip,
+              tracing_identifier: data.tracing_identifier
+            )
+
+            submit_signal signal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/modules/server.rb

@@ -0,0 +1,30 @@
+require 'sqreen/ecosystem/tracing/signals/tracing_server'
+require 'sqreen/ecosystem/module_api/tracing'
+require 'sqreen/ecosystem/module_api/tracing/server_data'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        class Server
+          include ModuleApi::Tracing
+
+          consumes    ModuleApi::Tracing::ServerData
+          fixed_scope 'server'
+
+          # @param [Sqreen::Ecosystem::ModuleApi::Tracing::ServerData] data
+          def receive(data)
+            signal = Tracing::Signals::TracingServer.new
+            signal.payload = Tracing::Signals::TracingServer::Payload.new(
+              transport: data.transport,
+              client_ip: data.client_ip,
+              tracing_identifier: data.tracing_identifier
+            )
+
+            submit_signal signal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/sampler.rb

@@ -0,0 +1,160 @@
+require 'thread' # for Mutex
+require 'singleton'
+require 'sqreen/ecosystem/loggable'
+
+# see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000024-sampling.md
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      class Sampler
+        # @param [Array<Hash{String=>Object}>] definition
+        def initialize(definition)
+          @lines = definition.map { |h| Line.new(h) }
+        end
+
+        def should_sample?
+          line = @lines.find(&:triggers?)
+          line ? line.saved_definition : false
+        end
+
+        class Line
+          include Loggable
+
+          attr_reader :saved_definition
+
+          # @param [Hash{String=>Object}] definition
+          def initialize(definition)
+            @saved_definition = definition
+            @primitives = []
+
+            unknown = definition.keys - PRIMITIVES_MAP.keys
+            unless unknown.empty?
+              logger.warn "Unknown primitives: #{unknown}"
+              @primitives << AlwaysFalsePrimitive.instance
+              return
+            end
+
+            PRIMITIVES_MAP.each do |key, prim_class|
+              next unless definition[key]
+              @primitives << prim_class.new(definition[key])
+            end
+            # if @primitives is empty the line will always
+            # return true: [].all?(&:triggers?) is true
+          end
+
+          def triggers?
+            @primitives.all?(&:triggers?)
+          end
+        end
+
+        class AlwaysFalsePrimitive
+          include Singleton
+
+          def triggers?
+            false
+          end
+        end
+
+        class CallsPrimitive
+          def initialize(calls_period)
+            @calls_period = calls_period
+            @count = 0
+            @mutex = Mutex.new
+          end
+
+          def triggers?
+            prev_count = nil
+            @mutex.synchronize do
+              prev_count = @count
+              @count += 1
+            end
+
+            (prev_count % @calls_period).zero?
+          end
+        end
+
+        class RandomPrimitive
+          def initialize(probability)
+            @probability = probability
+          end
+
+          def triggers?
+            @probability >= rand
+          end
+        end
+
+        class MaxDurationMinutesPrimitive
+          def initialize(time_in_minutes)
+            @deadline = Sqreen.time + time_in_minutes * 60
+            @passed = false # no locking needed
+          end
+
+          def triggers?
+            return false if @passed
+            if Sqreen.time > @deadline
+              @passed = true
+              return false
+            end
+            true
+          end
+        end
+
+        class TargetPerMinutePrimitive
+          def initialize(max_calls)
+            @max_calls = max_calls
+            @minute_last_call = cur_minute
+            @calls_accumulated = 0
+            @mutex = Mutex.new
+          end
+
+          def triggers?
+            this_minute = cur_minute
+            calls_cur_minute = @mutex.synchronize do
+              if @minute_last_call == this_minute
+                @calls_accumulated += 1
+              else
+                @minute_last_call = this_minute
+                @calls_accumulated = 1
+              end
+            end
+
+            calls_cur_minute <= @max_calls
+          end
+
+          private
+
+          def cur_minute
+            (Sqreen.time / 60).floor
+          end
+        end
+
+        class MaxCallsPrimitive
+          def initialize(max_calls)
+            @max_calls = max_calls
+            @disabled = false # to avoid lock
+            @mutex = Mutex.new
+            @num_calls = 0
+          end
+
+          def triggers?
+            return false if @disabled
+            num_calls = @mutex.synchronize do
+              @num_calls += 1
+            end
+
+            num_calls <= @max_calls
+          end
+        end
+
+        PRIMITIVES_MAP = {
+          "calls"                => CallsPrimitive,
+          "random"               => RandomPrimitive,
+          "max_duration_minutes" => MaxDurationMinutesPrimitive,
+          "target_per_minute"    => TargetPerMinutePrimitive,
+          "max_calls"            => MaxCallsPrimitive,
+        }.freeze
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/sampling_configuration.rb

@@ -0,0 +1,150 @@
+require 'thread'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/tracing/sampler'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      # tracing sampling configuration, as specified by the 2nd argument of the
+      # tracing_enable command.
+      # See https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000025-enabling-tracing.md
+      class SamplingConfiguration
+        include Loggable
+
+        DEFAULT_SCOPE = '*'.freeze
+
+        def initialize(sampling_config)
+          @sampling_config = sampling_config
+          @samplers = {}
+          @samplers_virtual = build_virtual_holders(sampling_config)
+        end
+
+        # @param [String] scope either the scope or the part behind @ in a virtual scope
+        # @param [String] qualifier the part after @ in a virtual scope or nil
+        def should_sample?(scope, qualifier = nil)
+          if qualifier
+            fetch_sampler_virtual(scope, qualifier).should_sample?
+          else
+            fetch_sampler(scope).should_sample?
+          end
+        end
+
+        def forget_virtual_scope(scope, qualifier)
+          holder = @samplers_virtual[scope]
+          return unless holder.delete!(qualifier)
+        end
+
+        private
+
+        def fetch_sampler_virtual(scope, qualifier)
+          holder = @samplers_virtual[scope]
+
+          # no virtual scope configured, fallback to plain scope
+          return fetch_sampler(scope) unless holder
+
+          holder[qualifier]
+        end
+
+        def fetch_sampler(scope)
+          sampler = @samplers[scope]
+          return sampler if sampler
+
+          cfg = @sampling_config[scope] || @sampling_config[DEFAULT_SCOPE]
+          if cfg
+            @samplers[scope] = SamplingConfiguration.build_sampler(scope, cfg)
+          else
+            logger.info { "Disabling scope #{scope} due to its not being configured" }
+            @samplers[scope] = DisabledScopeSampler
+          end
+        end
+
+        def build_virtual_holders(sampling_config)
+          Hash[
+            sampling_config
+            .select { |scope, _cfg| scope.end_with?('@*') }
+            .map do |scope, cfg|
+              parent = scope[0...-2] # remove trailing @*
+              [parent, VirtualScopesHolder.new(parent, cfg)]
+            end
+          ]
+        end
+
+        class << self
+          include Loggable
+
+          def build_sampler(scope, cfg)
+            do_build_sampler(cfg['enabled'], cfg['sampling'] || [{}])
+          rescue StandardError => e
+            logger.warn "Invalid sampling configuration for #{scope}: #{e.inspect}"
+            logger.debug { e.backtrace }
+            DisabledScopeSampler
+          end
+
+          private
+
+          def do_build_sampler(enabled, sampling)
+            if enabled
+              Sampler.new(sampling)
+            else
+              logger.debug { "Disabling scope #{scope}" }
+              DisabledScopeSampler
+            end
+          end
+        end
+      end
+
+      class VirtualScopesHolder
+        include Loggable
+
+        MAX_VIRTUAL_SCOPES = 120
+        DISCARD_SIZE = 20
+
+        def initialize(parent, cfg)
+          @parent = parent
+          @cfg = cfg
+          @virtual_scopes = {}
+          @mutex = Mutex.new
+        end
+
+        def [](qualifier)
+          return false unless @cfg['enabled']
+
+          @mutex.synchronize do
+            sampler = @virtual_scopes[qualifier]
+            return sampler if sampler
+
+            @virtual_scopes[qualifier] = create_virtual(qualifier)
+          end
+        end
+
+        def delete!(qualifier)
+          @virtual_scopes.delete(qualifier)
+        end
+
+        private
+
+        def create_virtual(qualifier)
+          discard if @virtual_scopes.size >= MAX_VIRTUAL_SCOPES
+
+          @virtual_scopes[qualifier] =
+            SamplingConfiguration.build_sampler("#{@parent}@#{qualifier}", @cfg)
+        end
+
+        def discard
+          logger.info { "Discarding excess virtual scopes for scope '#{@parent}'" }
+          discard_keys = @virtual_scopes.keys.sample(DISCARD_SIZE)
+          @virtual_scopes.delete_if { |k, _v| discard_keys.include? k }
+        end
+      end
+
+      # fake sampler that always returns false
+      class DisabledScopeSampler
+        class << self
+          def should_sample?
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_client.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/client-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingClient < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.client'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :host
+
+    SCHEMA_VERSION = 'tracing/client-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_server.rb

@@ -0,0 +1,53 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/server-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingServer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.server'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :transport, :client_ip
+
+    SCHEMA_VERSION = 'tracing/server-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :transport
+
+    # @return [String]
+    attr_accessor :client_ip
+
+    # @return [Array<String>]
+    attr_accessor :previous_hops
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end

data/lib/sqreen/ecosystem/tracing_broker.rb

@@ -0,0 +1,101 @@
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/exception_reporting'
+
+module Sqreen
+  module Ecosystem
+    class TracingBroker
+      include Loggable
+      include ExceptionReporting
+
+      # Stores a lookup resolution so that lookup (incl. sampling) is done
+      # only once, not in the beginning of the producer code (when it asks
+      # whether it should proceed) and again once it delivers the data
+      ObserverLookup = Struct.new(:modules)
+
+      # @return [Sqreen::Ecosystem::Tracing::SamplingConfiguration]
+      attr_writer :sampling_configuration
+
+      # @param [Array<Sqreen::Ecosystem::ModuleApi::Tracing>] tracing_modules
+      def initialize(tracing_modules)
+        @sampling_configuration = nil
+        @type_to_subscribers = {}
+        tracing_modules.each do |mod|
+          consumed_type = mod.consumed_type
+          @type_to_subscribers[consumed_type] ||= []
+          @type_to_subscribers[consumed_type] << mod
+        end
+      end
+
+      # @param [Object] data
+      # @param [Sqreen::Ecosystem::TracingBroker::ObserverLookup] prev_lookup
+      def publish(data, prev_lookup)
+        prev_lookup.modules.each do |mod|
+          mod_process_data mod, data
+        end
+      end
+
+      # @param [Module] data_type
+      # @param [Hash] _hints reserved for future use, e.g. virtual scopes
+      # @return [Sqreen::Ecosystem::TracingBroker::ObserverLookup]
+      def interested_consumers(data_type, _hints = {})
+        unless @sampling_configuration
+          logger.debug do
+            "Declaring no one is interested in #{data_type} " \
+            "because tracing hasn't been enabled yet"
+          end
+          return false
+        end
+
+        # if we have several modules with the same scope, we
+        # should ask whether we should sample only once
+        scope_to_should_sample = Hash.new do |hash, scope|
+          result = @sampling_configuration.should_sample?(scope)
+          if result
+            logger.debug { "Will sample scope #{scope}. Sampling line: #{result}" }
+          else
+            logger.debug { "Will NOT sample scope #{scope}" }
+          end
+
+          hash[scope] = result
+        end
+
+        res = subscribers(data_type).select do |mod|
+          scope_to_should_sample[mod.scope]
+        end
+
+        res.empty? ? false : ObserverLookup.new(res)
+      end
+
+      private
+
+      # @param [Sqreen::Ecosystem::ModuleApi::Tracing] mod
+      def mod_process_data(mod, data)
+        mod.receive(data)
+      rescue ::Exception => e # rubocop:disable Lint/RescueException
+        report_exception("Error invoking tracing module #{mod}", e)
+      end
+
+      # @param [Module] data_type
+      # @return Array<Sqreen::Ecosystem::ModuleApi::Tracing>
+      def subscribers(data_type)
+        subscribers = @type_to_subscribers[data_type]
+
+        # None of the modules subscribes to data_type directly,
+        # but maybe they subscribe to one of the ancestors
+        # Cache this lookup
+        unless subscribers
+          subscribers = parents(data_type).inject([]) do |accum, type|
+            accum + (@type_to_subscribers[type] || [])
+          end
+          @type_to_subscribers[data_type] = subscribers
+        end
+
+        subscribers
+      end
+
+      def parents(type)
+        type.ancestors - Object.ancestors - [type]
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing_id_setup.rb

@@ -0,0 +1,34 @@
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/module_api/signal_producer'
+
+module Sqreen
+  module Ecosystem
+    class TracingIdSetup
+      # @param [Array<Sqreen::Ecosystem::ModuleApi::SignalProducer>] signal_producer_modules
+      def initialize(signal_producer_modules)
+        @modules = signal_producer_modules
+        @tracing_id_prefix = nil
+      end
+
+      def setup_modules
+        inject_out_of_tx_tracing_id_gen
+      end
+
+      attr_writer :tracing_id_prefix
+
+      private
+
+      def inject_out_of_tx_tracing_id_gen
+        @modules.each do |mod|
+          mod.tracing_id_producer = method(:generate_tracing_id)
+        end
+      end
+
+      def generate_tracing_id
+        return nil unless @tracing_id_prefix
+        "#{@tracing_id_prefix}.#{SecureRandom.uuid}"
+      end
+    end
+  end
+end