sqreen 1.20.4-java → 1.21.0.beta3-java

data/lib/sqreen/ecosystem/transaction_storage.rb

@@ -0,0 +1,64 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    # The transaction storage is a mechanism for the modules to share
+    # request-scoped data with each other or to keep request-scoped objects
+    # themselves, although, for this last use case to be effective,
+    # propagation of request start/end events to the modules will likely be
+    # needed. A more generic notification of data availability to modules
+    # also may be needed in the future.
+    #
+    # This is not be used to share data with the Ecosystem client
+    #
+    # This class is not thread safe because it can call the lazy getter
+    # twice if [] is called concurrently.
+    class TransactionStorage
+      include Loggable
+
+      class << self
+        # @return [Sqreen::Ecosystem::TransactionStorage]
+        def create_thread_local
+          Thread.current[:tx_storage] = new
+        end
+
+        # @return [Sqreen::Ecosystem::TransactionStorage]
+        def fetch_thread_local
+          Thread.current[:tx_storage]
+        end
+
+        def destroy_thread_local
+          Thread.current[:tx_storage] = nil
+        end
+      end
+
+      def initialize
+        @values = {}
+        @values_lazy = {}
+      end
+
+      def []=(key, value)
+        @values[key] = value
+      end
+
+      def set_lazy(key, &block)
+        @values_lazy[key] = block
+      end
+
+      def [](key)
+        v = @values[key]
+        return v unless v.nil?
+
+        v = @values_lazy[key]
+        return nil if v.nil?
+
+        begin
+          @values[key] = v.call
+        rescue StandardError => e
+          logger.warn { "Error resolving key #{e} with lazy value: #{e.message}" }
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/util/call_writers_from_init.rb

@@ -0,0 +1,13 @@
+module Sqreen
+  module Ecosystem
+    module Util
+      module CallWritersFromInit
+        def initialize(values = {})
+          values.each do |attr, val|
+            public_send("#{attr}=", val)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem.rb

@@ -0,0 +1,96 @@
+require 'securerandom'
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/tracing/sampling_configuration'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_broker'
+require 'sqreen/ecosystem/tracing_id_setup'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing'
+
+module Sqreen
+  # The API for the ecosystem client (together with the dispatch table)
+  module Ecosystem
+    class << self
+      def init(opts = {})
+        @registry = ModuleRegistry.new
+        register_modules(opts[:modules])
+        @registry.init_all
+
+        # setup tracing generation
+        tracing_id_mods = @registry.module_subset(ModuleApi::TracingIdGeneration)
+        @tracing_id_setup = TracingIdSetup.new(tracing_id_mods)
+        @tracing_id_setup.setup_modules
+
+        # configure tracing broker with the consumers (tracing modules)
+        tracing_modules = @registry.module_subset(ModuleApi::Tracing)
+        @tracing_broker = TracingBroker.new(tracing_modules)
+
+        # inject tracing broker in message producers
+        @registry.each_module(ModuleApi::MessageProducer) do |mod|
+          mod.tracing_broker = @tracing_broker
+        end
+      rescue ::Exception # rubocop:disable Lint/RescueException
+        # TODO: modules must be disabled at this point
+        raise
+      end
+
+      def reset
+        instance_variables.each do |ia|
+          instance_variable_set(ia, nil)
+        end
+      end
+
+      # To be called by the Ecosystem client when a new transaction
+      # (generally: request) is started
+      # In the future, it's intended that request end/start detection be handled
+      # by the Ecosystem itself, so control will flow in the other direction,
+      # from the ecosystem to its client
+      def start_transaction
+        TransactionStorage.create_thread_local
+      end
+
+      def end_transaction
+        TransactionStorage.destroy_thread_local
+      end
+
+      # @param [String] tracing_id_prefix
+      # @param [Array<Hash{String=>Object}>] sampling_config
+      def configure_sampling(tracing_id_prefix, sampling_config)
+        @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
+        built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
+        @tracing_broker.sampling_configuration = built_samp_cfg
+      end
+
+      private
+
+      def register_modules(modules)
+        return register_all_modules unless modules
+
+        modules.each { |mod| register mod }
+      end
+
+      def register_all_modules
+        # replace with something more magical?
+        require_relative 'ecosystem/http/rack_request'
+        register Http::RackRequest.new
+
+        require_relative 'ecosystem/http/net_http'
+        register Http::NetHttp.new
+
+        require_relative 'ecosystem/redis/redis_connection'
+        register Redis::RedisConnection.new
+
+        require_relative 'ecosystem/tracing/modules/client'
+        register Tracing::Modules::Client.new
+
+        require_relative 'ecosystem/tracing/modules/server'
+        register Tracing::Modules::Server.new
+      end
+
+      def register(mod)
+        @registry.register mod
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/around_callbacks.rb

@@ -0,0 +1,99 @@
+require 'sqreen/metrics_store'
+require 'sqreen/log/loggable'
+require 'sqreen/events/remote_exception'
+require 'sqreen/mono_time'
+require 'sqreen/graft/call'
+
+module Sqreen
+  class EcosystemIntegration
+    module AroundCallbacks
+      class << self
+        include Log::Loggable::ClassMethods
+
+        # @return [Proc]
+        attr_accessor :create_metric
+
+        # for instrumentation hooks
+        # instrumentation hooks already handle budgets/metrics
+        def wrap_instrumentation_hook(module_name, action, callable)
+          # make tag similar to that of rules
+          # TODO: move to structured tags to avoid silly string manips
+          action_for_metric = if %w[pre post failing finally].include?(action)
+                                action
+                              else
+                                'pre'
+                              end
+          metric_name = "sq.ecosystem_#{module_name}.#{action_for_metric}"
+          create_metric.call(metric_name)
+
+          Proc.new do |*args|
+            begin
+              callable.call(*args)
+            rescue ::Exception => e # rubocop:disable Lint/RescueException
+              # 2) rescue exceptions
+              logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
+              logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+              Sqreen::RemoteException.record(e)
+            end # end begin
+          end # end proc
+        end
+
+        # XXX: not used yet
+        def wrap_generic_callback(module_name, action, callable)
+          timer_name = "ecosystem:#{module_name}@#{action}"
+          perf_notif_name = "ecosystem_#{module_name}"
+
+          # XXX: register metric
+
+          Proc.new do |*args|
+            begin
+              req_storage = Thread.current[:sqreen_http_request]
+
+              timer = Graft::Timer.new(timer_name) do |t|
+                # this is an epilogue to measure()
+                req_storage && req_storage[:timed_hooks] << t
+              end
+
+              req_timer = nil
+              timer.measure do
+                # not in a request, no budget; call cb
+                next callable.call(*args) unless req_storage
+
+                # 1) budget enforcement
+                # skip callback if budget already expended
+                next if req_storage[:time_budget_expended]
+
+                budget = req_storage[:time_budget]
+                if budget
+                  req_timer = req_storage[:timer]
+                  remaining = budget - req_timer.elapsed
+                  unless remaining > 0
+                    req_storage[:time_budget_expended] = true
+                    next # skip callback
+                  end
+                end
+
+                callable.call(*args)
+              end
+            rescue ::Exception => e # rubocop:disable Lint/RescueException
+              # 2) rescue exceptions
+              logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
+              logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+              Sqreen::RemoteException.record(e)
+            ensure
+              # 3) contribute to performance metrics
+              if timer
+                req_timer.include_measurements(timer) if req_timer
+
+                # XXX: PerformanceNotifications is used no more
+                Sqreen::PerformanceNotifications.notify(
+                  perf_notif_name, action, *timer.start_and_end
+                )
+              end
+            end # end begin
+          end # end proc
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/instrumentation_service.rb

@@ -0,0 +1,42 @@
+require 'sqreen/graft/hook'
+require 'sqreen/ecosystem_integration/around_callbacks'
+
+module Sqreen
+  class EcosystemIntegration
+    module InstrumentationService
+      class << self
+        # @param [String] module_name
+        # @param [String] method in form A::B#c or A::B.c
+        # @param [Hash{Symbol=>Proc}] spec
+        def instrument(module_name, method, spec)
+          hook = Sqreen::Graft::Hook[method].add do
+            if spec[:before]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'pre', spec[:before])
+              tag = "weave,rule=ecosystem_#{module_name}"
+              before(tag, flow: true, &cb)
+            end
+
+            if spec[:after]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'post', spec[:after])
+              tag = "weave,rule=ecosystem_#{module_name}"
+              after(tag, flow: true, &cb)
+            end
+
+            if spec[:raised]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'failing', spec[:raised])
+              tag = "weave,rule=ecosystem_#{module_name}"
+              raised(tag, flow: true, &cb)
+            end
+
+            if spec[:ensured]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'finally', spec[:ensured])
+              tag = "weave,rule=ecosystem_#{module_name}"
+              ensured(tag, flow: true, &cb)
+            end
+          end
+          hook.install
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb

@@ -0,0 +1,58 @@
+require 'sqreen/events/remote_exception'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    # This class gets notified of request start/end and
+    # 1) distributes such events to listeners (typically ecosystem modules;
+    #    the method add_start_observer is exposed to ecosystem modules through
+    #    +Sqreen::Ecosystem::ModuleApi::EventListener+ and the dispatch table).
+    # 2) keeps track of whether a request is active on this thread. This is
+    #    so that users of this class can have this information without needing
+    #    to subscribe to request start/events and keeping thread local state
+    #    themselves.
+    # XXX: Since the Ecosystem is also notified of request start/end, it could
+    # notify its modules of request start without going through the dispatch
+    # table and call add_start_observer. We need to think if we want to keep
+    # the transaction / request distinction or if they should just be
+    # assumed to be the same, though.
+    class RequestLifecycleTracking
+      include Sqreen::Log::Loggable
+
+      def initialize
+        @start_observers = []
+        @tl_key = "#{object_id}_req_in_flight"
+      end
+
+      # API for classes needing to know the request state
+
+      # @param cb A callback taking a Rack::Request
+      def add_start_observer(cb)
+        @start_observers << cb
+      end
+
+      def in_request?
+        Thread.current[@tl_key] ? true : false
+      end
+
+      # API for classes notifying this one of request events
+
+      def notify_request_start(rack_req)
+        Thread.current[@tl_key] = true
+        return if @start_observers.empty?
+        @start_observers.each do |cb|
+          begin
+            cb.call(rack_req)
+          rescue ::Exception => e # rubocop:disable Lint/RescueException
+            logger.warn { "Error calling #{cb} on request start: #{e.message}" }
+            Sqreen::RemoteException.record(e)
+          end
+        end
+      end
+
+      def notify_request_end
+        Thread.current[@tl_key] = false
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/signal_consumption.rb

@@ -0,0 +1,35 @@
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    class SignalConsumption
+      include Sqreen::Log::Loggable
+
+      PAYLOAD_CREATOR_SECTIONS = %w[request response params headers].freeze
+
+      # @param [Sqreen::Frameworks::GenericFramework] framework
+      # @param [Sqreen::EcosystemIntegration::RequestLifecycleTracking]
+      # @param [Sqreen::CappedQueue]
+      def initialize(framework, req_lifecycle, queue)
+        @framework = framework
+        @req_lifecycle = req_lifecycle
+        @queue = queue
+      end
+
+      def consume_signal(signal)
+        # transitional
+        unless Sqreen.features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+          logger.debug { "Discarding signal #{signal} (signals disabled)" }
+          return
+        end
+
+        if @req_lifecycle.in_request?
+          # add it to the request record
+          @framework.observe(:signals, signal, PAYLOAD_CREATOR_SECTIONS, true)
+        else
+          @queue.push signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration.rb

@@ -0,0 +1,87 @@
+require 'sqreen/log/loggable'
+require 'sqreen/metrics_store'
+require 'sqreen/ecosystem'
+require 'sqreen/ecosystem/dispatch_table'
+require 'sqreen/ecosystem_integration/around_callbacks'
+require 'sqreen/ecosystem_integration/instrumentation_service'
+require 'sqreen/ecosystem_integration/request_lifecycle_tracking'
+require 'sqreen/ecosystem_integration/signal_consumption'
+
+module Sqreen
+  # This class is the interface through which the agent interacts
+  # with the ecosystem.
+  #
+  # Other classes in the EcosystemIntegration module implement the
+  # functionality that the ecosystem requires in order to deliver
+  # data to the agent and to be informed by the agent of certain
+  # key events (see Sqreen::Ecosystem::DispatchTable).
+  class EcosystemIntegration
+    include Sqreen::Log::Loggable
+
+    # @param [Sqreen::Framework] framework
+    # @param [Proc] create_binning_metric
+    def initialize(framework, queue, create_binning_metric)
+      @framework = framework
+      @queue = queue
+      # XXX: created metrics are insulated from feature upgrades
+      @create_binning_metric = create_binning_metric
+      @request_lifecycle = RequestLifecycleTracking.new
+      @online = false
+    end
+
+    def init
+      raise 'already initialized' if @online
+
+      setup_dispatch_table
+      AroundCallbacks.create_metric = @create_binning_metric
+      Ecosystem.init
+      logger.info 'Ecosystem successfully initialized'
+      @online = true
+    rescue ::Exception => e # rubocop:disable Lint/RescueException
+      logger.warn { "Error initializing Ecosystem: #{e.message}" }
+      logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+      Sqreen::RemoteException.record(e)
+    end
+
+    def disable
+      raise NotImplementedYet
+    end
+
+    def request_start(rack_request)
+      return unless @online
+
+      Ecosystem.start_transaction
+      @request_lifecycle.notify_request_start(rack_request)
+    end
+
+    def request_end
+      return unless @online
+
+      Ecosystem.end_transaction
+    end
+
+    def handle_tracing_command(trace_id_prefix, scopes_config)
+      return unless @online
+
+      Ecosystem.configure_sampling(trace_id_prefix, scopes_config)
+    end
+
+    private
+
+    def setup_dispatch_table
+      Ecosystem::DispatchTable.consume_signal =
+        create_signal_consumption.method(:consume_signal)
+
+      Ecosystem::DispatchTable.add_request_start_listener =
+        @request_lifecycle.method(:add_start_observer)
+
+      Ecosystem::DispatchTable.fetch_logger = lambda { logger }
+
+      Ecosystem::DispatchTable.instrument = InstrumentationService.method(:instrument)
+    end
+
+    def create_signal_consumption
+      SignalConsumption.new(@framework, @request_lifecycle, @queue)
+    end
+  end
+end

data/lib/sqreen/frameworks/generic.rb

@@ -22,8 +22,17 @@ module Sqreen
       include RequestRecorder
       attr_accessor :sqreen_configuration
 
+      attr_writer :req_start_cb, :req_end_cb
+
       def initialize
         clean_request_record
+
+        # for notifying the ecosystem of request boundaries
+        # XXX: this should be refactored. It shouldn't be
+        # the framework doing these notifications to the ecosystem
+        # Probably the rule callback should do it itself
+        @req_start_cb = Proc.new {}
+        @req_end_cb = Proc.new {}
       end
 
       # What kind of database is this
@@ -260,8 +269,12 @@ module Sqreen
       # Nota: cleanup should be performed at end of request (see clean_request)
       def store_request(object)
         return unless ensure_rack_loaded
+
+        rack_req = Rack::Request.new(object)
+        @req_start_cb.call(rack_req)
+
         self.remaining_perf_budget = Sqreen.performance_budget
-        SharedStorage.set(:request, Rack::Request.new(object))
+        SharedStorage.set(:request, rack_req)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -290,6 +303,7 @@ module Sqreen
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
+        @req_end_cb.call
       end
 
       def remaining_perf_budget

data/lib/sqreen/graft/call.rb

@@ -27,6 +27,10 @@ module Sqreen
       def raise(value)
         Flow.raise(value)
       end
+
+      def noop
+        Flow.noop
+      end
     end
 
     class Flow
@@ -46,12 +50,17 @@ module Sqreen
         def raise(value)
           new(:raise, value)
         end
+
+        def noop
+          new(:noop, nil)
+        end
       end
 
       def initialize(action, value, brk = false)
         @action = action
         @value = value
         @break = brk
+        @passed_conditions = false
       end
 
       def return?
@@ -91,6 +100,16 @@ module Sqreen
       def break?
         @break ? true : false
       end
+
+      def passed_conditions!
+        @passed_conditions = true
+
+        self
+      end
+
+      def passed_conditions?
+        @passed_conditions
+      end
     end
 
     class TimerError < StandardError; end
@@ -101,6 +120,7 @@ module Sqreen
       end
 
       attr_reader :tag, :size
+      attr_accessor :conditions_passed
 
       def initialize(tag, &block)
         @tag = tag
@@ -195,8 +215,17 @@ module Sqreen
         @size.even?
       end
 
+      def include_measurements(another_timer)
+        @blips += another_timer.instance_variable_get(:@blips)
+      end
+
+      def start_and_end
+        raise 'Not exactly two measurements recorded' unless size == 2
+        @blips
+      end
+
       def to_s
-        "#{@tag}: time=%.03fus" % (duration * 1_000_000)
+        "#{@tag}: time=%.03fus" % (@tally * 1_000_000)
       end
 
       protected