sqreen 1.20.4.beta1 → 1.21.0

data/lib/sqreen/ecosystem_integration/around_callbacks.rb

@@ -0,0 +1,89 @@
+require 'sqreen/log/loggable'
+require 'sqreen/events/remote_exception'
+require 'sqreen/mono_time'
+
+module Sqreen
+  class EcosystemIntegration
+    module AroundCallbacks
+      class << self
+        include Log::Loggable::ClassMethods
+
+        # for instrumentation hooks
+        # instrumentation hooks already handle budgets, so nothing
+        # to do in that respect
+        def wrap_instrumentation_hook(module_name, action, callable)
+          perf_notif_name = "ecosystem_#{module_name}"
+
+          Proc.new do |*args|
+            begin
+              start = Sqreen.time
+              callable.call(*args)
+            rescue ::Exception => e # rubocop:disable Lint/RescueException
+              # 2) rescue exceptions
+              logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
+              logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+              Sqreen::RemoteException.record(e)
+            ensure
+              # 3) contribute to performance metrics
+              stop = Sqreen.time
+              Sqreen::PerformanceNotifications.notify(perf_notif_name, action, start, stop)
+            end # end proc
+          end # end begin
+        end
+
+        # XXX: not used yet
+        def wrap_generic_callback(module_name, action, callable)
+          timer_name = "ecosystem:#{module_name}@#{action}"
+          perf_notif_name = "ecosystem_#{module_name}"
+
+          Proc.new do |*args|
+            begin
+              req_storage = Thread.current[:sqreen_http_request]
+
+              timer = Graft::Timer.new(timer_name) do |t|
+                # this is an epilogue to measure()
+                req_storage && req_storage[:timed_hooks] << t
+              end
+
+              req_timer = nil
+              timer.measure do
+                # not in a request, no budget; call cb
+                next callable.call(*args) unless req_storage
+
+                # 1) budget enforcement
+                # skip callback if budget already expended
+                next if req_storage[:time_budget_expended]
+
+                budget = req_storage[:time_budget]
+                if budget
+                  req_timer = req_storage[:timer]
+                  remaining = budget - req_timer.elapsed
+                  unless remaining > 0
+                    req_storage[:time_budget_expended] = true
+                    next # skip callback
+                  end
+                end
+
+                callable.call(*args)
+              end
+            rescue ::Exception => e # rubocop:disable Lint/RescueException
+              # 2) rescue exceptions
+              logger.warn { "Error in #{module_name}:#{action}: #{e.message}" }
+              logger.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
+              Sqreen::RemoteException.record(e)
+            ensure
+              # 3) contribute to performance metrics
+              if timer
+                req_timer.include_measurements(timer) if req_timer
+
+                Sqreen::PerformanceNotifications.notify(
+                  perf_notif_name, action, *timer.start_and_end
+                )
+              end
+            end # end begin
+          end # end proc
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/instrumentation_service.rb

@@ -0,0 +1,38 @@
+require 'sqreen/graft/hook'
+require 'sqreen/ecosystem_integration/around_callbacks'
+
+module Sqreen
+  class EcosystemIntegration
+    module InstrumentationService
+      class << self
+        # @param [String] module_name
+        # @param [String] method in form A::B#c or A::B.c
+        # @param [Hash{Symbol=>Proc}] spec
+        def instrument(module_name, method, spec)
+          hook = Sqreen::Graft::Hook[method].add do
+            if spec[:before]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'pre', spec[:before])
+              before(nil, flow: true, &cb)
+            end
+
+            if spec[:after]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'post', spec[:after])
+              after(nil, flow: true, &cb)
+            end
+
+            if spec[:raised]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'failing', spec[:raised])
+              raised(nil, flow: true, &cb)
+            end
+
+            if spec[:ensured]
+              cb = AroundCallbacks.wrap_instrumentation_hook(module_name, 'finally', spec[:ensured])
+              ensured(nil, flow: true, &cb)
+            end
+          end
+          hook.install
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb

@@ -0,0 +1,58 @@
+require 'sqreen/events/remote_exception'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    # This class gets notified of request start/end and
+    # 1) distributes such events to listeners (typically ecosystem modules;
+    #    the method add_start_observer is exposed to ecosystem modules through
+    #    +Sqreen::Ecosystem::ModuleApi::EventListener+ and the dispatch table).
+    # 2) keeps track of whether a request is active on this thread. This is
+    #    so that users of this class can have this information without needing
+    #    to subscribe to request start/events and keeping thread local state
+    #    themselves.
+    # XXX: Since the Ecosystem is also notified of request start/end, it could
+    # notify its modules of request start without going through the dispatch
+    # table and call add_start_observer. We need to think if we want to keep
+    # the transaction / request distinction or if they should just be
+    # assumed to be the same, though.
+    class RequestLifecycleTracking
+      include Sqreen::Log::Loggable
+
+      def initialize
+        @start_observers = []
+        @tl_key = "#{object_id}_req_in_flight"
+      end
+
+      # API for classes needing to know the request state
+
+      # @param cb A callback taking a Rack::Request
+      def add_start_observer(cb)
+        @start_observers << cb
+      end
+
+      def in_request?
+        Thread.current[@tl_key] ? true : false
+      end
+
+      # API for classes notifying this one of request events
+
+      def notify_request_start(rack_req)
+        Thread.current[@tl_key] = true
+        return if @start_observers.empty?
+        @start_observers.each do |cb|
+          begin
+            cb.call(rack_req)
+          rescue ::Exception => e # rubocop:disable Lint/RescueException
+            logger.warn { "Error calling #{cb} on request start: #{e.message}" }
+            Sqreen::RemoteException.record(e)
+          end
+        end
+      end
+
+      def notify_request_end
+        Thread.current[@tl_key] = false
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem_integration/signal_consumption.rb

@@ -0,0 +1,35 @@
+require 'sqreen/log/loggable'
+
+module Sqreen
+  class EcosystemIntegration
+    class SignalConsumption
+      include Sqreen::Log::Loggable
+
+      PAYLOAD_CREATOR_SECTIONS = %w[request response params headers].freeze
+
+      # @param [Sqreen::Frameworks::GenericFramework] framework
+      # @param [Sqreen::EcosystemIntegration::RequestLifecycleTracking]
+      # @param [Sqreen::CappedQueue]
+      def initialize(framework, req_lifecycle, queue)
+        @framework = framework
+        @req_lifecycle = req_lifecycle
+        @queue = queue
+      end
+
+      def consume_signal(signal)
+        # transitional
+        unless Sqreen.features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+          logger.debug { "Discarding signal #{signal} (signals disabled)" }
+          return
+        end
+
+        if @req_lifecycle.in_request?
+          # add it to the request record
+          @framework.observe(:signals, signal, PAYLOAD_CREATOR_SECTIONS, true)
+        else
+          @queue.push signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/events/request_record.rb

@@ -8,7 +8,6 @@
 require 'json'
 require 'sqreen/log'
 require 'sqreen/event'
-require 'sqreen/encoding_sanitizer'
 require 'sqreen/sensitive_data_redactor'
 
 module Sqreen

data/lib/sqreen/frameworks/generic.rb

@@ -22,8 +22,17 @@ module Sqreen
       include RequestRecorder
       attr_accessor :sqreen_configuration
 
+      attr_writer :req_start_cb, :req_end_cb
+
       def initialize
         clean_request_record
+
+        # for notifying the ecosystem of request boundaries
+        # XXX: this should be refactored. It shouldn't be
+        # the framework doing these notifications to the ecosystem
+        # Probably the rule callback should do it itself
+        @req_start_cb = Proc.new {}
+        @req_end_cb = Proc.new {}
       end
 
       # What kind of database is this
@@ -260,8 +269,12 @@ module Sqreen
       # Nota: cleanup should be performed at end of request (see clean_request)
       def store_request(object)
         return unless ensure_rack_loaded
+
+        rack_req = Rack::Request.new(object)
+        @req_start_cb.call(rack_req)
+
         self.remaining_perf_budget = Sqreen.performance_budget
-        SharedStorage.set(:request, Rack::Request.new(object))
+        SharedStorage.set(:request, rack_req)
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
@@ -290,6 +303,7 @@ module Sqreen
         SharedStorage.set(:xss_params, nil)
         SharedStorage.set(:whitelisted, nil)
         SharedStorage.set(:request_overtime, nil)
+        @req_end_cb.call
       end
 
       def remaining_perf_budget

data/lib/sqreen/frameworks/request_recorder.rb

@@ -69,6 +69,8 @@ module Sqreen
 
       # signals require request section to be present
       payload_requests << 'request'
+      # for signals, response is optional, but the backend team wants them
+      payload_requests << 'response'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
 

data/lib/sqreen/graft/call.rb

@@ -195,6 +195,15 @@ module Sqreen
         @size.even?
       end
 
+      def include_measurements(another_timer)
+        @blips += another_timer.instance_variable_get(:@blips)
+      end
+
+      def start_and_end
+        raise 'Not exactly two measurements recorded' unless size == 2
+        @blips
+      end
+
       def to_s
         "#{@tag}: time=%.03fus" % (duration * 1_000_000)
       end

data/lib/sqreen/graft/hook.rb

@@ -11,13 +11,15 @@ require 'sqreen/graft/hook_point'
 module Sqreen
   module Graft
     class Hook
+      DEFAULT_STRATEGY = Sqreen::Graft::HookPoint::DEFAULT_STRATEGY
+
       @hooks = {}
 
-      def self.[](hook_point, strategy = :chain)
+      def self.[](hook_point, strategy = DEFAULT_STRATEGY)
         @hooks[hook_point] ||= new(hook_point, nil, strategy)
       end
 
-      def self.add(hook_point, strategy = :chain, &block)
+      def self.add(hook_point, strategy = DEFAULT_STRATEGY, &block)
         self[hook_point, strategy].add(&block)
       end
 
@@ -30,7 +32,7 @@ module Sqreen
 
       attr_reader :point
 
-      def initialize(hook_point, dependency_test = nil, strategy = :chain)
+      def initialize(hook_point, dependency_test = nil, strategy = DEFAULT_STRATEGY)
         @disabled = false
         @point = hook_point.is_a?(HookPoint) ? hook_point : HookPoint.new(hook_point, strategy)
         @before = []

data/lib/sqreen/graft/hook_point.rb

@@ -23,6 +23,8 @@ module Sqreen
     end
 
     class HookPoint
+      DEFAULT_STRATEGY = Module.respond_to?(:prepend) ? :prepend : :chain
+
       def self.parse(hook_point)
         klass_name, separator, method_name = hook_point.split(/(\#|\.)/, 2)
 
@@ -36,7 +38,7 @@ module Sqreen
 
       attr_reader :klass_name, :method_kind, :method_name
 
-      def initialize(hook_point, strategy = :chain)
+      def initialize(hook_point, strategy = DEFAULT_STRATEGY)
         @klass_name, @method_kind, @method_name = Sqreen::Graft::HookPoint.parse(hook_point)
         @strategy = strategy
       end
@@ -177,23 +179,30 @@ module Sqreen
 
       private
 
-      def prepend(key)
+      def hook_spot(key)
         target = klass_method? ? klass.singleton_class : klass
         mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
+        raise "Inconsistency detected: #{target} missing from its own ancestors" if mod.is_a?(Array)
+
+        [target, mod]
+      end
+
+      def prepend(key)
+        target, mod = hook_spot(key)
+
         mod ||= HookSpot.new(key)
+
         target.instance_eval { prepend(mod) }
       end
 
       def prepended?(key)
-        target = klass_method? ? klass.singleton_class : klass
-        mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
+        _, mod = hook_spot(key)
 
         mod != nil
       end
 
       def overridden?(key)
-        target = klass_method? ? klass.singleton_class : klass
-        mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
+        _, mod = hook_spot(key)
 
         (mod.instance_methods(false) + mod.protected_instance_methods(false) + mod.private_instance_methods(false)).include?(method_name)
       end
@@ -202,8 +211,7 @@ module Sqreen
         hook_point = self
         method_name = @method_name
 
-        target = klass_method? ? klass.singleton_class : klass
-        mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
+        _, mod = hook_spot(key)
 
         mod.instance_eval do
           if hook_point.private_method?
@@ -221,8 +229,7 @@ module Sqreen
       def unoverride(key)
         method_name = @method_name
 
-        target = klass_method? ? klass.singleton_class : klass
-        mod = target.ancestors.each { |e| break if e == target; break(e) if e.class == HookSpot && e.key == key }
+        _, mod = hook_spot(key)
 
         mod.instance_eval { remove_method(method_name) }
       end

data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/kit/configuration'
 require 'sqreen/kit/signals/point'
 require 'sqreen/kit/signals/dto_helper'
 
@@ -38,6 +39,7 @@ class Sqreen::Kit::Signals::Specialized::SqreenException < Sqreen::Kit::Signals:
     self.payload_schema = PAYLOAD_SCHEMA_VERSION
     self.signal_name = 'sq.agent.exception'
     self.time = values[:time] || Time.now
+    self.source = values[:source] || Sqreen::Kit::Configuration.default_source
     super
   end
 

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -6,6 +6,7 @@
 require 'sqreen/aggregated_metric'
 require 'sqreen/log/loggable'
 require 'sqreen/legacy/waf_redactions'
+require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Legacy
@@ -55,6 +56,12 @@ module Sqreen
               when AggregatedMetric
                 logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
                 next
+              when Sqreen::Kit::Signals::Signal
+                logger.warn "Signal event in non-signal mode"
+                next
+              when Sqreen::Kit::Signals::Trace
+                logger.warn "Trace event in non-signal mode"
+                next
               when Attack # in practice only found inside req rec
                 EventToHash.convert_attack event
               when RemoteException
@@ -72,7 +79,7 @@ module Sqreen
           tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
           "Doing batch with the following tally of event types: #{tally}"
         end
-        post('batch', { batch: batch }, {}, RETRY_MANY)
+        post('batch', { batch: batch.compact }, {}, RETRY_MANY)
       end
 
       private
@@ -166,7 +173,7 @@ module Sqreen
           res[:request][:parameters] = payload['params'] if payload['params']
           res[:request][:headers] = payload['headers'] if payload['headers']
 
-          res = Sqreen::EncodingSanitizer.sanitize(res)
+          res = Sqreen::Kit::StringSanitizer.sanitize(res)
 
           if rr.redactor
             res[:request], redacted = rr.redactor.redact(res[:request])