sqreen 1.19.1 → 1.20.2

data/lib/sqreen/events/remote_exception.rb

@@ -30,27 +30,5 @@ module Sqreen
     def klass
       payload['exception'].class.name
     end
-
-    def to_hash
-      exception = payload['exception']
-      ev = {
-        :klass => exception.class.name,
-        :message => exception.message,
-        :params => payload['request_params'],
-        :time => payload['time'],
-        :infos => {
-          :client_ip => payload['client_ip'],
-        },
-        :request => payload['request_infos'],
-        :headers => payload['headers'],
-        :rule_name => payload['rule_name'],
-        :rulespack_id => payload['rulespack_id'],
-      }
-
-      ev[:infos].merge!(payload['infos']) if payload['infos']
-      return ev unless exception.backtrace
-      ev[:context] = { :backtrace => exception.backtrace.map(&:to_s) }
-      ev
-    end
   end
 end

data/lib/sqreen/events/request_record.rb

@@ -14,6 +14,10 @@ require 'sqreen/sensitive_data_redactor'
 module Sqreen
   # When a request is deeemed worthy of being sent to the backend
   class RequestRecord < Sqreen::Event
+    attr_reader :redactor
+
+    # @param [Hash] payload
+    # @param [Sqreen::SensitiveDataRedactor] redactor
     def initialize(payload, redactor = nil)
       @redactor = redactor
       super(payload)
@@ -23,74 +27,18 @@ module Sqreen
       (payload && payload[:observed]) || {}
     end
 
-    def to_hash
-      res = { :version => '20171208' }
-      if payload[:observed]
-        res[:observed] = payload[:observed].dup
-        rulespack = nil
-        if observed[:attacks]
-          res[:observed][:attacks] = observed[:attacks].map do |att|
-            natt = att.dup
-            rulespack = natt.delete(:rulespack_id) || rulespack
-            natt
-          end
-        end
-        if observed[:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = observed[:sqreen_exceptions].map do |exc|
-            nex = exc.dup
-            excp = nex.delete(:exception)
-            if excp
-              nex[:message] = excp.message
-              nex[:klass] = excp.class.name
-            end
-            rulespack = nex.delete(:rulespack_id) || rulespack
-            nex
-          end
-        end
-        res[:rulespack_id] = rulespack unless rulespack.nil?
-        if observed[:observations]
-          res[:observed][:observations] = observed[:observations].map do |cat, key, value, time|
-            { :category => cat, :key => key, :value => value, :time => time }
-          end
-        end
-        if observed[:sdk]
-          res[:observed][:sdk] = processed_sdk_calls
-        end
-      end
-      res[:local] = payload['local'] if payload['local']
-      if payload['request']
-        res[:request] = payload['request'].dup
-        res[:client_ip] = res[:request].delete(:client_ip) if res[:request][:client_ip]
-      else
-        res[:request] = {}
-      end
-      if payload['response']
-        res[:response] = payload['response'].dup
-      else
-        res[:response] = {}
-      end
-
-      res[:request][:parameters] = payload['params'] if payload['params']
-      res[:request][:headers] = payload['headers'] if payload['headers']
-
-      res = Sqreen::EncodingSanitizer.sanitize(res)
+    def last_identify_args
+      return nil unless observed[:sdk]
 
-      if @redactor
-        res[:request], redacted = @redactor.redact(res[:request])
-        if redacted.any? && res[:observed] && res[:observed][:attacks]
-          res[:observed][:attacks] = @redactor.redact_attacks!(res[:observed][:attacks], redacted)
-        end
-        if redacted.any? && res[:observed] && res[:observed][:sqreen_exceptions]
-          res[:observed][:sqreen_exceptions] = @redactor.redact_exceptions!(res[:observed][:sqreen_exceptions], redacted)
-        end
+      observed[:sdk].reverse_each do |meth, _time, *args|
+        next unless meth == :identify
+        return args
       end
-
-      res
+      nil
     end
 
-    private
-
     def processed_sdk_calls
+      return [] unless observed[:sdk]
       auth_keys = last_identify_id
 
       observed[:sdk].map do |meth, time, *args|
@@ -102,6 +50,8 @@ module Sqreen
       end
     end
 
+    private
+
     def inject_identifiers(args, meth, auth_keys)
       return args unless meth == :track && auth_keys
 
@@ -118,13 +68,8 @@ module Sqreen
     end
 
     def last_identify_id
-      return nil unless observed[:sdk]
-
-      observed[:sdk].reverse_each do |meth, _time, *args|
-        next unless meth == :identify
-        return args.first if args.respond_to? :first
-      end
-      nil
+      args = last_identify_args
+      args.first if args.respond_to? :first
     end
   end
 end

data/lib/sqreen/frameworks/request_recorder.rb

@@ -58,12 +58,20 @@ module Sqreen
       Sqreen.log.debug { "close_request_record called. observed_items: #{observed_items}" }
 
       clean_request_record if observed_items.nil?
-      if only_metric_observation
+      if Sqreen.features['use_signals'] || only_metric_observation
         push_metrics(observations_queue, queue)
-        return clean_request_record
       end
+
+      if only_metric_observation
+        clean_request_record
+        return
+      end
+
+      # signals require request section to be present
+      payload_requests << 'request'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
+
       queue.push create_request_record(payload)
       clean_request_record
     end
@@ -79,10 +87,13 @@ module Sqreen
       @redactor ||= SensitiveDataRedactor.from_config
     end
 
+    # pushes metric observations to the observations queue
+    # and clears the list for the request record
     def push_metrics(observations_queue, event_queue)
       observed_items[:observations].each do |obs|
         observations_queue.push obs
       end
+      observed_items[:observations] = []
       return unless observations_queue.size > MAX_OBS_QUEUE_LENGTH / 2
       event_queue.push Sqreen::METRICS_EVENT
     end

data/lib/sqreen/graft/call.rb

@@ -98,53 +98,66 @@ module Sqreen
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      attr_reader :tag
+      attr_reader :tag, :size, :duration
 
       def initialize(tag, &block)
         @tag = tag
-        @blips = []
         @block = block
-      end
-
-      def duration
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e }
+        @duration = 0
+        @size = 0
       end
 
       def elapsed
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e } + Timer.read
+        @duration + Timer.read
       end
 
       def ignore
-        @blips << Timer.read
+        @size += 1
+        @duration += Timer.read
         yield(self)
       ensure
-        @blips << Timer.read
+        @size += 1
+        @duration -= Timer.read
       end
 
-      def measure
-        @blips << Timer.read
+      def measure(opts = nil)
+        now = Timer.read
+        ignore = opts[:ignore] if opts
+        if ignore
+          ignore.size += 1
+          ignore.duration += now
+        end
+        @size += 1
+        @duration -= now
         yield(self)
       ensure
-        @blips << Timer.read
+        now = Timer.read
+        if ignore
+          ignore.size += 1
+          ignore.duration -= now
+        end
+        @size += 1
+        @duration += now
         @block.call(self) if @block
-        Sqreen::Graft.logger.debug { "#{@tag}: time=%.03fus" % (duration * 1_000_000) }
       end
 
       def start
-        @blips << Timer.read
+        @size += 1
+        @duration -= Timer.read
       end
 
       def stop
-        @blips << Timer.read
-      end
-
-      def size
-        @blips.size
+        @size += 1
+        @duration += Timer.read
       end
 
       def to_s
         "#{@tag}: time=%.03fus" % (duration * 1_000_000)
       end
+
+      protected
+
+      attr_writer :size, :duration
     end
   end
 end

data/lib/sqreen/graft/callback.rb

@@ -21,7 +21,7 @@ module Sqreen
       end
 
       def call(*args, &block)
-        Sqreen::Graft.logger.debug { "[#{Process.pid}] Callback #{@name} disabled:#{disabled?}" }
+        # Sqreen::Graft.logger.debug { "[#{Process.pid}] Callback #{@name} disabled:#{disabled?}" } if Sqreen::Graft.logger.debug?
         return if @disabled
         @block.call(*args, &block)
       end

data/lib/sqreen/graft/hook.rb

@@ -46,27 +46,31 @@ module Sqreen
       end
 
       def before(tag = nil, opts = {}, &block)
-        return @before.sort_by(&:rank) if block.nil?
+        return @before if block.nil?
 
         @before << Callback.new(callback_name(:before, tag), opts, &block)
+        @before.sort_by!(&:rank)
       end
 
       def after(tag = nil, opts = {}, &block)
-        return @after.sort_by(&:rank) if block.nil?
+        return @after if block.nil?
 
         @after << Callback.new(callback_name(:after, tag), opts, &block)
+        @after.sort_by!(&:rank)
       end
 
       def raised(tag = nil, opts = {}, &block)
-        return @raised.sort_by(&:rank) if block.nil?
+        return @raised if block.nil?
 
         @raised << Callback.new(callback_name(:raised, tag), opts, &block)
+        @raised.sort_by!(&:rank)
       end
 
       def ensured(tag = nil, opts = {}, &block)
-        return @ensured.sort_by(&:rank) if block.nil?
+        return @ensured if block.nil?
 
         @ensured << Callback.new(callback_name(:ensured, tag), opts, &block)
+        @ensured.sort_by!(&:rank)
       end
 
       def depends_on(&block)
@@ -112,6 +116,13 @@ module Sqreen
       end
 
       def self.wrapper(hook)
+        timed_hooks_proc = proc do |t|
+          Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks] << t
+        end
+        timed_callbacks_proc = proc do |t|
+          Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
+        end
+
         Proc.new do |*args, &block|
           if Thread.current[:sqreen_hook_entered] || Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:time_budget_expended]
             if hook.point.super?
@@ -121,10 +132,12 @@ module Sqreen
             end
           end
 
-          Timer.new(hook.point) do |t|
-            Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks] << t
-          end.measure do |chrono|
-            Sqreen::Graft.logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} caller:#{Kernel.caller[2].inspect}" }
+          hook_point_super = hook.point.super?
+          logger = Sqreen::Graft.logger
+          logger_debug = Sqreen::Graft.logger.debug?
+
+          Timer.new(hook.point, &timed_hooks_proc).measure do |chrono|
+            logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} caller:#{Kernel.caller[2].inspect}" } if logger_debug
 
             budget = Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:time_budget]
             timer = Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timer] if budget
@@ -143,37 +156,29 @@ module Sqreen
               # TODO: TimedHookCall TimedCallbackCall
               # TODO: TimeBoundHookCall TimeBoundCallbackCall TimeBoundFlow?
 
-              Timer.new("#{hook.point}@before") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_before] << t
-              end.measure do |before_chrono|
-                hook.before.each do |c|
-                  next if c.ignore && c.ignore.call
-
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
-                    unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
-                      next
-                    end
+              hook.before.each do |c|
+                next if c.ignore && c.ignore.call
+
+                if timer && !c.mandatory
+                  remaining = budget - timer.elapsed
+                  unless remaining > 0
+                    Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                    next
                   end
+                end
 
-                  flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      before_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passed), ball)
-                      end
-                    end
+                flow = catch(Ball.new) do |ball|
+                  Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                    c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passed), ball)
                   end
+                end
 
-                  next unless c.flow && flow.is_a?(Flow)
-                  hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
-                  hooked_call.args_pass = flow.args and hooked_call.args_passing = true if flow.args?
-                  hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
-                  break if flow.break?
-                end unless hook.disabled?
-              end
+                next unless c.flow && flow.is_a?(Flow)
+                hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
+                hooked_call.args_pass = flow.args and hooked_call.args_passing = true if flow.args?
+                hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
+                break if flow.break?
+              end unless hook.disabled?
 
               raise hooked_call.raise if hooked_call.raising
               return hooked_call.return if hooked_call.returning
@@ -184,7 +189,7 @@ module Sqreen
 
             begin
               chrono.ignore do
-                if hook.point.super?
+                if hook_point_super
                   hooked_call.returned = super(*(hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed), &block)
                 else
                   hooked_call.returned = hook.point.apply(hooked_call.instance, 'sqreen_hook', *(hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed), &block)
@@ -195,40 +200,32 @@ module Sqreen
               Thread.current[:sqreen_hook_entered] = true
               hooked_call.raised = e
 
-              Sqreen::Graft.logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} exception:#{e}" }
+              logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} exception:#{e}" } if logger_debug
               raise if hook.raised.empty?
 
-              Timer.new("#{hook.point}@raised") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_raised] << t
-              end.measure do |raised_chrono|
-                hook.raised.each do |c|
-                  next if c.ignore && c.ignore.call
-
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
-                    unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
-                      next
-                    end
+              hook.raised.each do |c|
+                next if c.ignore && c.ignore.call
+
+                if timer && !c.mandatory
+                  remaining = budget - timer.elapsed
+                  unless remaining > 0
+                    Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                    next
                   end
+                end
 
-                  flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      raised_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, hooked_call.raised), ball)
-                      end
-                    end
+                flow = catch(Ball.new) do |ball|
+                  Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                    c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, hooked_call.raised), ball)
                   end
+                end
 
-                  next unless c.flow && flow.is_a?(Flow)
-                  hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
-                  hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
-                  hooked_call.retrying = true if flow.retry?
-                  break if flow.break?
-                end unless hook.disabled?
-              end
+                next unless c.flow && flow.is_a?(Flow)
+                hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
+                hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
+                hooked_call.retrying = true if flow.retry?
+                break if flow.break?
+              end unless hook.disabled?
 
               retry if hooked_call.retrying
               raise hooked_call.raise if hooked_call.raising
@@ -238,72 +235,56 @@ module Sqreen
               timer.start if timer
               Thread.current[:sqreen_hook_entered] = true
 
-              Timer.new("#{hook.point}@after") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_after] << t
-              end.measure do |after_chrono|
-                hook.after.each do |c|
-                  next if c.ignore && c.ignore.call
-
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
-                    unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
-                      next
-                    end
+              hook.after.each do |c|
+                next if c.ignore && c.ignore.call
+
+                if timer && !c.mandatory
+                  remaining = budget - timer.elapsed
+                  unless remaining > 0
+                    Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                    next
                   end
+                end
 
-                  flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      after_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
-                      end
-                    end
+                flow = catch(Ball.new) do |ball|
+                  Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                    c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
                   end
+                end
 
-                  next unless c.flow && flow.is_a?(Flow)
-                  hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
-                  hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
-                  break if flow.break?
-                end unless hook.disabled?
-              end
+                next unless c.flow && flow.is_a?(Flow)
+                hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
+                hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
+                break if flow.break?
+              end unless hook.disabled?
 
               raise hooked_call.raise if hooked_call.raising
               return hooked_call.returning ? hooked_call.return : hooked_call.returned
             ensure
               # TODO: timer.start if someone has thrown?
 
-              Timer.new("#{hook.point}@ensured") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_ensured] << t
-              end.measure do |ensured_chrono|
-                hook.ensured.each do |c|
-                  next if c.ignore && c.ignore.call
-
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
-                    unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
-                      next
-                    end
+              hook.ensured.each do |c|
+                next if c.ignore && c.ignore.call
+
+                if timer && !c.mandatory
+                  remaining = budget - timer.elapsed
+                  unless remaining > 0
+                    Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                    next
                   end
+                end
 
-                  flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      ensured_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
-                      end
-                    end
+                flow = catch(Ball.new) do |ball|
+                  Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                    c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
                   end
+                end
 
-                  next unless c.flow && flow.is_a?(Flow)
-                  hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
-                  hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
-                  break if flow.break?
-                end unless hook.disabled?
-              end
+                next unless c.flow && flow.is_a?(Flow)
+                hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
+                hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
+                break if flow.break?
+              end unless hook.disabled?
 
               Thread.current[:sqreen_hook_entered] = false
               timer.stop if timer