sqreen 1.20.0 → 1.20.4

data/lib/sqreen/events/request_record.rb

@@ -8,7 +8,6 @@
 require 'json'
 require 'sqreen/log'
 require 'sqreen/event'
-require 'sqreen/encoding_sanitizer'
 require 'sqreen/sensitive_data_redactor'
 
 module Sqreen

data/lib/sqreen/frameworks/generic.rb

@@ -209,7 +209,16 @@ module Sqreen
 
       # Should the agent not be starting up?
       def prevent_startup
+        # SQREEN-880 - prevent Sqreen startup on Sidekiq workers
+        return :sidekiq_cli if defined?(Sidekiq::CLI)
+        return :delayed_job if defined?(Delayed::Command)
+
+        # Prevent Sqreen startup on rake tasks - unless this is a Sqreen test
+        run_in_test = sqreen_configuration.get(:run_in_test)
+        return :rake if !run_in_test && $0.end_with?('rake')
+
         return :irb if $0 == 'irb'
+
         return if sqreen_configuration.nil?
         disable = sqreen_configuration.get(:disable)
         return :config_disable if disable == true || disable.to_s.to_i == 1

data/lib/sqreen/frameworks/rails.rb

@@ -103,13 +103,6 @@ module Sqreen
         run_in_test = sqreen_configuration.get(:run_in_test)
         return :rails_test if !run_in_test && (Rails.env.test? || Rails.env.cucumber?)
 
-        # SQREEN-880 - prevent Sqreen startup on Sidekiq workers
-        return :sidekiq_cli if defined?(Sidekiq::CLI)
-        return :delayed_job if defined?(Delayed::Command)
-
-        # Prevent Sqreen startup on rake tasks - unless this is a Sqreen test
-        return :rake if !run_in_test && $0.end_with?('rake')
-
         return nil unless defined?(Rails::CommandsTasks)
         return nil if defined?(Rails::Server)
         return :rails_console    if defined?(Rails::Console)

data/lib/sqreen/frameworks/request_recorder.rb

@@ -69,6 +69,8 @@ module Sqreen
 
       # signals require request section to be present
       payload_requests << 'request'
+      # for signals, response is optional, but the backend team wants them
+      payload_requests << 'response'
       payload = payload_creator.payload(payload_requests)
       payload[:observed] = observed_items
 

data/lib/sqreen/graft/call.rb

@@ -93,58 +93,116 @@ module Sqreen
       end
     end
 
+    class TimerError < StandardError; end
+
     class Timer
       def self.read
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
 
-      attr_reader :tag
+      attr_reader :tag, :size
 
       def initialize(tag, &block)
         @tag = tag
-        @blips = []
         @block = block
+        @tally = 0
+        @size = 0
       end
 
-      def duration
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e }
+      def elapsed
+        raise(TimerError, 'Timer#elapsed when paused') if @size.even?
+
+        @tally + Timer.read
       end
 
-      def elapsed
-        @blips.each_with_index.reduce(0) { |a, (e, i)| i.even? ? a - e : a + e } + Timer.read
+      def duration
+        raise(TimerError, 'Timer#duration when running') if @size.odd?
+
+        @tally
       end
 
       def ignore
-        @blips << Timer.read
+        raise(TimerError, 'Timer#ignore when paused') if @size.even?
+
+        @size += 1
+        @tally += Timer.read
         yield(self)
       ensure
-        @blips << Timer.read
+        @size += 1
+        @tally -= Timer.read
       end
 
-      def measure
-        @blips << Timer.read
+      def measure(opts = nil)
+        raise(TimerError, 'Timer#measure when running') if @size.odd?
+
+        now = Timer.read
+
+        ignore = opts[:ignore] if opts
+        if ignore
+          ignore.size += 1
+          ignore.tally += now
+        end
+
+        @size += 1
+        @tally -= now
+
         yield(self)
       ensure
-        @blips << Timer.read
+        now = Timer.read
+
+        if ignore
+          ignore.size += 1
+          ignore.tally -= now
+        end
+
+        @size += 1
+        @tally += now
+
         @block.call(self) if @block
-        Sqreen::Graft.logger.debug { "#{@tag}: time=%.03fus" % (duration * 1_000_000) }
       end
 
-      def start
-        @blips << Timer.read
+      def start(at = Timer.read)
+        raise(TimerError, 'Timer#start when started') unless @size.even?
+
+        @size += 1
+        @tally -= at
+
+        at
+      end
+
+      def stop(at = Timer.read)
+        raise(TimerError, 'Timer#stop when unstarted') unless @size.odd?
+
+        @size += 1
+        @tally += at
+
+        at
+      end
+
+      def started?
+        @size != 0 && @size.odd?
       end
 
-      def stop
-        @blips << Timer.read
+      def stopped?
+        @size != 0 && @size.even?
       end
 
-      def size
-        @blips.size
+      def running?
+        @size.odd?
+      end
+
+      def paused?
+        @size.even?
       end
 
       def to_s
         "#{@tag}: time=%.03fus" % (duration * 1_000_000)
       end
+
+      protected
+
+      attr_reader :tally
+      attr_writer :size, :tally
     end
   end
 end

data/lib/sqreen/graft/callback.rb

@@ -21,7 +21,7 @@ module Sqreen
       end
 
       def call(*args, &block)
-        Sqreen::Graft.logger.debug { "[#{Process.pid}] Callback #{@name} disabled:#{disabled?}" }
+        # Sqreen::Graft.logger.debug { "[#{Process.pid}] Callback #{@name} disabled:#{disabled?}" } if Sqreen::Graft.logger.debug?
         return if @disabled
         @block.call(*args, &block)
       end

data/lib/sqreen/graft/hook.rb

@@ -21,6 +21,13 @@ module Sqreen
         self[hook_point, strategy].add(&block)
       end
 
+      def self.ignore
+        Thread.current[:sqreen_hook_entered] = true
+        yield
+      ensure
+        Thread.current[:sqreen_hook_entered] = false
+      end
+
       attr_reader :point
 
       def initialize(hook_point, dependency_test = nil, strategy = :chain)
@@ -46,27 +53,31 @@ module Sqreen
       end
 
       def before(tag = nil, opts = {}, &block)
-        return @before.sort_by(&:rank) if block.nil?
+        return @before if block.nil?
 
         @before << Callback.new(callback_name(:before, tag), opts, &block)
+        @before.sort_by!(&:rank)
       end
 
       def after(tag = nil, opts = {}, &block)
-        return @after.sort_by(&:rank) if block.nil?
+        return @after if block.nil?
 
         @after << Callback.new(callback_name(:after, tag), opts, &block)
+        @after.sort_by!(&:rank)
       end
 
       def raised(tag = nil, opts = {}, &block)
-        return @raised.sort_by(&:rank) if block.nil?
+        return @raised if block.nil?
 
         @raised << Callback.new(callback_name(:raised, tag), opts, &block)
+        @raised.sort_by!(&:rank)
       end
 
       def ensured(tag = nil, opts = {}, &block)
-        return @ensured.sort_by(&:rank) if block.nil?
+        return @ensured if block.nil?
 
         @ensured << Callback.new(callback_name(:ensured, tag), opts, &block)
+        @ensured.sort_by!(&:rank)
       end
 
       def depends_on(&block)
@@ -109,11 +120,25 @@ module Sqreen
         @before = []
         @after = []
         @raised = []
+        @ensured = []
       end
 
       def self.wrapper(hook)
+        timed_hooks_proc = proc do |t|
+          if (request = Thread.current[:sqreen_http_request])
+            request[:timed_hooks] << t if request[:timed_level] >= 1
+          end
+        end
+        timed_callbacks_proc = proc do |t|
+          if (request = Thread.current[:sqreen_http_request])
+            request[:timed_callbacks] << t if request[:timed_level] >= 1
+          end
+        end
+
         Proc.new do |*args, &block|
-          if Thread.current[:sqreen_hook_entered] || Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:time_budget_expended]
+          request = Thread.current[:sqreen_http_request]
+
+          if Thread.current[:sqreen_hook_entered]
             if hook.point.super?
               return super(*args, &block)
             else
@@ -121,49 +146,88 @@ module Sqreen
             end
           end
 
-          Timer.new(hook.point) do |t|
-            Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks] << t
-          end.measure do |chrono|
-            Sqreen::Graft.logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} caller:#{Kernel.caller[2].inspect}" }
+          if request && request[:time_budget_expended] && (hook.before + hook.after + hook.raised + hook.ensured).none?(&:mandatory)
+            if request[:timed_level] >= 2
+              begin
+                request[:skipped_callbacks].concat(hook.before)
+
+                if hook.point.super?
+                  return super(*args, &block)
+                else
+                  return hook.point.apply(self, 'sqreen_hook', *args, &block)
+                end
+              rescue ::Exception # rubocop:disable Lint/RescueException
+                request[:skipped_callbacks].concat(hook.raised)
+                raise
+              else
+                request[:skipped_callbacks].concat(hook.after)
+              ensure
+                request[:skipped_callbacks].concat(hook.ensured)
+              end
+            else
+              if hook.point.super? # rubocop:disable Style/IfInsideElse
+                return super(*args, &block)
+              else
+                return hook.point.apply(self, 'sqreen_hook', *args, &block)
+              end
+            end
+          end
+
+          hook_point_super = hook.point.super?
+          logger = Sqreen::Graft.logger
+          logger_debug = Sqreen::Graft.logger.debug?
+
+          Timer.new(hook.point, &timed_hooks_proc).measure do |chrono|
+            # budget implies request
+            # TODO: make budget depend on a generic context (currently "request")
+            budget = request[:time_budget] if request
+            if budget
+              budget_threshold = request[:time_budget_threshold]
+              budget_ratio = request[:time_budget_ratio]
+              sqreen_timer = request[:sqreen_timer]
+              request_timer = request[:request_timer]
+            end
 
-            budget = Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:time_budget]
-            timer = Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timer] if budget
             hooked_call = HookedCall.new(self, args)
 
             begin
-              timer.start if timer
-              Thread.current[:sqreen_hook_entered] = true
-
-              # TODO: make Call have #ball to throw by cb
-              # TODO: can Call be the ball? r = catch(Call.new, &c)
-              # TODO: is catch return value a Call? a #dispatch?
-              # TODO: make before/after/raised return a CallbackCollection << Array (or extend with module)
-              # TODO: add CallbackCollection#each_with_call(instance, args) { |call| ... } ?
-              # TODO: HookCall x CallbackCollection#each_with_call x Flow
-              # TODO: TimedHookCall TimedCallbackCall
-              # TODO: TimeBoundHookCall TimeBoundCallbackCall TimeBoundFlow?
-
-              Timer.new("#{hook.point}@before") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_before] << t
-              end.measure do |before_chrono|
+              begin
+                sqreen_timer.start if budget
+                Thread.current[:sqreen_hook_entered] = true
+
+                # TODO: make Call have #ball to throw by cb
+                # TODO: can Call be the ball? r = catch(Call.new, &c)
+                # TODO: is catch return value a Call? a #dispatch?
+                # TODO: make before/after/raised return a CallbackCollection << Array (or extend with module)
+                # TODO: add CallbackCollection#each_with_call(instance, args) { |call| ... } ?
+                # TODO: HookCall x CallbackCollection#each_with_call x Flow
+                # TODO: TimedHookCall TimedCallbackCall
+                # TODO: TimeBoundHookCall TimeBoundCallbackCall TimeBoundFlow?
+
+                request_elapsed = request_timer.elapsed if budget
+
                 hook.before.each do |c|
                   next if c.ignore && c.ignore.call
 
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
+                  if budget && !c.mandatory
+                    sqreen_elapsed = sqreen_timer.elapsed
+                    if budget_ratio && !request[:time_budget_expended]
+                      fixed_budget = budget_threshold
+                      proportional_budget = (request_elapsed - sqreen_elapsed) * budget_ratio
+                      remaining = (fixed_budget > proportional_budget ? fixed_budget : proportional_budget) - sqreen_elapsed
+                    else
+                      remaining = budget_threshold - sqreen_elapsed
+                    end
                     unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                      request[:skipped_callbacks] << c
+                      request[:time_budget_expended] = true
                       next
                     end
                   end
 
                   flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      before_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passed), ball)
-                      end
+                    Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                      c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passed), ball)
                     end
                   end
 
@@ -173,52 +237,59 @@ module Sqreen
                   hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
                   break if flow.break?
                 end unless hook.disabled?
+              rescue StandardError => e
+                Sqreen::Weave.logger.debug { "exception:#{e.class} message:'#{e.message}' location:\"#{e.backtrace[0]}\"" }
+                Sqreen::RemoteException.record(e) if Sqreen.queue
               end
 
               raise hooked_call.raise if hooked_call.raising
               return hooked_call.return if hooked_call.returning
             ensure
               Thread.current[:sqreen_hook_entered] = false
-              timer.stop if timer
-            end
+              sqreen_timer.stop if budget
+            end unless hook.before.empty?
 
             begin
               chrono.ignore do
-                if hook.point.super?
+                if hook_point_super
                   hooked_call.returned = super(*(hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed), &block)
                 else
                   hooked_call.returned = hook.point.apply(hooked_call.instance, 'sqreen_hook', *(hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed), &block)
                 end
               end
             rescue ::Exception => e # rubocop:disable Lint/RescueException
-              timer.start if timer
-              Thread.current[:sqreen_hook_entered] = true
-              hooked_call.raised = e
+              begin
+                sqreen_timer.start if budget
+                Thread.current[:sqreen_hook_entered] = true
+                hooked_call.raised = e
 
-              Sqreen::Graft.logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} exception:#{e}" }
-              raise if hook.raised.empty?
+                logger.debug { "[#{Process.pid}] Hook #{hook.point} disabled:#{hook.disabled?} exception:#{e}" } if logger_debug
+                raise if hook.raised.empty?
+
+                request_elapsed = request_timer.elapsed if budget
 
-              Timer.new("#{hook.point}@raised") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_raised] << t
-              end.measure do |raised_chrono|
                 hook.raised.each do |c|
                   next if c.ignore && c.ignore.call
 
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
+                  if budget && !c.mandatory
+                    sqreen_elapsed = sqreen_timer.elapsed
+                    if budget_ratio && !request[:time_budget_expended]
+                      fixed_budget = budget_threshold
+                      proportional_budget = (request_elapsed - sqreen_elapsed) * budget_ratio
+                      remaining = (fixed_budget > proportional_budget ? fixed_budget : proportional_budget) - sqreen_elapsed
+                    else
+                      remaining = budget_threshold - sqreen_elapsed
+                    end
                     unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                      request[:skipped_callbacks] << c
+                      request[:time_budget_expended] = true
                       next
                     end
                   end
 
                   flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      raised_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, hooked_call.raised), ball)
-                      end
+                    Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                      c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, hooked_call.raised), ball)
                     end
                   end
 
@@ -228,6 +299,9 @@ module Sqreen
                   hooked_call.retrying = true if flow.retry?
                   break if flow.break?
                 end unless hook.disabled?
+              rescue StandardError => e
+                Sqreen::Weave.logger.debug { "exception:#{e.class} message:'#{e.message}' location:\"#{e.backtrace[0]}\"" }
+                Sqreen::RemoteException.record(e) if Sqreen.queue
               end
 
               retry if hooked_call.retrying
@@ -235,30 +309,37 @@ module Sqreen
               return hooked_call.return if hooked_call.returning
               raise
             else
-              timer.start if timer
-              Thread.current[:sqreen_hook_entered] = true
+              begin
+                sqreen_timer.start if budget
+                Thread.current[:sqreen_hook_entered] = true
+
+                # TODO: hooked_call.returning should be always false here?
+                return hooked_call.returning ? hooked_call.return : hooked_call.returned if hook.after.empty?
+
+                request_elapsed = request_timer.elapsed if budget
 
-              Timer.new("#{hook.point}@after") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_after] << t
-              end.measure do |after_chrono|
                 hook.after.each do |c|
                   next if c.ignore && c.ignore.call
 
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
+                  if budget && !c.mandatory
+                    sqreen_elapsed = sqreen_timer.elapsed
+                    if budget_ratio && !request[:time_budget_expended]
+                      fixed_budget = budget_threshold
+                      proportional_budget = (request_elapsed - sqreen_elapsed) * budget_ratio
+                      remaining = (fixed_budget > proportional_budget ? fixed_budget : proportional_budget) - sqreen_elapsed
+                    else
+                      remaining = budget_threshold - sqreen_elapsed
+                    end
                     unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                      request[:skipped_callbacks] << c
+                      request[:time_budget_expended] = true
                       next
                     end
                   end
 
                   flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      after_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
-                      end
+                    Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                      c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
                     end
                   end
 
@@ -267,34 +348,48 @@ module Sqreen
                   hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
                   break if flow.break?
                 end unless hook.disabled?
+              rescue StandardError => e
+                Sqreen::Weave.logger.debug { "exception:#{e.class} message:'#{e.message}' location:\"#{e.backtrace[0]}\"" }
+                Sqreen::RemoteException.record(e) if Sqreen.queue
               end
 
               raise hooked_call.raise if hooked_call.raising
               return hooked_call.returning ? hooked_call.return : hooked_call.returned
             ensure
-              # TODO: timer.start if someone has thrown?
+              begin
+                # TODO: sqreen_timer.start if someone has thrown?
+                # TODO: sqreen_timer.stop at end of rescue+else?
+                # TODO: Thread.current[:sqreen_hook_entered] = true if neither rescue nor else ie thrown?
+                # TODO: Thread.current[:sqreen_hook_entered] = false at end of rescue+else? (risky?)
+
+                # TODO: uniform early bail out? (but don't forget sqreen_hook_entered and sqreen_timer)
+                # return hooked_call.returning ? hooked_call.return : hooked_call.returned if hook.ensured.empty?
+
+                # done at either rescue or else
+                # request_elapsed = request_timer.elapsed if budget # && !hook.ensured.empty?
 
-              Timer.new("#{hook.point}@ensured") do |t|
-                Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_hooks_ensured] << t
-              end.measure do |ensured_chrono|
                 hook.ensured.each do |c|
                   next if c.ignore && c.ignore.call
 
-                  if timer && !c.mandatory
-                    remaining = budget - timer.elapsed
+                  if budget && !c.mandatory
+                    sqreen_elapsed = sqreen_timer.elapsed
+                    if budget_ratio && !request[:time_budget_expended]
+                      fixed_budget = budget_threshold
+                      proportional_budget = (request_elapsed - sqreen_elapsed) * budget_ratio
+                      remaining = (fixed_budget > proportional_budget ? fixed_budget : proportional_budget) - sqreen_elapsed
+                    else
+                      remaining = budget_threshold - sqreen_elapsed
+                    end
                     unless remaining > 0
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:skipped_callbacks] << c && Thread.current[:sqreen_http_request][:time_budget_expended] = true
+                      request[:skipped_callbacks] << c
+                      request[:time_budget_expended] = true
                       next
                     end
                   end
 
                   flow = catch(Ball.new) do |ball|
-                    Timer.new(c.name) do |t|
-                      Thread.current[:sqreen_http_request] && Thread.current[:sqreen_http_request][:timed_callbacks] << t
-                    end.measure do
-                      ensured_chrono.ignore do
-                        c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
-                      end
+                    Timer.new(c.name, &timed_callbacks_proc).measure(ignore: chrono) do
+                      c.call(CallbackCall.new(c, remaining, hooked_call.instance, hooked_call.args_passing ? hooked_call.args_pass : hooked_call.args_passed, nil, hooked_call.returned), ball)
                     end
                   end
 
@@ -302,11 +397,18 @@ module Sqreen
                   hooked_call.raise = flow.raise and hooked_call.raising = true if flow.raise?
                   hooked_call.return = flow.return and hooked_call.returning = true if flow.return?
                   break if flow.break?
-                end unless hook.disabled?
+                end unless hook.ensured.empty? || hook.disabled?
+
+                Thread.current[:sqreen_hook_entered] = false
+                sqreen_timer.stop if budget
+              rescue StandardError => e
+                Sqreen::Weave.logger.debug { "exception:#{e.class} message:'#{e.message}' location:\"#{e.backtrace[0]}\"" }
+                Sqreen::RemoteException.record(e) if Sqreen.queue
               end
 
-              Thread.current[:sqreen_hook_entered] = false
-              timer.stop if timer
+              # TODO: should we run the following?
+              # raise hooked_call.raise if hooked_call.raising
+              # return hooked_call.returning ? hooked_call.return : hooked_call.returned
             end
           end # chrono
         end