sqreen 1.20.4 → 1.21.0.beta1

data/lib/sqreen/session.rb

@@ -249,8 +249,10 @@ module Sqreen
       end
       Sqreen.log.info 'Login success.'
       @session_id = res['session_id']
+
       Kit::Configuration.session_key = @session_id
       Kit.reset
+
       Sqreen.log.debug { "received session_id #{@session_id}" }
       Sqreen.logged_in = true
       res

data/lib/sqreen/signals/conversions.rb

@@ -118,6 +118,7 @@ module Sqreen
           signals += req_rec.processed_sdk_calls
                             .select { |h| h[:name] == :track }
                             .map { |h| convert_track(h) }
+          signals += (observed[:signals] || [])
 
           trace = Kit::Signals::Specialized::HttpTrace.new(
             actor: Kit::Signals::Actor.new(
@@ -137,7 +138,7 @@ module Sqreen
           trace
         end
 
-        # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
+        # @return [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>]
         def convert_batch(batch)
           batch.map do |evt|
             case evt
@@ -147,6 +148,10 @@ module Sqreen
               convert_metric_sample(evt)
             when RequestRecord
               convert_req_record(evt)
+            when Sqreen::Kit::Signals::Signal
+              evt
+            when Sqreen::Kit::Signals::Trace
+              evt
             else
               raise NotImplementedError, "Unknown type of event in batch: #{evt}"
             end

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.20.4'.freeze
+  VERSION = '1.21.0.beta1'.freeze
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -4,13 +4,10 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 require 'sqreen/weave/legacy'
-require 'sqreen/weave/budget'
-require 'sqreen/graft/hook'
 require 'sqreen/graft/hook_point'
 require 'sqreen/call_countable'
 require 'sqreen/rules'
 require 'sqreen/rules/record_request_context'
-require 'sqreen/sqreen_signed_verifier'
 
 class Sqreen::Weave::Legacy::Instrumentation
   attr_accessor :metrics_engine
@@ -63,27 +60,6 @@ class Sqreen::Weave::Legacy::Instrumentation
       'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
     )
 
-    metrics_engine.create_metric(
-      'name' => 'req.sq.hook.overhead',
-      'period' => 60,
-      'kind' => 'Binning',
-      'options' => { 'base' => 2.0, 'factor' => 0.1 },
-    )
-
-    metrics_engine.create_metric(
-      'name' => 'sq.hook.overhead',
-      'period' => 60,
-      'kind' => 'Binning',
-      'options' => { 'base' => 2.0, 'factor' => 0.1 },
-    )
-
-    metrics_engine.create_metric(
-      'name' => 'sq.shrinkwrap',
-      'period' => 60,
-      'kind' => 'Binning',
-      'options' => { 'base' => 2.0, 'factor' => 0.1 },
-    )
-
     Sqreen.thread_cpu_time? && metrics_engine.create_metric(
       'name' => 'sq_thread_cpu_pct',
       'period' => opts[:period] || 60,
@@ -108,15 +84,6 @@ class Sqreen::Weave::Legacy::Instrumentation
 
     ### set up rule signature verifier
     verifier = nil
-    if Sqreen.features['rules_signature'] &&
-       Sqreen.config_get(:rules_verify_signature) == true &&
-       !defined?(::JRUBY_VERSION)
-      verifier = Sqreen::SqreenSignedVerifier.new
-      Sqreen::Weave.logger.debug('Rules signature enabled')
-    else
-      Sqreen::Weave.logger.debug('Rules signature disabled')
-    end
-
     ### force clean instrumentation callback list
     @hooks = []
     ### for each rule description
@@ -127,25 +94,6 @@ class Sqreen::Weave::Legacy::Instrumentation
       next unless rule_callback
       ### attach framework to callback
       rule_callback.framework = framework
-      ## create metric
-      Sqreen::Weave.logger.debug { "Adding rule metric: #{rule_callback}" }
-      [:pre, :post, :failing].each do |whence|
-        next unless rule_callback.send(:"#{whence}?")
-        metric_name = "sq.#{rule['name']}.#{whence}"
-        metrics_engine.create_metric(
-          'name' => metric_name,
-          'period' => 60,
-          'kind' => 'Binning',
-          'options' => { 'base' => 2.0, 'factor' => 0.1 },
-        )
-        metric_name = "req.sq.#{rule['name']}.#{whence}"
-        metrics_engine.create_metric(
-          'name' => metric_name,
-          'period' => 60,
-          'kind' => 'Binning',
-          'options' => { 'base' => 2.0, 'factor' => 0.1 },
-        )
-      end
       ### install callback, observing priority
       Sqreen::Weave.logger.debug { "Adding rule callback: #{rule_callback}" }
       @hooks << add_callback("weave,rule=#{rule['name']}", rule_callback, strategy)
@@ -159,43 +107,30 @@ class Sqreen::Weave::Legacy::Instrumentation
     end
 
     metrics_engine = self.metrics_engine
-
     request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
     @hooks << request_hook
     request_hook.add do
       before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
         next unless Sqreen.instrumentation_ready
 
-        # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
-        # shrinkwrap_timer.start
-
-        request_timer = Sqreen::Graft::Timer.new("request")
-        request_timer.start
-        sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
-        budget = Sqreen::Weave::Budget.current
-        request_budget_threshold = budget.threshold if budget
-        request_budget_ratio = budget.ratio if budget
-        request_budget_is_dynamic = !request_budget_ratio.nil?
-        request_budget = !request_budget_threshold.nil?
-        timed_level = (Sqreen.features['perf_level'] || 1).to_i
-        Sqreen::Weave.logger.debug { "request budget: #{budget.to_h} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
-
+        uuid = SecureRandom.uuid
+        now = Sqreen::Graft::Timer.read
         Thread.current[:sqreen_http_request] = {
-          request_timer: request_timer,
-          sqreen_timer: sqreen_timer,
+          uuid: uuid,
+          start_time: now,
+          time_budget: Sqreen.performance_budget,
           time_budget_expended: false,
-          time_budget_threshold: request_budget_threshold,
-          time_budget_dynamic: request_budget_is_dynamic,
-          time_budget_ratio: request_budget_ratio,
-          time_budget: request_budget,
+          timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
           timed_callbacks: [],
           timed_hooks: [],
-          timed_level: timed_level,
+          timed_hooks_before: [],
+          timed_hooks_after: [],
+          timed_hooks_raised: [],
+          timed_hooks_ensured: [],
           skipped_callbacks: [],
-          # timed_shrinkwrap: shrinkwrap_timer,
         }
 
-        # shrinkwrap_timer.stop
+        Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
       end
 
       ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
@@ -203,118 +138,105 @@ class Sqreen::Weave::Legacy::Instrumentation
 
         next if request.nil?
 
-        # shrinkwrap_timer = request[:timed_shrinkwrap]
-        # shrinkwrap_timer.start
-
         Thread.current[:sqreen_http_request] = nil
-        request_timer = request[:request_timer]
-        now = request_timer.stop
-
-        if request[:timed_level] >= 1
-          request[:timed_callbacks].each do |timer|
-            duration = timer.duration
-
-            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            next unless rule
-
-            whence = case timer.tag
-                     when /@before/ then 'pre'
-                     when /@after/  then 'post'
-                     when /@raised/ then 'failing'
-                     end
-            next unless whence
-
-            metric_name = "sq.#{rule}.#{whence}"
-            metrics_engine.update(metric_name, now, nil, duration * 1000)
-            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
-          end
-
-          request[:timed_hooks].each do |timer|
-            duration = timer.duration
-            metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
-            # Sqreen.observations_queue.push(['sq.hook.overhead', nil, duration * 1000, utc_now])
+        now = Sqreen::Graft::Timer.read
+        utc_now = Time.now.utc
+
+        request[:timed_callbacks].each do |timer|
+          duration = timer.duration
+          # stop = now
+          # start = now - duration
+          timer.tag =~ /weave,rule=(.*)$/ && rule = $1
+          timer.tag =~ /@before/ && whence = 'pre'
+          timer.tag =~ /@after/  && whence = 'post'
+          timer.tag =~ /@raised/ && whence = 'failing'
+
+          next unless rule && whence
+
+          # Sqreen::PerformanceNotifications.notify(rule, whence, start, stop)
+          # => BinnedMetrics
+          metric_name = "sq.#{rule}.#{whence}"
+          unless metrics_engine.metric?(metric_name)
+            metrics_engine.create_metric(
+              'name' => metric_name,
+              'period' => 60,
+              'kind' => 'Binning',
+              'options' => { 'base' => 2.0, 'factor' => 0.1 },
+            )
           end
+          metrics_engine.update(metric_name, now, nil, duration * 1000)
         end
 
-        sqreen_timer = request[:sqreen_timer]
-        total = sqreen_timer.duration
-        Sqreen::Weave.logger.debug { "request sqreen_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
-        total = request_timer.duration
-        Sqreen::Weave.logger.debug { "request request_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
-
-        if request[:timed_level] >= 2
-          skipped = request[:skipped_callbacks].map(&:name)
-          Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
-          timings = request[:timed_callbacks].map(&:to_s)
-          total = request[:timed_callbacks].sum(&:duration)
-          Sqreen::Weave.logger.debug { "request callback.total: #{'%.03fus' % (total * 1_000_000)} callback.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
-          timings = request[:timed_hooks].map(&:to_s)
-          total = request[:timed_hooks].sum(&:duration)
-          Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
+        metric_name = 'sq.hooks_pre.pre'
+        duration = request[:timed_hooks_before].sum(&:duration)
+        unless metrics_engine.metric?(metric_name)
+          metrics_engine.create_metric(
+            'name' => metric_name,
+            'period' => 60,
+            'kind' => 'Binning',
+            'options' => { 'base' => 2.0, 'factor' => 0.1 },
+          )
+        end
+        metrics_engine.update(metric_name, now, nil, duration * 1000)
+
+        metric_name = 'sq.hooks_post.post'
+        duration = request[:timed_hooks_after].sum(&:duration)
+        unless metrics_engine.metric?(metric_name)
+          metrics_engine.create_metric(
+            'name' => metric_name,
+            'period' => 60,
+            'kind' => 'Binning',
+            'options' => { 'base' => 2.0, 'factor' => 0.1 },
+          )
         end
+        metrics_engine.update(metric_name, now, nil, duration * 1000)
+
+        metric_name = 'sq.hooks_failing.failing'
+        duration = request[:timed_hooks_raised].sum(&:duration)
+        unless metrics_engine.metric?(metric_name)
+          metrics_engine.create_metric(
+            'name' => metric_name,
+            'period' => 60,
+            'kind' => 'Binning',
+            'options' => { 'base' => 2.0, 'factor' => 0.1 },
+          )
+        end
+        metrics_engine.update(metric_name, now, nil, duration * 1000)
+
+        skipped = request[:skipped_callbacks].map(&:name)
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
+        timer = request[:timer]
+        total = timer.duration
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
+        timings = request[:timed_callbacks].map(&:to_s)
+        total = request[:timed_callbacks].sum(&:duration)
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
+        timings = request[:timed_hooks].map(&:to_s)
+        total = request[:timed_hooks].sum(&:duration)
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
 
         skipped = request[:skipped_callbacks].map(&:name)
         skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
-        metrics_engine.update('request_overtime', now, skipped_rule_name, 1) if skipped_rule_name
-        # Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
+        Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
 
-        sqreen_request_duration = sqreen_timer.duration
-        metrics_engine.update('sq', now, nil, sqreen_request_duration * 1000)
-        # Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
+        sqreen_request_duration = total
+        Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
 
-        request_duration = request_timer.duration
-        metrics_engine.update('req', now, nil, request_duration * 1000)
-        # Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
+        request_duration = now - request[:start_time]
+        Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
 
         sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
-        metrics_engine.update('pct', now, nil, sqreen_request_ratio)
-        # Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
-        Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
-
-        if request[:timed_level] >= 2
-          tallies = Hash.new(0.0)
-          request[:timed_callbacks].each do |timer|
-            duration = timer.duration
-
-            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            next unless rule
-
-            whence = case timer.tag
-                     when /@before/ then 'pre'
-                     when /@after/  then 'post'
-                     when /@raised/ then 'failing'
-                     end
-            next unless whence
-
-            metric_name = "req.sq.#{rule}.#{whence}"
-            tallies[metric_name] += duration
-          end
-          tallies.each do |metric_name, duration|
-            metrics_engine.update(metric_name, now, nil, duration * 1000)
-            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
-          end
-
-          duration = request[:timed_hooks].sum(&:duration)
-          metrics_engine.update('req.sq.hook.overhead', now, nil, duration * 1000)
-          # Sqreen.observations_queue.push(['req.sq.hook.overhead', nil, duration * 1000, utc_now])
-        end
-
-        # shrinkwrap_timer.stop
-
-        # duration = shrinkwrap_timer.duration
-        # metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
+        Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
       end
     end.install
 
     ### globally declare instrumentation ready
     Sqreen.instrumentation_ready = true
-    Sqreen::Weave.logger.info { "Instrumentation activated" }
   end
 
   # needed by Sqreen::Runner
   def remove_all_callbacks
     Sqreen.instrumentation_ready = false
-    Sqreen::Weave.logger.info { "Instrumentation deactivated" }
 
     loop do
       hook = @hooks.pop
@@ -331,15 +253,6 @@ class Sqreen::Weave::Legacy::Instrumentation
     klass = callback.klass
     method = callback.method
 
-    if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
-      call_count = JSON.parse(call_count)
-      if callback.respond_to?(:rule_name) && call_count.key?(callback.rule_name)
-        count = call_count[callback.rule_name]
-        Sqreen::Weave.logger.debug { "override rule: #{callback.rule_name} call_count: #{count.inspect}" }
-        callback.instance_eval { @call_count_interval = call_count[callback.rule_name] }
-      end
-    end
-
     if Sqreen::Graft::HookPoint.new("#{klass}.#{method}").exist?
       hook_point = "#{klass}.#{method}"
     elsif Sqreen::Graft::HookPoint.new("#{klass}##{method}").exist?
@@ -362,6 +275,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
           begin
             ret = callback.pre(i, a, r)
           rescue StandardError => e
@@ -372,26 +286,17 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
 
-          next if ret.nil? || !ret.is_a?(Hash)
-
-          throw_val =
-            case ret[:status]
-            when :skip, 'skip'
-              b.return(ret[:new_return_value]).break! if ret.key?(:new_return_value)
-            when :modify_args, 'modify_args'
-              b.args(ret[:args])
-            when :raise, 'raise'
-              if ret.key?(:exception)
-                b.raise(ret[:exception])
-              else
-                b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
-              end
-            end
-
-          next unless throw_val
-          throw_val.break! if ret[:skip_rem_cbs]
-          throw(b, throw_val)
+          case ret[:status]
+          when :skip, 'skip'
+            throw(b, b.return(ret[:new_return_value]).break!) if ret.key?(:new_return_value)
+          when :modify_args, 'modify_args'
+            throw(b, b.args(ret[:args]))
+          when :raise, 'raise'
+            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
+            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -404,6 +309,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
           begin
             ret = callback.post(v, i, a, r)
           rescue StandardError => e
@@ -414,6 +320,7 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
 
           case ret[:status]
           when :override, 'override'
@@ -434,6 +341,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
           begin
             ret = callback.failing(e, i, a, r)
           rescue StandardError => e
@@ -444,6 +352,7 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
 
           throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)