sqreen 1.21.0.beta3-java → 1.21.0-java

data/lib/sqreen/kit/signals/specialized/sqreen_exception.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/kit/configuration'
 require 'sqreen/kit/signals/point'
 require 'sqreen/kit/signals/dto_helper'
 
@@ -38,6 +39,7 @@ class Sqreen::Kit::Signals::Specialized::SqreenException < Sqreen::Kit::Signals:
     self.payload_schema = PAYLOAD_SCHEMA_VERSION
     self.signal_name = 'sq.agent.exception'
     self.time = values[:time] || Time.now
+    self.source = values[:source] || Sqreen::Kit::Configuration.default_source
     super
   end
 

data/lib/sqreen/metrics.rb

@@ -7,4 +7,3 @@ require 'sqreen/metrics/collect'
 require 'sqreen/metrics/average'
 require 'sqreen/metrics/sum'
 require 'sqreen/metrics/binning'
-require 'sqreen/metrics/req_detailed'

data/lib/sqreen/rules/custom_error_cb.rb

@@ -55,7 +55,7 @@ module Sqreen
       end
 
       def respond_page
-        @page ||= File.read(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        @page ||= File.open(File.join(File.dirname(__FILE__), '../attack_detected.html'), 'rb', &:read)
         headers = {
           'Content-Type' => 'text/html',
           'Content-Length' => @page.size.to_s,

data/lib/sqreen/rules/rule_cb.rb

@@ -90,9 +90,9 @@ module Sqreen
         framework.observe(:sqreen_exceptions, payload)
       end
 
-      # Recommend taking an action (optionally adding more data/context)
+      # Recommend taking an action (optionnally adding more data/context)
       #
-      # This will format the requested action and optionally
+      # This will format the requested action and optionnally
       # override it if it should not be taken (should not block for example)
       def advise_action(action, additional_data = {})
         return if action.nil? && additional_data.empty?

data/lib/sqreen/runner.rb

@@ -141,12 +141,7 @@ module Sqreen
       end
 
       if @configuration.get(:weave) || needs_weave.call
-        # XXX: don't get updated
-        opts = {
-          perf_req_metrics_max_reqs: Sqreen.features['perf_req_metrics_max_reqs'],
-          perf_req_metrics_period: Sqreen.features['perf_req_metrics_period'],
-        }
-        @instrumenter = Sqreen::Weave::Legacy::Instrumentation.new(metrics_engine, opts)
+        @instrumenter = Sqreen::Weave::Legacy::Instrumentation.new(metrics_engine)
       else
         @instrumenter = Sqreen::Legacy::Instrumentation.new(metrics_engine)
       end
@@ -172,9 +167,7 @@ module Sqreen
       end
       self.features = wanted_features
 
-      @ecosystem_integration = EcosystemIntegration.new(framework,
-                                                        Sqreen.queue,
-                                                        create_binning_metric_proc)
+      @ecosystem_integration = EcosystemIntegration.new(framework, Sqreen.queue)
       framework.req_start_cb = @ecosystem_integration.method(:request_start)
       framework.req_end_cb = @ecosystem_integration.method(:request_end)
 
@@ -281,7 +274,7 @@ module Sqreen
 
       # XXX: ecosystem instrumentation should likely be deferred
       #      the same way the rest might be
-      @ecosystem_integration.init unless Sqreen.features['disable_ecosystem']
+      @ecosystem_integration.init
       rulespack_id.to_s
     end
 
@@ -406,8 +399,15 @@ module Sqreen
         prev = Sqreen::Weave::Budget.current
         prev = prev.to_h if prev
 
-        budget_s = budget.to_f / 1000.0 if budget
-        Sqreen::Weave::Budget.update(threshold: budget_s)
+        budget_s = budget.to_f / 1000 if budget
+
+        feature = features['performance_budget']
+        if feature
+          budget_s = feature['threshold'] if feature.key?('threshold')
+          ratio = feature['ratio'] if feature.key?('ratio')
+        end
+
+        Sqreen::Weave::Budget.update(threshold: budget_s, ratio: ratio)
       else
         prev = Sqreen.performance_budget
         Sqreen.update_performance_budget(budget)
@@ -555,21 +555,6 @@ module Sqreen
 
     private
 
-    def create_binning_metric_proc
-      lambda do |metric_name|
-        return if @metrics_engine.metric?(metric_name)
-        metrics_engine.create_metric(
-          'name' => metric_name,
-          'kind' => 'Binning',
-          'period' => Sqreen.features['performance_metrics_period'] || 60,
-          'options' => {
-            'base' => Sqreen.features['perf_base'] || PerformanceNotifications::BinnedMetrics::DEFAULT_PERF_BASE,
-            'factor' => Sqreen.features['perf_unit'] || PerformanceNotifications::BinnedMetrics::DEFAULT_PERF_UNIT,
-          },
-        )
-      end
-    end
-
     def post_endpoint_testing_msgs(chosen_endpoints)
       chosen_endpoints.messages.each do |msg|
         session.post_agent_message(@framework, msg)

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.21.0.beta3'.freeze
+  VERSION = '1.21.0'.freeze
 end

data/lib/sqreen/weave/budget.rb

@@ -9,14 +9,24 @@ require 'sqreen/weave'
 class Sqreen::Weave::Budget
   include Sqreen::Log::Loggable
 
-  def initialize(threshold)
+  def initialize(threshold, ratio = nil)
     @threshold = threshold
+    @ratio = ratio
+  end
+
+  def static?
+    threshold && !ratio
+  end
+
+  def dynamic?
+    threshold && ratio
   end
 
   attr_reader :threshold
+  attr_reader :ratio
 
   def to_h
-    { threshold: threshold }
+    { threshold: threshold, ratio: ratio }
   end
 
   class << self
@@ -28,8 +38,9 @@ class Sqreen::Weave::Budget
       return @current = nil if opts.nil? || opts.empty?
 
       threshold = opts[:threshold]
+      ratio = opts[:ratio]
 
-      @current = threshold
+      @current = new(threshold, ratio)
     end
   end
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -11,34 +11,19 @@ require 'sqreen/call_countable'
 require 'sqreen/rules'
 require 'sqreen/rules/record_request_context'
 require 'sqreen/sqreen_signed_verifier'
-require 'rack/request'
-begin
-  require 'sq_detailed_metrics'
-rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
-end
 
 class Sqreen::Weave::Legacy::Instrumentation
   attr_accessor :metrics_engine
 
-  HAS_SQ_DETAILED_METRICS = defined?(::SqDetailedMetrics)
-  REQ_LVL_2_METRIC = 'request_level_perf'.freeze
-
   def initialize(metrics_engine, opts = {})
     Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
     @hooks = []
 
-    unless HAS_SQ_DETAILED_METRICS
-      Sqreen::Weave.logger.warn { "Detailed metrics are unavailable" }
-    end
-
     self.metrics_engine = metrics_engine
 
     ### bail out if no metric engine
     return if metrics_engine.nil?
 
-    # XXX: these metric definitions do not support change of opts
-    #      due to features updates!
-
     ### init metric to count calls to sqreen
     metrics_engine.create_metric(
       'name' => 'sqreen_call_counts',
@@ -105,15 +90,6 @@ class Sqreen::Weave::Legacy::Instrumentation
       'kind' => 'Binning',
       'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
     )
-
-    if HAS_SQ_DETAILED_METRICS # rubocop:disable Style/GuardClause
-      @lvl_2_metric = metrics_engine.create_metric(
-        'name' => REQ_LVL_2_METRIC,
-        'period' => opts[:perf_req_metrics_period] || 60,
-        'kind' => 'ReqDetailed',
-      )
-      @lvl_2_max_reqs = opts[:perf_req_metrics_max_reqs] || 100
-    end
   end
 
   # needed by Sqreen::Runner#initialize
@@ -183,13 +159,11 @@ class Sqreen::Weave::Legacy::Instrumentation
     end
 
     metrics_engine = self.metrics_engine
-    lvl_2_metric = @lvl_2_metric
-    lvl_2_max_reqs = @lvl_2_max_reqs
 
     request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
     @hooks << request_hook
     request_hook.add do
-      before('wave,meta,request', rank: -100000, mandatory: true) do |call|
+      before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
         next unless Sqreen.instrumentation_ready
 
         # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
@@ -199,42 +173,25 @@ class Sqreen::Weave::Legacy::Instrumentation
         request_timer.start
         sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
         budget = Sqreen::Weave::Budget.current
-
+        request_budget_threshold = budget.threshold if budget
+        request_budget_ratio = budget.ratio if budget
+        request_budget_is_dynamic = !request_budget_ratio.nil?
+        request_budget = !request_budget_threshold.nil?
         timed_level = (Sqreen.features['perf_level'] || 1).to_i
-        timed_level = 1 if !HAS_SQ_DETAILED_METRICS && timed_level == 2
-        if timed_level == 2 && lvl_2_metric.num_requests >= lvl_2_max_reqs
-          timed_level = 1
-          Sqreen::Weave.logger.debug { "Reducing timed level to 1 (#{lvl_2_metric.num_requests} reqs accumulated)" }
-        end
-
-        Sqreen::Weave.logger.debug { "request budget: #{budget} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
-
-        route_found = nil
-        if timed_level >= 2
-          rack_env, = call.args
-          rack_request = Rack::Request.new(rack_env) if rack_env
-
-          # TODO: Rails engines
-          # TODO: Struct
-          # TODO: Sinatra
-          # TODO: Rack?
-          Rails.application.routes.router.recognize(rack_request) do |route, params|
-            route = ActionDispatch::Routing::RouteWrapper.new(route)
-            route_found = { name: route.name, verb: route.verb, path: route.path, reqs: route.reqs, params: params }
-          end if defined?(Rails) && Rails.application && defined?(ActionDispatch::Routing::RouteWrapper)
-        end
+        Sqreen::Weave.logger.debug { "request budget: #{budget.to_h} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
 
-        # TODO: Struct
         Thread.current[:sqreen_http_request] = {
           request_timer: request_timer,
           sqreen_timer: sqreen_timer,
           time_budget_expended: false,
-          time_budget: budget,
+          time_budget_threshold: request_budget_threshold,
+          time_budget_dynamic: request_budget_is_dynamic,
+          time_budget_ratio: request_budget_ratio,
+          time_budget: request_budget,
           timed_callbacks: [],
           timed_hooks: [],
           timed_level: timed_level,
           skipped_callbacks: [],
-          route: ("#{route_found[:verb]} #{route_found[:path]}" if route_found),
           # timed_shrinkwrap: shrinkwrap_timer,
         }
 
@@ -246,9 +203,6 @@ class Sqreen::Weave::Legacy::Instrumentation
 
         next if request.nil?
 
-        timed_level = request[:timed_level]
-        req_detailed = SqDetailedMetrics::Request.new if timed_level >= 2
-
         # shrinkwrap_timer = request[:timed_shrinkwrap]
         # shrinkwrap_timer.start
 
@@ -256,30 +210,39 @@ class Sqreen::Weave::Legacy::Instrumentation
         request_timer = request[:request_timer]
         now = request_timer.stop
 
-        if timed_level >= 1
+        if request[:timed_level] >= 1
           request[:timed_callbacks].each do |timer|
-            duration_ms = timer.duration * 1000.0
-            # XXX: the timer tag should have this structured data;
-            # it would be better than recomputing this for every measurement
-            metric_name = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(timer.tag)
+            duration = timer.duration
+
+            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
+            next unless rule
+
+            whence = case timer.tag
+                     when /@before/ then 'pre'
+                     when /@after/  then 'post'
+                     when /@raised/ then 'failing'
+                     end
+            next unless whence
 
-            next unless metric_name
+            metric_name = "sq.#{rule}.#{whence}"
+            metrics_engine.update(metric_name, now, nil, duration * 1000)
+            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
+          end
 
-            metrics_engine.update(metric_name, now, nil, duration_ms)
-            duration_ms *= -1.0 if timer.conditions_passed
-            req_detailed.add_measurement metric_name, duration_ms if req_detailed
+          request[:timed_hooks].each do |timer|
+            duration = timer.duration
+            metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
+            # Sqreen.observations_queue.push(['sq.hook.overhead', nil, duration * 1000, utc_now])
           end
         end
 
         sqreen_timer = request[:sqreen_timer]
-        Sqreen::Weave.logger.debug do
-          "request sqreen_timer.total: #{'%.03fus' % (sqreen_timer.duration * 1_000_000)}"
-        end if Sqreen::Weave.logger.debug?
-        Sqreen::Weave.logger.debug do
-          "request request_timer.total: #{'%.03fus' % (request_timer.duration * 1_000_000)}"
-        end if Sqreen::Weave.logger.debug?
-
-        if timed_level >= 1 && Sqreen::Weave.logger.debug?
+        total = sqreen_timer.duration
+        Sqreen::Weave.logger.debug { "request sqreen_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
+        total = request_timer.duration
+        Sqreen::Weave.logger.debug { "request request_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
+
+        if request[:timed_level] >= 2
           skipped = request[:skipped_callbacks].map(&:name)
           Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
           timings = request[:timed_callbacks].map(&:to_s)
@@ -290,27 +253,50 @@ class Sqreen::Weave::Legacy::Instrumentation
           Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
         end
 
-        overtime_cb = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(request[:overtime_cb]) \
-                      if request[:overtime_cb]
-        metrics_engine.update('request_overtime', now, overtime_cb, 1) if overtime_cb
+        skipped = request[:skipped_callbacks].map(&:name)
+        skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
+        metrics_engine.update('request_overtime', now, skipped_rule_name, 1) if skipped_rule_name
+        # Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
 
-        sqreen_request_duration = sqreen_timer.duration * 1000.0
-        metrics_engine.update('sq', now, nil, sqreen_request_duration)
+        sqreen_request_duration = sqreen_timer.duration
+        metrics_engine.update('sq', now, nil, sqreen_request_duration * 1000)
+        # Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
 
-        request_duration = request_timer.duration * 1000.0
-        metrics_engine.update('req', now, nil, request_duration)
+        request_duration = request_timer.duration
+        metrics_engine.update('req', now, nil, request_duration * 1000)
+        # Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
 
         sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
         metrics_engine.update('pct', now, nil, sqreen_request_ratio)
+        # Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
         Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
 
-        if req_detailed
-          req_detailed.route = request[:route]
-          req_detailed.overtime_cb = overtime_cb if overtime_cb
-          req_detailed.add_measurement 'sq', sqreen_request_duration
-          req_detailed.add_measurement 'req', request_duration
+        if request[:timed_level] >= 2
+          tallies = Hash.new(0.0)
+          request[:timed_callbacks].each do |timer|
+            duration = timer.duration
+
+            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
+            next unless rule
+
+            whence = case timer.tag
+                     when /@before/ then 'pre'
+                     when /@after/  then 'post'
+                     when /@raised/ then 'failing'
+                     end
+            next unless whence
+
+            metric_name = "req.sq.#{rule}.#{whence}"
+            tallies[metric_name] += duration
+          end
+          tallies.each do |metric_name, duration|
+            metrics_engine.update(metric_name, now, nil, duration * 1000)
+            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
+          end
 
-          metrics_engine.update(REQ_LVL_2_METRIC, now, nil, req_detailed)
+          duration = request[:timed_hooks].sum(&:duration)
+          metrics_engine.update('req.sq.hook.overhead', now, nil, duration * 1000)
+          # Sqreen.observations_queue.push(['req.sq.hook.overhead', nil, duration * 1000, utc_now])
         end
 
         # shrinkwrap_timer.stop
@@ -369,8 +355,7 @@ class Sqreen::Weave::Legacy::Instrumentation
     hook = Sqreen::Graft::Hook[hook_point, strategy]
     hook.add do
       if callback.pre?
-        use_flow = block || callback.is_a?(::Sqreen::Conditionable)
-        before(rule, rank: priority, mandatory: !callback.overtimeable, flow: use_flow, ignore: ignore) do |call, b|
+        before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
           next unless Thread.current[:sqreen_http_request]
 
           i = call.instance
@@ -402,12 +387,8 @@ class Sqreen::Weave::Legacy::Instrumentation
               else
                 b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
               end
-            end if block
+            end
 
-          if ret && ret[:passed_conditions]
-            throw_val ||= b.noop
-            throw_val.passed_conditions!
-          end
           next unless throw_val
           throw_val.break! if ret[:skip_rem_cbs]
           throw(b, throw_val)
@@ -434,21 +415,13 @@ class Sqreen::Weave::Legacy::Instrumentation
             end
           end
 
-          throw_val =
-            case ret[:status]
-            when :override, 'override'
-              b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
-            when :raise, 'raise'
-              b.raise(ret[:exception]) if ret.key?(:exception)
-              b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
-            end unless ret.nil? || !ret.is_a?(Hash) || !block
-
-          if ret && ret[:passed_conditions]
-            throw_val ||= b.noop
-            throw_val.passed_conditions!
-          end
-          next unless throw_val
-          throw(b, throw_val)
+          case ret[:status]
+          when :override, 'override'
+            throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
+          when :raise, 'raise'
+            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
+            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -474,27 +447,19 @@ class Sqreen::Weave::Legacy::Instrumentation
 
           throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
 
-          throw_val =
-            case ret[:status]
-            when :override, 'override'
-              b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
-            when :retry, 'retry'
-              b.retry
-            when :raise, 'raise'
-              b.raise(ret[:exception]) if ret.key?(:exception)
-              b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
-            when :reraise, 'reraise'
-              b.raise(e)
-            else
-              b.raise(e)
-            end unless ret.nil? || !ret.is_a?(Hash) || !block
-
-          if ret && ret[:passed_conditions]
-            throw_val ||= b.noop
-            throw_val.passed_conditions!
-          end
-          next unless throw_val
-          throw(b, throw_val)
+          case ret[:status]
+          when :override, 'override'
+            throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
+          when :retry, 'retry'
+            throw(b, b.retry)
+          when :raise, 'raise'
+            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
+            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
+          when :reraise, 'reraise'
+            throw(b, b.raise(e))
+          else
+            throw(b, b.raise(e))
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
     end.install
@@ -529,20 +494,4 @@ class Sqreen::Weave::Legacy::Instrumentation
       Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
     ]
   end
-
-  def self.tag_to_metric_name(tag)
-    cached = @cache_tag_to_metric[tag]
-    return cached unless cached.nil?
-
-    tag =~ /weave,rule=(.*)$/ && rule = $1 and # rubocop:disable Style/AndOr
-      (tag =~ /@before/ && whence = 'pre' or # rubocop:disable Style/AndOr
-        tag =~ /@after/  && whence = 'post' or # rubocop:disable Style/AndOr
-        tag =~ /@raised/ && whence = 'failing' or # rubocop:disable Style/AndOr
-        tag =~ /@ensured/ && whence = 'finally')
-
-    @cache_tag_to_metric[tag] =
-      rule && whence ? "sq.#{rule}.#{whence}" : false
-  end
-
-  @cache_tag_to_metric = {}
 end