sqreen 1.20.4-java → 1.21.0.beta3-java

data/lib/sqreen/weave/budget.rb

@@ -9,24 +9,14 @@ require 'sqreen/weave'
 class Sqreen::Weave::Budget
   include Sqreen::Log::Loggable
 
-  def initialize(threshold, ratio = nil)
+  def initialize(threshold)
     @threshold = threshold
-    @ratio = ratio
-  end
-
-  def static?
-    threshold && !ratio
-  end
-
-  def dynamic?
-    threshold && ratio
   end
 
   attr_reader :threshold
-  attr_reader :ratio
 
   def to_h
-    { threshold: threshold, ratio: ratio }
+    { threshold: threshold }
   end
 
   class << self
@@ -38,9 +28,8 @@ class Sqreen::Weave::Budget
       return @current = nil if opts.nil? || opts.empty?
 
       threshold = opts[:threshold]
-      ratio = opts[:ratio]
 
-      @current = new(threshold, ratio)
+      @current = threshold
     end
   end
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -11,19 +11,34 @@ require 'sqreen/call_countable'
 require 'sqreen/rules'
 require 'sqreen/rules/record_request_context'
 require 'sqreen/sqreen_signed_verifier'
+require 'rack/request'
+begin
+  require 'sq_detailed_metrics'
+rescue LoadError => _e # rubocop:disable Lint/HandleExceptions
+end
 
 class Sqreen::Weave::Legacy::Instrumentation
   attr_accessor :metrics_engine
 
+  HAS_SQ_DETAILED_METRICS = defined?(::SqDetailedMetrics)
+  REQ_LVL_2_METRIC = 'request_level_perf'.freeze
+
   def initialize(metrics_engine, opts = {})
     Sqreen::Weave.logger.debug { "#{self.class.name}#initialize #{metrics_engine}" }
     @hooks = []
 
+    unless HAS_SQ_DETAILED_METRICS
+      Sqreen::Weave.logger.warn { "Detailed metrics are unavailable" }
+    end
+
     self.metrics_engine = metrics_engine
 
     ### bail out if no metric engine
     return if metrics_engine.nil?
 
+    # XXX: these metric definitions do not support change of opts
+    #      due to features updates!
+
     ### init metric to count calls to sqreen
     metrics_engine.create_metric(
       'name' => 'sqreen_call_counts',
@@ -90,6 +105,15 @@ class Sqreen::Weave::Legacy::Instrumentation
       'kind' => 'Binning',
       'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
     )
+
+    if HAS_SQ_DETAILED_METRICS # rubocop:disable Style/GuardClause
+      @lvl_2_metric = metrics_engine.create_metric(
+        'name' => REQ_LVL_2_METRIC,
+        'period' => opts[:perf_req_metrics_period] || 60,
+        'kind' => 'ReqDetailed',
+      )
+      @lvl_2_max_reqs = opts[:perf_req_metrics_max_reqs] || 100
+    end
   end
 
   # needed by Sqreen::Runner#initialize
@@ -159,11 +183,13 @@ class Sqreen::Weave::Legacy::Instrumentation
     end
 
     metrics_engine = self.metrics_engine
+    lvl_2_metric = @lvl_2_metric
+    lvl_2_max_reqs = @lvl_2_max_reqs
 
     request_hook = Sqreen::Graft::Hook['Sqreen::ShrinkWrap#call', strategy]
     @hooks << request_hook
     request_hook.add do
-      before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
+      before('wave,meta,request', rank: -100000, mandatory: true) do |call|
         next unless Sqreen.instrumentation_ready
 
         # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
@@ -173,25 +199,42 @@ class Sqreen::Weave::Legacy::Instrumentation
         request_timer.start
         sqreen_timer = Sqreen::Graft::Timer.new("sqreen")
         budget = Sqreen::Weave::Budget.current
-        request_budget_threshold = budget.threshold if budget
-        request_budget_ratio = budget.ratio if budget
-        request_budget_is_dynamic = !request_budget_ratio.nil?
-        request_budget = !request_budget_threshold.nil?
+
         timed_level = (Sqreen.features['perf_level'] || 1).to_i
-        Sqreen::Weave.logger.debug { "request budget: #{budget.to_h} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
+        timed_level = 1 if !HAS_SQ_DETAILED_METRICS && timed_level == 2
+        if timed_level == 2 && lvl_2_metric.num_requests >= lvl_2_max_reqs
+          timed_level = 1
+          Sqreen::Weave.logger.debug { "Reducing timed level to 1 (#{lvl_2_metric.num_requests} reqs accumulated)" }
+        end
+
+        Sqreen::Weave.logger.debug { "request budget: #{budget} timed.level: #{timed_level}" } if Sqreen::Weave.logger.debug?
+
+        route_found = nil
+        if timed_level >= 2
+          rack_env, = call.args
+          rack_request = Rack::Request.new(rack_env) if rack_env
+
+          # TODO: Rails engines
+          # TODO: Struct
+          # TODO: Sinatra
+          # TODO: Rack?
+          Rails.application.routes.router.recognize(rack_request) do |route, params|
+            route = ActionDispatch::Routing::RouteWrapper.new(route)
+            route_found = { name: route.name, verb: route.verb, path: route.path, reqs: route.reqs, params: params }
+          end if defined?(Rails) && Rails.application && defined?(ActionDispatch::Routing::RouteWrapper)
+        end
 
+        # TODO: Struct
         Thread.current[:sqreen_http_request] = {
           request_timer: request_timer,
           sqreen_timer: sqreen_timer,
           time_budget_expended: false,
-          time_budget_threshold: request_budget_threshold,
-          time_budget_dynamic: request_budget_is_dynamic,
-          time_budget_ratio: request_budget_ratio,
-          time_budget: request_budget,
+          time_budget: budget,
           timed_callbacks: [],
           timed_hooks: [],
           timed_level: timed_level,
           skipped_callbacks: [],
+          route: ("#{route_found[:verb]} #{route_found[:path]}" if route_found),
           # timed_shrinkwrap: shrinkwrap_timer,
         }
 
@@ -203,6 +246,9 @@ class Sqreen::Weave::Legacy::Instrumentation
 
         next if request.nil?
 
+        timed_level = request[:timed_level]
+        req_detailed = SqDetailedMetrics::Request.new if timed_level >= 2
+
         # shrinkwrap_timer = request[:timed_shrinkwrap]
         # shrinkwrap_timer.start
 
@@ -210,39 +256,30 @@ class Sqreen::Weave::Legacy::Instrumentation
         request_timer = request[:request_timer]
         now = request_timer.stop
 
-        if request[:timed_level] >= 1
+        if timed_level >= 1
           request[:timed_callbacks].each do |timer|
-            duration = timer.duration
-
-            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            next unless rule
-
-            whence = case timer.tag
-                     when /@before/ then 'pre'
-                     when /@after/  then 'post'
-                     when /@raised/ then 'failing'
-                     end
-            next unless whence
+            duration_ms = timer.duration * 1000.0
+            # XXX: the timer tag should have this structured data;
+            # it would be better than recomputing this for every measurement
+            metric_name = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(timer.tag)
 
-            metric_name = "sq.#{rule}.#{whence}"
-            metrics_engine.update(metric_name, now, nil, duration * 1000)
-            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
-          end
+            next unless metric_name
 
-          request[:timed_hooks].each do |timer|
-            duration = timer.duration
-            metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
-            # Sqreen.observations_queue.push(['sq.hook.overhead', nil, duration * 1000, utc_now])
+            metrics_engine.update(metric_name, now, nil, duration_ms)
+            duration_ms *= -1.0 if timer.conditions_passed
+            req_detailed.add_measurement metric_name, duration_ms if req_detailed
           end
         end
 
         sqreen_timer = request[:sqreen_timer]
-        total = sqreen_timer.duration
-        Sqreen::Weave.logger.debug { "request sqreen_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
-        total = request_timer.duration
-        Sqreen::Weave.logger.debug { "request request_timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
-
-        if request[:timed_level] >= 2
+        Sqreen::Weave.logger.debug do
+          "request sqreen_timer.total: #{'%.03fus' % (sqreen_timer.duration * 1_000_000)}"
+        end if Sqreen::Weave.logger.debug?
+        Sqreen::Weave.logger.debug do
+          "request request_timer.total: #{'%.03fus' % (request_timer.duration * 1_000_000)}"
+        end if Sqreen::Weave.logger.debug?
+
+        if timed_level >= 1 && Sqreen::Weave.logger.debug?
           skipped = request[:skipped_callbacks].map(&:name)
           Sqreen::Weave.logger.debug { "request callback.skipped.count: #{skipped.count}" } if Sqreen::Weave.logger.debug?
           timings = request[:timed_callbacks].map(&:to_s)
@@ -253,50 +290,27 @@ class Sqreen::Weave::Legacy::Instrumentation
           Sqreen::Weave.logger.debug { "request hook.total: #{'%.03fus' % (total * 1_000_000)} hook.count: #{timings.count}" } if Sqreen::Weave.logger.debug?
         end
 
-        skipped = request[:skipped_callbacks].map(&:name)
-        skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
-        metrics_engine.update('request_overtime', now, skipped_rule_name, 1) if skipped_rule_name
-        # Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
+        overtime_cb = ::Sqreen::Weave::Legacy::Instrumentation.tag_to_metric_name(request[:overtime_cb]) \
+                      if request[:overtime_cb]
+        metrics_engine.update('request_overtime', now, overtime_cb, 1) if overtime_cb
 
-        sqreen_request_duration = sqreen_timer.duration
-        metrics_engine.update('sq', now, nil, sqreen_request_duration * 1000)
-        # Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
+        sqreen_request_duration = sqreen_timer.duration * 1000.0
+        metrics_engine.update('sq', now, nil, sqreen_request_duration)
 
-        request_duration = request_timer.duration
-        metrics_engine.update('req', now, nil, request_duration * 1000)
-        # Sqreen.observations_queue.push(['req', nil, request_duration * 1000, utc_now])
+        request_duration = request_timer.duration * 1000.0
+        metrics_engine.update('req', now, nil, request_duration)
 
         sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
         metrics_engine.update('pct', now, nil, sqreen_request_ratio)
-        # Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
         Sqreen::Weave.logger.debug { "request sqreen_timer.ratio: #{'%.03f' % (sqreen_request_ratio / 100.0)}" } if Sqreen::Weave.logger.debug?
 
-        if request[:timed_level] >= 2
-          tallies = Hash.new(0.0)
-          request[:timed_callbacks].each do |timer|
-            duration = timer.duration
-
-            timer.tag =~ /weave,rule=(.*)$/ && rule = $1
-            next unless rule
-
-            whence = case timer.tag
-                     when /@before/ then 'pre'
-                     when /@after/  then 'post'
-                     when /@raised/ then 'failing'
-                     end
-            next unless whence
-
-            metric_name = "req.sq.#{rule}.#{whence}"
-            tallies[metric_name] += duration
-          end
-          tallies.each do |metric_name, duration|
-            metrics_engine.update(metric_name, now, nil, duration * 1000)
-            # Sqreen.observations_queue.push([metric_name, nil, duration * 1000, utc_now])
-          end
+        if req_detailed
+          req_detailed.route = request[:route]
+          req_detailed.overtime_cb = overtime_cb if overtime_cb
+          req_detailed.add_measurement 'sq', sqreen_request_duration
+          req_detailed.add_measurement 'req', request_duration
 
-          duration = request[:timed_hooks].sum(&:duration)
-          metrics_engine.update('req.sq.hook.overhead', now, nil, duration * 1000)
-          # Sqreen.observations_queue.push(['req.sq.hook.overhead', nil, duration * 1000, utc_now])
+          metrics_engine.update(REQ_LVL_2_METRIC, now, nil, req_detailed)
         end
 
         # shrinkwrap_timer.stop
@@ -355,7 +369,8 @@ class Sqreen::Weave::Legacy::Instrumentation
     hook = Sqreen::Graft::Hook[hook_point, strategy]
     hook.add do
       if callback.pre?
-        before(rule, rank: priority, mandatory: !callback.overtimeable, flow: block, ignore: ignore) do |call, b|
+        use_flow = block || callback.is_a?(::Sqreen::Conditionable)
+        before(rule, rank: priority, mandatory: !callback.overtimeable, flow: use_flow, ignore: ignore) do |call, b|
           next unless Thread.current[:sqreen_http_request]
 
           i = call.instance
@@ -387,8 +402,12 @@ class Sqreen::Weave::Legacy::Instrumentation
               else
                 b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
               end
-            end
+            end if block
 
+          if ret && ret[:passed_conditions]
+            throw_val ||= b.noop
+            throw_val.passed_conditions!
+          end
           next unless throw_val
           throw_val.break! if ret[:skip_rem_cbs]
           throw(b, throw_val)
@@ -415,13 +434,21 @@ class Sqreen::Weave::Legacy::Instrumentation
             end
           end
 
-          case ret[:status]
-          when :override, 'override'
-            throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
-          when :raise, 'raise'
-            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
-            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil? || !ret.is_a?(Hash)
+          throw_val =
+            case ret[:status]
+            when :override, 'override'
+              b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
+            when :raise, 'raise'
+              b.raise(ret[:exception]) if ret.key?(:exception)
+              b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
+            end unless ret.nil? || !ret.is_a?(Hash) || !block
+
+          if ret && ret[:passed_conditions]
+            throw_val ||= b.noop
+            throw_val.passed_conditions!
+          end
+          next unless throw_val
+          throw(b, throw_val)
         end
       end
 
@@ -447,19 +474,27 @@ class Sqreen::Weave::Legacy::Instrumentation
 
           throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
 
-          case ret[:status]
-          when :override, 'override'
-            throw(b, b.return(ret[:new_return_value])) if ret.key?(:new_return_value)
-          when :retry, 'retry'
-            throw(b, b.retry)
-          when :raise, 'raise'
-            throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
-            throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          when :reraise, 'reraise'
-            throw(b, b.raise(e))
-          else
-            throw(b, b.raise(e))
-          end unless ret.nil? || !ret.is_a?(Hash)
+          throw_val =
+            case ret[:status]
+            when :override, 'override'
+              b.return(ret[:new_return_value]) if ret.key?(:new_return_value)
+            when :retry, 'retry'
+              b.retry
+            when :raise, 'raise'
+              b.raise(ret[:exception]) if ret.key?(:exception)
+              b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required."))
+            when :reraise, 'reraise'
+              b.raise(e)
+            else
+              b.raise(e)
+            end unless ret.nil? || !ret.is_a?(Hash) || !block
+
+          if ret && ret[:passed_conditions]
+            throw_val ||= b.noop
+            throw_val.passed_conditions!
+          end
+          next unless throw_val
+          throw(b, throw_val)
         end
       end
     end.install
@@ -494,4 +529,20 @@ class Sqreen::Weave::Legacy::Instrumentation
       Sqreen::Rules::RunUserActions.new(Sqreen, :auth_track, 1),
     ]
   end
+
+  def self.tag_to_metric_name(tag)
+    cached = @cache_tag_to_metric[tag]
+    return cached unless cached.nil?
+
+    tag =~ /weave,rule=(.*)$/ && rule = $1 and # rubocop:disable Style/AndOr
+      (tag =~ /@before/ && whence = 'pre' or # rubocop:disable Style/AndOr
+        tag =~ /@after/  && whence = 'post' or # rubocop:disable Style/AndOr
+        tag =~ /@raised/ && whence = 'failing' or # rubocop:disable Style/AndOr
+        tag =~ /@ensured/ && whence = 'finally')
+
+    @cache_tag_to_metric[tag] =
+      rule && whence ? "sq.#{rule}.#{whence}" : false
+  end
+
+  @cache_tag_to_metric = {}
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.20.4
+  version: 1.21.0.beta3
 platform: java
 authors:
 - Sqreen
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-16 00:00:00.000000000 Z
+date: 2020-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -123,6 +123,39 @@ files:
 - lib/sqreen/dependency/sentry.rb
 - lib/sqreen/dependency/sinatra.rb
 - lib/sqreen/deprecation.rb
+- lib/sqreen/ecosystem.rb
+- lib/sqreen/ecosystem/dispatch_table.rb
+- lib/sqreen/ecosystem/exception_reporting.rb
+- lib/sqreen/ecosystem/http/net_http.rb
+- lib/sqreen/ecosystem/http/rack_request.rb
+- lib/sqreen/ecosystem/loggable.rb
+- lib/sqreen/ecosystem/module_api.rb
+- lib/sqreen/ecosystem/module_api/event_listener.rb
+- lib/sqreen/ecosystem/module_api/instrumentation.rb
+- lib/sqreen/ecosystem/module_api/message_producer.rb
+- lib/sqreen/ecosystem/module_api/signal_producer.rb
+- lib/sqreen/ecosystem/module_api/tracing.rb
+- lib/sqreen/ecosystem/module_api/tracing/client_data.rb
+- lib/sqreen/ecosystem/module_api/tracing/server_data.rb
+- lib/sqreen/ecosystem/module_api/tracing_id_generation.rb
+- lib/sqreen/ecosystem/module_api/transaction_storage.rb
+- lib/sqreen/ecosystem/module_registry.rb
+- lib/sqreen/ecosystem/redis/redis_connection.rb
+- lib/sqreen/ecosystem/tracing/modules/client.rb
+- lib/sqreen/ecosystem/tracing/modules/server.rb
+- lib/sqreen/ecosystem/tracing/sampler.rb
+- lib/sqreen/ecosystem/tracing/sampling_configuration.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_client.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_server.rb
+- lib/sqreen/ecosystem/tracing_broker.rb
+- lib/sqreen/ecosystem/tracing_id_setup.rb
+- lib/sqreen/ecosystem/transaction_storage.rb
+- lib/sqreen/ecosystem/util/call_writers_from_init.rb
+- lib/sqreen/ecosystem_integration.rb
+- lib/sqreen/ecosystem_integration/around_callbacks.rb
+- lib/sqreen/ecosystem_integration/instrumentation_service.rb
+- lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb
+- lib/sqreen/ecosystem_integration/signal_consumption.rb
 - lib/sqreen/endpoint_testing.rb
 - lib/sqreen/error_handling_middleware.rb
 - lib/sqreen/event.rb
@@ -175,6 +208,7 @@ files:
 - lib/sqreen/metrics/base.rb
 - lib/sqreen/metrics/binning.rb
 - lib/sqreen/metrics/collect.rb
+- lib/sqreen/metrics/req_detailed.rb
 - lib/sqreen/metrics/sum.rb
 - lib/sqreen/metrics_store.rb
 - lib/sqreen/metrics_store/already_registered_metric.rb
@@ -278,7 +312,9 @@ metadata:
   changelog_uri: https://docs.sqreen.com/ruby/release-notes/
   source_code_uri: https://github.com/sqreen/ruby-agent
   bug_tracker_uri: https://github.com/sqreen/ruby-agent/issues
-post_install_message:
+post_install_message: |2
+      This is a Sqreen beta release and may not work in all situations.
+      Make sure to review CHANGELOG.md for important details.
 rdoc_options: []
 require_paths:
 - lib
@@ -289,9 +325,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.6.14.1