sqreen 1.19.0.beta1 → 1.20.0

data/lib/sqreen/legacy/waf_redactions.rb

@@ -0,0 +1,49 @@
+# typed: ignore
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+module Sqreen
+  module Legacy
+    module WafRedactions
+      class << self
+        def redact_attacks!(attacks, values)
+          return attacks if values.empty?
+
+          values = values.map { |v| v.downcase if v.is_a?(String) }
+
+          attacks.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf_data]
+
+            parsed = JSON.parse(e[:infos][:waf_data])
+            redacted = parsed.each do |w|
+              next unless (filters = w['filter'])
+
+              filters.each do |f|
+                next unless (v = f['resolved_value'])
+                next unless values.include?(v.downcase)
+
+                f['match_status'] = SensitiveDataRedactor::MASK
+                f['resolved_value'] = SensitiveDataRedactor::MASK
+              end
+            end
+            e[:infos][:waf_data] = JSON.dump(redacted)
+          end
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_exceptions!(exceptions, values)
+          return exceptions if values.empty?
+
+          exceptions.each do |e|
+            next(e) unless e[:infos]
+            next(e) unless e[:infos][:waf]
+
+            e[:infos][:waf].delete(:args)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/metrics/base.rb

@@ -12,6 +12,9 @@ module Sqreen
     FINISH_KEY = 'finish'.freeze
     # Base interface for a metric
     class Base
+      attr_accessor :name, :period # for signals serialization
+      attr_accessor :rule # optional
+
       def initialize(_opts={})
         @sample = nil
       end

data/lib/sqreen/metrics_store.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/aggregated_metric'
 require 'sqreen/metrics'
 require 'sqreen/mono_time'
 require 'sqreen/metrics_store/unknown_metric'
@@ -30,8 +31,9 @@ module Sqreen
 
     # Definition contains a name,period and aggregate at least
     # @param definition [Hash] a metric definition
+    # @param rule [RuleCB] the rule associated with this metric, if any
     # @param mklass [Object] Override metric object (used in testing)
-    def create_metric(definition, mklass = nil)
+    def create_metric(definition, rule = nil, mklass = nil)
       name = definition[NAME_KEY]
       kind = definition[KIND_KEY]
       klass = valid_metric(kind, name)
@@ -43,6 +45,9 @@ module Sqreen
         definition[PERIOD_KEY],
         nil # Start
       ]
+      metric.name = name
+      metric.rule = rule
+      metric.period = definition[PERIOD_KEY]
       metric
     end
 
@@ -50,7 +55,7 @@ module Sqreen
       @metrics.key?(name)
     end
 
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def update(name, at, key, value)
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
@@ -59,7 +64,7 @@ module Sqreen
     end
 
     # Drains every metrics and returns the store content
-    # @params at [Time] when is the store emptied
+    # @param at [Time] when is the store emptied
     def publish(flush = true, at = Sqreen.time)
       @metrics.each do |name, (_, period, start)|
         next_sample(name, at) if flush || !start.nil? && (start + period) < at
@@ -75,15 +80,20 @@ module Sqreen
       metric = @metrics[name][0]
       r = metric.next_sample(at)
       @metrics[name][2] = at # new start
-      if r
-        r[NAME_KEY] = name
-        obs = r[Metric::OBSERVATION_KEY]
-        start_of_mono = Time.now.utc - Sqreen.time
-        r[Metric::START_KEY] = start_of_mono + r[Metric::START_KEY]
-        r[Metric::FINISH_KEY] = start_of_mono + r[Metric::FINISH_KEY]
-        @store << r if obs && (!obs.respond_to?(:empty?) || !obs.empty?)
-      end
-      r
+      return unless r
+
+      r[NAME_KEY] = name
+      obs = r[Metric::OBSERVATION_KEY]
+      return unless obs && (!obs.respond_to?(:empty?) || !obs.empty?)
+      start_of_mono = Time.now.utc - Sqreen.time
+
+      agg = AggregatedMetric.new
+      agg.metric = metric
+      agg.rule = agg.metric.rule
+      agg.start = start_of_mono + r[Metric::START_KEY]
+      agg.finish = start_of_mono + r[Metric::FINISH_KEY]
+      agg.data = obs
+      @store << agg
     end
 
     def valid_metric(kind, name)

data/lib/sqreen/performance_notifications/binned_metrics.rb

@@ -122,10 +122,16 @@ module Sqreen
       attr_reader :metrics_store
       attr_reader :period
 
-      def ensure_metric(metric_name)
+      def ensure_metric(metric_name, rule = nil)
         return if metrics_store.metric?(metric_name)
         metrics_store.create_metric(
-          'name' => metric_name, 'period' => period, 'kind' => 'Binning', 'options' => @perf_metric_opts
+          {
+            'name' => metric_name,
+            'period' => period,
+            'kind' => 'Binning',
+            'options' => @perf_metric_opts,
+          },
+          rule
         )
       end
 

data/lib/sqreen/rules.rb

@@ -135,13 +135,15 @@ module Sqreen
         return nil
       end
 
+      rule_cb = cb_class.new(instr_class, instr_method, hash_rule)
+
       if metrics_store
         (hash_rule[Attrs::METRICS] || []).each do |metric|
-          metrics_store.create_metric(metric)
+          metrics_store.create_metric(metric, rule_cb)
         end
       end
 
-      cb_class.new(instr_class, instr_method, hash_rule)
+      rule_cb
     rescue => e
       rule_name = nil
       rulespack_id = nil

data/lib/sqreen/rules/not_found_cb.rb

@@ -24,6 +24,8 @@ module Sqreen
         exception   = env['action_dispatch.exception']
 
         record_from_env(ua, script_name, path_info, verb, override, host, exception)
+
+        nil
       end
 
       def record_from_env(ua, script_name, path_info, verb, override, host, exception)

data/lib/sqreen/rules/rule_cb.rb

@@ -61,7 +61,9 @@ module Sqreen
           :infos => infos,
           :rulespack_id => rulespack_id,
           :rule_name => rule_name,
+          :attack_type => @rule['attack_type'], # for signal
           :test => test,
+          :block => @rule['block'], # for signal
           :time => at,
         }
         if payload_tpl.include?('context')

data/lib/sqreen/rules/waf_cb.rb

@@ -11,11 +11,15 @@ require 'sqreen/safe_json'
 require 'sqreen/exception'
 require 'sqreen/util/capper'
 require 'sqreen/dependency/libsqreen'
+require 'sqreen/encoding_sanitizer'
 
 module Sqreen
   module Rules
     class WAFCB < RuleCB
-      BUDGET_MAX = 5
+      # 2^30 -1 or 2^62 -1
+      MAX_FIXNUM = 1.size == 4 ? 1_073_741_823 : 4_611_686_018_427_387_903
+      # will be converted to a long, so better not to overflow
+      INFINITE_BUDGET_US = MAX_FIXNUM
 
       def self.libsqreen?
         Sqreen::Dependency::LibSqreen.required?
@@ -25,7 +29,7 @@ module Sqreen
         Sqreen::Dependency.const_exist?('LibSqreen::WAF')
       end
 
-      attr_reader :binding_accessors, :budget, :waf_rule_name
+      attr_reader :binding_accessors, :max_run_budget_us, :waf_rule_name
 
       def initialize(*args)
         super(*args)
@@ -54,8 +58,12 @@ module Sqreen
         @binding_accessors = @data['values'].fetch('binding_accessors', []).each_with_object({}) do |e, h|
           h[e] = BindingAccessor.new(e)
         end
-        @budget = (@data['values'].fetch('budget_in_ms', nil) || BUDGET_MAX) * 1000
-        Sqreen.log.debug("WAF budget for #{@waf_rule_name} set to #{@budget}us")
+
+        # 0 for using defaults (PW_RUN_TIMEOUT)
+        @max_run_budget_us = (@data['values'].fetch('budget_in_ms', 0) * 1000).to_i
+        @max_run_budget_us = INFINITE_BUDGET_US if @max_run_budget_us >= INFINITE_BUDGET_US
+
+        Sqreen.log.debug { "Max WAF run budget for #{@waf_rule_name} set to #{@max_run_budget_us} us" }
 
         ObjectSpace.define_finalizer(self, WAFCB.finalizer(@waf_rule_name.dup))
       end
@@ -68,20 +76,32 @@ module Sqreen
 
         env = [binding, framework, instance, args]
 
+        start = Sqreen.time if budget
+
         capper = Sqreen::Util::Capper.new(string_size_cap: 4096, size_cap: 150, depth_cap: 10)
         waf_args = binding_accessors.each_with_object({}) do |(e, b), h|
           h[e] = capper.call(b.resolve(*env))
         end
         waf_args = Sqreen::EncodingSanitizer.sanitize(waf_args)
-        waf_budget = [self.budget, budget && budget * 1_000_000].compact.min.to_i
-        action, data = ::LibSqreen::WAF.run(waf_rule_name, waf_args, waf_budget)
+
+        if budget
+          rem_budget_s = budget - (Sqreen.time - start)
+          return advise_action(nil) if rem_budget_s <= 0.0
+
+          waf_gen_budget_us = [(rem_budget_s * 1_000_000).to_i, MAX_FIXNUM].min
+        else # no budget
+          waf_gen_budget_us = INFINITE_BUDGET_US
+        end
+
+        action, data = ::LibSqreen::WAF.run(waf_rule_name, waf_args,
+                                            waf_gen_budget_us, @max_run_budget_us)
 
         case action
         when :monitor
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(nil)
         when :block
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(:raise)
         when :good
           advise_action(nil)
@@ -112,20 +132,23 @@ module Sqreen
       end
 
       def record_exception(exception, infos = {}, at = Time.now.utc)
-        infos.merge!(exception_to_infos(exception)) if exception.is_a?(Sqreen::WAFError)
+        infos.merge!(waf_infos(exception)) if exception.is_a?(Sqreen::WAFError)
         super(exception, infos, at)
       end
 
       private
 
-      def exception_to_infos(e)
+      # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000016-waf-integration.md#error-management
+      def waf_infos(e)
         {
-          waf_rule: e.rule_name,
-          error_code: ERROR_CODES[e.error],
-        }.tap do |r|
-          r[:error_data] = e.data if e.data
-          r[:args] = e.args if e.args
-        end
+          waf:  {
+            waf_rule: e.rule_name,
+            error_code: ERROR_CODES[e.error],
+          }.tap do |r|
+            r[:error_data] = e.data if e.data
+            r[:args] = e.args if e.arg
+          end,
+        }
       end
 
       ERROR_CODES = {

data/lib/sqreen/runner.rb

@@ -23,6 +23,7 @@ require 'sqreen/performance_notifications/binned_metrics'
 require 'sqreen/legacy/instrumentation'
 require 'sqreen/call_countable'
 require 'sqreen/weave/legacy/instrumentation'
+require 'sqreen/kit/configuration'
 
 module Sqreen
   @features = {}
@@ -37,6 +38,8 @@ module Sqreen
   PERF_METRICS_PERIOD = 60 # 1 min
   DEFAULT_PERF_LEVEL = 0 # disabled
 
+  DEFAULT_USE_SIGNALS = false
+
   class << self
     attr_reader :features
     def update_features(features)
@@ -87,7 +90,9 @@ module Sqreen
 
     attr_accessor :heartbeat_delay
     attr_accessor :metrics_engine
+    # @return [Sqreen::Deliveries::Simple]
     attr_reader :deliverer
+    # @return [Sqreen::Session]
     attr_reader :session
     attr_reader :instrumenter
     attr_accessor :running
@@ -111,17 +116,26 @@ module Sqreen
       @token = @configuration.get(:token)
       @app_name = @configuration.get(:app_name)
       @url = @configuration.get(:url)
+      @proxy_url = @configuration.get(:proxy_url)
       Sqreen.update_whitelisted_paths([])
       Sqreen.update_whitelisted_ips({})
       Sqreen.update_performance_budget(nil)
       raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
+      Sqreen::Kit::Configuration.logger = Sqreen.log
+      Sqreen::Kit::Configuration.ingestion_url = @configuration.get(:ingestion_url)
+      Sqreen::Kit::Configuration.proxy_url = @configuration.get(:proxy_url)
+
       register_exit_cb if set_at_exit
 
       self.metrics_engine = MetricsStore.new
 
-      if @configuration.get(:weave)
+      needs_weave = proc do
+        Gem::Specification.select { |s| s.name == 'scout_apm' && Gem::Requirement.new('>= 2.5.2').satisfied_by?(Gem::Version.new(s.version)) }.any?
+      end
+
+      if @configuration.get(:weave) || needs_weave.call
         @instrumenter = Sqreen::Weave::Legacy::Instrumentation.new(metrics_engine)
       else
         @instrumenter = Sqreen::Legacy::Instrumentation.new(metrics_engine)
@@ -138,7 +152,7 @@ module Sqreen
           Sqreen.log.debug do
             "Override initial features with #{conf_features.inspect}"
           end
-          wanted_features = conf_features
+          wanted_features = wanted_features.merge(conf_features)
         rescue
           Sqreen.log.warn do
             "NOT using invalid inital features #{conf_initial_features}"
@@ -157,7 +171,7 @@ module Sqreen
     end
 
     def create_session(session_class)
-      @session = session_class.new(@url, @token, @app_name)
+      @session = session_class.new(@url, @token, @app_name, @proxy_url)
       session.login(@framework)
     end
 
@@ -166,8 +180,18 @@ module Sqreen
       @deliverer = new_deliverer
     end
 
-    def batch_events(batch_size, max_staleness = nil)
+    def batch_events(batch_size, max_staleness = nil, use_signals = false)
       size = batch_size.to_i
+
+      if size <= 1 && use_signals
+        Sqreen.log.warn do
+          "Using signals with no delivery batching is unsupported. " \
+          "Using instead batching with batch size = 30, max_staleness = 60"
+        end
+        size = 30
+        max_staleness = 60
+      end
+
       self.deliverer = if size < 1
                          Deliveries::Simple.new(session)
                        else
@@ -297,19 +321,37 @@ module Sqreen
     def do_heartbeat
       @last_heartbeat_request = Time.now
       @next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
-      res = session.heartbeat(next_command_results, next_metrics)
+      metrics_in_hb = use_signals? ? nil : next_metrics
+
+      res = session.heartbeat(next_command_results, metrics_in_hb)
       next_command_results.clear
+
+      deliver_metrics_as_event if use_signals?
       next_metrics.clear
+
       process_commands(res['commands'])
     end
 
+    def deliver_metrics_as_event
+      # this is disastrous withe simple delivery strategy,
+      # as each aggregated metric would trigger an http request
+      # Sending of metrics is therefore not supported with simple delivery strategy
+      # TODO: Confirm that only batch is used in production
+      next_metrics.each { |x| deliverer.post_event(x) }
+    end
+
     def features(_context_infos = {})
       Sqreen.features
     end
 
+    def use_signals?
+      features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+    end
+
     def features=(features)
       Sqreen.update_features(features)
       session.request_compression = features['request_compression'] if session
+      session.use_signals = use_signals?
       self.performance_metrics_period = features['performance_metrics_period']
 
       unless @configuration.get(:weave)
@@ -327,7 +369,7 @@ module Sqreen
       hd = features['heartbeat_delay'].to_i
       self.heartbeat_delay = hd if hd > 0
       return if features['batch_size'].nil?
-      batch_events(features['batch_size'], features['max_staleness'])
+      batch_events(features['batch_size'], features['max_staleness'], use_signals?)
     end
 
     def change_whitelisted_paths(paths, _context_infos = {})

data/lib/sqreen/sensitive_data_redactor.rb

@@ -61,7 +61,7 @@ module Sqreen
         obj.each do |k, v|
           ck = k.is_a?(String) ? k.downcase : k
           if @keys.include?(ck)
-            redacted << v
+            redacted += SensitiveDataRedactor.all_strings(v)
             v = MASK
           else
             v, r = redact(v)
@@ -74,39 +74,27 @@ module Sqreen
       [result, redacted]
     end
 
-    def redact_attacks!(attacks, values)
-      return attacks if values.empty?
-
-      values = values.map { |v| v.downcase if v.is_a?(String) }
-
-      attacks.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf_data]
-
-        parsed = JSON.parse(e[:infos][:waf_data])
-        redacted = parsed.each do |w|
-          next unless (filters = w['filter'])
-
-          filters.each do |f|
-            next unless (v = f['resolved_value'])
-            next unless values.include?(v.downcase)
+    class << self
+      def all_strings(v)
+        accum = []
+        all_strings_impl(v, accum)
+        accum
+      end
 
-            f['match_status'] = MASK
-            f['resolved_value'] = MASK
+      private
+
+      def all_strings_impl(obj, accum)
+        case obj
+        when String
+          accum << obj
+        when Array
+          obj.each { |el| all_strings_impl(el, accum) }
+        when Hash
+          obj.each do |k, v|
+            all_strings_impl(k, accum)
+            all_strings_impl(v, accum)
           end
         end
-        e[:infos][:waf_data] = JSON.dump(redacted)
-      end
-    end
-
-    def redact_exceptions!(exceptions, values)
-      return exceptions if values.empty?
-
-      exceptions.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf]
-
-        e[:infos][:waf].delete(:args)
       end
     end
   end