sqreen 1.19.1 → 1.20.2

data/lib/sqreen/signals/http_trace_redaction.rb

@@ -0,0 +1,111 @@
+require 'json'
+require 'sqreen/kit/loggable'
+require 'sqreen/kit/signals/specialized/http_trace'
+
+module Sqreen
+  module Signals
+    module HttpTraceRedaction
+      class << self
+        include Sqreen::Kit::Loggable
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        def redact_trace!(trace, redactor)
+          return unless redactor
+          # redact headers (keys unsafe)
+          # @type [Sqreen::Kit::Signals::Context::HttpContext]
+          http_context = trace.context
+
+          all_redacted = []
+
+          # Redact headers; save redacted values
+          # headers are encoded as [key, value], not a hash, so
+          # they require some transformation
+          orig_headers = http_context.headers
+          if orig_headers
+            headers = orig_headers.map { |(k, v)| { k => v } }
+            headers, redacted = redactor.redact(headers)
+            http_context.headers = headers.map(&:first)
+            all_redacted += redacted
+          end
+
+          # Redact params; save redacted values
+          Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
+            value = http_context.public_send(attr)
+            next unless value
+            value, redacted = redactor.redact(value)
+            all_redacted += redacted
+            http_context.public_send(:"#{attr}=", value)
+          end
+
+          all_redacted = all_redacted.uniq.map(&:downcase)
+
+          # Redact attacks and exceptions
+          # XXX: no redaction for infos in attacks/exceptions except for WAF data
+          # Is this the correct behavior?
+          redact_attacks!(trace, redactor, all_redacted)
+          redact_exceptions!(trace, redactor, all_redacted)
+        end
+
+        private
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        # Redacts WAF data according to specific rules therefor
+        # Redacts infos according to general rules
+        def redact_attacks!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::Attack)
+            # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
+            payload = signal.payload
+            next unless payload.infos
+
+            if payload.infos[:waf_data]
+              redact_waf_attack_data!(payload.infos, redacted_data)
+            end
+            payload.infos, = redactor.redact(payload.infos)
+          end
+        end
+
+        def redact_exceptions!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
+            infos = signal.infos
+            next unless infos
+
+            redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
+            signal.infos, = redactor.redact(infos)
+          end
+        end
+
+        # @param [Hash] infos from WAF attack
+        def redact_waf_attack_data!(infos, redacted_data)
+          begin
+            parsed = JSON.parse(infos[:waf_data])
+          rescue JSON::JSONError => e
+            logger.warn("waf_data is not valid json: #{e.message}")
+            return
+          end
+          redacted = parsed.each do |w|
+            next unless (filters = w['filter'])
+
+            filters.each do |f|
+              next unless (v = f['resolved_value'])
+              next unless redacted_data.include?(v.downcase)
+
+              f['match_status'] = SensitiveDataRedactor::MASK
+              f['resolved_value'] = SensitiveDataRedactor::MASK
+            end
+          end
+          infos[:waf_data] = JSON.dump(redacted)
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_waf_exception_data!(infos, redacted_data)
+          return if redacted_data.empty?
+          infos[:waf].delete(:args)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/signals/signals_submission_strategy.rb

@@ -0,0 +1,78 @@
+require 'sqreen/aggregated_metric'
+require 'sqreen/kit'
+require 'sqreen/kit/string_sanitizer'
+require 'sqreen/signals/conversions'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Signals
+    # see also Sqreen::Legacy::OldEventSubmissionStrategy
+    # usage in Sqreen:Session
+    class SignalsSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      # @param [Array<Sqreen::AggregatedMetric>] metrics
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+
+        guarded 'Failed to serialize or submit aggregated metrics' do
+          batch = metrics.map do |m|
+            Conversions.convert_metric_sample(m)
+          end
+          client.report_batch(batch)
+        end
+      end
+
+      # @param _attack [Sqreen::Attack]
+      # XXX: unused
+      def post_attack(_attack)
+        raise NotImplementedError
+      end
+
+      # @param request_record [Sqreen::RequestRecord]
+      def post_request_record(request_record)
+        guarded 'Failed to serialize or submit request record' do
+          trace = Conversions.convert_req_record(request_record)
+          append_sanitizing_filter(trace)
+          client.report_trace(trace)
+        end
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        guarded 'Failed to serialize or submit exception', false do
+          data = Conversions.convert_exception(exception)
+          append_sanitizing_filter(data)
+          client.report_signal(data)
+        end
+      end
+
+      def post_batch(events)
+        guarded 'Failed to serialize or submit batch of events' do
+          batch = Conversions.convert_batch(events)
+          batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
+          client.report_batch(batch)
+        end
+      end
+
+      private
+
+      def append_sanitizing_filter(sig_or_trace)
+        sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
+      end
+
+      # we don't want exceptions to propagate and kill the worker thread
+      def guarded(msg, report = true)
+        yield
+      rescue StandardError => e
+        logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| "  #{x}" }.join("\n")}"
+        post_sqreen_exception(RemoteException.new(e)) if report
+      end
+
+      def client
+        Sqreen::Kit.auth_signals_client
+      end
+    end
+  end
+end

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.19.1'.freeze
+  VERSION = '1.20.2'.freeze
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -60,6 +60,27 @@ class Sqreen::Weave::Legacy::Instrumentation
       'options' => opts[:perf_metric_percent] || { 'base' => 1.3, 'factor' => 1.0 },
     )
 
+    metrics_engine.create_metric(
+      'name' => 'req_sq_hook_overhead',
+      'period' => 60,
+      'kind' => 'Binning',
+      'options' => { 'base' => 2.0, 'factor' => 0.1 },
+    )
+
+    metrics_engine.create_metric(
+      'name' => 'sq.hook.overhead',
+      'period' => 60,
+      'kind' => 'Binning',
+      'options' => { 'base' => 2.0, 'factor' => 0.1 },
+    )
+
+    metrics_engine.create_metric(
+      'name' => 'sq.shrinkwrap',
+      'period' => 60,
+      'kind' => 'Binning',
+      'options' => { 'base' => 2.0, 'factor' => 0.1 },
+    )
+
     Sqreen.thread_cpu_time? && metrics_engine.create_metric(
       'name' => 'sq_thread_cpu_pct',
       'period' => opts[:period] || 60,
@@ -113,6 +134,9 @@ class Sqreen::Weave::Legacy::Instrumentation
       before('wave,meta,request', rank: -100000, mandatory: true) do |_call|
         next unless Sqreen.instrumentation_ready
 
+        # shrinkwrap_timer = Sqreen::Graft::Timer.new('weave,shrinkwrap')
+        # shrinkwrap_timer.start
+
         uuid = SecureRandom.uuid
         now = Sqreen::Graft::Timer.read
         Thread.current[:sqreen_http_request] = {
@@ -123,14 +147,13 @@ class Sqreen::Weave::Legacy::Instrumentation
           timer: Sqreen::Graft::Timer.new("request_#{uuid}"),
           timed_callbacks: [],
           timed_hooks: [],
-          timed_hooks_before: [],
-          timed_hooks_after: [],
-          timed_hooks_raised: [],
-          timed_hooks_ensured: [],
           skipped_callbacks: [],
+          # timed_shrinkwrap: shrinkwrap_timer,
         }
 
         Sqreen::Weave.logger.debug { "request.uuid: #{uuid}" }
+
+        # shrinkwrap_timer.stop
       end
 
       ensured('weave,meta,request', rank: 100000, mandatory: true) do |_call|
@@ -138,6 +161,9 @@ class Sqreen::Weave::Legacy::Instrumentation
 
         next if request.nil?
 
+        # shrinkwrap_timer = request[:timed_shrinkwrap]
+        # shrinkwrap_timer.start
+
         Thread.current[:sqreen_http_request] = nil
         now = Sqreen::Graft::Timer.read
         utc_now = Time.now.utc
@@ -167,59 +193,28 @@ class Sqreen::Weave::Legacy::Instrumentation
           metrics_engine.update(metric_name, now, nil, duration * 1000)
         end
 
-        metric_name = 'sq.hooks_pre.pre'
-        duration = request[:timed_hooks_before].sum(&:duration)
-        unless metrics_engine.metric?(metric_name)
-          metrics_engine.create_metric(
-            'name' => metric_name,
-            'period' => 60,
-            'kind' => 'Binning',
-            'options' => { 'base' => 2.0, 'factor' => 0.1 },
-          )
-        end
-        metrics_engine.update(metric_name, now, nil, duration * 1000)
-
-        metric_name = 'sq.hooks_post.post'
-        duration = request[:timed_hooks_after].sum(&:duration)
-        unless metrics_engine.metric?(metric_name)
-          metrics_engine.create_metric(
-            'name' => metric_name,
-            'period' => 60,
-            'kind' => 'Binning',
-            'options' => { 'base' => 2.0, 'factor' => 0.1 },
-          )
-        end
-        metrics_engine.update(metric_name, now, nil, duration * 1000)
-
-        metric_name = 'sq.hooks_failing.failing'
-        duration = request[:timed_hooks_raised].sum(&:duration)
-        unless metrics_engine.metric?(metric_name)
-          metrics_engine.create_metric(
-            'name' => metric_name,
-            'period' => 60,
-            'kind' => 'Binning',
-            'options' => { 'base' => 2.0, 'factor' => 0.1 },
-          )
+        request[:timed_hooks].each do |timer|
+          duration = timer.duration
+          metrics_engine.update('sq.hook.overhead', now, nil, duration * 1000)
         end
-        metrics_engine.update(metric_name, now, nil, duration * 1000)
 
         skipped = request[:skipped_callbacks].map(&:name)
-        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" }
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.skipped.size: #{skipped.count} callback.skipped: [#{skipped.join(', ')}]" } if Sqreen::Weave.logger.debug?
         timer = request[:timer]
         total = timer.duration
-        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)} timer.size: #{timer.size}" }
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} timer.total: #{'%.03fus' % (total * 1_000_000)}" } if Sqreen::Weave.logger.debug?
         timings = request[:timed_callbacks].map(&:to_s)
         total = request[:timed_callbacks].sum(&:duration)
-        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" }
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} callback.total: #{'%.03fus' % (total * 1_000_000)} callback.timings: [#{timings.join(', ')}]" } if Sqreen::Weave.logger.debug?
         timings = request[:timed_hooks].map(&:to_s)
         total = request[:timed_hooks].sum(&:duration)
-        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" }
+        Sqreen::Weave.logger.debug { "request:#{request[:uuid]} hook.total: #{'%.03fus' % (total * 1_000_000)} hook.timings: [#{timings.join(', ')}]" } if Sqreen::Weave.logger.debug?
 
         skipped = request[:skipped_callbacks].map(&:name)
         skipped_rule_name = skipped.first && skipped.first =~ /weave,rule=(.*)$/ && $1
         Sqreen.observations_queue.push(['request_overtime', skipped_rule_name, 1, utc_now]) if skipped_rule_name
 
-        sqreen_request_duration = total
+        sqreen_request_duration = request[:timed_hooks].sum(&:duration) + request[:timed_callbacks].sum(&:duration)
         Sqreen.observations_queue.push(['sq', nil, sqreen_request_duration * 1000, utc_now])
 
         request_duration = now - request[:start_time]
@@ -227,6 +222,14 @@ class Sqreen::Weave::Legacy::Instrumentation
 
         sqreen_request_ratio = (sqreen_request_duration * 100.0) / (request_duration - sqreen_request_duration)
         Sqreen.observations_queue.push(['pct', nil, sqreen_request_ratio, utc_now])
+
+        duration = request[:timed_hooks].sum(&:duration)
+        metrics_engine.update('req_sq_hook_overhead', now, nil, duration * 1000)
+
+        # shrinkwrap_timer.stop
+
+        # duration = shrinkwrap_timer.duration
+        # metrics_engine.update('sq.shrinkwrap', now, nil, duration * 1000)
       end
     end.install
 
@@ -275,7 +278,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i}" } if Sqreen::Weave.logger.debug?
           begin
             ret = callback.pre(i, a, r)
           rescue StandardError => e
@@ -286,7 +289,7 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#pre instance=#{i} => return=#{ret.inspect}" } if Sqreen::Weave.logger.debug?
 
           case ret[:status]
           when :skip, 'skip'
@@ -296,7 +299,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           when :raise, 'raise'
             throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
             throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -309,7 +312,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i}" } if Sqreen::Weave.logger.debug?
           begin
             ret = callback.post(v, i, a, r)
           rescue StandardError => e
@@ -320,7 +323,7 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#post instance=#{i} => return=#{ret.inspect}" } if Sqreen::Weave.logger.debug?
 
           case ret[:status]
           when :override, 'override'
@@ -328,7 +331,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           when :raise, 'raise'
             throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
             throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -341,7 +344,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           a = call.args
           r = call.remaining
 
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i}" } if Sqreen::Weave.logger.debug?
           begin
             ret = callback.failing(e, i, a, r)
           rescue StandardError => e
@@ -352,9 +355,9 @@ class Sqreen::Weave::Legacy::Instrumentation
               Sqreen::RemoteException.record(e)
             end
           end
-          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
+          Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" } if Sqreen::Weave.logger.debug?
 
-          raise e if ret.nil?
+          throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
 
           case ret[:status]
           when :override, 'override'
@@ -368,7 +371,7 @@ class Sqreen::Weave::Legacy::Instrumentation
             throw(b, b.raise(e))
           else
             throw(b, b.raise(e))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
     end.install

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.19.1
+  version: 1.20.2
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-01 00:00:00.000000000 Z
+date: 2020-07-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqreen-backport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: sqreen-kit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: sq_mini_racer
   requirement: !ruby/object:Gem::Requirement
@@ -65,11 +93,10 @@ files:
 - lib/sqreen/actions/user_action_class.rb
 - lib/sqreen/actions/users_index.rb
 - lib/sqreen/agent.rb
+- lib/sqreen/agent_message.rb
+- lib/sqreen/aggregated_metric.rb
 - lib/sqreen/attack_blocked.rb
 - lib/sqreen/attack_detected.html
-- lib/sqreen/backport.rb
-- lib/sqreen/backport/clock_gettime.rb
-- lib/sqreen/backport/original_name.rb
 - lib/sqreen/binding_accessor.rb
 - lib/sqreen/binding_accessor/path_elem.rb
 - lib/sqreen/binding_accessor/transforms.rb
@@ -96,6 +123,7 @@ files:
 - lib/sqreen/dependency/sentry.rb
 - lib/sqreen/dependency/sinatra.rb
 - lib/sqreen/encoding_sanitizer.rb
+- lib/sqreen/endpoint_testing.rb
 - lib/sqreen/error_handling_middleware.rb
 - lib/sqreen/event.rb
 - lib/sqreen/events/attack.rb
@@ -129,8 +157,16 @@ files:
 - lib/sqreen/js/mini_racer_adapter.rb
 - lib/sqreen/js/mini_racer_executable_js.rb
 - lib/sqreen/js/thread_local_exec_js_runnable.rb
+- lib/sqreen/kit/signals/specialized/aggregated_metric.rb
+- lib/sqreen/kit/signals/specialized/attack.rb
+- lib/sqreen/kit/signals/specialized/binning_metric.rb
+- lib/sqreen/kit/signals/specialized/http_trace.rb
+- lib/sqreen/kit/signals/specialized/sdk_track_call.rb
+- lib/sqreen/kit/signals/specialized/sqreen_exception.rb
 - lib/sqreen/legacy.rb
 - lib/sqreen/legacy/instrumentation.rb
+- lib/sqreen/legacy/old_event_submission_strategy.rb
+- lib/sqreen/legacy/waf_redactions.rb
 - lib/sqreen/log.rb
 - lib/sqreen/log/loggable.rb
 - lib/sqreen/logger.rb
@@ -201,6 +237,9 @@ files:
 - lib/sqreen/shared_storage.rb
 - lib/sqreen/shared_storage23.rb
 - lib/sqreen/shrink_wrap.rb
+- lib/sqreen/signals/conversions.rb
+- lib/sqreen/signals/http_trace_redaction.rb
+- lib/sqreen/signals/signals_submission_strategy.rb
 - lib/sqreen/signature_verifier.rb
 - lib/sqreen/sinatra_middleware.rb
 - lib/sqreen/sqreen_signed_verifier.rb
@@ -253,8 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Sqreen Ruby agent