sqreen 1.19.1 → 1.21.0.beta1

data/lib/sqreen/signals/http_trace_redaction.rb

@@ -0,0 +1,111 @@
+require 'json'
+require 'sqreen/kit/loggable'
+require 'sqreen/kit/signals/specialized/http_trace'
+
+module Sqreen
+  module Signals
+    module HttpTraceRedaction
+      class << self
+        include Sqreen::Kit::Loggable
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        def redact_trace!(trace, redactor)
+          return unless redactor
+          # redact headers (keys unsafe)
+          # @type [Sqreen::Kit::Signals::Context::HttpContext]
+          http_context = trace.context
+
+          all_redacted = []
+
+          # Redact headers; save redacted values
+          # headers are encoded as [key, value], not a hash, so
+          # they require some transformation
+          orig_headers = http_context.headers
+          if orig_headers
+            headers = orig_headers.map { |(k, v)| { k => v } }
+            headers, redacted = redactor.redact(headers)
+            http_context.headers = headers.map(&:first)
+            all_redacted += redacted
+          end
+
+          # Redact params; save redacted values
+          Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
+            value = http_context.public_send(attr)
+            next unless value
+            value, redacted = redactor.redact(value)
+            all_redacted += redacted
+            http_context.public_send(:"#{attr}=", value)
+          end
+
+          all_redacted = all_redacted.uniq.map(&:downcase)
+
+          # Redact attacks and exceptions
+          # XXX: no redaction for infos in attacks/exceptions except for WAF data
+          # Is this the correct behavior?
+          redact_attacks!(trace, redactor, all_redacted)
+          redact_exceptions!(trace, redactor, all_redacted)
+        end
+
+        private
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        # Redacts WAF data according to specific rules therefor
+        # Redacts infos according to general rules
+        def redact_attacks!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::Attack)
+            # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
+            payload = signal.payload
+            next unless payload.infos
+
+            if payload.infos[:waf_data]
+              redact_waf_attack_data!(payload.infos, redacted_data)
+            end
+            payload.infos, = redactor.redact(payload.infos)
+          end
+        end
+
+        def redact_exceptions!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
+            infos = signal.infos
+            next unless infos
+
+            redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
+            signal.infos, = redactor.redact(infos)
+          end
+        end
+
+        # @param [Hash] infos from WAF attack
+        def redact_waf_attack_data!(infos, redacted_data)
+          begin
+            parsed = JSON.parse(infos[:waf_data])
+          rescue JSON::JSONError => e
+            logger.warn("waf_data is not valid json: #{e.message}")
+            return
+          end
+          redacted = parsed.each do |w|
+            next unless (filters = w['filter'])
+
+            filters.each do |f|
+              next unless (v = f['resolved_value'])
+              next unless redacted_data.include?(v.downcase)
+
+              f['match_status'] = SensitiveDataRedactor::MASK
+              f['resolved_value'] = SensitiveDataRedactor::MASK
+            end
+          end
+          infos[:waf_data] = JSON.dump(redacted)
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_waf_exception_data!(infos, redacted_data)
+          return if redacted_data.empty?
+          infos[:waf].delete(:args)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/signals/signals_submission_strategy.rb

@@ -0,0 +1,78 @@
+require 'sqreen/aggregated_metric'
+require 'sqreen/kit'
+require 'sqreen/kit/string_sanitizer'
+require 'sqreen/signals/conversions'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Signals
+    # see also Sqreen::Legacy::OldEventSubmissionStrategy
+    # usage in Sqreen:Session
+    class SignalsSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      # @param [Array<Sqreen::AggregatedMetric>] metrics
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+
+        guarded 'Failed to serialize or submit aggregated metrics' do
+          batch = metrics.map do |m|
+            Conversions.convert_metric_sample(m)
+          end
+          client.report_batch(batch)
+        end
+      end
+
+      # @param _attack [Sqreen::Attack]
+      # XXX: unused
+      def post_attack(_attack)
+        raise NotImplementedError
+      end
+
+      # @param request_record [Sqreen::RequestRecord]
+      def post_request_record(request_record)
+        guarded 'Failed to serialize or submit request record' do
+          trace = Conversions.convert_req_record(request_record)
+          append_sanitizing_filter(trace)
+          client.report_trace(trace)
+        end
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        guarded 'Failed to serialize or submit exception', false do
+          data = Conversions.convert_exception(exception)
+          append_sanitizing_filter(data)
+          client.report_signal(data)
+        end
+      end
+
+      def post_batch(events)
+        guarded 'Failed to serialize or submit batch of events' do
+          batch = Conversions.convert_batch(events)
+          batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
+          client.report_batch(batch)
+        end
+      end
+
+      private
+
+      def append_sanitizing_filter(sig_or_trace)
+        sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
+      end
+
+      # we don't want exceptions to propagate and kill the worker thread
+      def guarded(msg, report = true)
+        yield
+      rescue StandardError => e
+        logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| "  #{x}" }.join("\n")}"
+        post_sqreen_exception(RemoteException.new(e)) if report
+      end
+
+      def client
+        Sqreen::Kit.auth_signals_client
+      end
+    end
+  end
+end

data/lib/sqreen/version.rb

@@ -4,5 +4,5 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 module Sqreen
-  VERSION = '1.19.1'.freeze
+  VERSION = '1.21.0.beta1'.freeze
 end

data/lib/sqreen/weave/legacy/instrumentation.rb

@@ -296,7 +296,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           when :raise, 'raise'
             throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
             throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -328,7 +328,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           when :raise, 'raise'
             throw(b, b.raise(ret[:exception])) if ret.key?(:exception)
             throw(b, b.raise(Sqreen::AttackBlocked.new("Sqreen blocked a security threat (type: #{callback.rule_name}). No action is required.")))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
 
@@ -354,7 +354,7 @@ class Sqreen::Weave::Legacy::Instrumentation
           end
           Sqreen::Weave.logger.debug { "#{rule} klass=#{callback.klass} method=#{callback.method} when=#failing instance=#{i} => return=#{ret.inspect}" }
 
-          raise e if ret.nil?
+          throw(b, b.raise(e)) if ret.nil? || !ret.is_a?(Hash)
 
           case ret[:status]
           when :override, 'override'
@@ -368,7 +368,7 @@ class Sqreen::Weave::Legacy::Instrumentation
             throw(b, b.raise(e))
           else
             throw(b, b.raise(e))
-          end unless ret.nil?
+          end unless ret.nil? || !ret.is_a?(Hash)
         end
       end
     end.install

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: sqreen
 version: !ruby/object:Gem::Version
-  version: 1.19.1
+  version: 1.21.0.beta1
 platform: ruby
 authors:
 - Sqreen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-06-01 00:00:00.000000000 Z
+date: 2020-06-29 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqreen-backport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: sqreen-kit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: sq_mini_racer
   requirement: !ruby/object:Gem::Requirement
@@ -65,11 +93,10 @@ files:
 - lib/sqreen/actions/user_action_class.rb
 - lib/sqreen/actions/users_index.rb
 - lib/sqreen/agent.rb
+- lib/sqreen/agent_message.rb
+- lib/sqreen/aggregated_metric.rb
 - lib/sqreen/attack_blocked.rb
 - lib/sqreen/attack_detected.html
-- lib/sqreen/backport.rb
-- lib/sqreen/backport/clock_gettime.rb
-- lib/sqreen/backport/original_name.rb
 - lib/sqreen/binding_accessor.rb
 - lib/sqreen/binding_accessor/path_elem.rb
 - lib/sqreen/binding_accessor/transforms.rb
@@ -95,7 +122,32 @@ files:
 - lib/sqreen/dependency/rails.rb
 - lib/sqreen/dependency/sentry.rb
 - lib/sqreen/dependency/sinatra.rb
+- lib/sqreen/ecosystem.rb
+- lib/sqreen/ecosystem/dispatch_table.rb
+- lib/sqreen/ecosystem/http/net_http.rb
+- lib/sqreen/ecosystem/http/rack_request.rb
+- lib/sqreen/ecosystem/loggable.rb
+- lib/sqreen/ecosystem/module_api.rb
+- lib/sqreen/ecosystem/module_api/event_listener.rb
+- lib/sqreen/ecosystem/module_api/instrumentation.rb
+- lib/sqreen/ecosystem/module_api/signal_producer.rb
+- lib/sqreen/ecosystem/module_api/tracing_push_down.rb
+- lib/sqreen/ecosystem/module_api/transaction_storage.rb
+- lib/sqreen/ecosystem/module_registry.rb
+- lib/sqreen/ecosystem/redis/redis_connection.rb
+- lib/sqreen/ecosystem/tracing/sampler.rb
+- lib/sqreen/ecosystem/tracing/sampling_configuration.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_client.rb
+- lib/sqreen/ecosystem/tracing/signals/tracing_server.rb
+- lib/sqreen/ecosystem/tracing_id_setup.rb
+- lib/sqreen/ecosystem/transaction_storage.rb
+- lib/sqreen/ecosystem_integration.rb
+- lib/sqreen/ecosystem_integration/around_callbacks.rb
+- lib/sqreen/ecosystem_integration/instrumentation_service.rb
+- lib/sqreen/ecosystem_integration/request_lifecycle_tracking.rb
+- lib/sqreen/ecosystem_integration/signal_consumption.rb
 - lib/sqreen/encoding_sanitizer.rb
+- lib/sqreen/endpoint_testing.rb
 - lib/sqreen/error_handling_middleware.rb
 - lib/sqreen/event.rb
 - lib/sqreen/events/attack.rb
@@ -129,8 +181,16 @@ files:
 - lib/sqreen/js/mini_racer_adapter.rb
 - lib/sqreen/js/mini_racer_executable_js.rb
 - lib/sqreen/js/thread_local_exec_js_runnable.rb
+- lib/sqreen/kit/signals/specialized/aggregated_metric.rb
+- lib/sqreen/kit/signals/specialized/attack.rb
+- lib/sqreen/kit/signals/specialized/binning_metric.rb
+- lib/sqreen/kit/signals/specialized/http_trace.rb
+- lib/sqreen/kit/signals/specialized/sdk_track_call.rb
+- lib/sqreen/kit/signals/specialized/sqreen_exception.rb
 - lib/sqreen/legacy.rb
 - lib/sqreen/legacy/instrumentation.rb
+- lib/sqreen/legacy/old_event_submission_strategy.rb
+- lib/sqreen/legacy/waf_redactions.rb
 - lib/sqreen/log.rb
 - lib/sqreen/log/loggable.rb
 - lib/sqreen/logger.rb
@@ -201,6 +261,9 @@ files:
 - lib/sqreen/shared_storage.rb
 - lib/sqreen/shared_storage23.rb
 - lib/sqreen/shrink_wrap.rb
+- lib/sqreen/signals/conversions.rb
+- lib/sqreen/signals/http_trace_redaction.rb
+- lib/sqreen/signals/signals_submission_strategy.rb
 - lib/sqreen/signature_verifier.rb
 - lib/sqreen/sinatra_middleware.rb
 - lib/sqreen/sqreen_signed_verifier.rb
@@ -238,7 +301,9 @@ metadata:
   changelog_uri: https://docs.sqreen.com/ruby/release-notes/
   source_code_uri: https://github.com/sqreen/ruby-agent
   bug_tracker_uri: https://github.com/sqreen/ruby-agent/issues
-post_install_message: 
+post_install_message: |2
+      This is a Sqreen beta release and may not work in all situations.
+      Make sure to review CHANGELOG.md for important details.
 rdoc_options: []
 require_paths:
 - lib
@@ -249,12 +314,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Sqreen Ruby agent

data/lib/sqreen/backport.rb

@@ -1,9 +0,0 @@
-# typed: strong
-
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.com/terms.html
-
-module Sqreen
-  module Backport
-  end
-end

data/lib/sqreen/backport/clock_gettime.rb

@@ -1,74 +0,0 @@
-# typed: ignore
-
-# Copyright (c) 2015 Sqreen. All Rights Reserved.
-# Please refer to our terms for more information: https://www.sqreen.com/terms.html
-
-require 'sqreen/backport'
-
-module Sqreen
-  module Backport
-    module ClockGettime
-      class << self
-        def supported?
-          Process.respond_to?(:clock_gettime)
-        end
-      end
-
-      unless supported?
-        require 'ffi'
-
-        class Timespec < FFI::Struct
-          layout :tv_sec => :time_t, :tv_nsec => :long
-        end
-
-        module LibC
-          extend FFI::Library
-          ffi_lib FFI::Library::LIBC
-
-          # TODO: FFI::NotFoundError
-
-          if RUBY_PLATFORM =~ /darwin/
-            attach_function :mach_absolute_time, [], :uint64
-          end
-
-          attach_function :clock_gettime, [:int, :pointer], :int
-        end
-
-        module Constants
-          case RUBY_PLATFORM
-          when /darwin/
-            CLOCK_REALTIME = 0
-            CLOCK_MONOTONIC = 6
-            CLOCK_PROCESS_CPUTIME_ID = 12
-            CLOCK_THERAD_CPUTIME_ID = 16
-          when /linux/
-            CLOCK_REALTIME = 0
-            CLOCK_MONOTONIC = 1
-            CLOCK_PROCESS_CPUTIME_ID = 2
-            CLOCK_THREAD_CPUTIME_ID = 3
-          end
-        end
-
-        def clock_gettime(clock_id, unit = :float_second)
-          unless unit == :float_second
-            raise "Process.clock_gettime: unsupported unit #{unit.inspect}"
-          end
-
-          t = Timespec.new
-          ret = LibC.clock_gettime(clock_id, t.pointer)
-
-          raise SystemCallError, "Errno #{FFI.errno}" if ret == -1
-
-          t[:tv_sec].to_f + t[:tv_nsec].to_f / 1_000_000_000
-        end
-      end
-    end
-  end
-end
-
-unless Sqreen::Backport::ClockGettime.supported?
-  Process.instance_eval do
-    extend Sqreen::Backport::ClockGettime
-    include Sqreen::Backport::ClockGettime::Constants
-  end
-end