sqreen 1.19.0 → 1.20.1

data/lib/sqreen/signals/conversions.rb

@@ -0,0 +1,283 @@
+require 'sqreen/version'
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+require 'sqreen/metrics/binning'
+require 'sqreen/signals/http_trace_redaction'
+require 'sqreen/kit/signals/signal_attributes'
+require 'sqreen/kit/signals/specialized/aggregated_metric'
+require 'sqreen/kit/signals/specialized/attack'
+require 'sqreen/kit/signals/specialized/binning_metric'
+require 'sqreen/kit/signals/specialized/sqreen_exception'
+require 'sqreen/kit/signals/specialized/http_trace'
+require 'sqreen/kit/signals/specialized/sdk_track_call'
+
+module Sqreen
+  module Signals
+    module Conversions # rubocop:disable Metrics/ModuleLength
+      class << self
+        # @param [Sqreen::AggregatedMetric] agg
+        # @return [Sqreen::Kit::Signals::Metric]
+        def convert_metric_sample(agg)
+          attrs = {
+            signal_name: "sq.agent.metric.#{agg.name}",
+            source: if agg.rule
+                      "sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
+                    else
+                      agent_gen_source
+                    end,
+            time: agg.finish,
+          }
+
+          if agg.metric.is_a?(Sqreen::Metric::Binning)
+            conv_binning_metric(agg, attrs)
+          else
+            conv_generic_metric(agg, attrs)
+          end
+        end
+
+        # @param [Sqreen::Attack] attack
+        # XXX: not used because we don't use Sqreen::Attack
+        def convert_attack(attack)
+          # no need to set actor/context as we only include them in request records/traces
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{attack.attack_type}",
+            source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
+            time: attack.time,
+            location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: attack.test?,
+              block: attack.block?,
+              infos: attack.infos
+            )
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_event
+        def convert_unstructured_attack(payload)
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{payload[:attack_type]}",
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: payload[:test],
+              block: payload[:block],
+              infos: payload[:infos]
+            )
+          )
+        end
+
+        # @param [Sqreen::RemoteException] exception
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_exception(exception)
+          payload = exception.payload
+
+          infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
+          infos.merge!(payload['infos'] || {})
+
+          Kit::Signals::Specialized::SqreenException.new(
+            source: if payload['rule_name']
+                      "sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
+                    else
+                      agent_gen_source
+                    end,
+            time: exception.time,
+            ruby_exception: payload['exception'],
+            infos: infos
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_exception
+        # @param [Hash] payload
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_unstructured_exception(payload)
+          Kit::Signals::Specialized::SqreenException.new(
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            ruby_exception: payload[:exception],
+            infos: payload[:infos]
+          )
+        end
+
+        # @param [Sqreen::RequestRecord] req_rec
+        # @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
+        def convert_req_record(req_rec)
+          payload = req_rec.payload
+
+          request_p = payload['request']
+          id_args = req_rec.last_identify_args
+          identifiers = id_args[0] if id_args
+          traits = id_args[1] if id_args
+
+          observed = payload[:observed] || {}
+          signals = []
+          signals += (observed[:attacks] || [])
+                     .map { |att| convert_unstructured_attack(att) }
+          signals += (observed[:sqreen_exceptions] || [])
+                     .map { |sq_exc| convert_unstructured_exception(sq_exc) }
+          signals += req_rec.processed_sdk_calls
+                            .select { |h| h[:name] == :track }
+                            .map { |h| convert_track(h) }
+
+          trace = Kit::Signals::Specialized::HttpTrace.new(
+            actor: Kit::Signals::Actor.new(
+              ip_addresses: [request_p[:client_ip]].compact,
+              user_agent: request_p[:user_agent],
+              identifiers: identifiers,
+              traits: traits,
+            ),
+            location_infra: location_infra,
+            context: convert_request(request_p,
+                                     payload['response'],
+                                     payload['headers'],
+                                     payload['params']),
+            data: signals
+          )
+          HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
+          trace
+        end
+
+        # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
+        def convert_batch(batch)
+          batch.map do |evt|
+            case evt
+            when RemoteException
+              convert_exception(evt)
+            when AggregatedMetric
+              convert_metric_sample(evt)
+            when RequestRecord
+              convert_req_record(evt)
+            else
+              raise NotImplementedError, "Unknown type of event in batch: #{evt}"
+            end
+          end
+        end
+
+        private
+
+        def agent_gen_source
+          "sqreen:agent:ruby:#{Sqreen::VERSION}"
+        end
+
+        def location_infra
+          @location_infra ||= begin
+            Kit::Signals::LocationInfra.new(
+              agent_version: Sqreen::VERSION,
+              os_type: RuntimeInfos.os[:os_type],
+              hostname: RuntimeInfos.hostname,
+              runtime_type: RuntimeInfos.runtime[:runtime_type],
+              runtime_version: RuntimeInfos.runtime[:runtime_version],
+              libsqreen_version: RuntimeInfos.libsqreen_version,
+            )
+          end
+        end
+
+        # see Sqreen::RequestRecord.processed_sdk_calls
+        def convert_track(call_info)
+          options = call_info[:args][1] || {}
+          Kit::Signals::Specialized::SdkTrackCall.new(
+            signal_name: "sq.sdk.#{call_info[:args][0]}",
+            time: call_info[:time],
+            payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
+              properties: options[:properties],
+              user_identifiers: options[:user_identifiers]
+            )
+          )
+        end
+
+        # @param [Hash] req_payload
+        # @param [Hash] headers_payload
+        # @param [Hash] params_payload
+        # see the PayloadCreator abomination for reference
+        # TODO: do not convert from the old payload to the new payload
+        # Have an intermediate object that gets the data from the framework.
+        # (Or convert directly from the framework, but this needs to be
+        # done during the request, not just before event is transmitted)
+        def convert_request(req_payload, resp_payload, headers_payload, params_payload)
+          req_payload ||= {}
+          headers_payload ||= {}
+          resp_payload ||= {}
+          params_payload ||= {}
+
+          other = params_payload['other']
+          other = merge_hash_append(other, params_payload['rack'])
+          other = merge_hash_append(other, params_payload['grape_params'])
+          other = merge_hash_append(other, params_payload['rack_routing'])
+
+          Sqreen::Kit::Signals::Context::HttpContext.new(
+            {
+              rid: req_payload[:rid],
+              headers: headers_payload,
+              user_agent: req_payload[:user_agent],
+              scheme: req_payload[:scheme],
+              verb: req_payload[:verb],
+              host: req_payload[:host],
+              port: req_payload[:port],
+              remote_ip: req_payload[:remote_ip],
+              remote_port: req_payload[:remote_port] || 0,
+              path: req_payload[:path],
+              referer: req_payload[:referer],
+              params_query: params_payload['query'],
+              params_form: params_payload['form'],
+              params_other: other,
+              # endpoint, is_reveal_replayed not set
+              status: resp_payload[:status],
+              content_length: resp_payload[:content_length],
+              content_type: resp_payload[:content_type],
+            }
+          )
+        end
+
+        def merge_hash_append(hash1, hash2)
+          return nil if hash1.nil? && hash2.nil?
+          return hash1 if hash2.nil? || hash2.empty?
+          return hash2 if hash1.nil? || hash1.empty?
+
+          pairs = (hash1.keys + hash2.keys).map do |key|
+            values1 = hash1[key]
+            values2 = hash2[key]
+            values = [values1, values2].compact
+            values = values.first if values.size == 1
+            [key, values]
+          end
+          Hash[pairs]
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_generic_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
+            kind: metric_kind(agg.metric),
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            values: agg.data
+          )
+
+          Kit::Signals::Specialized::AggregatedMetric.new(attrs)
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_binning_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            base: agg.data['b'],
+            unit: agg.data['u'],
+            max: agg.data['v']['max'],
+            bins: agg.data['v'].reject { |k, _v| k == 'max' }
+          )
+
+          Kit::Signals::Specialized::BinningMetric.new(attrs)
+        end
+
+        # @param [Sqreen::Metric::Base] metric
+        def metric_kind(metric)
+          metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/signals/http_trace_redaction.rb

@@ -0,0 +1,111 @@
+require 'json'
+require 'sqreen/kit/loggable'
+require 'sqreen/kit/signals/specialized/http_trace'
+
+module Sqreen
+  module Signals
+    module HttpTraceRedaction
+      class << self
+        include Sqreen::Kit::Loggable
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        def redact_trace!(trace, redactor)
+          return unless redactor
+          # redact headers (keys unsafe)
+          # @type [Sqreen::Kit::Signals::Context::HttpContext]
+          http_context = trace.context
+
+          all_redacted = []
+
+          # Redact headers; save redacted values
+          # headers are encoded as [key, value], not a hash, so
+          # they require some transformation
+          orig_headers = http_context.headers
+          if orig_headers
+            headers = orig_headers.map { |(k, v)| { k => v } }
+            headers, redacted = redactor.redact(headers)
+            http_context.headers = headers.map(&:first)
+            all_redacted += redacted
+          end
+
+          # Redact params; save redacted values
+          Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
+            value = http_context.public_send(attr)
+            next unless value
+            value, redacted = redactor.redact(value)
+            all_redacted += redacted
+            http_context.public_send(:"#{attr}=", value)
+          end
+
+          all_redacted = all_redacted.uniq.map(&:downcase)
+
+          # Redact attacks and exceptions
+          # XXX: no redaction for infos in attacks/exceptions except for WAF data
+          # Is this the correct behavior?
+          redact_attacks!(trace, redactor, all_redacted)
+          redact_exceptions!(trace, redactor, all_redacted)
+        end
+
+        private
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        # Redacts WAF data according to specific rules therefor
+        # Redacts infos according to general rules
+        def redact_attacks!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::Attack)
+            # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
+            payload = signal.payload
+            next unless payload.infos
+
+            if payload.infos[:waf_data]
+              redact_waf_attack_data!(payload.infos, redacted_data)
+            end
+            payload.infos, = redactor.redact(payload.infos)
+          end
+        end
+
+        def redact_exceptions!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
+            infos = signal.infos
+            next unless infos
+
+            redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
+            signal.infos, = redactor.redact(infos)
+          end
+        end
+
+        # @param [Hash] infos from WAF attack
+        def redact_waf_attack_data!(infos, redacted_data)
+          begin
+            parsed = JSON.parse(infos[:waf_data])
+          rescue JSON::JSONError => e
+            logger.warn("waf_data is not valid json: #{e.message}")
+            return
+          end
+          redacted = parsed.each do |w|
+            next unless (filters = w['filter'])
+
+            filters.each do |f|
+              next unless (v = f['resolved_value'])
+              next unless redacted_data.include?(v.downcase)
+
+              f['match_status'] = SensitiveDataRedactor::MASK
+              f['resolved_value'] = SensitiveDataRedactor::MASK
+            end
+          end
+          infos[:waf_data] = JSON.dump(redacted)
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_waf_exception_data!(infos, redacted_data)
+          return if redacted_data.empty?
+          infos[:waf].delete(:args)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/signals/signals_submission_strategy.rb

@@ -0,0 +1,78 @@
+require 'sqreen/aggregated_metric'
+require 'sqreen/kit'
+require 'sqreen/kit/string_sanitizer'
+require 'sqreen/signals/conversions'
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Signals
+    # see also Sqreen::Legacy::OldEventSubmissionStrategy
+    # usage in Sqreen:Session
+    class SignalsSubmissionStrategy
+      include Sqreen::Log::Loggable
+
+      # @param [Array<Sqreen::AggregatedMetric>] metrics
+      def post_metrics(metrics)
+        return if metrics.nil? || metrics.empty?
+
+        guarded 'Failed to serialize or submit aggregated metrics' do
+          batch = metrics.map do |m|
+            Conversions.convert_metric_sample(m)
+          end
+          client.report_batch(batch)
+        end
+      end
+
+      # @param _attack [Sqreen::Attack]
+      # XXX: unused
+      def post_attack(_attack)
+        raise NotImplementedError
+      end
+
+      # @param request_record [Sqreen::RequestRecord]
+      def post_request_record(request_record)
+        guarded 'Failed to serialize or submit request record' do
+          trace = Conversions.convert_req_record(request_record)
+          append_sanitizing_filter(trace)
+          client.report_trace(trace)
+        end
+      end
+
+      # Post an exception to Sqreen for analysis
+      # @param exception [RemoteException] Exception and context to be sent over
+      def post_sqreen_exception(exception)
+        guarded 'Failed to serialize or submit exception', false do
+          data = Conversions.convert_exception(exception)
+          append_sanitizing_filter(data)
+          client.report_signal(data)
+        end
+      end
+
+      def post_batch(events)
+        guarded 'Failed to serialize or submit batch of events' do
+          batch = Conversions.convert_batch(events)
+          batch.each { |sig_or_trace| append_sanitizing_filter(sig_or_trace) }
+          client.report_batch(batch)
+        end
+      end
+
+      private
+
+      def append_sanitizing_filter(sig_or_trace)
+        sig_or_trace.append_to_h_filter Kit::StringSanitizer.method(:sanitize)
+      end
+
+      # we don't want exceptions to propagate and kill the worker thread
+      def guarded(msg, report = true)
+        yield
+      rescue StandardError => e
+        logger.warn "#{msg}: #{e.message}\n#{e.backtrace.map { |x| "  #{x}" }.join("\n")}"
+        post_sqreen_exception(RemoteException.new(e)) if report
+      end
+
+      def client
+        Sqreen::Kit.auth_signals_client
+      end
+    end
+  end
+end