sqreen 1.19.1 → 1.21.0.beta1

data/lib/sqreen/sensitive_data_redactor.rb

@@ -61,7 +61,7 @@ module Sqreen
         obj.each do |k, v|
           ck = k.is_a?(String) ? k.downcase : k
           if @keys.include?(ck)
-            redacted << v
+            redacted += SensitiveDataRedactor.all_strings(v)
             v = MASK
           else
             v, r = redact(v)
@@ -74,39 +74,27 @@ module Sqreen
       [result, redacted]
     end
 
-    def redact_attacks!(attacks, values)
-      return attacks if values.empty?
-
-      values = values.map { |v| v.downcase if v.is_a?(String) }
-
-      attacks.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf_data]
-
-        parsed = JSON.parse(e[:infos][:waf_data])
-        redacted = parsed.each do |w|
-          next unless (filters = w['filter'])
-
-          filters.each do |f|
-            next unless (v = f['resolved_value'])
-            next unless values.include?(v.downcase)
+    class << self
+      def all_strings(v)
+        accum = []
+        all_strings_impl(v, accum)
+        accum
+      end
 
-            f['match_status'] = MASK
-            f['resolved_value'] = MASK
+      private
+
+      def all_strings_impl(obj, accum)
+        case obj
+        when String
+          accum << obj
+        when Array
+          obj.each { |el| all_strings_impl(el, accum) }
+        when Hash
+          obj.each do |k, v|
+            all_strings_impl(k, accum)
+            all_strings_impl(v, accum)
           end
         end
-        e[:infos][:waf_data] = JSON.dump(redacted)
-      end
-    end
-
-    def redact_exceptions!(exceptions, values)
-      return exceptions if values.empty?
-
-      exceptions.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf]
-
-        e[:infos][:waf].delete(:args)
       end
     end
   end

data/lib/sqreen/session.rb

@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
 require 'sqreen/events/request_record'
 require 'sqreen/exception'
 require 'sqreen/safe_json'
+require 'sqreen/kit'
+require 'sqreen/kit/configuration'
+require 'sqreen/signals/signals_submission_strategy'
+require 'sqreen/legacy/old_event_submission_strategy'
 
 require 'net/https'
 require 'uri'
@@ -41,13 +45,12 @@ module Sqreen
     RETRY_MANY = 301
 
     MUTEX = Mutex.new
-    METRICS_KEY = 'metrics'.freeze
 
     @@path_prefix = '/sqreen/v0/'
 
     attr_accessor :request_compression
 
-    def initialize(server_url, token, app_name = nil)
+    def initialize(server_url, cert_store, token, app_name = nil, proxy_url = nil)
       @token = token
       @app_name = app_name
       @session_id = nil
@@ -59,15 +62,29 @@ module Sqreen
       uri = parse_uri(server_url)
       use_ssl = (uri.scheme == 'https')
 
+      proxy_params = []
+      if proxy_url
+        proxy_uri = parse_uri(proxy_url)
+        proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
+      end
+
       @req_nb = 0
 
-      @http = Net::HTTP.new(uri.host, uri.port)
+      @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
       @http.use_ssl = use_ssl
-      if use_ssl
-        cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
-        cert_store = OpenSSL::X509::Store.new
-        cert_store.add_file cert_file
-        @http.cert_store = cert_store
+      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
+      @http.cert_store = cert_store if use_ssl
+      self.use_signals = false
+    end
+
+    def use_signals=(do_use)
+      return if do_use == @use_signals
+
+      @use_signals = do_use
+      if do_use
+        @evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
+      else
+        @evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
       end
     end
 
@@ -218,10 +235,7 @@ module Sqreen
     end
 
     def login(framework)
-      headers = {
-        'x-api-key'  => @token,
-        'x-app-name' => @app_name || framework.application_name,
-      }.reject { |k, v| v ==  nil }
+      headers = prelogin_auth_headers(framework)
 
       Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
 
@@ -235,6 +249,10 @@ module Sqreen
       end
       Sqreen.log.info 'Login success.'
       @session_id = res['session_id']
+
+      Kit::Configuration.session_key = @session_id
+      Kit.reset
+
       Sqreen.log.debug { "received session_id #{@session_id}" }
       Sqreen.logged_in = true
       res
@@ -246,20 +264,24 @@ module Sqreen
 
     def heartbeat(cmd_res = {}, metrics = [])
       payload = {}
-      payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
+      unless metrics.nil? || metrics.empty?
+        # never reached with signals
+        payload['metrics'] = metrics.map do |m|
+          Sqreen::Legacy::EventToHash.convert_agg_metric(m)
+        end
+      end
       payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
 
       post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
     end
 
     def post_metrics(metrics)
-      return if metrics.nil? || metrics.empty?
-      payload = { METRICS_KEY => metrics }
-      post(METRICS_KEY, payload, {}, RETRY_MANY)
+      @evt_sub_strategy.post_metrics(metrics)
     end
 
+    # XXX never called
     def post_attack(attack)
-      post('attack', attack.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_attack(attack)
     end
 
     def post_bundle(bundle_sig, dependencies)
@@ -271,33 +293,22 @@ module Sqreen
     end
 
     def post_request_record(request_record)
-      post('request_record', request_record.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_request_record(request_record)
     end
 
     # Post an exception to Sqreen for analysis
     # @param exception [RemoteException] Exception and context to be sent over
     def post_sqreen_exception(exception)
-      post('sqreen_exception', exception.to_hash, {}, 5)
-    rescue StandardError => e
-      Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
-                             e.inspect,
-                             exception.to_hash.inspect))
-      nil
+      @evt_sub_strategy.post_sqreen_exception(exception)
     end
 
-    BATCH_KEY = 'batch'.freeze
-    EVENT_TYPE_KEY = 'event_type'.freeze
     def post_batch(events)
-      batch = events.map do |event|
-        h = event.to_hash
-        h[EVENT_TYPE_KEY] = event_kind(event)
-        h
-      end
-      Sqreen.log.debug do
-        tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
-        "Doing batch with the following tally of event types: #{tally}"
-      end
-      post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
+      @evt_sub_strategy.post_batch(events)
+    end
+
+    def post_agent_message(framework, agent_message)
+      headers = prelogin_auth_headers(framework)
+      post('app_agent_message', agent_message.to_h, headers, 0)
     end
 
     # Perform agent logout
@@ -314,14 +325,13 @@ module Sqreen
       disconnect
     end
 
-    protected
+    private
 
-    def event_kind(event)
-      case event
-      when Sqreen::RemoteException then 'sqreen_exception'
-      when Sqreen::Attack then 'attack'
-      when Sqreen::RequestRecord then 'request_record'
-      end
+    def prelogin_auth_headers(framework)
+      {
+        'x-api-key' => @token,
+        'x-app-name' => @app_name || framework.application_name,
+      }.reject { |_k, v| v == nil }
     end
   end
 end

data/lib/sqreen/signals/conversions.rb

@@ -0,0 +1,288 @@
+require 'sqreen/version'
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+require 'sqreen/metrics/binning'
+require 'sqreen/signals/http_trace_redaction'
+require 'sqreen/kit/signals/signal_attributes'
+require 'sqreen/kit/signals/specialized/aggregated_metric'
+require 'sqreen/kit/signals/specialized/attack'
+require 'sqreen/kit/signals/specialized/binning_metric'
+require 'sqreen/kit/signals/specialized/sqreen_exception'
+require 'sqreen/kit/signals/specialized/http_trace'
+require 'sqreen/kit/signals/specialized/sdk_track_call'
+
+module Sqreen
+  module Signals
+    module Conversions # rubocop:disable Metrics/ModuleLength
+      class << self
+        # @param [Sqreen::AggregatedMetric] agg
+        # @return [Sqreen::Kit::Signals::Metric]
+        def convert_metric_sample(agg)
+          attrs = {
+            signal_name: "sq.agent.metric.#{agg.name}",
+            source: if agg.rule
+                      "sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
+                    else
+                      agent_gen_source
+                    end,
+            time: agg.finish,
+          }
+
+          if agg.metric.is_a?(Sqreen::Metric::Binning)
+            conv_binning_metric(agg, attrs)
+          else
+            conv_generic_metric(agg, attrs)
+          end
+        end
+
+        # @param [Sqreen::Attack] attack
+        # XXX: not used because we don't use Sqreen::Attack
+        def convert_attack(attack)
+          # no need to set actor/context as we only include them in request records/traces
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{attack.attack_type}",
+            source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
+            time: attack.time,
+            location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: attack.test?,
+              block: attack.block?,
+              infos: attack.infos
+            )
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_event
+        def convert_unstructured_attack(payload)
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{payload[:attack_type]}",
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: payload[:test],
+              block: payload[:block],
+              infos: payload[:infos]
+            )
+          )
+        end
+
+        # @param [Sqreen::RemoteException] exception
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_exception(exception)
+          payload = exception.payload
+
+          infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
+          infos.merge!(payload['infos'] || {})
+
+          Kit::Signals::Specialized::SqreenException.new(
+            source: if payload['rule_name']
+                      "sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
+                    else
+                      agent_gen_source
+                    end,
+            time: exception.time,
+            ruby_exception: payload['exception'],
+            infos: infos
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_exception
+        # @param [Hash] payload
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_unstructured_exception(payload)
+          Kit::Signals::Specialized::SqreenException.new(
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            ruby_exception: payload[:exception],
+            infos: payload[:infos]
+          )
+        end
+
+        # @param [Sqreen::RequestRecord] req_rec
+        # @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
+        def convert_req_record(req_rec)
+          payload = req_rec.payload
+
+          request_p = payload['request']
+          id_args = req_rec.last_identify_args
+          identifiers = id_args[0] if id_args
+          traits = id_args[1] if id_args
+
+          observed = payload[:observed] || {}
+          signals = []
+          signals += (observed[:attacks] || [])
+                     .map { |att| convert_unstructured_attack(att) }
+          signals += (observed[:sqreen_exceptions] || [])
+                     .map { |sq_exc| convert_unstructured_exception(sq_exc) }
+          signals += req_rec.processed_sdk_calls
+                            .select { |h| h[:name] == :track }
+                            .map { |h| convert_track(h) }
+          signals += (observed[:signals] || [])
+
+          trace = Kit::Signals::Specialized::HttpTrace.new(
+            actor: Kit::Signals::Actor.new(
+              ip_addresses: [request_p[:client_ip]].compact,
+              user_agent: request_p[:user_agent],
+              identifiers: identifiers,
+              traits: traits,
+            ),
+            location_infra: location_infra,
+            context: convert_request(request_p,
+                                     payload['response'],
+                                     payload['headers'],
+                                     payload['params']),
+            data: signals
+          )
+          HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
+          trace
+        end
+
+        # @return [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>]
+        def convert_batch(batch)
+          batch.map do |evt|
+            case evt
+            when RemoteException
+              convert_exception(evt)
+            when AggregatedMetric
+              convert_metric_sample(evt)
+            when RequestRecord
+              convert_req_record(evt)
+            when Sqreen::Kit::Signals::Signal
+              evt
+            when Sqreen::Kit::Signals::Trace
+              evt
+            else
+              raise NotImplementedError, "Unknown type of event in batch: #{evt}"
+            end
+          end
+        end
+
+        private
+
+        def agent_gen_source
+          "sqreen:agent:ruby:#{Sqreen::VERSION}"
+        end
+
+        def location_infra
+          @location_infra ||= begin
+            Kit::Signals::LocationInfra.new(
+              agent_version: Sqreen::VERSION,
+              os_type: RuntimeInfos.os[:os_type],
+              hostname: RuntimeInfos.hostname,
+              runtime_type: RuntimeInfos.runtime[:runtime_type],
+              runtime_version: RuntimeInfos.runtime[:runtime_version],
+              libsqreen_version: RuntimeInfos.libsqreen_version,
+            )
+          end
+        end
+
+        # see Sqreen::RequestRecord.processed_sdk_calls
+        def convert_track(call_info)
+          options = call_info[:args][1] || {}
+          Kit::Signals::Specialized::SdkTrackCall.new(
+            signal_name: "sq.sdk.#{call_info[:args][0]}",
+            time: call_info[:time],
+            payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
+              properties: options[:properties],
+              user_identifiers: options[:user_identifiers]
+            )
+          )
+        end
+
+        # @param [Hash] req_payload
+        # @param [Hash] headers_payload
+        # @param [Hash] params_payload
+        # see the PayloadCreator abomination for reference
+        # TODO: do not convert from the old payload to the new payload
+        # Have an intermediate object that gets the data from the framework.
+        # (Or convert directly from the framework, but this needs to be
+        # done during the request, not just before event is transmitted)
+        def convert_request(req_payload, resp_payload, headers_payload, params_payload)
+          req_payload ||= {}
+          headers_payload ||= {}
+          resp_payload ||= {}
+          params_payload ||= {}
+
+          other = params_payload['other']
+          other = merge_hash_append(other, params_payload['rack'])
+          other = merge_hash_append(other, params_payload['grape_params'])
+          other = merge_hash_append(other, params_payload['rack_routing'])
+
+          Sqreen::Kit::Signals::Context::HttpContext.new(
+            {
+              rid: req_payload[:rid],
+              headers: headers_payload,
+              user_agent: req_payload[:user_agent],
+              scheme: req_payload[:scheme],
+              verb: req_payload[:verb],
+              host: req_payload[:host],
+              port: req_payload[:port],
+              remote_ip: req_payload[:remote_ip],
+              remote_port: req_payload[:remote_port] || 0,
+              path: req_payload[:path],
+              referer: req_payload[:referer],
+              params_query: params_payload['query'],
+              params_form: params_payload['form'],
+              params_other: other,
+              # endpoint, is_reveal_replayed not set
+              status: resp_payload[:status],
+              content_length: resp_payload[:content_length],
+              content_type: resp_payload[:content_type],
+            }
+          )
+        end
+
+        def merge_hash_append(hash1, hash2)
+          return nil if hash1.nil? && hash2.nil?
+          return hash1 if hash2.nil? || hash2.empty?
+          return hash2 if hash1.nil? || hash1.empty?
+
+          pairs = (hash1.keys + hash2.keys).map do |key|
+            values1 = hash1[key]
+            values2 = hash2[key]
+            values = [values1, values2].compact
+            values = values.first if values.size == 1
+            [key, values]
+          end
+          Hash[pairs]
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_generic_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
+            kind: metric_kind(agg.metric),
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            values: agg.data
+          )
+
+          Kit::Signals::Specialized::AggregatedMetric.new(attrs)
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_binning_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            base: agg.data['b'],
+            unit: agg.data['u'],
+            max: agg.data['v']['max'],
+            bins: agg.data['v'].reject { |k, _v| k == 'max' }
+          )
+
+          Kit::Signals::Specialized::BinningMetric.new(attrs)
+        end
+
+        # @param [Sqreen::Metric::Base] metric
+        def metric_kind(metric)
+          metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
+        end
+      end
+    end
+  end
+end