sqreen 1.19.0.beta1 → 1.20.0

data/lib/sqreen/session.rb

@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
 require 'sqreen/events/request_record'
 require 'sqreen/exception'
 require 'sqreen/safe_json'
+require 'sqreen/kit'
+require 'sqreen/kit/configuration'
+require 'sqreen/signals/signals_submission_strategy'
+require 'sqreen/legacy/old_event_submission_strategy'
 
 require 'net/https'
 require 'uri'
@@ -41,13 +45,12 @@ module Sqreen
     RETRY_MANY = 301
 
     MUTEX = Mutex.new
-    METRICS_KEY = 'metrics'.freeze
 
     @@path_prefix = '/sqreen/v0/'
 
     attr_accessor :request_compression
 
-    def initialize(server_url, token, app_name = nil)
+    def initialize(server_url, token, app_name = nil, proxy_url = nil)
       @token = token
       @app_name = app_name
       @session_id = nil
@@ -59,16 +62,35 @@ module Sqreen
       uri = parse_uri(server_url)
       use_ssl = (uri.scheme == 'https')
 
+      proxy_params = []
+      if proxy_url
+        proxy_uri = parse_uri(proxy_url)
+        proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
+      end
+
       @req_nb = 0
 
-      @http = Net::HTTP.new(uri.host, uri.port)
+      @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
       @http.use_ssl = use_ssl
+      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
       if use_ssl
         cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
         cert_store = OpenSSL::X509::Store.new
         cert_store.add_file cert_file
         @http.cert_store = cert_store
       end
+      self.use_signals = false
+    end
+
+    def use_signals=(do_use)
+      return if do_use == @use_signals
+
+      @use_signals = do_use
+      if do_use
+        @evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
+      else
+        @evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
+      end
     end
 
     def parse_uri(uri)
@@ -235,6 +257,8 @@ module Sqreen
       end
       Sqreen.log.info 'Login success.'
       @session_id = res['session_id']
+      Kit::Configuration.session_key = @session_id
+      Kit.reset
       Sqreen.log.debug { "received session_id #{@session_id}" }
       Sqreen.logged_in = true
       res
@@ -246,20 +270,24 @@ module Sqreen
 
     def heartbeat(cmd_res = {}, metrics = [])
       payload = {}
-      payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
+      unless metrics.nil? || metrics.empty?
+        # never reached with signals
+        payload['metrics'] = metrics.map do |m|
+          Sqreen::Legacy::EventToHash.convert_agg_metric(m)
+        end
+      end
       payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
 
       post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
     end
 
     def post_metrics(metrics)
-      return if metrics.nil? || metrics.empty?
-      payload = { METRICS_KEY => metrics }
-      post(METRICS_KEY, payload, {}, RETRY_MANY)
+      @evt_sub_strategy.post_metrics(metrics)
     end
 
+    # XXX never called
     def post_attack(attack)
-      post('attack', attack.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_attack(attack)
     end
 
     def post_bundle(bundle_sig, dependencies)
@@ -271,33 +299,17 @@ module Sqreen
     end
 
     def post_request_record(request_record)
-      post('request_record', request_record.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_request_record(request_record)
     end
 
     # Post an exception to Sqreen for analysis
     # @param exception [RemoteException] Exception and context to be sent over
     def post_sqreen_exception(exception)
-      post('sqreen_exception', exception.to_hash, {}, 5)
-    rescue StandardError => e
-      Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
-                             e.inspect,
-                             exception.to_hash.inspect))
-      nil
+      @evt_sub_strategy.post_sqreen_exception(exception)
     end
 
-    BATCH_KEY = 'batch'.freeze
-    EVENT_TYPE_KEY = 'event_type'.freeze
     def post_batch(events)
-      batch = events.map do |event|
-        h = event.to_hash
-        h[EVENT_TYPE_KEY] = event_kind(event)
-        h
-      end
-      Sqreen.log.debug do
-        tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
-        "Doing batch with the following tally of event types: #{tally}"
-      end
-      post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
+      @evt_sub_strategy.post_batch(events)
     end
 
     # Perform agent logout
@@ -313,15 +325,5 @@ module Sqreen
       Sqreen.logged_in = false
       disconnect
     end
-
-    protected
-
-    def event_kind(event)
-      case event
-      when Sqreen::RemoteException then 'sqreen_exception'
-      when Sqreen::Attack then 'attack'
-      when Sqreen::RequestRecord then 'request_record'
-      end
-    end
   end
 end

data/lib/sqreen/signals/conversions.rb

@@ -0,0 +1,283 @@
+require 'sqreen/version'
+require 'sqreen/rules/rule_cb'
+require 'sqreen/metrics/base'
+require 'sqreen/metrics/binning'
+require 'sqreen/signals/http_trace_redaction'
+require 'sqreen/kit/signals/signal_attributes'
+require 'sqreen/kit/signals/specialized/aggregated_metric'
+require 'sqreen/kit/signals/specialized/attack'
+require 'sqreen/kit/signals/specialized/binning_metric'
+require 'sqreen/kit/signals/specialized/sqreen_exception'
+require 'sqreen/kit/signals/specialized/http_trace'
+require 'sqreen/kit/signals/specialized/sdk_track_call'
+
+module Sqreen
+  module Signals
+    module Conversions # rubocop:disable Metrics/ModuleLength
+      class << self
+        # @param [Sqreen::AggregatedMetric] agg
+        # @return [Sqreen::Kit::Signals::Metric]
+        def convert_metric_sample(agg)
+          attrs = {
+            signal_name: "sq.agent.metric.#{agg.name}",
+            source: if agg.rule
+                      "sqreen:rules:#{agg.rule.rulespack_id}:#{agg.rule.rule_name}"
+                    else
+                      agent_gen_source
+                    end,
+            time: agg.finish,
+          }
+
+          if agg.metric.is_a?(Sqreen::Metric::Binning)
+            conv_binning_metric(agg, attrs)
+          else
+            conv_generic_metric(agg, attrs)
+          end
+        end
+
+        # @param [Sqreen::Attack] attack
+        # XXX: not used because we don't use Sqreen::Attack
+        def convert_attack(attack)
+          # no need to set actor/context as we only include them in request records/traces
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{attack.attack_type}",
+            source: "sqreen:rule:#{attack.rulespack_id}:#{attack.rule_name}",
+            time: attack.time,
+            location: Kit::Signals::Location.new(stack_trace: attack.backtrace),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: attack.test?,
+              block: attack.block?,
+              infos: attack.infos
+            )
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_event
+        def convert_unstructured_attack(payload)
+          Kit::Signals::Specialized::Attack.new(
+            signal_name: "sq.agent.attack.#{payload[:attack_type]}",
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            location: (Kit::Signals::Location.new(stack_trace: payload[:backtrace]) if payload[:backtrace]),
+            payload: Kit::Signals::Specialized::Attack::Payload.new(
+              test: payload[:test],
+              block: payload[:block],
+              infos: payload[:infos]
+            )
+          )
+        end
+
+        # @param [Sqreen::RemoteException] exception
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_exception(exception)
+          payload = exception.payload
+
+          infos = payload['client_ip'] ? { client_ip: payload['client_ip'] } : {}
+          infos.merge!(payload['infos'] || {})
+
+          Kit::Signals::Specialized::SqreenException.new(
+            source: if payload['rule_name']
+                      "sqreen:rule:#{payload['rulespack_id']}:#{payload['rule_name']}"
+                    else
+                      agent_gen_source
+                    end,
+            time: exception.time,
+            ruby_exception: payload['exception'],
+            infos: infos
+          )
+        end
+
+        # see Sqreen::Rules::RuleCB.record_exception
+        # @param [Hash] payload
+        # @return [Sqreen::Kit::Signals::Specialized::SqreenException]
+        def convert_unstructured_exception(payload)
+          Kit::Signals::Specialized::SqreenException.new(
+            source: "sqreen:rule:#{payload[:rulespack_id]}:#{payload[:rule_name]}",
+            time: payload[:time],
+            ruby_exception: payload[:exception],
+            infos: payload[:infos]
+          )
+        end
+
+        # @param [Sqreen::RequestRecord] req_rec
+        # @return [Sqreen::Kit::Signals::Specialized::HttpTrace]
+        def convert_req_record(req_rec)
+          payload = req_rec.payload
+
+          request_p = payload['request']
+          id_args = req_rec.last_identify_args
+          identifiers = id_args[0] if id_args
+          traits = id_args[1] if id_args
+
+          observed = payload[:observed] || {}
+          signals = []
+          signals += (observed[:attacks] || [])
+                     .map { |att| convert_unstructured_attack(att) }
+          signals += (observed[:sqreen_exceptions] || [])
+                     .map { |sq_exc| convert_unstructured_exception(sq_exc) }
+          signals += req_rec.processed_sdk_calls
+                            .select { |h| h[:name] == :track }
+                            .map { |h| convert_track(h) }
+
+          trace = Kit::Signals::Specialized::HttpTrace.new(
+            actor: Kit::Signals::Actor.new(
+              ip_addresses: [request_p[:client_ip]].compact,
+              user_agent: request_p[:user_agent],
+              identifiers: identifiers,
+              traits: traits,
+            ),
+            location_infra: location_infra,
+            context: convert_request(request_p,
+                                     payload['response'],
+                                     payload['headers'],
+                                     payload['params']),
+            data: signals
+          )
+          HttpTraceRedaction.redact_trace!(trace, req_rec.redactor)
+          trace
+        end
+
+        # @param [Array<Sqreen::Kit::Signals::Signal|Sqreen::Kit::Signals::Trace>] batch
+        def convert_batch(batch)
+          batch.map do |evt|
+            case evt
+            when RemoteException
+              convert_exception(evt)
+            when AggregatedMetric
+              convert_metric_sample(evt)
+            when RequestRecord
+              convert_req_record(evt)
+            else
+              raise NotImplementedError, "Unknown type of event in batch: #{evt}"
+            end
+          end
+        end
+
+        private
+
+        def agent_gen_source
+          "sqreen:agent:ruby:#{Sqreen::VERSION}"
+        end
+
+        def location_infra
+          @location_infra ||= begin
+            Kit::Signals::LocationInfra.new(
+              agent_version: Sqreen::VERSION,
+              os_type: RuntimeInfos.os[:os_type],
+              hostname: RuntimeInfos.hostname,
+              runtime_type: RuntimeInfos.runtime[:runtime_type],
+              runtime_version: RuntimeInfos.runtime[:runtime_version],
+              libsqreen_version: RuntimeInfos.libsqreen_version,
+            )
+          end
+        end
+
+        # see Sqreen::RequestRecord.processed_sdk_calls
+        def convert_track(call_info)
+          options = call_info[:args][1] || {}
+          Kit::Signals::Specialized::SdkTrackCall.new(
+            signal_name: "sq.sdk.#{call_info[:args][0]}",
+            time: call_info[:time],
+            payload: Kit::Signals::Specialized::SdkTrackCall::Payload.new(
+              properties: options[:properties],
+              user_identifiers: options[:user_identifiers]
+            )
+          )
+        end
+
+        # @param [Hash] req_payload
+        # @param [Hash] headers_payload
+        # @param [Hash] params_payload
+        # see the PayloadCreator abomination for reference
+        # TODO: do not convert from the old payload to the new payload
+        # Have an intermediate object that gets the data from the framework.
+        # (Or convert directly from the framework, but this needs to be
+        # done during the request, not just before event is transmitted)
+        def convert_request(req_payload, resp_payload, headers_payload, params_payload)
+          req_payload ||= {}
+          headers_payload ||= {}
+          resp_payload ||= {}
+          params_payload ||= {}
+
+          other = params_payload['other']
+          other = merge_hash_append(other, params_payload['rack'])
+          other = merge_hash_append(other, params_payload['grape_params'])
+          other = merge_hash_append(other, params_payload['rack_routing'])
+
+          Sqreen::Kit::Signals::Context::HttpContext.new(
+            {
+              rid: req_payload[:rid],
+              headers: headers_payload,
+              user_agent: req_payload[:user_agent],
+              scheme: req_payload[:scheme],
+              verb: req_payload[:verb],
+              host: req_payload[:host],
+              port: req_payload[:port],
+              remote_ip: req_payload[:remote_ip],
+              remote_port: req_payload[:remote_port] || 0,
+              path: req_payload[:path],
+              referer: req_payload[:referer],
+              params_query: params_payload['query'],
+              params_form: params_payload['form'],
+              params_other: other,
+              # endpoint, is_reveal_replayed not set
+              status: resp_payload[:status],
+              content_length: resp_payload[:content_length],
+              content_type: resp_payload[:content_type],
+            }
+          )
+        end
+
+        def merge_hash_append(hash1, hash2)
+          return nil if hash1.nil? && hash2.nil?
+          return hash1 if hash2.nil? || hash2.empty?
+          return hash2 if hash1.nil? || hash1.empty?
+
+          pairs = (hash1.keys + hash2.keys).map do |key|
+            values1 = hash1[key]
+            values2 = hash2[key]
+            values = [values1, values2].compact
+            values = values.first if values.size == 1
+            [key, values]
+          end
+          Hash[pairs]
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_generic_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::AggregatedMetric::Payload.new(
+            kind: metric_kind(agg.metric),
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            values: agg.data
+          )
+
+          Kit::Signals::Specialized::AggregatedMetric.new(attrs)
+        end
+
+        # @param [Sqreen::AggregatedMetric] agg
+        # @param [Hash] attrs
+        def conv_binning_metric(agg, attrs)
+          attrs[:payload] = Kit::Signals::Specialized::BinningMetric::Payload.new(
+            capture_interval_s: agg.metric.period,
+            date_started: agg.start,
+            date_ended: agg.finish,
+            base: agg.data['b'],
+            unit: agg.data['u'],
+            max: agg.data['v']['max'],
+            bins: agg.data['v'].reject { |k, _v| k == 'max' }
+          )
+
+          Kit::Signals::Specialized::BinningMetric.new(attrs)
+        end
+
+        # @param [Sqreen::Metric::Base] metric
+        def metric_kind(metric)
+          metric.class.name.sub(/.*::/, '').sub(/Metric$/, '')
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/signals/http_trace_redaction.rb

@@ -0,0 +1,111 @@
+require 'json'
+require 'sqreen/kit/loggable'
+require 'sqreen/kit/signals/specialized/http_trace'
+
+module Sqreen
+  module Signals
+    module HttpTraceRedaction
+      class << self
+        include Sqreen::Kit::Loggable
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        def redact_trace!(trace, redactor)
+          return unless redactor
+          # redact headers (keys unsafe)
+          # @type [Sqreen::Kit::Signals::Context::HttpContext]
+          http_context = trace.context
+
+          all_redacted = []
+
+          # Redact headers; save redacted values
+          # headers are encoded as [key, value], not a hash, so
+          # they require some transformation
+          orig_headers = http_context.headers
+          if orig_headers
+            headers = orig_headers.map { |(k, v)| { k => v } }
+            headers, redacted = redactor.redact(headers)
+            http_context.headers = headers.map(&:first)
+            all_redacted += redacted
+          end
+
+          # Redact params; save redacted values
+          Kit::Signals::Context::HttpContext::PARAMS_ATTRS.each do |attr|
+            value = http_context.public_send(attr)
+            next unless value
+            value, redacted = redactor.redact(value)
+            all_redacted += redacted
+            http_context.public_send(:"#{attr}=", value)
+          end
+
+          all_redacted = all_redacted.uniq.map(&:downcase)
+
+          # Redact attacks and exceptions
+          # XXX: no redaction for infos in attacks/exceptions except for WAF data
+          # Is this the correct behavior?
+          redact_attacks!(trace, redactor, all_redacted)
+          redact_exceptions!(trace, redactor, all_redacted)
+        end
+
+        private
+
+        # @param [Sqreen::Kit::Signals::Specialized::HttpTrace] trace
+        # @param [Sqreen::SensitiveDataRedactor] redactor
+        # Redacts WAF data according to specific rules therefor
+        # Redacts infos according to general rules
+        def redact_attacks!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::Attack)
+            # @type [Sqreen::Kit::Signals::Specialized::Attack::Payload] payload
+            payload = signal.payload
+            next unless payload.infos
+
+            if payload.infos[:waf_data]
+              redact_waf_attack_data!(payload.infos, redacted_data)
+            end
+            payload.infos, = redactor.redact(payload.infos)
+          end
+        end
+
+        def redact_exceptions!(trace, redactor, redacted_data)
+          trace.data.each do |signal|
+            next unless signal.is_a?(Kit::Signals::Specialized::SqreenException)
+            infos = signal.infos
+            next unless infos
+
+            redact_waf_exception_data!(signal.infos, redacted_data) if signal.infos[:waf]
+            signal.infos, = redactor.redact(infos)
+          end
+        end
+
+        # @param [Hash] infos from WAF attack
+        def redact_waf_attack_data!(infos, redacted_data)
+          begin
+            parsed = JSON.parse(infos[:waf_data])
+          rescue JSON::JSONError => e
+            logger.warn("waf_data is not valid json: #{e.message}")
+            return
+          end
+          redacted = parsed.each do |w|
+            next unless (filters = w['filter'])
+
+            filters.each do |f|
+              next unless (v = f['resolved_value'])
+              next unless redacted_data.include?(v.downcase)
+
+              f['match_status'] = SensitiveDataRedactor::MASK
+              f['resolved_value'] = SensitiveDataRedactor::MASK
+            end
+          end
+          infos[:waf_data] = JSON.dump(redacted)
+        end
+
+        # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000022-waf-data-sanitization.md#changes-to-the-agents
+        def redact_waf_exception_data!(infos, redacted_data)
+          return if redacted_data.empty?
+          infos[:waf].delete(:args)
+        end
+      end
+    end
+  end
+end