sqreen 1.19.0 → 1.20.1

data/lib/sqreen/rules/not_found_cb.rb

@@ -24,6 +24,8 @@ module Sqreen
         exception   = env['action_dispatch.exception']
 
         record_from_env(ua, script_name, path_info, verb, override, host, exception)
+
+        nil
       end
 
       def record_from_env(ua, script_name, path_info, verb, override, host, exception)

data/lib/sqreen/rules/rule_cb.rb

@@ -61,7 +61,9 @@ module Sqreen
           :infos => infos,
           :rulespack_id => rulespack_id,
           :rule_name => rule_name,
+          :attack_type => @rule['attack_type'], # for signal
           :test => test,
+          :block => @rule['block'], # for signal
           :time => at,
         }
         if payload_tpl.include?('context')

data/lib/sqreen/rules/waf_cb.rb

@@ -98,10 +98,10 @@ module Sqreen
 
         case action
         when :monitor
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(nil)
         when :block
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(:raise)
         when :good
           advise_action(nil)
@@ -132,20 +132,23 @@ module Sqreen
       end
 
       def record_exception(exception, infos = {}, at = Time.now.utc)
-        infos.merge!(exception_to_infos(exception)) if exception.is_a?(Sqreen::WAFError)
+        infos.merge!(waf_infos(exception)) if exception.is_a?(Sqreen::WAFError)
         super(exception, infos, at)
       end
 
       private
 
-      def exception_to_infos(e)
+      # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000016-waf-integration.md#error-management
+      def waf_infos(e)
         {
-          waf_rule: e.rule_name,
-          error_code: ERROR_CODES[e.error],
-        }.tap do |r|
-          r[:error_data] = e.data if e.data
-          r[:args] = e.args if e.args
-        end
+          waf:  {
+            waf_rule: e.rule_name,
+            error_code: ERROR_CODES[e.error],
+          }.tap do |r|
+            r[:error_data] = e.data if e.data
+            r[:args] = e.args if e.arg
+          end,
+        }
       end
 
       ERROR_CODES = {

data/lib/sqreen/runner.rb

@@ -11,6 +11,7 @@ require 'sqreen/events/attack'
 
 require 'sqreen/log'
 
+require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
 require 'sqreen/remote_command'
@@ -18,11 +19,13 @@ require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
 require 'sqreen/deliveries/simple'
 require 'sqreen/deliveries/batch'
+require 'sqreen/endpoint_testing'
 require 'sqreen/performance_notifications/metrics'
 require 'sqreen/performance_notifications/binned_metrics'
 require 'sqreen/legacy/instrumentation'
 require 'sqreen/call_countable'
 require 'sqreen/weave/legacy/instrumentation'
+require 'sqreen/kit/configuration'
 
 module Sqreen
   @features = {}
@@ -37,6 +40,8 @@ module Sqreen
   PERF_METRICS_PERIOD = 60 # 1 min
   DEFAULT_PERF_LEVEL = 0 # disabled
 
+  DEFAULT_USE_SIGNALS = false
+
   class << self
     attr_reader :features
     def update_features(features)
@@ -87,7 +92,9 @@ module Sqreen
 
     attr_accessor :heartbeat_delay
     attr_accessor :metrics_engine
+    # @return [Sqreen::Deliveries::Simple]
     attr_reader :deliverer
+    # @return [Sqreen::Session]
     attr_reader :session
     attr_reader :instrumenter
     attr_accessor :running
@@ -108,15 +115,24 @@ module Sqreen
       @next_metrics = []
       @running = true
 
+      @proxy_url = @configuration.get(:proxy_url)
+      chosen_endpoints = determine_endpoints
+
       @token = @configuration.get(:token)
       @app_name = @configuration.get(:app_name)
-      @url = @configuration.get(:url)
+      @url = chosen_endpoints.control.url
+      @cert_store = chosen_endpoints.control.ca_store
+
       Sqreen.update_whitelisted_paths([])
       Sqreen.update_whitelisted_ips({})
       Sqreen.update_performance_budget(nil)
-      raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
+      Sqreen::Kit::Configuration.logger = Sqreen.log
+      Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
+      Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
+      Sqreen::Kit::Configuration.proxy_url = @proxy_url
+
       register_exit_cb if set_at_exit
 
       self.metrics_engine = MetricsStore.new
@@ -133,6 +149,7 @@ module Sqreen
 
       Sqreen.log.debug "Using token #{@token}"
       response = create_session(session_class)
+      post_endpoint_testing_msgs(chosen_endpoints)
       wanted_features = response.fetch('features', {})
       conf_initial_features = configuration.get(:initial_features)
       unless conf_initial_features.nil?
@@ -142,10 +159,10 @@ module Sqreen
           Sqreen.log.debug do
             "Override initial features with #{conf_features.inspect}"
           end
-          wanted_features = conf_features
+          wanted_features = wanted_features.merge(conf_features)
         rescue
           Sqreen.log.warn do
-            "NOT using invalid inital features #{conf_initial_features}"
+            "NOT using invalid initial features #{conf_initial_features}"
           end
         end
       end
@@ -161,7 +178,7 @@ module Sqreen
     end
 
     def create_session(session_class)
-      @session = session_class.new(@url, @token, @app_name)
+      @session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
       session.login(@framework)
     end
 
@@ -170,8 +187,18 @@ module Sqreen
       @deliverer = new_deliverer
     end
 
-    def batch_events(batch_size, max_staleness = nil)
+    def batch_events(batch_size, max_staleness = nil, use_signals = false)
       size = batch_size.to_i
+
+      if size <= 1 && use_signals
+        Sqreen.log.warn do
+          "Using signals with no delivery batching is unsupported. " \
+          "Using instead batching with batch size = 30, max_staleness = 60"
+        end
+        size = 30
+        max_staleness = 60
+      end
+
       self.deliverer = if size < 1
                          Deliveries::Simple.new(session)
                        else
@@ -301,19 +328,37 @@ module Sqreen
     def do_heartbeat
       @last_heartbeat_request = Time.now
       @next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
-      res = session.heartbeat(next_command_results, next_metrics)
+      metrics_in_hb = use_signals? ? nil : next_metrics
+
+      res = session.heartbeat(next_command_results, metrics_in_hb)
       next_command_results.clear
+
+      deliver_metrics_as_event if use_signals?
       next_metrics.clear
+
       process_commands(res['commands'])
     end
 
+    def deliver_metrics_as_event
+      # this is disastrous withe simple delivery strategy,
+      # as each aggregated metric would trigger an http request
+      # Sending of metrics is therefore not supported with simple delivery strategy
+      # TODO: Confirm that only batch is used in production
+      next_metrics.each { |x| deliverer.post_event(x) }
+    end
+
     def features(_context_infos = {})
       Sqreen.features
     end
 
+    def use_signals?
+      features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+    end
+
     def features=(features)
       Sqreen.update_features(features)
       session.request_compression = features['request_compression'] if session
+      session.use_signals = use_signals?
       self.performance_metrics_period = features['performance_metrics_period']
 
       unless @configuration.get(:weave)
@@ -331,7 +376,7 @@ module Sqreen
       hd = features['heartbeat_delay'].to_i
       self.heartbeat_delay = hd if hd > 0
       return if features['batch_size'].nil?
-      batch_events(features['batch_size'], features['max_staleness'])
+      batch_events(features['batch_size'], features['max_staleness'], use_signals?)
     end
 
     def change_whitelisted_paths(paths, _context_infos = {})
@@ -470,6 +515,28 @@ module Sqreen
 
     private
 
+    def post_endpoint_testing_msgs(chosen_endpoints)
+      chosen_endpoints.messages.each do |msg|
+        session.post_agent_message(@framework, msg)
+      end
+    rescue => e
+      Sqreen.log.warn "Error submitting agent message: #{e}"
+      RemoteException.record(e)
+    end
+
+    def determine_endpoints
+      # there's no sniffing going on; just a misnamed config setting
+      if @configuration.get(:no_sniff_domains)
+        # reproduces behaviour before endpoint testing was introduced
+        EndpointTesting.no_test_endpoints(@configuration.get(:url),
+                                          @configuration.get(:ingestion_url))
+      else
+        EndpointTesting.test_endpoints(@proxy_url,
+                                       @configuration.get(:url),
+                                       @configuration.get(:ingestion_url))
+      end
+    end
+
     def load_actions(hashes)
       unsupported = Set.new
 

data/lib/sqreen/sensitive_data_redactor.rb

@@ -61,7 +61,7 @@ module Sqreen
         obj.each do |k, v|
           ck = k.is_a?(String) ? k.downcase : k
           if @keys.include?(ck)
-            redacted << v
+            redacted += SensitiveDataRedactor.all_strings(v)
             v = MASK
           else
             v, r = redact(v)
@@ -74,39 +74,27 @@ module Sqreen
       [result, redacted]
     end
 
-    def redact_attacks!(attacks, values)
-      return attacks if values.empty?
-
-      values = values.map { |v| v.downcase if v.is_a?(String) }
-
-      attacks.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf_data]
-
-        parsed = JSON.parse(e[:infos][:waf_data])
-        redacted = parsed.each do |w|
-          next unless (filters = w['filter'])
-
-          filters.each do |f|
-            next unless (v = f['resolved_value'])
-            next unless values.include?(v.downcase)
+    class << self
+      def all_strings(v)
+        accum = []
+        all_strings_impl(v, accum)
+        accum
+      end
 
-            f['match_status'] = MASK
-            f['resolved_value'] = MASK
+      private
+
+      def all_strings_impl(obj, accum)
+        case obj
+        when String
+          accum << obj
+        when Array
+          obj.each { |el| all_strings_impl(el, accum) }
+        when Hash
+          obj.each do |k, v|
+            all_strings_impl(k, accum)
+            all_strings_impl(v, accum)
           end
         end
-        e[:infos][:waf_data] = JSON.dump(redacted)
-      end
-    end
-
-    def redact_exceptions!(exceptions, values)
-      return exceptions if values.empty?
-
-      exceptions.each do |e|
-        next(e) unless e[:infos]
-        next(e) unless e[:infos][:waf]
-
-        e[:infos][:waf].delete(:args)
       end
     end
   end

data/lib/sqreen/session.rb

@@ -11,6 +11,10 @@ require 'sqreen/events/attack'
 require 'sqreen/events/request_record'
 require 'sqreen/exception'
 require 'sqreen/safe_json'
+require 'sqreen/kit'
+require 'sqreen/kit/configuration'
+require 'sqreen/signals/signals_submission_strategy'
+require 'sqreen/legacy/old_event_submission_strategy'
 
 require 'net/https'
 require 'uri'
@@ -41,13 +45,12 @@ module Sqreen
     RETRY_MANY = 301
 
     MUTEX = Mutex.new
-    METRICS_KEY = 'metrics'.freeze
 
     @@path_prefix = '/sqreen/v0/'
 
     attr_accessor :request_compression
 
-    def initialize(server_url, token, app_name = nil)
+    def initialize(server_url, cert_store, token, app_name = nil, proxy_url = nil)
       @token = token
       @app_name = app_name
       @session_id = nil
@@ -59,15 +62,29 @@ module Sqreen
       uri = parse_uri(server_url)
       use_ssl = (uri.scheme == 'https')
 
+      proxy_params = []
+      if proxy_url
+        proxy_uri = parse_uri(proxy_url)
+        proxy_params = [proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
+      end
+
       @req_nb = 0
 
-      @http = Net::HTTP.new(uri.host, uri.port)
+      @http = Net::HTTP.new(uri.host, uri.port, *proxy_params)
       @http.use_ssl = use_ssl
-      if use_ssl
-        cert_file = File.join(File.dirname(__FILE__), 'ca.crt')
-        cert_store = OpenSSL::X509::Store.new
-        cert_store.add_file cert_file
-        @http.cert_store = cert_store
+      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE if ENV['SQREEN_SSL_NO_VERIFY'] # for testing
+      @http.cert_store = cert_store if use_ssl
+      self.use_signals = false
+    end
+
+    def use_signals=(do_use)
+      return if do_use == @use_signals
+
+      @use_signals = do_use
+      if do_use
+        @evt_sub_strategy = Sqreen::Signals::SignalsSubmissionStrategy.new
+      else
+        @evt_sub_strategy = Sqreen::Legacy::OldEventSubmissionStrategy.new(method(:post))
       end
     end
 
@@ -218,10 +235,7 @@ module Sqreen
     end
 
     def login(framework)
-      headers = {
-        'x-api-key'  => @token,
-        'x-app-name' => @app_name || framework.application_name,
-      }.reject { |k, v| v ==  nil }
+      headers = prelogin_auth_headers(framework)
 
       Sqreen.log.warn "Using app name: #{headers['x-app-name']}"
 
@@ -235,6 +249,8 @@ module Sqreen
       end
       Sqreen.log.info 'Login success.'
       @session_id = res['session_id']
+      Kit::Configuration.session_key = @session_id
+      Kit.reset
       Sqreen.log.debug { "received session_id #{@session_id}" }
       Sqreen.logged_in = true
       res
@@ -246,20 +262,24 @@ module Sqreen
 
     def heartbeat(cmd_res = {}, metrics = [])
       payload = {}
-      payload['metrics'] = metrics unless metrics.nil? || metrics.empty?
+      unless metrics.nil? || metrics.empty?
+        # never reached with signals
+        payload['metrics'] = metrics.map do |m|
+          Sqreen::Legacy::EventToHash.convert_agg_metric(m)
+        end
+      end
       payload['command_results'] = cmd_res unless cmd_res.nil? || cmd_res.empty?
 
       post('app-beat', payload.empty? ? nil : payload, {}, RETRY_MANY)
     end
 
     def post_metrics(metrics)
-      return if metrics.nil? || metrics.empty?
-      payload = { METRICS_KEY => metrics }
-      post(METRICS_KEY, payload, {}, RETRY_MANY)
+      @evt_sub_strategy.post_metrics(metrics)
     end
 
+    # XXX never called
     def post_attack(attack)
-      post('attack', attack.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_attack(attack)
     end
 
     def post_bundle(bundle_sig, dependencies)
@@ -271,33 +291,22 @@ module Sqreen
     end
 
     def post_request_record(request_record)
-      post('request_record', request_record.to_hash, {}, RETRY_MANY)
+      @evt_sub_strategy.post_request_record(request_record)
     end
 
     # Post an exception to Sqreen for analysis
     # @param exception [RemoteException] Exception and context to be sent over
     def post_sqreen_exception(exception)
-      post('sqreen_exception', exception.to_hash, {}, 5)
-    rescue StandardError => e
-      Sqreen.log.warn(format('Could not post exception (network down? %s) %s',
-                             e.inspect,
-                             exception.to_hash.inspect))
-      nil
+      @evt_sub_strategy.post_sqreen_exception(exception)
     end
 
-    BATCH_KEY = 'batch'.freeze
-    EVENT_TYPE_KEY = 'event_type'.freeze
     def post_batch(events)
-      batch = events.map do |event|
-        h = event.to_hash
-        h[EVENT_TYPE_KEY] = event_kind(event)
-        h
-      end
-      Sqreen.log.debug do
-        tally = Hash[events.group_by(&:class).map{ |k,v| [k, v.count] }]
-        "Doing batch with the following tally of event types: #{tally}"
-      end
-      post(BATCH_KEY, { BATCH_KEY => batch }, {}, RETRY_MANY)
+      @evt_sub_strategy.post_batch(events)
+    end
+
+    def post_agent_message(framework, agent_message)
+      headers = prelogin_auth_headers(framework)
+      post('app_agent_message', agent_message.to_h, headers, 0)
     end
 
     # Perform agent logout
@@ -314,14 +323,13 @@ module Sqreen
       disconnect
     end
 
-    protected
+    private
 
-    def event_kind(event)
-      case event
-      when Sqreen::RemoteException then 'sqreen_exception'
-      when Sqreen::Attack then 'attack'
-      when Sqreen::RequestRecord then 'request_record'
-      end
+    def prelogin_auth_headers(framework)
+      {
+        'x-api-key' => @token,
+        'x-app-name' => @app_name || framework.application_name,
+      }.reject { |_k, v| v == nil }
     end
   end
 end