sqreen 1.19.1-java → 1.21.0.beta3-java

data/lib/sqreen/null_logger.rb

@@ -9,6 +9,26 @@ module Sqreen
   class NullLogger
     include Singleton
 
+    def debug?
+      false
+    end
+
+    def info?
+      false
+    end
+
+    def warn?
+      false
+    end
+
+    def error?
+      false
+    end
+
+    def fatal?
+      false
+    end
+
     def debug(_msg = nil); end
 
     def info(_msg = nil); end
@@ -19,6 +39,8 @@ module Sqreen
 
     def fatal(_msg = nil); end
 
+    def unknown(_msg = nil); end
+
     def add(_severity, _msg = nil); end
 
     def formatter=(_); end

data/lib/sqreen/performance_notifications/binned_metrics.rb

@@ -122,10 +122,16 @@ module Sqreen
       attr_reader :metrics_store
       attr_reader :period
 
-      def ensure_metric(metric_name)
+      def ensure_metric(metric_name, rule = nil)
         return if metrics_store.metric?(metric_name)
         metrics_store.create_metric(
-          'name' => metric_name, 'period' => period, 'kind' => 'Binning', 'options' => @perf_metric_opts
+          {
+            'name' => metric_name,
+            'period' => period,
+            'kind' => 'Binning',
+            'options' => @perf_metric_opts,
+          },
+          rule
         )
       end
 

data/lib/sqreen/remote_command.rb

@@ -18,10 +18,12 @@ module Sqreen
       :features_get => :features,
       :features_change => :change_features,
       :force_logout => :shutdown,
+      :force_restart => :restart,
       :paths_whitelist => :change_whitelisted_paths,
       :ips_whitelist => :change_whitelisted_ips,
       :get_bundle => :upload_bundle,
       :performance_budget => :change_performance_budget,
+      :tracing_enable => :tracing_enable,
     }.freeze
 
     attr_reader :uuid
@@ -39,6 +41,8 @@ module Sqreen
       begin
         output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
       rescue => e
+        Sqreen.log.warn { "Command failed with #{e}" }
+        Sqreen.log.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
         Sqreen::RemoteException.record(e)
         return { :status => false, :reason => "error: #{e.inspect}" }
       end

data/lib/sqreen/rules.rb

@@ -114,15 +114,19 @@ module Sqreen
           Sqreen.log.warn('No JavaScript engine is available. ' \
               'JavaScript callbacks will be ignored')
         end
-        Sqreen.log.info("Ignoring JS callback #{rule_name}")
+        Sqreen.log.debug("Ignoring JS callback #{rule_name}")
         return nil
       end
 
       cb_class = ExecJSCB if js
 
-      if cbname && Rules.const_defined?(cbname)
-        # Only load callbacks from sqreen
-        cb_class = Rules.const_get(cbname)
+      if cbname
+        cb_class = if cbname.include?('::')
+                     # Only load callbacks from sqreen
+                     Rules.walk_const_get(cbname) if cbname.start_with?('::Sqreen::', 'Sqreen::')
+                   else
+                     Rules.const_get(cbname) if Rules.const_defined?(cbname) # rubocop:disable Style/IfInsideElse
+                   end
       end
 
       if cb_class.nil?
@@ -135,13 +139,15 @@ module Sqreen
         return nil
       end
 
+      rule_cb = cb_class.new(instr_class, instr_method, hash_rule)
+
       if metrics_store
         (hash_rule[Attrs::METRICS] || []).each do |metric|
-          metrics_store.create_metric(metric)
+          metrics_store.create_metric(metric, rule_cb)
         end
       end
 
-      cb_class.new(instr_class, instr_method, hash_rule)
+      rule_cb
     rescue => e
       rule_name = nil
       rulespack_id = nil

data/lib/sqreen/rules/blacklist_ips_cb.rb

@@ -33,7 +33,7 @@ module Sqreen
       private
 
       def insert_values(ranges)
-        Sqreen.log.info 'no ips given for IP blacklisting' if ranges.empty?
+        Sqreen.log.debug 'no ips given for IP blacklisting' if ranges.empty?
 
         ranges.map { |r| Prefix.from_str(r, r) }.each do |prefix|
           trie_for(prefix).insert prefix
@@ -50,7 +50,7 @@ module Sqreen
         begin
           ipa = IPAddr.new(rip)
         rescue StandardError
-          Sqreen.log.info "invalid IP address given by framework: #{rip}"
+          Sqreen.log.debug "invalid IP address given by framework: #{rip}"
           return nil
         end
 

data/lib/sqreen/rules/custom_error_cb.rb

@@ -55,12 +55,12 @@ module Sqreen
       end
 
       def respond_page
-        page = open(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        @page ||= File.read(File.join(File.dirname(__FILE__), '../attack_detected.html'))
         headers = {
           'Content-Type' => 'text/html',
-          'Content-Length' => page.size.to_s,
+          'Content-Length' => @page.size.to_s,
         }
-        [@status_code, headers, page]
+        [@status_code, headers, [@page]]
       end
     end
   end

data/lib/sqreen/rules/not_found_cb.rb

@@ -24,6 +24,8 @@ module Sqreen
         exception   = env['action_dispatch.exception']
 
         record_from_env(ua, script_name, path_info, verb, override, host, exception)
+
+        nil
       end
 
       def record_from_env(ua, script_name, path_info, verb, override, host, exception)

data/lib/sqreen/rules/rule_cb.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/deprecation'
 require 'sqreen/framework_cb'
 require 'sqreen/context'
 require 'sqreen/conditionable'
@@ -61,7 +62,9 @@ module Sqreen
           :infos => infos,
           :rulespack_id => rulespack_id,
           :rule_name => rule_name,
+          :attack_type => @rule['attack_type'], # for signal
           :test => test,
+          :block => @rule['block'], # for signal
           :time => at,
         }
         if payload_tpl.include?('context')
@@ -87,9 +90,9 @@ module Sqreen
         framework.observe(:sqreen_exceptions, payload)
       end
 
-      # Recommend taking an action (optionnally adding more data/context)
+      # Recommend taking an action (optionally adding more data/context)
       #
-      # This will format the requested action and optionnally
+      # This will format the requested action and optionally
       # override it if it should not be taken (should not block for example)
       def advise_action(action, additional_data = {})
         return if action.nil? && additional_data.empty?
@@ -107,6 +110,7 @@ module Sqreen
         )
         true
       end
+      Sqreen::Deprecation.deprecate(instance_method(:overtime!))
     end
   end
 end

data/lib/sqreen/rules/waf_cb.rb

@@ -11,7 +11,7 @@ require 'sqreen/safe_json'
 require 'sqreen/exception'
 require 'sqreen/util/capper'
 require 'sqreen/dependency/libsqreen'
-require 'sqreen/encoding_sanitizer'
+require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Rules
@@ -60,7 +60,7 @@ module Sqreen
         end
 
         # 0 for using defaults (PW_RUN_TIMEOUT)
-        @max_run_budget_us = (@data['values'].fetch('budget_in_ms', 0) * 1000).to_i
+        @max_run_budget_us = (@data['values'].fetch('max_budget_ms', 0) * 1000).to_i
         @max_run_budget_us = INFINITE_BUDGET_US if @max_run_budget_us >= INFINITE_BUDGET_US
 
         Sqreen.log.debug { "Max WAF run budget for #{@waf_rule_name} set to #{@max_run_budget_us} us" }
@@ -82,7 +82,7 @@ module Sqreen
         waf_args = binding_accessors.each_with_object({}) do |(e, b), h|
           h[e] = capper.call(b.resolve(*env))
         end
-        waf_args = Sqreen::EncodingSanitizer.sanitize(waf_args)
+        waf_args = Sqreen::Kit::StringSanitizer.sanitize(waf_args)
 
         if budget
           rem_budget_s = budget - (Sqreen.time - start)
@@ -98,10 +98,10 @@ module Sqreen
 
         case action
         when :monitor
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(nil)
         when :block
-          record_event({ 'waf_data' => data })
+          record_event({ waf_data: data })
           advise_action(:raise)
         when :good
           advise_action(nil)
@@ -132,20 +132,23 @@ module Sqreen
       end
 
       def record_exception(exception, infos = {}, at = Time.now.utc)
-        infos.merge!(exception_to_infos(exception)) if exception.is_a?(Sqreen::WAFError)
+        infos.merge!(waf_infos(exception)) if exception.is_a?(Sqreen::WAFError)
         super(exception, infos, at)
       end
 
       private
 
-      def exception_to_infos(e)
+      # see https://github.com/sqreen/TechDoc/blob/master/content/specs/spec000016-waf-integration.md#error-management
+      def waf_infos(e)
         {
-          waf_rule: e.rule_name,
-          error_code: ERROR_CODES[e.error],
-        }.tap do |r|
-          r[:error_data] = e.data if e.data
-          r[:args] = e.args if e.args
-        end
+          waf:  {
+            waf_rule: e.rule_name,
+            error_code: ERROR_CODES[e.error],
+          }.tap do |r|
+            r[:error_data] = e.data if e.data
+            r[:args] = e.args if e.arg
+          end,
+        }
       end
 
       ERROR_CODES = {

data/lib/sqreen/runner.rb

@@ -11,18 +11,23 @@ require 'sqreen/events/attack'
 
 require 'sqreen/log'
 
+require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
+require 'sqreen/version'
 require 'sqreen/remote_command'
 require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
 require 'sqreen/deliveries/simple'
 require 'sqreen/deliveries/batch'
+require 'sqreen/endpoint_testing'
 require 'sqreen/performance_notifications/metrics'
 require 'sqreen/performance_notifications/binned_metrics'
 require 'sqreen/legacy/instrumentation'
 require 'sqreen/call_countable'
 require 'sqreen/weave/legacy/instrumentation'
+require 'sqreen/kit/configuration'
+require 'sqreen/ecosystem_integration'
 
 module Sqreen
   @features = {}
@@ -37,6 +42,8 @@ module Sqreen
   PERF_METRICS_PERIOD = 60 # 1 min
   DEFAULT_PERF_LEVEL = 0 # disabled
 
+  DEFAULT_USE_SIGNALS = false
+
   class << self
     attr_reader :features
     def update_features(features)
@@ -47,10 +54,6 @@ module Sqreen
       @queue ||= CappedQueue.new(MAX_QUEUE_LENGTH)
     end
 
-    def update_queue(queue)
-      @queue = queue
-    end
-
     def observations_queue
       @observations_queue ||= CappedQueue.new(MAX_OBS_QUEUE_LENGTH)
     end
@@ -87,7 +90,9 @@ module Sqreen
 
     attr_accessor :heartbeat_delay
     attr_accessor :metrics_engine
+    # @return [Sqreen::Deliveries::Simple]
     attr_reader :deliverer
+    # @return [Sqreen::Session]
     attr_reader :session
     attr_reader :instrumenter
     attr_accessor :running
@@ -97,8 +102,8 @@ module Sqreen
     # we may want to do that in a thread in order to prevent delaying app
     # startup
     # set_at_exit do not place a global at_exit (used for testing)
+    # @param [Sqreen::Frameworks::GenericFramework] framework
     def initialize(configuration, framework, set_at_exit = true, session_class = Sqreen::Session)
-      Sqreen.update_queue(CappedQueue.new(MAX_QUEUE_LENGTH))
       @logged_out_tried = false
       @configuration = configuration
       @framework = framework
@@ -108,15 +113,25 @@ module Sqreen
       @next_metrics = []
       @running = true
 
+      @proxy_url = @configuration.get(:proxy_url)
+      chosen_endpoints = determine_endpoints
+
       @token = @configuration.get(:token)
       @app_name = @configuration.get(:app_name)
-      @url = @configuration.get(:url)
+      @url = chosen_endpoints.control.url
+      @cert_store = chosen_endpoints.control.ca_store
+
       Sqreen.update_whitelisted_paths([])
       Sqreen.update_whitelisted_ips({})
       Sqreen.update_performance_budget(nil)
-      raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
+      Sqreen::Kit::Configuration.logger = Sqreen.log
+      Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
+      Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
+      Sqreen::Kit::Configuration.proxy_url = @proxy_url
+      Sqreen::Kit::Configuration.default_source = "sqreen:agent:ruby:#{Sqreen::VERSION}"
+
       register_exit_cb if set_at_exit
 
       self.metrics_engine = MetricsStore.new
@@ -126,13 +141,19 @@ module Sqreen
       end
 
       if @configuration.get(:weave) || needs_weave.call
-        @instrumenter = Sqreen::Weave::Legacy::Instrumentation.new(metrics_engine)
+        # XXX: don't get updated
+        opts = {
+          perf_req_metrics_max_reqs: Sqreen.features['perf_req_metrics_max_reqs'],
+          perf_req_metrics_period: Sqreen.features['perf_req_metrics_period'],
+        }
+        @instrumenter = Sqreen::Weave::Legacy::Instrumentation.new(metrics_engine, opts)
       else
         @instrumenter = Sqreen::Legacy::Instrumentation.new(metrics_engine)
       end
 
       Sqreen.log.debug "Using token #{@token}"
       response = create_session(session_class)
+      post_endpoint_testing_msgs(chosen_endpoints)
       wanted_features = response.fetch('features', {})
       conf_initial_features = configuration.get(:initial_features)
       unless conf_initial_features.nil?
@@ -142,15 +163,21 @@ module Sqreen
           Sqreen.log.debug do
             "Override initial features with #{conf_features.inspect}"
           end
-          wanted_features = conf_features
+          wanted_features = wanted_features.merge(conf_features)
         rescue
           Sqreen.log.warn do
-            "NOT using invalid inital features #{conf_initial_features}"
+            "NOT using invalid initial features #{conf_initial_features}"
           end
         end
       end
       self.features = wanted_features
 
+      @ecosystem_integration = EcosystemIntegration.new(framework,
+                                                        Sqreen.queue,
+                                                        create_binning_metric_proc)
+      framework.req_start_cb = @ecosystem_integration.method(:request_start)
+      framework.req_end_cb = @ecosystem_integration.method(:request_end)
+
       # Ensure a deliverer is there unless features have set it first
       self.deliverer ||= Deliveries::Simple.new(session)
       context_infos = {}
@@ -161,7 +188,7 @@ module Sqreen
     end
 
     def create_session(session_class)
-      @session = session_class.new(@url, @token, @app_name)
+      @session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
       session.login(@framework)
     end
 
@@ -170,8 +197,18 @@ module Sqreen
       @deliverer = new_deliverer
     end
 
-    def batch_events(batch_size, max_staleness = nil)
+    def batch_events(batch_size, max_staleness = nil, use_signals = false)
       size = batch_size.to_i
+
+      if size <= 1 && use_signals
+        Sqreen.log.warn do
+          "Using signals with no delivery batching is unsupported. " \
+          "Using instead batching with batch size = 30, max_staleness = 60"
+        end
+        size = 30
+        max_staleness = 60
+      end
+
       self.deliverer = if size < 1
                          Deliveries::Simple.new(session)
                        else
@@ -241,6 +278,10 @@ module Sqreen
       rulespack_id, rules = load_rules(context_infos)
       @framework.instrument_when_ready!(instrumenter, rules)
       Sqreen.log.info 'Instrumentation set up'
+
+      # XXX: ecosystem instrumentation should likely be deferred
+      #      the same way the rest might be
+      @ecosystem_integration.init unless Sqreen.features['disable_ecosystem']
       rulespack_id.to_s
     end
 
@@ -301,19 +342,37 @@ module Sqreen
     def do_heartbeat
       @last_heartbeat_request = Time.now
       @next_metrics.concat(metrics_engine.publish(false)) if metrics_engine
-      res = session.heartbeat(next_command_results, next_metrics)
+      metrics_in_hb = use_signals? ? nil : next_metrics
+
+      res = session.heartbeat(next_command_results, metrics_in_hb)
       next_command_results.clear
+
+      deliver_metrics_as_event if use_signals?
       next_metrics.clear
+
       process_commands(res['commands'])
     end
 
+    def deliver_metrics_as_event
+      # this is disastrous withe simple delivery strategy,
+      # as each aggregated metric would trigger an http request
+      # Sending of metrics is therefore not supported with simple delivery strategy
+      # TODO: Confirm that only batch is used in production
+      next_metrics.each { |x| deliverer.post_event(x) }
+    end
+
     def features(_context_infos = {})
       Sqreen.features
     end
 
+    def use_signals?
+      features.fetch('use_signals', DEFAULT_USE_SIGNALS)
+    end
+
     def features=(features)
       Sqreen.update_features(features)
       session.request_compression = features['request_compression'] if session
+      session.use_signals = use_signals?
       self.performance_metrics_period = features['performance_metrics_period']
 
       unless @configuration.get(:weave)
@@ -331,7 +390,7 @@ module Sqreen
       hd = features['heartbeat_delay'].to_i
       self.heartbeat_delay = hd if hd > 0
       return if features['batch_size'].nil?
-      batch_events(features['batch_size'], features['max_staleness'])
+      batch_events(features['batch_size'], features['max_staleness'], use_signals?)
     end
 
     def change_whitelisted_paths(paths, _context_infos = {})
@@ -342,11 +401,28 @@ module Sqreen
 
     def change_performance_budget(budget, _context_infos = {})
       return false unless budget.nil? || budget.to_f > 0
-      prev = Sqreen.performance_budget
-      Sqreen.update_performance_budget(budget)
+
+      if @configuration.get(:weave)
+        prev = Sqreen::Weave::Budget.current
+        prev = prev.to_h if prev
+
+        budget_s = budget.to_f / 1000.0 if budget
+        Sqreen::Weave::Budget.update(threshold: budget_s)
+      else
+        prev = Sqreen.performance_budget
+        Sqreen.update_performance_budget(budget)
+      end
+
       { :was => prev }
     end
 
+    # @param [String] tracing_id_prefix
+    # @param [Array<Hash{String=>Object}>] sampling_config
+    def tracing_enable(tracing_id_prefix, sampling_config, _context_infos = {})
+      @ecosystem_integration.handle_tracing_command(tracing_id_prefix, sampling_config)
+      { status: true }
+    end
+
     def upload_bundle(_context_infos = {})
       t = Time.now
       session.post_bundle(RuntimeInfos.dependencies_signature, RuntimeInfos.dependencies)
@@ -433,6 +509,15 @@ module Sqreen
       logout
     end
 
+    def restart(_context_infos = {})
+      shutdown
+      heartbeat_delay = @heartbeat_delay
+      Thread.new do
+        sleep(2 * heartbeat_delay)
+        Sqreen::Worker.start(Sqreen.framework)
+      end
+    end
+
     def logout(retrying = true)
       return unless session
       Sqreen.log.debug("Logging out")
@@ -470,6 +555,43 @@ module Sqreen
 
     private
 
+    def create_binning_metric_proc
+      lambda do |metric_name|
+        return if @metrics_engine.metric?(metric_name)
+        metrics_engine.create_metric(
+          'name' => metric_name,
+          'kind' => 'Binning',
+          'period' => Sqreen.features['performance_metrics_period'] || 60,
+          'options' => {
+            'base' => Sqreen.features['perf_base'] || PerformanceNotifications::BinnedMetrics::DEFAULT_PERF_BASE,
+            'factor' => Sqreen.features['perf_unit'] || PerformanceNotifications::BinnedMetrics::DEFAULT_PERF_UNIT,
+          },
+        )
+      end
+    end
+
+    def post_endpoint_testing_msgs(chosen_endpoints)
+      chosen_endpoints.messages.each do |msg|
+        session.post_agent_message(@framework, msg)
+      end
+    rescue => e
+      Sqreen.log.warn "Error submitting agent message: #{e}"
+      RemoteException.record(e)
+    end
+
+    def determine_endpoints
+      # there's no sniffing going on; just a misnamed config setting
+      if @configuration.get(:no_sniff_domains)
+        # reproduces behaviour before endpoint testing was introduced
+        EndpointTesting.no_test_endpoints(@configuration.get(:url),
+                                          @configuration.get(:ingestion_url))
+      else
+        EndpointTesting.test_endpoints(@proxy_url,
+                                       @configuration.get(:url),
+                                       @configuration.get(:ingestion_url))
+      end
+    end
+
     def load_actions(hashes)
       unsupported = Set.new