sqreen 1.20.0 → 1.20.4

data/lib/sqreen/graft/hook_point.rb

@@ -42,7 +42,7 @@ module Sqreen
       end
 
       def to_s
-        "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
+        @to_s ||= "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
       end
 
       def exist?

data/lib/sqreen/legacy/instrumentation.rb

@@ -109,7 +109,7 @@ module Legacy
             break if res.is_a?(Hash) && res[:skip_rem_cbs]
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug e.backtrace
+            Sqreen.log.debug { e.backtrace }
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -162,7 +162,7 @@ module Legacy
             returns << res
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug e.backtrace
+            Sqreen.log.debug { e.backtrace }
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -597,16 +597,25 @@ module Legacy
         method = cb.method
         key = [klass, method]
 
+        if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
+          call_count = JSON.parse(call_count)
+          if cb.respond_to?(:rule_name) && call_count.key?(cb.rule_name)
+            count = call_count[cb.rule_name]
+            Sqreen.log.debug { "override rule:#{cb.rule_name} call_count:#{count.inspect}" }
+            cb.instance_eval { @call_count_interval = call_count[cb.rule_name] }
+          end
+        end
+
         @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
 
         already_overriden = @@overriden_methods.include? key
 
         if !already_overriden
           if is_class_method?(klass, method)
-            Sqreen.log.debug "overriding class method for #{cb}"
+            Sqreen.log.debug { "overriding class method for #{cb}" }
             success = override_class_method(klass, method)
           elsif is_instance_method?(klass, method)
-            Sqreen.log.debug "overriding instance method for #{cb}"
+            Sqreen.log.debug { "overriding instance method for #{cb}" }
             success = override_instance_method(klass, method)
           else
             # FIXME: Override define_method and other dynamic ways to
@@ -623,7 +632,7 @@ module Legacy
 
           @@overriden_methods += [key] if success
         else
-          Sqreen.log.debug "#{key} was already overriden"
+          Sqreen.log.debug { "#{key} was already overriden" }
         end
 
         if klass != Object && klass != Kernel && !Sqreen.features['instrument_all_instances'] && !defined?(::JRUBY_VERSION)
@@ -638,7 +647,7 @@ module Legacy
           end
         end
 
-        Sqreen.log.debug "Adding callback #{cb} for #{klass} #{method}"
+        Sqreen.log.debug { "Adding callback #{cb} for #{klass} #{method}" }
         @@registered_callbacks.add(cb)
         @@unovertimable_hookpoints << key unless cb.overtimeable
         @@instrumented_pid = Process.pid
@@ -659,7 +668,7 @@ module Legacy
 
       already_overriden = @@overriden_methods.include? key
       unless already_overriden
-        Sqreen.log.debug "#{key} apparently not overridden"
+        Sqreen.log.debug { "#{key} apparently not overridden" }
       end
 
       defined_cbs = @@registered_callbacks.get(klass, method).flatten
@@ -667,17 +676,17 @@ module Legacy
       nb_removed = 0
       defined_cbs.each do |found_cb|
         if found_cb == cb
-          Sqreen.log.debug "Removing callback #{found_cb}"
+          Sqreen.log.debug { "Removing callback #{found_cb}" }
           @@registered_callbacks.remove(found_cb)
           nb_removed += 1
         else
-          Sqreen.log.debug "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)"
+          Sqreen.log.debug { "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)" }
         end
       end
 
       return unless nb_removed == defined_cbs.size
 
-      Sqreen.log.debug "Removing overriden method #{key}"
+      Sqreen.log.debug { "Removing overriden method #{key}" }
       @@overriden_methods.delete(key)
 
       if is_class_method?(klass, method)
@@ -705,6 +714,7 @@ module Legacy
           remove_callback_no_lock(cb)
         end
         Sqreen.instrumentation_ready = false
+        Sqreen.log.info('Instrumentation deactivated')
       end
     end
 
@@ -757,6 +767,8 @@ module Legacy
       ### globally declare instrumentation ready
       ### from within instance method? not even thread local?
       Sqreen.instrumentation_ready = true
+
+      Sqreen.log.info('Instrumentation activated')
     end
 
     def initialize(metrics_engine = nil)

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -6,6 +6,7 @@
 require 'sqreen/aggregated_metric'
 require 'sqreen/log/loggable'
 require 'sqreen/legacy/waf_redactions'
+require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Legacy
@@ -166,7 +167,7 @@ module Sqreen
           res[:request][:parameters] = payload['params'] if payload['params']
           res[:request][:headers] = payload['headers'] if payload['headers']
 
-          res = Sqreen::EncodingSanitizer.sanitize(res)
+          res = Sqreen::Kit::StringSanitizer.sanitize(res)
 
           if rr.redactor
             res[:request], redacted = rr.redactor.redact(res[:request])

data/lib/sqreen/log.rb

@@ -14,16 +14,17 @@ require 'sqreen/deferred_logger'
 
 module Sqreen
   def self.log_init
+    deferred_logger = @logger
     @logger = Sqreen::Logger.new(
       Sqreen.config_get(:log_level).to_s.upcase,
       Sqreen.config_get(:log_location)
     )
-    Sqreen::DeferredLogger.instance.flush_to(@logger.instance_eval { @logger })
+    deferred_logger.flush_to(@logger.instance_eval { @logger })
   rescue => e
     warn "Sqreen logger exception: #{e}"
   end
 
   def self::log
-    @logger || Sqreen::DeferredLogger.instance
+    @logger ||= Sqreen::DeferredLogger.new
   end
 end

data/lib/sqreen/log/loggable.rb

@@ -4,6 +4,7 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 require 'logger'
+require 'sqreen/log'
 
 module Sqreen; end
 module Sqreen::Log; end
@@ -23,6 +24,6 @@ module Sqreen::Log::Loggable
   end
 
   def logger
-    @logger || self.class.logger
+    @logger || singleton_class.logger
   end
 end

data/lib/sqreen/logger.rb

@@ -28,6 +28,26 @@ module Sqreen
       create_error_logger
     end
 
+    def debug?
+      @logger.debug?
+    end
+
+    def info?
+      @logger.info?
+    end
+
+    def warn?
+      @logger.warn?
+    end
+
+    def error?
+      @logger.error?
+    end
+
+    def fatal?
+      @logger.fatal?
+    end
+
     def debug(msg = nil, &block)
       @logger.debug(msg, &block)
     end
@@ -45,6 +65,10 @@ module Sqreen
       @logger.error(msg, &block)
     end
 
+    def unknown(msg = nil, &block)
+      @logger.unknown(msg, &block)
+    end
+
     def add(severity, msg = nil, &block)
       send(SEVERITY_TO_METHOD[severity], msg, &block)
     end

data/lib/sqreen/metrics_store.rb

@@ -27,6 +27,7 @@ module Sqreen
     def initialize
       @store = []
       @metrics = {} # name => (metric, period, start)
+      @mutex = Mutex.new
     end
 
     # Definition contains a name,period and aggregate at least
@@ -34,6 +35,8 @@ module Sqreen
     # @param rule [RuleCB] the rule associated with this metric, if any
     # @param mklass [Object] Override metric object (used in testing)
     def create_metric(definition, rule = nil, mklass = nil)
+      @mutex.lock
+
       name = definition[NAME_KEY]
       kind = definition[KIND_KEY]
       klass = valid_metric(kind, name)
@@ -49,6 +52,8 @@ module Sqreen
       metric.rule = rule
       metric.period = definition[PERIOD_KEY]
       metric
+    ensure
+      @mutex.unlock
     end
 
     def metric?(name)
@@ -57,21 +62,27 @@ module Sqreen
 
     # @param at [Time] when is the store emptied
     def update(name, at, key, value)
+      @mutex.lock
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
       next_sample(name, at) if start.nil? || (start + period) < at
       metric.update(key, value)
+    ensure
+      @mutex.unlock
     end
 
     # Drains every metrics and returns the store content
     # @param at [Time] when is the store emptied
     def publish(flush = true, at = Sqreen.time)
+      @mutex.lock
       @metrics.each do |name, (_, period, start)|
         next_sample(name, at) if flush || !start.nil? && (start + period) < at
       end
       out = @store
       @store = []
       out
+    ensure
+      @mutex.unlock
     end
 
     protected

data/lib/sqreen/null_logger.rb

@@ -9,6 +9,26 @@ module Sqreen
   class NullLogger
     include Singleton
 
+    def debug?
+      false
+    end
+
+    def info?
+      false
+    end
+
+    def warn?
+      false
+    end
+
+    def error?
+      false
+    end
+
+    def fatal?
+      false
+    end
+
     def debug(_msg = nil); end
 
     def info(_msg = nil); end
@@ -19,6 +39,8 @@ module Sqreen
 
     def fatal(_msg = nil); end
 
+    def unknown(_msg = nil); end
+
     def add(_severity, _msg = nil); end
 
     def formatter=(_); end

data/lib/sqreen/remote_command.rb

@@ -18,6 +18,7 @@ module Sqreen
       :features_get => :features,
       :features_change => :change_features,
       :force_logout => :shutdown,
+      :force_restart => :restart,
       :paths_whitelist => :change_whitelisted_paths,
       :ips_whitelist => :change_whitelisted_ips,
       :get_bundle => :upload_bundle,

data/lib/sqreen/rules.rb

@@ -114,15 +114,19 @@ module Sqreen
           Sqreen.log.warn('No JavaScript engine is available. ' \
               'JavaScript callbacks will be ignored')
         end
-        Sqreen.log.info("Ignoring JS callback #{rule_name}")
+        Sqreen.log.debug("Ignoring JS callback #{rule_name}")
         return nil
       end
 
       cb_class = ExecJSCB if js
 
-      if cbname && Rules.const_defined?(cbname)
-        # Only load callbacks from sqreen
-        cb_class = Rules.const_get(cbname)
+      if cbname
+        cb_class = if cbname.include?('::')
+                     # Only load callbacks from sqreen
+                     Rules.walk_const_get(cbname) if cbname.start_with?('::Sqreen::', 'Sqreen::')
+                   else
+                     Rules.const_get(cbname) if Rules.const_defined?(cbname) # rubocop:disable Style/IfInsideElse
+                   end
       end
 
       if cb_class.nil?

data/lib/sqreen/rules/blacklist_ips_cb.rb

@@ -33,7 +33,7 @@ module Sqreen
       private
 
       def insert_values(ranges)
-        Sqreen.log.info 'no ips given for IP blacklisting' if ranges.empty?
+        Sqreen.log.debug 'no ips given for IP blacklisting' if ranges.empty?
 
         ranges.map { |r| Prefix.from_str(r, r) }.each do |prefix|
           trie_for(prefix).insert prefix
@@ -50,7 +50,7 @@ module Sqreen
         begin
           ipa = IPAddr.new(rip)
         rescue StandardError
-          Sqreen.log.info "invalid IP address given by framework: #{rip}"
+          Sqreen.log.debug "invalid IP address given by framework: #{rip}"
           return nil
         end
 

data/lib/sqreen/rules/custom_error_cb.rb

@@ -55,12 +55,12 @@ module Sqreen
       end
 
       def respond_page
-        page = open(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        @page ||= File.read(File.join(File.dirname(__FILE__), '../attack_detected.html'))
         headers = {
           'Content-Type' => 'text/html',
-          'Content-Length' => page.size.to_s,
+          'Content-Length' => @page.size.to_s,
         }
-        [@status_code, headers, page]
+        [@status_code, headers, [@page]]
       end
     end
   end

data/lib/sqreen/rules/rule_cb.rb

@@ -3,6 +3,7 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
+require 'sqreen/deprecation'
 require 'sqreen/framework_cb'
 require 'sqreen/context'
 require 'sqreen/conditionable'
@@ -109,6 +110,7 @@ module Sqreen
         )
         true
       end
+      Sqreen::Deprecation.deprecate(instance_method(:overtime!))
     end
   end
 end

data/lib/sqreen/rules/waf_cb.rb

@@ -11,7 +11,7 @@ require 'sqreen/safe_json'
 require 'sqreen/exception'
 require 'sqreen/util/capper'
 require 'sqreen/dependency/libsqreen'
-require 'sqreen/encoding_sanitizer'
+require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Rules
@@ -60,7 +60,7 @@ module Sqreen
         end
 
         # 0 for using defaults (PW_RUN_TIMEOUT)
-        @max_run_budget_us = (@data['values'].fetch('budget_in_ms', 0) * 1000).to_i
+        @max_run_budget_us = (@data['values'].fetch('max_budget_ms', 0) * 1000).to_i
         @max_run_budget_us = INFINITE_BUDGET_US if @max_run_budget_us >= INFINITE_BUDGET_US
 
         Sqreen.log.debug { "Max WAF run budget for #{@waf_rule_name} set to #{@max_run_budget_us} us" }
@@ -82,7 +82,7 @@ module Sqreen
         waf_args = binding_accessors.each_with_object({}) do |(e, b), h|
           h[e] = capper.call(b.resolve(*env))
         end
-        waf_args = Sqreen::EncodingSanitizer.sanitize(waf_args)
+        waf_args = Sqreen::Kit::StringSanitizer.sanitize(waf_args)
 
         if budget
           rem_budget_s = budget - (Sqreen.time - start)

data/lib/sqreen/runner.rb

@@ -11,6 +11,7 @@ require 'sqreen/events/attack'
 
 require 'sqreen/log'
 
+require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
 require 'sqreen/remote_command'
@@ -18,6 +19,7 @@ require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
 require 'sqreen/deliveries/simple'
 require 'sqreen/deliveries/batch'
+require 'sqreen/endpoint_testing'
 require 'sqreen/performance_notifications/metrics'
 require 'sqreen/performance_notifications/binned_metrics'
 require 'sqreen/legacy/instrumentation'
@@ -113,19 +115,23 @@ module Sqreen
       @next_metrics = []
       @running = true
 
+      @proxy_url = @configuration.get(:proxy_url)
+      chosen_endpoints = determine_endpoints
+
       @token = @configuration.get(:token)
       @app_name = @configuration.get(:app_name)
-      @url = @configuration.get(:url)
-      @proxy_url = @configuration.get(:proxy_url)
+      @url = chosen_endpoints.control.url
+      @cert_store = chosen_endpoints.control.ca_store
+
       Sqreen.update_whitelisted_paths([])
       Sqreen.update_whitelisted_ips({})
       Sqreen.update_performance_budget(nil)
-      raise(Sqreen::Exception, 'no url found') unless @url
       raise(Sqreen::TokenNotFoundException, 'no token found') unless @token
 
       Sqreen::Kit::Configuration.logger = Sqreen.log
-      Sqreen::Kit::Configuration.ingestion_url = @configuration.get(:ingestion_url)
-      Sqreen::Kit::Configuration.proxy_url = @configuration.get(:proxy_url)
+      Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
+      Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
+      Sqreen::Kit::Configuration.proxy_url = @proxy_url
 
       register_exit_cb if set_at_exit
 
@@ -143,6 +149,7 @@ module Sqreen
 
       Sqreen.log.debug "Using token #{@token}"
       response = create_session(session_class)
+      post_endpoint_testing_msgs(chosen_endpoints)
       wanted_features = response.fetch('features', {})
       conf_initial_features = configuration.get(:initial_features)
       unless conf_initial_features.nil?
@@ -155,7 +162,7 @@ module Sqreen
           wanted_features = wanted_features.merge(conf_features)
         rescue
           Sqreen.log.warn do
-            "NOT using invalid inital features #{conf_initial_features}"
+            "NOT using invalid initial features #{conf_initial_features}"
           end
         end
       end
@@ -171,7 +178,7 @@ module Sqreen
     end
 
     def create_session(session_class)
-      @session = session_class.new(@url, @token, @app_name, @proxy_url)
+      @session = session_class.new(@url, @cert_store, @token, @app_name, @proxy_url)
       session.login(@framework)
     end
 
@@ -380,8 +387,25 @@ module Sqreen
 
     def change_performance_budget(budget, _context_infos = {})
       return false unless budget.nil? || budget.to_f > 0
-      prev = Sqreen.performance_budget
-      Sqreen.update_performance_budget(budget)
+
+      if @configuration.get(:weave)
+        prev = Sqreen::Weave::Budget.current
+        prev = prev.to_h if prev
+
+        budget_s = budget.to_f / 1000 if budget
+
+        feature = features['performance_budget']
+        if feature
+          budget_s = feature['threshold'] if feature.key?('threshold')
+          ratio = feature['ratio'] if feature.key?('ratio')
+        end
+
+        Sqreen::Weave::Budget.update(threshold: budget_s, ratio: ratio)
+      else
+        prev = Sqreen.performance_budget
+        Sqreen.update_performance_budget(budget)
+      end
+
       { :was => prev }
     end
 
@@ -471,6 +495,15 @@ module Sqreen
       logout
     end
 
+    def restart(_context_infos = {})
+      shutdown
+      heartbeat_delay = @heartbeat_delay
+      Thread.new do
+        sleep(2 * heartbeat_delay)
+        Sqreen::Worker.start(Sqreen.framework)
+      end
+    end
+
     def logout(retrying = true)
       return unless session
       Sqreen.log.debug("Logging out")
@@ -508,6 +541,28 @@ module Sqreen
 
     private
 
+    def post_endpoint_testing_msgs(chosen_endpoints)
+      chosen_endpoints.messages.each do |msg|
+        session.post_agent_message(@framework, msg)
+      end
+    rescue => e
+      Sqreen.log.warn "Error submitting agent message: #{e}"
+      RemoteException.record(e)
+    end
+
+    def determine_endpoints
+      # there's no sniffing going on; just a misnamed config setting
+      if @configuration.get(:no_sniff_domains)
+        # reproduces behaviour before endpoint testing was introduced
+        EndpointTesting.no_test_endpoints(@configuration.get(:url),
+                                          @configuration.get(:ingestion_url))
+      else
+        EndpointTesting.test_endpoints(@proxy_url,
+                                       @configuration.get(:url),
+                                       @configuration.get(:ingestion_url))
+      end
+    end
+
     def load_actions(hashes)
       unsupported = Set.new