sqreen 1.20.4 → 1.21.0.beta1

data/lib/sqreen/graft/hook_point.rb

@@ -42,7 +42,7 @@ module Sqreen
       end
 
       def to_s
-        @to_s ||= "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
+        "#{@klass_name}#{@method_kind == :instance_method ? '#' : '.'}#{@method_name}"
       end
 
       def exist?

data/lib/sqreen/legacy/instrumentation.rb

@@ -109,7 +109,7 @@ module Legacy
             break if res.is_a?(Hash) && res[:skip_rem_cbs]
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug { e.backtrace }
+            Sqreen.log.debug e.backtrace
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -162,7 +162,7 @@ module Legacy
             returns << res
           rescue StandardError => e
             Sqreen.log.warn { "we catch an exception: #{e.inspect}" }
-            Sqreen.log.debug { e.backtrace }
+            Sqreen.log.debug e.backtrace
             if cb.respond_to?(:record_exception)
               cb.record_exception(e)
             else
@@ -597,25 +597,16 @@ module Legacy
         method = cb.method
         key = [klass, method]
 
-        if (call_count = ENV['SQREEN_DEBUG_CALL_COUNT'])
-          call_count = JSON.parse(call_count)
-          if cb.respond_to?(:rule_name) && call_count.key?(cb.rule_name)
-            count = call_count[cb.rule_name]
-            Sqreen.log.debug { "override rule:#{cb.rule_name} call_count:#{count.inspect}" }
-            cb.instance_eval { @call_count_interval = call_count[cb.rule_name] }
-          end
-        end
-
         @@record_request_hookpoints << key if cb.is_a?(Sqreen::Rules::RecordRequestContext)
 
         already_overriden = @@overriden_methods.include? key
 
         if !already_overriden
           if is_class_method?(klass, method)
-            Sqreen.log.debug { "overriding class method for #{cb}" }
+            Sqreen.log.debug "overriding class method for #{cb}"
             success = override_class_method(klass, method)
           elsif is_instance_method?(klass, method)
-            Sqreen.log.debug { "overriding instance method for #{cb}" }
+            Sqreen.log.debug "overriding instance method for #{cb}"
             success = override_instance_method(klass, method)
           else
             # FIXME: Override define_method and other dynamic ways to
@@ -632,7 +623,7 @@ module Legacy
 
           @@overriden_methods += [key] if success
         else
-          Sqreen.log.debug { "#{key} was already overriden" }
+          Sqreen.log.debug "#{key} was already overriden"
         end
 
         if klass != Object && klass != Kernel && !Sqreen.features['instrument_all_instances'] && !defined?(::JRUBY_VERSION)
@@ -647,7 +638,7 @@ module Legacy
           end
         end
 
-        Sqreen.log.debug { "Adding callback #{cb} for #{klass} #{method}" }
+        Sqreen.log.debug "Adding callback #{cb} for #{klass} #{method}"
         @@registered_callbacks.add(cb)
         @@unovertimable_hookpoints << key unless cb.overtimeable
         @@instrumented_pid = Process.pid
@@ -668,7 +659,7 @@ module Legacy
 
       already_overriden = @@overriden_methods.include? key
       unless already_overriden
-        Sqreen.log.debug { "#{key} apparently not overridden" }
+        Sqreen.log.debug "#{key} apparently not overridden"
       end
 
       defined_cbs = @@registered_callbacks.get(klass, method).flatten
@@ -676,17 +667,17 @@ module Legacy
       nb_removed = 0
       defined_cbs.each do |found_cb|
         if found_cb == cb
-          Sqreen.log.debug { "Removing callback #{found_cb}" }
+          Sqreen.log.debug "Removing callback #{found_cb}"
           @@registered_callbacks.remove(found_cb)
           nb_removed += 1
         else
-          Sqreen.log.debug { "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)" }
+          Sqreen.log.debug "Not removing callback #{found_cb} (remains #{defined_cbs.size} cbs)"
         end
       end
 
       return unless nb_removed == defined_cbs.size
 
-      Sqreen.log.debug { "Removing overriden method #{key}" }
+      Sqreen.log.debug "Removing overriden method #{key}"
       @@overriden_methods.delete(key)
 
       if is_class_method?(klass, method)
@@ -714,7 +705,6 @@ module Legacy
           remove_callback_no_lock(cb)
         end
         Sqreen.instrumentation_ready = false
-        Sqreen.log.info('Instrumentation deactivated')
       end
     end
 
@@ -767,8 +757,6 @@ module Legacy
       ### globally declare instrumentation ready
       ### from within instance method? not even thread local?
       Sqreen.instrumentation_ready = true
-
-      Sqreen.log.info('Instrumentation activated')
     end
 
     def initialize(metrics_engine = nil)

data/lib/sqreen/legacy/old_event_submission_strategy.rb

@@ -6,7 +6,6 @@
 require 'sqreen/aggregated_metric'
 require 'sqreen/log/loggable'
 require 'sqreen/legacy/waf_redactions'
-require 'sqreen/kit/string_sanitizer'
 
 module Sqreen
   module Legacy
@@ -56,6 +55,12 @@ module Sqreen
               when AggregatedMetric
                 logger.warn "Aggregated metric event in non-signal mode. Signals disabled at runtime?"
                 next
+              when Sqreen::Kit::Signals::Signal
+                logger.warn "Signal event in non-signal mode"
+                next
+              when Sqreen::Kit::Signals::Trace
+                logger.warn "Trace event in non-signal mode"
+                next
               when Attack # in practice only found inside req rec
                 EventToHash.convert_attack event
               when RemoteException
@@ -73,7 +78,7 @@ module Sqreen
           tally = Hash[events.group_by(&:class).map { |k, v| [k, v.count] }]
           "Doing batch with the following tally of event types: #{tally}"
         end
-        post('batch', { batch: batch }, {}, RETRY_MANY)
+        post('batch', { batch: batch.compact }, {}, RETRY_MANY)
       end
 
       private
@@ -167,7 +172,7 @@ module Sqreen
           res[:request][:parameters] = payload['params'] if payload['params']
           res[:request][:headers] = payload['headers'] if payload['headers']
 
-          res = Sqreen::Kit::StringSanitizer.sanitize(res)
+          res = Sqreen::EncodingSanitizer.sanitize(res)
 
           if rr.redactor
             res[:request], redacted = rr.redactor.redact(res[:request])

data/lib/sqreen/log.rb

@@ -14,17 +14,16 @@ require 'sqreen/deferred_logger'
 
 module Sqreen
   def self.log_init
-    deferred_logger = @logger
     @logger = Sqreen::Logger.new(
       Sqreen.config_get(:log_level).to_s.upcase,
       Sqreen.config_get(:log_location)
     )
-    deferred_logger.flush_to(@logger.instance_eval { @logger })
+    Sqreen::DeferredLogger.instance.flush_to(@logger.instance_eval { @logger })
   rescue => e
     warn "Sqreen logger exception: #{e}"
   end
 
   def self::log
-    @logger ||= Sqreen::DeferredLogger.new
+    @logger || Sqreen::DeferredLogger.instance
   end
 end

data/lib/sqreen/log/loggable.rb

@@ -4,7 +4,6 @@
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
 require 'logger'
-require 'sqreen/log'
 
 module Sqreen; end
 module Sqreen::Log; end

data/lib/sqreen/logger.rb

@@ -28,26 +28,6 @@ module Sqreen
       create_error_logger
     end
 
-    def debug?
-      @logger.debug?
-    end
-
-    def info?
-      @logger.info?
-    end
-
-    def warn?
-      @logger.warn?
-    end
-
-    def error?
-      @logger.error?
-    end
-
-    def fatal?
-      @logger.fatal?
-    end
-
     def debug(msg = nil, &block)
       @logger.debug(msg, &block)
     end
@@ -65,10 +45,6 @@ module Sqreen
       @logger.error(msg, &block)
     end
 
-    def unknown(msg = nil, &block)
-      @logger.unknown(msg, &block)
-    end
-
     def add(severity, msg = nil, &block)
       send(SEVERITY_TO_METHOD[severity], msg, &block)
     end

data/lib/sqreen/metrics_store.rb

@@ -27,7 +27,6 @@ module Sqreen
     def initialize
       @store = []
       @metrics = {} # name => (metric, period, start)
-      @mutex = Mutex.new
     end
 
     # Definition contains a name,period and aggregate at least
@@ -35,8 +34,6 @@ module Sqreen
     # @param rule [RuleCB] the rule associated with this metric, if any
     # @param mklass [Object] Override metric object (used in testing)
     def create_metric(definition, rule = nil, mklass = nil)
-      @mutex.lock
-
       name = definition[NAME_KEY]
       kind = definition[KIND_KEY]
       klass = valid_metric(kind, name)
@@ -52,8 +49,6 @@ module Sqreen
       metric.rule = rule
       metric.period = definition[PERIOD_KEY]
       metric
-    ensure
-      @mutex.unlock
     end
 
     def metric?(name)
@@ -62,27 +57,21 @@ module Sqreen
 
     # @param at [Time] when is the store emptied
     def update(name, at, key, value)
-      @mutex.lock
       metric, period, start = @metrics[name]
       raise UnregisteredMetric, "Unknown metric #{name}" unless metric
       next_sample(name, at) if start.nil? || (start + period) < at
       metric.update(key, value)
-    ensure
-      @mutex.unlock
     end
 
     # Drains every metrics and returns the store content
     # @param at [Time] when is the store emptied
     def publish(flush = true, at = Sqreen.time)
-      @mutex.lock
       @metrics.each do |name, (_, period, start)|
         next_sample(name, at) if flush || !start.nil? && (start + period) < at
       end
       out = @store
       @store = []
       out
-    ensure
-      @mutex.unlock
     end
 
     protected

data/lib/sqreen/null_logger.rb

@@ -9,26 +9,6 @@ module Sqreen
   class NullLogger
     include Singleton
 
-    def debug?
-      false
-    end
-
-    def info?
-      false
-    end
-
-    def warn?
-      false
-    end
-
-    def error?
-      false
-    end
-
-    def fatal?
-      false
-    end
-
     def debug(_msg = nil); end
 
     def info(_msg = nil); end
@@ -39,8 +19,6 @@ module Sqreen
 
     def fatal(_msg = nil); end
 
-    def unknown(_msg = nil); end
-
     def add(_severity, _msg = nil); end
 
     def formatter=(_); end

data/lib/sqreen/remote_command.rb

@@ -18,11 +18,11 @@ module Sqreen
       :features_get => :features,
       :features_change => :change_features,
       :force_logout => :shutdown,
-      :force_restart => :restart,
       :paths_whitelist => :change_whitelisted_paths,
       :ips_whitelist => :change_whitelisted_ips,
       :get_bundle => :upload_bundle,
       :performance_budget => :change_performance_budget,
+      :tracing_enable => :tracing_enable,
     }.freeze
 
     attr_reader :uuid
@@ -40,6 +40,8 @@ module Sqreen
       begin
         output = runner.send(KNOWN_COMMANDS[@name], *@params, context_infos)
       rescue => e
+        Sqreen.log.warn { "Command failed with #{e}" }
+        Sqreen.log.debug { e.backtrace.map { |x| "  #{x}" }.join("\n") }
         Sqreen::RemoteException.record(e)
         return { :status => false, :reason => "error: #{e.inspect}" }
       end

data/lib/sqreen/rules.rb

@@ -114,19 +114,15 @@ module Sqreen
           Sqreen.log.warn('No JavaScript engine is available. ' \
               'JavaScript callbacks will be ignored')
         end
-        Sqreen.log.debug("Ignoring JS callback #{rule_name}")
+        Sqreen.log.info("Ignoring JS callback #{rule_name}")
         return nil
       end
 
       cb_class = ExecJSCB if js
 
-      if cbname
-        cb_class = if cbname.include?('::')
-                     # Only load callbacks from sqreen
-                     Rules.walk_const_get(cbname) if cbname.start_with?('::Sqreen::', 'Sqreen::')
-                   else
-                     Rules.const_get(cbname) if Rules.const_defined?(cbname) # rubocop:disable Style/IfInsideElse
-                   end
+      if cbname && Rules.const_defined?(cbname)
+        # Only load callbacks from sqreen
+        cb_class = Rules.const_get(cbname)
       end
 
       if cb_class.nil?

data/lib/sqreen/rules/blacklist_ips_cb.rb

@@ -33,7 +33,7 @@ module Sqreen
       private
 
       def insert_values(ranges)
-        Sqreen.log.debug 'no ips given for IP blacklisting' if ranges.empty?
+        Sqreen.log.info 'no ips given for IP blacklisting' if ranges.empty?
 
         ranges.map { |r| Prefix.from_str(r, r) }.each do |prefix|
           trie_for(prefix).insert prefix
@@ -50,7 +50,7 @@ module Sqreen
         begin
           ipa = IPAddr.new(rip)
         rescue StandardError
-          Sqreen.log.debug "invalid IP address given by framework: #{rip}"
+          Sqreen.log.info "invalid IP address given by framework: #{rip}"
           return nil
         end
 

data/lib/sqreen/rules/custom_error_cb.rb

@@ -55,12 +55,12 @@ module Sqreen
       end
 
       def respond_page
-        @page ||= File.read(File.join(File.dirname(__FILE__), '../attack_detected.html'))
+        page = open(File.join(File.dirname(__FILE__), '../attack_detected.html'))
         headers = {
           'Content-Type' => 'text/html',
-          'Content-Length' => @page.size.to_s,
+          'Content-Length' => page.size.to_s,
         }
-        [@status_code, headers, [@page]]
+        [@status_code, headers, page]
       end
     end
   end

data/lib/sqreen/rules/rule_cb.rb

@@ -3,7 +3,6 @@
 # Copyright (c) 2015 Sqreen. All Rights Reserved.
 # Please refer to our terms for more information: https://www.sqreen.com/terms.html
 
-require 'sqreen/deprecation'
 require 'sqreen/framework_cb'
 require 'sqreen/context'
 require 'sqreen/conditionable'
@@ -110,7 +109,6 @@ module Sqreen
         )
         true
       end
-      Sqreen::Deprecation.deprecate(instance_method(:overtime!))
     end
   end
 end

data/lib/sqreen/rules/waf_cb.rb

@@ -11,7 +11,7 @@ require 'sqreen/safe_json'
 require 'sqreen/exception'
 require 'sqreen/util/capper'
 require 'sqreen/dependency/libsqreen'
-require 'sqreen/kit/string_sanitizer'
+require 'sqreen/encoding_sanitizer'
 
 module Sqreen
   module Rules
@@ -60,7 +60,7 @@ module Sqreen
         end
 
         # 0 for using defaults (PW_RUN_TIMEOUT)
-        @max_run_budget_us = (@data['values'].fetch('max_budget_ms', 0) * 1000).to_i
+        @max_run_budget_us = (@data['values'].fetch('budget_in_ms', 0) * 1000).to_i
         @max_run_budget_us = INFINITE_BUDGET_US if @max_run_budget_us >= INFINITE_BUDGET_US
 
         Sqreen.log.debug { "Max WAF run budget for #{@waf_rule_name} set to #{@max_run_budget_us} us" }
@@ -82,7 +82,7 @@ module Sqreen
         waf_args = binding_accessors.each_with_object({}) do |(e, b), h|
           h[e] = capper.call(b.resolve(*env))
         end
-        waf_args = Sqreen::Kit::StringSanitizer.sanitize(waf_args)
+        waf_args = Sqreen::EncodingSanitizer.sanitize(waf_args)
 
         if budget
           rem_budget_s = budget - (Sqreen.time - start)

data/lib/sqreen/runner.rb

@@ -14,6 +14,7 @@ require 'sqreen/log'
 require 'sqreen/agent_message'
 require 'sqreen/rules'
 require 'sqreen/session'
+require 'sqreen/version'
 require 'sqreen/remote_command'
 require 'sqreen/capped_queue'
 require 'sqreen/metrics_store'
@@ -26,6 +27,7 @@ require 'sqreen/legacy/instrumentation'
 require 'sqreen/call_countable'
 require 'sqreen/weave/legacy/instrumentation'
 require 'sqreen/kit/configuration'
+require 'sqreen/ecosystem_integration'
 
 module Sqreen
   @features = {}
@@ -52,10 +54,6 @@ module Sqreen
       @queue ||= CappedQueue.new(MAX_QUEUE_LENGTH)
     end
 
-    def update_queue(queue)
-      @queue = queue
-    end
-
     def observations_queue
       @observations_queue ||= CappedQueue.new(MAX_OBS_QUEUE_LENGTH)
     end
@@ -104,8 +102,8 @@ module Sqreen
     # we may want to do that in a thread in order to prevent delaying app
     # startup
     # set_at_exit do not place a global at_exit (used for testing)
+    # @param [Sqreen::Frameworks::GenericFramework] framework
     def initialize(configuration, framework, set_at_exit = true, session_class = Sqreen::Session)
-      Sqreen.update_queue(CappedQueue.new(MAX_QUEUE_LENGTH))
       @logged_out_tried = false
       @configuration = configuration
       @framework = framework
@@ -132,6 +130,7 @@ module Sqreen
       Sqreen::Kit::Configuration.ingestion_url = chosen_endpoints.ingestion.url
       Sqreen::Kit::Configuration.certificate_store = chosen_endpoints.ingestion.ca_store
       Sqreen::Kit::Configuration.proxy_url = @proxy_url
+      Sqreen::Kit::Configuration.default_source = "sqreen:agent:ruby:#{Sqreen::VERSION}"
 
       register_exit_cb if set_at_exit
 
@@ -168,6 +167,10 @@ module Sqreen
       end
       self.features = wanted_features
 
+      @ecosystem_integration = EcosystemIntegration.new(framework, Sqreen.queue)
+      framework.req_start_cb = @ecosystem_integration.method(:request_start)
+      framework.req_end_cb = @ecosystem_integration.method(:request_end)
+
       # Ensure a deliverer is there unless features have set it first
       self.deliverer ||= Deliveries::Simple.new(session)
       context_infos = {}
@@ -268,6 +271,10 @@ module Sqreen
       rulespack_id, rules = load_rules(context_infos)
       @framework.instrument_when_ready!(instrumenter, rules)
       Sqreen.log.info 'Instrumentation set up'
+
+      # XXX: ecosystem instrumentation should likely be deferred
+      #      the same way the rest might be
+      @ecosystem_integration.init
       rulespack_id.to_s
     end
 
@@ -387,28 +394,18 @@ module Sqreen
 
     def change_performance_budget(budget, _context_infos = {})
       return false unless budget.nil? || budget.to_f > 0
-
-      if @configuration.get(:weave)
-        prev = Sqreen::Weave::Budget.current
-        prev = prev.to_h if prev
-
-        budget_s = budget.to_f / 1000 if budget
-
-        feature = features['performance_budget']
-        if feature
-          budget_s = feature['threshold'] if feature.key?('threshold')
-          ratio = feature['ratio'] if feature.key?('ratio')
-        end
-
-        Sqreen::Weave::Budget.update(threshold: budget_s, ratio: ratio)
-      else
-        prev = Sqreen.performance_budget
-        Sqreen.update_performance_budget(budget)
-      end
-
+      prev = Sqreen.performance_budget
+      Sqreen.update_performance_budget(budget)
       { :was => prev }
     end
 
+    # @param [String] tracing_id_prefix
+    # @param [Array<Hash{String=>Object}>] sampling_config
+    def tracing_enable(tracing_id_prefix, sampling_config, _context_infos = {})
+      @ecosystem_integration.handle_tracing_command(tracing_id_prefix, sampling_config)
+      { status: true }
+    end
+
     def upload_bundle(_context_infos = {})
       t = Time.now
       session.post_bundle(RuntimeInfos.dependencies_signature, RuntimeInfos.dependencies)
@@ -495,15 +492,6 @@ module Sqreen
       logout
     end
 
-    def restart(_context_infos = {})
-      shutdown
-      heartbeat_delay = @heartbeat_delay
-      Thread.new do
-        sleep(2 * heartbeat_delay)
-        Sqreen::Worker.start(Sqreen.framework)
-      end
-    end
-
     def logout(retrying = true)
       return unless session
       Sqreen.log.debug("Logging out")