sqreen 1.20.2 → 1.21.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20f83c83a19022c3bfba53af705716e2d4b4d791912bf32c6e7e2853496a4b05
-  data.tar.gz: 6592f5a6c88895016924c4a342d6c1f48f885527c18fc0b182495d2905cef926
+  metadata.gz: 88934827e2aa5f84ae21f96fb33afdada009fb118d94e8940fe61973812da4d4
+  data.tar.gz: b30da2199b4dc96d0193c671872ac192396c988ef67845c26fc1ed36a7704bfd
 SHA512:
-  metadata.gz: 349fecb7524cb1740965162e596708fede99c5ab12bd5ed0a009c324e360585a15f2237c78614b666e77d5bbbf5b044a9a1ee7be8d3ce791b377cbc1ecbe15aa
-  data.tar.gz: ad4b8725a6531e80a4e47aac8c820f88284391a8f674ae59e8908846714c8b1c76be7fd95034cdf4885588f39e413308f39cc5688f6ed207558b48d0b14def5f
+  metadata.gz: 19942da7597a18aee5b46aa4ef664c5083eadf3a878c5f431f9dde83196fc15f3263509c6cda6537a5c823de310f117dbb7c600130478a0828d2dbc73f0bf53c
+  data.tar.gz: c8b1293e8480043d3b494d6335725f649107c56e7bf282580592de7d680eaeb9475bd9091ea280ad2d2e473b100789a9b80c3dac93b9ddebbddb2f8ec4f3ced6

data/CHANGELOG.md

@@ -1,6 +1,18 @@
-## 1.20.2
-
-* Fix performance regression in instrumentation engine
+## 1.21.0.beta3
+
+* Avoid fd leak in `custom_error_cb` (ARB-109)
+* Support `skip_rem_cbs` (ARB-107)
+* Fix instrumentation in Ruby 2.0
+* Fix encoding exception on `hash_key_include` (ARB-53)
+* Fix erroneous start in non-Rails context (SQREEN-880)
+* Make metrics thread-safe
+* Add restart command
+* Fix `overtime_cb`
+* Add `perf_level` 2
+* Several performance optimizations
+* WAF: rename `budget_in_ms` to `max_budget_ms`
+* Transport/Tracing with http module
+* Update the blocking page
 
 ## 1.20.1
 

data/lib/sqreen/actions/block_user.rb

@@ -22,7 +22,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info(
+        Sqreen.log.debug(
           "Will raise due to user being blocked by action #{id}. " \
           "Blocked user identity: #{identity_params}"
         )

data/lib/sqreen/actions/redirect_ip.rb

@@ -25,7 +25,7 @@ module Sqreen
       end
 
       def do_run(client_ip)
-        Sqreen.log.info "Will request redirect for client with IP #{client_ip} " \
+        Sqreen.log.debug "Will request redirect for client with IP #{client_ip} " \
           "(action: #{id})."
         {
           :status => :skip,

data/lib/sqreen/actions/redirect_user.rb

@@ -24,7 +24,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info 'Will request redirect for user with identity ' \
+        Sqreen.log.debug 'Will request redirect for user with identity ' \
           "#{identity_params} (action: #{id})."
 
         e = Sqreen::AttackBlocked.new(

data/lib/sqreen/condition_evaluator.rb

@@ -67,7 +67,7 @@ module Sqreen
       return true if rem <= 0
       if hash.is_a?(Array)
         return hash.any? do |v|
-          ConditionEvaluator.hash_key_include?(values, v, min_value_size, rem - 1)
+          hash_key_include?(values, v, min_value_size, rem - 1)
         end
       end
 
@@ -81,7 +81,14 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            values.include?(hkey.to_s) || ConditionEvaluator.hash_key_include?(values, hval, min_value_size, rem - 1)
+            key_incl =
+              if values.is_a?(String)
+                str_include? values, hkey.to_s
+              else
+                values.include?(hkey.to_s)
+              end
+
+            key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
           end
         end
       end

data/lib/sqreen/conditionable.rb

@@ -28,21 +28,39 @@ module Sqreen
     end
 
     def pre_with_conditions(inst, args, budget = nil, &block)
+      return pre_without_conditions(inst, args, budget, &block) if pre_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, nil]
-      return nil if !pre_conditions.nil? && !pre_conditions.evaluate(*eargs)
-      pre_without_conditions(inst, args, budget, &block)
+      return nil unless pre_conditions.evaluate(*eargs)
+
+      res = pre_without_conditions(inst, args, budget, &block)
+      return { passed_conditions: true } unless res.is_a?(Hash)
+      res[:passed_conditions] = true
+      res
     end
 
     def post_with_conditions(rv, inst, args, budget = nil, &block)
+      return post_without_conditions(rv, inst, args, budget, &block) if post_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !post_conditions.nil? && !post_conditions.evaluate(*eargs)
-      post_without_conditions(rv, inst, args, budget, &block)
+      return nil unless post_conditions.evaluate(*eargs)
+
+      res = post_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     def failing_with_conditions(rv, inst, args, budget = nil, &block)
+      return failing_without_conditions(rv, inst, args, budget, &block) if failing_conditions.nil?
+
       eargs = [nil, framework, inst, args, @data, rv]
-      return nil if !failing_conditions.nil? && !failing_conditions.evaluate(*eargs)
-      failing_without_conditions(rv, inst, args, budget, &block)
+      return nil unless failing_conditions.evaluate(*eargs)
+
+      res = failing_without_conditions(rv, inst, args, budget, &block)
+      return { passed_conditions: true } if res.nil?
+      res[:passed_conditions] = true
+      res
     end
 
     protected

data/lib/sqreen/configuration.rb

@@ -57,7 +57,7 @@ module Sqreen
     { :env => :SQREEN_RULES_SIGNATURE, :name => :rules_verify_signature,
       :default => true },
     { :env => :SQREEN_LOG_LEVEL, :name => :log_level,
-      :default => 'WARN', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
+      :default => 'INFO', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
     { :env => :SQREEN_LOG_LOCATION, :name => :log_location,
       :default => 'log/sqreen.log' },
     { :env => :SQREEN_RUN_IN_TEST, :name => :run_in_test,

data/lib/sqreen/deferred_logger.rb

@@ -9,39 +9,70 @@ require 'sqreen/logger'
 
 module Sqreen
   class DeferredLogger
-    include Singleton
+    MAX_ENTRIES = 1000
+
+    Entry = Struct.new(:severity, :message)
 
     def initialize
       @buffer = StringIO.new
       @logger = ::Logger.new(@buffer)
+      @entries = []
+      @mutex = Mutex.new
     end
 
     def debug?
       true
     end
 
+    def info?
+      true
+    end
+
+    def warn?
+      true
+    end
+
+    def error?
+      true
+    end
+
+    def fatal?
+      true
+    end
+
     def debug(msg = nil, &block)
-      @logger.debug(msg, &block)
+      add(::Logger::DEBUG, msg, &block)
     end
 
     def info(msg = nil, &block)
-      @logger.info(msg, &block)
+      add(::Logger::INFO, msg, &block)
     end
 
     def warn(msg = nil, &block)
-      @logger.warn(msg, &block)
+      add(::Logger::WARN, msg, &block)
     end
 
     def error(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::ERROR, msg, &block)
     end
 
     def fatal(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::FATAL, msg, &block)
+    end
+
+    def unknown(msg = nil, &block)
+      add(::Logger::UNKNOWN, msg, &block)
     end
 
     def add(severity, msg = nil, &block)
-      send(Sqreen::Logger::SEVERITY_TO_METHOD[severity], msg, &block)
+      @mutex.synchronize do
+        @entries.shift if @entries.count >= MAX_ENTRIES
+        mark = @buffer.pos
+        @logger.add(severity, msg, &block)
+        @buffer.seek(mark)
+        @entries << Entry.new(severity, @buffer.read)
+        @buffer.truncate(0)
+      end
     end
 
     def formatter=(value)
@@ -49,21 +80,22 @@ module Sqreen
     end
 
     def flush_to(logger)
-      logger.instance_eval { @logdev }.write(read).tap { reset }
+      @mutex.synchronize do
+        @entries.each do |entry|
+          next if entry.severity < logger.level
+          logger.instance_eval { @logdev }.write(entry.message)
+        end
+        reset
+      end
     end
 
     private
 
-    def read
-      @buffer.rewind
-      @buffer.read
-    end
-
     def reset
       buffer = StringIO.new
       logger = ::Logger.new(buffer)
       logger.formatter = @logger.formatter
-      @buffer, @logger = buffer, logger
+      @buffer, @logger, @entries = buffer, logger, []
     end
   end
 end

data/lib/sqreen/deliveries/batch.rb

@@ -13,6 +13,8 @@ require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
 require 'sqreen/deliveries/simple'
+require 'sqreen/kit/signals/signal'
+require 'sqreen/kit/signals/trace'
 
 module Sqreen
   module Deliveries
@@ -58,7 +60,7 @@ module Sqreen
       def post_batch_needed?(event)
         now = Sqreen.time
         # do not use any? {} due to side effects inside block
-        event_keys(event).map do |key|
+        event_keys(event).uniq.map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
           was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
@@ -86,6 +88,7 @@ module Sqreen
         res += event.observed.fetch(:sdk, []).select { |e|
             e[0] == :track
         }.map { |e| "sdk-track".freeze }
+        res += event.observed.fetch(:signals, []).map { "signal".freeze }
         return res
       end
 
@@ -97,6 +100,10 @@ module Sqreen
           "rex-#{event.klass}"
         when Sqreen::AggregatedMetric
           "agg-metric"
+        when Sqreen::Kit::Signals::Signal
+          "signal"
+        when Sqreen::Kit::Signals::Trace
+          "signal"
         end
       end
     end

data/lib/sqreen/deprecation.rb

@@ -0,0 +1,38 @@
+# typed: strong
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Deprecation
+    include Sqreen::Log::Loggable
+
+    module_function
+
+    def deprecate(method)
+      return unless ENV['SQREEN_DEBUG_DEPRECATION']
+
+      owner = method.owner
+      deprecated = :"_deprecated_#{method.name}"
+      klass = owner.is_a?(Module)
+      target = klass ? owner.to_s : owner.class.to_s
+
+      method.owner.instance_eval do
+        alias_method deprecated, method.name
+
+        define_method(method.name) do |*args, &block|
+          msg = [
+            "deprecation",
+            "target:#{target}",
+            "method:#{method.name}",
+            "caller:#{Kernel.caller_locations[0]}",
+          ].join(' ')
+          Sqreen::Deprecation.logger.info(msg)
+          send(deprecated, *args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem.rb

@@ -0,0 +1,96 @@
+require 'securerandom'
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/tracing/sampling_configuration'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_broker'
+require 'sqreen/ecosystem/tracing_id_setup'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing'
+
+module Sqreen
+  # The API for the ecosystem client (together with the dispatch table)
+  module Ecosystem
+    class << self
+      def init(opts = {})
+        @registry = ModuleRegistry.new
+        register_modules(opts[:modules])
+        @registry.init_all
+
+        # setup tracing generation
+        tracing_id_mods = @registry.module_subset(ModuleApi::TracingIdGeneration)
+        @tracing_id_setup = TracingIdSetup.new(tracing_id_mods)
+        @tracing_id_setup.setup_modules
+
+        # configure tracing broker with the consumers (tracing modules)
+        tracing_modules = @registry.module_subset(ModuleApi::Tracing)
+        @tracing_broker = TracingBroker.new(tracing_modules)
+
+        # inject tracing broker in message producers
+        @registry.each_module(ModuleApi::MessageProducer) do |mod|
+          mod.tracing_broker = @tracing_broker
+        end
+      rescue ::Exception # rubocop:disable Lint/RescueException
+        # TODO: modules must be disabled at this point
+        raise
+      end
+
+      def reset
+        instance_variables.each do |ia|
+          instance_variable_set(ia, nil)
+        end
+      end
+
+      # To be called by the Ecosystem client when a new transaction
+      # (generally: request) is started
+      # In the future, it's intended that request end/start detection be handled
+      # by the Ecosystem itself, so control will flow in the other direction,
+      # from the ecosystem to its client
+      def start_transaction
+        TransactionStorage.create_thread_local
+      end
+
+      def end_transaction
+        TransactionStorage.destroy_thread_local
+      end
+
+      # @param [String] tracing_id_prefix
+      # @param [Array<Hash{String=>Object}>] sampling_config
+      def configure_sampling(tracing_id_prefix, sampling_config)
+        @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
+        built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
+        @tracing_broker.sampling_configuration = built_samp_cfg
+      end
+
+      private
+
+      def register_modules(modules)
+        return register_all_modules unless modules
+
+        modules.each { |mod| register mod }
+      end
+
+      def register_all_modules
+        # replace with something more magical?
+        require_relative 'ecosystem/http/rack_request'
+        register Http::RackRequest.new
+
+        require_relative 'ecosystem/http/net_http'
+        register Http::NetHttp.new
+
+        require_relative 'ecosystem/redis/redis_connection'
+        register Redis::RedisConnection.new
+
+        require_relative 'ecosystem/tracing/modules/client'
+        register Tracing::Modules::Client.new
+
+        require_relative 'ecosystem/tracing/modules/server'
+        register Tracing::Modules::Server.new
+      end
+
+      def register(mod)
+        @registry.register mod
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/dispatch_table.rb

@@ -0,0 +1,43 @@
+require 'logger'
+
+module Sqreen
+  module Ecosystem
+    # Configured by the ecosystem client
+    module DispatchTable
+      class << self
+        # data consumption
+        # argument: +Sqreen::Kit::Signals::Signal+
+        # see +Sqreen::EcosystemIntegration::SignalConsumption#consume_signal+
+        attr_accessor :consume_signal
+
+        # argument: block taking a Rack::Request
+        # see +Sqreen::EcosystemIntegration::RequestLifecycleTracking#add_start_observer+
+        attr_accessor :add_request_start_listener
+
+        attr_accessor :fetch_logger
+
+        # argument: callback taking:
+        # * the method to instrument
+        # * A Hash{Symbol=>Proc} with the advice. The proc takes the
+        # arguments and the ball, so these details of the instrumentation
+        # implementation leak through the abstraction
+        # see +Sqreen::EcosystemIntegration::InstrumentationService+
+        attr_accessor :instrument
+
+        def reset
+          instance_variables.each do |ia|
+            instance_variable_set(ia, nil)
+          end
+
+          # set default logger
+          logger = ::Logger.new(STDERR)
+          logger.level = ::Logger::WARN
+          logger.progname = 'sqreen-ecosystem'
+          self.fetch_logger = proc { logger }
+        end
+      end
+
+      reset
+    end
+  end
+end