sqreen 1.21.0.beta1 → 1.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49a6c95ab34d19ae0f769e475ae67c2c3e4d19b17582ee09d8a6d231ac3d9150
-  data.tar.gz: ad1ea7a80f3582ba90ffc6aa1cef7040459c56f65ea4712a023923b8bc7801e3
+  metadata.gz: 72fc8c4943ce7cb8cd45a80553ae02ae8c28086ca1895a89f2ec5840b2e6a883
+  data.tar.gz: ca46dc3483df4a16fca77e0226ca7c9a3d832c5a837155b86f0ee70c5fc12226
 SHA512:
-  metadata.gz: 0da6438f20a00a84914db2bb06c24feab53e164feae1cb7d2af0b53afe24c5d5ea54a0cb68a8872a49aadedae4450de57e63da6a4f4a2ec21ac85eaa84c13acf
-  data.tar.gz: 3e75918e810529674e10d693fd2a62c0eeeeecb18bac06a25fdd686d29c45d55333abb0feea790ec1d89e29ce71097bd1de16c2d31d98a6219e985d806793008
+  metadata.gz: 112a455abff8baca0c586f1479351e8e900e73d7f89b31e83fcbdc07ae99dbfa86f7439541a2f1688d56b481ce9094e16cd6bf860cb9959c7edd7be80ed66f92
+  data.tar.gz: 3f867ee75cf103c8817007540151e189c10116e0da8fa0bdad324de3bfa611d2bb4f888fdccf1074248f7976f26eb68fc81f987fcae0db5208ee9c00569f9797

data/CHANGELOG.md

@@ -1,3 +1,43 @@
+## 1.22.0
+
+* Update WAF via libsqreen
+* Add support for raw body
+* Improve signature check
+* Improve APM detection
+
+## 1.21.1
+
+* Work around NewRelic initialisation (see https://github.com/newrelic/newrelic-ruby-agent/issues/461)
+
+## 1.21.0
+
+* Add support for transport and tracing facilities
+
+## 1.20.4
+
+* Fix missing budget check
+* Improve performance
+* Align internal setting name for WAF
+* Include response information in all payloads
+* Improve robustness against invalid Unicode
+* Prevent rule execution to pursue in early block cases
+
+## 1.20.4.beta1
+
+* Add optional dynamic time budget
+* Add advanced per request metrics
+* Improve robustness against exception in instrumentation
+* Improve metric engine thread safety
+* Restrict deferred logger to final logger severity on agent boot
+
+## 1.20.3
+
+* Fix signature check
+
+## 1.20.2
+
+* Fix performance regression in instrumentation engine
+
 ## 1.20.1
 
 * Add fallback mechanisms when connecting to new Sqreen backend API domains

data/lib/sqreen/actions/block_user.rb

@@ -22,7 +22,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info(
+        Sqreen.log.debug(
           "Will raise due to user being blocked by action #{id}. " \
           "Blocked user identity: #{identity_params}"
         )

data/lib/sqreen/actions/redirect_ip.rb

@@ -25,7 +25,7 @@ module Sqreen
       end
 
       def do_run(client_ip)
-        Sqreen.log.info "Will request redirect for client with IP #{client_ip} " \
+        Sqreen.log.debug "Will request redirect for client with IP #{client_ip} " \
           "(action: #{id})."
         {
           :status => :skip,

data/lib/sqreen/actions/redirect_user.rb

@@ -24,7 +24,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.info 'Will request redirect for user with identity ' \
+        Sqreen.log.debug 'Will request redirect for user with identity ' \
           "#{identity_params} (action: #{id})."
 
         e = Sqreen::AttackBlocked.new(

data/lib/sqreen/condition_evaluator.rb

@@ -67,7 +67,7 @@ module Sqreen
       return true if rem <= 0
       if hash.is_a?(Array)
         return hash.any? do |v|
-          ConditionEvaluator.hash_key_include?(values, v, min_value_size, rem - 1)
+          hash_key_include?(values, v, min_value_size, rem - 1)
         end
       end
 
@@ -81,7 +81,13 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            values.include?(hkey.to_s) || ConditionEvaluator.hash_key_include?(values, hval, min_value_size, rem - 1)
+            key_incl = if values.is_a?(String)
+                         str_include?(values, hkey.to_s)
+                       else
+                         values.include?(hkey.to_s)
+                       end
+
+            key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
           end
         end
       end

data/lib/sqreen/configuration.rb

@@ -57,7 +57,7 @@ module Sqreen
     { :env => :SQREEN_RULES_SIGNATURE, :name => :rules_verify_signature,
       :default => true },
     { :env => :SQREEN_LOG_LEVEL, :name => :log_level,
-      :default => 'WARN', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
+      :default => 'INFO', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
     { :env => :SQREEN_LOG_LOCATION, :name => :log_location,
       :default => 'log/sqreen.log' },
     { :env => :SQREEN_RUN_IN_TEST, :name => :run_in_test,

data/lib/sqreen/deferred_logger.rb

@@ -9,35 +9,70 @@ require 'sqreen/logger'
 
 module Sqreen
   class DeferredLogger
-    include Singleton
+    MAX_ENTRIES = 1000
+
+    Entry = Struct.new(:severity, :message)
 
     def initialize
       @buffer = StringIO.new
       @logger = ::Logger.new(@buffer)
+      @entries = []
+      @mutex = Mutex.new
+    end
+
+    def debug?
+      true
+    end
+
+    def info?
+      true
+    end
+
+    def warn?
+      true
+    end
+
+    def error?
+      true
+    end
+
+    def fatal?
+      true
     end
 
     def debug(msg = nil, &block)
-      @logger.debug(msg, &block)
+      add(::Logger::DEBUG, msg, &block)
     end
 
     def info(msg = nil, &block)
-      @logger.info(msg, &block)
+      add(::Logger::INFO, msg, &block)
     end
 
     def warn(msg = nil, &block)
-      @logger.warn(msg, &block)
+      add(::Logger::WARN, msg, &block)
     end
 
     def error(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::ERROR, msg, &block)
     end
 
     def fatal(msg = nil, &block)
-      @logger.error(msg, &block)
+      add(::Logger::FATAL, msg, &block)
+    end
+
+    def unknown(msg = nil, &block)
+      add(::Logger::UNKNOWN, msg, &block)
     end
 
     def add(severity, msg = nil, &block)
-      send(Sqreen::Logger::SEVERITY_TO_METHOD[severity], msg, &block)
+      @mutex.synchronize do
+        @entries.shift if @entries.count >= MAX_ENTRIES
+        mark = @buffer.pos
+        @logger.add(severity, msg, &block)
+        @buffer.seek(mark)
+        @entries << Entry.new(severity, @buffer.read)
+        @buffer.truncate(0)
+      end
     end
 
     def formatter=(value)
@@ -45,21 +80,22 @@ module Sqreen
     end
 
     def flush_to(logger)
-      logger.instance_eval { @logdev }.write(read).tap { reset }
+      @mutex.synchronize do
+        @entries.each do |entry|
+          next if entry.severity < logger.level
+          logger.instance_eval { @logdev }.write(entry.message)
+        end
+        reset
+      end
     end
 
     private
 
-    def read
-      @buffer.rewind
-      @buffer.read
-    end
-
     def reset
       buffer = StringIO.new
       logger = ::Logger.new(buffer)
       logger.formatter = @logger.formatter
-      @buffer, @logger = buffer, logger
+      @buffer, @logger, @entries = buffer, logger, []
     end
   end
 end

data/lib/sqreen/dependency/detector.rb

@@ -25,6 +25,14 @@ module Sqreen
         end
       end
 
+      def to_app_hook_strategy
+        if Sqreen::Dependency::NewRelic.bundled? || Sqreen::Dependency::NewRelic.required?
+          :chain
+        else
+          :prepend
+        end
+      end
+
       def hook(&block)
         Sqreen.log.debug "[#{Process.pid}] Startup command: #{$0}"
 
@@ -34,7 +42,7 @@ module Sqreen
           Sqreen::Dependency::Rails.insert_sqreen_middlewares
         end if Sqreen::Dependency::Rails.required?
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do
             Sqreen::Dependency::Rails.inspect_middlewares
           end
@@ -48,7 +56,7 @@ module Sqreen
           end
         end.install if Sqreen::Dependency::Sinatra.required?
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do |call|
             builder = call.instance
 
@@ -58,7 +66,7 @@ module Sqreen
 
         # ensure startup of thread in request handling processes
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do |call|
             callback = call.callback
 

data/lib/sqreen/dependency/new_relic.rb

@@ -8,8 +8,17 @@ module Sqreen
     module NewRelic
       module_function
 
+      def bundled?
+        defined?(Gem) && Gem.respond_to?(:loaded_specs) && !Gem.loaded_specs['newrelic_rpm'].nil?
+      end
+
+      def required?
+        Sqreen::Dependency.const_exist?('NewRelic::Agent::Agent')
+      end
+
       def ignore_sqreen_exceptions
-        return unless defined?(NewRelic::Agent::Agent)
+        return unless required?
+
         NewRelic::Agent::Agent.instance.error_collector.ignore(['Sqreen::AttackBlocked'])
       rescue ::Exception => e # rubocop:disable Lint/RescueException
         Sqreen.log.warn "Failed ignoring AttackBlocked on NewRelic: #{e.inspect}"

data/lib/sqreen/deprecation.rb

@@ -0,0 +1,38 @@
+# typed: strong
+
+# Copyright (c) 2015 Sqreen. All Rights Reserved.
+# Please refer to our terms for more information: https://www.sqreen.com/terms.html
+
+require 'sqreen/log/loggable'
+
+module Sqreen
+  module Deprecation
+    include Sqreen::Log::Loggable
+
+    module_function
+
+    def deprecate(method)
+      return unless ENV['SQREEN_DEBUG_DEPRECATION']
+
+      owner = method.owner
+      deprecated = :"_deprecated_#{method.name}"
+      klass = owner.is_a?(Module)
+      target = klass ? owner.to_s : owner.class.to_s
+
+      method.owner.instance_eval do
+        alias_method deprecated, method.name
+
+        define_method(method.name) do |*args, &block|
+          msg = [
+            "deprecation",
+            "target:#{target}",
+            "method:#{method.name}",
+            "caller:#{Kernel.caller_locations[0]}",
+          ].join(' ')
+          Sqreen::Deprecation.logger.info(msg)
+          send(deprecated, *args, &block)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem.rb

@@ -2,8 +2,11 @@ require 'securerandom'
 require 'sqreen/ecosystem/module_registry'
 require 'sqreen/ecosystem/tracing/sampling_configuration'
 require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_broker'
 require 'sqreen/ecosystem/tracing_id_setup'
-require 'sqreen/ecosystem/module_api/tracing_push_down'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing'
 
 module Sqreen
   # The API for the ecosystem client (together with the dispatch table)
@@ -14,8 +17,22 @@ module Sqreen
         register_modules(opts[:modules])
         @registry.init_all
 
-        @tracing_id_setup = TracingIdSetup.new(@registry)
+        # setup tracing generation
+        tracing_id_mods = @registry.module_subset(ModuleApi::TracingIdGeneration)
+        @tracing_id_setup = TracingIdSetup.new(tracing_id_mods)
         @tracing_id_setup.setup_modules
+
+        # configure tracing broker with the consumers (tracing modules)
+        tracing_modules = @registry.module_subset(ModuleApi::Tracing)
+        @tracing_broker = TracingBroker.new(tracing_modules)
+
+        # inject tracing broker in message producers
+        @registry.each_module(ModuleApi::MessageProducer) do |mod|
+          mod.tracing_broker = @tracing_broker
+        end
+      rescue ::Exception # rubocop:disable Lint/RescueException
+        # TODO: modules must be disabled at this point
+        raise
       end
 
       def reset
@@ -42,7 +59,7 @@ module Sqreen
       def configure_sampling(tracing_id_prefix, sampling_config)
         @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
         built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
-        inject_sampling_config(built_samp_cfg)
+        @tracing_broker.sampling_configuration = built_samp_cfg
       end
 
       private
@@ -61,20 +78,46 @@ module Sqreen
         require_relative 'ecosystem/http/net_http'
         register Http::NetHttp.new
 
-        require_relative 'ecosystem/redis/redis_connection'
-        register Redis::RedisConnection.new
+        require_relative 'ecosystem/databases/postgres'
+        register Databases::Postgres.new
+
+        require_relative 'ecosystem/databases/mysql'
+        register Databases::Mysql.new
+
+        require_relative 'ecosystem/databases/mongo'
+        register Databases::Mongo.new
+
+        require_relative 'ecosystem/databases/redis'
+        register Databases::Redis.new
+
+        require_relative 'ecosystem/messaging/sqs'
+        register Messaging::Sqs.new
+
+        require_relative 'ecosystem/messaging/kinesis'
+        register Messaging::Kinesis.new
+
+        require_relative 'ecosystem/messaging/bunny'
+        register Messaging::Bunny.new
+
+        require_relative 'ecosystem/messaging/kafka'
+        register Messaging::Kafka.new
+
+        require_relative 'ecosystem/tracing/modules/client'
+        register Tracing::Modules::Client.new
+
+        require_relative 'ecosystem/tracing/modules/server'
+        register Tracing::Modules::Server.new
+
+        require_relative 'ecosystem/tracing/modules/producer'
+        register Tracing::Modules::Producer.new
+
+        require_relative 'ecosystem/tracing/modules/consumer'
+        register Tracing::Modules::Consumer.new
       end
 
       def register(mod)
         @registry.register mod
       end
-
-      # @param [Sqreen::Ecosystem::SamplingConfiguration] config
-      def inject_sampling_config(config)
-        @registry.each_module(Sqreen::Ecosystem::ModuleApi::TracingPushDown) do |mod|
-          mod.sampling_config = config
-        end
-      end
     end
   end
 end

data/lib/sqreen/ecosystem/databases/database_connection_data.rb

@@ -0,0 +1,23 @@
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class DatabaseConnectionData
+        include ModuleApi::Tracing::ClientData
+
+        # @return [Integer]
+        attr_accessor :port
+
+        # @return [String]
+        attr_accessor :unix_socket
+
+        # @return [String]
+        attr_accessor :username
+
+        # @return [String]
+        attr_accessor :db
+      end
+    end
+  end
+end