sqreen 1.20.4 → 1.21.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7324df5cecbb626299a4e048550c6f7c5a3e15bef2eecc8d89011d0342914aa
-  data.tar.gz: eabb5203769dcf898d8a4e373c6cfee7e5b0eac8bbfc3977fb8924a54504b126
+  metadata.gz: 49a6c95ab34d19ae0f769e475ae67c2c3e4d19b17582ee09d8a6d231ac3d9150
+  data.tar.gz: ad1ea7a80f3582ba90ffc6aa1cef7040459c56f65ea4712a023923b8bc7801e3
 SHA512:
-  metadata.gz: 5e7f4bb53e01c0f5328a5190b189c82b596bcf28eada2104b1491c5601b1275866ef8950ae9ad92c91303011636a3c11ebcb3c3dbccd75280d6b89c1c69d301a
-  data.tar.gz: 77770ff7b00b9d07ea7f4137eadce1e840bc1ad94e885b42623354707306ebd42425187833e0403e33d02bea7471010bf934bda9010c5fe1d604715c5daf88c1
+  metadata.gz: 0da6438f20a00a84914db2bb06c24feab53e164feae1cb7d2af0b53afe24c5d5ea54a0cb68a8872a49aadedae4450de57e63da6a4f4a2ec21ac85eaa84c13acf
+  data.tar.gz: 3e75918e810529674e10d693fd2a62c0eeeeecb18bac06a25fdd686d29c45d55333abb0feea790ec1d89e29ce71097bd1de16c2d31d98a6219e985d806793008

data/CHANGELOG.md

@@ -1,28 +1,3 @@
-## 1.20.4
-
-* Fix missing budget check
-* Improve performance
-* Align internal setting name for WAF
-* Include response information in all payloads
-* Improve robustness against invalid Unicode
-* Prevent rule execution to pursue in early block cases
-
-## 1.20.4.beta1
-
-* Add optional dynamic time budget
-* Add advanced per request metrics
-* Improve robustness against exception in instrumentation
-* Improve metric engine thread safety
-* Restrict deferred logger to final logger severity on agent boot
-
-## 1.20.3
-
-* Fix signature check
-
-## 1.20.2
-
-* Fix performance regression in instrumentation engine
-
 ## 1.20.1
 
 * Add fallback mechanisms when connecting to new Sqreen backend API domains

data/lib/sqreen/actions/block_user.rb

@@ -22,7 +22,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.debug(
+        Sqreen.log.info(
           "Will raise due to user being blocked by action #{id}. " \
           "Blocked user identity: #{identity_params}"
         )

data/lib/sqreen/actions/redirect_ip.rb

@@ -25,7 +25,7 @@ module Sqreen
       end
 
       def do_run(client_ip)
-        Sqreen.log.debug "Will request redirect for client with IP #{client_ip} " \
+        Sqreen.log.info "Will request redirect for client with IP #{client_ip} " \
           "(action: #{id})."
         {
           :status => :skip,

data/lib/sqreen/actions/redirect_user.rb

@@ -24,7 +24,7 @@ module Sqreen
       end
 
       def do_run(identity_params)
-        Sqreen.log.debug 'Will request redirect for user with identity ' \
+        Sqreen.log.info 'Will request redirect for user with identity ' \
           "#{identity_params} (action: #{id})."
 
         e = Sqreen::AttackBlocked.new(

data/lib/sqreen/condition_evaluator.rb

@@ -67,7 +67,7 @@ module Sqreen
       return true if rem <= 0
       if hash.is_a?(Array)
         return hash.any? do |v|
-          hash_key_include?(values, v, min_value_size, rem - 1)
+          ConditionEvaluator.hash_key_include?(values, v, min_value_size, rem - 1)
         end
       end
 
@@ -81,13 +81,7 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            key_incl = if values.is_a?(String)
-                         str_include?(values, hkey.to_s)
-                       else
-                         values.include?(hkey.to_s)
-                       end
-
-            key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
+            values.include?(hkey.to_s) || ConditionEvaluator.hash_key_include?(values, hval, min_value_size, rem - 1)
           end
         end
       end

data/lib/sqreen/configuration.rb

@@ -57,7 +57,7 @@ module Sqreen
     { :env => :SQREEN_RULES_SIGNATURE, :name => :rules_verify_signature,
       :default => true },
     { :env => :SQREEN_LOG_LEVEL, :name => :log_level,
-      :default => 'INFO', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
+      :default => 'WARN', :choice => %w[UNKNOWN FATAL ERROR WARN INFO DEBUG] },
     { :env => :SQREEN_LOG_LOCATION, :name => :log_location,
       :default => 'log/sqreen.log' },
     { :env => :SQREEN_RUN_IN_TEST, :name => :run_in_test,

data/lib/sqreen/deferred_logger.rb

@@ -9,70 +9,35 @@ require 'sqreen/logger'
 
 module Sqreen
   class DeferredLogger
-    MAX_ENTRIES = 1000
-
-    Entry = Struct.new(:severity, :message)
+    include Singleton
 
     def initialize
       @buffer = StringIO.new
       @logger = ::Logger.new(@buffer)
-      @entries = []
-      @mutex = Mutex.new
-    end
-
-    def debug?
-      true
-    end
-
-    def info?
-      true
-    end
-
-    def warn?
-      true
-    end
-
-    def error?
-      true
-    end
-
-    def fatal?
-      true
     end
 
     def debug(msg = nil, &block)
-      add(::Logger::DEBUG, msg, &block)
+      @logger.debug(msg, &block)
     end
 
     def info(msg = nil, &block)
-      add(::Logger::INFO, msg, &block)
+      @logger.info(msg, &block)
     end
 
     def warn(msg = nil, &block)
-      add(::Logger::WARN, msg, &block)
+      @logger.warn(msg, &block)
     end
 
     def error(msg = nil, &block)
-      add(::Logger::ERROR, msg, &block)
+      @logger.error(msg, &block)
     end
 
     def fatal(msg = nil, &block)
-      add(::Logger::FATAL, msg, &block)
-    end
-
-    def unknown(msg = nil, &block)
-      add(::Logger::UNKNOWN, msg, &block)
+      @logger.error(msg, &block)
     end
 
     def add(severity, msg = nil, &block)
-      @mutex.synchronize do
-        @entries.shift if @entries.count >= MAX_ENTRIES
-        mark = @buffer.pos
-        @logger.add(severity, msg, &block)
-        @buffer.seek(mark)
-        @entries << Entry.new(severity, @buffer.read)
-        @buffer.truncate(0)
-      end
+      send(Sqreen::Logger::SEVERITY_TO_METHOD[severity], msg, &block)
     end
 
     def formatter=(value)
@@ -80,22 +45,21 @@ module Sqreen
     end
 
     def flush_to(logger)
-      @mutex.synchronize do
-        @entries.each do |entry|
-          next if entry.severity < logger.level
-          logger.instance_eval { @logdev }.write(entry.message)
-        end
-        reset
-      end
+      logger.instance_eval { @logdev }.write(read).tap { reset }
     end
 
     private
 
+    def read
+      @buffer.rewind
+      @buffer.read
+    end
+
     def reset
       buffer = StringIO.new
       logger = ::Logger.new(buffer)
       logger.formatter = @logger.formatter
-      @buffer, @logger, @entries = buffer, logger, []
+      @buffer, @logger = buffer, logger
     end
   end
 end

data/lib/sqreen/deliveries/batch.rb

@@ -13,6 +13,8 @@ require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
 require 'sqreen/deliveries/simple'
+require 'sqreen/kit/signals/signal'
+require 'sqreen/kit/signals/trace'
 
 module Sqreen
   module Deliveries
@@ -58,7 +60,7 @@ module Sqreen
       def post_batch_needed?(event)
         now = Sqreen.time
         # do not use any? {} due to side effects inside block
-        event_keys(event).map do |key|
+        event_keys(event).uniq.map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
           was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
@@ -86,6 +88,7 @@ module Sqreen
         res += event.observed.fetch(:sdk, []).select { |e|
             e[0] == :track
         }.map { |e| "sdk-track".freeze }
+        res += event.observed.fetch(:signals, []).map { "signal".freeze }
         return res
       end
 
@@ -97,6 +100,10 @@ module Sqreen
           "rex-#{event.klass}"
         when Sqreen::AggregatedMetric
           "agg-metric"
+        when Sqreen::Kit::Signals::Signal
+          "signal"
+        when Sqreen::Kit::Signals::Trace
+          "signal"
         end
       end
     end

data/lib/sqreen/ecosystem.rb

@@ -0,0 +1,80 @@
+require 'securerandom'
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/tracing/sampling_configuration'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_id_setup'
+require 'sqreen/ecosystem/module_api/tracing_push_down'
+
+module Sqreen
+  # The API for the ecosystem client (together with the dispatch table)
+  module Ecosystem
+    class << self
+      def init(opts = {})
+        @registry = ModuleRegistry.new
+        register_modules(opts[:modules])
+        @registry.init_all
+
+        @tracing_id_setup = TracingIdSetup.new(@registry)
+        @tracing_id_setup.setup_modules
+      end
+
+      def reset
+        instance_variables.each do |ia|
+          instance_variable_set(ia, nil)
+        end
+      end
+
+      # To be called by the Ecosystem client when a new transaction
+      # (generally: request) is started
+      # In the future, it's intended that request end/start detection be handled
+      # by the Ecosystem itself, so control will flow in the other direction,
+      # from the ecosystem to its client
+      def start_transaction
+        TransactionStorage.create_thread_local
+      end
+
+      def end_transaction
+        TransactionStorage.destroy_thread_local
+      end
+
+      # @param [String] tracing_id_prefix
+      # @param [Array<Hash{String=>Object}>] sampling_config
+      def configure_sampling(tracing_id_prefix, sampling_config)
+        @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
+        built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
+        inject_sampling_config(built_samp_cfg)
+      end
+
+      private
+
+      def register_modules(modules)
+        return register_all_modules unless modules
+
+        modules.each { |mod| register mod }
+      end
+
+      def register_all_modules
+        # replace with something more magical?
+        require_relative 'ecosystem/http/rack_request'
+        register Http::RackRequest.new
+
+        require_relative 'ecosystem/http/net_http'
+        register Http::NetHttp.new
+
+        require_relative 'ecosystem/redis/redis_connection'
+        register Redis::RedisConnection.new
+      end
+
+      def register(mod)
+        @registry.register mod
+      end
+
+      # @param [Sqreen::Ecosystem::SamplingConfiguration] config
+      def inject_sampling_config(config)
+        @registry.each_module(Sqreen::Ecosystem::ModuleApi::TracingPushDown) do |mod|
+          mod.sampling_config = config
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/dispatch_table.rb

@@ -0,0 +1,43 @@
+require 'logger'
+
+module Sqreen
+  module Ecosystem
+    # Configured by the ecosystem client
+    module DispatchTable
+      class << self
+        # data consumption
+        # argument: +Sqreen::Kit::Signals::Signal+
+        # see +Sqreen::EcosystemIntegration::SignalConsumption#consume_signal+
+        attr_accessor :consume_signal
+
+        # argument: block taking a Rack::Request
+        # see +Sqreen::EcosystemIntegration::RequestLifecycleTracking#add_start_observer+
+        attr_accessor :add_request_start_listener
+
+        attr_accessor :fetch_logger
+
+        # argument: callback taking:
+        # * the method to instrument
+        # * A Hash{Symbol=>Proc} with the advice. The proc takes the
+        # arguments and the ball, so these details of the instrumentation
+        # implementation leak through the abstraction
+        # see +Sqreen::EcosystemIntegration::InstrumentationService+
+        attr_accessor :instrument
+
+        def reset
+          instance_variables.each do |ia|
+            instance_variable_set(ia, nil)
+          end
+
+          # set default logger
+          logger = ::Logger.new(STDERR)
+          logger.level = ::Logger::WARN
+          logger.progname = 'sqreen-ecosystem'
+          self.fetch_logger = proc { logger }
+        end
+      end
+
+      reset
+    end
+  end
+end

data/lib/sqreen/ecosystem/http/net_http.rb

@@ -0,0 +1,51 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/tracing_push_down'
+require 'sqreen/ecosystem/module_api/signal_producer'
+require 'sqreen/ecosystem/module_api/transaction_storage'
+require 'sqreen/ecosystem/tracing/signals/tracing_client'
+
+module Sqreen
+  module Ecosystem
+    module Http
+      class NetHttp
+        include ModuleApi::Instrumentation
+        include ModuleApi::SignalProducer
+        include ModuleApi::TracingPushDown
+
+        def setup
+          instrument 'Net::HTTP#request', before: method(:before_advice)
+        end
+
+        private
+
+        # instr. def request(req, body = nil, &block)  # :yield: +response+
+        # req is of type +Net::HTTPGenericRequest+
+        def before_advice(call, _ball)
+          return unless should_sample?('client')
+
+          tracing_id = create_tracing_id
+
+          # build & submit signal
+          host = call.instance.address
+          port = call.instance.port
+
+          host += ':' + port.to_s if port != 80 && port != 443
+
+          signal = Tracing::Signals::TracingClient.new
+          signal.payload = Tracing::Signals::TracingClient::Payload.new(
+            transport: 'http',
+            host: host,
+            tracing_identifier: tracing_id
+          )
+
+          submit_signal signal
+
+          # add tracing header, if available
+          req = call.args[0]
+          req[ModuleApi::TRACE_ID_HEADER] = tracing_id
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/http/rack_request.rb

@@ -0,0 +1,38 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/event_listener'
+require 'sqreen/ecosystem/module_api/signal_producer'
+require 'sqreen/ecosystem/module_api/tracing_push_down'
+require 'sqreen/ecosystem/tracing/signals/tracing_server'
+
+module Sqreen
+  module Ecosystem
+    module Http
+      class RackRequest
+        include ModuleApi::EventListener
+        include ModuleApi::TracingPushDown
+        include ModuleApi::SignalProducer
+
+        def setup
+          on_request_start(&method(:handle_request))
+        end
+
+        private
+
+        def handle_request(rack_request)
+          return unless should_sample?('server')
+
+          trace_id = rack_request.env[ModuleApi::TRACE_ID_ENV_KEY]
+
+          signal = Tracing::Signals::TracingServer.new
+          signal.payload = Tracing::Signals::TracingServer::Payload.new(
+            transport: 'http',
+            client_ip: rack_request.ip,
+            tracing_identifier: trace_id
+          )
+
+          submit_signal signal
+        end
+      end
+    end
+  end
+end