sqreen 1.20.4.beta1 → 1.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18fc090589982cc7e09d74263a3bbb17ae9180f00afd8425c47498a0f8b75e36
-  data.tar.gz: 074c3c9cad84ef1a52e6fd7ce1a4db5431920f59298e5b930206ca9a996141e9
+  metadata.gz: acfd752e97a36e2f1f2d0cb224e72a409334d379a6e8c3e05b002bade311e097
+  data.tar.gz: cb15f77dbc7e21faa5136a3cfc2493dd576164f2e49c532573c9f8adfa331e77
 SHA512:
-  metadata.gz: afd27c8a52c0c5e21685b9bf6b346a28ca43ff297c529c1976d7197bc132d51a92f2ac52dcb4de150cd8c752c4875b4fdd3baf8e59560a0c1952c4df12585572
-  data.tar.gz: 2e6d09aceaf1dcc351537b0a34e4459e7f1d4659a231cd515e6f01ec42f0515dc40ff89f14adb7094a6971f86fe3983e984bf35f8242b8f047405c164ef0874c
+  metadata.gz: a4025d0a2500ff46db50b33eb7f3340065ae687f3e8d218a14ca3f2690ccecee4412b4901a4dcd93fd67a7fc0036ee025f4f4cf08a6d254101741ddaf4bbcf07
+  data.tar.gz: 54755c74c6211c7aad3ae75aa8f2a5c307a074acef5ebf182ee31a5f50580a3115bd499be80d24cd5566ce16ef82c169f50ee9bed18531e7d0944e6864e7f610

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## 1.21.0
+
+* Add support for transport and tracing facilities
+
+## 1.20.4
+
+* Fix missing budget check
+* Improve performance
+* Align internal setting name for WAF
+* Include response information in all payloads
+* Improve robustness against invalid Unicode
+* Prevent rule execution to pursue in early block cases
+
 ## 1.20.4.beta1
 
 * Add optional dynamic time budget

data/lib/sqreen/condition_evaluator.rb

@@ -67,7 +67,7 @@ module Sqreen
       return true if rem <= 0
       if hash.is_a?(Array)
         return hash.any? do |v|
-          ConditionEvaluator.hash_key_include?(values, v, min_value_size, rem - 1)
+          hash_key_include?(values, v, min_value_size, rem - 1)
         end
       end
 
@@ -81,7 +81,13 @@ module Sqreen
           if hkey.respond_to?(:empty?) && hkey.empty?
             false
           else
-            values.include?(hkey.to_s) || ConditionEvaluator.hash_key_include?(values, hval, min_value_size, rem - 1)
+            key_incl = if values.is_a?(String)
+                         str_include?(values, hkey.to_s)
+                       else
+                         values.include?(hkey.to_s)
+                       end
+
+            key_incl || hash_key_include?(values, hval, min_value_size, rem - 1)
           end
         end
       end

data/lib/sqreen/deliveries/batch.rb

@@ -13,6 +13,8 @@ require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
 require 'sqreen/deliveries/simple'
+require 'sqreen/kit/signals/signal'
+require 'sqreen/kit/signals/trace'
 
 module Sqreen
   module Deliveries
@@ -58,7 +60,7 @@ module Sqreen
       def post_batch_needed?(event)
         now = Sqreen.time
         # do not use any? {} due to side effects inside block
-        event_keys(event).map do |key|
+        event_keys(event).uniq.map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
           was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
@@ -86,6 +88,7 @@ module Sqreen
         res += event.observed.fetch(:sdk, []).select { |e|
             e[0] == :track
         }.map { |e| "sdk-track".freeze }
+        res += event.observed.fetch(:signals, []).map { "signal".freeze }
         return res
       end
 
@@ -97,6 +100,10 @@ module Sqreen
           "rex-#{event.klass}"
         when Sqreen::AggregatedMetric
           "agg-metric"
+        when Sqreen::Kit::Signals::Signal
+          "signal"
+        when Sqreen::Kit::Signals::Trace
+          "signal"
         end
       end
     end

data/lib/sqreen/ecosystem.rb

@@ -0,0 +1,123 @@
+require 'securerandom'
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/tracing/sampling_configuration'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_broker'
+require 'sqreen/ecosystem/tracing_id_setup'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing'
+
+module Sqreen
+  # The API for the ecosystem client (together with the dispatch table)
+  module Ecosystem
+    class << self
+      def init(opts = {})
+        @registry = ModuleRegistry.new
+        register_modules(opts[:modules])
+        @registry.init_all
+
+        # setup tracing generation
+        tracing_id_mods = @registry.module_subset(ModuleApi::TracingIdGeneration)
+        @tracing_id_setup = TracingIdSetup.new(tracing_id_mods)
+        @tracing_id_setup.setup_modules
+
+        # configure tracing broker with the consumers (tracing modules)
+        tracing_modules = @registry.module_subset(ModuleApi::Tracing)
+        @tracing_broker = TracingBroker.new(tracing_modules)
+
+        # inject tracing broker in message producers
+        @registry.each_module(ModuleApi::MessageProducer) do |mod|
+          mod.tracing_broker = @tracing_broker
+        end
+      rescue ::Exception # rubocop:disable Lint/RescueException
+        # TODO: modules must be disabled at this point
+        raise
+      end
+
+      def reset
+        instance_variables.each do |ia|
+          instance_variable_set(ia, nil)
+        end
+      end
+
+      # To be called by the Ecosystem client when a new transaction
+      # (generally: request) is started
+      # In the future, it's intended that request end/start detection be handled
+      # by the Ecosystem itself, so control will flow in the other direction,
+      # from the ecosystem to its client
+      def start_transaction
+        TransactionStorage.create_thread_local
+      end
+
+      def end_transaction
+        TransactionStorage.destroy_thread_local
+      end
+
+      # @param [String] tracing_id_prefix
+      # @param [Array<Hash{String=>Object}>] sampling_config
+      def configure_sampling(tracing_id_prefix, sampling_config)
+        @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
+        built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
+        @tracing_broker.sampling_configuration = built_samp_cfg
+      end
+
+      private
+
+      def register_modules(modules)
+        return register_all_modules unless modules
+
+        modules.each { |mod| register mod }
+      end
+
+      def register_all_modules
+        # replace with something more magical?
+        require_relative 'ecosystem/http/rack_request'
+        register Http::RackRequest.new
+
+        require_relative 'ecosystem/http/net_http'
+        register Http::NetHttp.new
+
+        require_relative 'ecosystem/databases/postgres'
+        register Databases::Postgres.new
+
+        require_relative 'ecosystem/databases/mysql'
+        register Databases::Mysql.new
+
+        require_relative 'ecosystem/databases/mongo'
+        register Databases::Mongo.new
+
+        require_relative 'ecosystem/databases/redis'
+        register Databases::Redis.new
+
+        require_relative 'ecosystem/messaging/sqs'
+        register Messaging::Sqs.new
+
+        require_relative 'ecosystem/messaging/kinesis'
+        register Messaging::Kinesis.new
+
+        require_relative 'ecosystem/messaging/bunny'
+        register Messaging::Bunny.new
+
+        require_relative 'ecosystem/messaging/kafka'
+        register Messaging::Kafka.new
+
+        require_relative 'ecosystem/tracing/modules/client'
+        register Tracing::Modules::Client.new
+
+        require_relative 'ecosystem/tracing/modules/server'
+        register Tracing::Modules::Server.new
+
+        require_relative 'ecosystem/tracing/modules/producer'
+        register Tracing::Modules::Producer.new
+
+        require_relative 'ecosystem/tracing/modules/consumer'
+        register Tracing::Modules::Consumer.new
+      end
+
+      def register(mod)
+        @registry.register mod
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/database_connection_data.rb

@@ -0,0 +1,23 @@
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class DatabaseConnectionData
+        include ModuleApi::Tracing::ClientData
+
+        # @return [Integer]
+        attr_accessor :port
+
+        # @return [String]
+        attr_accessor :unix_socket
+
+        # @return [String]
+        attr_accessor :username
+
+        # @return [String]
+        attr_accessor :db
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/mongo.rb

@@ -0,0 +1,39 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Mongo
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Mongo::Client#initialize', after: advice
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_advice(call, _ball)
+          return if call.raised
+
+          client = call.instance
+          server_addrs = client.cluster.servers.map(&:address)
+
+          server_addrs.map do |addr|
+            DatabaseConnectionData.new(
+              transport: :mongo,
+              host: addr.host,
+              port: addr.port,
+              db: client.database.name,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/mysql.rb

@@ -0,0 +1,54 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Mysql
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Mysql2::Client#connect', after: advice
+        end
+
+        private
+
+        # instance is of type +Mysql2::Client+
+        # VALUE rb_mysql_connect(VALUE self, VALUE user, VALUE pass, VALUE host, VALUE port, VALUE database,
+        #                        VALUE socket, VALUE flags, VALUE conn_attrs) {
+        # @param [Sqreen::Graft::CallbackCall]
+        def after_advice(call, _ball)
+          args = call.args
+
+          # build & submit signal
+          signal = DatabaseConnectionData.new(transport: :mysql)
+
+          user = args[0]
+          host = args[2]
+          port = args[3]
+          db = args[4]
+          socket = args[5]
+
+          if socket && !socket.empty?
+            signal.unix_socket = socket
+            signal.host = 'localhost'
+            signal.ip = '::1'
+          else
+            signal.host = host
+          end
+
+          signal.port = port if port != 0
+          signal.username = user
+          signal.db = db
+
+          signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/postgres.rb

@@ -0,0 +1,51 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Postgres
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'PG::Connection#initialize', after: advice
+        end
+
+        private
+
+        # instance is of type +PG::Connection+
+        # > c = PG::Connection.new(host: '172.17.0.2', password: 'mysecretpassword', user: 'postgres')
+        #  => #<PG::Connection:0x000055b44d077d10>
+        # > %i{host port user db}.map { |m| c.send m }
+        #  => ["172.17.0.2", 5432, "postgres", "postgres"]
+        def after_advice(call, _ball)
+          conn = call.instance
+
+          # build & submit signal
+          signal = DatabaseConnectionData.new(transport: :postgres)
+
+          host = conn.host
+          if host
+            if host.include?('/')
+              signal.unix_socket = host
+              signal.host = 'localhost'
+              signal.ip = '::1'
+            else
+              signal.host = host
+            end
+          end
+
+          signal.port = conn.port
+          signal.username = conn.user
+          signal.db = conn.db
+
+          signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/redis.rb

@@ -0,0 +1,36 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Redis
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Redis#initialize', after: advice
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_advice(call, _ball)
+          return if call.raised
+
+          conn = call.instance.connection
+
+          DatabaseConnectionData.new(
+            transport: :redis,
+            host: conn[:host],
+            port: conn[:port],
+            db: conn[:db].to_s,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/dispatch_table.rb

@@ -0,0 +1,43 @@
+require 'logger'
+
+module Sqreen
+  module Ecosystem
+    # Configured by the ecosystem client
+    module DispatchTable
+      class << self
+        # data consumption
+        # argument: +Sqreen::Kit::Signals::Signal+
+        # see +Sqreen::EcosystemIntegration::SignalConsumption#consume_signal+
+        attr_accessor :consume_signal
+
+        # argument: block taking a Rack::Request
+        # see +Sqreen::EcosystemIntegration::RequestLifecycleTracking#add_start_observer+
+        attr_accessor :add_request_start_listener
+
+        attr_accessor :fetch_logger
+
+        # argument: callback taking:
+        # * the method to instrument
+        # * A Hash{Symbol=>Proc} with the advice. The proc takes the
+        # arguments and the ball, so these details of the instrumentation
+        # implementation leak through the abstraction
+        # see +Sqreen::EcosystemIntegration::InstrumentationService+
+        attr_accessor :instrument
+
+        def reset
+          instance_variables.each do |ia|
+            instance_variable_set(ia, nil)
+          end
+
+          # set default logger
+          logger = ::Logger.new(STDERR)
+          logger.level = ::Logger::WARN
+          logger.progname = 'sqreen-ecosystem'
+          self.fetch_logger = proc { logger }
+        end
+      end
+
+      reset
+    end
+  end
+end