sqreen 1.20.4 → 1.21.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7324df5cecbb626299a4e048550c6f7c5a3e15bef2eecc8d89011d0342914aa
-  data.tar.gz: eabb5203769dcf898d8a4e373c6cfee7e5b0eac8bbfc3977fb8924a54504b126
+  metadata.gz: 6611caf7add24e05a248fce4026be48a4c47612335c5fc9a26c64c33cd0409e9
+  data.tar.gz: 8381eba611e37df32a8744848661bcf144eaffe00c818e306355508fe0267394
 SHA512:
-  metadata.gz: 5e7f4bb53e01c0f5328a5190b189c82b596bcf28eada2104b1491c5601b1275866ef8950ae9ad92c91303011636a3c11ebcb3c3dbccd75280d6b89c1c69d301a
-  data.tar.gz: 77770ff7b00b9d07ea7f4137eadce1e840bc1ad94e885b42623354707306ebd42425187833e0403e33d02bea7471010bf934bda9010c5fe1d604715c5daf88c1
+  metadata.gz: 65c820c306961ed812360c2c73a1f37396433238cff8fd79e2676132d7842eaa86d3555bf860d7c2f6fd465783f8c1aeb59100c94349b8c53588d83513265d2e
+  data.tar.gz: 38cda532df6f1037ac243c8cd5ccaecd708b880ff377d3fcbcecad329be184d40ec9f27deb61b69124be3d37ff74a16f88a7ab1032a6e6a0f6b5387645181027

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 1.21.1
+
+* Work around NewRelic initialisation (see https://github.com/newrelic/newrelic-ruby-agent/issues/461)
+
+## 1.21.0
+
+* Add support for transport and tracing facilities
+
 ## 1.20.4
 
 * Fix missing budget check

data/lib/sqreen/deliveries/batch.rb

@@ -13,6 +13,8 @@ require 'sqreen/events/attack'
 require 'sqreen/events/remote_exception'
 require 'sqreen/mono_time'
 require 'sqreen/deliveries/simple'
+require 'sqreen/kit/signals/signal'
+require 'sqreen/kit/signals/trace'
 
 module Sqreen
   module Deliveries
@@ -58,7 +60,7 @@ module Sqreen
       def post_batch_needed?(event)
         now = Sqreen.time
         # do not use any? {} due to side effects inside block
-        event_keys(event).map do |key|
+        event_keys(event).uniq.map do |key|
           was = @first_seen[key]
           @first_seen[key] ||= now
           was.nil? || current_batch.size > max_batch || now > (was + max_staleness)
@@ -86,6 +88,7 @@ module Sqreen
         res += event.observed.fetch(:sdk, []).select { |e|
             e[0] == :track
         }.map { |e| "sdk-track".freeze }
+        res += event.observed.fetch(:signals, []).map { "signal".freeze }
         return res
       end
 
@@ -97,6 +100,10 @@ module Sqreen
           "rex-#{event.klass}"
         when Sqreen::AggregatedMetric
           "agg-metric"
+        when Sqreen::Kit::Signals::Signal
+          "signal"
+        when Sqreen::Kit::Signals::Trace
+          "signal"
         end
       end
     end

data/lib/sqreen/dependency/detector.rb

@@ -25,6 +25,14 @@ module Sqreen
         end
       end
 
+      def to_app_hook_strategy
+        if Sqreen::Dependency::NewRelic.bundled? || Sqreen::Dependency::NewRelic.required?
+          :chain
+        else
+          :prepend
+        end
+      end
+
       def hook(&block)
         Sqreen.log.debug "[#{Process.pid}] Startup command: #{$0}"
 
@@ -34,7 +42,7 @@ module Sqreen
           Sqreen::Dependency::Rails.insert_sqreen_middlewares
         end if Sqreen::Dependency::Rails.required?
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do
             Sqreen::Dependency::Rails.inspect_middlewares
           end
@@ -48,7 +56,7 @@ module Sqreen
           end
         end.install if Sqreen::Dependency::Sinatra.required?
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do |call|
             builder = call.instance
 
@@ -58,7 +66,7 @@ module Sqreen
 
         # ensure startup of thread in request handling processes
 
-        Sqreen::Graft::Hook.add('Rack::Builder#to_app') do
+        Sqreen::Graft::Hook.add('Rack::Builder#to_app', to_app_hook_strategy) do
           after do |call|
             callback = call.callback
 

data/lib/sqreen/dependency/new_relic.rb

@@ -8,8 +8,17 @@ module Sqreen
     module NewRelic
       module_function
 
+      def bundled?
+        defined?(Gem) && Gem.respond_to?(:loaded_specs) && !Gem.loaded_specs['newrelic_rpm'].nil?
+      end
+
+      def required?
+        Sqreen::Dependency.const_exist?('NewRelic::Agent::Agent')
+      end
+
       def ignore_sqreen_exceptions
-        return unless defined?(NewRelic::Agent::Agent)
+        return unless required?
+
         NewRelic::Agent::Agent.instance.error_collector.ignore(['Sqreen::AttackBlocked'])
       rescue ::Exception => e # rubocop:disable Lint/RescueException
         Sqreen.log.warn "Failed ignoring AttackBlocked on NewRelic: #{e.inspect}"

data/lib/sqreen/ecosystem.rb

@@ -0,0 +1,123 @@
+require 'securerandom'
+require 'sqreen/ecosystem/module_registry'
+require 'sqreen/ecosystem/tracing/sampling_configuration'
+require 'sqreen/ecosystem/transaction_storage'
+require 'sqreen/ecosystem/tracing_broker'
+require 'sqreen/ecosystem/tracing_id_setup'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing'
+
+module Sqreen
+  # The API for the ecosystem client (together with the dispatch table)
+  module Ecosystem
+    class << self
+      def init(opts = {})
+        @registry = ModuleRegistry.new
+        register_modules(opts[:modules])
+        @registry.init_all
+
+        # setup tracing generation
+        tracing_id_mods = @registry.module_subset(ModuleApi::TracingIdGeneration)
+        @tracing_id_setup = TracingIdSetup.new(tracing_id_mods)
+        @tracing_id_setup.setup_modules
+
+        # configure tracing broker with the consumers (tracing modules)
+        tracing_modules = @registry.module_subset(ModuleApi::Tracing)
+        @tracing_broker = TracingBroker.new(tracing_modules)
+
+        # inject tracing broker in message producers
+        @registry.each_module(ModuleApi::MessageProducer) do |mod|
+          mod.tracing_broker = @tracing_broker
+        end
+      rescue ::Exception # rubocop:disable Lint/RescueException
+        # TODO: modules must be disabled at this point
+        raise
+      end
+
+      def reset
+        instance_variables.each do |ia|
+          instance_variable_set(ia, nil)
+        end
+      end
+
+      # To be called by the Ecosystem client when a new transaction
+      # (generally: request) is started
+      # In the future, it's intended that request end/start detection be handled
+      # by the Ecosystem itself, so control will flow in the other direction,
+      # from the ecosystem to its client
+      def start_transaction
+        TransactionStorage.create_thread_local
+      end
+
+      def end_transaction
+        TransactionStorage.destroy_thread_local
+      end
+
+      # @param [String] tracing_id_prefix
+      # @param [Array<Hash{String=>Object}>] sampling_config
+      def configure_sampling(tracing_id_prefix, sampling_config)
+        @tracing_id_setup.tracing_id_prefix = tracing_id_prefix
+        built_samp_cfg = Tracing::SamplingConfiguration.new(sampling_config)
+        @tracing_broker.sampling_configuration = built_samp_cfg
+      end
+
+      private
+
+      def register_modules(modules)
+        return register_all_modules unless modules
+
+        modules.each { |mod| register mod }
+      end
+
+      def register_all_modules
+        # replace with something more magical?
+        require_relative 'ecosystem/http/rack_request'
+        register Http::RackRequest.new
+
+        require_relative 'ecosystem/http/net_http'
+        register Http::NetHttp.new
+
+        require_relative 'ecosystem/databases/postgres'
+        register Databases::Postgres.new
+
+        require_relative 'ecosystem/databases/mysql'
+        register Databases::Mysql.new
+
+        require_relative 'ecosystem/databases/mongo'
+        register Databases::Mongo.new
+
+        require_relative 'ecosystem/databases/redis'
+        register Databases::Redis.new
+
+        require_relative 'ecosystem/messaging/sqs'
+        register Messaging::Sqs.new
+
+        require_relative 'ecosystem/messaging/kinesis'
+        register Messaging::Kinesis.new
+
+        require_relative 'ecosystem/messaging/bunny'
+        register Messaging::Bunny.new
+
+        require_relative 'ecosystem/messaging/kafka'
+        register Messaging::Kafka.new
+
+        require_relative 'ecosystem/tracing/modules/client'
+        register Tracing::Modules::Client.new
+
+        require_relative 'ecosystem/tracing/modules/server'
+        register Tracing::Modules::Server.new
+
+        require_relative 'ecosystem/tracing/modules/producer'
+        register Tracing::Modules::Producer.new
+
+        require_relative 'ecosystem/tracing/modules/consumer'
+        register Tracing::Modules::Consumer.new
+      end
+
+      def register(mod)
+        @registry.register mod
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/database_connection_data.rb

@@ -0,0 +1,23 @@
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class DatabaseConnectionData
+        include ModuleApi::Tracing::ClientData
+
+        # @return [Integer]
+        attr_accessor :port
+
+        # @return [String]
+        attr_accessor :unix_socket
+
+        # @return [String]
+        attr_accessor :username
+
+        # @return [String]
+        attr_accessor :db
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/mongo.rb

@@ -0,0 +1,39 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Mongo
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Mongo::Client#initialize', after: advice
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_advice(call, _ball)
+          return if call.raised
+
+          client = call.instance
+          server_addrs = client.cluster.servers.map(&:address)
+
+          server_addrs.map do |addr|
+            DatabaseConnectionData.new(
+              transport: :mongo,
+              host: addr.host,
+              port: addr.port,
+              db: client.database.name,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/mysql.rb

@@ -0,0 +1,54 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing_id_generation'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Mysql
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Mysql2::Client#connect', after: advice
+        end
+
+        private
+
+        # instance is of type +Mysql2::Client+
+        # VALUE rb_mysql_connect(VALUE self, VALUE user, VALUE pass, VALUE host, VALUE port, VALUE database,
+        #                        VALUE socket, VALUE flags, VALUE conn_attrs) {
+        # @param [Sqreen::Graft::CallbackCall]
+        def after_advice(call, _ball)
+          args = call.args
+
+          # build & submit signal
+          signal = DatabaseConnectionData.new(transport: :mysql)
+
+          user = args[0]
+          host = args[2]
+          port = args[3]
+          db = args[4]
+          socket = args[5]
+
+          if socket && !socket.empty?
+            signal.unix_socket = socket
+            signal.host = 'localhost'
+            signal.ip = '::1'
+          else
+            signal.host = host
+          end
+
+          signal.port = port if port != 0
+          signal.username = user
+          signal.db = db
+
+          signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/postgres.rb

@@ -0,0 +1,51 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Postgres
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'PG::Connection#initialize', after: advice
+        end
+
+        private
+
+        # instance is of type +PG::Connection+
+        # > c = PG::Connection.new(host: '172.17.0.2', password: 'mysecretpassword', user: 'postgres')
+        #  => #<PG::Connection:0x000055b44d077d10>
+        # > %i{host port user db}.map { |m| c.send m }
+        #  => ["172.17.0.2", 5432, "postgres", "postgres"]
+        def after_advice(call, _ball)
+          conn = call.instance
+
+          # build & submit signal
+          signal = DatabaseConnectionData.new(transport: :postgres)
+
+          host = conn.host
+          if host
+            if host.include?('/')
+              signal.unix_socket = host
+              signal.host = 'localhost'
+              signal.ip = '::1'
+            else
+              signal.host = host
+            end
+          end
+
+          signal.port = conn.port
+          signal.username = conn.user
+          signal.db = conn.db
+
+          signal
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/databases/redis.rb

@@ -0,0 +1,36 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/databases/database_connection_data'
+
+module Sqreen
+  module Ecosystem
+    module Databases
+      class Redis
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice = wrap_for_interest(DatabaseConnectionData, &method(:after_advice))
+          instrument 'Redis#initialize', after: advice
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_advice(call, _ball)
+          return if call.raised
+
+          conn = call.instance.connection
+
+          DatabaseConnectionData.new(
+            transport: :redis,
+            host: conn[:host],
+            port: conn[:port],
+            db: conn[:db].to_s,
+          )
+        end
+      end
+    end
+  end
+end