sqreen 1.21.0.beta1 → 1.22.0

data/lib/sqreen/ecosystem/messaging/kinesis.rb

@@ -0,0 +1,66 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+# see https://aws.amazon.com/blogs/developer/announcing-amazon-kinesis-subscribetoshard-api-support-in-the-aws-sdk-for-ruby/
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Kinesis
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::Kinesis::Client#put_record',  after: advice_send
+          instrument 'Aws::Kinesis::Client#put_records', after: advice_send
+          # more sophisticated usages (register_stream_consumer, possibly with AsyncClient)
+          # are not supported. They are more difficult to test, as kinesalite doesn't support them
+          instrument 'Aws::Kinesis::Client#get_shard_iterator', after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to put_record(s)"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to get_shared_iterator"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(client, args, clazz)
+          hash = args[0]
+          stream_name = hash[:stream_name] || hash['stream_name']
+          return unless stream_name
+
+          endpoint = client.instance_variable_get(:@config).endpoint
+
+          clazz.new(
+            message_type: :'aws-kinesis',
+            host: endpoint.host,
+            topic: stream_name,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/messaging/sqs.rb

@@ -0,0 +1,68 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Sqs
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::SQS::Client#send_message',       after: advice_send
+          instrument 'Aws::SQS::Client#send_message_batch', after: advice_send
+          instrument 'Aws::SQS::Client#receive_message',    after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to send_message(_batch)"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to receive_message"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(args, clazz)
+          hash = args[0]
+          queue_url = hash[:queue_url] || hash['queue_url']
+          return unless queue_url
+
+          begin
+            uri = URI.parse(queue_url)
+          rescue URI::InvalidURIError
+            logger.info { "Invalid URI: #{uri}" }
+            return
+          end
+
+          clazz.new(
+            message_type: :'aws-sqs',
+            host: uri.host,
+            topic: ($1 if uri.path =~ /\A\/queue\/(?:[^\/]+\/)?([^\/]+)\z/),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/message_producer.rb

@@ -0,0 +1,57 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module MessageProducer
+        include Loggable
+
+        # method for ecosystem to inject the config
+        # @param [Sqreen::Ecosystem::TracingBroker]
+        attr_writer :tracing_broker
+
+        private
+
+        def determine_interest(type, hints = {})
+          @tracing_broker.interested_consumers(type, hints)
+        end
+
+        def publish(data, interested)
+          @tracing_broker.publish(data, interested)
+        end
+
+        # Convenience wrapper.
+        # Wraps a callback, skipping it if there is no interest in the type
+        # produced and submitting the return value as a message to the
+        # tracing broker
+        def wrap_for_interest(type, gen_hints = nil, &block)
+          raise ArgumentError, 'no block passed' if block.nil?
+
+          proc do |*args|
+            hints = gen_hints[*args] if gen_hints
+            interested = determine_interest(type, hints || {})
+
+            unless interested
+              logger.debug { "No interested consumers in #{type}" }
+              next
+            end
+
+            res = block[*args]
+
+            next if res.nil?
+
+            if res.is_a?(Array)
+              res.each do |d|
+                raise "unexpected return type: #{d.class}" unless d.is_a?(type)
+                @tracing_broker.publish(d, interested)
+              end
+            else
+              raise "unexpected return type: #{res.class}" unless res.is_a?(type)
+              @tracing_broker.publish(res, interested)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/signal_producer.rb

@@ -1,13 +1,11 @@
 require 'sqreen/ecosystem/dispatch_table'
-require 'sqreen/ecosystem/module_api/transaction_storage'
 
 module Sqreen
   module Ecosystem
     module ModuleApi
       module SignalProducer
-        include TransactionStorage
-
         # for injection
+        # callable taking no arguments and generating a tracing id
         attr_writer :tracing_id_producer
 
         private

data/lib/sqreen/ecosystem/module_api/tracing.rb

@@ -0,0 +1,45 @@
+require 'sqreen/ecosystem/module_api/signal_producer'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        include SignalProducer
+
+        def self.included(mod)
+          mod.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          private
+
+          # @param [Module] type The type the including module is interested in
+          def consumes(type)
+            @consumes = type
+          end
+
+          # A fixed (non-virtual) scope for this tracing module
+          # @param [String] scope
+          def fixed_scope(scope)
+            @fixed_scope = scope
+          end
+        end
+
+        def consumed_type
+          self.class.instance_variable_get(:@consumes) \
+            || raise('@consumes not specified')
+        end
+
+        def scope(_hints = {})
+          self.class.instance_variable_get(:@fixed_scope) \
+            || raise('@fixed_scope not set')
+        end
+
+        # including class must implement it
+        def receive(_data)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb

@@ -0,0 +1,31 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingClient::Payload+
+        #
+        # Signals are not produced by the data producers (transport)
+        # because of superior orders, as the only current use of this
+        # data is to generate signals.
+        module ClientData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :transport
+
+          # @return [String]
+          attr_accessor :host
+
+          # @return [String]
+          attr_accessor :ip
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/module_api/tracing/messaging_data'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        class ConsumerData
+          include MessagingData
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/messaging_data.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingConsumer::Payload+ and
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingProducer::Payload+.
+        #
+        # Signals are not produced by the data producers (transport)
+        # because of superior orders, as the only current use of this
+        # data is to generate signals.
+        module MessagingData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :message_type
+
+          # @return [String]
+          attr_accessor :host
+
+          # @return [String]
+          attr_accessor :ip
+
+          # @return [String]
+          attr_accessor :topic
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/producer_data.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/module_api/tracing/messaging_data'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        class ProducerData
+          include MessagingData
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/server_data.rb

@@ -0,0 +1,27 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingServer::Payload+
+        module ServerData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :transport
+
+          # @return [String]
+          attr_accessor :client_ip
+
+          # @return [Array<String>]
+          attr_accessor :previous_hops
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing_id_generation.rb

@@ -0,0 +1,16 @@
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module TracingIdGeneration
+        # for injection
+        attr_writer :tracing_id_producer
+
+        private
+
+        def create_tracing_id
+          @tracing_id_producer.call
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_registry.rb

@@ -16,6 +16,11 @@ module Sqreen
       def init_all
         logger.info { "Initializing #{@mods.size} ecosystem modules" }
         each_module do |mod|
+          unless mod.respond_to? :setup
+            logger.debug { "Module with type #{mod.class} requires no initialization" }
+            next
+          end
+
           logger.debug { "Initializing module with type #{mod.class}" }
           mod.setup
         end
@@ -34,6 +39,10 @@ module Sqreen
           selected_mods.each
         end
       end
+
+      def module_subset(type)
+        each_module(type).to_a
+      end
     end
   end
 end

data/lib/sqreen/ecosystem/tracing/modules/client.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/tracing/modules/determine_ip'
+require 'sqreen/ecosystem/tracing/signals/tracing_client'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/tracing'
+require 'sqreen/ecosystem/module_api/tracing/client_data'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        class Client
+          include ModuleApi::Tracing
+          include ModuleApi::TracingIdGeneration
+          include ModuleApi::Loggable
+
+          consumes    ModuleApi::Tracing::ClientData
+          fixed_scope 'client'
+
+          def receive(data)
+            signal = Tracing::Signals::TracingClient.new
+            signal.payload = Tracing::Signals::TracingClient::Payload.new(
+              transport: data.transport,
+              host: data.host || '',
+              ip: data.ip,
+              tracing_identifier: data.tracing_identifier || create_tracing_id
+            )
+
+            submit_signal signal
+          end
+        end
+      end
+    end
+  end
+end