sqreen 1.20.1-java → 1.22.0-java

data/lib/sqreen/ecosystem/messaging/kafka.rb

@@ -0,0 +1,70 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Kafka
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive))
+          instrument 'Kafka::Broker#produce',        after: advice_send
+          instrument 'Kafka::Broker#fetch_messages', after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send(call, _ball)
+          return if call.raised
+          if call.args.length != 1
+            logger.info "Expected 1 arguments to Kafka::Broker#produce"
+            return
+          end
+          options = call.args.first
+          topics = options[:messages_for_topics].keys
+
+          create_signal_data(ModuleApi::Tracing::ProducerData,
+                             call.instance,
+                             topics)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive(call, _ball)
+          return if call.raised
+          if call.args.length != 1
+            logger.info "Expected 1 arguments to Kafka::Broker#fetch_messages"
+            return
+          end
+          options = call.args.first
+          topics = options[:topics].keys
+
+          create_signal_data(ModuleApi::Tracing::ConsumerData,
+                             call.instance,
+                             topics)
+        end
+
+        # @param [Class] clazz
+        # @param [Kafka::Broker] broker
+        # @param [Array<String>] topics
+        def create_signal_data(clazz, broker, topics)
+          host = broker.instance_variable_get :@host
+          topics.map do |top|
+            clazz.new(
+              message_type: :kafka,
+              host: host,
+              topic: top,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/messaging/kinesis.rb

@@ -0,0 +1,66 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+# see https://aws.amazon.com/blogs/developer/announcing-amazon-kinesis-subscribetoshard-api-support-in-the-aws-sdk-for-ruby/
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Kinesis
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::Kinesis::Client#put_record',  after: advice_send
+          instrument 'Aws::Kinesis::Client#put_records', after: advice_send
+          # more sophisticated usages (register_stream_consumer, possibly with AsyncClient)
+          # are not supported. They are more difficult to test, as kinesalite doesn't support them
+          instrument 'Aws::Kinesis::Client#get_shard_iterator', after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to put_record(s)"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to get_shared_iterator"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(client, args, clazz)
+          hash = args[0]
+          stream_name = hash[:stream_name] || hash['stream_name']
+          return unless stream_name
+
+          endpoint = client.instance_variable_get(:@config).endpoint
+
+          clazz.new(
+            message_type: :'aws-kinesis',
+            host: endpoint.host,
+            topic: stream_name,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/messaging/sqs.rb

@@ -0,0 +1,68 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Sqs
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::SQS::Client#send_message',       after: advice_send
+          instrument 'Aws::SQS::Client#send_message_batch', after: advice_send
+          instrument 'Aws::SQS::Client#receive_message',    after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to send_message(_batch)"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to receive_message"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(args, clazz)
+          hash = args[0]
+          queue_url = hash[:queue_url] || hash['queue_url']
+          return unless queue_url
+
+          begin
+            uri = URI.parse(queue_url)
+          rescue URI::InvalidURIError
+            logger.info { "Invalid URI: #{uri}" }
+            return
+          end
+
+          clazz.new(
+            message_type: :'aws-sqs',
+            host: uri.host,
+            topic: ($1 if uri.path =~ /\A\/queue\/(?:[^\/]+\/)?([^\/]+)\z/),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api.rb

@@ -0,0 +1,30 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    # The API that the transport/tracing modules are written against
+    module ModuleApi
+      TRACE_ID_HEADER = 'X-Sqreen-Trace-Identifier'.freeze
+      TRACE_ID_ENV_KEY = 'HTTP_X_SQREEN_TRACE_IDENTIFIER'.freeze
+
+      Loggable = Sqreen::Ecosystem::Loggable
+
+      module ClassMethods
+        attr_writer :module_name
+
+        def module_name
+          if instance_variable_defined?(:@module_name)
+            @module_name
+          else
+            # to snake case
+            @module_name = to_s.sub(/.*::/, '').gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+          end
+        end
+      end
+
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/event_listener.rb

@@ -0,0 +1,18 @@
+require 'sqreen/ecosystem/dispatch_table'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module EventListener
+        private
+
+        # XXX: callbacks need to be wrapped in order ot handle
+        # perfcap, exceptions, and maybe other concerns applying
+        # across the board
+        def on_request_start(&cb)
+          DispatchTable.add_request_start_listener.call(cb)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/instrumentation.rb

@@ -0,0 +1,23 @@
+require 'sqreen/ecosystem/module_api'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Instrumentation
+        def self.included(mod)
+          mod.send :include, ModuleApi unless mod.ancestors.include?(ModuleApi)
+        end
+
+        private
+
+        # Just forwards the call to the instrumentation service
+        # @param [String] method
+        # @param [Hash{Symbol=>Proc}] advice keys are one of: :before, :after,
+        #                             :raised,
+        def instrument(method, advice)
+          DispatchTable.instrument.call(self.class.module_name, method, advice)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/message_producer.rb

@@ -0,0 +1,57 @@
+require 'sqreen/ecosystem/loggable'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module MessageProducer
+        include Loggable
+
+        # method for ecosystem to inject the config
+        # @param [Sqreen::Ecosystem::TracingBroker]
+        attr_writer :tracing_broker
+
+        private
+
+        def determine_interest(type, hints = {})
+          @tracing_broker.interested_consumers(type, hints)
+        end
+
+        def publish(data, interested)
+          @tracing_broker.publish(data, interested)
+        end
+
+        # Convenience wrapper.
+        # Wraps a callback, skipping it if there is no interest in the type
+        # produced and submitting the return value as a message to the
+        # tracing broker
+        def wrap_for_interest(type, gen_hints = nil, &block)
+          raise ArgumentError, 'no block passed' if block.nil?
+
+          proc do |*args|
+            hints = gen_hints[*args] if gen_hints
+            interested = determine_interest(type, hints || {})
+
+            unless interested
+              logger.debug { "No interested consumers in #{type}" }
+              next
+            end
+
+            res = block[*args]
+
+            next if res.nil?
+
+            if res.is_a?(Array)
+              res.each do |d|
+                raise "unexpected return type: #{d.class}" unless d.is_a?(type)
+                @tracing_broker.publish(d, interested)
+              end
+            else
+              raise "unexpected return type: #{res.class}" unless res.is_a?(type)
+              @tracing_broker.publish(res, interested)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/signal_producer.rb

@@ -0,0 +1,24 @@
+require 'sqreen/ecosystem/dispatch_table'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module SignalProducer
+        # for injection
+        # callable taking no arguments and generating a tracing id
+        attr_writer :tracing_id_producer
+
+        private
+
+        def create_tracing_id
+          @tracing_id_producer.call
+        end
+
+        # @param [Sqreen::Kit::Signals::Signal] signal
+        def submit_signal(signal)
+          DispatchTable.consume_signal.call(signal)
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing.rb

@@ -0,0 +1,45 @@
+require 'sqreen/ecosystem/module_api/signal_producer'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        include SignalProducer
+
+        def self.included(mod)
+          mod.extend(ClassMethods)
+        end
+
+        module ClassMethods
+          private
+
+          # @param [Module] type The type the including module is interested in
+          def consumes(type)
+            @consumes = type
+          end
+
+          # A fixed (non-virtual) scope for this tracing module
+          # @param [String] scope
+          def fixed_scope(scope)
+            @fixed_scope = scope
+          end
+        end
+
+        def consumed_type
+          self.class.instance_variable_get(:@consumes) \
+            || raise('@consumes not specified')
+        end
+
+        def scope(_hints = {})
+          self.class.instance_variable_get(:@fixed_scope) \
+            || raise('@fixed_scope not set')
+        end
+
+        # including class must implement it
+        def receive(_data)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/client_data.rb

@@ -0,0 +1,31 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingClient::Payload+
+        #
+        # Signals are not produced by the data producers (transport)
+        # because of superior orders, as the only current use of this
+        # data is to generate signals.
+        module ClientData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :transport
+
+          # @return [String]
+          attr_accessor :host
+
+          # @return [String]
+          attr_accessor :ip
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/module_api/tracing/messaging_data'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        class ConsumerData
+          include MessagingData
+        end
+      end
+    end
+  end
+end