sqreen 1.21.0.beta3-java → 1.21.0-java

data/lib/sqreen/ecosystem/messaging/kafka.rb

@@ -0,0 +1,70 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Kafka
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive))
+          instrument 'Kafka::Broker#produce',        after: advice_send
+          instrument 'Kafka::Broker#fetch_messages', after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send(call, _ball)
+          return if call.raised
+          if call.args.length != 1
+            logger.info "Expected 1 arguments to Kafka::Broker#produce"
+            return
+          end
+          options = call.args.first
+          topics = options[:messages_for_topics].keys
+
+          create_signal_data(ModuleApi::Tracing::ProducerData,
+                             call.instance,
+                             topics)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive(call, _ball)
+          return if call.raised
+          if call.args.length != 1
+            logger.info "Expected 1 arguments to Kafka::Broker#fetch_messages"
+            return
+          end
+          options = call.args.first
+          topics = options[:topics].keys
+
+          create_signal_data(ModuleApi::Tracing::ConsumerData,
+                             call.instance,
+                             topics)
+        end
+
+        # @param [Class] clazz
+        # @param [Kafka::Broker] broker
+        # @param [Array<String>] topics
+        def create_signal_data(clazz, broker, topics)
+          host = broker.instance_variable_get :@host
+          topics.map do |top|
+            clazz.new(
+              message_type: :kafka,
+              host: host,
+              topic: top,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/messaging/kinesis.rb

@@ -0,0 +1,66 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+# see https://aws.amazon.com/blogs/developer/announcing-amazon-kinesis-subscribetoshard-api-support-in-the-aws-sdk-for-ruby/
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Kinesis
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::Kinesis::Client#put_record',  after: advice_send
+          instrument 'Aws::Kinesis::Client#put_records', after: advice_send
+          # more sophisticated usages (register_stream_consumer, possibly with AsyncClient)
+          # are not supported. They are more difficult to test, as kinesalite doesn't support them
+          instrument 'Aws::Kinesis::Client#get_shard_iterator', after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to put_record(s)"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to get_shared_iterator"
+            return
+          end
+
+          create_signal(call.instance, call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(client, args, clazz)
+          hash = args[0]
+          stream_name = hash[:stream_name] || hash['stream_name']
+          return unless stream_name
+
+          endpoint = client.instance_variable_get(:@config).endpoint
+
+          clazz.new(
+            message_type: :'aws-kinesis',
+            host: endpoint.host,
+            topic: stream_name,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/messaging/sqs.rb

@@ -0,0 +1,68 @@
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/instrumentation'
+require 'sqreen/ecosystem/module_api/message_producer'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Messaging
+      class Sqs
+        include ModuleApi::Loggable
+        include ModuleApi::Instrumentation
+        include ModuleApi::MessageProducer
+
+        def setup
+          advice_send = wrap_for_interest(ModuleApi::Tracing::ProducerData, &method(:after_send_advice))
+          advice_receive = wrap_for_interest(ModuleApi::Tracing::ConsumerData, &method(:after_receive_advice))
+          instrument 'Aws::SQS::Client#send_message',       after: advice_send
+          instrument 'Aws::SQS::Client#send_message_batch', after: advice_send
+          instrument 'Aws::SQS::Client#receive_message',    after: advice_receive
+        end
+
+        private
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_send_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to send_message(_batch)"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ProducerData)
+        end
+
+        # @param [Sqreen::Graft::CallbackCall] call
+        def after_receive_advice(call, _ball)
+          return if call.raised
+          unless call.args.length > 0 && call.args[0].is_a?(Hash)
+            logger.info "Unexpected arguments to receive_message"
+            return
+          end
+
+          create_signal(call.args, ModuleApi::Tracing::ConsumerData)
+        end
+
+        def create_signal(args, clazz)
+          hash = args[0]
+          queue_url = hash[:queue_url] || hash['queue_url']
+          return unless queue_url
+
+          begin
+            uri = URI.parse(queue_url)
+          rescue URI::InvalidURIError
+            logger.info { "Invalid URI: #{uri}" }
+            return
+          end
+
+          clazz.new(
+            message_type: :'aws-sqs',
+            host: uri.host,
+            topic: ($1 if uri.path =~ /\A\/queue\/(?:[^\/]+\/)?([^\/]+)\z/),
+          )
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/message_producer.rb

@@ -40,9 +40,15 @@ module Sqreen
 
             next if res.nil?
 
-            raise 'unexpected return type' unless res.is_a?(type)
-
-            @tracing_broker.publish(res, interested)
+            if res.is_a?(Array)
+              res.each do |d|
+                raise "unexpected return type: #{d.class}" unless d.is_a?(type)
+                @tracing_broker.publish(d, interested)
+              end
+            else
+              raise "unexpected return type: #{res.class}" unless res.is_a?(type)
+              @tracing_broker.publish(res, interested)
+            end
           end
         end
       end

data/lib/sqreen/ecosystem/module_api/tracing/consumer_data.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/module_api/tracing/messaging_data'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        class ConsumerData
+          include MessagingData
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/messaging_data.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/util/call_writers_from_init'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        # The data the tracing module needs in order to populate
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingConsumer::Payload+ and
+        # +Sqreen::Ecosystem::Tracing::Signals::TracingProducer::Payload+.
+        #
+        # Signals are not produced by the data producers (transport)
+        # because of superior orders, as the only current use of this
+        # data is to generate signals.
+        module MessagingData
+          include Util::CallWritersFromInit
+
+          # @return [Symbol]
+          attr_accessor :message_type
+
+          # @return [String]
+          attr_accessor :host
+
+          # @return [String]
+          attr_accessor :ip
+
+          # @return [String]
+          attr_accessor :topic
+
+          # @return [String]
+          attr_accessor :tracing_identifier
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_api/tracing/producer_data.rb

@@ -0,0 +1,13 @@
+require 'sqreen/ecosystem/module_api/tracing/messaging_data'
+
+module Sqreen
+  module Ecosystem
+    module ModuleApi
+      module Tracing
+        class ProducerData
+          include MessagingData
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/module_registry.rb

@@ -16,7 +16,11 @@ module Sqreen
       def init_all
         logger.info { "Initializing #{@mods.size} ecosystem modules" }
         each_module do |mod|
-          next unless mod.respond_to? :setup
+          unless mod.respond_to? :setup
+            logger.debug { "Module with type #{mod.class} requires no initialization" }
+            next
+          end
+
           logger.debug { "Initializing module with type #{mod.class}" }
           mod.setup
         end

data/lib/sqreen/ecosystem/tracing/modules/client.rb

@@ -1,4 +1,7 @@
+require 'sqreen/ecosystem/tracing/modules/determine_ip'
 require 'sqreen/ecosystem/tracing/signals/tracing_client'
+require 'sqreen/ecosystem/loggable'
+require 'sqreen/ecosystem/module_api'
 require 'sqreen/ecosystem/module_api/tracing'
 require 'sqreen/ecosystem/module_api/tracing/client_data'
 
@@ -8,18 +11,19 @@ module Sqreen
       module Modules
         class Client
           include ModuleApi::Tracing
+          include ModuleApi::TracingIdGeneration
+          include ModuleApi::Loggable
 
           consumes    ModuleApi::Tracing::ClientData
           fixed_scope 'client'
 
-          # @param [Sqreen::Ecosystem::ModuleApi::Tracing::ClientData] data
           def receive(data)
             signal = Tracing::Signals::TracingClient.new
             signal.payload = Tracing::Signals::TracingClient::Payload.new(
               transport: data.transport,
-              host: data.host,
+              host: data.host || '',
               ip: data.ip,
-              tracing_identifier: data.tracing_identifier
+              tracing_identifier: data.tracing_identifier || create_tracing_id
             )
 
             submit_signal signal

data/lib/sqreen/ecosystem/tracing/modules/consumer.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/tracing/modules/determine_ip'
+require 'sqreen/ecosystem/tracing/signals/tracing_consumer'
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/tracing/consumer_data'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        class Consumer
+          include ModuleApi::Tracing
+          include ModuleApi::TracingIdGeneration
+          include ModuleApi::Loggable
+
+          consumes    ModuleApi::Tracing::ConsumerData
+          fixed_scope 'consumer'
+
+          # @param [ModuleApi::Tracing::ConsumerData] data
+          def receive(data)
+            signal = Tracing::Signals::TracingConsumer.new
+            signal.payload = Tracing::Signals::TracingConsumer::Payload.new(
+              message_type: data.message_type,
+              host: data.host || '',
+              ip: data.ip,
+              topic: data.topic,
+              tracing_identifier: data.tracing_identifier || create_tracing_id
+            )
+
+            submit_signal signal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/modules/determine_ip.rb

@@ -0,0 +1,28 @@
+require 'sqreen/ecosystem/module_api'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        module DetermineIp
+          class << self
+            include ModuleApi::Loggable
+
+            def [](data)
+              return data.ip if data.ip
+
+              return nil unless data.host && !data.host.empty?
+
+              begin
+                IPSocket.getaddress data.host
+              rescue SocketError => e
+                logger.info { "Error resolving #{data.host}: #{e.message}" }
+                nil
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/modules/producer.rb

@@ -0,0 +1,35 @@
+require 'sqreen/ecosystem/tracing/modules/determine_ip'
+require 'sqreen/ecosystem/tracing/signals/tracing_producer'
+require 'sqreen/ecosystem/module_api'
+require 'sqreen/ecosystem/module_api/tracing/producer_data'
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Modules
+        class Producer
+          include ModuleApi::Tracing
+          include ModuleApi::TracingIdGeneration
+          include ModuleApi::Loggable
+
+          consumes    ModuleApi::Tracing::ProducerData
+          fixed_scope 'producer'
+
+          # @param [ModuleApi::Tracing::ProducerData] data
+          def receive(data)
+            signal = Tracing::Signals::TracingProducer.new
+            signal.payload = Tracing::Signals::TracingProducer::Payload.new(
+              message_type: data.message_type,
+              host: data.host || '',
+              ip: data.ip,
+              topic: data.topic,
+              tracing_identifier: data.tracing_identifier || create_tracing_id
+            )
+
+            submit_signal signal
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sqreen/ecosystem/tracing/signals/tracing_consumer.rb

@@ -0,0 +1,56 @@
+require 'sqreen/kit/configuration'
+require 'sqreen/kit/signals/point'
+require 'sqreen/kit/signals/dto_helper'
+
+# reference: https://github.com/sqreen/SignalsSchemas/blob/master/schemas/payload/tracing/consumer-2020-04-21/schema.cue
+
+module Sqreen
+  module Ecosystem
+    module Tracing
+      module Signals
+      end
+    end
+  end
+end
+
+class Sqreen::Ecosystem::Tracing::Signals::TracingConsumer < Sqreen::Kit::Signals::Point
+  readonly_attrs :payload_schema, :source, :signal_name
+
+  def initialize(values = {})
+    self.payload_schema = Payload::SCHEMA_VERSION
+    self.source = Sqreen::Kit::Configuration.default_source
+    self.signal_name = 'tracing.consumer'
+    self.time = values[:time] || Time.now
+    super
+  end
+
+  def payload=(payload)
+    unless payload.is_a?(Payload)
+      raise ArgumentError, "Payload should be a #{Payload}"
+    end
+    super
+  end
+
+  class Payload
+    include Sqreen::Kit::Signals::DtoHelper
+
+    add_mandatory_attrs :message_type, :host, :tracing_identifier
+
+    SCHEMA_VERSION = 'tracing/consumer-2020-04-21'.freeze
+
+    # @return [Symbol]
+    attr_accessor :message_type
+
+    # @return [String]
+    attr_accessor :host
+
+    # @return [String]
+    attr_accessor :ip
+
+    # @return [String]
+    attr_accessor :topic
+
+    # @return [String]
+    attr_accessor :tracing_identifier
+  end
+end