julewire-karafka 1.0.0

data/lib/julewire/karafka/message_context.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module MessageContext
+      class << self
+        def call(message, configuration:, fields: nil, &)
+          raise ArgumentError, "block required" unless block_given?
+
+          fields ||= PayloadReader.message_payload(message)
+          carrier = carrier_for(fields, configuration)
+
+          Julewire::Core::Propagation::Carrier.restore(carrier, key: configuration.carrier_key) do
+            Julewire::Core::Integration::Facade.with_neutral(message_neutral(fields)) do
+              Julewire::Core::Integration::Facade.with_attributes(message_attributes(fields), &)
+            end
+          end
+        end
+
+        private
+
+        def carrier_for(fields, configuration)
+          return {} unless configuration.propagation?
+
+          headers = fields[:headers].is_a?(Hash) ? fields[:headers] : {}
+          filter = configuration.carrier_filter
+          return headers unless filter
+
+          filtered = filter.call(headers, message: fields)
+          filtered.is_a?(Hash) ? filtered : {}
+        rescue StandardError => e
+          IntegrationHealth.record_failure(e, action: :carrier_filter, component: :message_context)
+          {}
+        end
+
+        def message_attributes(fields) = { karafka: fields }
+
+        def message_neutral(fields) = MessagingAttributes.message(fields)
+      end
+    end
+  end
+end

data/lib/julewire/karafka/message_execution.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module MessageExecution
+      DEFAULT_TYPE = :karafka_message
+      DEFAULT_SUMMARY_EVENT = "message.completed"
+
+      class << self
+        def call(message, configuration: Configuration.new, **options, &)
+          raise ArgumentError, "block required" unless block_given?
+
+          fields = PayloadReader.message_payload(message)
+          execution_fields = execution_fields(options)
+          type = execution_fields.delete(:type) || DEFAULT_TYPE
+          id = execution_fields.delete(:id) || execution_id(fields)
+          emit_summary = execution_fields.delete(:emit_summary) { true }
+          summary_event = execution_fields.delete(:summary_event) || DEFAULT_SUMMARY_EVENT
+          summary_severity = execution_fields.delete(:summary_severity)
+          summary_source = execution_fields.delete(:summary_source) || configuration.source
+          MessageContext.call(message, configuration: configuration, fields: fields) do
+            Julewire::Core::Integration::Facade.with_execution(
+              type: type,
+              id: id,
+              emit_summary: emit_summary,
+              fields: execution_fields,
+              summary_event: summary_event,
+              summary_severity: summary_severity,
+              summary_source: summary_source,
+              &
+            )
+          end
+        end
+
+        private
+
+        def execution_fields(options)
+          values = Core::Integration::Values::Shape
+          fields = values.hash_or_empty(options)
+          fields.empty? ? {} : fields
+        end
+
+        def execution_id(fields)
+          topic = fields[:topic]
+          partition = fields[:partition]
+          offset = fields[:offset]
+          return if topic.nil? || partition.nil? || offset.nil?
+
+          "#{topic}:#{partition}:#{offset}"
+        end
+      end
+    end
+  end
+end

data/lib/julewire/karafka/messaging_attributes.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module MessagingAttributes
+      class << self
+        def message(fields)
+          Core::Fields::AttributeKeys.fields(
+            Core::Fields::AttributeKeys::MESSAGING_SYSTEM => "kafka",
+            Core::Fields::AttributeKeys::MESSAGING_OPERATION_NAME => "process",
+            Core::Fields::AttributeKeys::MESSAGING_OPERATION_TYPE => "process",
+            Core::Fields::AttributeKeys::MESSAGING_DESTINATION_NAME => fields[:topic],
+            Core::Fields::AttributeKeys::MESSAGING_DESTINATION_PARTITION_ID => string_value(fields[:partition]),
+            Core::Fields::AttributeKeys::MESSAGING_KAFKA_OFFSET => string_value(fields[:offset]),
+            Core::Fields::AttributeKeys::MESSAGING_KAFKA_MESSAGE_KEY => string_value(fields[:key])
+          )
+        end
+
+        def monitor(name, payload, role:)
+          Core::Fields::AttributeKeys.fields(
+            Core::Fields::AttributeKeys::MESSAGING_SYSTEM => "kafka",
+            Core::Fields::AttributeKeys::MESSAGING_OPERATION_NAME => name.to_s,
+            Core::Fields::AttributeKeys::MESSAGING_OPERATION_TYPE => operation_type(name, role: role),
+            Core::Fields::AttributeKeys::MESSAGING_DESTINATION_NAME => payload[:topic] || payload.dig(:message, :topic),
+            Core::Fields::AttributeKeys::MESSAGING_DESTINATION_PARTITION_ID => string_value(
+              payload[:partition] || payload.dig(:message, :partition)
+            ),
+            Core::Fields::AttributeKeys::MESSAGING_BATCH_MESSAGE_COUNT => message_count(payload),
+            Core::Fields::AttributeKeys::MESSAGING_CONSUMER_GROUP_NAME => payload[:consumer_group],
+            Core::Fields::AttributeKeys::MESSAGING_KAFKA_OFFSET => string_value(first_offset(payload))
+          )
+        end
+
+        private
+
+        def message_count(payload)
+          payload[:messages_count] || payload.dig(:messages, :count)
+        end
+
+        def first_offset(payload)
+          payload[:first_offset] || payload.dig(:message, :offset)
+        end
+
+        def operation_type(name, role:)
+          return "send" if role == :producer
+
+          "receive" if name.to_s.include?("consume")
+        end
+
+        def string_value(value)
+          value&.to_s
+        end
+      end
+    end
+  end
+end

data/lib/julewire/karafka/monitor_listener.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    class MonitorListener
+      Profile = Data.define(
+        :component,
+        :event_prefix,
+        :logger_name,
+        :messaging_role,
+        :config_method,
+        :important_events,
+        :severity
+      )
+
+      CONSUMER_PROFILE = Profile.new(
+        component: :listener,
+        event_prefix: "karafka",
+        logger_name: "Karafka.monitor",
+        messaging_role: :consumer,
+        config_method: :consumer_event_names,
+        important_events: Configuration::IMPORTANT_CONSUMER_EVENT_NAMES,
+        severity: ->(name, event, payload) { EventSeverity.consumer(name, event: event, payload: payload) }
+      ).freeze
+      PRODUCER_PROFILE = Profile.new(
+        component: :waterdrop_listener,
+        event_prefix: "waterdrop",
+        logger_name: "WaterDrop.monitor",
+        messaging_role: :producer,
+        config_method: :producer_event_names,
+        important_events: Configuration::IMPORTANT_PRODUCER_EVENT_NAMES,
+        severity: ->(name, _event, payload) { EventSeverity.producer(name, payload) }
+      ).freeze
+      private_constant :Profile, :CONSUMER_PROFILE, :PRODUCER_PROFILE
+
+      class << self
+        def consumer(configuration = Configuration.new)
+          new(configuration, profile: CONSUMER_PROFILE)
+        end
+
+        def producer(configuration = Configuration.new)
+          new(configuration, profile: PRODUCER_PROFILE)
+        end
+      end
+
+      def initialize(configuration = Configuration.new, profile:)
+        @configuration = configuration
+        @profile = profile
+      end
+
+      attr_writer :configuration
+
+      def emit(name, event)
+        IntegrationHealth.with_failure_health(
+          action: :emit,
+          component: @profile.component,
+          event: name
+        ) do
+          payload = EventPayload.call(name, event)
+          Core::Integration::Facade.emit(
+            severity: @profile.severity.call(name, event, payload),
+            event: "#{@profile.event_prefix}.#{name.tr(".", "_")}",
+            logger: @profile.logger_name,
+            source: @configuration.source,
+            error: EventPayload.error(event),
+            neutral: messaging_attributes(name, payload),
+            attributes: event_attributes(name, payload)
+          )
+        end
+        nil
+      end
+
+      def event_attributes(_name, payload)
+        Core.deep_compact_empty(@profile.event_prefix.to_sym => payload)
+      end
+
+      def messaging_attributes(name, payload)
+        MessagingAttributes.monitor(name, payload, role: @profile.messaging_role)
+      end
+    end
+  end
+end

data/lib/julewire/karafka/monitor_subscription.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module MonitorSubscription
+      PROFILE_CONSTANTS = {
+        consumer: :CONSUMER_PROFILE,
+        producer: :PRODUCER_PROFILE
+      }.freeze
+      private_constant :PROFILE_CONSTANTS
+
+      class << self
+        def install!(monitor, profile:, configuration: Configuration.new)
+          profile = monitor_listener_profile(profile)
+          state = subscription_state(monitor, profile)
+          listener = listener_for(state, configuration, profile)
+          subscriptions = subscriptions_for(state)
+          desired_events = event_names(monitor, configuration, profile)
+
+          unsubscribe_removed_events(monitor, subscriptions, desired_events, profile)
+
+          desired_events.each do |event_name|
+            next if subscriptions.key?(event_name)
+
+            callback = ->(event) { listener.emit(event_name, event) }
+            subscriptions[event_name] = callback if subscribe_event(monitor, event_name, profile, &callback)
+          end
+          store_subscription_state(monitor, listener: listener, subscriptions: subscriptions, profile: profile)
+          listener
+        end
+
+        private
+
+        def monitor_listener_profile(profile)
+          constant_name = PROFILE_CONSTANTS.fetch(profile) { return profile }
+          MonitorListener.const_get(constant_name, false)
+        end
+
+        def listener_for(state, configuration, profile)
+          listener = state && state[:listener]
+          if listener
+            listener.configuration = configuration
+            listener
+          else
+            MonitorListener.new(configuration, profile: profile)
+          end
+        end
+
+        def subscriptions_for(state)
+          return {} unless state
+
+          state[:subscriptions].is_a?(Hash) ? state[:subscriptions].dup : {}
+        end
+
+        def subscription_state(monitor, profile)
+          subscription_state_store(profile).fetch(monitor)
+        end
+
+        def store_subscription_state(monitor, listener:, subscriptions:, profile:)
+          subscription_state_store(profile).store(
+            monitor,
+            { listener: listener, subscriptions: subscriptions.freeze }.freeze
+          )
+        end
+
+        def install_marker(profile)
+          :"@julewire_karafka_#{profile.component}_state"
+        end
+
+        def subscription_state_store(profile)
+          Core::Integration::IvarState.new(install_marker(profile))
+        end
+
+        def event_names(monitor, configuration, profile)
+          configured = configuration.public_send(profile.config_method)
+          return profile.important_events if configured == :important
+
+          if all_events?(configured)
+            available_events = available_events_for(monitor)
+            return available_events unless available_events.empty?
+
+            return profile.important_events
+          end
+
+          Array(configured)
+        end
+
+        def available_events_for(monitor)
+          available = direct_available_events(monitor)
+          return available unless available.empty?
+
+          available = notification_bus_available_events(monitor)
+          return available unless available.empty?
+
+          listener_event_names(monitor)
+        end
+
+        def direct_available_events(monitor)
+          return [] unless monitor.respond_to?(:available_events)
+
+          Array(monitor.available_events)
+        rescue StandardError
+          []
+        end
+
+        def notification_bus_available_events(monitor)
+          return [] unless monitor.respond_to?(:notifications_bus)
+
+          bus = monitor.notifications_bus
+          return [] unless bus.respond_to?(:available_events)
+
+          Array(bus.available_events)
+        rescue StandardError
+          []
+        end
+
+        def listener_event_names(monitor)
+          return [] unless monitor.respond_to?(:listeners)
+
+          listeners = monitor.listeners
+          listeners.is_a?(Hash) ? listeners.keys : []
+        rescue StandardError
+          []
+        end
+
+        def all_events?(configured)
+          %i[all available].include?(configured)
+        end
+
+        def unsubscribe_removed_events(monitor, subscriptions, desired_events, profile)
+          return unless monitor.respond_to?(:unsubscribe)
+
+          desired = desired_events.to_h { [it, true] }
+          subscriptions.each_key.to_a.each do |event_name|
+            next if desired.key?(event_name)
+
+            callback = subscriptions.delete(event_name)
+            unsubscribe_event(monitor, event_name, callback, profile)
+          end
+        end
+
+        def unsubscribe_event(monitor, event_name, callback, profile)
+          IntegrationHealth.with_failure_health(
+            action: :unsubscribe,
+            component: profile.component,
+            event: event_name
+          ) do
+            monitor.unsubscribe(callback || event_name)
+            true
+          end
+        end
+
+        def subscribe_event(monitor, event_name, profile, &)
+          return false unless monitor.respond_to?(:subscribe)
+
+          IntegrationHealth.with_failure_health(action: :subscribe, component: profile.component, event: event_name) do
+            monitor.subscribe(event_name, &)
+            true
+          end || false
+        end
+      end
+    end
+
+    private_constant :MonitorSubscription
+  end
+end

data/lib/julewire/karafka/payload_reader.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module PayloadReader
+      class << self
+        def consumer_payload(payload)
+          consumer = value(payload, :caller)
+          message_set = value(consumer, :messages)
+          metadata = value(message_set, :metadata)
+          messages = messages_for(consumer)
+          first = messages.first
+          last = messages.last
+
+          {
+            consumer_class: consumer&.class&.name,
+            consumer_id: value(consumer, :id),
+            consumer_group: nested_value(consumer, :topic, :consumer_group, :id),
+            subscription_group: nested_value(consumer, :topic, :subscription_group, :id),
+            topic: consumer_topic(consumer, metadata, first),
+            partition: consumer_partition(consumer, metadata, first),
+            messages_count: count(messages, metadata),
+            first_offset: value(metadata, :first_offset) || value(first, :offset),
+            last_offset: value(metadata, :last_offset) || value(last, :offset),
+            processing_lag_ms: value(metadata, :processing_lag),
+            consumption_lag_ms: value(metadata, :consumption_lag)
+          }.compact
+        end
+
+        def message_payload(message)
+          {
+            topic: value(message, :topic),
+            partition: value(message, :partition),
+            offset: value(message, :offset),
+            key: value(message, :key),
+            headers: headers(message)
+          }.compact
+        end
+
+        def messages_for(consumer)
+          messages = value(consumer, :messages)
+          return Array(messages) if messages
+
+          message = value(consumer, :message)
+          message ? [message] : []
+        end
+
+        def headers(message)
+          value(message, :headers) || {}
+        end
+
+        def count(messages, metadata)
+          value(metadata, :size) || messages.size.nonzero?
+        end
+
+        def consumer_topic(consumer, metadata, first)
+          nested_value(consumer, :topic, :name) || value(metadata, :topic) || value(first, :topic)
+        end
+
+        def consumer_partition(consumer, metadata, first)
+          value(consumer, :partition) || value(metadata, :partition) || value(first, :partition)
+        end
+
+        def nested_value(object, *method_names)
+          Core::Integration::Values::Read.nested_value(object, *method_names)
+        end
+
+        def value(object, method_name)
+          Core::Integration::Values::Read.value(object, method_name)
+        end
+      end
+    end
+  end
+end

data/lib/julewire/karafka/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    VERSION = "1.0.0"
+  end
+end

data/lib/julewire/karafka/waterdrop_installer.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    module WaterdropInstaller
+      MIDDLEWARE_INSTALL = Core::Integration::IvarState.new(:@julewire_karafka_waterdrop_middleware)
+
+      class << self
+        def install!(producer, configuration: Configuration.new)
+          return false unless configuration.enabled?
+
+          install_or_update_middleware(producer, configuration) if middleware_needed?(producer, configuration)
+          install_listener(producer, configuration) if configuration.producer_events?
+          producer
+        end
+
+        private
+
+        def middleware_needed?(producer, configuration)
+          configuration.propagation? || installed_middleware(producer)
+        end
+
+        def install_or_update_middleware(producer, configuration)
+          install_middleware(producer, configuration)
+        end
+
+        def install_middleware(producer, configuration)
+          existing = MIDDLEWARE_INSTALL.fetch(producer)
+          if existing
+            existing.configuration = configuration
+            return existing
+          end
+
+          middleware = producer.middleware if producer.respond_to?(:middleware)
+          return unless middleware.respond_to?(:prepend)
+
+          installed = WaterdropMiddleware.new(configuration: configuration)
+          middleware.prepend(installed)
+          MIDDLEWARE_INSTALL.store(producer, installed)
+          installed
+        rescue StandardError => e
+          IntegrationHealth.record_failure(e, action: :install, component: :waterdrop_installer)
+          nil
+        end
+
+        def install_listener(producer, configuration)
+          monitor = producer.monitor if producer.respond_to?(:monitor)
+          MonitorSubscription.install!(monitor, configuration: configuration, profile: :producer) if monitor
+        rescue StandardError => e
+          IntegrationHealth.record_failure(e, action: :install, component: :waterdrop_installer)
+          nil
+        end
+
+        def installed_middleware(producer)
+          MIDDLEWARE_INSTALL.fetch(producer)
+        end
+      end
+    end
+  end
+end

data/lib/julewire/karafka/waterdrop_middleware.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Julewire
+  module Karafka
+    class WaterdropMiddleware
+      def initialize(configuration: Configuration.new)
+        @configuration = configuration
+      end
+
+      attr_writer :configuration
+
+      def call(message)
+        inject_carrier(message) if @configuration.propagation?
+        message
+      end
+
+      private
+
+      def inject_carrier(message)
+        IntegrationHealth.with_failure_health(action: :carrier_inject, component: :waterdrop_middleware) do
+          headers = headers_for(message)
+          Julewire::Core::Propagation::Carrier.inject(
+            headers,
+            key: @configuration.carrier_key,
+            max_bytes: @configuration.carrier_max_bytes
+          )
+        end
+      end
+
+      def headers_for(message)
+        if message.respond_to?(:headers)
+          message.headers ||= {}
+        elsif message.is_a?(Hash)
+          message[:headers] ||= {}
+        else
+          {}
+        end
+      end
+    end
+  end
+end

data/lib/julewire/karafka.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "julewire/core"
+
+module Julewire
+  module Karafka
+    class Error < Julewire::Error; end
+    IntegrationHealth = Core::Integration::Health.scoped(:karafka)
+
+    extend Core::Integration::Configurable
+
+    configurable_with { Configuration }
+
+    InstallResult = Data.define(:consumer, :producer)
+
+    class << self
+      def install!(app: nil, monitor: nil, producer: nil, consumer: true, configuration: config)
+        return false unless configuration.enabled?
+
+        consumer_result = Installer.install!(app: app, monitor: monitor, configuration: configuration) if consumer
+        producer_result = WaterdropInstaller.install!(producer, configuration: configuration) if producer
+
+        return InstallResult.new(consumer_result, producer_result) if consumer && producer
+
+        producer ? producer_result : consumer_result
+      end
+
+      def inject!(message, configuration: config)
+        WaterdropMiddleware.new(configuration: configuration).call(message)
+      end
+
+      def with_message(message, configuration: config, &)
+        MessageContext.call(message, configuration: configuration, &)
+      end
+
+      def with_message_execution(message, configuration: config, **execution_options, &)
+        MessageExecution.call(message, configuration: configuration, **execution_options, &)
+      end
+    end
+  end
+
+  loader = Zeitwerk::Loader.for_gem_extension(self)
+  loader.setup
+end

data/lib/julewire-karafka.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "julewire/karafka"