eh_messaging 1.0.0

data/lib/eh_messaging/subscriber.rb

@@ -0,0 +1,300 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  class Subscriber
+    attr_reader :subscriptions
+
+    def initialize(configuration, consumer_factory: nil, producer: nil, logger: nil)
+      @configuration, @consumer_factory, @producer, @logger = configuration, consumer_factory, producer, logger
+      @logger ||= AuditLogger.new(configuration)
+      @dead_letter_router = DeadLetterRouter.new(configuration, producer: producer, logger: @logger)
+      @subscriptions, @threads, @consumers, @mutex = [], [], [], Mutex.new
+      @stopped = ConditionVariable.new
+      @stopping = false
+      @booted = false
+      @closed = false
+    end
+
+    def subscribe(handler)
+      raise Error, "subscriber is shut down" if @closed
+      raise Error, "cannot subscribe after boot" if @booted
+      raise ArgumentError, "handler must define topic and group" unless handler.respond_to?(:topic) && handler.respond_to?(:group)
+      @subscriptions << handler
+      self
+    end
+
+    def boot
+      raise Error, "subscriber is shut down" if @closed
+      return self if @booted
+      @stopping = false
+      @subscriptions.each do |handler|
+        raise ArgumentError, "Topic #{handler.topic} is unavailable" unless topic_available?(handler, handler.topic, mode: :normal)
+        start_loop(handler, :normal, handler.topic, handler.group)
+        recovery_topic = Topics.recovery(handler.topic, handler.group)
+        start_loop(handler, :recovery, recovery_topic, Topics.recovery_group(handler.group)) if topic_available?(handler, recovery_topic, mode: :recovery)
+      end
+      @booted = true
+      self
+    rescue
+      shutdown_started_threads
+      raise
+    end
+
+    def reconsume(handler, until_time: nil, partitions: nil, group: nil)
+      raise Error, "subscriber is shut down" if @closed
+      raise ArgumentError, "exactly one of until_time or partitions is required" if until_time.nil? == partitions.nil?
+      group ||= Topics.reconsume_group(handler.group)
+      consumer = build_consumer(handler, :reconsume, handler.topic, group)
+      @mutex.synchronize { @consumers << consumer }
+      consumer.subscribe(handler.topic) if consumer.respond_to?(:subscribe)
+      deadline = until_time&.utc
+      ends = partitions&.to_h
+      completed = {}
+      if consumer.respond_to?(:assignment)
+        assigned = assigned_partitions(consumer, handler.topic)
+        if ends && !assigned.empty?
+          ends.each_key { |partition| completed[partition] = true unless assigned.include?(partition) }
+        elsif deadline
+          assigned.each { |partition| completed[partition] = false }
+        end
+      end
+      until_complete = false
+      until (ends && completed.size == ends.size) || until_complete || @stopping
+        begin
+          message = poll_message(consumer)
+        rescue Rdkafka::RdkafkaError => error
+          log_poll_error(error)
+          break if @stopping || unknown_topic_error?(error) || closed_consumer_error?(error)
+          next
+        rescue => error
+          log_error(error)
+          break if @stopping || closed_consumer_error?(error)
+          next
+        end
+        unless message
+          if ends
+            assigned = assigned_partitions(consumer, handler.topic)
+            ends.each_key { |partition| completed[partition] = true unless assigned.include?(partition) } unless assigned.empty?
+          end
+          # EOF does not prove that a timestamp boundary has been crossed;
+          # a future UNTIL remains active until a later record or shutdown.
+          next
+        end
+        partition = message.partition
+        offset = message.offset
+        if ends
+          next unless ends.key?(partition)
+          if offset > ends.fetch(partition)
+            completed[partition] = true
+            next
+          end
+          completed[partition] = true if offset == ends.fetch(partition)
+        elsif deadline && broker_time(message) && broker_time(message) > deadline
+          completed[partition] = true
+          until_complete = completed.values.all? { |value| value }
+          @logger.info("Reconsume reached #{deadline.iso8601} on partition #{partition}")
+          break if until_complete
+          next
+        end
+        process_message(handler, :reconsume, group, message)
+      end
+    ensure
+      @mutex&.synchronize { @consumers.delete(consumer) if consumer }
+      consumer.close if consumer&.respond_to?(:close)
+    end
+
+    def run
+      @mutex.synchronize { @stopped.wait(@mutex) until @stopping }
+    end
+
+    def shutdown
+      @stopping = true
+      @mutex.synchronize { @stopped.broadcast }
+      close_registered_consumers
+      join_threads
+      clear_runtime
+      @booted = false
+      @closed = true
+      self
+    end
+
+    private
+
+    def start_loop(handler, mode, topic, group)
+      thread = Thread.new do
+        consumer = nil
+        consumer = build_consumer(handler, mode, topic, group)
+        registered = @mutex.synchronize do
+          if @stopping
+            consumer.close if consumer.respond_to?(:close)
+            false
+          else
+            @consumers << consumer
+            true
+          end
+        end
+        next unless registered
+        consumer.subscribe(topic) if consumer.respond_to?(:subscribe)
+        unless mode == :normal || topic_available?(handler, topic, consumer: consumer)
+          @logger.warn("EhMessaging: topic unavailable: #{topic}")
+          next
+        end
+        run_polling_loop(handler, mode, group, consumer)
+      rescue => error
+        log_error(error)
+      ensure
+        close_consumer(consumer) if consumer
+      end
+      join_thread = @mutex.synchronize do
+        if @stopping
+          true
+        else
+          @threads << thread
+          false
+        end
+      end
+      thread.join if join_thread
+    end
+
+    def build_consumer(handler, mode, topic, group)
+      broker_name = if mode == :recovery && handler.respond_to?(:recovery_broker)
+        handler.recovery_broker
+      else
+        handler.respond_to?(:broker) ? handler.broker : :default
+      end
+      connection = @configuration.broker_config(broker_name)
+      if @consumer_factory
+        @consumer_factory.call(handler, mode, topic, group, connection)
+      else
+        Rdkafka::Config.new(@configuration.kafka_config(connection, kind: :consumer, group_id: group)).consumer
+      end
+    end
+
+    def process_message(handler, mode, group, message)
+      headers = (message.headers || {}).transform_keys(&:to_s)
+      begin
+        return if handler.respond_to?(:accept?) && !handler.accept?(headers)
+      rescue => error
+        log_error(error)
+        return
+      end
+      begin
+        envelope = Envelope.from_message(message, mode: mode, consumer_group: group)
+        Telemetry.in_span("eh_messaging.consume", attributes: {
+          "messaging.destination.name" => message.topic,
+          "messaging.kafka.consumer.group" => group,
+          "messaging.kafka.partition" => message.partition,
+          "messaging.kafka.offset" => message.offset
+        }, parent_headers: envelope.headers) do |span|
+          handler.new.process(envelope)
+          span.set_status(:ok) if span.respond_to?(:set_status)
+        end
+        @logger.message("consume", envelope: envelope)
+      rescue JSON::ParserError => error
+        log_error(error)
+        envelope = invalid_envelope(message, mode, group)
+        @logger.message("consume", envelope: envelope, error: error)
+        publish_dlt(handler, envelope, error) if mode == :normal
+      rescue RecoverableError => error
+        log_error(error)
+        @logger.message("consume", envelope: envelope, error: error) if envelope
+        span&.record_exception(error) if defined?(span) && span.respond_to?(:record_exception)
+      rescue => error
+        log_error(error)
+        @logger.message("consume", envelope: envelope, error: error) if envelope
+        span&.record_exception(error) if defined?(span) && span.respond_to?(:record_exception)
+        publish_dlt(handler, envelope, error) if mode == :normal && envelope
+      end
+    end
+
+    def invalid_envelope(message, mode, group)
+      headers = Envelope.headers_from_message(message)
+      timestamp = message.timestamp if message.respond_to?(:timestamp)
+      timestamp = Time.at(timestamp / 1000.0).utc if timestamp.is_a?(Numeric)
+      Envelope.new(payload: message.payload, partition_key: message.key, headers: headers,
+        mode: mode, consumer_group: group, topic: message.topic, partition: message.partition,
+        offset: message.offset, enqueued_at: timestamp)
+    end
+
+    def topic_available?(handler, topic, mode: :recovery, consumer: nil)
+      temporary_consumer = consumer.nil?
+      if @consumer_factory
+        return true if mode == :normal && consumer.nil?
+
+        group = (mode == :normal) ? handler.group : Topics.recovery_group(handler.group)
+        consumer ||= build_consumer(handler, mode, topic, group)
+        return true unless consumer.respond_to?(:topic_exists?)
+        return consumer.topic_exists?(topic)
+      end
+
+      broker_method = (mode == :dlt) ? :dead_letter_broker : :recovery_broker
+      broker = handler.respond_to?(broker_method) ? handler.public_send(broker_method) : :default
+      connection = @configuration.broker_config(broker)
+      consumer ||= build_consumer(handler, mode, topic, (mode == :normal) ? handler.group : Topics.recovery_group(handler.group))
+      metadata = if consumer.respond_to?(:metadata)
+        consumer.metadata(topic)
+      else
+        admin = Rdkafka::Config.new(@configuration.kafka_config(connection)).admin
+        admin.metadata(topic)
+      end
+      metadata_topic_names(metadata).include?(topic)
+    rescue => error
+      @logger.warn("EhMessaging: topic unavailable: #{topic}: #{error.message}")
+      false
+    ensure
+      admin&.close
+      begin
+        consumer&.close if temporary_consumer
+      rescue
+        nil
+      end
+    end
+
+    def close_consumer(consumer)
+      registered = @mutex.synchronize { @consumers.delete(consumer) }
+      return unless registered
+
+      consumer.close if consumer.respond_to?(:close)
+    rescue => error
+      log_error(error)
+    end
+
+    def shutdown_started_threads
+      @stopping = true
+      close_registered_consumers
+      join_threads
+      clear_runtime
+      @booted = false
+    end
+
+    def close_registered_consumers
+      @mutex.synchronize { @consumers.dup }.each { |consumer| close_consumer(consumer) }
+    end
+
+    def join_threads
+      @mutex.synchronize { @threads.dup }.each(&:join)
+    end
+
+    def clear_runtime
+      @mutex.synchronize do
+        @threads.clear
+        @consumers.clear
+      end
+    end
+
+    def publish_dlt(handler, envelope, error)
+      @dead_letter_router.route(handler, envelope, error)
+    end
+
+    def broker_time(message)
+      value = message.timestamp if message.respond_to?(:timestamp)
+      return nil if value.nil?
+      value.is_a?(Numeric) ? Time.at(value / 1000.0).utc : value.utc
+    end
+
+    def log_poll_error(error) = @logger.error(error.message)
+    def log_error(error) = @logger.error("EhMessaging: #{error.class}: #{error.message}")
+    def unknown_topic_error?(error) = error.message.include?("UNKNOWN_TOPIC_OR_PART") || error.message.include?("Unknown topic")
+    def closed_consumer_error?(error) = error.class.name.to_s.include?("ClosedConsumer")
+  end
+end

data/lib/eh_messaging/subscriber_metadata.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  class Subscriber
+    # Kafka metadata shapes vary slightly between rdkafka versions and test doubles.
+    module Metadata
+      private
+
+      def assigned_partitions(consumer, topic)
+        assignment = consumer.assignment
+        partitions = if assignment.is_a?(Array)
+          assignment
+        elsif assignment.respond_to?(:to_h)
+          assignment.to_h.values.flatten
+        elsif assignment.respond_to?(:each)
+          assignment.each.to_a
+        else
+          Array(assignment)
+        end
+        assigned = partitions.filter_map do |partition|
+          next partition if partition.is_a?(Integer)
+          if partition.is_a?(Array)
+            partition.last
+          elsif partition.is_a?(Hash)
+            partition[:id] || partition["id"]
+          elsif partition.respond_to?(:partition)
+            partition.partition
+          elsif partition.respond_to?(:id)
+            partition.id
+          end
+        end
+        return assigned unless assigned.empty? && consumer.respond_to?(:metadata)
+
+        metadata_partitions(consumer.metadata(topic), topic) || assigned
+      rescue => error
+        log_error(error)
+        assigned
+      end
+
+      def metadata_partitions(metadata, requested_topic = nil)
+        topics = metadata.respond_to?(:topics) ? metadata.topics : metadata
+        topic = if topics.is_a?(Hash)
+          topics[requested_topic] || topics[requested_topic.to_s] || topics.values.first
+        else
+          Array(topics).find { |candidate| candidate.respond_to?(:name) && candidate.name == requested_topic } ||
+            Array(topics).find { |candidate| candidate.respond_to?(:topic_name) && candidate.topic_name == requested_topic } ||
+            Array(topics).first
+        end
+        partitions = if topic.respond_to?(:partitions)
+          topic.partitions
+        elsif topic.is_a?(Hash)
+          topic[:partitions] || topic["partitions"]
+        end
+        return unless partitions
+
+        return partitions.keys.filter_map { |partition| Integer(partition, exception: false) } if partitions.is_a?(Hash)
+
+        Array(partitions).filter_map do |partition|
+          next partition if partition.is_a?(Integer)
+          if partition.respond_to?(:id)
+            partition.id
+          elsif partition.is_a?(Hash)
+            partition[:id] || partition["id"]
+          end
+        end
+      end
+
+      def metadata_topic_names(metadata)
+        topics = metadata.topics
+        topics = topics.keys if topics.is_a?(Hash)
+        Array(topics).filter_map do |item|
+          if item.is_a?(String)
+            item
+          elsif item.respond_to?(:name)
+            item.name
+          elsif item.respond_to?(:topic_name)
+            item.topic_name
+          elsif item.is_a?(Hash)
+            item[:topic_name] || item["topic_name"] || item[:name] || item["name"]
+          end
+        end
+      end
+    end
+
+    include Metadata
+  end
+end

data/lib/eh_messaging/subscriber_polling.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  class Subscriber
+    # Kafka polling is isolated from lifecycle and message-routing concerns.
+    module Polling
+      private
+
+      def run_polling_loop(handler, mode, group, consumer)
+        until @stopping
+          begin
+            stop_polling = poll_and_process(handler, mode, group, consumer)
+            break if stop_polling
+          rescue Rdkafka::RdkafkaError => error
+            log_poll_error(error)
+            break if @stopping || unknown_topic_error?(error) || closed_consumer_error?(error)
+          rescue => error
+            log_error(error)
+            break if @stopping || closed_consumer_error?(error)
+          end
+        end
+      end
+
+      def poll_and_process(handler, mode, group, consumer)
+        poll_messages(consumer).each do |message|
+          if message.is_a?(Rdkafka::RdkafkaError)
+            log_poll_error(message)
+            return true if @stopping || unknown_topic_error?(message) || closed_consumer_error?(message)
+
+            next
+          end
+          process_message(handler, mode, group, message)
+        end
+        false
+      end
+
+      def poll_message(consumer)
+        timeout_ms = (@configuration.poll_timeout * 1000).to_i
+        return consumer.poll(timeout_ms) if consumer.respond_to?(:poll)
+
+        consumer.each.first
+      end
+
+      def poll_messages(consumer)
+        timeout_ms = (@configuration.poll_timeout * 1000).to_i
+        return consumer.poll_batch(timeout_ms, max_items: 100) if consumer.respond_to?(:poll_batch)
+
+        message = poll_message(consumer)
+        message ? [message] : []
+      end
+    end
+
+    include Polling
+  end
+end

data/lib/eh_messaging/telemetry.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  module Telemetry
+    def self.tracer
+      OpenTelemetry.tracer_provider.tracer("eh_messaging", EhMessaging::VERSION)
+    end
+
+    def self.in_span(name, attributes: {}, parent_headers: nil)
+      parent_context = context_from_headers(parent_headers)
+      if parent_context
+        OpenTelemetry::Context.with_current(parent_context) do
+          tracer.in_span(name, attributes: attributes) { |span| yield span }
+        end
+      else
+        tracer.in_span(name, attributes: attributes) { |span| yield span }
+      end
+    end
+
+    def self.headers(headers, current_span: nil)
+      return headers if headers.key?("operation-id") && headers.key?("parent-id")
+
+      context = current_span&.context || OpenTelemetry::Trace.current_span.context
+      return fallback_headers(headers) unless context.respond_to?(:valid?) && context.valid?
+
+      headers.merge(
+        "operation-id" => context.hex_trace_id,
+        "parent-id" => context.hex_span_id
+      )
+    rescue
+      fallback_headers(headers)
+    end
+
+    def self.context_from_headers(headers)
+      return unless headers
+      operation_id = headers["operation-id"] || headers[:"operation-id"]
+      parent_id = headers["parent-id"] || headers[:"parent-id"]
+      return if operation_id.to_s.empty? || parent_id.to_s.empty?
+
+      span_context = OpenTelemetry::Trace::SpanContext.new(
+        trace_id: operation_id.to_s.delete("-").rjust(32, "0"),
+        span_id: parent_id.to_s.delete("-").rjust(16, "0"),
+        trace_flags: OpenTelemetry::Trace::TraceFlags::SAMPLED,
+        remote: true
+      )
+      OpenTelemetry::Trace.context_with_span(OpenTelemetry::Trace.non_recording_span(span_context))
+    rescue
+      nil
+    end
+
+    def self.fallback_headers(headers)
+      headers.merge("operation-id" => SecureRandom.hex(16), "parent-id" => SecureRandom.hex(8))
+    end
+  end
+end

data/lib/eh_messaging/topics.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  module Topics
+    def self.dead_letter(topic, group) = "#{topic}.#{group}.dlt"
+    def self.recovery(topic, group) = "#{topic}.#{group}.rec"
+    def self.recovery_group(group) = "#{group}_rec"
+    def self.reconsume_group(group) = "#{group}_reconsume"
+  end
+end

data/lib/eh_messaging/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  VERSION = "1.0.0"
+end

data/lib/eh_messaging.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module EhMessaging
+  require "json"
+  require "logger"
+  require "opentelemetry-api"
+  require "rdkafka"
+  require "securerandom"
+  require "time"
+
+  class Error < StandardError; end
+
+  require_relative "eh_messaging/version"
+  require_relative "eh_messaging/recoverable_error"
+  require_relative "eh_messaging/telemetry"
+  require_relative "eh_messaging/audit_logger"
+  require_relative "eh_messaging/topics"
+  require_relative "eh_messaging/envelope"
+  require_relative "eh_messaging/configuration"
+  require_relative "eh_messaging/publisher"
+  require_relative "eh_messaging/dead_letter_router"
+  require_relative "eh_messaging/subscriber"
+  require_relative "eh_messaging/subscriber_polling"
+  require_relative "eh_messaging/subscriber_metadata"
+  require_relative "eh_messaging/client"
+
+  class << self
+    def configure(**options, &block) = client.configure(**options, &block) && self
+    def configuration = client.configuration
+    def publish(topic, **options) = client.publish(topic, **options)
+    def subscribe(handler) = client.subscribe(handler)
+    def reconsume(handler, **options) = client.reconsume(handler, **options)
+    def boot = client.boot && self
+    def run = client.run
+    def shutdown = client.shutdown
+
+    def reset!
+      client.reset!
+      @client = nil
+      self
+    end
+
+    private
+
+    def client = @client ||= Client.new
+  end
+end