phobos_temp_fork 0.0.1

data/lib/phobos/handler.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Handler
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def consume(_payload, _metadata)
+      raise NotImplementedError
+    end
+
+    def around_consume(payload, metadata)
+      yield payload, metadata
+    end
+
+    module ClassMethods
+      def start(kafka_client); end
+
+      def stop; end
+    end
+  end
+end

data/lib/phobos/instrumentation.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'active_support/notifications'
+
+module Phobos
+  module Instrumentation
+    NAMESPACE = 'phobos'
+
+    def self.subscribe(event)
+      ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
+        yield ActiveSupport::Notifications::Event.new(*args) if block_given?
+      end
+    end
+
+    def self.unsubscribe(subscriber)
+      ActiveSupport::Notifications.unsubscribe(subscriber)
+    end
+
+    def instrument(event, extra = {})
+      ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do |args|
+        yield(args) if block_given?
+      end
+    end
+  end
+end

data/lib/phobos/listener.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Phobos
+  # rubocop:disable Metrics/ParameterLists, Metrics/ClassLength
+  class Listener
+    include Phobos::Instrumentation
+    include Phobos::Log
+
+    DEFAULT_MAX_BYTES_PER_PARTITION = 1_048_576 # 1 MB
+    DELIVERY_OPTS = %w[batch message inline_batch].freeze
+
+    attr_reader :group_id, :topic, :id
+    attr_reader :handler_class, :encoding, :consumer
+
+    # rubocop:disable Metrics/MethodLength
+    def initialize(handler:, group_id:, topic:, min_bytes: nil, max_wait_time: nil,
+                   force_encoding: nil, start_from_beginning: true, backoff: nil,
+                   delivery: 'batch', max_bytes_per_partition: DEFAULT_MAX_BYTES_PER_PARTITION,
+                   session_timeout: nil, offset_commit_interval: nil,
+                   heartbeat_interval: nil, offset_commit_threshold: nil,
+                   offset_retention_time: nil)
+      @id = SecureRandom.hex[0...6]
+      @handler_class = handler
+      @group_id = group_id
+      @topic = topic
+      @backoff = backoff
+      @delivery = delivery.to_s
+      @subscribe_opts = {
+        start_from_beginning: start_from_beginning, max_bytes_per_partition: max_bytes_per_partition
+      }
+      @kafka_consumer_opts = compact(
+        session_timeout: session_timeout, offset_retention_time: offset_retention_time,
+        offset_commit_interval: offset_commit_interval, heartbeat_interval: heartbeat_interval,
+        offset_commit_threshold: offset_commit_threshold
+      )
+      @encoding = Encoding.const_get(force_encoding.to_sym) if force_encoding
+      @message_processing_opts = compact(min_bytes: min_bytes, max_wait_time: max_wait_time)
+      @kafka_client = Phobos.create_kafka_client(:consumer)
+      @producer_enabled = @handler_class.ancestors.include?(Phobos::Producer)
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    def start
+      @signal_to_stop = false
+
+      start_listener
+
+      begin
+        start_consumer_loop
+      rescue Kafka::ProcessingError, Phobos::AbortError
+        # Abort is an exception to prevent the consumer from committing the offset.
+        # Since "listener" had a message being retried while "stop" was called
+        # it's wise to not commit the batch offset to avoid data loss. This will
+        # cause some messages to be reprocessed
+        instrument('listener.retry_aborted', listener_metadata) do
+          log_info('Retry loop aborted, listener is shutting down', listener_metadata)
+        end
+      end
+    ensure
+      stop_listener
+    end
+
+    def stop
+      return if should_stop?
+
+      instrument('listener.stopping', listener_metadata) do
+        log_info('Listener stopping', listener_metadata)
+        @consumer&.stop
+        @signal_to_stop = true
+      end
+    end
+
+    def create_exponential_backoff
+      Phobos.create_exponential_backoff(@backoff)
+    end
+
+    def should_stop?
+      @signal_to_stop == true
+    end
+
+    def send_heartbeat_if_necessary
+      raise Phobos::AbortError if should_stop?
+
+      @consumer&.send_heartbeat_if_necessary
+    end
+
+    private
+
+    def listener_metadata
+      { listener_id: id, group_id: group_id, topic: topic, handler: handler_class.to_s }
+    end
+
+    def start_listener
+      instrument('listener.start', listener_metadata) do
+        @consumer = create_kafka_consumer
+        @consumer.subscribe(topic, **@subscribe_opts)
+
+        # This is done here because the producer client is bound to the current thread and
+        # since "start" blocks a thread might be used to call it
+        @handler_class.producer.configure_kafka_client(@kafka_client) if @producer_enabled
+
+        instrument('listener.start_handler', listener_metadata) do
+          @handler_class.start(@kafka_client)
+        end
+        log_info('Listener started', listener_metadata)
+      end
+    end
+
+    def stop_listener
+      instrument('listener.stop', listener_metadata) do
+        instrument('listener.stop_handler', listener_metadata) { @handler_class.stop }
+
+        @consumer&.stop
+
+        if @producer_enabled
+          @handler_class.producer.async_producer_shutdown
+          @handler_class.producer.configure_kafka_client(nil)
+        end
+
+        @kafka_client.close
+        log_info('Listener stopped', listener_metadata) if should_stop?
+      end
+    end
+
+    def start_consumer_loop
+      # validate batch handling
+      case @delivery
+      when 'batch'
+        consume_each_batch
+      when 'inline_batch'
+        consume_each_batch_inline
+      else
+        consume_each_message
+      end
+    end
+
+    def consume_each_batch
+      @consumer.each_batch(**@message_processing_opts) do |batch|
+        batch_processor = Phobos::Actions::ProcessBatch.new(
+          listener: self,
+          batch: batch,
+          listener_metadata: listener_metadata
+        )
+
+        batch_processor.execute
+        log_debug('Committed offset', batch_processor.metadata)
+        return nil if should_stop?
+      end
+    end
+
+    def consume_each_batch_inline
+      @consumer.each_batch(**@message_processing_opts) do |batch|
+        batch_processor = Phobos::Actions::ProcessBatchInline.new(
+          listener: self,
+          batch: batch,
+          metadata: listener_metadata
+        )
+
+        batch_processor.execute
+        log_debug('Committed offset', batch_processor.metadata)
+        return nil if should_stop?
+      end
+    end
+
+    def consume_each_message
+      @consumer.each_message(**@message_processing_opts) do |message|
+        message_processor = Phobos::Actions::ProcessMessage.new(
+          listener: self,
+          message: message,
+          listener_metadata: listener_metadata
+        )
+
+        message_processor.execute
+        log_debug('Committed offset', message_processor.metadata)
+        return nil if should_stop?
+      end
+    end
+
+    def create_kafka_consumer
+      configs = Phobos.config.consumer_hash.select do |k|
+        Constants::KAFKA_CONSUMER_OPTS.include?(k)
+      end
+      configs.merge!(@kafka_consumer_opts)
+      @kafka_client.consumer(**{ group_id: group_id }.merge(configs))
+    end
+
+    def compact(hash)
+      hash.delete_if { |_, v| v.nil? }
+    end
+  end
+  # rubocop:enable Metrics/ParameterLists, Metrics/ClassLength
+end

data/lib/phobos/log.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Log
+    def log_info(msg, metadata = {})
+      LoggerHelper.log(:info, msg, metadata)
+    end
+
+    def log_debug(msg, metadata = {})
+      LoggerHelper.log(:debug, msg, metadata)
+    end
+
+    def log_error(msg, metadata)
+      LoggerHelper.log(:error, msg, metadata)
+    end
+  end
+
+  module LoggerHelper
+    def self.log(method, msg, metadata)
+      Phobos.logger.send(method, Hash(message: msg).merge(metadata))
+    end
+  end
+end

data/lib/phobos/processor.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'active_support/concern'
+
+module Phobos
+  module Processor
+    extend ActiveSupport::Concern
+    include Phobos::Instrumentation
+
+    MAX_SLEEP_INTERVAL = 3
+
+    def snooze(interval)
+      remaining_interval = interval
+
+      @listener.send_heartbeat_if_necessary
+
+      while remaining_interval.positive?
+        sleep [remaining_interval, MAX_SLEEP_INTERVAL].min
+        remaining_interval -= MAX_SLEEP_INTERVAL
+        @listener.send_heartbeat_if_necessary
+      end
+    end
+
+    private
+
+    def force_encoding(value)
+      @listener.encoding ? value&.force_encoding(@listener.encoding) : value
+    end
+
+    def handle_error(error, instrumentation_key, error_message)
+      error_hash = {
+        waiting_time: backoff_interval,
+        exception_class: error.class.name,
+        exception_message: error.message,
+        backtrace: error.backtrace
+      }
+
+      instrument(instrumentation_key, error_hash.merge(@metadata)) do
+        Phobos.logger.error do
+          { message: error_message }
+            .merge(error_hash)
+            .merge(@metadata)
+        end
+
+        snooze(backoff_interval)
+      end
+
+      increment_retry_count
+    end
+
+    def retry_count
+      @metadata[:retry_count]
+    end
+
+    def increment_retry_count
+      @metadata[:retry_count] = retry_count + 1
+    end
+
+    def backoff
+      @backoff ||= @listener.create_exponential_backoff
+    end
+
+    def backoff_interval
+      backoff.interval_at(retry_count).round(2)
+    end
+  end
+end

data/lib/phobos/producer.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Producer
+    def self.included(base)
+      base.extend(Phobos::Producer::ClassMethods)
+    end
+
+    def producer
+      Phobos::Producer::PublicAPI.new(self)
+    end
+
+    class PublicAPI
+      def initialize(host_obj)
+        @host_obj = host_obj
+      end
+
+      def publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+        class_producer.publish(topic: topic,
+                               payload: payload,
+                               key: key,
+                               partition_key: partition_key,
+                               headers: headers)
+      end
+
+      def async_publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+        class_producer.async_publish(topic: topic,
+                                     payload: payload,
+                                     key: key,
+                                     partition_key: partition_key,
+                                     headers: headers)
+      end
+
+      # @param messages [Array(Hash(:topic, :payload, :key, :headers))]
+      #        e.g.: [
+      #          { topic: 'A', payload: 'message-1', key: '1', headers: { foo: 'bar' } },
+      #          { topic: 'B', payload: 'message-2', key: '2', headers: { foo: 'bar' } }
+      #        ]
+      #
+      def publish_list(messages)
+        class_producer.publish_list(messages)
+      end
+
+      def async_publish_list(messages)
+        class_producer.async_publish_list(messages)
+      end
+
+      private
+
+      def class_producer
+        @host_obj.class.producer
+      end
+    end
+
+    module ClassMethods
+      def producer
+        Phobos::Producer::ClassMethods::PublicAPI.new
+      end
+
+      class PublicAPI
+        NAMESPACE = :phobos_producer_store
+        ASYNC_PRODUCER_PARAMS = [:max_queue_size, :delivery_threshold, :delivery_interval].freeze
+        INTERNAL_PRODUCER_PARAMS = [:persistent_connections].freeze
+
+        # This method configures the kafka client used with publish operations
+        # performed by the host class
+        #
+        # @param kafka_client [Kafka::Client]
+        #
+        def configure_kafka_client(kafka_client)
+          async_producer_shutdown
+          producer_store[:kafka_client] = kafka_client
+        end
+
+        def kafka_client
+          producer_store[:kafka_client]
+        end
+
+        def create_sync_producer
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          sync_producer = client.producer(**regular_configs)
+          if Phobos.config.producer_hash[:persistent_connections]
+            producer_store[:sync_producer] = sync_producer
+          end
+          sync_producer
+        end
+
+        def sync_producer
+          producer_store[:sync_producer]
+        end
+
+        def sync_producer_shutdown
+          sync_producer&.shutdown
+          producer_store[:sync_producer] = nil
+        end
+
+        def publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+          publish_list([{ topic: topic, payload: payload, key: key,
+                          partition_key: partition_key, headers: headers }])
+        end
+
+        def publish_list(messages)
+          producer = sync_producer || create_sync_producer
+          produce_messages(producer, messages)
+          producer.deliver_messages
+        ensure
+          producer&.shutdown unless Phobos.config.producer_hash[:persistent_connections]
+        end
+
+        def create_async_producer
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client(:producer))
+          async_producer = client.async_producer(**async_configs)
+          producer_store[:async_producer] = async_producer
+        end
+
+        def async_producer
+          producer_store[:async_producer]
+        end
+
+        def async_publish(topic:, payload:, key: nil, partition_key: nil, headers: nil)
+          async_publish_list([{ topic: topic, payload: payload, key: key,
+                                partition_key: partition_key, headers: headers }])
+        end
+
+        def async_publish_list(messages)
+          producer = async_producer || create_async_producer
+          produce_messages(producer, messages)
+          producer.deliver_messages unless async_automatic_delivery?
+        end
+
+        def async_producer_shutdown
+          async_producer&.deliver_messages
+          async_producer&.shutdown
+          producer_store[:async_producer] = nil
+        end
+
+        def regular_configs
+          Phobos.config.producer_hash
+                .reject { |k, _| ASYNC_PRODUCER_PARAMS.include?(k) }
+                .reject { |k, _| INTERNAL_PRODUCER_PARAMS.include?(k) }
+        end
+
+        def async_configs
+          Phobos.config.producer_hash
+                .reject { |k, _| INTERNAL_PRODUCER_PARAMS.include?(k) }
+        end
+
+        private
+
+        def produce_messages(producer, messages)
+          messages.each do |message|
+            partition_key = message[:partition_key] || message[:key]
+            producer.produce(message[:payload], topic: message[:topic],
+                                                key: message[:key],
+                                                headers: message[:headers],
+                                                partition_key: partition_key)
+          end
+        end
+
+        def async_automatic_delivery?
+          async_configs.fetch(:delivery_threshold, 0).positive? ||
+            async_configs.fetch(:delivery_interval, 0).positive?
+        end
+
+        def producer_store
+          Thread.current[NAMESPACE] ||= {}
+        end
+      end
+    end
+  end
+end