phobos 1.0.0

data/lib/phobos/cli/start.rb

@@ -0,0 +1,47 @@
+require 'phobos/cli/runner'
+
+module Phobos
+  module CLI
+    class Start
+      def initialize(options)
+        @config_file = File.expand_path(options[:config])
+        @boot_file = File.expand_path(options[:boot])
+      end
+
+      def execute
+        validate_config_file!
+        Phobos.configure(config_file)
+        load_boot_file
+        validate_listeners!
+
+        Phobos::CLI::Runner.new.run!
+      end
+
+      private
+
+      attr_reader :config_file, :boot_file
+
+      def validate_config_file!
+        unless File.exist?(config_file)
+          Phobos::CLI.logger.error { Hash(message: "Config file not found (#{config_file})") }
+          exit(1)
+        end
+      end
+
+      def validate_listeners!
+        Phobos.config.listeners.collect(&:handler).each do |handler_class|
+          begin
+            handler_class.constantize
+          rescue NameError
+            Phobos::CLI.logger.error { Hash(message: "Handler '#{handler_class}' not defined") }
+            exit(1)
+          end
+        end
+      end
+
+      def load_boot_file
+        load(boot_file) if File.exist?(boot_file)
+      end
+    end
+  end
+end

data/lib/phobos/echo_handler.rb

@@ -0,0 +1,9 @@
+module Phobos
+  class EchoHandler
+    include Phobos::Handler
+
+    def consume(message, metadata)
+      Phobos.logger.info { Hash(message: message).merge(metadata) }
+    end
+  end
+end

data/lib/phobos/errors.rb

@@ -0,0 +1,4 @@
+module Phobos
+  class Error < StandardError; end
+  class AbortError < Error; end
+end

data/lib/phobos/executor.rb

@@ -0,0 +1,83 @@
+module Phobos
+  class Executor
+    include Phobos::Instrumentation
+    LISTENER_OPTS = %i(handler group_id topic start_from_beginning max_bytes_per_partition).freeze
+
+    def initialize
+      @threads = Concurrent::Array.new
+      @listeners = Phobos.config.listeners.flat_map do |config|
+        handler_class = config.handler.constantize
+        listener_configs = config.to_hash.symbolize_keys
+        max_concurrency = listener_configs[:max_concurrency] || 1
+        max_concurrency.times.map do
+          configs = listener_configs.select {|k| LISTENER_OPTS.include?(k)}
+          Phobos::Listener.new(configs.merge(handler: handler_class))
+        end
+      end
+    end
+
+    def start
+      @signal_to_stop = false
+      @threads.clear
+      @thread_pool = Concurrent::FixedThreadPool.new(@listeners.size)
+
+      @listeners.each do |listener|
+        @thread_pool.post do
+          thread = Thread.current
+          thread.abort_on_exception = true
+          @threads << thread
+          run_listener(listener)
+        end
+      end
+
+      true
+    end
+
+    def stop
+      return if @signal_to_stop
+      instrument('executor.stop') do
+        @signal_to_stop = true
+        @listeners.map(&:stop)
+        @threads.select(&:alive?).each { |thread| thread.wakeup rescue nil }
+        @thread_pool&.shutdown
+        @thread_pool&.wait_for_termination
+        Phobos.logger.info { Hash(message: 'Executor stopped') }
+      end
+    end
+
+    private
+
+    def run_listener(listener)
+      retry_count = 0
+      backoff = Phobos.create_exponential_backoff
+
+      begin
+        listener.start
+      rescue Exception => e
+        #
+        # When "listener#start" is interrupted it's safe to assume that the consumer
+        # and the kafka client were properly stopped, it's safe to call start
+        # again
+        #
+        interval = backoff.interval_at(retry_count).round(2)
+        metadata = {
+          listener_id: listener.id,
+          retry_count: retry_count,
+          waiting_time: interval,
+          exception_class: e.class.name,
+          exception_message: e.message,
+          backtrace: e.backtrace
+        }
+
+        instrument('executor.retry_listener_error', metadata) do
+          Phobos.logger.error { Hash(message: "Listener crashed, waiting #{interval}s (#{e.message})").merge(metadata)}
+          sleep interval
+        end
+
+        retry_count += 1
+        retry unless @signal_to_stop
+      end
+    end
+
+  end
+end

data/lib/phobos/handler.rb

@@ -0,0 +1,23 @@
+module Phobos
+  module Handler
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def consume(payload, metadata)
+      raise NotImplementedError
+    end
+
+    module ClassMethods
+      def start(kafka_client)
+      end
+
+      def stop
+      end
+
+      def around_consume(payload, metadata)
+        yield
+      end
+    end
+  end
+end

data/lib/phobos/instrumentation.rb

@@ -0,0 +1,21 @@
+module Phobos
+  module Instrumentation
+    NAMESPACE = 'phobos'
+
+    def instrument(event, extra = {})
+      ActiveSupport::Notifications.instrument("#{NAMESPACE}.#{event}", extra) do
+        yield if block_given?
+      end
+    end
+
+    def self.subscribe(event)
+      ActiveSupport::Notifications.subscribe("#{NAMESPACE}.#{event}") do |*args|
+        yield ActiveSupport::Notifications::Event.new(*args) if block_given?
+      end
+    end
+
+    def self.unsubscribe(subscriber)
+      ActiveSupport::Notifications.unsubscribe(subscriber)
+    end
+  end
+end

data/lib/phobos/listener.rb

@@ -0,0 +1,153 @@
+module Phobos
+  class Listener
+    include Phobos::Instrumentation
+
+    KAFKA_CONSUMER_OPTS = %i(session_timeout offset_commit_interval offset_commit_threshold heartbeat_interval).freeze
+    DEFAULT_MAX_BYTES_PER_PARTITION = 524288 # 512 KB
+
+    attr_reader :group_id, :topic, :id
+
+    def initialize(handler:, group_id:, topic:, start_from_beginning: true, max_bytes_per_partition: DEFAULT_MAX_BYTES_PER_PARTITION)
+      @id = SecureRandom.hex[0...6]
+      @handler_class = handler
+      @group_id = group_id
+      @topic = topic
+      @subscribe_opts = {
+        start_from_beginning: start_from_beginning,
+        max_bytes_per_partition: max_bytes_per_partition
+      }
+      @kafka_client = Phobos.create_kafka_client
+      @producer_enabled = @handler_class.ancestors.include?(Phobos::Producer)
+    end
+
+    def start
+      @signal_to_stop = false
+      instrument('listener.start', listener_metadata) do
+        @consumer = create_kafka_consumer
+        @consumer.subscribe(topic, @subscribe_opts)
+
+        # This is done here because the producer client is bound to the current thread and
+        # since "start" blocks a thread might be used to call it
+        @handler_class.producer.configure_kafka_client(@kafka_client) if @producer_enabled
+
+        instrument('listener.start_handler', listener_metadata) { @handler_class.start(@kafka_client) }
+        Phobos.logger.info { Hash(message: 'Listener started').merge(listener_metadata) }
+      end
+
+      begin
+        @consumer.each_batch do |batch|
+          batch_metadata = {
+            batch_size: batch.messages.count,
+            partition: batch.partition,
+            offset_lag: batch.offset_lag,
+            # the offset of the most recent message in the partition
+            highwater_mark_offset: batch.highwater_mark_offset
+          }.merge(listener_metadata)
+
+          instrument('listener.process_batch', batch_metadata) do
+            process_batch(batch)
+            Phobos.logger.info { Hash(message: 'Committed offset').merge(batch_metadata) }
+          end
+
+          return if @signal_to_stop
+        end
+
+      # Abort is an exception to prevent the consumer from committing the offset.
+      # Since "listener" had a message being retried while "stop" was called
+      # it's wise to not commit the batch offset to avoid data loss. This will
+      # cause some messages to be reprocessed
+      #
+      rescue Phobos::AbortError
+        instrument('listener.retry_aborted', listener_metadata) do
+          Phobos.logger.info do
+            {message: 'Retry loop aborted, listener is shutting down'}.merge(listener_metadata)
+          end
+        end
+      end
+
+    ensure
+      instrument('listener.stop', listener_metadata) do
+        instrument('listener.stop_handler', listener_metadata) { @handler_class.stop }
+
+        @consumer&.stop
+
+        if @producer_enabled
+          @handler_class.producer.async_producer_shutdown
+          @handler_class.producer.configure_kafka_client(nil)
+        end
+
+        @kafka_client.close
+        if @signal_to_stop
+          Phobos.logger.info { Hash(message: 'Listener stopped').merge(listener_metadata) }
+        end
+      end
+    end
+
+    def stop
+      return if @signal_to_stop
+      instrument('listener.stopping', listener_metadata) do
+        Phobos.logger.info { Hash(message: 'Listener stopping').merge(listener_metadata) }
+        @consumer&.stop
+        @signal_to_stop = true
+      end
+    end
+
+    private
+
+    def listener_metadata
+      { listener_id: id, group_id: group_id, topic: topic }
+    end
+
+    def process_batch(batch)
+      batch.messages.each do |message|
+        backoff = Phobos.create_exponential_backoff
+        metadata = listener_metadata.merge(
+          key: message.key,
+          partition: message.partition,
+          offset: message.offset,
+          retry_count: 0
+        )
+
+        begin
+          instrument('listener.process_message', metadata) do
+            process_message(message, metadata)
+          end
+        rescue => e
+          retry_count = metadata[:retry_count]
+          interval = backoff.interval_at(retry_count).round(2)
+
+          error = {
+            waiting_time: interval,
+            exception_class: e.class.name,
+            exception_message: e.message,
+            backtrace: e.backtrace
+          }
+
+          instrument('listener.retry_handler_error', error.merge(metadata)) do
+            Phobos.logger.error do
+              {message: "error processing message, waiting #{interval}s"}.merge(error).merge(metadata)
+            end
+
+            sleep interval
+            metadata.merge!(retry_count: retry_count + 1)
+          end
+
+          raise Phobos::AbortError if @signal_to_stop
+          retry
+        end
+      end
+    end
+
+    def process_message(message, metadata)
+      payload = message.value
+      @handler_class.around_consume(payload, metadata) do
+        @handler_class.new.consume(payload, metadata)
+      end
+    end
+
+    def create_kafka_consumer
+      configs = Phobos.config.consumer_hash.select { |k| KAFKA_CONSUMER_OPTS.include?(k) }
+      @kafka_client.consumer({group_id: group_id}.merge(configs))
+    end
+  end
+end

data/lib/phobos/producer.rb

@@ -0,0 +1,122 @@
+module Phobos
+  module Producer
+    def self.included(base)
+      base.extend(Phobos::Producer::ClassMethods)
+    end
+
+    def producer
+      Phobos::Producer::PublicAPI.new(self)
+    end
+
+    class PublicAPI
+      def initialize(host_obj)
+        @host_obj = host_obj
+      end
+
+      def publish(topic, payload, key = nil)
+        class_producer.publish(topic, payload, key)
+      end
+
+      def async_publish(topic, payload, key = nil)
+        class_producer.async_publish(topic, payload, key)
+      end
+
+      # @param messages [Array(Hash(:topic, :payload, :key))]
+      #        e.g.: [
+      #          { topic: 'A', payload: 'message-1', key: '1' },
+      #          { topic: 'B', payload: 'message-2', key: '2' }
+      #        ]
+      #
+      def publish_list(messages)
+        class_producer.publish_list(messages)
+      end
+
+      def async_publish_list(messages)
+        class_producer.async_publish_list(messages)
+      end
+
+      private
+
+      def class_producer
+        @host_obj.class.producer
+      end
+    end
+
+    module ClassMethods
+      def producer
+        Phobos::Producer::ClassMethods::PublicAPI.new
+      end
+
+      class PublicAPI
+        NAMESPACE = :phobos_producer_store
+
+        # This method configures the kafka client used with publish operations
+        # performed by the host class
+        #
+        # @param kafka_client [Kafka::Client]
+        #
+        def configure_kafka_client(kafka_client)
+          async_producer_shutdown
+          producer_store[:kafka_client] = kafka_client
+        end
+
+        def kafka_client
+          producer_store[:kafka_client]
+        end
+
+        def publish(topic, payload, key = nil)
+          publish_list([{ topic: topic, payload: payload, key: key }])
+        end
+
+        def publish_list(messages)
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
+          producer = client.producer(Phobos.config.producer_hash)
+          produce_messages(producer, messages)
+        ensure
+          producer&.shutdown
+        end
+
+        def create_async_producer
+          client = kafka_client || configure_kafka_client(Phobos.create_kafka_client)
+          async_producer = client.async_producer(Phobos.config.producer_hash)
+          producer_store[:async_producer] = async_producer
+        end
+
+        def async_producer
+          producer_store[:async_producer]
+        end
+
+        def async_publish(topic, payload, key = nil)
+          async_publish_list([{ topic: topic, payload: payload, key: key }])
+        end
+
+        def async_publish_list(messages)
+          producer = async_producer || create_async_producer
+          produce_messages(producer, messages)
+        end
+
+        def async_producer_shutdown
+          async_producer&.deliver_messages
+          async_producer&.shutdown
+          producer_store[:async_producer] = nil
+        end
+
+        private
+
+        def produce_messages(producer, messages)
+          messages.each do |message|
+            producer.produce(message[:payload], topic: message[:topic],
+                                                key: message[:key],
+                                                partition_key: message[:key]
+            )
+          end
+          producer.deliver_messages
+        end
+
+        def producer_store
+          Thread.current[NAMESPACE] ||= {}
+        end
+      end
+    end
+  end
+end