phobos_temp_fork 0.0.1

data/lib/phobos/actions/process_message.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'phobos/processor'
+
+module Phobos
+  module Actions
+    class ProcessMessage
+      include Phobos::Processor
+
+      attr_reader :metadata
+
+      def initialize(listener:, message:, listener_metadata:)
+        @listener = listener
+        @message = message
+        @metadata = listener_metadata.merge(
+          key: message.key,
+          partition: message.partition,
+          offset: message.offset,
+          retry_count: 0,
+          headers: message.headers
+        )
+      end
+
+      def execute
+        payload = force_encoding(@message.value)
+
+        begin
+          process_message(payload)
+        rescue StandardError => e
+          handle_error(e, 'listener.retry_handler_error',
+                       "error processing message, waiting #{backoff_interval}s")
+          retry
+        end
+      end
+
+      private
+
+      def process_message(payload)
+        instrument('listener.process_message', @metadata) do
+          handler = @listener.handler_class.new
+
+          handler.around_consume(payload, @metadata) do |around_payload, around_metadata|
+            handler.consume(around_payload, around_metadata)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/phobos/batch_handler.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Phobos
+  module BatchHandler
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def consume_batch(_payloads, _metadata)
+      raise NotImplementedError
+    end
+
+    def around_consume_batch(payloads, metadata)
+      yield payloads, metadata
+    end
+
+    module ClassMethods
+      def start(kafka_client); end
+
+      def stop; end
+    end
+  end
+end

data/lib/phobos/batch_message.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Phobos
+  class BatchMessage
+    attr_accessor :key, :partition, :offset, :payload, :headers
+
+    def initialize(key:, partition:, offset:, payload:, headers:)
+      @key = key
+      @partition = partition
+      @offset = offset
+      @payload = payload
+      @headers = headers
+    end
+
+    def ==(other)
+      [:key, :partition, :offset, :payload, :headers].all? do |s|
+        public_send(s) == other.public_send(s)
+      end
+    end
+  end
+end

data/lib/phobos/cli.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'thor'
+require 'phobos/cli/start'
+
+module Phobos
+  module CLI
+    def self.logger
+      @logger ||= Logging.logger[self].tap do |l|
+        l.appenders = [Logging.appenders.stdout]
+      end
+    end
+
+    class Commands < Thor
+      include Thor::Actions
+
+      map '-v' => :version
+      map '--version' => :version
+
+      desc 'version', 'Outputs the version number. Can be used with: phobos -v or phobos --version'
+      def version
+        puts Phobos::VERSION
+      end
+
+      desc 'init', 'Initialize your project with Phobos'
+      def init
+        copy_file 'config/phobos.yml.example', 'config/phobos.yml'
+        create_file 'phobos_boot.rb' do
+          <<~EXAMPLE
+            # Use this file to load your code
+            puts <<~ART
+              ______ _           _
+              | ___ \\\\ |         | |
+              | |_/ / |__   ___ | |__   ___  ___
+              |  __/| '_ \\\\ / _ \\\\| '_ \\\\ / _ \\\\/ __|
+              | |   | | | | (_) | |_) | (_) \\\\__ \\\\
+              \\\\_|   |_| |_|\\\\___/|_.__/ \\\\___/|___/
+            ART
+            puts "\nphobos_boot.rb - find this file at \#{File.expand_path(__FILE__)}\n\n"
+          EXAMPLE
+        end
+      end
+
+      desc 'start', 'Starts Phobos'
+      method_option :config,
+                    aliases: ['-c'],
+                    default: 'config/phobos.yml',
+                    banner: 'Configuration file'
+      method_option :boot,
+                    aliases: ['-b'],
+                    banner: 'File path to load application specific code',
+                    default: 'phobos_boot.rb'
+      method_option :listeners,
+                    aliases: ['-l'],
+                    banner: 'Separate listeners config file (optional)'
+      method_option :skip_config,
+                    default: false,
+                    type: :boolean,
+                    banner: 'Skip config file'
+      def start
+        Phobos::CLI::Start.new(options).execute
+      end
+
+      def self.source_root
+        File.expand_path(File.join(File.dirname(__FILE__), '../..'))
+      end
+    end
+  end
+end

data/lib/phobos/cli/runner.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Phobos
+  module CLI
+    class Runner
+      SIGNALS = [:INT, :TERM, :QUIT].freeze
+
+      def initialize
+        @signal_queue = []
+        @reader, @writer = IO.pipe
+        @executor = Phobos::Executor.new
+      end
+
+      def run!
+        setup_signals
+        executor.start
+
+        loop do
+          case signal_queue.pop
+          when *SIGNALS
+            executor.stop
+            break
+          else
+            ready = IO.select([reader, writer])
+
+            # drain the self-pipe so it won't be returned again next time
+            reader.read_nonblock(1) if ready[0].include?(reader)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :reader, :writer, :signal_queue, :executor
+
+      def setup_signals
+        SIGNALS.each do |signal|
+          Signal.trap(signal) { unblock(signal) }
+        end
+      end
+
+      def unblock(signal)
+        writer.write_nonblock('.')
+        signal_queue << signal
+      end
+    end
+  end
+end

data/lib/phobos/cli/start.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'phobos/cli/runner'
+
+module Phobos
+  module CLI
+    class Start
+      def initialize(options)
+        @config_file = File.expand_path(options[:config]) unless options[:skip_config]
+        @boot_file = File.expand_path(options[:boot])
+
+        @listeners_file = File.expand_path(options[:listeners]) if options[:listeners]
+      end
+
+      def execute
+        load_boot_file
+
+        if config_file
+          validate_config_file!
+          Phobos.configure(config_file)
+        end
+
+        Phobos.add_listeners(listeners_file) if listeners_file
+
+        validate_listeners!
+
+        Phobos::CLI::Runner.new.run!
+      end
+
+      private
+
+      attr_reader :config_file, :boot_file, :listeners_file
+
+      def validate_config_file!
+        File.exist?(config_file) || error_exit("Config file not found (#{config_file})")
+      end
+
+      def validate_listeners! # rubocop:disable Metrics/MethodLength
+        Phobos.config.listeners.each do |listener|
+          handler = listener.handler
+
+          begin
+            handler.constantize
+          rescue NameError
+            error_exit("Handler '#{handler}' not defined")
+          end
+
+          delivery = listener.delivery
+          if delivery.nil?
+            Phobos::CLI.logger.warn do
+              Hash(message: "Delivery option should be specified, defaulting to 'batch'"\
+                ' - specify this option to silence this message')
+            end
+          elsif !Listener::DELIVERY_OPTS.include?(delivery)
+            error_exit("Invalid delivery option '#{delivery}'. Please specify one of: "\
+              "#{Listener::DELIVERY_OPTS.join(', ')}")
+          end
+        end
+      end
+
+      def error_exit(msg)
+        Phobos::CLI.logger.error { Hash(message: msg) }
+        exit(1)
+      end
+
+      def load_boot_file
+        load(boot_file) if File.exist?(boot_file)
+      end
+    end
+  end
+end

data/lib/phobos/constants.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Phobos
+  module Constants
+    LOG_DATE_PATTERN = '%Y-%m-%dT%H:%M:%S:%L%zZ'
+
+    KAFKA_CONSUMER_OPTS = [
+      :session_timeout,
+      :offset_commit_interval,
+      :offset_commit_threshold,
+      :heartbeat_interval,
+      :offset_retention_time
+    ].freeze
+
+    LISTENER_OPTS = [
+      :handler,
+      :group_id,
+      :topic,
+      :min_bytes,
+      :max_wait_time,
+      :force_encoding,
+      :start_from_beginning,
+      :max_bytes_per_partition,
+      :backoff,
+      :delivery,
+      :session_timeout,
+      :offset_commit_interval,
+      :offset_commit_threshold,
+      :heartbeat_interval,
+      :offset_retention_time
+    ].freeze
+  end
+end

data/lib/phobos/deep_struct.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Please use this with at least the same consideration as you would when using OpenStruct.
+# Right now we only use this to parse our internal configuration files. It is not meant to
+# be used on incoming data.
+module Phobos
+  class DeepStruct < OpenStruct
+    # Based on
+    # https://docs.omniref.com/ruby/2.3.0/files/lib/ostruct.rb#line=88
+    def initialize(hash = nil)
+      super
+      @hash_table = {}
+
+      hash&.each_pair do |key, value|
+        key = key.to_sym
+        @table[key] = to_deep_struct(value)
+        @hash_table[key] = value
+      end
+    end
+
+    def to_h
+      @hash_table.dup
+    end
+    alias to_hash to_h
+
+    private
+
+    def to_deep_struct(value)
+      case value
+      when Hash
+        self.class.new(value)
+      when Enumerable
+        value.map { |el| to_deep_struct(el) }
+      else
+        value
+      end
+    end
+  end
+end

data/lib/phobos/echo_handler.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Phobos
+  class EchoHandler
+    include Phobos::Handler
+
+    def consume(message, metadata)
+      Phobos.logger.info { Hash(message: message).merge(metadata) }
+    end
+  end
+end

data/lib/phobos/errors.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Phobos
+  class Error < StandardError; end
+  class AbortError < Error; end
+end

data/lib/phobos/executor.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Phobos
+  class Executor
+    include Phobos::Instrumentation
+    include Phobos::Log
+
+    def initialize
+      @threads = Concurrent::Array.new
+      @listeners = Phobos.config.listeners.flat_map do |config|
+        handler_class = config.handler.constantize
+        listener_configs = config.to_hash.deep_symbolize_keys
+        max_concurrency = listener_configs[:max_concurrency] || 1
+        Array.new(max_concurrency).map do
+          configs = listener_configs.select { |k| Constants::LISTENER_OPTS.include?(k) }
+          Phobos::Listener.new(**configs.merge(handler: handler_class))
+        end
+      end
+    end
+
+    def start
+      @signal_to_stop = false
+      @threads.clear
+      @thread_pool = Concurrent::FixedThreadPool.new(@listeners.size)
+
+      @listeners.each do |listener|
+        @thread_pool.post do
+          thread = Thread.current
+          thread.abort_on_exception = true
+          @threads << thread
+          run_listener(listener)
+        end
+      end
+
+      true
+    end
+
+    def stop
+      return if @signal_to_stop
+
+      instrument('executor.stop') do
+        @signal_to_stop = true
+        @listeners.each(&:stop)
+        @threads.select(&:alive?).each do |thread|
+          begin
+            thread.wakeup
+          rescue StandardError
+            nil
+          end
+        end
+        @thread_pool&.shutdown
+        @thread_pool&.wait_for_termination
+        Phobos.logger.info { Hash(message: 'Executor stopped') }
+      end
+    end
+
+    private
+
+    def error_metadata(exception)
+      {
+        exception_class: exception.class.name,
+        exception_message: exception.message,
+        backtrace: exception.backtrace
+      }
+    end
+
+    # rubocop:disable Lint/RescueException
+    def run_listener(listener)
+      retry_count = 0
+
+      begin
+        listener.start
+      rescue Exception => e
+        handle_crashed_listener(listener, e, retry_count)
+        retry_count += 1
+        retry unless @signal_to_stop
+      end
+    rescue Exception => e
+      log_error("Failed to run listener (#{e.message})", error_metadata(e))
+      raise e
+    end
+    # rubocop:enable Lint/RescueException
+
+    # When "listener#start" is interrupted it's safe to assume that the consumer
+    # and the kafka client were properly stopped, it's safe to call start
+    # again
+    def handle_crashed_listener(listener, error, retry_count)
+      backoff = listener.create_exponential_backoff
+      interval = backoff.interval_at(retry_count).round(2)
+
+      metadata = {
+        listener_id: listener.id,
+        retry_count: retry_count,
+        waiting_time: interval
+      }.merge(error_metadata(error))
+
+      instrument('executor.retry_listener_error', metadata) do
+        log_error("Listener crashed, waiting #{interval}s (#{error.message})", metadata)
+        sleep interval
+      end
+    end
+  end
+end