deimos-ruby 1.7.0.pre.beta1 → 1.8.0.pre.beta1

data/lib/deimos/base_consumer.rb

@@ -1,100 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  # Shared methods for Kafka Consumers
-  class BaseConsumer
-    include SharedConfig
-
-    class << self
-      # @return [Deimos::SchemaBackends::Base]
-      def decoder
-        @decoder ||= Deimos.schema_backend(schema: config[:schema],
-                                           namespace: config[:namespace])
-      end
-
-      # @return [Deimos::SchemaBackends::Base]
-      def key_decoder
-        @key_decoder ||= Deimos.schema_backend(schema: config[:key_schema],
-                                               namespace: config[:namespace])
-      end
-    end
-
-    # Helper method to decode an encoded key.
-    # @param key [String]
-    # @return [Object] the decoded key.
-    def decode_key(key)
-      return nil if key.nil?
-
-      config = self.class.config
-      unless config[:key_configured]
-        raise 'No key config given - if you are not decoding keys, please use '\
-          '`key_config plain: true`'
-      end
-
-      if config[:key_field]
-        self.class.decoder.decode_key(key, config[:key_field])
-      elsif config[:key_schema]
-        self.class.key_decoder.decode(key, schema: config[:key_schema])
-      else # no encoding
-        key
-      end
-    end
-
-  protected
-
-    def _with_span
-      @span = Deimos.config.tracer&.start(
-        'deimos-consumer',
-        resource: self.class.name.gsub('::', '-')
-      )
-      yield
-    ensure
-      Deimos.config.tracer&.finish(@span)
-    end
-
-    def _report_time_delayed(payload, metadata)
-      return if payload.nil? || payload['timestamp'].blank?
-
-      begin
-        time_delayed = Time.now.in_time_zone - payload['timestamp'].to_datetime
-      rescue ArgumentError
-        Deimos.config.logger.info(
-          message: "Error parsing timestamp! #{payload['timestamp']}"
-        )
-        return
-      end
-      Deimos.config.metrics&.histogram('handler', time_delayed, tags: %W(
-                                         time:time_delayed
-                                         topic:#{metadata[:topic]}
-                                       ))
-    end
-
-    # Overrideable method to determine if a given error should be considered
-    # "fatal" and always be reraised.
-    # @param error [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    # @return [Boolean]
-    def fatal_error?(_error, _payload, _metadata)
-      false
-    end
-
-    # @param exception [Exception]
-    # @param payload [Hash]
-    # @param metadata [Hash]
-    def _handle_error(exception, payload, metadata)
-      Deimos.config.tracer&.set_error(@span, exception)
-
-      raise if Deimos.config.consumers.reraise_errors ||
-               Deimos.config.consumers.fatal_error&.call(exception, payload, metadata) ||
-               fatal_error?(exception, payload, metadata)
-    end
-
-    # @param _time_taken [Float]
-    # @param _payload [Hash]
-    # @param _metadata [Hash]
-    def _handle_success(_time_taken, _payload, _metadata)
-      raise NotImplementedError
-    end
-  end
-end

data/lib/deimos/utils/executor.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-# rubocop:disable Lint/RescueException
-module Deimos
-  module Utils
-    # Mostly copied from Phobos::Executor. We should DRY this up by putting in a
-    # PR to make it more generic. Might even make sense to move to a separate
-    # gem.
-    class Executor
-      # @return [Array<#start, #stop, #id>]
-      attr_accessor :runners
-
-      # @param runners [Array<#start, #stop, #id>] A list of objects that can be
-      # started or stopped.
-      # @param logger [Logger]
-      # @param sleep_seconds [Integer] Use a fixed time to sleep between
-      # failed runs instead of using an exponential backoff.
-      def initialize(runners, sleep_seconds: nil, logger: Logger.new(STDOUT))
-        @threads = Concurrent::Array.new
-        @runners = runners
-        @logger = logger
-        @sleep_seconds = sleep_seconds
-      end
-
-      # Start the executor.
-      def start
-        @logger.info('Starting executor')
-        @signal_to_stop = false
-        @threads.clear
-        @thread_pool = Concurrent::FixedThreadPool.new(@runners.size)
-
-        @runners.each do |runner|
-          @thread_pool.post do
-            thread = Thread.current
-            thread.abort_on_exception = true
-            @threads << thread
-            run_object(runner)
-          end
-        end
-
-        true
-      end
-
-      # Stop the executor.
-      def stop
-        return if @signal_to_stop
-
-        @logger.info('Stopping executor')
-        @signal_to_stop = true
-        @runners.each(&:stop)
-        @threads.select(&:alive?).each do |thread|
-          begin
-            thread.wakeup
-          rescue StandardError
-            nil
-          end
-        end
-        @thread_pool&.shutdown
-        @thread_pool&.wait_for_termination
-        @logger.info('Executor stopped')
-      end
-
-    private
-
-      # @param exception [Throwable]
-      # @return [Hash]
-      def error_metadata(exception)
-        {
-          exception_class: exception.class.name,
-          exception_message: exception.message,
-          backtrace: exception.backtrace
-        }
-      end
-
-      def run_object(runner)
-        retry_count = 0
-
-        begin
-          @logger.info("Running #{runner.id}")
-          runner.start
-          retry_count = 0 # success - reset retry count
-        rescue Exception => e
-          handle_crashed_runner(runner, e, retry_count)
-          retry_count += 1
-          retry unless @signal_to_stop
-        end
-      rescue Exception => e
-        @logger.error("Failed to run executor (#{e.message}) #{error_metadata(e)}")
-        raise e
-      end
-
-      # @return [ExponentialBackoff]
-      def create_exponential_backoff
-        min = 1
-        max = 60
-        ExponentialBackoff.new(min, max).tap do |backoff|
-          backoff.randomize_factor = rand
-        end
-      end
-
-      # When "runner#start" is interrupted / crashes we assume it's
-      # safe to be called again
-      def handle_crashed_runner(runner, error, retry_count)
-        interval = if @sleep_seconds
-                     @sleep_seconds
-                   else
-                     backoff = create_exponential_backoff
-                     backoff.interval_at(retry_count).round(2)
-                   end
-
-        metadata = {
-          listener_id: runner.id,
-          retry_count: retry_count,
-          waiting_time: interval
-        }.merge(error_metadata(error))
-
-        @logger.error("Runner crashed, waiting #{interval}s (#{error.message}) #{metadata}")
-        sleep(interval)
-      end
-    end
-  end
-end
-
-# rubocop:enable Lint/RescueException

data/lib/deimos/utils/signal_handler.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-module Deimos
-  module Utils
-    # Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
-    # basically replace that implementation with this one to make it more generic.
-    class SignalHandler
-      SIGNALS = %i(INT TERM QUIT).freeze
-
-      # Takes any object that responds to the `start` and `stop` methods.
-      # @param runner[#start, #stop]
-      def initialize(runner)
-        @signal_queue = []
-        @reader, @writer = IO.pipe
-        @runner = runner
-      end
-
-      # Run the runner.
-      def run!
-        setup_signals
-        @runner.start
-
-        loop do
-          case signal_queue.pop
-          when *SIGNALS
-            @runner.stop
-            break
-          else
-            ready = IO.select([reader, writer])
-
-            # drain the self-pipe so it won't be returned again next time
-            reader.read_nonblock(1) if ready[0].include?(reader)
-          end
-        end
-      end
-
-    private
-
-      attr_reader :reader, :writer, :signal_queue, :executor
-
-      # https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
-      def prepend_handler(signal)
-        previous = Signal.trap(signal) do
-          previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
-          yield
-          previous.call
-        end
-      end
-
-      # Trap signals using the self-pipe trick.
-      def setup_signals
-        at_exit { @runner&.stop }
-        SIGNALS.each do |signal|
-          prepend_handler(signal) do
-            unblock(signal)
-          end
-        end
-      end
-
-      # Save the signal to the queue and continue on.
-      # @param signal [Symbol]
-      def unblock(signal)
-        writer.write_nonblock('.')
-        signal_queue << signal
-      end
-    end
-  end
-end

data/spec/utils/executor_spec.rb

@@ -1,53 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Deimos::Utils::Executor do
-
-  let(:executor) { described_class.new(runners) }
-  let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
-
-  it 'starts and stops configured runners' do
-    runners.each do |r|
-      expect(r.started).to be_falsey
-      expect(r.stopped).to be_falsey
-    end
-    executor.start
-    wait_for do
-      runners.each do |r|
-        expect(r.started).to be_truthy
-        expect(r.stopped).to be_falsey
-      end
-      executor.stop
-      runners.each do |r|
-        expect(r.started).to be_truthy
-        expect(r.stopped).to be_truthy
-      end
-    end
-  end
-
-  it 'sleeps X seconds' do
-    executor = described_class.new(runners, sleep_seconds: 5)
-    allow(executor).to receive(:handle_crashed_runner).and_call_original
-    expect(executor).to receive(:sleep).with(5).twice
-    runners.each { |r| r.should_error = true }
-    executor.start
-    wait_for do
-      runners.each { |r| expect(r.started).to be_truthy }
-      executor.stop
-    end
-  end
-
-  it 'reconnects crashed runners' do
-    allow(executor).to receive(:handle_crashed_runner).and_call_original
-    runners.each { |r| r.should_error = true }
-    executor.start
-    wait_for do
-      expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
-      expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
-      runners.each { |r| expect(r.started).to be_truthy }
-      executor.stop
-    end
-  end
-
-end

data/spec/utils/signal_handler_spec.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe Deimos::Utils::SignalHandler do
-  describe '#run!' do
-
-    it 'starts and stops the runner' do
-      runner = TestRunners::TestRunner.new
-      expect(runner).to receive(:start)
-      expect(runner).to receive(:stop)
-
-      signal_handler = described_class.new(runner)
-      signal_handler.send(:unblock, described_class::SIGNALS.first)
-      signal_handler.run!
-    end
-  end
-end