deimos-ruby 1.0.0.pre.beta23 → 1.0.0.pre.beta24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7de207dcf4a09b8881881cfbb6af9c007c48083e091344ed3401e4a6fb944fb
-  data.tar.gz: 662ecac0adcd71cd8bf4bb44fa70436a88844d4ef65c76f8a2f341c964ae2cd8
+  metadata.gz: 8604edf8b63251b1ae08c2080b8ca2434be662c0174191fa41eb58c32601c392
+  data.tar.gz: 17b37eede19bbec30c0a428de98a5cfaee0b30b788cec01ba2708443173f1f40
 SHA512:
-  metadata.gz: 4508f6fb52f68a89b57cdda8bf46d1069d2359c3fafd9e12e099601ed5b474a36739958a56fa4c796584eec73ae1eac3e8e31aab121d139a9a8ee6dac7797fc2
-  data.tar.gz: b2b31653474653660fd0e4fac2af4843954c39b1525604fb7ff9fa39aa14f6e982627ad1da48c655077f4ad3143253ee923043229b36f6cf44d03135fb4c873a
+  metadata.gz: b5eefc789b0604d743e24fb2f0ada5dc1c16aa9325c1bbf5b3c714b9959f663b9d9dd9957b5964f1e073f9facf2f985d948d1ba21bf83ba2c7972e833af874a5
+  data.tar.gz: 03130de06b20d0b05dc4388013aec6db96317e4b1e66e4873bc274c4c7a6ea785dbf507dfd8805c1effd813057a524bce5f7acbcd6c9a483df8c0045b4b5a481

data/CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## UNRELEASED
+
+## 1.0.0-beta24 - 2019-08-26
+- Reconnect DB backend if database goes away.
+- Sleep only 5 seconds between attempts instead of using exponential backoff.
+- Fix for null payload being Avro-encoded.
+
 ## [1.0.0-beta23] - 2019-08-22
 - Fix bug where nil payloads were not being saved to the DB.
 - Fix DB producer rake task looking at THREADS env var instead of THREAD_COUNT.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.0.0.pre.beta23)
+    deimos-ruby (1.0.0.pre.beta24)
       avro-patches (~> 0.3)
       avro_turf (~> 0.8)
       phobos (~> 1.8)

data/lib/deimos/backends/db.rb

@@ -12,7 +12,7 @@ module Deimos
         def execute(producer_class:, messages:)
           records = messages.map do |m|
             message = Deimos::KafkaMessage.new(
-              message: m.encoded_payload.to_s.b,
+              message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
               topic: m.topic,
               partition_key: m.partition_key || m.key
             )

data/lib/deimos/kafka_message.rb

@@ -11,7 +11,7 @@ module Deimos
     # should already be a string.
     # @param mess [Object]
     def message=(mess)
-      write_attribute(:message, mess.to_s)
+      write_attribute(:message, mess ? mess.to_s : nil)
     end
 
     # @return [Hash]
@@ -23,6 +23,5 @@ module Deimos
         topic: self.topic
       }
     end
-
   end
 end

data/lib/deimos/utils/db_producer.rb

@@ -20,6 +20,7 @@ module Deimos
       def start
         @logger.info('Starting...')
         @signal_to_stop = false
+        ActiveRecord::Base.connection.reconnect!
         loop do
           if @signal_to_stop
             @logger.info('Shutting down')
@@ -69,8 +70,8 @@ module Deimos
                                  consumer.config.merge!(producer.config)
                                  messages.map do |message|
                                    {
-                                     :key => message[:key].present? ? consumer.new.decode_key(message[:key]) : nil,
-                                     :message => consumer.decoder.decode(message[:payload])
+                                     key: message[:key].present? ? consumer.new.decode_key(message[:key]) : nil,
+                                     message: consumer.decoder.decode(message[:payload])
                                    }
                                  end
                                else

data/lib/deimos/utils/executor.rb

@@ -13,10 +13,13 @@ module Deimos
       # @param runners [Array<#start, #stop, #id>] A list of objects that can be
       # started or stopped.
       # @param logger [Logger]
-      def initialize(runners, logger=Logger.new(STDOUT))
+      # @param sleep_seconds [Integer] Use a fixed time to sleep between
+      # failed runs instead of using an exponential backoff.
+      def initialize(runners, sleep_seconds: nil, logger: Logger.new(STDOUT))
         @threads = Concurrent::Array.new
         @runners = runners
         @logger = logger
+        @sleep_seconds = sleep_seconds
       end
 
       # Start the executor.
@@ -98,8 +101,12 @@ module Deimos
       # When "runner#start" is interrupted / crashes we assume it's
       # safe to be called again
       def handle_crashed_runner(runner, error, retry_count)
-        backoff = create_exponential_backoff
-        interval = backoff.interval_at(retry_count).round(2)
+        interval = if @sleep_seconds
+                     @sleep_seconds
+                   else
+                     backoff = create_exponential_backoff
+                     backoff.interval_at(retry_count).round(2)
+                   end
 
         metadata = {
           listener_id: runner.id,

data/lib/deimos/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Deimos
-  VERSION = '1.0.0-beta23'
+  VERSION = '1.0.0-beta24'
 end

data/lib/deimos.rb

@@ -90,7 +90,8 @@ module Deimos
         Deimos::Utils::DbProducer.new(self.config.logger)
       end
       executor = Deimos::Utils::Executor.new(producers,
-                                             self.config.logger)
+                                             sleep_seconds: 5,
+                                             logger: self.config.logger)
       signal_handler = Deimos::Utils::SignalHandler.new(executor)
       signal_handler.run!
     end

data/spec/backends/db_spec.rb

@@ -26,8 +26,9 @@ each_db_config(Deimos::Backends::Db) do
 
   it 'should add nil messages' do
     described_class.publish(producer_class: MyProducer,
-                            messages: [build_message(nil, 'my-topic', "foo1")])
+                            messages: [build_message(nil, 'my-topic', 'foo1')])
     expect(Deimos::KafkaMessage.count).to eq(1)
+    expect(Deimos::KafkaMessage.last.message).to eq(nil)
   end
 
   it 'should add to non-keyed messages' do

data/spec/deimos_spec.rb

@@ -79,6 +79,8 @@ describe Deimos do
     it 'should start if backend is db and num_producer_threads is > 0' do
       signal_handler = instance_double(Deimos::Utils::SignalHandler)
       allow(signal_handler).to receive(:run!)
+      expect(Deimos::Utils::Executor).to receive(:new).
+        with(anything, sleep_seconds: 5, logger: anything).and_call_original
       expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
         expect(executor.runners.size).to eq(2)
         signal_handler

data/spec/utils/executor_spec.rb

@@ -26,8 +26,19 @@ RSpec.describe Deimos::Utils::Executor do
     end
   end
 
-  it 'reconnects crashed runners' do
+  it 'sleeps X seconds' do
+    executor = described_class.new(runners, sleep_seconds: 5)
     allow(executor).to receive(:handle_crashed_runner).and_call_original
+    expect(executor).to receive(:sleep).with(5).twice
+    runners.each { |r| r.should_error = true }
+    executor.start
+    wait_for do
+      runners.each { |r| expect(r.started).to be_truthy }
+      executor.stop
+    end
+  end
+
+  it 'reconnects crashed runners' do
     allow(executor).to receive(:handle_crashed_runner).and_call_original
     runners.each { |r| r.should_error = true }
     executor.start

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: deimos-ruby
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.beta23
+  version: 1.0.0.pre.beta24
 platform: ruby
 authors:
 - Daniel Orner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-22 00:00:00.000000000 Z
+date: 2019-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: avro-patches