deimos-ruby 1.0.0.pre.beta23 → 1.0.0.pre.beta24
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/Gemfile.lock +1 -1
- data/lib/deimos/backends/db.rb +1 -1
- data/lib/deimos/kafka_message.rb +1 -2
- data/lib/deimos/utils/db_producer.rb +3 -2
- data/lib/deimos/utils/executor.rb +10 -3
- data/lib/deimos/version.rb +1 -1
- data/lib/deimos.rb +2 -1
- data/spec/backends/db_spec.rb +2 -1
- data/spec/deimos_spec.rb +2 -0
- data/spec/utils/executor_spec.rb +12 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8604edf8b63251b1ae08c2080b8ca2434be662c0174191fa41eb58c32601c392
|
4
|
+
data.tar.gz: 17b37eede19bbec30c0a428de98a5cfaee0b30b788cec01ba2708443173f1f40
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b5eefc789b0604d743e24fb2f0ada5dc1c16aa9325c1bbf5b3c714b9959f663b9d9dd9957b5964f1e073f9facf2f985d948d1ba21bf83ba2c7972e833af874a5
|
7
|
+
data.tar.gz: 03130de06b20d0b05dc4388013aec6db96317e4b1e66e4873bc274c4c7a6ea785dbf507dfd8805c1effd813057a524bce5f7acbcd6c9a483df8c0045b4b5a481
|
data/CHANGELOG.md
CHANGED
@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
|
+
## UNRELEASED
|
9
|
+
|
10
|
+
## 1.0.0-beta24 - 2019-08-26
|
11
|
+
- Reconnect DB backend if database goes away.
|
12
|
+
- Sleep only 5 seconds between attempts instead of using exponential backoff.
|
13
|
+
- Fix for null payload being Avro-encoded.
|
14
|
+
|
8
15
|
## [1.0.0-beta23] - 2019-08-22
|
9
16
|
- Fix bug where nil payloads were not being saved to the DB.
|
10
17
|
- Fix DB producer rake task looking at THREADS env var instead of THREAD_COUNT.
|
data/Gemfile.lock
CHANGED
data/lib/deimos/backends/db.rb
CHANGED
@@ -12,7 +12,7 @@ module Deimos
|
|
12
12
|
def execute(producer_class:, messages:)
|
13
13
|
records = messages.map do |m|
|
14
14
|
message = Deimos::KafkaMessage.new(
|
15
|
-
message: m.encoded_payload.to_s.b,
|
15
|
+
message: m.encoded_payload ? m.encoded_payload.to_s.b : nil,
|
16
16
|
topic: m.topic,
|
17
17
|
partition_key: m.partition_key || m.key
|
18
18
|
)
|
data/lib/deimos/kafka_message.rb
CHANGED
@@ -11,7 +11,7 @@ module Deimos
|
|
11
11
|
# should already be a string.
|
12
12
|
# @param mess [Object]
|
13
13
|
def message=(mess)
|
14
|
-
write_attribute(:message, mess.to_s)
|
14
|
+
write_attribute(:message, mess ? mess.to_s : nil)
|
15
15
|
end
|
16
16
|
|
17
17
|
# @return [Hash]
|
@@ -23,6 +23,5 @@ module Deimos
|
|
23
23
|
topic: self.topic
|
24
24
|
}
|
25
25
|
end
|
26
|
-
|
27
26
|
end
|
28
27
|
end
|
@@ -20,6 +20,7 @@ module Deimos
|
|
20
20
|
def start
|
21
21
|
@logger.info('Starting...')
|
22
22
|
@signal_to_stop = false
|
23
|
+
ActiveRecord::Base.connection.reconnect!
|
23
24
|
loop do
|
24
25
|
if @signal_to_stop
|
25
26
|
@logger.info('Shutting down')
|
@@ -69,8 +70,8 @@ module Deimos
|
|
69
70
|
consumer.config.merge!(producer.config)
|
70
71
|
messages.map do |message|
|
71
72
|
{
|
72
|
-
:
|
73
|
-
:
|
73
|
+
key: message[:key].present? ? consumer.new.decode_key(message[:key]) : nil,
|
74
|
+
message: consumer.decoder.decode(message[:payload])
|
74
75
|
}
|
75
76
|
end
|
76
77
|
else
|
@@ -13,10 +13,13 @@ module Deimos
|
|
13
13
|
# @param runners [Array<#start, #stop, #id>] A list of objects that can be
|
14
14
|
# started or stopped.
|
15
15
|
# @param logger [Logger]
|
16
|
-
|
16
|
+
# @param sleep_seconds [Integer] Use a fixed time to sleep between
|
17
|
+
# failed runs instead of using an exponential backoff.
|
18
|
+
def initialize(runners, sleep_seconds: nil, logger: Logger.new(STDOUT))
|
17
19
|
@threads = Concurrent::Array.new
|
18
20
|
@runners = runners
|
19
21
|
@logger = logger
|
22
|
+
@sleep_seconds = sleep_seconds
|
20
23
|
end
|
21
24
|
|
22
25
|
# Start the executor.
|
@@ -98,8 +101,12 @@ module Deimos
|
|
98
101
|
# When "runner#start" is interrupted / crashes we assume it's
|
99
102
|
# safe to be called again
|
100
103
|
def handle_crashed_runner(runner, error, retry_count)
|
101
|
-
|
102
|
-
|
104
|
+
interval = if @sleep_seconds
|
105
|
+
@sleep_seconds
|
106
|
+
else
|
107
|
+
backoff = create_exponential_backoff
|
108
|
+
backoff.interval_at(retry_count).round(2)
|
109
|
+
end
|
103
110
|
|
104
111
|
metadata = {
|
105
112
|
listener_id: runner.id,
|
data/lib/deimos/version.rb
CHANGED
data/lib/deimos.rb
CHANGED
@@ -90,7 +90,8 @@ module Deimos
|
|
90
90
|
Deimos::Utils::DbProducer.new(self.config.logger)
|
91
91
|
end
|
92
92
|
executor = Deimos::Utils::Executor.new(producers,
|
93
|
-
|
93
|
+
sleep_seconds: 5,
|
94
|
+
logger: self.config.logger)
|
94
95
|
signal_handler = Deimos::Utils::SignalHandler.new(executor)
|
95
96
|
signal_handler.run!
|
96
97
|
end
|
data/spec/backends/db_spec.rb
CHANGED
@@ -26,8 +26,9 @@ each_db_config(Deimos::Backends::Db) do
|
|
26
26
|
|
27
27
|
it 'should add nil messages' do
|
28
28
|
described_class.publish(producer_class: MyProducer,
|
29
|
-
messages: [build_message(nil, 'my-topic',
|
29
|
+
messages: [build_message(nil, 'my-topic', 'foo1')])
|
30
30
|
expect(Deimos::KafkaMessage.count).to eq(1)
|
31
|
+
expect(Deimos::KafkaMessage.last.message).to eq(nil)
|
31
32
|
end
|
32
33
|
|
33
34
|
it 'should add to non-keyed messages' do
|
data/spec/deimos_spec.rb
CHANGED
@@ -79,6 +79,8 @@ describe Deimos do
|
|
79
79
|
it 'should start if backend is db and num_producer_threads is > 0' do
|
80
80
|
signal_handler = instance_double(Deimos::Utils::SignalHandler)
|
81
81
|
allow(signal_handler).to receive(:run!)
|
82
|
+
expect(Deimos::Utils::Executor).to receive(:new).
|
83
|
+
with(anything, sleep_seconds: 5, logger: anything).and_call_original
|
82
84
|
expect(Deimos::Utils::SignalHandler).to receive(:new) do |executor|
|
83
85
|
expect(executor.runners.size).to eq(2)
|
84
86
|
signal_handler
|
data/spec/utils/executor_spec.rb
CHANGED
@@ -26,8 +26,19 @@ RSpec.describe Deimos::Utils::Executor do
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
-
it '
|
29
|
+
it 'sleeps X seconds' do
|
30
|
+
executor = described_class.new(runners, sleep_seconds: 5)
|
30
31
|
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
32
|
+
expect(executor).to receive(:sleep).with(5).twice
|
33
|
+
runners.each { |r| r.should_error = true }
|
34
|
+
executor.start
|
35
|
+
wait_for do
|
36
|
+
runners.each { |r| expect(r.started).to be_truthy }
|
37
|
+
executor.stop
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'reconnects crashed runners' do
|
31
42
|
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
32
43
|
runners.each { |r| r.should_error = true }
|
33
44
|
executor.start
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: deimos-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.pre.
|
4
|
+
version: 1.0.0.pre.beta24
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Orner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-08-
|
11
|
+
date: 2019-08-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: avro-patches
|