deimos-ruby 1.6.1 → 1.8.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +30 -0
- data/Gemfile.lock +87 -80
- data/README.md +139 -15
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +3 -2
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +7 -6
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +148 -0
- data/lib/deimos/consume/message_consumption.rb +93 -0
- data/lib/deimos/consumer.rb +79 -69
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +29 -23
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +8 -3
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +22 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +23 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_source_spec.rb +53 -0
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +61 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +61 -20
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
File without changes
|
@@ -1,68 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
module Utils
|
5
|
-
# Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
|
6
|
-
# basically replace that implementation with this one to make it more generic.
|
7
|
-
class SignalHandler
|
8
|
-
SIGNALS = %i(INT TERM QUIT).freeze
|
9
|
-
|
10
|
-
# Takes any object that responds to the `start` and `stop` methods.
|
11
|
-
# @param runner[#start, #stop]
|
12
|
-
def initialize(runner)
|
13
|
-
@signal_queue = []
|
14
|
-
@reader, @writer = IO.pipe
|
15
|
-
@runner = runner
|
16
|
-
end
|
17
|
-
|
18
|
-
# Run the runner.
|
19
|
-
def run!
|
20
|
-
setup_signals
|
21
|
-
@runner.start
|
22
|
-
|
23
|
-
loop do
|
24
|
-
case signal_queue.pop
|
25
|
-
when *SIGNALS
|
26
|
-
@runner.stop
|
27
|
-
break
|
28
|
-
else
|
29
|
-
ready = IO.select([reader, writer])
|
30
|
-
|
31
|
-
# drain the self-pipe so it won't be returned again next time
|
32
|
-
reader.read_nonblock(1) if ready[0].include?(reader)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
private
|
38
|
-
|
39
|
-
attr_reader :reader, :writer, :signal_queue, :executor
|
40
|
-
|
41
|
-
# https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
|
42
|
-
def prepend_handler(signal)
|
43
|
-
previous = Signal.trap(signal) do
|
44
|
-
previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
|
45
|
-
yield
|
46
|
-
previous.call
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
# Trap signals using the self-pipe trick.
|
51
|
-
def setup_signals
|
52
|
-
at_exit { @runner&.stop }
|
53
|
-
SIGNALS.each do |signal|
|
54
|
-
prepend_handler(signal) do
|
55
|
-
unblock(signal)
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
# Save the signal to the queue and continue on.
|
61
|
-
# @param signal [Symbol]
|
62
|
-
def unblock(signal)
|
63
|
-
writer.write_nonblock('.')
|
64
|
-
signal_queue << signal
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
data/spec/utils/executor_spec.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
RSpec.describe Deimos::Utils::Executor do
|
6
|
-
|
7
|
-
let(:executor) { described_class.new(runners) }
|
8
|
-
let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
|
9
|
-
|
10
|
-
it 'starts and stops configured runners' do
|
11
|
-
runners.each do |r|
|
12
|
-
expect(r.started).to be_falsey
|
13
|
-
expect(r.stopped).to be_falsey
|
14
|
-
end
|
15
|
-
executor.start
|
16
|
-
wait_for do
|
17
|
-
runners.each do |r|
|
18
|
-
expect(r.started).to be_truthy
|
19
|
-
expect(r.stopped).to be_falsey
|
20
|
-
end
|
21
|
-
executor.stop
|
22
|
-
runners.each do |r|
|
23
|
-
expect(r.started).to be_truthy
|
24
|
-
expect(r.stopped).to be_truthy
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'sleeps X seconds' do
|
30
|
-
executor = described_class.new(runners, sleep_seconds: 5)
|
31
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
32
|
-
expect(executor).to receive(:sleep).with(5).twice
|
33
|
-
runners.each { |r| r.should_error = true }
|
34
|
-
executor.start
|
35
|
-
wait_for do
|
36
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
37
|
-
executor.stop
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'reconnects crashed runners' do
|
42
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
43
|
-
runners.each { |r| r.should_error = true }
|
44
|
-
executor.start
|
45
|
-
wait_for do
|
46
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
|
47
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
|
48
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
49
|
-
executor.stop
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
RSpec.describe Deimos::Utils::SignalHandler do
|
4
|
-
describe '#run!' do
|
5
|
-
|
6
|
-
it 'starts and stops the runner' do
|
7
|
-
runner = TestRunners::TestRunner.new
|
8
|
-
expect(runner).to receive(:start)
|
9
|
-
expect(runner).to receive(:stop)
|
10
|
-
|
11
|
-
signal_handler = described_class.new(runner)
|
12
|
-
signal_handler.send(:unblock, described_class::SIGNALS.first)
|
13
|
-
signal_handler.run!
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|