deimos-ruby 1.6.1 → 1.8.0.pre.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +9 -0
- data/.rubocop.yml +15 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +30 -0
- data/Gemfile.lock +87 -80
- data/README.md +139 -15
- data/Rakefile +1 -1
- data/deimos-ruby.gemspec +3 -2
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +27 -0
- data/lib/deimos.rb +7 -6
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/active_record_producer.rb +23 -0
- data/lib/deimos/batch_consumer.rb +2 -140
- data/lib/deimos/config/configuration.rb +28 -10
- data/lib/deimos/consume/batch_consumption.rb +148 -0
- data/lib/deimos/consume/message_consumption.rb +93 -0
- data/lib/deimos/consumer.rb +79 -69
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_source.rb +29 -23
- data/lib/deimos/kafka_topic_info.rb +1 -1
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/metrics/provider.rb +0 -2
- data/lib/deimos/poll_info.rb +9 -0
- data/lib/deimos/tracing/provider.rb +0 -2
- data/lib/deimos/utils/db_poller.rb +149 -0
- data/lib/deimos/utils/db_producer.rb +8 -3
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/db_poller/templates/migration +11 -0
- data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
- data/lib/generators/deimos/db_poller_generator.rb +48 -0
- data/lib/tasks/deimos.rake +7 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +22 -11
- data/spec/active_record_producer_spec.rb +66 -88
- data/spec/batch_consumer_spec.rb +23 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +8 -8
- data/spec/deimos_spec.rb +57 -49
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_source_spec.rb +53 -0
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +3 -3
- data/spec/rake_spec.rb +1 -1
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/spec_helper.rb +61 -6
- data/spec/utils/db_poller_spec.rb +320 -0
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- metadata +61 -20
- data/lib/deimos/base_consumer.rb +0 -104
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
File without changes
|
@@ -1,68 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
module Utils
|
5
|
-
# Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
|
6
|
-
# basically replace that implementation with this one to make it more generic.
|
7
|
-
class SignalHandler
|
8
|
-
SIGNALS = %i(INT TERM QUIT).freeze
|
9
|
-
|
10
|
-
# Takes any object that responds to the `start` and `stop` methods.
|
11
|
-
# @param runner[#start, #stop]
|
12
|
-
def initialize(runner)
|
13
|
-
@signal_queue = []
|
14
|
-
@reader, @writer = IO.pipe
|
15
|
-
@runner = runner
|
16
|
-
end
|
17
|
-
|
18
|
-
# Run the runner.
|
19
|
-
def run!
|
20
|
-
setup_signals
|
21
|
-
@runner.start
|
22
|
-
|
23
|
-
loop do
|
24
|
-
case signal_queue.pop
|
25
|
-
when *SIGNALS
|
26
|
-
@runner.stop
|
27
|
-
break
|
28
|
-
else
|
29
|
-
ready = IO.select([reader, writer])
|
30
|
-
|
31
|
-
# drain the self-pipe so it won't be returned again next time
|
32
|
-
reader.read_nonblock(1) if ready[0].include?(reader)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
private
|
38
|
-
|
39
|
-
attr_reader :reader, :writer, :signal_queue, :executor
|
40
|
-
|
41
|
-
# https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
|
42
|
-
def prepend_handler(signal)
|
43
|
-
previous = Signal.trap(signal) do
|
44
|
-
previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
|
45
|
-
yield
|
46
|
-
previous.call
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
# Trap signals using the self-pipe trick.
|
51
|
-
def setup_signals
|
52
|
-
at_exit { @runner&.stop }
|
53
|
-
SIGNALS.each do |signal|
|
54
|
-
prepend_handler(signal) do
|
55
|
-
unblock(signal)
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
# Save the signal to the queue and continue on.
|
61
|
-
# @param signal [Symbol]
|
62
|
-
def unblock(signal)
|
63
|
-
writer.write_nonblock('.')
|
64
|
-
signal_queue << signal
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
data/spec/utils/executor_spec.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
RSpec.describe Deimos::Utils::Executor do
|
6
|
-
|
7
|
-
let(:executor) { described_class.new(runners) }
|
8
|
-
let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
|
9
|
-
|
10
|
-
it 'starts and stops configured runners' do
|
11
|
-
runners.each do |r|
|
12
|
-
expect(r.started).to be_falsey
|
13
|
-
expect(r.stopped).to be_falsey
|
14
|
-
end
|
15
|
-
executor.start
|
16
|
-
wait_for do
|
17
|
-
runners.each do |r|
|
18
|
-
expect(r.started).to be_truthy
|
19
|
-
expect(r.stopped).to be_falsey
|
20
|
-
end
|
21
|
-
executor.stop
|
22
|
-
runners.each do |r|
|
23
|
-
expect(r.started).to be_truthy
|
24
|
-
expect(r.stopped).to be_truthy
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'sleeps X seconds' do
|
30
|
-
executor = described_class.new(runners, sleep_seconds: 5)
|
31
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
32
|
-
expect(executor).to receive(:sleep).with(5).twice
|
33
|
-
runners.each { |r| r.should_error = true }
|
34
|
-
executor.start
|
35
|
-
wait_for do
|
36
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
37
|
-
executor.stop
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'reconnects crashed runners' do
|
42
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
43
|
-
runners.each { |r| r.should_error = true }
|
44
|
-
executor.start
|
45
|
-
wait_for do
|
46
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
|
47
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
|
48
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
49
|
-
executor.stop
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
RSpec.describe Deimos::Utils::SignalHandler do
|
4
|
-
describe '#run!' do
|
5
|
-
|
6
|
-
it 'starts and stops the runner' do
|
7
|
-
runner = TestRunners::TestRunner.new
|
8
|
-
expect(runner).to receive(:start)
|
9
|
-
expect(runner).to receive(:stop)
|
10
|
-
|
11
|
-
signal_handler = described_class.new(runner)
|
12
|
-
signal_handler.send(:unblock, described_class::SIGNALS.first)
|
13
|
-
signal_handler.run!
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|