deimos-ruby 1.7.0.pre.beta1 → 1.8.1.pre.beta3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +8 -4
- data/CHANGELOG.md +50 -0
- data/Gemfile.lock +109 -75
- data/README.md +147 -16
- data/deimos-ruby.gemspec +4 -2
- data/docs/ARCHITECTURE.md +144 -0
- data/docs/CONFIGURATION.md +4 -0
- data/lib/deimos.rb +8 -7
- data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
- data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
- data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
- data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
- data/lib/deimos/active_record_consumer.rb +33 -75
- data/lib/deimos/batch_consumer.rb +2 -142
- data/lib/deimos/config/configuration.rb +8 -10
- data/lib/deimos/consume/batch_consumption.rb +150 -0
- data/lib/deimos/consume/message_consumption.rb +94 -0
- data/lib/deimos/consumer.rb +79 -72
- data/lib/deimos/instrumentation.rb +10 -5
- data/lib/deimos/kafka_message.rb +1 -1
- data/lib/deimos/kafka_topic_info.rb +21 -2
- data/lib/deimos/message.rb +6 -1
- data/lib/deimos/schema_backends/avro_base.rb +33 -1
- data/lib/deimos/schema_backends/avro_schema_coercer.rb +30 -11
- data/lib/deimos/schema_backends/base.rb +21 -2
- data/lib/deimos/utils/db_poller.rb +6 -6
- data/lib/deimos/utils/db_producer.rb +57 -15
- data/lib/deimos/utils/deadlock_retry.rb +68 -0
- data/lib/deimos/utils/lag_reporter.rb +19 -26
- data/lib/deimos/utils/schema_controller_mixin.rb +111 -0
- data/lib/deimos/version.rb +1 -1
- data/lib/generators/deimos/active_record/templates/migration.rb.tt +28 -0
- data/lib/generators/deimos/active_record/templates/model.rb.tt +5 -0
- data/lib/generators/deimos/active_record_generator.rb +79 -0
- data/lib/generators/deimos/db_backend/templates/migration +1 -0
- data/lib/generators/deimos/db_backend/templates/rails3_migration +1 -0
- data/spec/active_record_batch_consumer_spec.rb +481 -0
- data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
- data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
- data/spec/active_record_consumer_spec.rb +3 -11
- data/spec/batch_consumer_spec.rb +24 -7
- data/spec/config/configuration_spec.rb +4 -0
- data/spec/consumer_spec.rb +6 -6
- data/spec/deimos_spec.rb +57 -49
- data/spec/generators/active_record_generator_spec.rb +56 -0
- data/spec/handlers/my_batch_consumer.rb +6 -1
- data/spec/handlers/my_consumer.rb +6 -1
- data/spec/kafka_listener_spec.rb +54 -0
- data/spec/kafka_topic_info_spec.rb +39 -16
- data/spec/message_spec.rb +19 -0
- data/spec/producer_spec.rb +34 -0
- data/spec/schemas/com/my-namespace/Generated.avsc +71 -0
- data/spec/schemas/com/my-namespace/MyNestedSchema.avsc +55 -0
- data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
- data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
- data/spec/schemas/com/my-namespace/request/Index.avsc +11 -0
- data/spec/schemas/com/my-namespace/request/UpdateRequest.avsc +11 -0
- data/spec/schemas/com/my-namespace/response/Index.avsc +11 -0
- data/spec/schemas/com/my-namespace/response/UpdateResponse.avsc +11 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/utils/db_poller_spec.rb +2 -2
- data/spec/utils/db_producer_spec.rb +84 -10
- data/spec/utils/deadlock_retry_spec.rb +74 -0
- data/spec/utils/lag_reporter_spec.rb +29 -22
- data/spec/utils/schema_controller_mixin_spec.rb +68 -0
- metadata +87 -30
- data/lib/deimos/base_consumer.rb +0 -100
- data/lib/deimos/utils/executor.rb +0 -124
- data/lib/deimos/utils/platform_schema_validation.rb +0 -0
- data/lib/deimos/utils/signal_handler.rb +0 -68
- data/spec/utils/executor_spec.rb +0 -53
- data/spec/utils/signal_handler_spec.rb +0 -16
File without changes
|
@@ -1,68 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Deimos
|
4
|
-
module Utils
|
5
|
-
# Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
|
6
|
-
# basically replace that implementation with this one to make it more generic.
|
7
|
-
class SignalHandler
|
8
|
-
SIGNALS = %i(INT TERM QUIT).freeze
|
9
|
-
|
10
|
-
# Takes any object that responds to the `start` and `stop` methods.
|
11
|
-
# @param runner[#start, #stop]
|
12
|
-
def initialize(runner)
|
13
|
-
@signal_queue = []
|
14
|
-
@reader, @writer = IO.pipe
|
15
|
-
@runner = runner
|
16
|
-
end
|
17
|
-
|
18
|
-
# Run the runner.
|
19
|
-
def run!
|
20
|
-
setup_signals
|
21
|
-
@runner.start
|
22
|
-
|
23
|
-
loop do
|
24
|
-
case signal_queue.pop
|
25
|
-
when *SIGNALS
|
26
|
-
@runner.stop
|
27
|
-
break
|
28
|
-
else
|
29
|
-
ready = IO.select([reader, writer])
|
30
|
-
|
31
|
-
# drain the self-pipe so it won't be returned again next time
|
32
|
-
reader.read_nonblock(1) if ready[0].include?(reader)
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
private
|
38
|
-
|
39
|
-
attr_reader :reader, :writer, :signal_queue, :executor
|
40
|
-
|
41
|
-
# https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
|
42
|
-
def prepend_handler(signal)
|
43
|
-
previous = Signal.trap(signal) do
|
44
|
-
previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
|
45
|
-
yield
|
46
|
-
previous.call
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
# Trap signals using the self-pipe trick.
|
51
|
-
def setup_signals
|
52
|
-
at_exit { @runner&.stop }
|
53
|
-
SIGNALS.each do |signal|
|
54
|
-
prepend_handler(signal) do
|
55
|
-
unblock(signal)
|
56
|
-
end
|
57
|
-
end
|
58
|
-
end
|
59
|
-
|
60
|
-
# Save the signal to the queue and continue on.
|
61
|
-
# @param signal [Symbol]
|
62
|
-
def unblock(signal)
|
63
|
-
writer.write_nonblock('.')
|
64
|
-
signal_queue << signal
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
data/spec/utils/executor_spec.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'spec_helper'
|
4
|
-
|
5
|
-
RSpec.describe Deimos::Utils::Executor do
|
6
|
-
|
7
|
-
let(:executor) { described_class.new(runners) }
|
8
|
-
let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
|
9
|
-
|
10
|
-
it 'starts and stops configured runners' do
|
11
|
-
runners.each do |r|
|
12
|
-
expect(r.started).to be_falsey
|
13
|
-
expect(r.stopped).to be_falsey
|
14
|
-
end
|
15
|
-
executor.start
|
16
|
-
wait_for do
|
17
|
-
runners.each do |r|
|
18
|
-
expect(r.started).to be_truthy
|
19
|
-
expect(r.stopped).to be_falsey
|
20
|
-
end
|
21
|
-
executor.stop
|
22
|
-
runners.each do |r|
|
23
|
-
expect(r.started).to be_truthy
|
24
|
-
expect(r.stopped).to be_truthy
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
it 'sleeps X seconds' do
|
30
|
-
executor = described_class.new(runners, sleep_seconds: 5)
|
31
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
32
|
-
expect(executor).to receive(:sleep).with(5).twice
|
33
|
-
runners.each { |r| r.should_error = true }
|
34
|
-
executor.start
|
35
|
-
wait_for do
|
36
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
37
|
-
executor.stop
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
it 'reconnects crashed runners' do
|
42
|
-
allow(executor).to receive(:handle_crashed_runner).and_call_original
|
43
|
-
runners.each { |r| r.should_error = true }
|
44
|
-
executor.start
|
45
|
-
wait_for do
|
46
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
|
47
|
-
expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
|
48
|
-
runners.each { |r| expect(r.started).to be_truthy }
|
49
|
-
executor.stop
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
RSpec.describe Deimos::Utils::SignalHandler do
|
4
|
-
describe '#run!' do
|
5
|
-
|
6
|
-
it 'starts and stops the runner' do
|
7
|
-
runner = TestRunners::TestRunner.new
|
8
|
-
expect(runner).to receive(:start)
|
9
|
-
expect(runner).to receive(:stop)
|
10
|
-
|
11
|
-
signal_handler = described_class.new(runner)
|
12
|
-
signal_handler.send(:unblock, described_class::SIGNALS.first)
|
13
|
-
signal_handler.run!
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|