deimos-ruby 1.6.1 → 1.8.0.pre.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +15 -13
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +30 -0
  6. data/Gemfile.lock +87 -80
  7. data/README.md +139 -15
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +3 -2
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +7 -6
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +148 -0
  22. data/lib/deimos/consume/message_consumption.rb +93 -0
  23. data/lib/deimos/consumer.rb +79 -69
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_source.rb +29 -23
  26. data/lib/deimos/kafka_topic_info.rb +1 -1
  27. data/lib/deimos/message.rb +6 -1
  28. data/lib/deimos/metrics/provider.rb +0 -2
  29. data/lib/deimos/poll_info.rb +9 -0
  30. data/lib/deimos/tracing/provider.rb +0 -2
  31. data/lib/deimos/utils/db_poller.rb +149 -0
  32. data/lib/deimos/utils/db_producer.rb +8 -3
  33. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  34. data/lib/deimos/utils/lag_reporter.rb +19 -26
  35. data/lib/deimos/version.rb +1 -1
  36. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  37. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  38. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  39. data/lib/tasks/deimos.rake +7 -0
  40. data/spec/active_record_batch_consumer_spec.rb +481 -0
  41. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  42. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  43. data/spec/active_record_consumer_spec.rb +22 -11
  44. data/spec/active_record_producer_spec.rb +66 -88
  45. data/spec/batch_consumer_spec.rb +23 -7
  46. data/spec/config/configuration_spec.rb +4 -0
  47. data/spec/consumer_spec.rb +8 -8
  48. data/spec/deimos_spec.rb +57 -49
  49. data/spec/handlers/my_batch_consumer.rb +6 -1
  50. data/spec/handlers/my_consumer.rb +6 -1
  51. data/spec/kafka_source_spec.rb +53 -0
  52. data/spec/message_spec.rb +19 -0
  53. data/spec/producer_spec.rb +3 -3
  54. data/spec/rake_spec.rb +1 -1
  55. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  56. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  57. data/spec/spec_helper.rb +61 -6
  58. data/spec/utils/db_poller_spec.rb +320 -0
  59. data/spec/utils/deadlock_retry_spec.rb +74 -0
  60. data/spec/utils/lag_reporter_spec.rb +29 -22
  61. metadata +61 -20
  62. data/lib/deimos/base_consumer.rb +0 -104
  63. data/lib/deimos/utils/executor.rb +0 -124
  64. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  65. data/lib/deimos/utils/signal_handler.rb +0 -68
  66. data/spec/utils/executor_spec.rb +0 -53
  67. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -1,68 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Deimos
4
- module Utils
5
- # Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
6
- # basically replace that implementation with this one to make it more generic.
7
- class SignalHandler
8
- SIGNALS = %i(INT TERM QUIT).freeze
9
-
10
- # Takes any object that responds to the `start` and `stop` methods.
11
- # @param runner[#start, #stop]
12
- def initialize(runner)
13
- @signal_queue = []
14
- @reader, @writer = IO.pipe
15
- @runner = runner
16
- end
17
-
18
- # Run the runner.
19
- def run!
20
- setup_signals
21
- @runner.start
22
-
23
- loop do
24
- case signal_queue.pop
25
- when *SIGNALS
26
- @runner.stop
27
- break
28
- else
29
- ready = IO.select([reader, writer])
30
-
31
- # drain the self-pipe so it won't be returned again next time
32
- reader.read_nonblock(1) if ready[0].include?(reader)
33
- end
34
- end
35
- end
36
-
37
- private
38
-
39
- attr_reader :reader, :writer, :signal_queue, :executor
40
-
41
- # https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
42
- def prepend_handler(signal)
43
- previous = Signal.trap(signal) do
44
- previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
45
- yield
46
- previous.call
47
- end
48
- end
49
-
50
- # Trap signals using the self-pipe trick.
51
- def setup_signals
52
- at_exit { @runner&.stop }
53
- SIGNALS.each do |signal|
54
- prepend_handler(signal) do
55
- unblock(signal)
56
- end
57
- end
58
- end
59
-
60
- # Save the signal to the queue and continue on.
61
- # @param signal [Symbol]
62
- def unblock(signal)
63
- writer.write_nonblock('.')
64
- signal_queue << signal
65
- end
66
- end
67
- end
68
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- RSpec.describe Deimos::Utils::Executor do
6
-
7
- let(:executor) { described_class.new(runners) }
8
- let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
9
-
10
- it 'starts and stops configured runners' do
11
- runners.each do |r|
12
- expect(r.started).to be_falsey
13
- expect(r.stopped).to be_falsey
14
- end
15
- executor.start
16
- wait_for do
17
- runners.each do |r|
18
- expect(r.started).to be_truthy
19
- expect(r.stopped).to be_falsey
20
- end
21
- executor.stop
22
- runners.each do |r|
23
- expect(r.started).to be_truthy
24
- expect(r.stopped).to be_truthy
25
- end
26
- end
27
- end
28
-
29
- it 'sleeps X seconds' do
30
- executor = described_class.new(runners, sleep_seconds: 5)
31
- allow(executor).to receive(:handle_crashed_runner).and_call_original
32
- expect(executor).to receive(:sleep).with(5).twice
33
- runners.each { |r| r.should_error = true }
34
- executor.start
35
- wait_for do
36
- runners.each { |r| expect(r.started).to be_truthy }
37
- executor.stop
38
- end
39
- end
40
-
41
- it 'reconnects crashed runners' do
42
- allow(executor).to receive(:handle_crashed_runner).and_call_original
43
- runners.each { |r| r.should_error = true }
44
- executor.start
45
- wait_for do
46
- expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
47
- expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
48
- runners.each { |r| expect(r.started).to be_truthy }
49
- executor.stop
50
- end
51
- end
52
-
53
- end
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- RSpec.describe Deimos::Utils::SignalHandler do
4
- describe '#run!' do
5
-
6
- it 'starts and stops the runner' do
7
- runner = TestRunners::TestRunner.new
8
- expect(runner).to receive(:start)
9
- expect(runner).to receive(:stop)
10
-
11
- signal_handler = described_class.new(runner)
12
- signal_handler.send(:unblock, described_class::SIGNALS.first)
13
- signal_handler.run!
14
- end
15
- end
16
- end