deimos-ruby 1.6.1 → 1.8.0.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +9 -0
  3. data/.rubocop.yml +15 -13
  4. data/.ruby-version +1 -1
  5. data/CHANGELOG.md +30 -0
  6. data/Gemfile.lock +87 -80
  7. data/README.md +139 -15
  8. data/Rakefile +1 -1
  9. data/deimos-ruby.gemspec +3 -2
  10. data/docs/ARCHITECTURE.md +144 -0
  11. data/docs/CONFIGURATION.md +27 -0
  12. data/lib/deimos.rb +7 -6
  13. data/lib/deimos/active_record_consume/batch_consumption.rb +159 -0
  14. data/lib/deimos/active_record_consume/batch_slicer.rb +27 -0
  15. data/lib/deimos/active_record_consume/message_consumption.rb +58 -0
  16. data/lib/deimos/active_record_consume/schema_model_converter.rb +52 -0
  17. data/lib/deimos/active_record_consumer.rb +33 -75
  18. data/lib/deimos/active_record_producer.rb +23 -0
  19. data/lib/deimos/batch_consumer.rb +2 -140
  20. data/lib/deimos/config/configuration.rb +28 -10
  21. data/lib/deimos/consume/batch_consumption.rb +148 -0
  22. data/lib/deimos/consume/message_consumption.rb +93 -0
  23. data/lib/deimos/consumer.rb +79 -69
  24. data/lib/deimos/kafka_message.rb +1 -1
  25. data/lib/deimos/kafka_source.rb +29 -23
  26. data/lib/deimos/kafka_topic_info.rb +1 -1
  27. data/lib/deimos/message.rb +6 -1
  28. data/lib/deimos/metrics/provider.rb +0 -2
  29. data/lib/deimos/poll_info.rb +9 -0
  30. data/lib/deimos/tracing/provider.rb +0 -2
  31. data/lib/deimos/utils/db_poller.rb +149 -0
  32. data/lib/deimos/utils/db_producer.rb +8 -3
  33. data/lib/deimos/utils/deadlock_retry.rb +68 -0
  34. data/lib/deimos/utils/lag_reporter.rb +19 -26
  35. data/lib/deimos/version.rb +1 -1
  36. data/lib/generators/deimos/db_poller/templates/migration +11 -0
  37. data/lib/generators/deimos/db_poller/templates/rails3_migration +16 -0
  38. data/lib/generators/deimos/db_poller_generator.rb +48 -0
  39. data/lib/tasks/deimos.rake +7 -0
  40. data/spec/active_record_batch_consumer_spec.rb +481 -0
  41. data/spec/active_record_consume/batch_slicer_spec.rb +42 -0
  42. data/spec/active_record_consume/schema_model_converter_spec.rb +105 -0
  43. data/spec/active_record_consumer_spec.rb +22 -11
  44. data/spec/active_record_producer_spec.rb +66 -88
  45. data/spec/batch_consumer_spec.rb +23 -7
  46. data/spec/config/configuration_spec.rb +4 -0
  47. data/spec/consumer_spec.rb +8 -8
  48. data/spec/deimos_spec.rb +57 -49
  49. data/spec/handlers/my_batch_consumer.rb +6 -1
  50. data/spec/handlers/my_consumer.rb +6 -1
  51. data/spec/kafka_source_spec.rb +53 -0
  52. data/spec/message_spec.rb +19 -0
  53. data/spec/producer_spec.rb +3 -3
  54. data/spec/rake_spec.rb +1 -1
  55. data/spec/schemas/com/my-namespace/MySchemaCompound-key.avsc +18 -0
  56. data/spec/schemas/com/my-namespace/Wibble.avsc +43 -0
  57. data/spec/spec_helper.rb +61 -6
  58. data/spec/utils/db_poller_spec.rb +320 -0
  59. data/spec/utils/deadlock_retry_spec.rb +74 -0
  60. data/spec/utils/lag_reporter_spec.rb +29 -22
  61. metadata +61 -20
  62. data/lib/deimos/base_consumer.rb +0 -104
  63. data/lib/deimos/utils/executor.rb +0 -124
  64. data/lib/deimos/utils/platform_schema_validation.rb +0 -0
  65. data/lib/deimos/utils/signal_handler.rb +0 -68
  66. data/spec/utils/executor_spec.rb +0 -53
  67. data/spec/utils/signal_handler_spec.rb +0 -16
@@ -1,68 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Deimos
4
- module Utils
5
- # Mostly copied free-form from Phobos::Cli::Runner. We should add a PR to
6
- # basically replace that implementation with this one to make it more generic.
7
- class SignalHandler
8
- SIGNALS = %i(INT TERM QUIT).freeze
9
-
10
- # Takes any object that responds to the `start` and `stop` methods.
11
- # @param runner[#start, #stop]
12
- def initialize(runner)
13
- @signal_queue = []
14
- @reader, @writer = IO.pipe
15
- @runner = runner
16
- end
17
-
18
- # Run the runner.
19
- def run!
20
- setup_signals
21
- @runner.start
22
-
23
- loop do
24
- case signal_queue.pop
25
- when *SIGNALS
26
- @runner.stop
27
- break
28
- else
29
- ready = IO.select([reader, writer])
30
-
31
- # drain the self-pipe so it won't be returned again next time
32
- reader.read_nonblock(1) if ready[0].include?(reader)
33
- end
34
- end
35
- end
36
-
37
- private
38
-
39
- attr_reader :reader, :writer, :signal_queue, :executor
40
-
41
- # https://stackoverflow.com/questions/29568298/run-code-when-signal-is-sent-but-do-not-trap-the-signal-in-ruby
42
- def prepend_handler(signal)
43
- previous = Signal.trap(signal) do
44
- previous = -> { raise SignalException, signal } unless previous.respond_to?(:call)
45
- yield
46
- previous.call
47
- end
48
- end
49
-
50
- # Trap signals using the self-pipe trick.
51
- def setup_signals
52
- at_exit { @runner&.stop }
53
- SIGNALS.each do |signal|
54
- prepend_handler(signal) do
55
- unblock(signal)
56
- end
57
- end
58
- end
59
-
60
- # Save the signal to the queue and continue on.
61
- # @param signal [Symbol]
62
- def unblock(signal)
63
- writer.write_nonblock('.')
64
- signal_queue << signal
65
- end
66
- end
67
- end
68
- end
@@ -1,53 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- RSpec.describe Deimos::Utils::Executor do
6
-
7
- let(:executor) { described_class.new(runners) }
8
- let(:runners) { (1..2).map { |i| TestRunners::TestRunner.new(i) } }
9
-
10
- it 'starts and stops configured runners' do
11
- runners.each do |r|
12
- expect(r.started).to be_falsey
13
- expect(r.stopped).to be_falsey
14
- end
15
- executor.start
16
- wait_for do
17
- runners.each do |r|
18
- expect(r.started).to be_truthy
19
- expect(r.stopped).to be_falsey
20
- end
21
- executor.stop
22
- runners.each do |r|
23
- expect(r.started).to be_truthy
24
- expect(r.stopped).to be_truthy
25
- end
26
- end
27
- end
28
-
29
- it 'sleeps X seconds' do
30
- executor = described_class.new(runners, sleep_seconds: 5)
31
- allow(executor).to receive(:handle_crashed_runner).and_call_original
32
- expect(executor).to receive(:sleep).with(5).twice
33
- runners.each { |r| r.should_error = true }
34
- executor.start
35
- wait_for do
36
- runners.each { |r| expect(r.started).to be_truthy }
37
- executor.stop
38
- end
39
- end
40
-
41
- it 'reconnects crashed runners' do
42
- allow(executor).to receive(:handle_crashed_runner).and_call_original
43
- runners.each { |r| r.should_error = true }
44
- executor.start
45
- wait_for do
46
- expect(executor).to have_received(:handle_crashed_runner).with(runners[0], anything, 0).once
47
- expect(executor).to have_received(:handle_crashed_runner).with(runners[1], anything, 0).once
48
- runners.each { |r| expect(r.started).to be_truthy }
49
- executor.stop
50
- end
51
- end
52
-
53
- end
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- RSpec.describe Deimos::Utils::SignalHandler do
4
- describe '#run!' do
5
-
6
- it 'starts and stops the runner' do
7
- runner = TestRunners::TestRunner.new
8
- expect(runner).to receive(:start)
9
- expect(runner).to receive(:stop)
10
-
11
- signal_handler = described_class.new(runner)
12
- signal_handler.send(:unblock, described_class::SIGNALS.first)
13
- signal_handler.run!
14
- end
15
- end
16
- end