deimos-ruby 1.8.6 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/Gemfile.lock +106 -103
- data/deimos-ruby.gemspec +4 -3
- data/docs/ARCHITECTURE.md +1 -5
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos.rb +2 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +5 -0
- data/lib/deimos/config/configuration.rb +13 -13
- data/lib/deimos/kafka_source.rb +8 -12
- data/lib/deimos/test_helpers.rb +1 -1
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/config/configuration_spec.rb +17 -17
- data/spec/kafka_source_spec.rb +69 -0
- metadata +30 -14
- data/lib/deimos/config/configurable.rb +0 -278
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +0 -85
- data/spec/config/configurable_spec.rb +0 -136
@@ -1,85 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Kafka
|
4
|
-
class Heartbeat
|
5
|
-
def initialize(group:, interval:, instrumenter:)
|
6
|
-
@group = group
|
7
|
-
@interval = interval
|
8
|
-
@last_heartbeat = Time.now
|
9
|
-
@instrumenter = instrumenter
|
10
|
-
end
|
11
|
-
|
12
|
-
def trigger!
|
13
|
-
@instrumenter.instrument('heartbeat.consumer',
|
14
|
-
group_id: @group.group_id,
|
15
|
-
topic_partitions: @group.assigned_partitions) do
|
16
|
-
@group.heartbeat
|
17
|
-
@last_heartbeat = Time.now
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
class Client
|
23
|
-
def consumer(
|
24
|
-
group_id:,
|
25
|
-
session_timeout: 30,
|
26
|
-
offset_commit_interval: 10,
|
27
|
-
offset_commit_threshold: 0,
|
28
|
-
heartbeat_interval: 10,
|
29
|
-
offset_retention_time: nil,
|
30
|
-
fetcher_max_queue_size: 100
|
31
|
-
)
|
32
|
-
cluster = initialize_cluster
|
33
|
-
|
34
|
-
instrumenter = DecoratingInstrumenter.new(@instrumenter,
|
35
|
-
group_id: group_id)
|
36
|
-
|
37
|
-
# The Kafka protocol expects the retention time to be in ms.
|
38
|
-
retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
|
39
|
-
|
40
|
-
group = ConsumerGroup.new(
|
41
|
-
cluster: cluster,
|
42
|
-
logger: @logger,
|
43
|
-
group_id: group_id,
|
44
|
-
session_timeout: session_timeout,
|
45
|
-
retention_time: retention_time,
|
46
|
-
instrumenter: instrumenter
|
47
|
-
)
|
48
|
-
|
49
|
-
fetcher = Fetcher.new(
|
50
|
-
cluster: initialize_cluster,
|
51
|
-
group: group,
|
52
|
-
logger: @logger,
|
53
|
-
instrumenter: instrumenter,
|
54
|
-
max_queue_size: fetcher_max_queue_size
|
55
|
-
)
|
56
|
-
|
57
|
-
offset_manager = OffsetManager.new(
|
58
|
-
cluster: cluster,
|
59
|
-
group: group,
|
60
|
-
fetcher: fetcher,
|
61
|
-
logger: @logger,
|
62
|
-
commit_interval: offset_commit_interval,
|
63
|
-
commit_threshold: offset_commit_threshold,
|
64
|
-
offset_retention_time: offset_retention_time
|
65
|
-
)
|
66
|
-
|
67
|
-
heartbeat = Heartbeat.new(
|
68
|
-
group: group,
|
69
|
-
interval: heartbeat_interval,
|
70
|
-
instrumenter: instrumenter
|
71
|
-
)
|
72
|
-
|
73
|
-
Consumer.new(
|
74
|
-
cluster: cluster,
|
75
|
-
logger: @logger,
|
76
|
-
instrumenter: instrumenter,
|
77
|
-
group: group,
|
78
|
-
offset_manager: offset_manager,
|
79
|
-
fetcher: fetcher,
|
80
|
-
session_timeout: session_timeout,
|
81
|
-
heartbeat: heartbeat
|
82
|
-
)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# :nodoc:
|
4
|
-
class MyConfig
|
5
|
-
include Deimos::Configurable
|
6
|
-
|
7
|
-
define_settings do
|
8
|
-
setting :set1
|
9
|
-
setting :set2, 'hi mom'
|
10
|
-
setting :group do
|
11
|
-
setting :set3, default_proc: proc { false }
|
12
|
-
setting :set5, (proc { 5 })
|
13
|
-
end
|
14
|
-
|
15
|
-
setting_object :listy do
|
16
|
-
setting :list1, 10
|
17
|
-
setting :list2, 5
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe Deimos::Configurable do
|
23
|
-
it 'should configure correctly with default values' do
|
24
|
-
expect(MyConfig.config.set1).to be_nil
|
25
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
26
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
27
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
28
|
-
expect { MyConfig.config.blah }.to raise_error(NameError)
|
29
|
-
expect { MyConfig.config.group.set4 }.to raise_error(NameError)
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'should not call the proc until it has to' do
|
33
|
-
num_calls = 0
|
34
|
-
value_proc = proc do
|
35
|
-
num_calls += 1
|
36
|
-
num_calls
|
37
|
-
end
|
38
|
-
MyConfig.define_settings do
|
39
|
-
setting :set_with_proc, default_proc: value_proc
|
40
|
-
end
|
41
|
-
expect(num_calls).to eq(0)
|
42
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
43
|
-
# calling twice should not call the proc again
|
44
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
45
|
-
expect(num_calls).to eq(1)
|
46
|
-
end
|
47
|
-
|
48
|
-
it "should raise error when setting configs that don't exist" do
|
49
|
-
expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
|
50
|
-
end
|
51
|
-
|
52
|
-
it 'should add values' do
|
53
|
-
MyConfig.configure do |config|
|
54
|
-
config.set1 = 5 # config.x syntax
|
55
|
-
set2 nil # method_missing syntax
|
56
|
-
config.group.set3 = true
|
57
|
-
end
|
58
|
-
|
59
|
-
# second configure should not blow anything away
|
60
|
-
MyConfig.configure do
|
61
|
-
listy do
|
62
|
-
list1 0
|
63
|
-
list2 1
|
64
|
-
end
|
65
|
-
listy do
|
66
|
-
list1 100
|
67
|
-
list2 200
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
expect(MyConfig.config.set1).to eq(5)
|
72
|
-
expect(MyConfig.config.set2).to be_nil
|
73
|
-
expect(MyConfig.config.group.set3).to eq(true)
|
74
|
-
expect(MyConfig.config.listy_objects.map(&:to_h)).
|
75
|
-
to eq([
|
76
|
-
{ list1: 0, list2: 1 },
|
77
|
-
{ list1: 100, list2: 200 }
|
78
|
-
])
|
79
|
-
|
80
|
-
# test reset!
|
81
|
-
MyConfig.config.reset!
|
82
|
-
expect(MyConfig.config.set1).to be_nil
|
83
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
84
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
85
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
86
|
-
end
|
87
|
-
|
88
|
-
it 'should add with block syntax' do
|
89
|
-
MyConfig.configure do
|
90
|
-
group do
|
91
|
-
set5(proc { 10 })
|
92
|
-
end
|
93
|
-
end
|
94
|
-
expect(MyConfig.config.group.set5.call).to eq(10)
|
95
|
-
end
|
96
|
-
|
97
|
-
it 'should add or redefine settings' do
|
98
|
-
MyConfig.define_settings do
|
99
|
-
setting :group do
|
100
|
-
setting :set6, 15
|
101
|
-
setting :set5, (proc { 15 })
|
102
|
-
end
|
103
|
-
setting_object :notey do
|
104
|
-
setting :note_title, 'some-title'
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
expect(MyConfig.config.group.set6).to eq(15)
|
109
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
110
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
111
|
-
expect(MyConfig.config.notey_objects).to be_empty
|
112
|
-
|
113
|
-
MyConfig.configure do
|
114
|
-
notey do
|
115
|
-
note_title 'hi mom'
|
116
|
-
end
|
117
|
-
listy do
|
118
|
-
list1 0
|
119
|
-
end
|
120
|
-
end
|
121
|
-
expect(MyConfig.config.notey_objects.size).to eq(1)
|
122
|
-
expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
|
123
|
-
expect(MyConfig.config.listy_objects.size).to eq(1)
|
124
|
-
expect(MyConfig.config.listy_objects.first.list1).to eq(0)
|
125
|
-
|
126
|
-
# This should not remove any keys
|
127
|
-
MyConfig.define_settings do
|
128
|
-
setting :group do
|
129
|
-
setting :set6, 20
|
130
|
-
end
|
131
|
-
end
|
132
|
-
expect(MyConfig.config.group.set6).to eq(20)
|
133
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
134
|
-
end
|
135
|
-
|
136
|
-
end
|