deimos-ruby 1.8.7 → 1.10.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/Gemfile.lock +112 -107
- data/deimos-ruby.gemspec +3 -2
- data/docs/ARCHITECTURE.md +1 -5
- data/docs/CONFIGURATION.md +1 -0
- data/lib/deimos.rb +2 -1
- data/lib/deimos/active_record_consume/message_consumption.rb +5 -0
- data/lib/deimos/config/configuration.rb +13 -13
- data/lib/deimos/kafka_source.rb +8 -12
- data/lib/deimos/schema_backends/base.rb +2 -1
- data/lib/deimos/test_helpers.rb +1 -1
- data/lib/deimos/utils/db_poller.rb +1 -1
- data/lib/deimos/version.rb +1 -1
- data/spec/config/configuration_spec.rb +17 -17
- data/spec/kafka_source_spec.rb +69 -0
- data/spec/schema_backends/base_spec.rb +4 -0
- metadata +22 -12
- data/lib/deimos/config/configurable.rb +0 -278
- data/lib/deimos/monkey_patches/ruby_kafka_heartbeat.rb +0 -85
- data/spec/config/configurable_spec.rb +0 -136
@@ -1,85 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Kafka
|
4
|
-
class Heartbeat
|
5
|
-
def initialize(group:, interval:, instrumenter:)
|
6
|
-
@group = group
|
7
|
-
@interval = interval
|
8
|
-
@last_heartbeat = Time.now
|
9
|
-
@instrumenter = instrumenter
|
10
|
-
end
|
11
|
-
|
12
|
-
def trigger!
|
13
|
-
@instrumenter.instrument('heartbeat.consumer',
|
14
|
-
group_id: @group.group_id,
|
15
|
-
topic_partitions: @group.assigned_partitions) do
|
16
|
-
@group.heartbeat
|
17
|
-
@last_heartbeat = Time.now
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
class Client
|
23
|
-
def consumer(
|
24
|
-
group_id:,
|
25
|
-
session_timeout: 30,
|
26
|
-
offset_commit_interval: 10,
|
27
|
-
offset_commit_threshold: 0,
|
28
|
-
heartbeat_interval: 10,
|
29
|
-
offset_retention_time: nil,
|
30
|
-
fetcher_max_queue_size: 100
|
31
|
-
)
|
32
|
-
cluster = initialize_cluster
|
33
|
-
|
34
|
-
instrumenter = DecoratingInstrumenter.new(@instrumenter,
|
35
|
-
group_id: group_id)
|
36
|
-
|
37
|
-
# The Kafka protocol expects the retention time to be in ms.
|
38
|
-
retention_time = (offset_retention_time && offset_retention_time * 1_000) || -1
|
39
|
-
|
40
|
-
group = ConsumerGroup.new(
|
41
|
-
cluster: cluster,
|
42
|
-
logger: @logger,
|
43
|
-
group_id: group_id,
|
44
|
-
session_timeout: session_timeout,
|
45
|
-
retention_time: retention_time,
|
46
|
-
instrumenter: instrumenter
|
47
|
-
)
|
48
|
-
|
49
|
-
fetcher = Fetcher.new(
|
50
|
-
cluster: initialize_cluster,
|
51
|
-
group: group,
|
52
|
-
logger: @logger,
|
53
|
-
instrumenter: instrumenter,
|
54
|
-
max_queue_size: fetcher_max_queue_size
|
55
|
-
)
|
56
|
-
|
57
|
-
offset_manager = OffsetManager.new(
|
58
|
-
cluster: cluster,
|
59
|
-
group: group,
|
60
|
-
fetcher: fetcher,
|
61
|
-
logger: @logger,
|
62
|
-
commit_interval: offset_commit_interval,
|
63
|
-
commit_threshold: offset_commit_threshold,
|
64
|
-
offset_retention_time: offset_retention_time
|
65
|
-
)
|
66
|
-
|
67
|
-
heartbeat = Heartbeat.new(
|
68
|
-
group: group,
|
69
|
-
interval: heartbeat_interval,
|
70
|
-
instrumenter: instrumenter
|
71
|
-
)
|
72
|
-
|
73
|
-
Consumer.new(
|
74
|
-
cluster: cluster,
|
75
|
-
logger: @logger,
|
76
|
-
instrumenter: instrumenter,
|
77
|
-
group: group,
|
78
|
-
offset_manager: offset_manager,
|
79
|
-
fetcher: fetcher,
|
80
|
-
session_timeout: session_timeout,
|
81
|
-
heartbeat: heartbeat
|
82
|
-
)
|
83
|
-
end
|
84
|
-
end
|
85
|
-
end
|
@@ -1,136 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# :nodoc:
|
4
|
-
class MyConfig
|
5
|
-
include Deimos::Configurable
|
6
|
-
|
7
|
-
define_settings do
|
8
|
-
setting :set1
|
9
|
-
setting :set2, 'hi mom'
|
10
|
-
setting :group do
|
11
|
-
setting :set3, default_proc: proc { false }
|
12
|
-
setting :set5, (proc { 5 })
|
13
|
-
end
|
14
|
-
|
15
|
-
setting_object :listy do
|
16
|
-
setting :list1, 10
|
17
|
-
setting :list2, 5
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
describe Deimos::Configurable do
|
23
|
-
it 'should configure correctly with default values' do
|
24
|
-
expect(MyConfig.config.set1).to be_nil
|
25
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
26
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
27
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
28
|
-
expect { MyConfig.config.blah }.to raise_error(NameError)
|
29
|
-
expect { MyConfig.config.group.set4 }.to raise_error(NameError)
|
30
|
-
end
|
31
|
-
|
32
|
-
it 'should not call the proc until it has to' do
|
33
|
-
num_calls = 0
|
34
|
-
value_proc = proc do
|
35
|
-
num_calls += 1
|
36
|
-
num_calls
|
37
|
-
end
|
38
|
-
MyConfig.define_settings do
|
39
|
-
setting :set_with_proc, default_proc: value_proc
|
40
|
-
end
|
41
|
-
expect(num_calls).to eq(0)
|
42
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
43
|
-
# calling twice should not call the proc again
|
44
|
-
expect(MyConfig.config.set_with_proc).to eq(1)
|
45
|
-
expect(num_calls).to eq(1)
|
46
|
-
end
|
47
|
-
|
48
|
-
it "should raise error when setting configs that don't exist" do
|
49
|
-
expect { MyConfig.configure { set15 'some_value' } }.to raise_error(NameError)
|
50
|
-
end
|
51
|
-
|
52
|
-
it 'should add values' do
|
53
|
-
MyConfig.configure do |config|
|
54
|
-
config.set1 = 5 # config.x syntax
|
55
|
-
set2 nil # method_missing syntax
|
56
|
-
config.group.set3 = true
|
57
|
-
end
|
58
|
-
|
59
|
-
# second configure should not blow anything away
|
60
|
-
MyConfig.configure do
|
61
|
-
listy do
|
62
|
-
list1 0
|
63
|
-
list2 1
|
64
|
-
end
|
65
|
-
listy do
|
66
|
-
list1 100
|
67
|
-
list2 200
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
expect(MyConfig.config.set1).to eq(5)
|
72
|
-
expect(MyConfig.config.set2).to be_nil
|
73
|
-
expect(MyConfig.config.group.set3).to eq(true)
|
74
|
-
expect(MyConfig.config.listy_objects.map(&:to_h)).
|
75
|
-
to eq([
|
76
|
-
{ list1: 0, list2: 1 },
|
77
|
-
{ list1: 100, list2: 200 }
|
78
|
-
])
|
79
|
-
|
80
|
-
# test reset!
|
81
|
-
MyConfig.config.reset!
|
82
|
-
expect(MyConfig.config.set1).to be_nil
|
83
|
-
expect(MyConfig.config.set2).to eq('hi mom')
|
84
|
-
expect(MyConfig.config.group.set3).to eq(false)
|
85
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
86
|
-
end
|
87
|
-
|
88
|
-
it 'should add with block syntax' do
|
89
|
-
MyConfig.configure do
|
90
|
-
group do
|
91
|
-
set5(proc { 10 })
|
92
|
-
end
|
93
|
-
end
|
94
|
-
expect(MyConfig.config.group.set5.call).to eq(10)
|
95
|
-
end
|
96
|
-
|
97
|
-
it 'should add or redefine settings' do
|
98
|
-
MyConfig.define_settings do
|
99
|
-
setting :group do
|
100
|
-
setting :set6, 15
|
101
|
-
setting :set5, (proc { 15 })
|
102
|
-
end
|
103
|
-
setting_object :notey do
|
104
|
-
setting :note_title, 'some-title'
|
105
|
-
end
|
106
|
-
end
|
107
|
-
|
108
|
-
expect(MyConfig.config.group.set6).to eq(15)
|
109
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
110
|
-
expect(MyConfig.config.listy_objects).to be_empty
|
111
|
-
expect(MyConfig.config.notey_objects).to be_empty
|
112
|
-
|
113
|
-
MyConfig.configure do
|
114
|
-
notey do
|
115
|
-
note_title 'hi mom'
|
116
|
-
end
|
117
|
-
listy do
|
118
|
-
list1 0
|
119
|
-
end
|
120
|
-
end
|
121
|
-
expect(MyConfig.config.notey_objects.size).to eq(1)
|
122
|
-
expect(MyConfig.config.notey_objects.first.note_title).to eq('hi mom')
|
123
|
-
expect(MyConfig.config.listy_objects.size).to eq(1)
|
124
|
-
expect(MyConfig.config.listy_objects.first.list1).to eq(0)
|
125
|
-
|
126
|
-
# This should not remove any keys
|
127
|
-
MyConfig.define_settings do
|
128
|
-
setting :group do
|
129
|
-
setting :set6, 20
|
130
|
-
end
|
131
|
-
end
|
132
|
-
expect(MyConfig.config.group.set6).to eq(20)
|
133
|
-
expect(MyConfig.config.group.set5.call).to eq(15)
|
134
|
-
end
|
135
|
-
|
136
|
-
end
|