hivent 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +7 -0
  2. data/.codeclimate.yml +19 -0
  3. data/.gitignore +14 -0
  4. data/.rspec +1 -0
  5. data/.rubocop.yml +1063 -0
  6. data/.ruby-version +1 -0
  7. data/.simplecov.template +1 -0
  8. data/.travis.yml +23 -0
  9. data/.version +1 -0
  10. data/Gemfile +4 -0
  11. data/LICENSE +21 -0
  12. data/README.md +196 -0
  13. data/bin/hivent +5 -0
  14. data/hivent.gemspec +34 -0
  15. data/lib/hivent.rb +32 -0
  16. data/lib/hivent/abstract_signal.rb +63 -0
  17. data/lib/hivent/cli/consumer.rb +60 -0
  18. data/lib/hivent/cli/runner.rb +50 -0
  19. data/lib/hivent/cli/start_option_parser.rb +53 -0
  20. data/lib/hivent/config.rb +22 -0
  21. data/lib/hivent/config/options.rb +51 -0
  22. data/lib/hivent/emitter.rb +41 -0
  23. data/lib/hivent/life_cycle_event_handler.rb +41 -0
  24. data/lib/hivent/redis/consumer.rb +82 -0
  25. data/lib/hivent/redis/extensions.rb +26 -0
  26. data/lib/hivent/redis/lua/consumer.lua +179 -0
  27. data/lib/hivent/redis/lua/producer.lua +27 -0
  28. data/lib/hivent/redis/producer.rb +24 -0
  29. data/lib/hivent/redis/redis.rb +14 -0
  30. data/lib/hivent/redis/signal.rb +36 -0
  31. data/lib/hivent/rspec.rb +11 -0
  32. data/lib/hivent/signal.rb +14 -0
  33. data/lib/hivent/spec.rb +11 -0
  34. data/lib/hivent/spec/matchers.rb +14 -0
  35. data/lib/hivent/spec/matchers/emit.rb +116 -0
  36. data/lib/hivent/spec/signal.rb +60 -0
  37. data/lib/hivent/version.rb +6 -0
  38. data/spec/codeclimate_helper.rb +5 -0
  39. data/spec/fixtures/cli/bootstrap_consumers.rb +7 -0
  40. data/spec/fixtures/cli/life_cycle_event_test.rb +8 -0
  41. data/spec/hivent/abstract_signal_spec.rb +161 -0
  42. data/spec/hivent/cli/consumer_spec.rb +68 -0
  43. data/spec/hivent/cli/runner_spec.rb +75 -0
  44. data/spec/hivent/cli/start_option_parser_spec.rb +48 -0
  45. data/spec/hivent/life_cycle_event_handler_spec.rb +38 -0
  46. data/spec/hivent/redis/consumer_spec.rb +348 -0
  47. data/spec/hivent/redis/signal_spec.rb +155 -0
  48. data/spec/hivent_spec.rb +100 -0
  49. data/spec/spec/matchers/emit_spec.rb +66 -0
  50. data/spec/spec/signal_spec.rb +72 -0
  51. data/spec/spec_helper.rb +27 -0
  52. data/spec/support/matchers/exit_with_code.rb +28 -0
  53. data/spec/support/stdout_helpers.rb +25 -0
  54. metadata +267 -0
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+ require "spec_helper"
3
+
4
+ describe Hivent::CLI::Consumer do
5
+
6
+ let(:consumer) { Hivent::CLI::Consumer.new(options) }
7
+ let(:options) { {} }
8
+
9
+ let(:life_cycle_event_handler) { double("Hivent::LifeCycleEventHandler").as_null_object }
10
+ let(:require_file) { File.expand_path("../../../fixtures/cli/bootstrap_consumers.rb", __FILE__) }
11
+
12
+ before :each do
13
+ allow(Hivent::Config).to receive(:life_cycle_event_handler).and_return(life_cycle_event_handler)
14
+ end
15
+
16
+ describe "#run!" do
17
+
18
+ subject { silence { consumer.run! } }
19
+ let(:options) { { require: require_file } }
20
+ let(:redis) { Redis.new(url: REDIS_URL) }
21
+ let(:service_name) { "test" }
22
+ let(:partition_count) { 2 }
23
+ let(:redis_consumer_double) { double("Hivent::Redis::Consumer").as_null_object }
24
+
25
+ before :each do
26
+ allow(Hivent::Redis::Consumer).to receive(:new).and_return(redis_consumer_double)
27
+ end
28
+
29
+ after :each do
30
+ redis.flushall
31
+ Hivent.emitter.events.clear
32
+ end
33
+
34
+ it "registers the partition count for the service" do
35
+ expect { subject }.to change {
36
+ redis.get("#{service_name}:partition_count")
37
+ }.from(nil).to(partition_count.to_s)
38
+ end
39
+
40
+ it "registers events for the service" do
41
+ Hivent.emitter.events.push({ name: "my:event", version: 1 }, name: "my:event2")
42
+ expect { subject }.to change {
43
+ [redis.smembers("my:event"), redis.smembers("my:event2")].flatten
44
+ }.from([]).to([service_name, service_name])
45
+ end
46
+
47
+ it "notifies the life cycle event handler about registration of events" do
48
+ events = [{ name: "my:event", version: 1 }, { name: "my:event2" }]
49
+ Hivent.emitter.events.push(*events)
50
+
51
+ expect(life_cycle_event_handler).to receive(:application_registered).with(service_name, events, partition_count)
52
+ subject
53
+ end
54
+
55
+ it "starts the consumer" do
56
+ double = double().as_null_object
57
+
58
+ allow(Hivent::Redis::Consumer).to receive(:new)
59
+ .with(instance_of(Redis), service_name, "#{Socket.gethostname}:#{Process.pid}", life_cycle_event_handler)
60
+ .and_return(double)
61
+
62
+ subject
63
+ expect(double).to have_received(:run!).once
64
+ end
65
+
66
+ end
67
+
68
+ end
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+ require "spec_helper"
3
+
4
+ describe Hivent::CLI::Runner do
5
+
6
+ describe "#run" do
7
+
8
+ subject { runner.run }
9
+
10
+ let(:runner) { described_class.new(([command] + args).compact) }
11
+ let(:args) { [] }
12
+ let(:require_file) { File.expand_path("../../../fixtures/cli/bootstrap_consumers.rb", __FILE__) }
13
+
14
+ context "with no command" do
15
+
16
+ let(:command) { nil }
17
+
18
+ it "prints help for available commands" do
19
+ expect(with_captured_stdout { subject }).to include("Available COMMANDs are")
20
+ end
21
+
22
+ end
23
+
24
+ context "with unknown command" do
25
+
26
+ let(:command) { "unknown" }
27
+
28
+ it "prints help for available commands" do
29
+ expect(with_captured_stdout { subject }).to include("Available COMMANDs are")
30
+ end
31
+
32
+ end
33
+
34
+ context "with --help option" do
35
+
36
+ let(:args) { ["--help"] }
37
+
38
+ ["start"].each do |cmd|
39
+
40
+ context "with #{cmd} command" do
41
+
42
+ let(:command) { cmd }
43
+
44
+ it "prints help for #{cmd} command options" do
45
+ output = with_captured_stdout do
46
+ begin
47
+ subject
48
+ rescue SystemExit
49
+ # do nothing
50
+ end
51
+ end
52
+ expect(output).to include("Usage: hivent #{cmd} [options]")
53
+ end
54
+
55
+ end
56
+
57
+ end
58
+
59
+ end
60
+
61
+ context "with start command and all required arguments" do
62
+
63
+ let(:command) { "start" }
64
+ let(:args) { ["--require", require_file] }
65
+
66
+ it "starts the consumer" do
67
+ expect(Hivent::CLI::Consumer).to receive(:run!).once
68
+ subject
69
+ end
70
+
71
+ end
72
+
73
+ end
74
+
75
+ end
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+ require "spec_helper"
3
+
4
+ describe Hivent::CLI::StartOptionParser do
5
+
6
+ let(:parser) { described_class.new(:start, args) }
7
+ let(:args) { [] }
8
+ let(:require_file) { File.expand_path("../../../fixtures/cli/bootstrap_consumers.rb", __FILE__) }
9
+
10
+ describe "#parse" do
11
+
12
+ subject { silence { parser.parse } }
13
+
14
+ context "when --require option is omitted" do
15
+
16
+ it "terminates" do
17
+ expect { subject }.to exit_with_code(1)
18
+ end
19
+
20
+ end
21
+
22
+ context "when --require is given" do
23
+
24
+ let(:args) { ["--require", require_file] }
25
+
26
+ it "does not terminate" do
27
+ expect { subject }.not_to exit_with_code(1)
28
+ end
29
+
30
+ it "sets options for require" do
31
+ expect(subject[:require]).to eq(args.last)
32
+ end
33
+
34
+ context "but does not exist" do
35
+
36
+ let(:args) { ["--require", "/does/not/exist.rb"] }
37
+
38
+ it "terminates" do
39
+ expect { subject }.to exit_with_code(1)
40
+ end
41
+
42
+ end
43
+
44
+ end
45
+
46
+ end
47
+
48
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+ require "spec_helper"
3
+
4
+ describe Hivent::LifeCycleEventHandler do
5
+
6
+ subject { silence { consumer.run! } }
7
+
8
+ let(:consumer) { Hivent::CLI::Consumer.new(require: require_file) }
9
+ let(:require_file) { File.expand_path("../../fixtures/cli/life_cycle_event_test.rb", __FILE__) }
10
+ let(:redis_consumer_double) { double("Hivent::Redis::Consumer").as_null_object }
11
+
12
+ let(:redis) { Redis.new(url: REDIS_URL) }
13
+ let(:handler_class) { Class.new(described_class) }
14
+
15
+ let(:event) { { name: "my:event", version: 1 } }
16
+
17
+ before :each do
18
+ stub_const("MyHandler", handler_class) # MyHandler is used in require file
19
+ allow(Hivent::Redis::Consumer).to receive(:new).and_return(redis_consumer_double)
20
+ end
21
+
22
+ after :each do
23
+ redis.flushall
24
+ Hivent.emitter.events.clear
25
+ end
26
+
27
+ it "notifies custom life cycle event handler about registration of events" do
28
+ Hivent.emitter.events.push(event)
29
+ expect_any_instance_of(handler_class).to receive(:application_registered)
30
+ subject
31
+ end
32
+
33
+ it "passes life cycle event handler to redis consumer" do
34
+ expect(Hivent::Redis::Consumer).to receive(:new).with(anything, anything, anything, instance_of(handler_class))
35
+ subject
36
+ end
37
+
38
+ end
@@ -0,0 +1,348 @@
1
+ # frozen_string_literal: true
2
+ require "spec_helper"
3
+
4
+ describe Hivent::Redis::Consumer do
5
+
6
+ let(:consumer) { described_class.new(redis, service_name, consumer_name, life_cycle_event_handler) }
7
+ let(:redis) { Redis.new(url: REDIS_URL) }
8
+ let(:service_name) { "a_service" }
9
+ let(:consumer_name) { "a_consumer" }
10
+ let(:life_cycle_event_handler) { double("Hivent::LifeCycleEventHandler").as_null_object }
11
+
12
+ after :each do
13
+ redis.flushall
14
+
15
+ Hivent.emitter.__events.clear
16
+ end
17
+
18
+ describe "#queues" do
19
+ def balance(consumers)
20
+ # 1. Marks every consumer as "alive"
21
+ # 2. Resets every consumer
22
+ # 3. Distributes partitions evenly
23
+ 3.times do
24
+ consumers.each(&:queues)
25
+ end
26
+ end
27
+
28
+ before :each do
29
+ redis.set("#{service_name}:partition_count", partition_count)
30
+ end
31
+
32
+ context "with a single consumer" do
33
+ subject { consumer.queues }
34
+
35
+ let(:partition_count) { 2 }
36
+
37
+ it "returns all available partitions" do
38
+ expect(subject.length).to eq(partition_count)
39
+ end
40
+ end
41
+
42
+ context "with two consumers and two partitions" do
43
+ let(:consumer1) { described_class.new(redis, service_name, "#{consumer_name}1", life_cycle_event_handler) }
44
+ let(:consumer2) { described_class.new(redis, service_name, "#{consumer_name}2", life_cycle_event_handler) }
45
+ let(:partition_count) { 2 }
46
+
47
+ context "when only one consumer is alive" do
48
+ before :each do
49
+ # Hearbeat from first consumer,
50
+ # marking it as "alive"
51
+ consumer1.queues
52
+ end
53
+
54
+ it "assigns all available partitions to the living consumer" do
55
+ distribution = [consumer1.queues, consumer2.queues]
56
+
57
+ expect(distribution.map(&:length)).to eq([2, 0])
58
+ end
59
+
60
+ describe "balancing" do
61
+ it "resets the first consumer for rebalancing" do
62
+ # Marks consumer 1 as alive, assigning all partitions
63
+ consumer1.queues
64
+ # Marks consumer 2 as alive, assigning 0 partitions to
65
+ # start rebalancing
66
+ consumer2.queues
67
+
68
+ # Assigns 0 partitions to finish resetting
69
+ expect(consumer1.queues.length).to eq(0)
70
+ end
71
+
72
+ it "assigns half the partitions after reset" do
73
+ # Fully resets
74
+ consumer1.queues
75
+ consumer2.queues
76
+ consumer1.queues
77
+
78
+ # Distributes partitions across consumers
79
+ expect(consumer2.queues.length).to eq(1)
80
+ end
81
+
82
+ it "rebalances partitions across both consumers" do
83
+ consumer1.queues
84
+ consumer2.queues
85
+ consumer1.queues
86
+ consumer2.queues
87
+
88
+ # Distributes partitions across consumers
89
+ expect(consumer1.queues.length).to eq(1)
90
+ end
91
+
92
+ context "when one of the consumers dies" do
93
+ before :each do
94
+ stub_const("#{described_class}::CONSUMER_TTL", 50)
95
+
96
+ balance([consumer1, consumer2])
97
+ count = 0
98
+
99
+ while count <= 2
100
+ consumer2.queues
101
+ count += 1
102
+
103
+ sleep described_class::CONSUMER_TTL.to_f / 1000
104
+ end
105
+ end
106
+
107
+ it "assigns those consumer's partitions to another consumer" do
108
+ expect(consumer2.queues.length).to eq(2)
109
+ end
110
+ end
111
+ end
112
+ end
113
+
114
+ context "when both consumers are alive" do
115
+ subject do
116
+ [consumer1.queues, consumer2.queues]
117
+ end
118
+
119
+ before :each do
120
+ balance([consumer1, consumer2])
121
+ end
122
+
123
+ it "returns all available partitions" do
124
+ expect(subject.map(&:length)).to eq([1, 1])
125
+ end
126
+ end
127
+ end
128
+
129
+ context "with more consumers than partitions" do
130
+ subject do
131
+ [consumer1.queues, consumer2.queues]
132
+ end
133
+
134
+ let(:consumer1) { described_class.new(redis, service_name, "#{consumer_name}1", life_cycle_event_handler) }
135
+ let(:consumer2) { described_class.new(redis, service_name, "#{consumer_name}2", life_cycle_event_handler) }
136
+ let(:partition_count) { 1 }
137
+
138
+ before :each do
139
+ balance([consumer1, consumer2])
140
+ end
141
+
142
+ it "returns all available partitions" do
143
+ expect(subject.map(&:length)).to eq([1, 0])
144
+ end
145
+ end
146
+
147
+ context "with fewer consumers than partitions" do
148
+ subject do
149
+ [consumer1.queues, consumer2.queues]
150
+ end
151
+
152
+ let(:consumer1) { described_class.new(redis, service_name, "#{consumer_name}1", life_cycle_event_handler) }
153
+ let(:consumer2) { described_class.new(redis, service_name, "#{consumer_name}2", life_cycle_event_handler) }
154
+ let(:partition_count) { 3 }
155
+
156
+ before :each do
157
+ balance([consumer1, consumer2])
158
+ end
159
+
160
+ it "returns all available partitions" do
161
+ expect(subject.map(&:length)).to eq([2, 1])
162
+ end
163
+ end
164
+
165
+ end
166
+
167
+ describe "#consume" do
168
+ subject { consumer.consume }
169
+
170
+ let(:partition_count) { 1 }
171
+ let(:event) do
172
+ {
173
+ payload: { foo: "bar" },
174
+ meta: {
175
+ name: 'my:event',
176
+ version: 1,
177
+ event_uuid: SecureRandom.hex
178
+ }
179
+ }
180
+ end
181
+ let(:event_name_with_version) do
182
+ "#{event[:meta][:name]}:#{event[:meta][:version]}"
183
+ end
184
+ let(:producer) { Hivent::Redis::Producer.new(redis) }
185
+
186
+ before :each do
187
+ redis.set("#{service_name}:partition_count", partition_count)
188
+ redis.sadd(event[:meta][:name], service_name)
189
+
190
+ producer.write(event[:meta][:name], event.to_json, 0)
191
+ end
192
+
193
+ context "when there are items ready to be consumed" do
194
+
195
+ it "emits the item with indifferent access" do
196
+ Hivent.emitter.on(event[:meta][:name]) do |received|
197
+ expect(received[:payload][:foo]).to eq("bar")
198
+ expect(received["payload"]["foo"]).to eq("bar")
199
+ end
200
+
201
+ subject
202
+ end
203
+
204
+ it "emits the item with name only and name with version" do
205
+ counter = 0
206
+ Hivent.emitter.on(event_name_with_version) do |_|
207
+ counter += 1
208
+ end
209
+ Hivent.emitter.on(event[:meta][:name]) do |_|
210
+ counter += 1
211
+ end
212
+
213
+ expect { subject }.to change { counter }.by(2)
214
+ end
215
+
216
+ it "removes the item from the queue" do
217
+ subject
218
+
219
+ expect(redis.llen("#{service_name}:0").to_i).to eq(0)
220
+ end
221
+
222
+ it "notifies life cycle event handler about the processed event" do
223
+ expect(life_cycle_event_handler).to receive(:event_processing_succeeded)
224
+ .with(event[:meta][:name], event[:meta][:version], event.with_indifferent_access)
225
+ subject
226
+ end
227
+
228
+ context "when several items are produced" do
229
+ let(:event2) { { foo: "bar" }.merge(event) }
230
+
231
+ before :each do
232
+ producer.write(event2[:meta][:name], event2.to_json, 0)
233
+ end
234
+
235
+ it "consumes all events in the order they were produced" do
236
+ events = []
237
+ Hivent.emitter.on(event_name_with_version) do |event|
238
+ events << event
239
+ end
240
+
241
+ 2.times { consumer.consume }
242
+
243
+ expect(events).to eq([event, event2].map(&:with_indifferent_access))
244
+ end
245
+ end
246
+
247
+ context "when processing fails" do
248
+
249
+ let(:dead_letter_queue) { "#{service_name}:0:dead_letter" }
250
+
251
+ before :each do
252
+ Hivent.emitter.on(event_name_with_version) do |_|
253
+ raise "something went wrong!"
254
+ end
255
+ end
256
+
257
+ it "puts the item into a dead letter queue" do
258
+ expect { subject }.to change { redis.llen(dead_letter_queue) }.by(1)
259
+ end
260
+
261
+ it "notifies life cycle event handler about the error" do
262
+ expect(life_cycle_event_handler).to receive(:event_processing_failed)
263
+ .with(instance_of(RuntimeError), event.with_indifferent_access, event.to_json, dead_letter_queue)
264
+ subject
265
+ end
266
+ end
267
+
268
+ end
269
+
270
+ context "when there are no items ready to be consumed" do
271
+ before :each do
272
+ allow(Kernel).to receive(:sleep)
273
+
274
+ redis.ltrim("#{service_name}:0", 1, -1)
275
+ end
276
+
277
+ it "sleeps for a little while" do
278
+ subject
279
+ expect(Kernel).to have_received(:sleep).with(described_class::SLEEP_TIME.to_f / 1000)
280
+ end
281
+ end
282
+
283
+ describe "Wildcard event consumption" do
284
+ before :each do
285
+ redis.set("#{service_name}:partition_count", partition_count)
286
+ redis.sadd("*", service_name)
287
+
288
+ producer.write("some_event_name", { foo: "bar", meta: { name: "some_event_name" } }.to_json, 0)
289
+ end
290
+
291
+ it "consumes all events" do
292
+ counter = 0
293
+ Hivent.emitter.on(Hivent::Emitter::WILDCARD) do |_|
294
+ counter += 1
295
+ end
296
+
297
+ expect { subject }.to change { counter }.by(1)
298
+ end
299
+ end
300
+ end
301
+
302
+ describe "#run!" do
303
+ subject { Thread.new { consumer.run! } }
304
+
305
+ let(:partition_count) { 2 }
306
+
307
+ before :each do
308
+ redis.set("#{service_name}:partition_count", partition_count)
309
+
310
+ allow(consumer).to receive(:consume)
311
+ end
312
+
313
+ it "processes items" do
314
+ thread = subject
315
+
316
+ sleep 0.1
317
+
318
+ thread.kill
319
+
320
+ expect(consumer).to have_received(:consume).at_least(:once)
321
+ end
322
+
323
+ end
324
+
325
+ describe "#stop!" do
326
+
327
+ let(:partition_count) { 2 }
328
+
329
+ before :each do
330
+ redis.set("#{service_name}:partition_count", partition_count)
331
+ end
332
+
333
+ it "stops processing" do
334
+ thread = Thread.new do
335
+ consumer.run!
336
+ end
337
+
338
+ sleep 0.1
339
+
340
+ consumer.stop!
341
+
342
+ # nil is returned if timeout expires
343
+ expect(thread.join(2)).to eq(thread)
344
+ end
345
+
346
+ end
347
+
348
+ end