fastly_nsq 0.13.2 → 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. checksums.yaml +5 -5
  2. data/.env +4 -0
  3. data/.overcommit.yml +3 -3
  4. data/.rubocop.yml +11 -1
  5. data/.travis.yml +8 -1
  6. data/Gemfile +9 -0
  7. data/README.md +52 -82
  8. data/Rakefile +2 -0
  9. data/bin/fastly_nsq +1 -0
  10. data/docker-compose.yml +23 -0
  11. data/examples/.sample.env +0 -3
  12. data/fastly_nsq.gemspec +7 -8
  13. data/lib/fastly_nsq.rb +44 -50
  14. data/lib/fastly_nsq/cli.rb +20 -14
  15. data/lib/fastly_nsq/consumer.rb +26 -30
  16. data/lib/fastly_nsq/feeder.rb +16 -0
  17. data/lib/fastly_nsq/http/nsqd.rb +7 -1
  18. data/lib/fastly_nsq/http/nsqlookupd.rb +1 -1
  19. data/lib/fastly_nsq/launcher.rb +31 -23
  20. data/lib/fastly_nsq/listener.rb +34 -103
  21. data/lib/fastly_nsq/manager.rb +48 -72
  22. data/lib/fastly_nsq/message.rb +2 -0
  23. data/lib/fastly_nsq/messenger.rb +5 -5
  24. data/lib/fastly_nsq/priority_queue.rb +12 -0
  25. data/lib/fastly_nsq/priority_thread_pool.rb +32 -0
  26. data/lib/fastly_nsq/producer.rb +52 -32
  27. data/lib/fastly_nsq/testing.rb +239 -0
  28. data/lib/fastly_nsq/tls_options.rb +2 -0
  29. data/lib/fastly_nsq/version.rb +3 -1
  30. data/spec/{lib/fastly_nsq/cli_spec.rb → cli_spec.rb} +2 -0
  31. data/spec/consumer_spec.rb +59 -0
  32. data/spec/fastly_nsq_spec.rb +72 -0
  33. data/spec/feeder_spec.rb +22 -0
  34. data/spec/{lib/fastly_nsq/http → http}/nsqd_spec.rb +1 -1
  35. data/spec/{lib/fastly_nsq/http → http}/nsqlookupd_spec.rb +1 -1
  36. data/spec/{lib/fastly_nsq/http_spec.rb → http_spec.rb} +3 -1
  37. data/spec/integration_spec.rb +48 -0
  38. data/spec/launcher_spec.rb +50 -0
  39. data/spec/listener_spec.rb +184 -0
  40. data/spec/manager_spec.rb +111 -0
  41. data/spec/matchers/delegate.rb +32 -0
  42. data/spec/{lib/fastly_nsq/message_spec.rb → message_spec.rb} +2 -0
  43. data/spec/{lib/fastly_nsq/messenger_spec.rb → messenger_spec.rb} +7 -5
  44. data/spec/priority_thread_pool_spec.rb +19 -0
  45. data/spec/producer_spec.rb +94 -0
  46. data/spec/spec_helper.rb +32 -28
  47. data/spec/support/http.rb +37 -0
  48. data/spec/support/webmock.rb +22 -0
  49. data/spec/{lib/fastly_nsq/tls_options_spec.rb → tls_options_spec.rb} +2 -0
  50. metadata +54 -96
  51. data/env_configuration_for_local_gem_tests.yml +0 -5
  52. data/example_config_class.rb +0 -20
  53. data/examples/Rakefile +0 -41
  54. data/lib/fastly_nsq/fake_backend.rb +0 -114
  55. data/lib/fastly_nsq/listener/config.rb +0 -35
  56. data/lib/fastly_nsq/rake_task.rb +0 -78
  57. data/lib/fastly_nsq/strategy.rb +0 -36
  58. data/spec/lib/fastly_nsq/consumer_spec.rb +0 -72
  59. data/spec/lib/fastly_nsq/fake_backend_spec.rb +0 -135
  60. data/spec/lib/fastly_nsq/fastly_nsq_spec.rb +0 -10
  61. data/spec/lib/fastly_nsq/launcher_spec.rb +0 -56
  62. data/spec/lib/fastly_nsq/listener_spec.rb +0 -213
  63. data/spec/lib/fastly_nsq/manager_spec.rb +0 -127
  64. data/spec/lib/fastly_nsq/producer_spec.rb +0 -60
  65. data/spec/lib/fastly_nsq/rake_task_spec.rb +0 -142
  66. data/spec/lib/fastly_nsq/strategy_spec.rb +0 -36
  67. data/spec/lib/fastly_nsq_spec.rb +0 -18
  68. data/spec/support/env_helpers.rb +0 -15
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe FastlyNsq do
6
+ describe '#configure' do
7
+ specify { expect { |b| described_class.configure(&b) }.to yield_with_args(described_class) }
8
+ end
9
+
10
+ describe '#listen' do
11
+ let!(:default_channel) { subject.channel }
12
+ let!(:topic) { 'fnsq' }
13
+
14
+ before { subject.channel = 'fnsq' }
15
+ after { subject.channel = default_channel }
16
+
17
+ it 'creates a listener' do
18
+ expect { subject.listen topic, ->(*) {} }.to change { subject.manager.topics }.to([topic])
19
+ end
20
+
21
+ it 'creates a listener with a specific priority' do
22
+ listener = subject.listen topic, ->(*) {}, priority: 10
23
+ expect(listener.priority).to eq(10)
24
+ end
25
+ end
26
+
27
+ describe '#channel=' do
28
+ let!(:default_channel) { subject.channel }
29
+ after { subject.channel = default_channel }
30
+
31
+ it 'allows the channel to be set and retrieved' do
32
+ expect(subject.channel).to be_nil
33
+ subject.channel = 'foo'
34
+ expect(subject.channel).to eq('foo')
35
+ end
36
+ end
37
+
38
+ describe '#logger=' do
39
+ let!(:default_logger) { subject.logger }
40
+ after { subject.logger = default_logger }
41
+
42
+ it 'allows the logger to be set and retrieved' do
43
+ logger = Logger.new(STDOUT)
44
+ subject.logger = logger
45
+
46
+ expect(subject.logger).to eq logger
47
+ end
48
+ end
49
+
50
+ describe '#manager' do
51
+ it 'represents the active default manager' do
52
+ expect(subject.manager).not_to be_stopped
53
+ end
54
+ end
55
+
56
+ describe '#manager=' do
57
+ it 'transfers to specified manager' do
58
+ old_manager = subject.manager
59
+ new_manager = FastlyNsq::Manager.new
60
+
61
+ expect(old_manager).to receive(:transfer).with(new_manager)
62
+
63
+ subject.manager = new_manager
64
+ end
65
+ end
66
+
67
+ describe '#lookupd_http_addresses' do
68
+ it 'retreives NSQLOOKUPD_HTTP_ADDRESS' do
69
+ expect(subject.lookupd_http_addresses).to eq(ENV['NSQLOOKUPD_HTTP_ADDRESS'].split(','))
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe FastlyNsq::Feeder do
6
+ describe '#push' do
7
+ it 'sends message to processor with the specified priority' do
8
+ messages = []
9
+ processor = ->(m) { messages << m }
10
+ priority = 5
11
+ message = 'foo'
12
+
13
+ feeder = described_class.new(processor, priority)
14
+
15
+ expect(FastlyNsq.manager.pool).to receive(:post).with(priority).and_call_original
16
+
17
+ feeder.push(message)
18
+
19
+ expect { messages }.to eventually(contain_exactly(message)).within(2)
20
+ end
21
+ end
22
+ end
@@ -3,7 +3,7 @@
3
3
  require 'spec_helper'
4
4
  require 'fastly_nsq/http/nsqd'
5
5
 
6
- RSpec.describe FastlyNsq::Http::Nsqd do
6
+ RSpec.describe FastlyNsq::Http::Nsqd, :webmock do
7
7
  let(:base_uri) { 'http://example.com' }
8
8
 
9
9
  it 'makes simple get requests' do
@@ -3,7 +3,7 @@
3
3
  require 'spec_helper'
4
4
  require 'fastly_nsq/http/nsqlookupd'
5
5
 
6
- RSpec.describe FastlyNsq::Http::Nsqlookupd do
6
+ RSpec.describe FastlyNsq::Http::Nsqlookupd, :webmock do
7
7
  let(:base_uri) { 'http://example.com' }
8
8
 
9
9
  it 'makes simple get requests' do
@@ -1,7 +1,9 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'spec_helper'
2
4
  require 'fastly_nsq/http'
3
5
 
4
- RSpec.describe FastlyNsq::Http do
6
+ RSpec.describe FastlyNsq::Http, :webmock do
5
7
  let(:base_url) { 'http://example.com' }
6
8
  describe 'get' do
7
9
  it 'can make simple requests' do
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe 'integration' do
6
+ let!(:topic) { 'fnsq-topic' }
7
+ let!(:channel) { 'fnsq-channel' }
8
+ let!(:message) { { 'foo' => 'bar' } }
9
+
10
+ before { reset_topic(topic, channel: channel) }
11
+
12
+ it 'processes jobs' do
13
+ received = nil
14
+ producer = FastlyNsq::Producer.new(topic: topic)
15
+ FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(m) { received = m })
16
+ producer.write JSON.dump(message)
17
+
18
+ expect { received&.body }.to eventually(eq(message)).within(2)
19
+ end
20
+
21
+ describe 'inline', :inline do
22
+ it 'processes job' do
23
+ received = nil
24
+ producer = FastlyNsq::Producer.new(topic: topic)
25
+ FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(m) { received = m })
26
+ producer.write JSON.dump(message)
27
+
28
+ expect(received.body).to eq(message)
29
+ end
30
+ end
31
+
32
+ describe 'fake', :fake do
33
+ it 'stores jobs' do
34
+ received = nil
35
+ encoded_message = JSON.dump(message)
36
+ producer = FastlyNsq::Producer.new(topic: topic)
37
+ listener = FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(m) { received = m })
38
+ expect { producer.write encoded_message }.to change { listener.messages.size }.by(1)
39
+
40
+ queued_message = listener.messages.shift
41
+ expect(queued_message.body).to eq(encoded_message)
42
+
43
+ listener.drain
44
+
45
+ expect(received.body).to eq(message)
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe FastlyNsq::Launcher do
6
+ let!(:options) { { max_threads: 3, timeout: 9 } }
7
+ let!(:launcher) { FastlyNsq::Launcher.new options }
8
+ let!(:topic) { 'fnsq' }
9
+ let!(:channel) { 'fnsq' }
10
+ let(:listener) { FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(*) {}) }
11
+
12
+ before { reset_topic(topic, channel: channel) }
13
+ before { expect { listener }.to eventually(be_connected).within(5) }
14
+ after { listener.terminate if listener.connected? }
15
+
16
+ let(:manager) { launcher.manager }
17
+
18
+ it 'creates a manager with correct options' do
19
+ expect(FastlyNsq.manager.pool.max_threads).to eq(3)
20
+ end
21
+
22
+ describe '#beat' do
23
+ let!(:logger) { Logger.new(nil).tap { |l| l.level = Logger::DEBUG } }
24
+ let!(:launcher) { FastlyNsq::Launcher.new pulse: 0.01, logger: logger }
25
+
26
+ it 'creates a heartbeat thread' do
27
+ expect(logger).not_to receive(:error)
28
+ expect { launcher.beat }.to eventually_not(eq('dead')).pause_for(1)
29
+ end
30
+ end
31
+
32
+ describe '#stop_listeners' do
33
+ it 'stops listeners and sets done' do
34
+ expect(launcher).not_to be_stopping
35
+ expect(manager).to receive(:stop_listeners)
36
+ expect(manager).not_to receive(:terminate)
37
+
38
+ launcher.stop_listeners
39
+
40
+ expect(launcher).to be_stopping
41
+ end
42
+ end
43
+
44
+ describe '#stop' do
45
+ it 'stops the manager within a deadline' do
46
+ expect(manager).to receive(:terminate).with(options[:timeout])
47
+ launcher.stop
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,184 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe FastlyNsq::Listener do
6
+ let!(:topic) { 'fnsq' }
7
+ let!(:channel) { 'fnsq' }
8
+ let!(:messages) { [] }
9
+ let(:processor) { ->(m) { messages << m.body } }
10
+
11
+ before { reset_topic(topic, channel: channel) }
12
+ before { expect { subject }.to eventually(be_connected).within(5) }
13
+ after { subject.terminate if subject.connected? }
14
+
15
+ subject { described_class.new(topic: topic, channel: channel, processor: processor) }
16
+
17
+ describe '#initialize' do
18
+ describe 'with FastlyNsq.channel set' do
19
+ let!(:default_channel) { FastlyNsq.channel }
20
+ before { FastlyNsq.channel = 'fnsq' }
21
+ after { FastlyNsq.channel = default_channel }
22
+
23
+ it 'defaults to FastlyNsq.channel' do
24
+ listener = described_class.new(topic: topic, processor: processor)
25
+ expect(listener.channel).to eq(FastlyNsq.channel)
26
+ end
27
+ end
28
+
29
+ describe 'with FastlyNsq.preprocessor set' do
30
+ let!(:default_preprocessor) { FastlyNsq.preprocessor }
31
+ before { FastlyNsq.preprocessor = 'fnsq' }
32
+ after { FastlyNsq.preprocessor = default_preprocessor }
33
+
34
+ it 'defaults to FastlyNsq.preprocessor' do
35
+ listener = described_class.new(topic: topic, processor: processor, channel: channel)
36
+ expect(listener.preprocessor).to eq(FastlyNsq.preprocessor)
37
+ end
38
+ end
39
+
40
+ describe 'with FastlyNsq.logger set' do
41
+ let!(:default_logger) { FastlyNsq.logger }
42
+ before { FastlyNsq.logger = Logger.new(nil) }
43
+ after { FastlyNsq.logger = default_logger }
44
+
45
+ it 'defaults to FastlyNsq.logger' do
46
+ listener = described_class.new(topic: topic, processor: processor, channel: channel)
47
+ expect(listener.logger).to eq(FastlyNsq.logger)
48
+ end
49
+ end
50
+
51
+ it 'warns when creating a listener for the same topic' do
52
+ expect(FastlyNsq.manager.logger).to receive(:warn).and_yield.and_return(match("#{topic} was added more than once"))
53
+
54
+ described_class.new(topic: topic, channel: channel, processor: processor)
55
+ end
56
+ end
57
+
58
+ describe '#priority' do
59
+ specify { expect(subject.priority).to eq(described_class::DEFAULT_PRIORITY) }
60
+ end
61
+
62
+ describe '#consumer' do
63
+ specify { expect(subject.consumer).to be_a(FastlyNsq::Consumer) }
64
+ end
65
+
66
+ describe 'connect_timeout' do
67
+ specify { expect(subject.consumer.connect_timeout).to eq(described_class::DEFAULT_CONNECTION_TIMEOUT) }
68
+ end
69
+
70
+ it 'requires processor to respond_to #call' do
71
+ expect { described_class.new(topic: topic, channel: channel, processor: 'foo') }.
72
+ to raise_error(ArgumentError, match('#call'))
73
+ end
74
+
75
+ it 'requires priority to be a Fixnum' do
76
+ expect { described_class.new(topic: topic, channel: channel, processor: ->(*) {}, priority: 'foo') }.
77
+ to raise_error(ArgumentError, match('Integer'))
78
+ end
79
+
80
+ describe '#call' do
81
+ it 'processes a message' do
82
+ body = { 'foo' => 'bar' }
83
+ message = spy('message', body: JSON.dump(body))
84
+ expect { subject.call(message) }.to change { messages }.to([body])
85
+ end
86
+
87
+ describe 'when the processor returns true' do
88
+ let(:processor) { ->(_) { true } }
89
+
90
+ it 'finishes the message' do
91
+ message = spy('message', body: '{}')
92
+ subject.call(message)
93
+
94
+ expect(message).to have_received(:finish)
95
+ end
96
+ end
97
+
98
+ describe 'when the processor returns false' do
99
+ let(:processor) { ->(_) { false } }
100
+
101
+ it 'finishes the message' do
102
+ message = spy('message', body: '{}')
103
+ subject.call(message)
104
+
105
+ expect(message).not_to have_received(:finish)
106
+ end
107
+ end
108
+ end
109
+
110
+ it { should be_connected }
111
+
112
+ it 'should terminate' do
113
+ expect { subject.terminate }.to change(subject, :connected?).to(false)
114
+ end
115
+
116
+ describe 'faking', :fake do
117
+ let!(:message) { { 'foo' => 'bar' } }
118
+
119
+ before { subject }
120
+
121
+ it { should be_connected }
122
+
123
+ it 'should terminate' do
124
+ expect { subject.terminate }.to change(subject, :connected?).to(false)
125
+ end
126
+
127
+ it "stores messages produced to the listener's topic" do
128
+ expect do
129
+ FastlyNsq::Producer.new(topic: topic).write(message)
130
+ end.to change { subject.messages.size }.by(1)
131
+
132
+ test_message = subject.messages.pop
133
+ expect(test_message.raw_body).to eq(message)
134
+ end
135
+
136
+ describe 'when the processor returns true' do
137
+ let(:processor) { ->(_) { true } }
138
+
139
+ it 'drains queued messages' do
140
+ FastlyNsq::Producer.new(topic: topic).write(message)
141
+ expect { subject.drain }.to change { subject.messages.size }.by(-1)
142
+ end
143
+ end
144
+
145
+ describe 'when the processor returns false' do
146
+ let(:processor) { ->(_) { false } }
147
+
148
+ it 'does not remove messages' do
149
+ FastlyNsq::Producer.new(topic: topic).write(message)
150
+ expect { subject.drain }.not_to change { subject.messages.size }
151
+ end
152
+ end
153
+ end
154
+
155
+ describe 'inline', :inline do
156
+ let!(:message) { { 'foo' => 'bar' } }
157
+ let!(:processor) { ->(m) { messages << m.raw_body } }
158
+
159
+ before { subject }
160
+
161
+ it { should be_connected }
162
+
163
+ it 'should terminate' do
164
+ expect { subject.terminate }.to change(subject, :connected?).to(false)
165
+ end
166
+
167
+ describe 'when the processor returns true' do
168
+ it 'processes and removes messages' do
169
+ expect { FastlyNsq::Producer.new(topic: topic).write(message) }.to change { messages.size }.by(1)
170
+ expect(messages).to contain_exactly(message)
171
+ expect(subject.messages).to be_empty
172
+ end
173
+ end
174
+
175
+ describe 'when the processor returns false' do
176
+ let(:processor) { ->(_) { false } }
177
+
178
+ it 'does not remove messages' do
179
+ FastlyNsq::Producer.new(topic: topic).write(message)
180
+ expect { subject.drain }.not_to change { subject.messages.size }
181
+ end
182
+ end
183
+ end
184
+ end
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'spec_helper'
4
+
5
+ RSpec.describe FastlyNsq::Manager do
6
+ let!(:topic) { 'fnsq' }
7
+ let!(:channel) { 'fnsq' }
8
+
9
+ subject { FastlyNsq.manager }
10
+
11
+ before { reset_topic(topic, channel: channel) }
12
+
13
+ after { subject.terminate(1) unless subject.stopped? }
14
+
15
+ it { should_not be_stopped }
16
+
17
+ describe '#initialize' do
18
+ it 'allows max_threads to be specified' do
19
+ max_threads = described_class::DEFAULT_POOL_SIZE * 2
20
+ manager = described_class.new(max_threads: max_threads)
21
+
22
+ expect(manager.pool.max_threads).to eq(max_threads)
23
+ end
24
+
25
+ it 'defaults max_threads to DEFAULT_POOL_SIZE' do
26
+ expect(subject.pool.max_threads).to eq(described_class::DEFAULT_POOL_SIZE)
27
+ end
28
+
29
+ it 'allows fallback_policy to be specified' do
30
+ manager = described_class.new(fallback_policy: :abort)
31
+
32
+ expect(manager.pool.fallback_policy).to eq(:abort)
33
+ end
34
+
35
+ it 'defaults fallback_policy to caller_runs' do
36
+ expect(subject.pool.fallback_policy).to eq(:caller_runs)
37
+ end
38
+
39
+ it 'defaults logger to FastlyNsq.logger' do
40
+ expect(subject.logger).to eq(FastlyNsq.logger)
41
+ end
42
+
43
+ it 'allows logger to be specified' do
44
+ logger = Logger.new(nil)
45
+ manager = described_class.new(logger: logger)
46
+
47
+ expect(manager.logger).to eq(logger)
48
+ end
49
+ end
50
+
51
+ context 'with a listener' do
52
+ let!(:listener) { FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(*) {}) }
53
+ before { expect { listener }.to eventually(be_connected).within(5) }
54
+
55
+ it 'tracks listener' do
56
+ expect(subject.listeners).to contain_exactly(listener)
57
+ end
58
+
59
+ it 'tracks topic listeners' do
60
+ expect(subject.topic_listeners).to eq(topic => listener)
61
+ end
62
+
63
+ it 'tracks topics' do
64
+ expect(subject.topics).to contain_exactly(topic)
65
+ end
66
+
67
+ describe '#terminate' do
68
+ it 'terminates listeners' do
69
+ expect { subject.terminate(2) }.to change(listener, :connected?).to(false)
70
+ end
71
+
72
+ it 'terminates the processing pool' do
73
+ expect { subject.terminate(2) }.to change(subject.pool, :shutdown?).to(true)
74
+ end
75
+
76
+ it 'stops' do
77
+ expect { subject.terminate(2) } .to change(subject, :stopped?).from(false).to(true)
78
+ end
79
+
80
+ context 'when the pool does not terminate within a the specified timeframe' do
81
+ before { expect(subject.pool).to receive(:shutdown).and_return(false) }
82
+
83
+ it 'kills the pool' do
84
+ expect(subject.pool).to receive(:kill).once.and_call_original
85
+
86
+ expect { subject.terminate(0.1) }.to change(subject.pool, :shutdown?).to(true)
87
+ end
88
+ end
89
+ end
90
+ end
91
+
92
+ it 'transfers' do
93
+ manager = described_class.new
94
+
95
+ listener = nil
96
+ # register listener with default manager
97
+ expect { listener = FastlyNsq::Listener.new(topic: topic, channel: channel, processor: ->(*) {}) }.
98
+ to change { FastlyNsq.manager.listeners.size }.by(1)
99
+ expect { listener }.to eventually(be_connected).within(5)
100
+
101
+ # transfer listener to new manager
102
+ expect { FastlyNsq.manager.transfer(manager) }.
103
+ to change { manager.listeners.size }.by(1).
104
+ and change { FastlyNsq.manager.listeners.size }.by(-1)
105
+
106
+ # old manager processing is disabled
107
+ expect(FastlyNsq.manager.pool).to be_shutdown
108
+ # listener is still connected
109
+ expect(listener).to be_connected
110
+ end
111
+ end