kicks 3.0.0.pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (73) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/ci.yml +24 -0
  3. data/.gitignore +12 -0
  4. data/ChangeLog.md +142 -0
  5. data/Dockerfile +24 -0
  6. data/Dockerfile.slim +20 -0
  7. data/Gemfile +8 -0
  8. data/Guardfile +8 -0
  9. data/LICENSE.txt +22 -0
  10. data/README.md +209 -0
  11. data/Rakefile +12 -0
  12. data/bin/sneakers +6 -0
  13. data/docker-compose.yml +24 -0
  14. data/examples/benchmark_worker.rb +22 -0
  15. data/examples/max_retry_handler.rb +68 -0
  16. data/examples/metrics_worker.rb +34 -0
  17. data/examples/middleware_worker.rb +36 -0
  18. data/examples/newrelic_metrics_worker.rb +40 -0
  19. data/examples/profiling_worker.rb +69 -0
  20. data/examples/sneakers.conf.rb.example +11 -0
  21. data/examples/title_scraper.rb +36 -0
  22. data/examples/workflow_worker.rb +23 -0
  23. data/kicks.gemspec +44 -0
  24. data/lib/sneakers/cli.rb +122 -0
  25. data/lib/sneakers/concerns/logging.rb +34 -0
  26. data/lib/sneakers/concerns/metrics.rb +34 -0
  27. data/lib/sneakers/configuration.rb +125 -0
  28. data/lib/sneakers/content_encoding.rb +47 -0
  29. data/lib/sneakers/content_type.rb +47 -0
  30. data/lib/sneakers/error_reporter.rb +33 -0
  31. data/lib/sneakers/errors.rb +2 -0
  32. data/lib/sneakers/handlers/maxretry.rb +219 -0
  33. data/lib/sneakers/handlers/oneshot.rb +26 -0
  34. data/lib/sneakers/metrics/logging_metrics.rb +16 -0
  35. data/lib/sneakers/metrics/newrelic_metrics.rb +32 -0
  36. data/lib/sneakers/metrics/null_metrics.rb +13 -0
  37. data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
  38. data/lib/sneakers/middleware/config.rb +23 -0
  39. data/lib/sneakers/publisher.rb +49 -0
  40. data/lib/sneakers/queue.rb +87 -0
  41. data/lib/sneakers/runner.rb +91 -0
  42. data/lib/sneakers/spawner.rb +30 -0
  43. data/lib/sneakers/support/production_formatter.rb +11 -0
  44. data/lib/sneakers/support/utils.rb +18 -0
  45. data/lib/sneakers/tasks.rb +66 -0
  46. data/lib/sneakers/version.rb +3 -0
  47. data/lib/sneakers/worker.rb +162 -0
  48. data/lib/sneakers/workergroup.rb +60 -0
  49. data/lib/sneakers.rb +125 -0
  50. data/log/.gitkeep +0 -0
  51. data/scripts/local_integration +2 -0
  52. data/scripts/local_worker +3 -0
  53. data/spec/fixtures/integration_worker.rb +18 -0
  54. data/spec/fixtures/require_worker.rb +23 -0
  55. data/spec/gzip_helper.rb +15 -0
  56. data/spec/sneakers/cli_spec.rb +75 -0
  57. data/spec/sneakers/concerns/logging_spec.rb +39 -0
  58. data/spec/sneakers/concerns/metrics_spec.rb +38 -0
  59. data/spec/sneakers/configuration_spec.rb +97 -0
  60. data/spec/sneakers/content_encoding_spec.rb +81 -0
  61. data/spec/sneakers/content_type_spec.rb +81 -0
  62. data/spec/sneakers/integration_spec.rb +158 -0
  63. data/spec/sneakers/publisher_spec.rb +179 -0
  64. data/spec/sneakers/queue_spec.rb +169 -0
  65. data/spec/sneakers/runner_spec.rb +70 -0
  66. data/spec/sneakers/sneakers_spec.rb +77 -0
  67. data/spec/sneakers/support/utils_spec.rb +44 -0
  68. data/spec/sneakers/tasks/sneakers_run_spec.rb +115 -0
  69. data/spec/sneakers/worker_handlers_spec.rb +469 -0
  70. data/spec/sneakers/worker_spec.rb +712 -0
  71. data/spec/sneakers/workergroup_spec.rb +83 -0
  72. data/spec/spec_helper.rb +21 -0
  73. metadata +352 -0
@@ -0,0 +1,169 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ describe Sneakers::Queue do
5
+ let :queue_vars do
6
+ {
7
+ :prefetch => 25,
8
+ :ack => true,
9
+ :heartbeat => 2,
10
+ :vhost => '/',
11
+ :exchange => "sneakers",
12
+ :exchange_options => {
13
+ :type => :direct,
14
+ durable: true,
15
+ :arguments => { 'x-arg' => 'value' }
16
+ },
17
+ queue_options: {
18
+ durable: true
19
+ }
20
+ }
21
+ end
22
+
23
+ before do
24
+ Sneakers.clear!
25
+ Sneakers.configure
26
+
27
+ @mkworker = Object.new
28
+ stub(@mkworker).opts { { :exchange => 'test-exchange' } }
29
+ @mkchan = Object.new
30
+ mock(@mkchan).prefetch(25)
31
+ @mkex = Object.new
32
+ @mkqueue = Object.new
33
+ end
34
+
35
+ describe 'with our own Bunny object' do
36
+ before do
37
+ @mkbunny = Object.new
38
+ @mkqueue_nondurable = Object.new
39
+
40
+ mock(@mkbunny).start {}
41
+ mock(@mkbunny).create_channel{ @mkchan }
42
+ mock(Bunny).new(
43
+ anything,
44
+ hash_including(:vhost => '/', :heartbeat => 2)
45
+ ){ @mkbunny }
46
+ end
47
+
48
+ describe "#subscribe with sneakers exchange" do
49
+ before do
50
+ mock(@mkchan).exchange("sneakers",
51
+ :type => :direct,
52
+ :durable => true,
53
+ :arguments => { 'x-arg' => 'value' }){ @mkex }
54
+ end
55
+
56
+ it "should setup a bunny queue according to configuration values" do
57
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
58
+ q = Sneakers::Queue.new("downloads", queue_vars)
59
+
60
+ mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
61
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
62
+
63
+ q.subscribe(@mkworker)
64
+ end
65
+
66
+ it "supports multiple routing_keys" do
67
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
68
+ q = Sneakers::Queue.new("downloads",
69
+ queue_vars.merge(:routing_key => ["alpha", "beta"]))
70
+
71
+ mock(@mkqueue).bind(@mkex, :routing_key => "alpha")
72
+ mock(@mkqueue).bind(@mkex, :routing_key => "beta")
73
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
74
+
75
+ q.subscribe(@mkworker)
76
+ end
77
+
78
+ it "supports setting arguments when binding" do
79
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
80
+ q = Sneakers::Queue.new("downloads",
81
+ queue_vars.merge(:bind_arguments => { "os" => "linux", "cores" => 8 }))
82
+
83
+ mock(@mkqueue).bind(@mkex, :routing_key => "downloads", :arguments => { "os" => "linux", "cores" => 8 })
84
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
85
+
86
+ q.subscribe(@mkworker)
87
+ end
88
+
89
+ it "will use whatever handler the worker specifies" do
90
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
91
+ @handler = Object.new
92
+ worker_opts = { :handler => @handler }
93
+ stub(@mkworker).opts { worker_opts }
94
+ mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
95
+
96
+ stub(@mkqueue).bind
97
+ stub(@mkqueue).subscribe
98
+ q = Sneakers::Queue.new("downloads", queue_vars)
99
+ q.subscribe(@mkworker)
100
+ end
101
+
102
+ it "creates a non-durable queue if :queue_durable => false" do
103
+ mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
104
+ queue_vars[:queue_options][:durable] = false
105
+ q = Sneakers::Queue.new("test_nondurable", queue_vars)
106
+
107
+ mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
108
+ mock(@mkqueue_nondurable).subscribe(:block => false, :manual_ack => true)
109
+
110
+ q.subscribe(@mkworker)
111
+ myqueue = q.instance_variable_get(:@queue)
112
+ end
113
+ end
114
+
115
+ describe "#subscribe with default exchange" do
116
+ before do
117
+ # expect default exchange
118
+ queue_vars[:exchange] = ""
119
+ mock(@mkchan).exchange("",
120
+ :type => :direct,
121
+ :durable => true,
122
+ :arguments => {"x-arg" => "value"}){ @mkex }
123
+ end
124
+
125
+ it "does not bind to exchange" do
126
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
127
+ @handler = Object.new
128
+ worker_opts = { :handler => @handler }
129
+ stub(@mkworker).opts { worker_opts }
130
+ mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
131
+
132
+ stub(@mkqueue).bind do
133
+ raise "bind should not be called"
134
+ end
135
+
136
+ stub(@mkqueue).subscribe
137
+ q = Sneakers::Queue.new("downloads", queue_vars)
138
+ q.subscribe(@mkworker)
139
+ end
140
+ end
141
+ end
142
+
143
+ describe 'with an externally-provided connection' do
144
+ describe '#subscribe' do
145
+ before do
146
+ @external_connection = Bunny.new
147
+ mock(@external_connection).start {}
148
+ mock(@external_connection).create_channel{ @mkchan }
149
+ mock(@mkchan).exchange("sneakers",
150
+ :type => :direct,
151
+ :durable => true,
152
+ :arguments => { 'x-arg' => 'value' }){ @mkex }
153
+
154
+ queue_name = 'foo'
155
+ mock(@mkchan).queue(queue_name, :durable => true) { @mkqueue }
156
+ mock(@mkqueue).bind(@mkex, :routing_key => queue_name)
157
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
158
+
159
+ my_vars = queue_vars.merge(:connection => @external_connection)
160
+ @q = Sneakers::Queue.new(queue_name, my_vars)
161
+ end
162
+
163
+ it 'uses that object' do
164
+ @q.subscribe(@mkworker)
165
+ _(@q.instance_variable_get(:@bunny)).must_equal @external_connection
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,70 @@
1
+ require 'logger'
2
+ require 'spec_helper'
3
+ require 'sneakers'
4
+ require 'sneakers/runner'
5
+
6
+ describe Sneakers::Runner do
7
+ let(:logger) { Logger.new('log/logtest.log') }
8
+
9
+ describe "with configuration that specifies a logger object" do
10
+ before do
11
+ Sneakers.configure(log: logger)
12
+ @runner = Sneakers::Runner.new([])
13
+ end
14
+
15
+ it 'passes the logger to serverengine' do
16
+ # Stub out ServerEngine::Daemon.run so we only exercise the way we invoke
17
+ # ServerEngine.create
18
+ any_instance_of(ServerEngine::Daemon) do |daemon|
19
+ stub(daemon).main{ return 0 }
20
+ end
21
+
22
+ @runner.run
23
+ # look at @runner's @se instance variable (actually of type Daemon)...and
24
+ # figure out what it's logger is...
25
+ end
26
+ end
27
+ end
28
+
29
+ describe Sneakers::RunnerConfig do
30
+ let(:logger) { Logger.new("log/logtest.log") }
31
+ let(:runner_config) { Sneakers::Runner.new([]).instance_variable_get("@runnerconfig") }
32
+
33
+ describe "with a connection" do
34
+ let(:connection) { Object.new }
35
+
36
+ before { Sneakers.configure(log: logger, connection: connection) }
37
+
38
+ describe "#reload_config!" do
39
+ it "does not throw exception" do
40
+ runner_config.reload_config!
41
+ end
42
+
43
+ it "must not have :log key" do
44
+ _(runner_config.reload_config!.has_key?(:log)).must_equal false
45
+ end
46
+
47
+ it "must have :logger key as an instance of Logger" do
48
+ _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true
49
+ end
50
+
51
+ it "must have :connection" do
52
+ _(runner_config.reload_config![:connection].is_a?(Object)).must_equal true
53
+ end
54
+ end
55
+ end
56
+
57
+ describe "without a connection" do
58
+ before { Sneakers.configure(log: logger) }
59
+
60
+ describe "#reload_config!" do
61
+ it "must not have :log key" do
62
+ _(runner_config.reload_config!.has_key?(:log)).must_equal false
63
+ end
64
+
65
+ it "must have :logger key as an instance of Logger" do
66
+ _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,77 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ class EnvWorker
5
+ include Sneakers::Worker
6
+ from_queue 'defaults'
7
+
8
+ def work(msg)
9
+ end
10
+ end
11
+
12
+
13
+ describe Sneakers do
14
+ before do
15
+ Sneakers.clear!
16
+ end
17
+
18
+ describe 'self' do
19
+ it 'should have defaults set up' do
20
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
21
+ end
22
+
23
+ it 'should configure itself' do
24
+ Sneakers.configure
25
+ _(Sneakers.logger).wont_be_nil
26
+ _(Sneakers.configured?).must_equal(true)
27
+ end
28
+ end
29
+
30
+ describe '.daemonize!' do
31
+ it 'should set a logger to a default info level and not daemonize' do
32
+ Sneakers.daemonize!
33
+ _(Sneakers::CONFIG[:log]).must_equal('sneakers.log')
34
+ _(Sneakers::CONFIG[:daemonize]).must_equal(true)
35
+ _(Sneakers.logger.level).must_equal(Logger::INFO)
36
+ end
37
+
38
+ it 'should set a logger to a level given that level' do
39
+ Sneakers.daemonize!(Logger::DEBUG)
40
+ _(Sneakers.logger.level).must_equal(Logger::DEBUG)
41
+ end
42
+ end
43
+
44
+
45
+ describe '.clear!' do
46
+ it 'must reset dirty configuration to default' do
47
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
48
+ Sneakers.configure(:log => 'foobar.log')
49
+ _(Sneakers::CONFIG[:log]).must_equal('foobar.log')
50
+ Sneakers.clear!
51
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
52
+ end
53
+ end
54
+
55
+
56
+ describe '#setup_general_logger' do
57
+ let(:logger_class) { ServerEngine::DaemonLogger }
58
+
59
+ it 'should detect a string and configure a logger' do
60
+ Sneakers.configure(:log => 'sneakers.log')
61
+ _(Sneakers.logger.kind_of?(logger_class)).must_equal(true)
62
+ end
63
+
64
+ it 'should detect a file-like thing and configure a logger' do
65
+ Sneakers.configure(:log => STDOUT)
66
+ _(Sneakers.logger.kind_of?(logger_class)).must_equal(true)
67
+ end
68
+
69
+ it 'should detect an actual logger and configure it' do
70
+ logger = Logger.new(STDOUT)
71
+ Sneakers.configure(:log => logger)
72
+ _(Sneakers.logger).must_equal(logger)
73
+ end
74
+ end
75
+
76
+ end
77
+
@@ -0,0 +1,44 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ describe Sneakers::Utils do
5
+ describe '::parse_workers' do
6
+ before(:all) do
7
+ class Foo; end
8
+ class Bar; end
9
+ class Baz
10
+ class Quux; end
11
+ class Corge; end
12
+ end
13
+ end
14
+
15
+ describe 'given a single class name' do
16
+ describe 'without namespace' do
17
+ it 'returns the worker class name' do
18
+ _(Sneakers::Utils.parse_workers('Foo')).must_equal([[Foo],[]])
19
+ end
20
+ end
21
+
22
+ describe 'with namespace' do
23
+ it 'returns the worker class name' do
24
+ _(Sneakers::Utils.parse_workers('Baz::Quux')).must_equal([[Baz::Quux],[]])
25
+ end
26
+ end
27
+ end
28
+
29
+ describe 'given a list of class names' do
30
+ describe 'without namespaces' do
31
+ it 'returns all worker class names' do
32
+ _(Sneakers::Utils.parse_workers('Foo,Bar')).must_equal([[Foo,Bar],[]])
33
+ end
34
+ end
35
+
36
+ describe 'with namespaces' do
37
+ it 'returns all worker class names' do
38
+ workers = Sneakers::Utils.parse_workers('Baz::Quux,Baz::Corge')
39
+ _(workers).must_equal([[Baz::Quux,Baz::Corge],[]])
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,115 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+ require 'rake'
4
+ require 'sneakers/tasks'
5
+
6
+ describe 'Worker classes run by rake sneakers:run' do
7
+ class TestWorker
8
+ include Sneakers::Worker
9
+ end
10
+ class TestClass1 < TestWorker; end
11
+ class TestClass2 < TestWorker; end
12
+
13
+ def with_workers_env(workers)
14
+ undefine, restore = if ENV.key?('WORKERS')
15
+ [false, ENV['WORKERS']]
16
+ else
17
+ true
18
+ end
19
+ ENV['WORKERS'] = workers
20
+ yield
21
+ ensure
22
+ undefine ? ENV.delete('WORKERS') : ENV['WORKERS'] = restore
23
+ end
24
+
25
+ def with_rake_worker_classes(workers)
26
+ restore = Sneakers.rake_worker_classes
27
+ Sneakers.rake_worker_classes = workers
28
+ yield
29
+ ensure
30
+ Sneakers.rake_worker_classes = restore
31
+ end
32
+
33
+ def with_sneakers_worker_classes_reset
34
+ restore = Sneakers::Worker::Classes.clone
35
+ Sneakers::Worker::Classes.replace([])
36
+ yield
37
+ ensure
38
+ Sneakers::Worker::Classes.replace(restore)
39
+ end
40
+
41
+ let(:opts) { {} }
42
+
43
+ let :runner do
44
+ mock = Minitest::Mock.new
45
+ mock.expect(:run, nil)
46
+ mock.expect(:call, mock, [expected_workers, opts])
47
+ mock
48
+ end
49
+
50
+ let :run_rake_task do
51
+ Rake::Task['sneakers:run'].reenable
52
+ Rake.application.invoke_task 'sneakers:run'
53
+ end
54
+
55
+ describe 'without any settings' do
56
+ let(:expected_workers) { [worker_class] }
57
+ let(:worker_class) { Class.new.tap { |klass| klass.send(:include, Sneakers::Worker) } }
58
+
59
+ it 'runs classes directly including the Worker' do
60
+ with_workers_env(nil) do
61
+ with_sneakers_worker_classes_reset do
62
+ Sneakers::Runner.stub :new, runner do
63
+ run_rake_task
64
+ end
65
+ runner.verify
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ describe 'with rake_worker_classes set' do
72
+ let(:expected_workers) { [TestClass1, TestClass2] }
73
+
74
+ it 'runs the classes from the setting' do
75
+ with_workers_env(nil) do
76
+ with_rake_worker_classes([TestClass1, TestClass2]) do
77
+ Sneakers::Runner.stub :new, runner do
78
+ run_rake_task
79
+ end
80
+ runner.verify
81
+ end
82
+ end
83
+ end
84
+ end
85
+
86
+ describe 'with rake_worker_classes set, overriden by WORKERS env' do
87
+ let(:expected_workers) { [TestClass2] }
88
+
89
+ it 'runs the classes from the setting' do
90
+ with_rake_worker_classes([TestClass1, TestClass2]) do
91
+ with_workers_env('TestClass2') do
92
+ Sneakers::Runner.stub :new, runner do
93
+ run_rake_task
94
+ end
95
+ runner.verify
96
+ end
97
+ end
98
+ end
99
+ end
100
+
101
+ describe 'with rake_worker_classes responding to call' do
102
+ let(:expected_workers) { [TestClass1] }
103
+
104
+ it 'runs the classes from the setting' do
105
+ with_workers_env(nil) do
106
+ with_rake_worker_classes(-> { [TestClass1] }) do
107
+ Sneakers::Runner.stub :new, runner do
108
+ run_rake_task
109
+ end
110
+ runner.verify
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end