kicks 3.0.0.pre

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +7 -0
  2. data/.github/workflows/ci.yml +24 -0
  3. data/.gitignore +12 -0
  4. data/ChangeLog.md +142 -0
  5. data/Dockerfile +24 -0
  6. data/Dockerfile.slim +20 -0
  7. data/Gemfile +8 -0
  8. data/Guardfile +8 -0
  9. data/LICENSE.txt +22 -0
  10. data/README.md +209 -0
  11. data/Rakefile +12 -0
  12. data/bin/sneakers +6 -0
  13. data/docker-compose.yml +24 -0
  14. data/examples/benchmark_worker.rb +22 -0
  15. data/examples/max_retry_handler.rb +68 -0
  16. data/examples/metrics_worker.rb +34 -0
  17. data/examples/middleware_worker.rb +36 -0
  18. data/examples/newrelic_metrics_worker.rb +40 -0
  19. data/examples/profiling_worker.rb +69 -0
  20. data/examples/sneakers.conf.rb.example +11 -0
  21. data/examples/title_scraper.rb +36 -0
  22. data/examples/workflow_worker.rb +23 -0
  23. data/kicks.gemspec +44 -0
  24. data/lib/sneakers/cli.rb +122 -0
  25. data/lib/sneakers/concerns/logging.rb +34 -0
  26. data/lib/sneakers/concerns/metrics.rb +34 -0
  27. data/lib/sneakers/configuration.rb +125 -0
  28. data/lib/sneakers/content_encoding.rb +47 -0
  29. data/lib/sneakers/content_type.rb +47 -0
  30. data/lib/sneakers/error_reporter.rb +33 -0
  31. data/lib/sneakers/errors.rb +2 -0
  32. data/lib/sneakers/handlers/maxretry.rb +219 -0
  33. data/lib/sneakers/handlers/oneshot.rb +26 -0
  34. data/lib/sneakers/metrics/logging_metrics.rb +16 -0
  35. data/lib/sneakers/metrics/newrelic_metrics.rb +32 -0
  36. data/lib/sneakers/metrics/null_metrics.rb +13 -0
  37. data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
  38. data/lib/sneakers/middleware/config.rb +23 -0
  39. data/lib/sneakers/publisher.rb +49 -0
  40. data/lib/sneakers/queue.rb +87 -0
  41. data/lib/sneakers/runner.rb +91 -0
  42. data/lib/sneakers/spawner.rb +30 -0
  43. data/lib/sneakers/support/production_formatter.rb +11 -0
  44. data/lib/sneakers/support/utils.rb +18 -0
  45. data/lib/sneakers/tasks.rb +66 -0
  46. data/lib/sneakers/version.rb +3 -0
  47. data/lib/sneakers/worker.rb +162 -0
  48. data/lib/sneakers/workergroup.rb +60 -0
  49. data/lib/sneakers.rb +125 -0
  50. data/log/.gitkeep +0 -0
  51. data/scripts/local_integration +2 -0
  52. data/scripts/local_worker +3 -0
  53. data/spec/fixtures/integration_worker.rb +18 -0
  54. data/spec/fixtures/require_worker.rb +23 -0
  55. data/spec/gzip_helper.rb +15 -0
  56. data/spec/sneakers/cli_spec.rb +75 -0
  57. data/spec/sneakers/concerns/logging_spec.rb +39 -0
  58. data/spec/sneakers/concerns/metrics_spec.rb +38 -0
  59. data/spec/sneakers/configuration_spec.rb +97 -0
  60. data/spec/sneakers/content_encoding_spec.rb +81 -0
  61. data/spec/sneakers/content_type_spec.rb +81 -0
  62. data/spec/sneakers/integration_spec.rb +158 -0
  63. data/spec/sneakers/publisher_spec.rb +179 -0
  64. data/spec/sneakers/queue_spec.rb +169 -0
  65. data/spec/sneakers/runner_spec.rb +70 -0
  66. data/spec/sneakers/sneakers_spec.rb +77 -0
  67. data/spec/sneakers/support/utils_spec.rb +44 -0
  68. data/spec/sneakers/tasks/sneakers_run_spec.rb +115 -0
  69. data/spec/sneakers/worker_handlers_spec.rb +469 -0
  70. data/spec/sneakers/worker_spec.rb +712 -0
  71. data/spec/sneakers/workergroup_spec.rb +83 -0
  72. data/spec/spec_helper.rb +21 -0
  73. metadata +352 -0
@@ -0,0 +1,169 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ describe Sneakers::Queue do
5
+ let :queue_vars do
6
+ {
7
+ :prefetch => 25,
8
+ :ack => true,
9
+ :heartbeat => 2,
10
+ :vhost => '/',
11
+ :exchange => "sneakers",
12
+ :exchange_options => {
13
+ :type => :direct,
14
+ durable: true,
15
+ :arguments => { 'x-arg' => 'value' }
16
+ },
17
+ queue_options: {
18
+ durable: true
19
+ }
20
+ }
21
+ end
22
+
23
+ before do
24
+ Sneakers.clear!
25
+ Sneakers.configure
26
+
27
+ @mkworker = Object.new
28
+ stub(@mkworker).opts { { :exchange => 'test-exchange' } }
29
+ @mkchan = Object.new
30
+ mock(@mkchan).prefetch(25)
31
+ @mkex = Object.new
32
+ @mkqueue = Object.new
33
+ end
34
+
35
+ describe 'with our own Bunny object' do
36
+ before do
37
+ @mkbunny = Object.new
38
+ @mkqueue_nondurable = Object.new
39
+
40
+ mock(@mkbunny).start {}
41
+ mock(@mkbunny).create_channel{ @mkchan }
42
+ mock(Bunny).new(
43
+ anything,
44
+ hash_including(:vhost => '/', :heartbeat => 2)
45
+ ){ @mkbunny }
46
+ end
47
+
48
+ describe "#subscribe with sneakers exchange" do
49
+ before do
50
+ mock(@mkchan).exchange("sneakers",
51
+ :type => :direct,
52
+ :durable => true,
53
+ :arguments => { 'x-arg' => 'value' }){ @mkex }
54
+ end
55
+
56
+ it "should setup a bunny queue according to configuration values" do
57
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
58
+ q = Sneakers::Queue.new("downloads", queue_vars)
59
+
60
+ mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
61
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
62
+
63
+ q.subscribe(@mkworker)
64
+ end
65
+
66
+ it "supports multiple routing_keys" do
67
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
68
+ q = Sneakers::Queue.new("downloads",
69
+ queue_vars.merge(:routing_key => ["alpha", "beta"]))
70
+
71
+ mock(@mkqueue).bind(@mkex, :routing_key => "alpha")
72
+ mock(@mkqueue).bind(@mkex, :routing_key => "beta")
73
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
74
+
75
+ q.subscribe(@mkworker)
76
+ end
77
+
78
+ it "supports setting arguments when binding" do
79
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
80
+ q = Sneakers::Queue.new("downloads",
81
+ queue_vars.merge(:bind_arguments => { "os" => "linux", "cores" => 8 }))
82
+
83
+ mock(@mkqueue).bind(@mkex, :routing_key => "downloads", :arguments => { "os" => "linux", "cores" => 8 })
84
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
85
+
86
+ q.subscribe(@mkworker)
87
+ end
88
+
89
+ it "will use whatever handler the worker specifies" do
90
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
91
+ @handler = Object.new
92
+ worker_opts = { :handler => @handler }
93
+ stub(@mkworker).opts { worker_opts }
94
+ mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
95
+
96
+ stub(@mkqueue).bind
97
+ stub(@mkqueue).subscribe
98
+ q = Sneakers::Queue.new("downloads", queue_vars)
99
+ q.subscribe(@mkworker)
100
+ end
101
+
102
+ it "creates a non-durable queue if :queue_durable => false" do
103
+ mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
104
+ queue_vars[:queue_options][:durable] = false
105
+ q = Sneakers::Queue.new("test_nondurable", queue_vars)
106
+
107
+ mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
108
+ mock(@mkqueue_nondurable).subscribe(:block => false, :manual_ack => true)
109
+
110
+ q.subscribe(@mkworker)
111
+ myqueue = q.instance_variable_get(:@queue)
112
+ end
113
+ end
114
+
115
+ describe "#subscribe with default exchange" do
116
+ before do
117
+ # expect default exchange
118
+ queue_vars[:exchange] = ""
119
+ mock(@mkchan).exchange("",
120
+ :type => :direct,
121
+ :durable => true,
122
+ :arguments => {"x-arg" => "value"}){ @mkex }
123
+ end
124
+
125
+ it "does not bind to exchange" do
126
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
127
+ @handler = Object.new
128
+ worker_opts = { :handler => @handler }
129
+ stub(@mkworker).opts { worker_opts }
130
+ mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
131
+
132
+ stub(@mkqueue).bind do
133
+ raise "bind should not be called"
134
+ end
135
+
136
+ stub(@mkqueue).subscribe
137
+ q = Sneakers::Queue.new("downloads", queue_vars)
138
+ q.subscribe(@mkworker)
139
+ end
140
+ end
141
+ end
142
+
143
+ describe 'with an externally-provided connection' do
144
+ describe '#subscribe' do
145
+ before do
146
+ @external_connection = Bunny.new
147
+ mock(@external_connection).start {}
148
+ mock(@external_connection).create_channel{ @mkchan }
149
+ mock(@mkchan).exchange("sneakers",
150
+ :type => :direct,
151
+ :durable => true,
152
+ :arguments => { 'x-arg' => 'value' }){ @mkex }
153
+
154
+ queue_name = 'foo'
155
+ mock(@mkchan).queue(queue_name, :durable => true) { @mkqueue }
156
+ mock(@mkqueue).bind(@mkex, :routing_key => queue_name)
157
+ mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
158
+
159
+ my_vars = queue_vars.merge(:connection => @external_connection)
160
+ @q = Sneakers::Queue.new(queue_name, my_vars)
161
+ end
162
+
163
+ it 'uses that object' do
164
+ @q.subscribe(@mkworker)
165
+ _(@q.instance_variable_get(:@bunny)).must_equal @external_connection
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,70 @@
1
+ require 'logger'
2
+ require 'spec_helper'
3
+ require 'sneakers'
4
+ require 'sneakers/runner'
5
+
6
+ describe Sneakers::Runner do
7
+ let(:logger) { Logger.new('log/logtest.log') }
8
+
9
+ describe "with configuration that specifies a logger object" do
10
+ before do
11
+ Sneakers.configure(log: logger)
12
+ @runner = Sneakers::Runner.new([])
13
+ end
14
+
15
+ it 'passes the logger to serverengine' do
16
+ # Stub out ServerEngine::Daemon.run so we only exercise the way we invoke
17
+ # ServerEngine.create
18
+ any_instance_of(ServerEngine::Daemon) do |daemon|
19
+ stub(daemon).main{ return 0 }
20
+ end
21
+
22
+ @runner.run
23
+ # look at @runner's @se instance variable (actually of type Daemon)...and
24
+ # figure out what it's logger is...
25
+ end
26
+ end
27
+ end
28
+
29
+ describe Sneakers::RunnerConfig do
30
+ let(:logger) { Logger.new("log/logtest.log") }
31
+ let(:runner_config) { Sneakers::Runner.new([]).instance_variable_get("@runnerconfig") }
32
+
33
+ describe "with a connection" do
34
+ let(:connection) { Object.new }
35
+
36
+ before { Sneakers.configure(log: logger, connection: connection) }
37
+
38
+ describe "#reload_config!" do
39
+ it "does not throw exception" do
40
+ runner_config.reload_config!
41
+ end
42
+
43
+ it "must not have :log key" do
44
+ _(runner_config.reload_config!.has_key?(:log)).must_equal false
45
+ end
46
+
47
+ it "must have :logger key as an instance of Logger" do
48
+ _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true
49
+ end
50
+
51
+ it "must have :connection" do
52
+ _(runner_config.reload_config![:connection].is_a?(Object)).must_equal true
53
+ end
54
+ end
55
+ end
56
+
57
+ describe "without a connection" do
58
+ before { Sneakers.configure(log: logger) }
59
+
60
+ describe "#reload_config!" do
61
+ it "must not have :log key" do
62
+ _(runner_config.reload_config!.has_key?(:log)).must_equal false
63
+ end
64
+
65
+ it "must have :logger key as an instance of Logger" do
66
+ _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,77 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ class EnvWorker
5
+ include Sneakers::Worker
6
+ from_queue 'defaults'
7
+
8
+ def work(msg)
9
+ end
10
+ end
11
+
12
+
13
+ describe Sneakers do
14
+ before do
15
+ Sneakers.clear!
16
+ end
17
+
18
+ describe 'self' do
19
+ it 'should have defaults set up' do
20
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
21
+ end
22
+
23
+ it 'should configure itself' do
24
+ Sneakers.configure
25
+ _(Sneakers.logger).wont_be_nil
26
+ _(Sneakers.configured?).must_equal(true)
27
+ end
28
+ end
29
+
30
+ describe '.daemonize!' do
31
+ it 'should set a logger to a default info level and not daemonize' do
32
+ Sneakers.daemonize!
33
+ _(Sneakers::CONFIG[:log]).must_equal('sneakers.log')
34
+ _(Sneakers::CONFIG[:daemonize]).must_equal(true)
35
+ _(Sneakers.logger.level).must_equal(Logger::INFO)
36
+ end
37
+
38
+ it 'should set a logger to a level given that level' do
39
+ Sneakers.daemonize!(Logger::DEBUG)
40
+ _(Sneakers.logger.level).must_equal(Logger::DEBUG)
41
+ end
42
+ end
43
+
44
+
45
+ describe '.clear!' do
46
+ it 'must reset dirty configuration to default' do
47
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
48
+ Sneakers.configure(:log => 'foobar.log')
49
+ _(Sneakers::CONFIG[:log]).must_equal('foobar.log')
50
+ Sneakers.clear!
51
+ _(Sneakers::CONFIG[:log]).must_equal(STDOUT)
52
+ end
53
+ end
54
+
55
+
56
+ describe '#setup_general_logger' do
57
+ let(:logger_class) { ServerEngine::DaemonLogger }
58
+
59
+ it 'should detect a string and configure a logger' do
60
+ Sneakers.configure(:log => 'sneakers.log')
61
+ _(Sneakers.logger.kind_of?(logger_class)).must_equal(true)
62
+ end
63
+
64
+ it 'should detect a file-like thing and configure a logger' do
65
+ Sneakers.configure(:log => STDOUT)
66
+ _(Sneakers.logger.kind_of?(logger_class)).must_equal(true)
67
+ end
68
+
69
+ it 'should detect an actual logger and configure it' do
70
+ logger = Logger.new(STDOUT)
71
+ Sneakers.configure(:log => logger)
72
+ _(Sneakers.logger).must_equal(logger)
73
+ end
74
+ end
75
+
76
+ end
77
+
@@ -0,0 +1,44 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+
4
+ describe Sneakers::Utils do
5
+ describe '::parse_workers' do
6
+ before(:all) do
7
+ class Foo; end
8
+ class Bar; end
9
+ class Baz
10
+ class Quux; end
11
+ class Corge; end
12
+ end
13
+ end
14
+
15
+ describe 'given a single class name' do
16
+ describe 'without namespace' do
17
+ it 'returns the worker class name' do
18
+ _(Sneakers::Utils.parse_workers('Foo')).must_equal([[Foo],[]])
19
+ end
20
+ end
21
+
22
+ describe 'with namespace' do
23
+ it 'returns the worker class name' do
24
+ _(Sneakers::Utils.parse_workers('Baz::Quux')).must_equal([[Baz::Quux],[]])
25
+ end
26
+ end
27
+ end
28
+
29
+ describe 'given a list of class names' do
30
+ describe 'without namespaces' do
31
+ it 'returns all worker class names' do
32
+ _(Sneakers::Utils.parse_workers('Foo,Bar')).must_equal([[Foo,Bar],[]])
33
+ end
34
+ end
35
+
36
+ describe 'with namespaces' do
37
+ it 'returns all worker class names' do
38
+ workers = Sneakers::Utils.parse_workers('Baz::Quux,Baz::Corge')
39
+ _(workers).must_equal([[Baz::Quux,Baz::Corge],[]])
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,115 @@
1
+ require 'spec_helper'
2
+ require 'sneakers'
3
+ require 'rake'
4
+ require 'sneakers/tasks'
5
+
6
+ describe 'Worker classes run by rake sneakers:run' do
7
+ class TestWorker
8
+ include Sneakers::Worker
9
+ end
10
+ class TestClass1 < TestWorker; end
11
+ class TestClass2 < TestWorker; end
12
+
13
+ def with_workers_env(workers)
14
+ undefine, restore = if ENV.key?('WORKERS')
15
+ [false, ENV['WORKERS']]
16
+ else
17
+ true
18
+ end
19
+ ENV['WORKERS'] = workers
20
+ yield
21
+ ensure
22
+ undefine ? ENV.delete('WORKERS') : ENV['WORKERS'] = restore
23
+ end
24
+
25
+ def with_rake_worker_classes(workers)
26
+ restore = Sneakers.rake_worker_classes
27
+ Sneakers.rake_worker_classes = workers
28
+ yield
29
+ ensure
30
+ Sneakers.rake_worker_classes = restore
31
+ end
32
+
33
+ def with_sneakers_worker_classes_reset
34
+ restore = Sneakers::Worker::Classes.clone
35
+ Sneakers::Worker::Classes.replace([])
36
+ yield
37
+ ensure
38
+ Sneakers::Worker::Classes.replace(restore)
39
+ end
40
+
41
+ let(:opts) { {} }
42
+
43
+ let :runner do
44
+ mock = Minitest::Mock.new
45
+ mock.expect(:run, nil)
46
+ mock.expect(:call, mock, [expected_workers, opts])
47
+ mock
48
+ end
49
+
50
+ let :run_rake_task do
51
+ Rake::Task['sneakers:run'].reenable
52
+ Rake.application.invoke_task 'sneakers:run'
53
+ end
54
+
55
+ describe 'without any settings' do
56
+ let(:expected_workers) { [worker_class] }
57
+ let(:worker_class) { Class.new.tap { |klass| klass.send(:include, Sneakers::Worker) } }
58
+
59
+ it 'runs classes directly including the Worker' do
60
+ with_workers_env(nil) do
61
+ with_sneakers_worker_classes_reset do
62
+ Sneakers::Runner.stub :new, runner do
63
+ run_rake_task
64
+ end
65
+ runner.verify
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ describe 'with rake_worker_classes set' do
72
+ let(:expected_workers) { [TestClass1, TestClass2] }
73
+
74
+ it 'runs the classes from the setting' do
75
+ with_workers_env(nil) do
76
+ with_rake_worker_classes([TestClass1, TestClass2]) do
77
+ Sneakers::Runner.stub :new, runner do
78
+ run_rake_task
79
+ end
80
+ runner.verify
81
+ end
82
+ end
83
+ end
84
+ end
85
+
86
+ describe 'with rake_worker_classes set, overriden by WORKERS env' do
87
+ let(:expected_workers) { [TestClass2] }
88
+
89
+ it 'runs the classes from the setting' do
90
+ with_rake_worker_classes([TestClass1, TestClass2]) do
91
+ with_workers_env('TestClass2') do
92
+ Sneakers::Runner.stub :new, runner do
93
+ run_rake_task
94
+ end
95
+ runner.verify
96
+ end
97
+ end
98
+ end
99
+ end
100
+
101
+ describe 'with rake_worker_classes responding to call' do
102
+ let(:expected_workers) { [TestClass1] }
103
+
104
+ it 'runs the classes from the setting' do
105
+ with_workers_env(nil) do
106
+ with_rake_worker_classes(-> { [TestClass1] }) do
107
+ Sneakers::Runner.stub :new, runner do
108
+ run_rake_task
109
+ end
110
+ runner.verify
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end