sneakers_custom_bunny 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.travis.yml +6 -0
  4. data/CHANGELOG.md +20 -0
  5. data/Gemfile +3 -0
  6. data/Guardfile +8 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +172 -0
  9. data/ROADMAP.md +18 -0
  10. data/Rakefile +11 -0
  11. data/bin/sneakers +5 -0
  12. data/examples/benchmark_worker.rb +20 -0
  13. data/examples/max_retry_handler.rb +78 -0
  14. data/examples/metrics_worker.rb +28 -0
  15. data/examples/newrelic_metrics_worker.rb +40 -0
  16. data/examples/profiling_worker.rb +69 -0
  17. data/examples/sneakers.conf.rb.example +11 -0
  18. data/examples/title_scraper.rb +23 -0
  19. data/examples/workflow_worker.rb +23 -0
  20. data/lib/sneakers.rb +83 -0
  21. data/lib/sneakers/cli.rb +115 -0
  22. data/lib/sneakers/concerns/logging.rb +34 -0
  23. data/lib/sneakers/concerns/metrics.rb +34 -0
  24. data/lib/sneakers/configuration.rb +59 -0
  25. data/lib/sneakers/handlers/maxretry.rb +191 -0
  26. data/lib/sneakers/handlers/oneshot.rb +30 -0
  27. data/lib/sneakers/metrics/logging_metrics.rb +16 -0
  28. data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
  29. data/lib/sneakers/metrics/null_metrics.rb +13 -0
  30. data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
  31. data/lib/sneakers/publisher.rb +34 -0
  32. data/lib/sneakers/queue.rb +65 -0
  33. data/lib/sneakers/runner.rb +82 -0
  34. data/lib/sneakers/spawner.rb +27 -0
  35. data/lib/sneakers/support/production_formatter.rb +11 -0
  36. data/lib/sneakers/support/utils.rb +18 -0
  37. data/lib/sneakers/tasks.rb +34 -0
  38. data/lib/sneakers/version.rb +3 -0
  39. data/lib/sneakers/worker.rb +151 -0
  40. data/lib/sneakers/workergroup.rb +47 -0
  41. data/sneakers.gemspec +35 -0
  42. data/spec/fixtures/require_worker.rb +17 -0
  43. data/spec/sneakers/cli_spec.rb +63 -0
  44. data/spec/sneakers/concerns/logging_spec.rb +39 -0
  45. data/spec/sneakers/concerns/metrics_spec.rb +38 -0
  46. data/spec/sneakers/configuration_spec.rb +75 -0
  47. data/spec/sneakers/publisher_spec.rb +83 -0
  48. data/spec/sneakers/queue_spec.rb +115 -0
  49. data/spec/sneakers/runner_spec.rb +26 -0
  50. data/spec/sneakers/sneakers_spec.rb +75 -0
  51. data/spec/sneakers/support/utils_spec.rb +44 -0
  52. data/spec/sneakers/worker_handlers_spec.rb +390 -0
  53. data/spec/sneakers/worker_spec.rb +463 -0
  54. data/spec/spec_helper.rb +13 -0
  55. metadata +306 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 28b5234a60deb884a5376ceb93a671a113464131
4
+ data.tar.gz: 25a77893cca3406cc82d0a0e54b593152d305b03
5
+ SHA512:
6
+ metadata.gz: 5e0fce7746274d11045da39af007e433cd5034dca1a323a0ac0fa26cd0791c0244c5db865f7cd1f8c94015d0e1f342e8e5b483f40f3e85d0883e74c4242062c1
7
+ data.tar.gz: a7882fb13af0c27005f72c4b5cd3135a2565b2965292eaf98471742c49be1150170578d3c79f37168e09fb6fba82917a51884d470dfb23734ce610d4cffde486
@@ -0,0 +1,9 @@
1
+ sneakers.yaml
2
+ Gemfile.lock
3
+ *.log
4
+ sneakers.pid
5
+ pkg/
6
+ coverage/
7
+ tmp/
8
+ .ruby-version
9
+ .ruby-gemset
@@ -0,0 +1,6 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2
4
+ - 2.1
5
+ - 2.0.0
6
+ - 1.9.3
@@ -0,0 +1,20 @@
1
+ # 0.1.0, Not released yet
2
+
3
+ + Added newrelic stats reporter (@arielze)
4
+ + Added explicit routing_key at queue (@arielze)
5
+ + Depracating self-daemonization (@jondot)
6
+ + updating bunny (@michaelklishin)
7
+ + Fix reference in publisher (@sergei-matheson)
8
+ + Allow binding of multiple routing keys (@SebastianEdwards)
9
+ + added work_with_params for advanced handling of work items with amqp
10
+ headers. (#12)
11
+ + Sneakers.not_environmental! will no disable auto-environment discovery.
12
+ (#15)
13
+
14
+
15
+
16
+
17
+ # 0.0.7
18
+
19
+ + Sneakers core.
20
+
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,8 @@
1
+
2
+ guard :minitest do
3
+ watch(%r{^spec/(.*)_spec\.rb})
4
+ watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
5
+ watch(%r{^spec/spec_helper\.rb}) { 'spec' }
6
+ end
7
+
8
+
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013-2014 Dotan Nahum
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,172 @@
1
+ # Sneakers
2
+
3
+
4
+ ```
5
+ __
6
+ ,--' >
7
+ `=====
8
+
9
+ ```
10
+
11
+
12
+ A high-performance RabbitMQ background processing framework for
13
+ Ruby.
14
+
15
+
16
+ Sneakers is being used in production for both I/O and CPU intensive workloads, and have achieved the goals of high-performance and 0-maintenance, as designed.
17
+
18
+
19
+ Visit the [wiki](https://github.com/jondot/sneakers/wiki) for
20
+ complete docs.
21
+
22
+
23
+ [![Build Status](https://travis-ci.org/jondot/sneakers.svg?branch=master)](https://travis-ci.org/jondot/sneakers)
24
+
25
+
26
+ ## Installation
27
+
28
+ Add this line to your application's Gemfile:
29
+
30
+ gem 'sneakers'
31
+
32
+ And then execute:
33
+
34
+ $ bundle
35
+
36
+ Or install it yourself as:
37
+
38
+ $ gem install sneakers
39
+
40
+
41
+ ## Quick start
42
+
43
+ Set up a Gemfile
44
+
45
+ ```ruby
46
+ source 'https://rubygems.org'
47
+ gem 'sneakers'
48
+ gem 'json'
49
+ gem 'redis'
50
+ ```
51
+
52
+ And a worker
53
+
54
+ ```ruby
55
+ require 'sneakers'
56
+ require 'redis'
57
+ require 'json'
58
+
59
+ $redis = Redis.new
60
+
61
+ class Processor
62
+ include Sneakers::Worker
63
+ from_queue :logs
64
+
65
+
66
+ def work(msg)
67
+ err = JSON.parse(msg)
68
+ if err["type"] == "error"
69
+ $redis.incr "processor:#{err["error"]}"
70
+ end
71
+
72
+ ack!
73
+ end
74
+ end
75
+ ```
76
+
77
+
78
+ As an example, make a message look like this:
79
+ We'll count errors and error types with Redis. Specifically for an error that looks like this:
80
+
81
+ ```javascript
82
+ {
83
+ "type": "error",
84
+ "message": "HALP!",
85
+ "error": "CODE001"
86
+ }
87
+ ```
88
+
89
+
90
+ Let's test it out quickly from the command line:
91
+
92
+
93
+ ```bash
94
+ sneakers work Processor --require boot.rb
95
+ ```
96
+
97
+ We just told Sneakers to spawn a worker named `Processor`, but first `--require` a file that we dedicate to setting up environment, including workers and what-not.
98
+
99
+ If you go to your RabbitMQ admin now, you'll see a new queue named `logs` was created. Push a couple messages, and this is the output you should see at your terminal.
100
+
101
+
102
+ ```
103
+ 2013-10-11T19:26:36Z p-4718 t-ovqgyb31o DEBUG: [worker-logs:1:213mmy][#<Thread:0x007fae6b05cc58>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
104
+ 2013-10-11T19:26:36Z p-4718 t-ovqgyrxu4 INFO: log log
105
+ 2013-10-11T19:26:40Z p-4719 t-ovqgyb364 DEBUG: [worker-logs:1:h23iit][#<Thread:0x007fae6b05cd98>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
106
+ 2013-10-11T19:26:40Z p-4719 t-ovqgyrx8g INFO: log log
107
+ ```
108
+
109
+ And redis will show this:
110
+
111
+
112
+ ```
113
+ ➜ ~ redis-cli monitor
114
+ 1381520329.888581 [0 127.0.0.1:49182] "incr" "processor:CODE001"
115
+ ```
116
+
117
+
118
+ We're basically done with the ceremonies and all is left is to do some real work.
119
+
120
+
121
+
122
+ ### Looking at metrics
123
+
124
+ Let's use the `logging_metrics` provider just for the sake of fun of seeing the metrics as they happen.
125
+
126
+ ```ruby
127
+ # boot.rb
128
+ require 'sneakers'
129
+ require 'redis'
130
+ require 'json'
131
+ require 'sneakers/metrics/logging_metrics'
132
+ Sneakers.configure :metrics => Sneakers::Metrics::LoggingMetrics.new
133
+
134
+ # ... rest of code
135
+ ```
136
+
137
+ Now push a message again and you'll see:
138
+
139
+ ```
140
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.started
141
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: TIME: work.Processor.time 0.00242
142
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.handled.ack
143
+ ```
144
+
145
+ Which increments start + end, and times the work unit.
146
+
147
+
148
+
149
+ From here, you can continue over to the
150
+ [Wiki](https://github.com/jondot/sneakers/wiki)
151
+
152
+ # Contributing
153
+
154
+ Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
155
+
156
+
157
+ ### Thanks:
158
+
159
+ To all Sneakers [Contributors](https://github.com/jondot/sneakers/graphs/contributors) - you make this happen, thanks!
160
+
161
+
162
+
163
+ # Copyright
164
+
165
+ Copyright (c) 2015 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
166
+
167
+
168
+
169
+
170
+
171
+
172
+
@@ -0,0 +1,18 @@
1
+ # Roadmap
2
+
3
+ It is essential to keep a clear roadmap.
4
+ In its core, Sneakers is already production-ready. However, it is vital
5
+ to take in feedback and improve what can be improved :)
6
+
7
+ This document will communicate the stages that the next version is at.
8
+ See `CHANGELOG.md` for new features that are in.
9
+
10
+
11
+ # v0.1.0 (next version)
12
+
13
+ - Taking in community pull requests and feedback
14
+ - Verify production readiness and performance
15
+ - Full performance test suite
16
+
17
+
18
+
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require 'metric_fu'
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new do |t|
6
+ t.libs << "spec"
7
+ t.test_files = FileList['spec/**/*_spec.rb']
8
+ end
9
+
10
+ task default: :test
11
+
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ require 'sneakers'
3
+ require 'sneakers/cli'
4
+
5
+ Sneakers::CLI.start
@@ -0,0 +1,20 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+
4
+ class BenchmarkWorker
5
+ include Sneakers::Worker
6
+ from_queue 'downloads',
7
+ :durable => false,
8
+ :ack => true,
9
+ :threads => 50,
10
+ :prefetch => 50,
11
+ :timeout_job_after => 1,
12
+ :exchange => 'dummy',
13
+ :heartbeat => 5
14
+ def work(msg)
15
+ ack!
16
+ end
17
+ end
18
+
19
+
20
+
@@ -0,0 +1,78 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/handlers/maxretry'
5
+ require 'logger'
6
+
7
+ Sneakers.configure(:handler => Sneakers::Handlers::Maxretry,
8
+ :workers => 1,
9
+ :threads => 1,
10
+ :prefetch => 1,
11
+ :exchange => 'sneakers',
12
+ :exchange_type => 'topic',
13
+ :routing_key => ['#', 'something'],
14
+ :durable => true,
15
+ )
16
+ Sneakers.logger.level = Logger::DEBUG
17
+
18
+ WORKER_OPTIONS = {
19
+ :ack => true,
20
+ :threads => 1,
21
+ :prefetch => 1,
22
+ :timeout_job_after => 60,
23
+ :heartbeat => 5,
24
+ :retry_timeout => 5000
25
+ }
26
+
27
+ # Example of how to write a retry worker. If your rabbit system is empty, then
28
+ # you must run this twice. Once to setup the exchanges, queues and bindings a
29
+ # second time to have the sent message end up on the downloads queue.
30
+ #
31
+ # Run this via:
32
+ # bundle exec ruby examples/max_retry_handler.rb
33
+ #
34
+ class MaxRetryWorker
35
+ include Sneakers::Worker
36
+ from_queue 'downloads',
37
+ WORKER_OPTIONS.merge({
38
+ :arguments => {
39
+ :'x-dead-letter-exchange' => 'downloads-retry'
40
+ },
41
+ })
42
+
43
+ def work(msg)
44
+ logger.info("MaxRetryWorker rejecting msg: #{msg.inspect}")
45
+
46
+ # We always want to reject to see if we do the proper timeout
47
+ reject!
48
+ end
49
+ end
50
+
51
+ # Example of a worker on the same exchange that does not fail, so it should only
52
+ # see the message once.
53
+ class SucceedingWorker
54
+ include Sneakers::Worker
55
+ from_queue 'uploads',
56
+ WORKER_OPTIONS.merge({
57
+ :arguments => {
58
+ :'x-dead-letter-exchange' => 'uploads-retry'
59
+ },
60
+ })
61
+
62
+ def work(msg)
63
+ logger.info("SucceedingWorker succeeding on msg: #{msg.inspect}")
64
+ ack!
65
+ end
66
+ end
67
+
68
+ messages = 1
69
+ puts "feeding messages in"
70
+ messages.times {
71
+ Sneakers.publish(" -- message -- ",
72
+ :to_queue => 'anywhere',
73
+ :persistence => true)
74
+ }
75
+ puts "done"
76
+
77
+ r = Sneakers::Runner.new([MaxRetryWorker, SucceedingWorker])
78
+ r.run
@@ -0,0 +1,28 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/metrics/logging_metrics'
5
+ require 'open-uri'
6
+ require 'nokogiri'
7
+
8
+
9
+ class MetricsWorker
10
+ include Sneakers::Worker
11
+
12
+ from_queue 'downloads'
13
+
14
+ def work(msg)
15
+ doc = Nokogiri::HTML(open(msg))
16
+ logger.info "FOUND <#{doc.css('title').text}>"
17
+ ack!
18
+ end
19
+ end
20
+
21
+
22
+ Sneakers.configure(:metrics => Sneakers::Metrics::LoggingMetrics.new)
23
+ r = Sneakers::Runner.new([ MetricsWorker ])
24
+ r.run
25
+
26
+
27
+
28
+
@@ -0,0 +1,40 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/metrics/newrelic_metrics'
5
+ require 'open-uri'
6
+ require 'nokogiri'
7
+ require 'newrelic_rpm'
8
+
9
+ # With this configuration will send two types of data to newrelic server:
10
+ # 1. Transaction data which you would see under 'Applications'
11
+ # 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts
12
+ #
13
+ # You should have newrelic.yml in the 'config' folder with the proper account settings
14
+
15
+ Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic
16
+ Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new
17
+
18
+ class MetricsWorker
19
+ include Sneakers::Worker
20
+ include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
21
+
22
+ from_queue 'downloads'
23
+
24
+ def work(msg)
25
+ doc = Nokogiri::HTML(open(msg))
26
+ logger.info "FOUND <#{doc.css('title').text}>"
27
+ ack!
28
+ end
29
+
30
+ add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
31
+
32
+ end
33
+
34
+
35
+ r = Sneakers::Runner.new([ MetricsWorker ])
36
+ r.run
37
+
38
+
39
+
40
+