sneakers_custom_bunny 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +20 -0
- data/Gemfile +3 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +172 -0
- data/ROADMAP.md +18 -0
- data/Rakefile +11 -0
- data/bin/sneakers +5 -0
- data/examples/benchmark_worker.rb +20 -0
- data/examples/max_retry_handler.rb +78 -0
- data/examples/metrics_worker.rb +28 -0
- data/examples/newrelic_metrics_worker.rb +40 -0
- data/examples/profiling_worker.rb +69 -0
- data/examples/sneakers.conf.rb.example +11 -0
- data/examples/title_scraper.rb +23 -0
- data/examples/workflow_worker.rb +23 -0
- data/lib/sneakers.rb +83 -0
- data/lib/sneakers/cli.rb +115 -0
- data/lib/sneakers/concerns/logging.rb +34 -0
- data/lib/sneakers/concerns/metrics.rb +34 -0
- data/lib/sneakers/configuration.rb +59 -0
- data/lib/sneakers/handlers/maxretry.rb +191 -0
- data/lib/sneakers/handlers/oneshot.rb +30 -0
- data/lib/sneakers/metrics/logging_metrics.rb +16 -0
- data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
- data/lib/sneakers/metrics/null_metrics.rb +13 -0
- data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
- data/lib/sneakers/publisher.rb +34 -0
- data/lib/sneakers/queue.rb +65 -0
- data/lib/sneakers/runner.rb +82 -0
- data/lib/sneakers/spawner.rb +27 -0
- data/lib/sneakers/support/production_formatter.rb +11 -0
- data/lib/sneakers/support/utils.rb +18 -0
- data/lib/sneakers/tasks.rb +34 -0
- data/lib/sneakers/version.rb +3 -0
- data/lib/sneakers/worker.rb +151 -0
- data/lib/sneakers/workergroup.rb +47 -0
- data/sneakers.gemspec +35 -0
- data/spec/fixtures/require_worker.rb +17 -0
- data/spec/sneakers/cli_spec.rb +63 -0
- data/spec/sneakers/concerns/logging_spec.rb +39 -0
- data/spec/sneakers/concerns/metrics_spec.rb +38 -0
- data/spec/sneakers/configuration_spec.rb +75 -0
- data/spec/sneakers/publisher_spec.rb +83 -0
- data/spec/sneakers/queue_spec.rb +115 -0
- data/spec/sneakers/runner_spec.rb +26 -0
- data/spec/sneakers/sneakers_spec.rb +75 -0
- data/spec/sneakers/support/utils_spec.rb +44 -0
- data/spec/sneakers/worker_handlers_spec.rb +390 -0
- data/spec/sneakers/worker_spec.rb +463 -0
- data/spec/spec_helper.rb +13 -0
- metadata +306 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 28b5234a60deb884a5376ceb93a671a113464131
|
4
|
+
data.tar.gz: 25a77893cca3406cc82d0a0e54b593152d305b03
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 5e0fce7746274d11045da39af007e433cd5034dca1a323a0ac0fa26cd0791c0244c5db865f7cd1f8c94015d0e1f342e8e5b483f40f3e85d0883e74c4242062c1
|
7
|
+
data.tar.gz: a7882fb13af0c27005f72c4b5cd3135a2565b2965292eaf98471742c49be1150170578d3c79f37168e09fb6fba82917a51884d470dfb23734ce610d4cffde486
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# 0.1.0, Not released yet
|
2
|
+
|
3
|
+
+ Added newrelic stats reporter (@arielze)
|
4
|
+
+ Added explicit routing_key at queue (@arielze)
|
5
|
+
+ Depracating self-daemonization (@jondot)
|
6
|
+
+ updating bunny (@michaelklishin)
|
7
|
+
+ Fix reference in publisher (@sergei-matheson)
|
8
|
+
+ Allow binding of multiple routing keys (@SebastianEdwards)
|
9
|
+
+ added work_with_params for advanced handling of work items with amqp
|
10
|
+
headers. (#12)
|
11
|
+
+ Sneakers.not_environmental! will no disable auto-environment discovery.
|
12
|
+
(#15)
|
13
|
+
|
14
|
+
|
15
|
+
|
16
|
+
|
17
|
+
# 0.0.7
|
18
|
+
|
19
|
+
+ Sneakers core.
|
20
|
+
|
data/Gemfile
ADDED
data/Guardfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2013-2014 Dotan Nahum
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,172 @@
|
|
1
|
+
# Sneakers
|
2
|
+
|
3
|
+
|
4
|
+
```
|
5
|
+
__
|
6
|
+
,--' >
|
7
|
+
`=====
|
8
|
+
|
9
|
+
```
|
10
|
+
|
11
|
+
|
12
|
+
A high-performance RabbitMQ background processing framework for
|
13
|
+
Ruby.
|
14
|
+
|
15
|
+
|
16
|
+
Sneakers is being used in production for both I/O and CPU intensive workloads, and have achieved the goals of high-performance and 0-maintenance, as designed.
|
17
|
+
|
18
|
+
|
19
|
+
Visit the [wiki](https://github.com/jondot/sneakers/wiki) for
|
20
|
+
complete docs.
|
21
|
+
|
22
|
+
|
23
|
+
[](https://travis-ci.org/jondot/sneakers)
|
24
|
+
|
25
|
+
|
26
|
+
## Installation
|
27
|
+
|
28
|
+
Add this line to your application's Gemfile:
|
29
|
+
|
30
|
+
gem 'sneakers'
|
31
|
+
|
32
|
+
And then execute:
|
33
|
+
|
34
|
+
$ bundle
|
35
|
+
|
36
|
+
Or install it yourself as:
|
37
|
+
|
38
|
+
$ gem install sneakers
|
39
|
+
|
40
|
+
|
41
|
+
## Quick start
|
42
|
+
|
43
|
+
Set up a Gemfile
|
44
|
+
|
45
|
+
```ruby
|
46
|
+
source 'https://rubygems.org'
|
47
|
+
gem 'sneakers'
|
48
|
+
gem 'json'
|
49
|
+
gem 'redis'
|
50
|
+
```
|
51
|
+
|
52
|
+
And a worker
|
53
|
+
|
54
|
+
```ruby
|
55
|
+
require 'sneakers'
|
56
|
+
require 'redis'
|
57
|
+
require 'json'
|
58
|
+
|
59
|
+
$redis = Redis.new
|
60
|
+
|
61
|
+
class Processor
|
62
|
+
include Sneakers::Worker
|
63
|
+
from_queue :logs
|
64
|
+
|
65
|
+
|
66
|
+
def work(msg)
|
67
|
+
err = JSON.parse(msg)
|
68
|
+
if err["type"] == "error"
|
69
|
+
$redis.incr "processor:#{err["error"]}"
|
70
|
+
end
|
71
|
+
|
72
|
+
ack!
|
73
|
+
end
|
74
|
+
end
|
75
|
+
```
|
76
|
+
|
77
|
+
|
78
|
+
As an example, make a message look like this:
|
79
|
+
We'll count errors and error types with Redis. Specifically for an error that looks like this:
|
80
|
+
|
81
|
+
```javascript
|
82
|
+
{
|
83
|
+
"type": "error",
|
84
|
+
"message": "HALP!",
|
85
|
+
"error": "CODE001"
|
86
|
+
}
|
87
|
+
```
|
88
|
+
|
89
|
+
|
90
|
+
Let's test it out quickly from the command line:
|
91
|
+
|
92
|
+
|
93
|
+
```bash
|
94
|
+
sneakers work Processor --require boot.rb
|
95
|
+
```
|
96
|
+
|
97
|
+
We just told Sneakers to spawn a worker named `Processor`, but first `--require` a file that we dedicate to setting up environment, including workers and what-not.
|
98
|
+
|
99
|
+
If you go to your RabbitMQ admin now, you'll see a new queue named `logs` was created. Push a couple messages, and this is the output you should see at your terminal.
|
100
|
+
|
101
|
+
|
102
|
+
```
|
103
|
+
2013-10-11T19:26:36Z p-4718 t-ovqgyb31o DEBUG: [worker-logs:1:213mmy][#<Thread:0x007fae6b05cc58>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
|
104
|
+
2013-10-11T19:26:36Z p-4718 t-ovqgyrxu4 INFO: log log
|
105
|
+
2013-10-11T19:26:40Z p-4719 t-ovqgyb364 DEBUG: [worker-logs:1:h23iit][#<Thread:0x007fae6b05cd98>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
|
106
|
+
2013-10-11T19:26:40Z p-4719 t-ovqgyrx8g INFO: log log
|
107
|
+
```
|
108
|
+
|
109
|
+
And redis will show this:
|
110
|
+
|
111
|
+
|
112
|
+
```
|
113
|
+
➜ ~ redis-cli monitor
|
114
|
+
1381520329.888581 [0 127.0.0.1:49182] "incr" "processor:CODE001"
|
115
|
+
```
|
116
|
+
|
117
|
+
|
118
|
+
We're basically done with the ceremonies and all is left is to do some real work.
|
119
|
+
|
120
|
+
|
121
|
+
|
122
|
+
### Looking at metrics
|
123
|
+
|
124
|
+
Let's use the `logging_metrics` provider just for the sake of fun of seeing the metrics as they happen.
|
125
|
+
|
126
|
+
```ruby
|
127
|
+
# boot.rb
|
128
|
+
require 'sneakers'
|
129
|
+
require 'redis'
|
130
|
+
require 'json'
|
131
|
+
require 'sneakers/metrics/logging_metrics'
|
132
|
+
Sneakers.configure :metrics => Sneakers::Metrics::LoggingMetrics.new
|
133
|
+
|
134
|
+
# ... rest of code
|
135
|
+
```
|
136
|
+
|
137
|
+
Now push a message again and you'll see:
|
138
|
+
|
139
|
+
```
|
140
|
+
2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.started
|
141
|
+
2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: TIME: work.Processor.time 0.00242
|
142
|
+
2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.handled.ack
|
143
|
+
```
|
144
|
+
|
145
|
+
Which increments start + end, and times the work unit.
|
146
|
+
|
147
|
+
|
148
|
+
|
149
|
+
From here, you can continue over to the
|
150
|
+
[Wiki](https://github.com/jondot/sneakers/wiki)
|
151
|
+
|
152
|
+
# Contributing
|
153
|
+
|
154
|
+
Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
|
155
|
+
|
156
|
+
|
157
|
+
### Thanks:
|
158
|
+
|
159
|
+
To all Sneakers [Contributors](https://github.com/jondot/sneakers/graphs/contributors) - you make this happen, thanks!
|
160
|
+
|
161
|
+
|
162
|
+
|
163
|
+
# Copyright
|
164
|
+
|
165
|
+
Copyright (c) 2015 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
|
166
|
+
|
167
|
+
|
168
|
+
|
169
|
+
|
170
|
+
|
171
|
+
|
172
|
+
|
data/ROADMAP.md
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# Roadmap
|
2
|
+
|
3
|
+
It is essential to keep a clear roadmap.
|
4
|
+
In its core, Sneakers is already production-ready. However, it is vital
|
5
|
+
to take in feedback and improve what can be improved :)
|
6
|
+
|
7
|
+
This document will communicate the stages that the next version is at.
|
8
|
+
See `CHANGELOG.md` for new features that are in.
|
9
|
+
|
10
|
+
|
11
|
+
# v0.1.0 (next version)
|
12
|
+
|
13
|
+
- Taking in community pull requests and feedback
|
14
|
+
- Verify production readiness and performance
|
15
|
+
- Full performance test suite
|
16
|
+
|
17
|
+
|
18
|
+
|
data/Rakefile
ADDED
data/bin/sneakers
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
$: << File.expand_path('../lib', File.dirname(__FILE__))
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
class BenchmarkWorker
|
5
|
+
include Sneakers::Worker
|
6
|
+
from_queue 'downloads',
|
7
|
+
:durable => false,
|
8
|
+
:ack => true,
|
9
|
+
:threads => 50,
|
10
|
+
:prefetch => 50,
|
11
|
+
:timeout_job_after => 1,
|
12
|
+
:exchange => 'dummy',
|
13
|
+
:heartbeat => 5
|
14
|
+
def work(msg)
|
15
|
+
ack!
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
|
20
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
$: << File.expand_path('../lib', File.dirname(__FILE__))
|
2
|
+
require 'sneakers'
|
3
|
+
require 'sneakers/runner'
|
4
|
+
require 'sneakers/handlers/maxretry'
|
5
|
+
require 'logger'
|
6
|
+
|
7
|
+
Sneakers.configure(:handler => Sneakers::Handlers::Maxretry,
|
8
|
+
:workers => 1,
|
9
|
+
:threads => 1,
|
10
|
+
:prefetch => 1,
|
11
|
+
:exchange => 'sneakers',
|
12
|
+
:exchange_type => 'topic',
|
13
|
+
:routing_key => ['#', 'something'],
|
14
|
+
:durable => true,
|
15
|
+
)
|
16
|
+
Sneakers.logger.level = Logger::DEBUG
|
17
|
+
|
18
|
+
WORKER_OPTIONS = {
|
19
|
+
:ack => true,
|
20
|
+
:threads => 1,
|
21
|
+
:prefetch => 1,
|
22
|
+
:timeout_job_after => 60,
|
23
|
+
:heartbeat => 5,
|
24
|
+
:retry_timeout => 5000
|
25
|
+
}
|
26
|
+
|
27
|
+
# Example of how to write a retry worker. If your rabbit system is empty, then
|
28
|
+
# you must run this twice. Once to setup the exchanges, queues and bindings a
|
29
|
+
# second time to have the sent message end up on the downloads queue.
|
30
|
+
#
|
31
|
+
# Run this via:
|
32
|
+
# bundle exec ruby examples/max_retry_handler.rb
|
33
|
+
#
|
34
|
+
class MaxRetryWorker
|
35
|
+
include Sneakers::Worker
|
36
|
+
from_queue 'downloads',
|
37
|
+
WORKER_OPTIONS.merge({
|
38
|
+
:arguments => {
|
39
|
+
:'x-dead-letter-exchange' => 'downloads-retry'
|
40
|
+
},
|
41
|
+
})
|
42
|
+
|
43
|
+
def work(msg)
|
44
|
+
logger.info("MaxRetryWorker rejecting msg: #{msg.inspect}")
|
45
|
+
|
46
|
+
# We always want to reject to see if we do the proper timeout
|
47
|
+
reject!
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Example of a worker on the same exchange that does not fail, so it should only
|
52
|
+
# see the message once.
|
53
|
+
class SucceedingWorker
|
54
|
+
include Sneakers::Worker
|
55
|
+
from_queue 'uploads',
|
56
|
+
WORKER_OPTIONS.merge({
|
57
|
+
:arguments => {
|
58
|
+
:'x-dead-letter-exchange' => 'uploads-retry'
|
59
|
+
},
|
60
|
+
})
|
61
|
+
|
62
|
+
def work(msg)
|
63
|
+
logger.info("SucceedingWorker succeeding on msg: #{msg.inspect}")
|
64
|
+
ack!
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
messages = 1
|
69
|
+
puts "feeding messages in"
|
70
|
+
messages.times {
|
71
|
+
Sneakers.publish(" -- message -- ",
|
72
|
+
:to_queue => 'anywhere',
|
73
|
+
:persistence => true)
|
74
|
+
}
|
75
|
+
puts "done"
|
76
|
+
|
77
|
+
r = Sneakers::Runner.new([MaxRetryWorker, SucceedingWorker])
|
78
|
+
r.run
|
@@ -0,0 +1,28 @@
|
|
1
|
+
$: << File.expand_path('../lib', File.dirname(__FILE__))
|
2
|
+
require 'sneakers'
|
3
|
+
require 'sneakers/runner'
|
4
|
+
require 'sneakers/metrics/logging_metrics'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'nokogiri'
|
7
|
+
|
8
|
+
|
9
|
+
class MetricsWorker
|
10
|
+
include Sneakers::Worker
|
11
|
+
|
12
|
+
from_queue 'downloads'
|
13
|
+
|
14
|
+
def work(msg)
|
15
|
+
doc = Nokogiri::HTML(open(msg))
|
16
|
+
logger.info "FOUND <#{doc.css('title').text}>"
|
17
|
+
ack!
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
Sneakers.configure(:metrics => Sneakers::Metrics::LoggingMetrics.new)
|
23
|
+
r = Sneakers::Runner.new([ MetricsWorker ])
|
24
|
+
r.run
|
25
|
+
|
26
|
+
|
27
|
+
|
28
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
$: << File.expand_path('../lib', File.dirname(__FILE__))
|
2
|
+
require 'sneakers'
|
3
|
+
require 'sneakers/runner'
|
4
|
+
require 'sneakers/metrics/newrelic_metrics'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'nokogiri'
|
7
|
+
require 'newrelic_rpm'
|
8
|
+
|
9
|
+
# With this configuration will send two types of data to newrelic server:
|
10
|
+
# 1. Transaction data which you would see under 'Applications'
|
11
|
+
# 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts
|
12
|
+
#
|
13
|
+
# You should have newrelic.yml in the 'config' folder with the proper account settings
|
14
|
+
|
15
|
+
Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic
|
16
|
+
Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new
|
17
|
+
|
18
|
+
class MetricsWorker
|
19
|
+
include Sneakers::Worker
|
20
|
+
include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
|
21
|
+
|
22
|
+
from_queue 'downloads'
|
23
|
+
|
24
|
+
def work(msg)
|
25
|
+
doc = Nokogiri::HTML(open(msg))
|
26
|
+
logger.info "FOUND <#{doc.css('title').text}>"
|
27
|
+
ack!
|
28
|
+
end
|
29
|
+
|
30
|
+
add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
|
35
|
+
r = Sneakers::Runner.new([ MetricsWorker ])
|
36
|
+
r.run
|
37
|
+
|
38
|
+
|
39
|
+
|
40
|
+
|