sneakers_custom_bunny 1.0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +9 -0
  3. data/.travis.yml +6 -0
  4. data/CHANGELOG.md +20 -0
  5. data/Gemfile +3 -0
  6. data/Guardfile +8 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +172 -0
  9. data/ROADMAP.md +18 -0
  10. data/Rakefile +11 -0
  11. data/bin/sneakers +5 -0
  12. data/examples/benchmark_worker.rb +20 -0
  13. data/examples/max_retry_handler.rb +78 -0
  14. data/examples/metrics_worker.rb +28 -0
  15. data/examples/newrelic_metrics_worker.rb +40 -0
  16. data/examples/profiling_worker.rb +69 -0
  17. data/examples/sneakers.conf.rb.example +11 -0
  18. data/examples/title_scraper.rb +23 -0
  19. data/examples/workflow_worker.rb +23 -0
  20. data/lib/sneakers.rb +83 -0
  21. data/lib/sneakers/cli.rb +115 -0
  22. data/lib/sneakers/concerns/logging.rb +34 -0
  23. data/lib/sneakers/concerns/metrics.rb +34 -0
  24. data/lib/sneakers/configuration.rb +59 -0
  25. data/lib/sneakers/handlers/maxretry.rb +191 -0
  26. data/lib/sneakers/handlers/oneshot.rb +30 -0
  27. data/lib/sneakers/metrics/logging_metrics.rb +16 -0
  28. data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
  29. data/lib/sneakers/metrics/null_metrics.rb +13 -0
  30. data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
  31. data/lib/sneakers/publisher.rb +34 -0
  32. data/lib/sneakers/queue.rb +65 -0
  33. data/lib/sneakers/runner.rb +82 -0
  34. data/lib/sneakers/spawner.rb +27 -0
  35. data/lib/sneakers/support/production_formatter.rb +11 -0
  36. data/lib/sneakers/support/utils.rb +18 -0
  37. data/lib/sneakers/tasks.rb +34 -0
  38. data/lib/sneakers/version.rb +3 -0
  39. data/lib/sneakers/worker.rb +151 -0
  40. data/lib/sneakers/workergroup.rb +47 -0
  41. data/sneakers.gemspec +35 -0
  42. data/spec/fixtures/require_worker.rb +17 -0
  43. data/spec/sneakers/cli_spec.rb +63 -0
  44. data/spec/sneakers/concerns/logging_spec.rb +39 -0
  45. data/spec/sneakers/concerns/metrics_spec.rb +38 -0
  46. data/spec/sneakers/configuration_spec.rb +75 -0
  47. data/spec/sneakers/publisher_spec.rb +83 -0
  48. data/spec/sneakers/queue_spec.rb +115 -0
  49. data/spec/sneakers/runner_spec.rb +26 -0
  50. data/spec/sneakers/sneakers_spec.rb +75 -0
  51. data/spec/sneakers/support/utils_spec.rb +44 -0
  52. data/spec/sneakers/worker_handlers_spec.rb +390 -0
  53. data/spec/sneakers/worker_spec.rb +463 -0
  54. data/spec/spec_helper.rb +13 -0
  55. metadata +306 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 28b5234a60deb884a5376ceb93a671a113464131
4
+ data.tar.gz: 25a77893cca3406cc82d0a0e54b593152d305b03
5
+ SHA512:
6
+ metadata.gz: 5e0fce7746274d11045da39af007e433cd5034dca1a323a0ac0fa26cd0791c0244c5db865f7cd1f8c94015d0e1f342e8e5b483f40f3e85d0883e74c4242062c1
7
+ data.tar.gz: a7882fb13af0c27005f72c4b5cd3135a2565b2965292eaf98471742c49be1150170578d3c79f37168e09fb6fba82917a51884d470dfb23734ce610d4cffde486
@@ -0,0 +1,9 @@
1
+ sneakers.yaml
2
+ Gemfile.lock
3
+ *.log
4
+ sneakers.pid
5
+ pkg/
6
+ coverage/
7
+ tmp/
8
+ .ruby-version
9
+ .ruby-gemset
@@ -0,0 +1,6 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2
4
+ - 2.1
5
+ - 2.0.0
6
+ - 1.9.3
@@ -0,0 +1,20 @@
1
+ # 0.1.0, Not released yet
2
+
3
+ + Added newrelic stats reporter (@arielze)
4
+ + Added explicit routing_key at queue (@arielze)
5
+ + Depracating self-daemonization (@jondot)
6
+ + updating bunny (@michaelklishin)
7
+ + Fix reference in publisher (@sergei-matheson)
8
+ + Allow binding of multiple routing keys (@SebastianEdwards)
9
+ + added work_with_params for advanced handling of work items with amqp
10
+ headers. (#12)
11
+ + Sneakers.not_environmental! will no disable auto-environment discovery.
12
+ (#15)
13
+
14
+
15
+
16
+
17
+ # 0.0.7
18
+
19
+ + Sneakers core.
20
+
data/Gemfile ADDED
@@ -0,0 +1,3 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
@@ -0,0 +1,8 @@
1
+
2
+ guard :minitest do
3
+ watch(%r{^spec/(.*)_spec\.rb})
4
+ watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" }
5
+ watch(%r{^spec/spec_helper\.rb}) { 'spec' }
6
+ end
7
+
8
+
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013-2014 Dotan Nahum
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,172 @@
1
+ # Sneakers
2
+
3
+
4
+ ```
5
+ __
6
+ ,--' >
7
+ `=====
8
+
9
+ ```
10
+
11
+
12
+ A high-performance RabbitMQ background processing framework for
13
+ Ruby.
14
+
15
+
16
+ Sneakers is being used in production for both I/O and CPU intensive workloads, and have achieved the goals of high-performance and 0-maintenance, as designed.
17
+
18
+
19
+ Visit the [wiki](https://github.com/jondot/sneakers/wiki) for
20
+ complete docs.
21
+
22
+
23
+ [![Build Status](https://travis-ci.org/jondot/sneakers.svg?branch=master)](https://travis-ci.org/jondot/sneakers)
24
+
25
+
26
+ ## Installation
27
+
28
+ Add this line to your application's Gemfile:
29
+
30
+ gem 'sneakers'
31
+
32
+ And then execute:
33
+
34
+ $ bundle
35
+
36
+ Or install it yourself as:
37
+
38
+ $ gem install sneakers
39
+
40
+
41
+ ## Quick start
42
+
43
+ Set up a Gemfile
44
+
45
+ ```ruby
46
+ source 'https://rubygems.org'
47
+ gem 'sneakers'
48
+ gem 'json'
49
+ gem 'redis'
50
+ ```
51
+
52
+ And a worker
53
+
54
+ ```ruby
55
+ require 'sneakers'
56
+ require 'redis'
57
+ require 'json'
58
+
59
+ $redis = Redis.new
60
+
61
+ class Processor
62
+ include Sneakers::Worker
63
+ from_queue :logs
64
+
65
+
66
+ def work(msg)
67
+ err = JSON.parse(msg)
68
+ if err["type"] == "error"
69
+ $redis.incr "processor:#{err["error"]}"
70
+ end
71
+
72
+ ack!
73
+ end
74
+ end
75
+ ```
76
+
77
+
78
+ As an example, make a message look like this:
79
+ We'll count errors and error types with Redis. Specifically for an error that looks like this:
80
+
81
+ ```javascript
82
+ {
83
+ "type": "error",
84
+ "message": "HALP!",
85
+ "error": "CODE001"
86
+ }
87
+ ```
88
+
89
+
90
+ Let's test it out quickly from the command line:
91
+
92
+
93
+ ```bash
94
+ sneakers work Processor --require boot.rb
95
+ ```
96
+
97
+ We just told Sneakers to spawn a worker named `Processor`, but first `--require` a file that we dedicate to setting up environment, including workers and what-not.
98
+
99
+ If you go to your RabbitMQ admin now, you'll see a new queue named `logs` was created. Push a couple messages, and this is the output you should see at your terminal.
100
+
101
+
102
+ ```
103
+ 2013-10-11T19:26:36Z p-4718 t-ovqgyb31o DEBUG: [worker-logs:1:213mmy][#<Thread:0x007fae6b05cc58>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
104
+ 2013-10-11T19:26:36Z p-4718 t-ovqgyrxu4 INFO: log log
105
+ 2013-10-11T19:26:40Z p-4719 t-ovqgyb364 DEBUG: [worker-logs:1:h23iit][#<Thread:0x007fae6b05cd98>][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log
106
+ 2013-10-11T19:26:40Z p-4719 t-ovqgyrx8g INFO: log log
107
+ ```
108
+
109
+ And redis will show this:
110
+
111
+
112
+ ```
113
+ ➜ ~ redis-cli monitor
114
+ 1381520329.888581 [0 127.0.0.1:49182] "incr" "processor:CODE001"
115
+ ```
116
+
117
+
118
+ We're basically done with the ceremonies and all is left is to do some real work.
119
+
120
+
121
+
122
+ ### Looking at metrics
123
+
124
+ Let's use the `logging_metrics` provider just for the sake of fun of seeing the metrics as they happen.
125
+
126
+ ```ruby
127
+ # boot.rb
128
+ require 'sneakers'
129
+ require 'redis'
130
+ require 'json'
131
+ require 'sneakers/metrics/logging_metrics'
132
+ Sneakers.configure :metrics => Sneakers::Metrics::LoggingMetrics.new
133
+
134
+ # ... rest of code
135
+ ```
136
+
137
+ Now push a message again and you'll see:
138
+
139
+ ```
140
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.started
141
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: TIME: work.Processor.time 0.00242
142
+ 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.handled.ack
143
+ ```
144
+
145
+ Which increments start + end, and times the work unit.
146
+
147
+
148
+
149
+ From here, you can continue over to the
150
+ [Wiki](https://github.com/jondot/sneakers/wiki)
151
+
152
+ # Contributing
153
+
154
+ Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
155
+
156
+
157
+ ### Thanks:
158
+
159
+ To all Sneakers [Contributors](https://github.com/jondot/sneakers/graphs/contributors) - you make this happen, thanks!
160
+
161
+
162
+
163
+ # Copyright
164
+
165
+ Copyright (c) 2015 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
166
+
167
+
168
+
169
+
170
+
171
+
172
+
@@ -0,0 +1,18 @@
1
+ # Roadmap
2
+
3
+ It is essential to keep a clear roadmap.
4
+ In its core, Sneakers is already production-ready. However, it is vital
5
+ to take in feedback and improve what can be improved :)
6
+
7
+ This document will communicate the stages that the next version is at.
8
+ See `CHANGELOG.md` for new features that are in.
9
+
10
+
11
+ # v0.1.0 (next version)
12
+
13
+ - Taking in community pull requests and feedback
14
+ - Verify production readiness and performance
15
+ - Full performance test suite
16
+
17
+
18
+
@@ -0,0 +1,11 @@
1
+ require "bundler/gem_tasks"
2
+ require 'metric_fu'
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new do |t|
6
+ t.libs << "spec"
7
+ t.test_files = FileList['spec/**/*_spec.rb']
8
+ end
9
+
10
+ task default: :test
11
+
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ require 'sneakers'
3
+ require 'sneakers/cli'
4
+
5
+ Sneakers::CLI.start
@@ -0,0 +1,20 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+
4
+ class BenchmarkWorker
5
+ include Sneakers::Worker
6
+ from_queue 'downloads',
7
+ :durable => false,
8
+ :ack => true,
9
+ :threads => 50,
10
+ :prefetch => 50,
11
+ :timeout_job_after => 1,
12
+ :exchange => 'dummy',
13
+ :heartbeat => 5
14
+ def work(msg)
15
+ ack!
16
+ end
17
+ end
18
+
19
+
20
+
@@ -0,0 +1,78 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/handlers/maxretry'
5
+ require 'logger'
6
+
7
+ Sneakers.configure(:handler => Sneakers::Handlers::Maxretry,
8
+ :workers => 1,
9
+ :threads => 1,
10
+ :prefetch => 1,
11
+ :exchange => 'sneakers',
12
+ :exchange_type => 'topic',
13
+ :routing_key => ['#', 'something'],
14
+ :durable => true,
15
+ )
16
+ Sneakers.logger.level = Logger::DEBUG
17
+
18
+ WORKER_OPTIONS = {
19
+ :ack => true,
20
+ :threads => 1,
21
+ :prefetch => 1,
22
+ :timeout_job_after => 60,
23
+ :heartbeat => 5,
24
+ :retry_timeout => 5000
25
+ }
26
+
27
+ # Example of how to write a retry worker. If your rabbit system is empty, then
28
+ # you must run this twice. Once to setup the exchanges, queues and bindings a
29
+ # second time to have the sent message end up on the downloads queue.
30
+ #
31
+ # Run this via:
32
+ # bundle exec ruby examples/max_retry_handler.rb
33
+ #
34
+ class MaxRetryWorker
35
+ include Sneakers::Worker
36
+ from_queue 'downloads',
37
+ WORKER_OPTIONS.merge({
38
+ :arguments => {
39
+ :'x-dead-letter-exchange' => 'downloads-retry'
40
+ },
41
+ })
42
+
43
+ def work(msg)
44
+ logger.info("MaxRetryWorker rejecting msg: #{msg.inspect}")
45
+
46
+ # We always want to reject to see if we do the proper timeout
47
+ reject!
48
+ end
49
+ end
50
+
51
+ # Example of a worker on the same exchange that does not fail, so it should only
52
+ # see the message once.
53
+ class SucceedingWorker
54
+ include Sneakers::Worker
55
+ from_queue 'uploads',
56
+ WORKER_OPTIONS.merge({
57
+ :arguments => {
58
+ :'x-dead-letter-exchange' => 'uploads-retry'
59
+ },
60
+ })
61
+
62
+ def work(msg)
63
+ logger.info("SucceedingWorker succeeding on msg: #{msg.inspect}")
64
+ ack!
65
+ end
66
+ end
67
+
68
+ messages = 1
69
+ puts "feeding messages in"
70
+ messages.times {
71
+ Sneakers.publish(" -- message -- ",
72
+ :to_queue => 'anywhere',
73
+ :persistence => true)
74
+ }
75
+ puts "done"
76
+
77
+ r = Sneakers::Runner.new([MaxRetryWorker, SucceedingWorker])
78
+ r.run
@@ -0,0 +1,28 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/metrics/logging_metrics'
5
+ require 'open-uri'
6
+ require 'nokogiri'
7
+
8
+
9
+ class MetricsWorker
10
+ include Sneakers::Worker
11
+
12
+ from_queue 'downloads'
13
+
14
+ def work(msg)
15
+ doc = Nokogiri::HTML(open(msg))
16
+ logger.info "FOUND <#{doc.css('title').text}>"
17
+ ack!
18
+ end
19
+ end
20
+
21
+
22
+ Sneakers.configure(:metrics => Sneakers::Metrics::LoggingMetrics.new)
23
+ r = Sneakers::Runner.new([ MetricsWorker ])
24
+ r.run
25
+
26
+
27
+
28
+
@@ -0,0 +1,40 @@
1
+ $: << File.expand_path('../lib', File.dirname(__FILE__))
2
+ require 'sneakers'
3
+ require 'sneakers/runner'
4
+ require 'sneakers/metrics/newrelic_metrics'
5
+ require 'open-uri'
6
+ require 'nokogiri'
7
+ require 'newrelic_rpm'
8
+
9
+ # With this configuration will send two types of data to newrelic server:
10
+ # 1. Transaction data which you would see under 'Applications'
11
+ # 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts
12
+ #
13
+ # You should have newrelic.yml in the 'config' folder with the proper account settings
14
+
15
+ Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic
16
+ Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new
17
+
18
+ class MetricsWorker
19
+ include Sneakers::Worker
20
+ include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
21
+
22
+ from_queue 'downloads'
23
+
24
+ def work(msg)
25
+ doc = Nokogiri::HTML(open(msg))
26
+ logger.info "FOUND <#{doc.css('title').text}>"
27
+ ack!
28
+ end
29
+
30
+ add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
31
+
32
+ end
33
+
34
+
35
+ r = Sneakers::Runner.new([ MetricsWorker ])
36
+ r.run
37
+
38
+
39
+
40
+