boardintel_frenzy_bunnies 0.0.17-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.ruby-version +1 -0
  4. data/Gemfile +4 -0
  5. data/Guardfile +19 -0
  6. data/LICENSE +22 -0
  7. data/README.md +165 -0
  8. data/Rakefile +2 -0
  9. data/bin/frenzy_bunnies +6 -0
  10. data/examples/feed.rb +20 -0
  11. data/examples/feed_worker.rb +33 -0
  12. data/examples/feed_workers_bin.rb +21 -0
  13. data/fb-cap.png +0 -0
  14. data/frenzy_bunnies.gemspec +27 -0
  15. data/lib/frenzy_bunnies/cli.rb +29 -0
  16. data/lib/frenzy_bunnies/context.rb +40 -0
  17. data/lib/frenzy_bunnies/handlers/maxretry.rb +199 -0
  18. data/lib/frenzy_bunnies/handlers/oneshot.rb +31 -0
  19. data/lib/frenzy_bunnies/health/collector.rb +21 -0
  20. data/lib/frenzy_bunnies/health/providers/jvm.rb +43 -0
  21. data/lib/frenzy_bunnies/health.rb +10 -0
  22. data/lib/frenzy_bunnies/queue_factory.rb +23 -0
  23. data/lib/frenzy_bunnies/version.rb +3 -0
  24. data/lib/frenzy_bunnies/web/public/css/bootstrap.min.css +9 -0
  25. data/lib/frenzy_bunnies/web/public/img/bunny16.png +0 -0
  26. data/lib/frenzy_bunnies/web/public/img/bunny32.png +0 -0
  27. data/lib/frenzy_bunnies/web/public/index.html +225 -0
  28. data/lib/frenzy_bunnies/web/public/js/app.coffee +90 -0
  29. data/lib/frenzy_bunnies/web/public/js/app.js +202 -0
  30. data/lib/frenzy_bunnies/web/public/js/backbone-min.js +40 -0
  31. data/lib/frenzy_bunnies/web/public/js/bootstrap.js +2027 -0
  32. data/lib/frenzy_bunnies/web/public/js/bootstrap.min.js +6 -0
  33. data/lib/frenzy_bunnies/web/public/js/jquery-1.8.0.min.js +2 -0
  34. data/lib/frenzy_bunnies/web/public/js/jquery.filesize.js +52 -0
  35. data/lib/frenzy_bunnies/web/public/js/jquery.timeago.js +152 -0
  36. data/lib/frenzy_bunnies/web/public/js/underscore-min.js +32 -0
  37. data/lib/frenzy_bunnies/web.rb +51 -0
  38. data/lib/frenzy_bunnies/worker.rb +105 -0
  39. data/lib/frenzy_bunnies.rb +15 -0
  40. data/spec/frenzy_bunnies/worker_spec.rb +117 -0
  41. data/spec/spec_helper.rb +35 -0
  42. metadata +184 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 941f5d93eece02f51b31f8e07ef2bdb9b85d3a0e
4
+ data.tar.gz: 75cf26720b0994787f895a7f01ccac9d2512d6d2
5
+ SHA512:
6
+ metadata.gz: b619e8b3f35bc92980bfa4f441aad08a64130d2191ac810a6bb810a32e364d418b7dac8e61df0ca49e4341370cff417e709245220d8b5339fa6ed14e57991003
7
+ data.tar.gz: 6b4bf910338902af5dde50eaba1482e8d37e9668ba13a6c1f6ae9073b50c6178bf5b794818555578afc56d0537fc733ec8ceeb65b3b6020e4cd5b4b87bf97c3d
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ jruby-9.1.15.0
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in frenzy_bunnies.gemspec
4
+ gemspec
data/Guardfile ADDED
@@ -0,0 +1,19 @@
1
+ guard 'minitest' do
2
+
3
+ # with Minitest::Spec
4
+ watch(%r|^spec/(.*)_spec\.rb|)
5
+ watch(%r|^lib/(.*)([^/]+)\.rb|) { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
6
+ watch(%r|^spec/spec_helper\.rb|) { "spec" }
7
+
8
+ # Rails 3.2
9
+ # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/controllers/#{m[1]}_test.rb" }
10
+ # watch(%r|^app/helpers/(.*)\.rb|) { |m| "test/helpers/#{m[1]}_test.rb" }
11
+ # watch(%r|^app/models/(.*)\.rb|) { |m| "test/unit/#{m[1]}_test.rb" }
12
+
13
+ # Rails
14
+ # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/functional/#{m[1]}_test.rb" }
15
+ # watch(%r|^app/helpers/(.*)\.rb|) { |m| "test/helpers/#{m[1]}_test.rb" }
16
+ # watch(%r|^app/models/(.*)\.rb|) { |m| "test/unit/#{m[1]}_test.rb" }
17
+ end
18
+
19
+ guard 'coffeescript', :input => 'lib/frenzy_bunnies/web/public/js', :output => 'lib/frenzy_bunnies/web/public/js', :all_on_start => true
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2012 Dotan Nahum
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,165 @@
1
+ # FrenzyBunnies
2
+
3
+ A lightweight background workers library based on JRuby and the very efficient `march_hare` RabbitMQ driver for very fast and
4
+ efficient processing.
5
+
6
+ Unlike other background job processing libraries, a Frenzy Bunnies worker is offering its work to a native JVM-based thread pool, where threads are allocated and cached.
7
+
8
+ This firstly means that the processing model isn't process-per-worker (saving memory) and it also isn't fixed-thread-per-worker based allowing workers to be pooled(saving memory even further).
9
+
10
+ RabbitMQ is a really awesome queue solution for background jobs as well as more real-time messaging processing. Within its strengths are its [performance](http://www.rabbitmq.com/blog/2012/04/17/rabbitmq-performance-measurements-part-1/), portability - [almost every worthy server-side language and platform](http://www.rabbitmq.com/devtools.html) has a RabbitMQ driver and you're not limited to process on a single platform, and high-availability out of the box (as opposed to Redis, although [Sentinel](http://redis.io/topics/sentinel-spec) is quite a progress - hurray!).
11
+
12
+
13
+ Here are [great background slides](https://speakerdeck.com/u/hungryblank/p/rails-underground-2009-rabbitmq) given by Paolo Negri over Rails Underground 2009 about [RabbitMQ](http://www.rabbitmq.com/).
14
+
15
+ ## Quick Start
16
+
17
+ Add this line to your application's Gemfile:
18
+
19
+ gem 'frenzy_bunnies'
20
+
21
+ And then execute:
22
+
23
+ $ bundle
24
+
25
+ Or install it yourself as:
26
+
27
+ $ gem install frenzy_bunnies
28
+
29
+ Then, you basically just need to define a worker in its own class, and then
30
+ decide if you want to use the Frenzy Bunnies runner
31
+ `frenzy_bunnies` to run it, or do it programmatically via the
32
+ `FrenzyBunnies::Context` API.
33
+
34
+ ```ruby
35
+ class FeedWorker
36
+ include FrenzyBunnies::Worker
37
+ from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
38
+
39
+ def work(msg)
40
+ puts msg
41
+ ack!
42
+ end
43
+ end
44
+ ```
45
+
46
+ You indicate that a class is a worker by `include
47
+ FrenzyBunnies::Worker`. Set up a queue with `from_queue` and implement a
48
+ `work(msg)` method.
49
+
50
+ You should indicate successful processing with
51
+ `ack!`, otherwise it will be rejected and lost (per RabbitMQ semantics,
52
+ in future versions, they'll add a feature where rejected messages goes
53
+ to an error queue).
54
+
55
+ ### Running with CLI
56
+
57
+ Running a worker with the command-line executable is easy
58
+
59
+ $ frenzy_bunnies start_workers worker_file.rb
60
+
61
+ Where `worker_file.rb` is a file containing all of your worker(s)
62
+ definition. FrenzyBunnies will require the file and immediately start
63
+ handing work to your workers.
64
+
65
+ ### Running Programatically
66
+
67
+ Assuming that workers are already `require`d in your code, their classes
68
+ should be visible by the moment you write this code:
69
+
70
+ ```ruby
71
+ f = FrenzyBunnies::Context.new
72
+ f.run FeedWorker, FeedDownloader
73
+ ```
74
+
75
+ In the listing above, `f.run` accepts your worker _classes_, and will run your workers immediately.
76
+
77
+
78
+ ## Web Dashboard
79
+
80
+ When FrenzyBunnies run, it will automatically create a web dashboard for you, on `localhost:11333` by default.
81
+
82
+
83
+ Currently, the dashboard displays your job statistics (passed vs. failed), JVM
84
+ health (heap usage) and threads overview.
85
+
86
+
87
+ <img src="https://raw.github.com/jondot/frenzy_bunnies/master/fb-cap.png"/><br/>
88
+
89
+
90
+ Changing the bound address is easy to do through the many options you can pass to the running `Context`:
91
+
92
+ ```ruby
93
+ f = FrenzyBunnies::Context.new :web_host=>'0.0.0.0', :web_port=>11222
94
+ ```
95
+
96
+
97
+ context definitions
98
+
99
+ ## In Detail
100
+
101
+ ### Worker Configuration
102
+
103
+ In your worker class, say `from_queue 'queue_name'` and pass any of these options:
104
+
105
+ ```ruby
106
+ :exchange # default frenzy_bunnies. name of exchange.
107
+ :exchange_type # default :direct. type of exchange used.
108
+ :routing_key # default queue_name. allows for other routing keys, useful for topic exchanges.
109
+ :prefetch # default 10. number of messages to prefetch each time
110
+ :durable # default false. durability of the queue
111
+ :timeout_job_after # default 5. reject the message if not processed for number of seconds
112
+ :threads # default none. number of threads in the threadpool. leave empty to let the threadpool manage it.
113
+ ```
114
+
115
+ Example:
116
+
117
+
118
+ ```ruby
119
+ class FeedWorker
120
+ include FrenzyBunnies::Worker
121
+ from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
122
+
123
+ ...
124
+ ```
125
+
126
+ ### General Configuration
127
+
128
+ Global / running configuration can be set through the running context `FrenzyBunnies::Context`, pass any of these as options (shown with defaults).
129
+
130
+ ```ruby
131
+ :host # default 'localhost'
132
+ :heartbeat # default 5
133
+ :web_host # default 'localhost'
134
+ :web_port # default 11333
135
+ :web_threadfilter # default /^pool-.*/
136
+ :env # default ''
137
+ ```
138
+
139
+
140
+ Example:
141
+
142
+ ```ruby
143
+ FrenzyBunnies::Context.new :heartbeat => 10
144
+ ```
145
+
146
+ ### AMQP Queue Wiring Under the Hood
147
+
148
+ If you're interested with the mechanics, in order to mimic a background-job / work-queue
149
+ semantics, the following is the AMQP wireup used within this library:
150
+
151
+ * Durable per configuration
152
+ * The exchange is created and named by default `frenzy_bunnies`
153
+ * Each worker is bound to an AMQP queue named `my_queue_environment` with the environment postfix appended automatically.
154
+ * The routing key on the exchange is of the same name and bound to the queue.
155
+
156
+ # Contributing
157
+
158
+ Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
159
+
160
+
161
+ # Copyright
162
+
163
+ Copyright (c) 2012 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
164
+
165
+
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ require 'frenzy_bunnies'
3
+ require 'frenzy_bunnies/cli'
4
+
5
+ FrenzyBunnies::CLI.start
6
+
data/examples/feed.rb ADDED
@@ -0,0 +1,20 @@
1
+ require 'rubygems'
2
+ require 'march_hare'
3
+
4
+
5
+
6
+
7
+ connection = MarchHare.connect(:host => 'localhost')
8
+ channel = connection.create_channel
9
+ channel.prefetch = 10
10
+
11
+ exchange = channel.exchange('frenzy_bunnies', :type => :direct, :durable => true)
12
+
13
+
14
+
15
+ 100_000.times do |i|
16
+ exchange.publish("hello world! #{i}", :routing_key => 'new.feeds')
17
+ end
18
+ puts "done"
19
+
20
+
@@ -0,0 +1,33 @@
1
+ $:<< File.expand_path('../lib', File.dirname(__FILE__))
2
+
3
+ require 'rubygems'
4
+ require 'frenzy_bunnies'
5
+
6
+ class FeedWorker
7
+ include FrenzyBunnies::Worker
8
+ from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
9
+
10
+ def work(msg)
11
+ puts msg
12
+ ack!
13
+ end
14
+ end
15
+
16
+ class FeedDownloader
17
+ include FrenzyBunnies::Worker
18
+ from_queue 'new.downloads', :durable => true
19
+ def work(msg)
20
+ puts msg
21
+ ack!
22
+ end
23
+ end
24
+
25
+ f = FrenzyBunnies::Context.new
26
+
27
+ f.run FeedWorker,FeedDownloader
28
+
29
+
30
+ trap "INT" do
31
+ f.stop
32
+ exit!
33
+ end
@@ -0,0 +1,21 @@
1
+ class FeedWorker
2
+ include FrenzyBunnies::Worker
3
+ from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
4
+
5
+ def work(msg)
6
+ puts msg
7
+ ack!
8
+ end
9
+ end
10
+
11
+ class FeedDownloader
12
+ include FrenzyBunnies::Worker
13
+ from_queue 'new.downloads', :durable => true
14
+ def work(msg)
15
+ puts msg
16
+ ack!
17
+ end
18
+ end
19
+
20
+
21
+
data/fb-cap.png ADDED
Binary file
@@ -0,0 +1,27 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/frenzy_bunnies/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Dotan Nahum"]
6
+ gem.email = ["jondotan@gmail.com"]
7
+ gem.description = %q{RabbitMQ JRuby based workers on top of march_hare}
8
+ gem.summary = %q{RabbitMQ JRuby based workers on top of march_hare}
9
+ gem.homepage = ""
10
+
11
+ gem.files = `git ls-files`.split($\)
12
+ gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
13
+ gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
14
+ gem.name = "boardintel_frenzy_bunnies"
15
+ gem.require_paths = ["lib"]
16
+ gem.version = FrenzyBunnies::VERSION
17
+ gem.platform = "java"
18
+
19
+ gem.add_runtime_dependency 'march_hare', "= 3.0.0"
20
+ gem.add_runtime_dependency 'sinatra', "= 1.4.8"
21
+ gem.add_runtime_dependency 'atomic', "= 1.1.99"
22
+ gem.add_runtime_dependency 'json', "= 1.8.6"
23
+
24
+ gem.add_development_dependency 'guard-coffeescript'
25
+ gem.add_development_dependency 'rr'
26
+ gem.add_development_dependency 'guard-minitest'
27
+ end
@@ -0,0 +1,29 @@
1
+ require 'thor'
2
+
3
+
4
+ class FrenzyBunnies::CLI < Thor
5
+ BUNNIES =<<-EOF
6
+
7
+ (\\___/)
8
+ (='.'=) Frenzy Bunnies!
9
+ (")_(") JRuby based workers on top of march_hare
10
+
11
+ EOF
12
+
13
+ desc 'run', "run workers from a file"
14
+ def start_workers(workerfile)
15
+
16
+ require workerfile
17
+ # enumerate all workers
18
+ workers = []
19
+ ObjectSpace.each_object(Class){|o| workers << o if o.ancestors.map(&:name).include? "FrenzyBunnies::Worker"}
20
+ workers.uniq!
21
+
22
+ puts BUNNIES
23
+
24
+ c = FrenzyBunnies::Context.new
25
+ c.logger.info "Discovered #{workers.inspect}"
26
+ c.run *workers
27
+ Signal.trap('INT') { c.stop; exit! }
28
+ end
29
+ end
@@ -0,0 +1,40 @@
1
+ require 'logger'
2
+ require 'frenzy_bunnies/web'
3
+
4
+ class FrenzyBunnies::Context
5
+ attr_reader :connection, :queue_factory, :logger, :env, :opts
6
+
7
+ def initialize(opts={})
8
+ @opts = opts
9
+ @opts[:host] ||= 'localhost'
10
+ @opts[:heartbeat] ||= 5
11
+ @opts[:web_host] ||= 'localhost'
12
+ @opts[:web_port] ||= 11333
13
+ @opts[:web_threadfilter] ||= /^pool-.*/
14
+ @opts[:env] ||= 'development'
15
+
16
+ @env = @opts[:env]
17
+ @logger = @opts[:logger] || Logger.new(STDOUT)
18
+ params = {:host => @opts[:host], :heartbeat_interval => @opts[:heartbeat]}
19
+ (params[:username], params[:password] = @opts[:username], @opts[:password]) if @opts[:username] && @opts[:password]
20
+ (params[:port] = @opts[:port]) if @opts[:port]
21
+ @connection = MarchHare.connect(params)
22
+ @connection.add_shutdown_listener(lambda { |cause| @logger.error("Disconnected: #{cause}"); stop;})
23
+
24
+ @queue_factory = FrenzyBunnies::QueueFactory.new(@connection)
25
+ end
26
+
27
+ def run(*klasses)
28
+ @klasses = []
29
+ klasses.each{|klass| klass.start(self); @klasses << klass}
30
+ return nil if @opts[:disable_web_stats]
31
+ Thread.new do
32
+ FrenzyBunnies::Web.run_with(@klasses, :host => @opts[:web_host], :port => @opts[:web_port], :threadfilter => @opts[:web_threadfilter], :logger => @logger)
33
+ end
34
+ end
35
+
36
+ def stop
37
+ @klasses.each{|klass| klass.stop }
38
+ end
39
+ end
40
+
@@ -0,0 +1,199 @@
1
+ require 'base64'
2
+ require 'json'
3
+
4
+ module FrenzyBunnies
5
+ module Handlers
6
+ #
7
+ # Maxretry uses dead letter policies on Rabbitmq to requeue and retry
8
+ # messages after failure (rejections, errors and timeouts). When the maximum
9
+ # number of retries is reached it will put the message on an error queue.
10
+ # This handler will only retry at the queue level. To accomplish that, the
11
+ # setup is a bit complex.
12
+ #
13
+ # Input:
14
+ # worker_exchange (eXchange)
15
+ # worker_queue (Queue)
16
+ # We create:
17
+ # worker_queue-retry - (X) where we setup the worker queue to dead-letter.
18
+ # worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to
19
+ # worker_queue-retry-requeue.
20
+ # worker_queue-error - (X) where to send max-retry failures
21
+ # worker_queue-error - (Q) bound to worker_queue-error.
22
+ # worker_queue-retry-requeue - (X) exchange to bind worker_queue to for
23
+ # requeuing directly to the worker_queue.
24
+ #
25
+ # This requires that you setup arguments to the worker queue to line up the
26
+ # dead letter queue. See the example for more information.
27
+ #
28
+ # Many of these can be override with options:
29
+ # - retry_exchange - sets retry exchange & queue
30
+ # - retry_error_exchange - sets error exchange and queue
31
+ # - retry_requeue_exchange - sets the exchange created to re-queue things
32
+ # back to the worker queue.
33
+ #
34
+ class Maxretry
35
+
36
+ def initialize(channel, queue, logger, opts)
37
+ @logger = logger
38
+ @worker_queue_name = queue.name
39
+ @logger.debug do
40
+ "#{log_prefix} creating handler, opts=#{opts}"
41
+ end
42
+
43
+ @channel = channel
44
+ @opts = opts
45
+
46
+ # Construct names, defaulting where suitable
47
+ retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry"
48
+ error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error"
49
+ requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue"
50
+
51
+ # Create the exchanges
52
+ @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name|
53
+ @logger.debug { "#{log_prefix} creating exchange=#{name}" }
54
+ @channel.exchange(name,
55
+ :type => 'topic',
56
+ :durable => exchange_durable?)
57
+ end
58
+
59
+ # Create the queues and bindings
60
+ @logger.debug do
61
+ "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}"
62
+ end
63
+ @retry_queue = @channel.queue(retry_name,
64
+ :durable => queue_durable?,
65
+ :arguments => {
66
+ 'x-dead-letter-exchange' => requeue_name,
67
+ 'x-message-ttl' => @opts[:retry_timeout] || 60000
68
+ })
69
+ @retry_queue.bind(@retry_exchange, :routing_key => '#')
70
+
71
+ @logger.debug do
72
+ "#{log_prefix} creating queue=#{error_name}"
73
+ end
74
+ @error_queue = @channel.queue(error_name,
75
+ :durable => queue_durable?)
76
+ @error_queue.bind(@error_exchange, :routing_key => '#')
77
+
78
+ # Finally, bind the worker queue to our requeue exchange
79
+ queue.bind(@requeue_exchange, :routing_key => '#')
80
+
81
+ @max_retries = @opts[:retry_max_times] || 5
82
+ end
83
+
84
+ def acknowledge(hdr, msg)
85
+ @channel.acknowledge(hdr.delivery_tag, false)
86
+ end
87
+
88
+ def reject(hdr, msg, requeue = false)
89
+ if requeue
90
+ # This was explicitly rejected specifying it be requeued so we do not
91
+ # want it to pass through our retry logic.
92
+ @channel.reject(hdr.delivery_tag, requeue)
93
+ else
94
+ handle_retry(hdr, msg, :reject)
95
+ end
96
+ end
97
+
98
+
99
+ def error(hdr, msg, err)
100
+ handle_retry(hdr, msg, err)
101
+ end
102
+
103
+ def timeout(hdr, msg)
104
+ handle_retry(hdr, msg, :timeout)
105
+ end
106
+
107
+ def noop(hdr, props, msg)
108
+
109
+ end
110
+
111
+ # Helper logic for retry handling. This will reject the message if there
112
+ # are remaining retries left on it, otherwise it will publish it to the
113
+ # error exchange along with the reason.
114
+ # @param hdr [MarchHare::Headers]
115
+ # @param msg [String] The message
116
+ # @param reason [String, Symbol, Exception] Reason for the retry, included
117
+ # in the JSON we put on the error exchange.
118
+ def handle_retry(hdr, msg, reason)
119
+ # +1 for the current attempt
120
+ num_attempts = failure_count(hdr.headers) + 1
121
+ if num_attempts <= @max_retries
122
+ # We call reject which will route the message to the
123
+ # x-dead-letter-exchange (ie. retry exchange) on the queue
124
+ @logger.info do
125
+ "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{hdr.headers}"
126
+ end
127
+ @channel.reject(hdr.delivery_tag, false)
128
+ # TODO: metrics
129
+ else
130
+ # Retried more than the max times
131
+ # Publish the original message with the routing_key to the error exchange
132
+ @logger.info do
133
+ "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
134
+ end
135
+ data = {
136
+ error: reason,
137
+ num_attempts: num_attempts,
138
+ failed_at: Time.now.iso8601,
139
+ payload: Base64.encode64(msg.to_s)
140
+ }.tap do |hash|
141
+ if reason.is_a?(Exception)
142
+ hash[:error_class] = reason.class.to_s
143
+ hash[:error_message] = "#{reason}"
144
+ if reason.backtrace
145
+ hash[:backtrace] = reason.backtrace.take(10).join(', ')
146
+ end
147
+ end
148
+ end.to_json
149
+ @error_exchange.publish(data, :routing_key => hdr.routing_key)
150
+ @channel.acknowledge(hdr.delivery_tag, false)
151
+ # TODO: metrics
152
+ end
153
+ end
154
+ private :handle_retry
155
+
156
+ # Uses the x-death header to determine the number of failures this job has
157
+ # seen in the past. This does not count the current failure. So for
158
+ # instance, the first time the job fails, this will return 0, the second
159
+ # time, 1, etc.
160
+ # @param headers [Hash] Hash of headers that Rabbit delivers as part of
161
+ # the message
162
+ # @return [Integer] Count of number of failures.
163
+ def failure_count(headers)
164
+ if headers.nil? || headers['x-death'].nil?
165
+ 0
166
+ else
167
+ x_death_array = headers['x-death'].select do |x_death|
168
+ x_death['queue'] == @worker_queue_name
169
+ end
170
+ if x_death_array.count > 0 && x_death_array.first['count']
171
+ # Newer versions of RabbitMQ return headers with a count key
172
+ x_death_array.inject(0) {|sum, x_death| sum + x_death['count']}
173
+ else
174
+ # Older versions return a separate x-death header for each failure
175
+ x_death_array.count
176
+ end
177
+ end
178
+ end
179
+ private :failure_count
180
+
181
+ # Prefix all of our log messages so they are easier to find. We don't have
182
+ # the worker, so the next best thing is the queue name.
183
+ def log_prefix
184
+ "Maxretry handler [queue=#{@worker_queue_name}]"
185
+ end
186
+ private :log_prefix
187
+
188
+ private
189
+
190
+ def queue_durable?
191
+ @opts.fetch(:queue_options, {}).fetch(:durable, false)
192
+ end
193
+
194
+ def exchange_durable?
195
+ queue_durable?
196
+ end
197
+ end
198
+ end
199
+ end
@@ -0,0 +1,31 @@
1
+ module FrenzyBunnies
2
+ module Handlers
3
+ class Oneshot
4
+ def initialize(channel, queue, logger, opts)
5
+ @channel = channel
6
+ @opts = opts
7
+ @logger = logger
8
+ end
9
+
10
+ def acknowledge(hdr, msg)
11
+ @channel.acknowledge(hdr.delivery_tag, false)
12
+ end
13
+
14
+ def reject(hdr, msg, requeue=false)
15
+ @channel.reject(hdr.delivery_tag, requeue)
16
+ end
17
+
18
+ def error(hdr, msg, err)
19
+ reject(hdr, msg)
20
+ end
21
+
22
+ def timeout(hdr, msg)
23
+ reject(hdr, msg)
24
+ end
25
+
26
+ def noop(hdr, msg)
27
+
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,21 @@
1
+ class FrenzyBunnies::Health::Collector
2
+ def initialize(opts={})
3
+ @providers = []
4
+ Dir["#{File.dirname(__FILE__)}/providers/*.rb"].each do |f|
5
+ require f
6
+ name = File.basename(f, '.*')
7
+ provider_klass = FrenzyBunnies::Health::Providers.const_get(camelize name)
8
+ @providers << provider_klass.new(opts[name.to_sym])
9
+ end
10
+ end
11
+
12
+ def collect
13
+ @providers.map{|p| p.report }.inject(:merge)
14
+ end
15
+
16
+ # real basic camelizer, beware!. meant to avoid including active-support here.
17
+ def camelize(str)
18
+ str.split('_').map {|s| s.capitalize}.join
19
+ end
20
+ end
21
+