boardintel_frenzy_bunnies 0.0.9

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c56073398071068aa0a6bf829ea788f9dc053cde
+  data.tar.gz: 26f193cdee2e88e234ffdf2bf5bb46da3f64b696
+SHA512:
+  metadata.gz: 54c2512224c825d1794f5d303f2860bca45c1317a007285e4d2f22a056c1028bb77881ef40a5b9b33dc26f0ec22b249c4409dd5fef9a7ff5b04ebf1021f9ea9d
+  data.tar.gz: 38e44b804b9c089d435009fdec7ffe4984ab6425c935fcc20afc4aaf24bb0f438127aa1e5271295f6639f47ec500e84c52500e77cfd8b3c72fe0a1f49ebd2794

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in frenzy_bunnies.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,19 @@
+guard 'minitest' do
+
+  # with Minitest::Spec
+   watch(%r|^spec/(.*)_spec\.rb|)
+   watch(%r|^lib/(.*)([^/]+)\.rb|)     { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+   watch(%r|^spec/spec_helper\.rb|)    { "spec" }
+
+  # Rails 3.2
+  # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/controllers/#{m[1]}_test.rb" }
+  # watch(%r|^app/helpers/(.*)\.rb|)     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r|^app/models/(.*)\.rb|)      { |m| "test/unit/#{m[1]}_test.rb" }  
+  
+  # Rails
+  # watch(%r|^app/controllers/(.*)\.rb|) { |m| "test/functional/#{m[1]}_test.rb" }
+  # watch(%r|^app/helpers/(.*)\.rb|)     { |m| "test/helpers/#{m[1]}_test.rb" }
+  # watch(%r|^app/models/(.*)\.rb|)      { |m| "test/unit/#{m[1]}_test.rb" }  
+end
+
+guard 'coffeescript', :input => 'lib/frenzy_bunnies/web/public/js', :output => 'lib/frenzy_bunnies/web/public/js', :all_on_start => true

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Dotan Nahum
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,165 @@
+# FrenzyBunnies
+
+A lightweight background workers library based on JRuby and the very efficient `march_hare` RabbitMQ driver for very fast and
+efficient processing.
+
+Unlike other background job processing libraries, a Frenzy Bunnies worker is offering its work to a native JVM-based thread pool, where threads are allocated and cached.  
+
+This firstly means that the processing model isn't process-per-worker (saving memory) and it also isn't fixed-thread-per-worker based allowing workers to be pooled(saving memory even further).
+
+RabbitMQ is a really awesome queue solution for background jobs as well as more real-time messaging processing. Within its strengths are its [performance](http://www.rabbitmq.com/blog/2012/04/17/rabbitmq-performance-measurements-part-1/), portability - [almost every worthy server-side language and platform](http://www.rabbitmq.com/devtools.html) has a RabbitMQ driver and you're not limited to process on a single platform, and high-availability out of the box (as opposed to Redis, although [Sentinel](http://redis.io/topics/sentinel-spec) is quite a progress - hurray!).  
+
+
+Here are [great background slides](https://speakerdeck.com/u/hungryblank/p/rails-underground-2009-rabbitmq)  given by Paolo Negri over Rails Underground 2009 about [RabbitMQ](http://www.rabbitmq.com/).
+
+## Quick Start
+
+Add this line to your application's Gemfile:
+
+    gem 'frenzy_bunnies'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install frenzy_bunnies
+
+Then, you basically just need to define a worker in its own class, and then
+decide if you want to use the Frenzy Bunnies runner
+`frenzy_bunnies` to run it, or do it programmatically via the
+`FrenzyBunnies::Context` API.
+
+```ruby
+class FeedWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
+
+  def work(msg)
+    puts msg
+    ack!
+  end
+end
+```
+
+You indicate that a class is a worker by `include
+FrenzyBunnies::Worker`. Set up a queue with `from_queue` and implement a
+`work(msg)` method. 
+
+You should indicate successful processing with
+`ack!`, otherwise it will be rejected and lost (per RabbitMQ semantics,
+in future versions, they'll add a feature where rejected messages goes
+to an error queue).
+
+### Running with CLI
+
+Running a worker with the command-line executable is easy
+
+    $ frenzy_bunnies start_workers worker_file.rb
+
+Where `worker_file.rb` is a file containing all of your worker(s)
+definition. FrenzyBunnies will require the file and immediately start
+handing work to your workers.
+
+### Running Programatically
+
+Assuming that workers are already `require`d in your code, their classes
+should be visible by the moment you write this code:
+
+```ruby
+f = FrenzyBunnies::Context.new
+f.run FeedWorker, FeedDownloader
+```
+
+In the listing above, `f.run` accepts your worker _classes_, and will run your workers immediately.
+
+
+## Web Dashboard
+
+When FrenzyBunnies run, it will automatically create a web dashboard for you, on `localhost:11333` by default.
+
+
+Currently, the dashboard displays your job statistics (passed vs. failed), JVM
+health (heap usage) and threads overview.
+
+
+<img src="https://raw.github.com/jondot/frenzy_bunnies/master/fb-cap.png"/><br/>
+
+
+Changing the bound address is easy to do through the many options you can pass to the running `Context`:
+
+```ruby
+f = FrenzyBunnies::Context.new :web_host=>'0.0.0.0', :web_port=>11222
+```
+
+
+context definitions
+
+## In Detail
+
+### Worker Configuration
+
+In your worker class, say `from_queue 'queue_name'` and pass any of these options:
+
+```ruby
+:exchange # default frenzy_bunnies. name of exchange.
+:exchange_type # default :direct. type of exchange used.
+:routing_key # default queue_name. allows for other routing keys, useful for topic exchanges.
+:prefetch  # default 10. number of messages to prefetch each time
+:durable   # default false. durability of the queue
+:timeout_job_after # default 5. reject the message if not processed for number of seconds
+:threads  # default none. number of threads in the threadpool. leave empty to let the threadpool manage it.
+```
+
+Example:
+
+
+```ruby
+class FeedWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
+
+...  
+```
+
+### General Configuration
+
+Global / running configuration can be set through the running context `FrenzyBunnies::Context`, pass any of these as options (shown with defaults).
+
+```ruby
+:host       # default 'localhost'
+:heartbeat  # default 5
+:web_host   # default 'localhost'
+:web_port   # default 11333
+:web_threadfilter # default /^pool-.*/
+:env        # default ''
+```
+
+
+Example:
+
+```ruby
+FrenzyBunnies::Context.new :heartbeat => 10
+```
+
+### AMQP Queue Wiring Under the Hood
+
+If you're interested with the mechanics, in order to mimic a background-job / work-queue 
+semantics, the following is the AMQP wireup used within this library:
+
+* Durable per configuration
+* The exchange is created and named by default `frenzy_bunnies`
+* Each worker is bound to an AMQP queue named `my_queue_environment` with the environment postfix appended automatically.
+* The routing key on the exchange is of the same name and bound to the queue.
+
+# Contributing
+
+Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :).
+
+
+# Copyright
+
+Copyright (c) 2012 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). See MIT-LICENSE for further details.
+
+

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/bin/frenzy_bunnies

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+require 'frenzy_bunnies'
+require 'frenzy_bunnies/cli'
+
+FrenzyBunnies::CLI.start
+

data/examples/feed.rb

@@ -0,0 +1,20 @@
+require 'rubygems'
+require 'march_hare'
+
+
+
+
+connection = MarchHare.connect(:host => 'localhost')
+channel = connection.create_channel
+channel.prefetch = 10
+
+exchange = channel.exchange('frenzy_bunnies', :type => :direct, :durable => true)
+
+
+
+100_000.times do |i|
+  exchange.publish("hello world! #{i}", :routing_key => 'new.feeds')
+end
+puts "done"
+
+

data/examples/feed_worker.rb

@@ -0,0 +1,33 @@
+$:<< File.expand_path('../lib', File.dirname(__FILE__))
+
+require 'rubygems'
+require 'frenzy_bunnies'
+
+class FeedWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
+
+  def work(msg)
+    puts msg
+    ack!
+  end
+end
+
+class FeedDownloader
+  include FrenzyBunnies::Worker
+  from_queue 'new.downloads', :durable => true
+  def work(msg)
+    puts msg
+    ack!
+  end
+end
+
+f = FrenzyBunnies::Context.new
+
+f.run FeedWorker,FeedDownloader
+
+
+trap "INT" do
+  f.stop
+  exit!
+end

data/examples/feed_workers_bin.rb

@@ -0,0 +1,21 @@
+class FeedWorker
+  include FrenzyBunnies::Worker
+  from_queue 'new.feeds', :prefetch => 20, :threads => 13, :durable => true
+
+  def work(msg)
+    puts msg
+    ack!
+  end
+end
+
+class FeedDownloader
+  include FrenzyBunnies::Worker
+  from_queue 'new.downloads', :durable => true
+  def work(msg)
+    puts msg
+    ack!
+  end
+end
+
+
+

data/frenzy_bunnies.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/frenzy_bunnies/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Dotan Nahum"]
+  gem.email         = ["jondotan@gmail.com"]
+  gem.description   = %q{RabbitMQ JRuby based workers on top of march_hare}
+  gem.summary       = %q{RabbitMQ JRuby based workers on top of march_hare}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "boardintel_frenzy_bunnies"
+  gem.require_paths = ["lib"]
+  gem.version       = FrenzyBunnies::VERSION
+
+  gem.add_runtime_dependency 'march_hare', '~> 2.3'
+  gem.add_runtime_dependency 'thor'
+  gem.add_runtime_dependency 'sinatra'
+  gem.add_runtime_dependency 'atomic'
+  gem.add_runtime_dependency 'json'
+
+  gem.add_development_dependency 'guard-coffeescript'
+  gem.add_development_dependency 'rr'
+  gem.add_development_dependency 'guard-minitest'
+end

data/lib/frenzy_bunnies/cli.rb

@@ -0,0 +1,29 @@
+require 'thor'
+
+
+class FrenzyBunnies::CLI < Thor
+  BUNNIES =<<-EOF
+
+    (\\___/)
+    (='.'=)  Frenzy Bunnies!
+    (")_(")  JRuby based workers on top of march_hare
+
+  EOF
+
+  desc 'run', "run workers from a file"
+  def start_workers(workerfile)
+
+    require workerfile
+    # enumerate all workers
+    workers = []
+    ObjectSpace.each_object(Class){|o| workers << o if o.ancestors.map(&:name).include? "FrenzyBunnies::Worker"}
+    workers.uniq!
+    
+    puts BUNNIES
+
+    c = FrenzyBunnies::Context.new
+    c.logger.info "Discovered #{workers.inspect}"
+    c.run *workers
+    Signal.trap('INT') { c.stop; exit! }
+  end
+end

data/lib/frenzy_bunnies/context.rb

@@ -0,0 +1,40 @@
+require 'logger'
+require 'frenzy_bunnies/web'
+
+class FrenzyBunnies::Context
+  attr_reader :queue_factory, :logger, :env, :opts
+
+  def initialize(opts={})
+    @opts = opts
+    @opts[:host]     ||= 'localhost'
+    @opts[:heartbeat] ||= 5
+    @opts[:web_host] ||= 'localhost'
+    @opts[:web_port] ||= 11333
+    @opts[:web_threadfilter] ||= /^pool-.*/
+    @opts[:env] ||= 'development'
+
+    @env = @opts[:env]
+    @logger = @opts[:logger] || Logger.new(STDOUT)
+    params = {:host => @opts[:host], :heartbeat_interval => @opts[:heartbeat]}
+    (params[:username], params[:password] = @opts[:username], @opts[:password]) if @opts[:username] && @opts[:password]
+    (params[:port] = @opts[:port]) if @opts[:port]
+    @connection = MarchHare.connect(params)
+    @connection.add_shutdown_listener(lambda { |cause| @logger.error("Disconnected: #{cause}"); stop;})
+
+    @queue_factory = FrenzyBunnies::QueueFactory.new(@connection)
+  end
+
+  def run(*klasses)
+    @klasses = []
+    klasses.each{|klass| klass.start(self); @klasses << klass}
+    return nil if @opts[:disable_web_stats]
+    Thread.new do
+      FrenzyBunnies::Web.run_with(@klasses, :host => @opts[:web_host], :port => @opts[:web_port], :threadfilter => @opts[:web_threadfilter], :logger => @logger)
+    end
+  end
+
+  def stop
+    @klasses.each{|klass| klass.stop }
+  end
+end
+

data/lib/frenzy_bunnies/handlers/maxretry.rb

@@ -0,0 +1,199 @@
+require 'base64'
+require 'json'
+
+module FrenzyBunnies
+  module Handlers
+    #
+    # Maxretry uses dead letter policies on Rabbitmq to requeue and retry
+    # messages after failure (rejections, errors and timeouts). When the maximum
+    # number of retries is reached it will put the message on an error queue.
+    # This handler will only retry at the queue level. To accomplish that, the
+    # setup is a bit complex.
+    #
+    # Input:
+    #   worker_exchange (eXchange)
+    #   worker_queue (Queue)
+    # We create:
+    #   worker_queue-retry - (X) where we setup the worker queue to dead-letter.
+    #   worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to
+    #                        worker_queue-retry-requeue.
+    #   worker_queue-error - (X) where to send max-retry failures
+    #   worker_queue-error - (Q) bound to worker_queue-error.
+    #   worker_queue-retry-requeue - (X) exchange to bind worker_queue to for
+    #                                requeuing directly to the worker_queue.
+    #
+    # This requires that you setup arguments to the worker queue to line up the
+    # dead letter queue. See the example for more information.
+    #
+    # Many of these can be override with options:
+    # - retry_exchange - sets retry exchange & queue
+    # - retry_error_exchange - sets error exchange and queue
+    # - retry_requeue_exchange - sets the exchange created to re-queue things
+    #   back to the worker queue.
+    #
+    class Maxretry
+
+      def initialize(channel, queue, logger, opts)
+        @logger = logger
+        @worker_queue_name = queue.name
+        @logger.debug do
+          "#{log_prefix} creating handler, opts=#{opts}"
+        end
+
+        @channel = channel
+        @opts = opts
+
+        # Construct names, defaulting where suitable
+        retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry"
+        error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error"
+        requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue"
+
+        # Create the exchanges
+        @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name|
+          @logger.debug { "#{log_prefix} creating exchange=#{name}" }
+          @channel.exchange(name,
+                            :type => 'topic',
+                            :durable => exchange_durable?)
+        end
+
+        # Create the queues and bindings
+        @logger.debug do
+          "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}"
+        end
+        @retry_queue = @channel.queue(retry_name,
+                                     :durable => queue_durable?,
+                                     :arguments => {
+                                       'x-dead-letter-exchange' => requeue_name,
+                                       'x-message-ttl' => @opts[:retry_timeout] || 60000
+                                     })
+        @retry_queue.bind(@retry_exchange, :routing_key => '#')
+
+        @logger.debug do
+          "#{log_prefix} creating queue=#{error_name}"
+        end
+        @error_queue = @channel.queue(error_name,
+                                      :durable => queue_durable?)
+        @error_queue.bind(@error_exchange, :routing_key => '#')
+
+        # Finally, bind the worker queue to our requeue exchange
+        queue.bind(@requeue_exchange, :routing_key => '#')
+
+        @max_retries = @opts[:retry_max_times] || 5
+      end
+
+      def acknowledge(hdr, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, msg, requeue = false)
+        if requeue
+          # This was explicitly rejected specifying it be requeued so we do not
+          # want it to pass through our retry logic.
+          @channel.reject(hdr.delivery_tag, requeue)
+        else
+          handle_retry(hdr, msg, :reject)
+        end
+      end
+
+
+      def error(hdr, msg, err)
+        handle_retry(hdr, msg, err)
+      end
+
+      def timeout(hdr, msg)
+        handle_retry(hdr, msg, :timeout)
+      end
+
+      def noop(hdr, props, msg)
+
+      end
+
+      # Helper logic for retry handling. This will reject the message if there
+      # are remaining retries left on it, otherwise it will publish it to the
+      # error exchange along with the reason.
+      # @param hdr [MarchHare::Headers]
+      # @param msg [String] The message
+      # @param reason [String, Symbol, Exception] Reason for the retry, included
+      #   in the JSON we put on the error exchange.
+      def handle_retry(hdr, msg, reason)
+        # +1 for the current attempt
+        num_attempts = failure_count(hdr.headers) + 1
+        if num_attempts <= @max_retries
+          # We call reject which will route the message to the
+          # x-dead-letter-exchange (ie. retry exchange) on the queue
+          @logger.info do
+            "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{hdr.headers}"
+          end
+          @channel.reject(hdr.delivery_tag, false)
+          # TODO: metrics
+        else
+          # Retried more than the max times
+          # Publish the original message with the routing_key to the error exchange
+          @logger.info do
+            "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
+          end
+          data = {
+            error: reason,
+            num_attempts: num_attempts,
+            failed_at: Time.now.iso8601,
+            payload: Base64.encode64(msg.to_s)
+          }.tap do |hash|
+            if reason.is_a?(Exception)
+              hash[:error_class] = reason.class.to_s
+              hash[:error_message] = "#{reason}"
+              if reason.backtrace
+                hash[:backtrace] = reason.backtrace.take(10).join(', ')
+              end
+            end
+          end.to_json
+          @error_exchange.publish(data, :routing_key => hdr.routing_key)
+          @channel.acknowledge(hdr.delivery_tag, false)
+          # TODO: metrics
+        end
+      end
+      private :handle_retry
+
+      # Uses the x-death header to determine the number of failures this job has
+      # seen in the past. This does not count the current failure. So for
+      # instance, the first time the job fails, this will return 0, the second
+      # time, 1, etc.
+      # @param headers [Hash] Hash of headers that Rabbit delivers as part of
+      #   the message
+      # @return [Integer] Count of number of failures.
+      def failure_count(headers)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          x_death_array = headers['x-death'].select do |x_death|
+            x_death['queue'] == @worker_queue_name
+          end
+          if x_death_array.count > 0 && x_death_array.first['count']
+            # Newer versions of RabbitMQ return headers with a count key
+            x_death_array.inject(0) {|sum, x_death| sum + x_death['count']}
+          else
+            # Older versions return a separate x-death header for each failure
+            x_death_array.count
+          end
+        end
+      end
+      private :failure_count
+
+      # Prefix all of our log messages so they are easier to find. We don't have
+      # the worker, so the next best thing is the queue name.
+      def log_prefix
+        "Maxretry handler [queue=#{@worker_queue_name}]"
+      end
+      private :log_prefix
+
+      private
+
+      def queue_durable?
+        @opts.fetch(:queue_options, {}).fetch(:durable, false)
+      end
+
+      def exchange_durable?
+        queue_durable?
+      end
+    end
+  end
+end

data/lib/frenzy_bunnies/handlers/oneshot.rb

@@ -0,0 +1,31 @@
+module FrenzyBunnies
+  module Handlers
+    class Oneshot
+      def initialize(channel, queue, logger, opts)
+        @channel = channel
+        @opts    = opts
+        @logger  = logger
+      end
+
+      def acknowledge(hdr, msg)
+        @channel.acknowledge(hdr.delivery_tag, false)
+      end
+
+      def reject(hdr, msg, requeue=false)
+        @channel.reject(hdr.delivery_tag, requeue)
+      end
+
+      def error(hdr, msg, err)
+        reject(hdr, msg)
+      end
+
+      def timeout(hdr, msg)
+        reject(hdr, msg)
+      end
+
+      def noop(hdr, msg)
+
+      end
+    end
+  end
+end

data/lib/frenzy_bunnies/health/collector.rb

@@ -0,0 +1,21 @@
+class FrenzyBunnies::Health::Collector
+  def initialize(opts={})
+    @providers = []
+    Dir["#{File.dirname(__FILE__)}/providers/*.rb"].each do |f| 
+      require f
+      name = File.basename(f, '.*')
+      provider_klass = FrenzyBunnies::Health::Providers.const_get(camelize name)
+      @providers << provider_klass.new(opts[name.to_sym])
+    end
+  end
+
+  def collect
+    @providers.map{|p| p.report }.inject(:merge)
+  end
+
+  # real basic camelizer, beware!. meant to avoid including active-support here.
+  def camelize(str)
+    str.split('_').map {|s| s.capitalize}.join
+  end
+end
+