gongren 0.1.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 François Beausoleil
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= gongren
+
+Gongren distributes jobs to workers, with support for failed worker daemons and load balancing.
+
+Gongren is Chinese worker: 工人.  See http://www.ehow.com/video_4403851_say-worker-chinese.html for pronunciation.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 François Beausoleil. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,82 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "gongren"
+    gem.summary = %Q{Gongren distributes jobs to workers, with support for failed worker daemons and load balancing.}
+    gem.description = %Q{A gem that's currently tied to Rails to distribute jobs.}
+    gem.email = "francois@teksol.info"
+    gem.homepage = "http://github.com/francois/gongren"
+    gem.authors = ["François Beausoleil"]
+    gem.require_paths << "vendor/qusion/lib"
+    gem.files = FileList["lib/**/*", "vendor/**/*", "LICENSE", "README.rdoc", "Rakefile"]
+    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_development_dependency "yard", ">= 0"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+
+    gem.add_dependency "amqp", "~> 0.6.6"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+begin
+  require 'reek/adapters/rake_task'
+  Reek::RakeTask.new do |t|
+    t.fail_on_error = true
+    t.verbose = false
+    t.source_files = 'lib/**/*.rb'
+  end
+rescue LoadError
+  task :reek do
+    abort "Reek is not available. In order to run reek, you must: sudo gem install reek"
+  end
+end
+
+begin
+  require 'roodi'
+  require 'roodi_task'
+  RoodiTask.new do |t|
+    t.verbose = false
+  end
+rescue LoadError
+  task :roodi do
+    abort "Roodi is not available. In order to run roodi, you must: sudo gem install roodi"
+  end
+end
+
+task :default => :test
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new
+rescue LoadError
+  task :yardoc do
+    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
+  end
+end

data/lib/gongren.rb

@@ -0,0 +1,4 @@
+# Nothing interesting here: see {Gongren::Server} or {Gongren::Worker}.
+module Gongren
+  VERSION = "0.0.1"
+end

data/lib/gongren/server.rb

@@ -0,0 +1,58 @@
+require "logger"
+require "qusion"
+
+module Gongren
+  # This version is intimately tied to Rails.
+  class Server
+    def initialize(options={})
+      @options = options.inject(Hash.new) {|memo, (k,v)| memo[k.to_sym] = v; memo} # #symbolize_keys
+      @logger  = options[:logger] || Logger.new(options[:log] || STDERR)
+    end
+
+    # Submits a unit of work to the pool of workers.
+    def self.submit(name, unit)
+      data = Marshal.dump(unit)
+      topic.publish(data, :persistent => true, :key => "unit.#{name}")
+    end
+
+    # A quick way to instantiate a server with some options.
+    def self.start(options={})
+      new(options).start
+    end
+
+    # Starts the reactor / event loop.
+    def start
+      logger.info { "Gongren::Server #{Process.pid} starting" }
+      Qusion.start(@options)
+      self.class.control_topic # Instantiates the control topic, for later use
+    end
+
+    private
+
+    attr_reader :options, :logger
+
+    def self.exchange_name
+      "gongren.work"
+    end
+
+    def self.exchange_options
+      {:durable => true}
+    end
+
+    def self.topic
+      Qusion.channel.topic(exchange_name, exchange_options)
+    end
+
+    def self.control_exchange_name
+      "gongren.worker.control"
+    end
+
+    def self.control_exchange_options
+      {}
+    end
+
+    def self.control_topic
+      Qusion.channel.topic(control_exchange_name, control_exchange_options)
+    end
+  end
+end

data/lib/gongren/worker.rb

@@ -0,0 +1,166 @@
+require "json"
+require "mq"
+require "logger"
+
+module Gongren
+  # A worker does work.  Units of work are received and processed locally.  All units of work will be
+  # acknowledged back to the server, ensuring units of work are executed exactly once.
+  #
+  # Units of work are received as Hashes from the {Gongren::Server}.  The Hash that is passed to the
+  # #run block will be dynamically injected with {Gongren::Worker::Unit}, which includes the {Gongren::Worker::Unit#ack}
+  # method.  If the block returns and the message hasn't been acknowledged, it will be for you.
+  #
+  # == Notes on use
+  #
+  # If you do any database work, it is important to wrap your work in a transaction, because if your
+  # worker dies, the same work unit will be resubmitted.
+  #
+  # @example
+  #
+  #   # In a Rails context, this would live in script/gongren-worker:
+  #   require File.dirname(__FILE__) + "/../config/environment"
+  #   require "gongren/worker"
+  #
+  #   Gengren::Worker.run do |unit|
+  #     ActiveRecord::Base.transaction do
+  #       klass_name = unit[:class_name]
+  #       klass      = klass_name.constantize
+  #       instance   = klass.find(unit[:id])
+  #       results    = instance.send(unit[:selector], *unit[:args])
+  #     end
+  #
+  #     # We don't have two phase commit yet, this acknowledging outside the
+  #     # transaction might execute a message twice.
+  #     unit.ack
+  #   end
+  class Worker
+    def initialize(options={})
+      @options = options.inject(Hash.new) {|memo, (k,v)| memo[k.to_sym] = v; memo} # #symbolize_keys
+      @logger  = options[:logger] || Logger.new(options[:log] || STDERR)
+    end
+
+    def run
+      raise ArgumentError, "#run must be called with a block" unless block_given?
+
+      logger.info { "Gongren::Worker #{worker_id} ready to work" }
+
+      EM.run do
+        MQ.queue(control_queue_name, control_queue_options).bind(control_exchange_name, control_exchange_options) do |header, data|
+          message = Marshal.load(data)
+          logger.info { message.inspect }
+
+          if message[:selector].to_s.strip.empty? then
+            logger.error { "Received control request without :selector key: ignoring" }
+          else
+            begin
+              send(message[:selector], message)
+            rescue Exception => e
+              log_failure(header, message, e)
+            end
+          end
+        end
+
+        MQ.queue(queue_name, queue_options).bind(exchange_name, exchange_options).subscribe do |header, data|
+          message = Marshal.load(data)
+          class << message; include Unit; end # Dynamically add our #ack method
+          message.gongren_header = header
+
+          logger.info { message.inspect }
+
+          begin
+            yield message
+
+            # Automatically ack messages, but do it only once
+            logger.debug { "Block ack'd? #{message.acked?}" }
+            unless message.acked?
+              logger.debug { "Ack'ing for the block" }
+              message.ack
+            end
+          rescue Exception => e
+            log_failure(header, message, e)
+          end
+        end
+      end
+    end
+
+    # A quick way to run a worker. Creates an instance with the options and runs the block
+    # whenever a message is received, passing the exact object that was sent from the server.
+    def self.run(options={}, &block)
+      new(options).run(&block)
+    end
+
+    module Unit
+      attr_writer :gongren_header
+
+      def ack
+        @gongren_header.ack
+        @acked = true
+      end
+
+      def acked?
+        !!@acked
+      end
+    end
+
+    private
+
+    def log_failure(header, message, exception)
+      logger.error do
+        <<-EOF.gsub(/^\s{10}/, "")
+          ============
+          #{header.inspect}
+          #{message.inspect}
+          message.acked? #{message.acked?}
+          ============
+          #{e.class_name}: #{e.message}
+          #{e.backtrace.join("\n")}.
+        EOF
+      end
+    end
+
+    def ping(message)
+      return logger.error { "Received #ping message without corresponding :exchange key: ignoring" } if message[:exchange].to_s.strip.empty?
+      MQ.direct(data[:exchange]).publish(worker_id)
+    end
+
+    attr_reader :logger, :options
+
+    def queue_name
+      "gongren.worker"
+    end
+
+    def queue_options
+      # We want a durable queue: one that survives server restarts, and that will hold messages until
+      # a worker is available to grab them, even if all worker process are down.
+      {:durable => true}
+    end
+
+    def exchange_name
+      "gongren.work"
+    end
+
+    def exchange_options
+      {:key => "unit.#"}
+    end
+
+    def control_queue_name
+      "gongren.worker.control.#{worker_id}"
+    end
+
+    def control_queue_options
+      {}
+    end
+
+    def control_exchange_name
+      "gongren.worker.control"
+    end
+
+    def control_exchange_options
+      {}
+    end
+
+    def worker_id
+      "#{Process.pid}.#{Thread.current.object_id}"
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'gongren'
+
+class Test::Unit::TestCase
+end

data/test/test_gongren.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestGongren < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

data/vendor/qusion/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Daniel DeLeo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+

data/vendor/qusion/README.rdoc

@@ -0,0 +1,102 @@
+= Qusion
+Qusion makes AMQP[http://github.com/tmm1/amqp] work with your webserver with no fuss. It's a simple library/plugin with three features:
+* A set of monkey patches that sets up the required callbacks and/or worker threads so that AMQP will work with Passenger, Thin, or Mongrel. WEBrick, SCGI, and Evented Mongrel are experimentally supported, but not heavily tested.
+* A Channel Pool. You can cause problems for yourself if you create new channels (with MQ.new) for every request. The pool sets up a few of these when your app starts and reuses them.
+* YAML configuration files. If you're using Rails or Merb, create config/amqp.yml, then fill in the details for development, test, and production. Use Qusion.start() in your environment.rb file and you're good to go.
+
+= Before You Start
+Qusion makes it easy to just install the plugin and start using AMQP in your application. But there are many ways to use background jobs within a Rails app, so it's worth taking some time to consider the tradeoffs of each approach.
+
+* If your background job needs are simple and you're using a relational database, Delayed::Job[http://github.com/tobi/delayed_job/] lets you schedule background tasks through the database. You won't need to run another application (the AMQP Broker) to keep your app running. 
+* The 0.6.x version of the ruby amqp library may drop messages when the AMQP broker goes down. Pivotal Labs has discussed this problem on their blog[http://pivots.pivotallabs.com/users/will/blog/articles/966-how-to-not-test-rabbitmq-part-1]. This issue will likely be addressed in the 0.7.0 release of amqp, but can be avoided entirely using a synchronous amqp library such as bunny[http://github.com/celldee/bunny]. For a ready-made background job solution using Bunny to publish jobs to the queue, see Minion[http://github.com/orionz/minion/].
+* Qusion runs EventMachine in a separate thread on Phusion Passenger, Mongrel, and other non-evented servers. There are some inefficiencies in Ruby 1.8's threading model that make running EM in a thread quite slow. Joe Damato and Aman Gupta have created a patch[http://github.com/ice799/matzruby/tree/heap_stacks] for the problem which is included in an experimental branch of REE. You can learn more about the patch from Phusion's Blog[http://blog.phusion.nl/2009/12/15/google-tech-talk-on-ruby-enterprise-edition/]. 
+
+= Getting Started
+First you'll need the amqp library and a working RabbitMQ installation. This entails:
+* Install Erlang for your platform
+* Install RabbitMQ for your platform
+* (sudo) gem install amqp
+Ezmobius has a good walk-through on the readme for nanite[http://github.com/ezmobius/nanite/] if you haven't done this yet.
+== Install Qusion
+Start by installing Qusion as a plugin:
+
+  script/plugin install git://github.com/danielsdeleo/qusion.git
+
+Next, in your config/environment.rb, add something like:
+  
+  # Add eventmachine and amqp gems to config.gem to get config.gem goodies:
+  config.gem "eventmachine"
+  config.gem "amqp"
+
+  # Start AMQP after rails loads:
+  config.after_initialize do
+    Qusion.start # no options needed if you're using config/amqp.yml or the default settings.
+  end
+  
+And that's it! This will set up AMQP for any ruby app server (tested on mongrel, thin, and passenger). Now, you can use all of AMQP's functionality as normal. In your controllers or models, you might have:
+
+  MQ.new.queue("my-work-queue").publish("do work, son!")
+  
+and it should just work.
+
+= Channel Pools
+It's considered bad practice to use MQ.new over and over, as it creates a new AMQP channel, and that creates a new Erlang process in RabbitMQ. Erlang processes are super light weight, but you'll be wasting them and causing the Erlang VM GC headaches if you create them wantonly. So don't do that. Instead, use the channel pool provided by Qusion. It's simple: wherever you'd normally put MQ.new, just replace it with Qusion.channel. Examples:
+
+  # Create a queue:
+  Qusion.channel.queue("my-worker-queue")
+  # Topics:
+  Qusion.channel.topic("my-topic-exchange")
+  # etc.
+  
+This feature is a bit experimental, so the optimal pool size isn't known yet. The default is 5. You can change it by adding something like the following to your environment.rb:
+
+  Qusion.channel_pool_size(3)
+  
+= Configuration
+If you're using rails or merb, you can put your AMQP server details in config/amqp.yml and Qusion will load it when you call Qusion.start(). Example:
+  
+  # Put this in config/amqp.yml
+  development:
+    host: localhost
+    port: 5672
+    user: guest
+    pass: guest
+    vhost: /
+    timeout: 3600
+    logging: false
+    ssl: false
+  
+  test:
+    host: localhost
+    port: 5672
+    ...
+  
+  production:
+    host: localhost
+    port: 5672
+    ...
+
+If you're too hardcore for rails or merb (maybe you're using Sinatra or Ramaze), you can still use a YAML config file, but there's no support for different environments. So do something like this:
+
+  # Tell Qusion where your config file is:
+  Qusion.start("/path/to/amqp.yml")
+  
+  # Your configuration looks like this:
+  application:
+    host: localhost
+    port: 5672
+    ...
+
+If you just want to get started without configuring anything, Qusion.start() will use the default options if it can't find a config file. And, finally, you can give options directly to Qusion.start() like this:
+
+  Qusion.start(:host => "my-amqp-broker.mydomain.com", :user => "me", :pass => "am_I_really_putting_this_in_VCS?")
+
+
+= Bugs? Hacking?
+If you find any bugs, or feel the need to add a feature, fork away. You can also contact me directly via the email address in my profile if you have any quesions.
+
+= Shouts
+* Qusion's code for Phusion Passenger's starting_worker_process event was originally posted by Aman Gupta (tmm1[http://github.com/tmm1]) on the AMQP list[http://groups.google.com/group/ruby-amqp]
+* Brightbox's Warren[http://github.com/brightbox/warren] library provides some similar functionality. It doesn't support webserver-specific EventMachine setup, but it does have built-in encryption and support for the synchronous (non-EventMachine) Bunny[http://github.com/celldee/bunny] AMQP client.
+
+dan@kallistec.com

data/vendor/qusion/Rakefile

@@ -0,0 +1,35 @@
+# encoding: UTF-8
+require "spec/rake/spectask"
+require "cucumber"
+require "cucumber/rake/task"
+
+task :default => :spec
+
+desc "Run Cucumber Features"
+Cucumber::Rake::Task.new do |t|
+  t.cucumber_opts = "-c -n"
+end
+
+desc "Run all of the specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "spec/spec.opts"]
+  t.fail_on_error = false
+end
+
+namespace :spec do
+
+  desc "Generate HTML report for failing examples"
+  Spec::Rake::SpecTask.new('report') do |t|
+    t.spec_files = FileList['failing_examples/**/*.rb']
+    t.spec_opts = ["--format", "html:doc/tools/reports/failing_examples.html", "--diff", '--options', '"spec/spec.opts"']
+    t.fail_on_error = false
+  end
+
+  desc "Run all spec with RCov" 
+  Spec::Rake::SpecTask.new(:rcov) do |t|
+    t.rcov = true
+    t.rcov_dir = 'doc/tools/coverage/'
+    t.rcov_opts = ['--exclude', 'spec']
+  end
+
+end

data/vendor/qusion/init.rb

@@ -0,0 +1 @@
+# Don't need to do anything special.

data/vendor/qusion/install.rb

@@ -0,0 +1 @@
+# Nothing to see here (yet), folks

data/vendor/qusion/lib/qusion.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+unless defined?(QUSION_ROOT)
+  QUSION_ROOT = File.dirname(__FILE__) + '/'
+end
+
+require "eventmachine"
+require "mq"
+
+require QUSION_ROOT + "qusion/server_spy"
+require QUSION_ROOT + "qusion/em"
+require QUSION_ROOT + "qusion/amqp"
+require QUSION_ROOT + "qusion/channel_pool"
+require QUSION_ROOT + "qusion/amqp_config"
+
+module Qusion
+  def self.start(*opts)
+    amqp_opts = AmqpConfig.new(*opts).config_opts
+    AMQP.start_web_dispatcher(amqp_opts)
+  end
+  
+  def self.channel
+    ChannelPool.instance.channel
+  end
+  
+  def self.channel_pool_size(new_pool_size)
+    ChannelPool.pool_size = new_pool_size
+  end
+end

data/vendor/qusion/lib/qusion/amqp.rb

@@ -0,0 +1,31 @@
+# encoding: UTF-8
+
+module AMQP
+  def self.start_web_dispatcher(amqp_settings={})
+    @settings = settings.merge(amqp_settings)
+    case Qusion::ServerSpy.server_type
+    when :passenger
+      PhusionPassenger.on_event(:starting_worker_process) do |forked| 
+        if forked
+          EM.kill_reactor
+          Thread.current[:mq], @conn = nil, nil
+        end 
+        Thread.new { start }
+        die_gracefully_on_signal
+      end
+    when :standard
+      Thread.new { start }
+      die_gracefully_on_signal
+    when :evented
+      die_gracefully_on_signal
+    when :none
+    else
+      raise ArgumentError, "AMQP#start_web_dispatcher requires an argument of [:standard|:evented|:passenger|:none]"
+    end
+  end
+  
+  def self.die_gracefully_on_signal
+    Signal.trap("INT")  { AMQP.stop { EM.stop } }
+    Signal.trap("TERM") { AMQP.stop { EM.stop } }
+  end
+end

data/vendor/qusion/lib/qusion/amqp_config.rb

@@ -0,0 +1,70 @@
+# encoding: UTF-8
+require 'yaml'
+
+module Qusion
+  
+  class AmqpConfig
+    attr_reader :config_path, :framework_env
+    
+    def initialize(opts=nil)
+      if opts && opts.respond_to?(:keys)
+        @config_path = nil
+        @config_opts = opts
+      elsif opts
+        @config_path = opts
+      else
+        load_framework_config
+      end
+    end
+    
+    def load_framework_config
+      if defined?(RAILS_ROOT)
+        @config_path = RAILS_ROOT + "/config/amqp.yml"
+        @framework_env = RAILS_ENV
+      elsif defined?(Merb)
+        @config_path = Merb.root + "/config/amqp.yml"
+        @framework_env = Merb.environment
+      else
+        nil
+      end
+    end
+    
+    def config_opts
+      @config_opts ||= load_config_opts
+    end
+    
+    def load_config_opts
+      if config_path && config_from_yaml = load_amqp_config_file
+        if framework_env
+          framework_amqp_opts(config_from_yaml)
+        else
+          amqp_opts(config_from_yaml)
+        end
+      else
+        {}
+      end
+    end
+    
+    def framework_amqp_opts(config_hash)
+      symbolize_keys(config_hash[framework_env.to_s])
+    end
+    
+    def amqp_opts(config_hash)
+      symbolize_keys(config_hash.first.last)
+    end
+    
+    def symbolize_keys(config_hash)
+      symbolized_hsh = {}
+      config_hash.each {|option, value| symbolized_hsh[option.to_sym] = value }
+      symbolized_hsh
+    end
+    
+    def load_amqp_config_file
+      begin
+        YAML.load_file(config_path)
+      rescue Errno::ENOENT
+      end
+    end
+    
+  end
+end

data/vendor/qusion/lib/qusion/channel_pool.rb

@@ -0,0 +1,64 @@
+# encoding: UTF-8
+require "singleton"
+
+module Qusion
+  
+  # ChannelPool maintains a pool of AMQP channel objects that can be reused.
+  # The motivation behind this is that if you were to use MQ.new to create a 
+  # new channel for every request, your AMQP broker could be swamped trying to
+  # maintain a bunch of channels that you're only using once.
+  #
+  # To use the channel pool, just replace <tt>MQ.new</tt> in your code with <tt>Qusion.channel</tt>
+  # 
+  #   # Instead of this:
+  #   MQ.new.queue("my-worker-queue")
+  #   # Do this:
+  #   Qusion.channel.queue("my-worker-queue")
+  #
+  # By default, ChannelPool maintains a pool of 5 channels. This can be adjusted with
+  # <tt>ChannelPool.pool_size=()</tt> or <tt>Qusion.channel_pool_size()</tt>
+  # The optimal pool size is not yet known, but I suspect you might need a 
+  # larger value if using Thin in production, and a smaller value otherwise.
+  class ChannelPool
+    include Singleton
+    
+    class << self
+      
+      def pool_size=(new_pool_size)
+        reset
+        @pool_size = new_pool_size
+      end
+      
+      def pool_size
+        @pool_size ||= 5
+      end
+      
+      def reset
+        @pool_size = nil
+        instance.reset
+      end
+      
+    end
+    
+    attr_reader :pool
+    
+    def channel
+      @i ||= 1
+      @i = (@i + 1) % pool_size
+      pool[@i]
+    end
+    
+    def pool
+      @pool ||= Array.new(pool_size) { MQ.new }
+    end
+    
+    def reset
+      @pool = nil
+    end
+    
+    def pool_size
+      self.class.pool_size
+    end
+    
+  end
+end

data/vendor/qusion/lib/qusion/em.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+
+module EM
+  
+  def self.kill_reactor
+    if self.reactor_running? 
+      self.stop_event_loop 
+      self.release_machine 
+      @reactor_running = false
+    end 
+  end
+  
+end

data/vendor/qusion/lib/qusion/server_spy.rb

@@ -0,0 +1,24 @@
+# encoding: UTF-8
+
+module Qusion
+  module ServerSpy
+    extend self
+    def server_type
+      if defined?(::PhusionPassenger)
+        :passenger
+      elsif defined?(::Mongrel) && defined?(::Mongrel::MongrelProtocol)
+        :evented
+      elsif defined?(::Mongrel)
+        :standard
+      elsif defined?(::SCGI)
+        :standard
+      elsif defined?(::WEBrick)
+        :standard
+      elsif defined?(::Thin)
+        :evented
+      else
+        :none
+      end
+    end
+  end
+end

data/vendor/qusion/spec/fixtures/framework-amqp.yml

@@ -0,0 +1,28 @@
+development:
+  host: localhost
+  port: 5672
+  user: guest
+  pass: guest
+  vhost: /
+  timeout: 600
+  logging: false
+  ssl: false
+  
+test:
+  host: localhost
+  port: 5672
+  user: guest
+  pass: guest
+  vhost: /
+  logging: false
+  ssl: false
+  
+production:
+  host: localhost
+  port: 5672
+  user: guest
+  pass: guest
+  vhost: /
+  timeout: 3600
+  logging: false
+  ssl: false

data/vendor/qusion/spec/fixtures/hardcoded-amqp.yml

@@ -0,0 +1,9 @@
+application:
+  host: localhost
+  port: 5672
+  user: guest
+  pass: guest
+  vhost: /
+  timeout: 600
+  logging: false
+  ssl: false

data/vendor/qusion/spec/spec.opts

@@ -0,0 +1 @@
+-f specdoc -c -t 2

data/vendor/qusion/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+# encoding: UTF-8
+begin
+  require 'rubygems'
+rescue LoadError
+end
+
+require File.dirname(__FILE__) + '/../lib/qusion.rb'
+
+include Qusion
+

data/vendor/qusion/spec/unit/amqp_config_spec.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe AmqpConfig do
+  
+  after(:each) do
+    Object.send(:remove_const, :RAILS_ROOT) if defined? ::RAILS_ROOT
+    Object.send(:remove_const, :Merb) if defined? ::Merb
+  end
+  
+  it "should use RAILS_ROOT/config/amqp.yml if RAILS_ROOT is defined" do
+    ::RAILS_ROOT = "/path/to/rails"
+    ::RAILS_ENV = nil
+    AmqpConfig.new.config_path.should == "/path/to/rails/config/amqp.yml"
+  end
+  
+  it "should use \#{Merb.root}/config/amqp.yml if RAILS_ROOT is undefined and Merb is defined" do
+    ::Merb = mock("merby")
+    ::Merb.should_receive(:root).and_return("/path/to/merb")
+    ::Merb.should_receive(:environment).and_return(nil)
+    AmqpConfig.new.config_path.should == "/path/to/merb/config/amqp.yml"
+  end
+  
+  it "should use the provided path no matter what" do
+    ::RAILS_ROOT = nil
+    ::Merb = nil
+    path = AmqpConfig.new("/custom/path/to/amqp.yml").config_path
+    path.should == "/custom/path/to/amqp.yml"
+  end
+  
+  it "should use a provided options hash if given" do
+    ::RAILS_ROOT = nil
+    ::Merb = nil
+    conf = AmqpConfig.new(:host => "my-broker.mydomain.com")
+    conf.config_path.should be_nil
+    conf.config_opts.should == {:host => "my-broker.mydomain.com"}
+  end
+  
+  it "should use the default amqp options in rails if amqp.yml doesn't exist" do
+    ::RAILS_ROOT = File.dirname(__FILE__) + '/../'
+    AmqpConfig.new.config_opts.should == {}
+  end
+  
+  it "should load a YAML file when using a framework" do
+    conf = AmqpConfig.new
+    conf.stub!(:config_path).and_return(File.dirname(__FILE__) + "/../fixtures/framework-amqp.yml")
+    conf.stub!(:framework_env).and_return("production")
+    conf.config_opts.should == {:host => 'localhost',:port => 5672,
+                                :user => 'guest', :pass => 'guest',
+                                :vhost => '/', :timeout => 3600,
+                                :logging => false, :ssl => false}
+  end
+  
+  it "should use the first set of opts when given a explicit file path" do
+    conf = AmqpConfig.new
+    conf.stub!(:config_path).and_return(File.dirname(__FILE__) + "/../fixtures/hardcoded-amqp.yml")
+    conf.config_opts.should == {:host => 'localhost',:port => 5672,
+                                :user => 'guest', :pass => 'guest',
+                                :vhost => '/', :timeout => 600,
+                                :logging => false, :ssl => false}
+  end
+  
+end

data/vendor/qusion/spec/unit/amqp_spec.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe AMQP do
+  
+  before do
+    AMQP.stub!(:settings).and_return({})
+  end
+  
+  after(:each) do
+    Object.send(:remove_const, :PhusionPassenger) if defined? ::PhusionPassenger
+    Object.send(:remove_const, :Thin) if defined? ::Thin
+    Object.send(:remove_const, :Mongrel) if defined? ::Mongrel
+  end
+  
+  it "should kill the reactor and start a new AMQP connection when forked in Passenger" do
+    AMQP.should_receive(:die_gracefully_on_signal)
+    ::PhusionPassenger = Module.new
+    forked = mock("starting_worker_process_callback_obj")
+    ::PhusionPassenger.should_receive(:on_event).with(:starting_worker_process).and_yield(forked)
+    EM.should_receive(:kill_reactor)
+    AMQP.should_receive(:start)
+    AMQP.start_web_dispatcher
+    sleep 0.1 # give the thread time to run, esp. on ruby 1.9
+  end
+  
+  it "should set AMQP's connection settings when running under Thin" do
+    AMQP.should_receive(:die_gracefully_on_signal)
+    ::Thin = Module.new
+    AMQP.start_web_dispatcher({:cookie => "yummy"})
+    AMQP.instance_variable_get(:@settings)[:cookie].should == "yummy"
+  end
+  
+  it "should start a worker thread when running under Mongrel" do
+    AMQP.should_receive(:die_gracefully_on_signal)
+    ::Mongrel = Module.new
+    AMQP.should_receive(:start)
+    AMQP.start_web_dispatcher
+    sleep 0.1 # give the thread time to run, esp. on ruby 1.9
+  end
+  
+end

data/vendor/qusion/spec/unit/channel_pool_spec.rb

@@ -0,0 +1,48 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe ChannelPool do
+  MQ = Object.new
+  
+  before(:each) do
+    ChannelPool.reset
+    @channel_pool = ChannelPool.instance
+  end
+  
+  it "should be singleton" do
+    lambda { ChannelPool.new }.should raise_error
+  end
+  
+  it "should adjust the pool size" do
+    ChannelPool.pool_size = 5
+    ChannelPool.pool_size.should == 5
+  end
+  
+  it "should reset itself when the pool size is set" do
+    ChannelPool.should_receive(:reset)
+    ChannelPool.pool_size = 23
+  end
+  
+  it "should create a pool of AMQP channels" do
+    ChannelPool.pool_size = 3
+    ::MQ.should_receive(:new).exactly(3).times
+    @channel_pool.pool
+  end
+  
+  it "should default to a pool size of 5" do
+    ::MQ.should_receive(:new).exactly(5).times.and_return("swanky")
+    @channel_pool.pool
+    @channel_pool.instance_variable_get(:@pool).should == %w{ swanky swanky swanky swanky swanky}
+  end
+  
+  it "should return a channel in a round-robin" do
+    @channel_pool.instance_variable_set(:@pool, [1,2,3,4,5])
+    @channel_pool.channel.should == 3
+    @channel_pool.channel.should == 4
+    @channel_pool.channel.should == 5
+    @channel_pool.channel.should == 1
+    @channel_pool.channel.should == 2
+    @channel_pool.channel.should == 3
+  end
+  
+end

data/vendor/qusion/spec/unit/em_spec.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe EM do
+  
+  it "should kill the reactor for forking" do
+    EM.should_receive(:reactor_running?).and_return(true)
+    EM.should_receive(:stop_event_loop)
+    EM.should_receive(:release_machine)
+    EM.kill_reactor
+    EM.instance_variable_get(:@reactor_running).should be_false
+  end
+  
+end

data/vendor/qusion/spec/unit/qusion_spec.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe "Qusion Convenience Methods" do
+  
+  it "should get a channel from the pool" do
+    channel_pool = mock("channel pool")
+    ChannelPool.should_receive(:instance).and_return(channel_pool)
+    channel_pool.should_receive(:channel)
+    Qusion.channel
+  end
+  
+  it "should set the channel pool size" do
+    ChannelPool.should_receive(:pool_size=).with(7)
+    Qusion.channel_pool_size(7)
+  end
+  
+  it "should load the configuration and setup AMQP for the webserver" do
+    config = mock("config")
+    AmqpConfig.should_receive(:new).and_return(config)
+    config.should_receive(:config_opts).and_return("tasty cookie")
+    AMQP.should_receive(:start_web_dispatcher).with("tasty cookie")
+    Qusion.start
+  end
+  
+end

data/vendor/qusion/spec/unit/server_spy_spec.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+require File.dirname(__FILE__) + "/../spec_helper"
+
+describe ServerSpy do
+  
+  after do
+    Object.send(:remove_const, :SCGI) if defined? ::SCGI
+    Object.send(:remove_const, :WEBrick) if defined? ::WEBrick
+    Object.send(:remove_const, :PhusionPassenger) if defined? ::PhusionPassenger
+    Object.send(:remove_const, :Thin) if defined? ::Thin
+    Mongrel.send(:remove_const, :MongrelProtocol) if defined?(::Mongrel::MongrelProtocol)
+    Object.send(:remove_const, :Mongrel) if defined? ::Mongrel
+  end
+  
+  it "maps evented mongrel to :evented" do
+    ::Mongrel = Module.new
+    ::Mongrel::MongrelProtocol = Module.new
+    ServerSpy.server_type.should == :evented
+  end
+  
+  it "maps Mongrel to :standard" do
+    ::Mongrel = Module.new
+    ServerSpy.server_type.should == :standard
+  end
+  
+  it "maps WEBrick to :standard" do
+    ::WEBrick = Module.new
+    ServerSpy.server_type.should == :standard
+  end
+  
+  it "maps SCGI to :standard" do
+    ::SCGI = Module.new
+    ServerSpy.server_type.should == :standard
+  end
+  
+  it "maps PhusionPassenger to :passenger" do
+    ::PhusionPassenger = Module.new
+    ServerSpy.server_type.should == :passenger
+  end
+  
+  it "maps Thin to :evented" do
+    ::Thin = Module.new
+    ServerSpy.server_type.should == :evented
+  end
+  
+  # Rails after 2.2(?) to edge circa Aug 2009 loads thin if it's installed no matter what
+  it "gives the server type as :standard if both Thin and Mongrel are defined" do
+    ::Mongrel = Module.new
+    ::Thin = Module.new
+    ServerSpy.server_type.should == :standard
+  end
+  
+  it "gives the server type as :passenger if both Thin and PhusionPassenger" do
+    ::PhusionPassenger = Module.new
+    ::Thin = Module.new
+    ServerSpy.server_type.should == :passenger
+  end
+  
+  it "gives the server type as :none if no supported server is found" do
+    ServerSpy.server_type.should == :none
+  end
+  
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: gongren
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- "Fran\xC3\xA7ois Beausoleil"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-01-16 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: yard
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 0.6.6
+    version: 
+description: A gem that's currently tied to Rails to distribute jobs.
+email: francois@teksol.info
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- lib/gongren.rb
+- lib/gongren/server.rb
+- lib/gongren/worker.rb
+- vendor/qusion/LICENSE
+- vendor/qusion/README.rdoc
+- vendor/qusion/Rakefile
+- vendor/qusion/init.rb
+- vendor/qusion/install.rb
+- vendor/qusion/lib/qusion.rb
+- vendor/qusion/lib/qusion/amqp.rb
+- vendor/qusion/lib/qusion/amqp_config.rb
+- vendor/qusion/lib/qusion/channel_pool.rb
+- vendor/qusion/lib/qusion/em.rb
+- vendor/qusion/lib/qusion/server_spy.rb
+- vendor/qusion/spec/fixtures/framework-amqp.yml
+- vendor/qusion/spec/fixtures/hardcoded-amqp.yml
+- vendor/qusion/spec/spec.opts
+- vendor/qusion/spec/spec_helper.rb
+- vendor/qusion/spec/unit/amqp_config_spec.rb
+- vendor/qusion/spec/unit/amqp_spec.rb
+- vendor/qusion/spec/unit/channel_pool_spec.rb
+- vendor/qusion/spec/unit/em_spec.rb
+- vendor/qusion/spec/unit/qusion_spec.rb
+- vendor/qusion/spec/unit/server_spy_spec.rb
+has_rdoc: true
+homepage: http://github.com/francois/gongren
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+- vendor/qusion/lib
+- vendor/qusion/lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Gongren distributes jobs to workers, with support for failed worker daemons and load balancing.
+test_files: 
+- test/helper.rb
+- test/test_gongren.rb