famoseagle-sweat_shop 0.3.0

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-01-13
+
+* 1 major enhancement
+
+  * Birthday!
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Amos Elliston
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,65 @@
+# SweatShop
+
+SweatShop provides an api to background resource intensive tasks. Much of the api design was copied from Workling, with a few tweaks.
+Currently, it runs kestrel, but it can support any number of queues.
+
+## Installing 
+
+   gem install sweat_shop
+   freeze in your rails directory
+   cd vendor/gems/sweat_shop
+   rake setup
+
+## Writing workers
+
+Put `email_worker.rb` into app/workers and sublcass `SweatShop::Worker`:
+
+    class EmailWorker
+      def send_mail(to)
+        user = User.find_by_id(to)
+        Mailer.deliver_welcome(to)
+      end
+    end
+
+Then, anywhere in your app you can execute:
+
+    EmailWorker.async_send_mail(1)
+
+The `async` signifies that this task will be placed on a queue to be serviced by the EmailWorker possibly on another machine. You can also
+call:
+
+    EmailWorker.send_mail(1)
+
+That will do the work immediately, without placing the task on the queue. You can also define a `queue_group` at the top of the file
+which will allow you to split workers out into logical groups. This is important if you have various machines serving different
+queues. 
+
+## Running the queue
+
+SweatShop has been tested with Kestrel, but it will also work with Starling. You can install and start kestrel following the instructions here:
+
+http://github.com/robey/kestrel/tree/master
+
+config/sweatshop.yml specifies the machine address of the queue (default localhost:22133).
+
+## Running the workers
+
+Assuming you ran `rake setup` in Rails, you can type:
+
+    script/sweatshop
+
+By default, the script will run all workers defined in the app/workers dir. Every task will be processed on each queue using a round-robin algorithm. You can also add the `-d` flag which will put the worker in daemon mode. The daemon also takes other params.  Add a `-h` for more details.
+
+    script/sweatshop -d
+    script/sweatshop -d stop
+
+If you would like to run SweatShop as a daemon on a linux machine, use the initd.sh script provided in the sweat_shop/script dir.
+
+# REQUIREMENTS
+
+    i_can_daemonize
+    memcache (for kestrel)
+
+# LICENSE
+
+Copyright (c) 2009 Amos Elliston, Geni.com; Published under The MIT License, see License

data/Rakefile

@@ -0,0 +1,41 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rcov/rcovtask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |s|
+    s.name = "sweat_shop"
+    s.summary = %Q{SweatShop is a simple asynchronous worker queue build on top of rabbitmq/ampq}
+    s.email = "amos@geni.com"
+    s.homepage = "http://github.com/famoseagle/sweat-shop"
+    s.description = "TODO"
+    s.authors = ["Amos Elliston"]
+    s.files =  FileList["[A-Z]*", "{lib,test,config}/**/*"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+Rake::TestTask.new
+
+Rake::RDocTask.new do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'new_project'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Rcov::RcovTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test
+
+task :setup do
+  require File.dirname(__FILE__) + '/install'
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 0
+:major: 0
+:minor: 3

data/config/defaults.yml

@@ -0,0 +1,4 @@
+# default options
+servers:
+  - localhost:22133
+enable: true

data/config/sweatshop.yml

@@ -0,0 +1,12 @@
+development:
+  servers:
+    - localhost:22133
+  enable: true
+test:
+  servers:
+    - localhost:22133
+  enable: true
+production:
+  servers:
+    - localhost:22133
+  enable: true

data/lib/kestrel.rb

@@ -0,0 +1,29 @@
+class Kestrel
+  attr_reader :client, :servers
+
+  def initialize(opts)
+    @servers = opts[:servers]
+    @client  = opts[:client] || MemCache.new(@servers) 
+  end
+
+  def queue_size(queue)
+    size  = 0
+    stats = client.stats
+    servers.each do |server|
+      size += stats[server]["queue_#{queue}_items"].to_i
+    end
+    size
+  end
+
+  def enqueue(queue, data)
+    client.set(queue, data)
+  end
+
+  def dequeue(queue)
+    client.get("#{queue}/open")
+  end
+
+  def confirm(queue)
+    client.get("#{queue}/close")
+  end
+end

data/lib/sweat_shop.rb

@@ -0,0 +1,107 @@
+require 'rubygems'
+require 'digest'
+require 'yaml'
+
+$:.unshift(File.dirname(__FILE__))
+require 'kestrel'
+require 'sweat_shop/worker'
+
+module SweatShop
+  extend self
+
+  def workers
+    @workers ||= []
+  end
+
+  def workers=(workers)
+    @workers = workers 
+  end
+
+  def workers_in_group(groups)
+    groups = [groups] unless groups.is_a?(Array)
+    if groups.include?(:all)
+      workers
+    else
+      workers.select do |worker|
+        groups.include?(worker.queue_group)
+      end
+    end
+  end
+
+  def do_tasks(workers)
+    loop do
+      wait = true
+      workers.each do |worker|
+        if task = worker.dequeue
+          worker.do_task(task)
+          wait = false
+        end
+      end
+      exit if stop?
+      sleep 1 if wait
+    end
+  end
+
+  def do_all_tasks
+    do_tasks(
+      workers_in_group(:all)
+    )
+  end
+
+  def do_default_tasks
+    do_tasks(
+      workers_in_group(:default)
+    )
+  end
+
+  def stop
+    @stop = true
+  end
+
+  def stop?
+    @stop
+  end
+
+  def config
+    @config ||= begin
+      defaults = YAML.load_file(File.dirname(__FILE__) + '/../config/defaults.yml')
+      if defined?(RAILS_ROOT)
+        file = RAILS_ROOT + '/config/sweatshop.yml'
+        if File.exist?(file)
+          YAML.load_file(file)[RAILS_ENV || 'development']
+        else
+          defaults['enable'] = false
+          defaults
+        end
+      else
+        defaults
+      end
+    end
+  end
+
+  def queue_sizes
+    workers.inject([]) do |all, worker|
+      all << [worker, worker.queue_size]
+      all
+    end
+  end
+
+  def pp_sizes
+    max_width = workers.collect{|w| w.to_s.size}.max
+    puts '-' * (max_width + 10)
+    puts queue_sizes.collect{|p| sprintf("%-#{max_width}s %2s", p.first, p.last)}.join("\n")
+    puts '-' * (max_width + 10)
+  end
+
+  def queue
+    @queue ||= Kestrel.new(:servers => config['servers'])
+  end
+
+  def queue=(queue)
+    @queue = queue
+  end
+end
+
+if defined?(RAILS_ROOT)
+  Dir.glob(RAILS_ROOT + '/app/workers/*.rb').each{|worker| require worker }
+end

data/lib/sweat_shop/metaid.rb

@@ -0,0 +1,5 @@
+def metaclass; class << self; self; end; end
+def meta_eval(&blk); metaclass.instance_eval(&blk); end
+def meta_def(name, &blk)
+  meta_eval { define_method name, &blk }
+end

data/lib/sweat_shop/sweatd.rb

@@ -0,0 +1,67 @@
+require File.dirname(__FILE__) + '/../sweat_shop'
+require 'i_can_daemonize'
+
+module SweatShop
+  class Sweatd
+    include ICanDaemonize
+    queues     = []
+    groups     = []
+    rails_root = nil
+
+    arg '--workers=Worker,Worker', 'Workers to service (Default is all)' do |value|
+      queues = value.split(',')
+    end
+
+    arg '--groups=GROUP,GROUP', 'Groups of queues to service' do |value|
+      groups = value.split(',').collect{|g| g.to_sym}
+    end
+
+    arg '--worker-file=WORKERFILE', 'Worker file to load'  do |value|
+      require value
+    end
+
+    arg '--worker-dir=WORKERDIR', 'Directory containing workers'  do |value|
+      Dir.glob(value + '*.rb').each{|worker| require worker}
+    end
+
+    arg '--rails=DIR', 'Pass in RAILS_ROOT to run this daemon in a rails environment' do |value|
+      rails_root = value
+    end
+
+    sig(:term, :int) do
+      puts "Shutting down sweatd..."
+      SweatShop.stop
+    end
+    
+    before do
+      if rails_root
+        puts "Loading Rails..."
+        require rails_root + '/config/environment' 
+      end
+    end
+
+    daemonize(:kill_timeout => 20) do
+      workers = []
+
+      if groups.any?
+        workers += SweatShop.workers_in_group(groups)
+      end
+
+      if queues.any?
+        workers += queues.collect{|q| Object.module_eval(q)}
+      end
+
+      if workers.any?
+        worker_str = workers.join(',')
+        puts "Starting #{worker_str}..." 
+        $0 = "Sweatd: #{worker_str}"
+        SweatShop.do_tasks(workers)
+      else
+        puts "Starting all workers..." 
+        $0 = 'Sweatd: all'
+        SweatShop.do_all_tasks
+      end
+    end
+
+  end
+end

data/lib/sweat_shop/worker.rb

@@ -0,0 +1,130 @@
+require File.dirname(__FILE__) + '/metaid'
+
+module SweatShop
+  class Worker
+    @@logger = nil
+
+    def self.inherited(subclass)
+      self.workers << subclass
+    end
+
+    def self.method_missing(method, *args, &block)
+      if method.to_s =~ /^async_(.*)/ and config['enable']
+        method        = $1
+        expected_args = instance.method(method).arity
+        if expected_args != args.size 
+          raise ArgumentError.new("#{method} expects #{expected_args} arguments")
+        end
+
+        uid  = ::Digest::MD5.hexdigest("#{name}:#{method}:#{args}:#{Time.now.to_f}")
+        task = {:args => args, :method => method, :uid => uid, :queued_at => Time.now.to_i}
+
+        log("Putting #{uid} on #{queue_name}")
+        enqueue(task)
+
+        uid
+      elsif instance.respond_to?(method)
+        instance.send(method, *args)
+      else
+        super
+      end
+    end
+
+    def self.instance
+      @instance ||= new
+    end
+
+    def self.config
+      SweatShop.config
+    end
+
+    def self.queue_name
+      @queue_name ||= self.to_s
+    end
+
+    def self.queue_size
+      queue.queue_size(queue_name)
+    end
+
+    def self.enqueue(task)
+      queue.enqueue(queue_name, task)
+    end
+
+    def self.dequeue
+      queue.dequeue(queue_name)
+    end
+
+    def self.confirm
+      queue.confirm(queue_name)
+    end
+    
+    def self.do_tasks
+      while task = dequeue
+        do_task(task)
+      end
+    end
+
+    def self.do_task(task)
+      call_before_task(task)
+
+      queued_at = task[:queued_at] ? "(queued #{Time.at(task[:queued_at]).strftime('%Y/%m/%d %H:%M:%S')})" : ''
+      log("Dequeuing #{queue_name}::#{task[:method]} #{queued_at}")
+      task[:result] = instance.send(task[:method], *task[:args])
+
+      call_after_task(task)
+      confirm
+    end
+
+    def self.call_before_task(task)
+      superclass.call_before_task(task) if superclass.respond_to?(:call_before_task)
+      before_task.call(task) if before_task
+    end
+
+    def self.call_after_task(task)
+      superclass.call_after_task(task) if superclass.respond_to?(:call_after_task)
+      after_task.call(task) if after_task
+    end
+
+    def self.workers
+      SweatShop.workers
+    end
+
+    def self.log(msg)
+      return if logger == :silent
+      logger ? logger.debug(msg) : puts(msg)
+    end
+
+    def self.logger
+      @@logger
+    end
+
+    def self.logger=(logger)
+      @@logger = logger
+    end
+
+    def self.before_task(&block)
+      if block
+        @before_task = block
+      else
+        @before_task
+      end
+    end
+
+    def self.after_task(&block)
+      if block
+        @after_task = block
+      else
+        @after_task
+      end
+    end
+
+    def self.queue
+      SweatShop.queue
+    end
+
+    def self.queue_group(group=nil)
+      group ? meta_def(:_queue_group){ group } : _queue_group
+    end
+    queue_group :default
+  end
+end

data/test/hello_worker.rb

@@ -0,0 +1,16 @@
+# hack for functional tests
+require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_extended'
+require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_util'
+
+class HelloWorker < SweatShop::Worker
+  TEST_FILE = File.dirname(__FILE__) + '/test.txt' unless defined?(TEST_FILE)
+
+  def hello(name)
+    puts name
+    "Hi, #{name}"
+  end
+
+  after_task do |task|
+    File.open(TEST_FILE, 'w'){|f| f << task[:result]}
+  end
+end

data/test/test_functional_worker.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
+require File.dirname(__FILE__) + '/test_helper'
+require File.dirname(__FILE__) + '/hello_worker'
+
+class WorkerTest < Test::Unit::TestCase
+
+  def setup
+    File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
+  end
+
+  def teardown
+    File.delete(HelloWorker::TEST_FILE) if File.exist?(HelloWorker::TEST_FILE)
+  end
+
+  # remove 'x' and start kestrel to run
+  test "daemon" do
+    begin
+      SweatShop.queue = nil
+      SweatShop::Worker.logger = :silent
+
+      worker = File.expand_path(File.dirname(__FILE__) + '/hello_worker')
+      sweatd = "#{File.dirname(__FILE__)}/../lib/sweat_shop/sweatd.rb" 
+      uid = HelloWorker.async_hello('Amos')
+
+      `ruby #{sweatd} --worker-file #{worker} start`
+      `ruby #{sweatd} stop`
+
+      File.delete('sweatd.log') if File.exist?('sweatd.log')
+      assert_equal 'Hi, Amos', File.read(HelloWorker::TEST_FILE)
+    rescue MemCache::MemCacheError => e
+      puts "\n\n*** Start kestrel on localhost to run all functional tests. ***\n\n"
+    end
+  end
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,14 @@
+require 'test/unit'
+
+class << Test::Unit::TestCase
+  def test(name, &block)
+    test_name = "test_#{name.gsub(/[\s\W]/,'_')}"
+    raise ArgumentError, "#{test_name} is already defined" if self.instance_methods.include? test_name
+    define_method test_name, &block
+  end
+
+  def xtest(name, &block)
+    # no-op, an empty test method is defined to prevent "no tests in testcase" errors when all tests are disabled
+    define_method(:test_disabled) { assert true }
+  end
+end

data/test/test_sweatshop.rb

@@ -0,0 +1,67 @@
+require File.dirname(__FILE__) + '/../../../memcache/lib/memcache_mock'
+require File.dirname(__FILE__) + '/test_helper'
+require File.dirname(__FILE__) + '/../lib/sweat_shop'
+
+class SweatShopTest < Test::Unit::TestCase
+  SweatShop.workers = []
+  SweatShop.queue   = Kestrel.new(:client => MemCacheMock.new)
+
+  class HelloWorker < SweatShop::Worker
+    def hello(name)
+      "Hi, #{name}"
+    end
+  end
+
+  class GroupedWorker < SweatShop::Worker
+    queue_group :foo
+  end
+
+  test "group workers" do
+    assert_equal [HelloWorker, GroupedWorker], SweatShop.workers_in_group(:all)
+    assert_equal [HelloWorker],   SweatShop.workers_in_group(:default)
+    assert_equal [GroupedWorker], SweatShop.workers_in_group(:foo)
+  end
+
+  test "synch call" do
+    worker = HelloWorker.new
+    assert_equal "Hi, Amos", worker.hello('Amos')
+  end
+
+  test "uid" do
+    SweatShop::Worker.logger = :silent
+    uid = HelloWorker.async_hello('Amos')
+    assert_not_nil uid
+  end
+
+  test "before task" do
+    HelloWorker.before_task do
+      "hello"
+    end
+    assert_equal "hello", HelloWorker.before_task.call
+  end
+
+  test "after task" do
+    HelloWorker.after_task do
+      "goodbye"
+    end
+    assert_equal "goodbye", HelloWorker.after_task.call
+  end
+
+  test "chainable before tasks" do
+    MESSAGES = []
+    class BaseWorker < SweatShop::Worker
+      before_task do |task|
+        MESSAGES << 'base'
+      end
+    end
+    class SubWorker < BaseWorker
+      before_task do |task|
+        MESSAGES << 'sub'
+      end
+    end
+    SubWorker.call_before_task('foo')
+    assert_equal ['base', 'sub'], MESSAGES
+    SweatShop.workers.delete(BaseWorker)
+    SweatShop.workers.delete(SubWorker)
+  end
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification 
+name: famoseagle-sweat_shop
+version: !ruby/object:Gem::Version 
+  version: 0.3.0
+platform: ruby
+authors: 
+- Amos Elliston
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-18 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: TODO
+email: amos@geni.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- History.txt
+- LICENSE
+- Rakefile
+- README.markdown
+- VERSION.yml
+- lib/kestrel.rb
+- lib/sweat_shop
+- lib/sweat_shop/metaid.rb
+- lib/sweat_shop/sweatd.rb
+- lib/sweat_shop/worker.rb
+- lib/sweat_shop.rb
+- test/hello_worker.rb
+- test/test_functional_worker.rb
+- test/test_helper.rb
+- test/test_sweatshop.rb
+- config/defaults.yml
+- config/sweatshop.yml
+has_rdoc: true
+homepage: http://github.com/famoseagle/sweat-shop
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: SweatShop is a simple asynchronous worker queue build on top of rabbitmq/ampq
+test_files: []
+