beanstalker 0.0.1

data/README

@@ -0,0 +1,27 @@
+This is Async Observer -- a Rails plugin that provides deep integration with
+Beanstalk.
+
+For more information, see http://async-observer.rubyforge.org/.
+
+For more information on Beanstalk, see its home page at
+http://xph.us/software/beanstalkd/.
+
+
+Worker Options:
+-d                                : daemonize
+--pid [path to pidfile]           : drop a pid file to a path
+-e [test,production,development]  : set the rails environment
+
+Example Usage:
+
+start 3 workers
+  ./vendor/plugins/async_observer/bin/worker -d --pid log/worker1.pid -e production
+  ./vendor/plugins/async_observer/bin/worker -d --pid log/worker2.pid -e production
+  ./vendor/plugins/async_observer/bin/worker -d --pid log/worker3.pid -e production
+
+kill one
+  kill -s INT `cat log/worker1.pid`
+
+Remember kill a worker will cause it to go into a shutdown phase.
+Run the above again to kill immediately, but remember all jobs in
+the workers queue is lost at that point...

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "beanstalker"
+    gem.summary = %Q{Beanstalker provides deep integration with Beanstalk. Fork from http://github.com/kristjan/async_observer}
+    gem.description = %Q{Beanstalker provides deep integration with Beanstalk. Fork from http://github.com/kristjan/async_observer}
+    gem.email = "glebpom@gmail.com"
+    gem.homepage = "http://github.com/glebpom/beanstalker"
+    gem.authors = ["Gleb Pomykalov"]
+#    gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
+    gem.add_dependency "daemonizer"
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/test_*.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "beanstalker #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/worker

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Use the same pointer (and therefore same buffer) for stdout and stderr.
+$VERBOSE = nil; STDERR = $stderr = STDOUT = $stdout; $VERBOSE = false
+
+require 'time'
+
+def load_rails_and_run
+  # Rails initialization.
+  # We do this here instead of using script/runner because script/runner
+  # breaks __FILE__, which we use below.
+  begin
+    puts "#!load-rails!begin!#{Time.now.utc.xmlschema(6)}"
+    require File.expand_path(File.dirname(__FILE__) + '/../../../../config/boot')
+    puts "RAILS_ROOT=#{RAILS_ROOT.inspect}"
+    require RAILS_ROOT + '/config/environment'
+  ensure
+    puts "#!load-rails!end!#{Time.now.utc.xmlschema(6)}"
+  end
+  require 'async_observer/worker'
+  AsyncObserver::Worker.new(binding).run()
+end
+
+# set environment
+if ARGV.include?('-e') # check rails env
+  env=nil
+  ARGV.each_with_index{|arg,i| env = ARGV[i+1] if arg == '-e' }
+  RAILS_ENV=env unless env.nil?
+end
+
+if ARGV.include?('-d')
+  pidpath = 'log/worker.pid'
+  if ARGV.include?('--pid') # look up the pid path
+    ARGV.each_with_index{|arg,i| pidpath = ARGV[i+1] if arg == '--pid' }
+    STDERR.puts "Missing pid file path!" and exit(1) if pidpath.nil?
+  end
+  unless File.writable?(File.dirname(pidpath))
+    STDERR.puts "#{pidpath} not writable!"
+    exit(1)
+  end
+
+  if File.exist?(pidpath)
+    STDERR.puts "#{pidpath} exits! Make sure the worker isn't still running and try again after rm #{pidpath}"
+    exit(1)
+  end
+
+  require File.dirname(__FILE__) + '/../lib/async_observer/daemonize'
+  AsyncObserver::Daemonize.detach(pidpath) do
+    load_rails_and_run
+  end
+else
+  load_rails_and_run
+end

data/init.rb

@@ -0,0 +1 @@
+require 'async_observer/extend'

data/lib/async_observer/daemonize.rb

@@ -0,0 +1,35 @@
+# async-observer - Rails plugin for asynchronous job execution
+# Copyright (C) 2009 Todd A. Fisher.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+module AsyncObserver; end
+
+class AsyncObserver::Daemonize
+  def self.detach(pidfile='log/worker.pid',&block)
+    # daemonize, create a pipe to send status to the parent process, after the child has successfully started or failed
+    fork do
+      Process.setsid
+      fork do
+        Process.setsid
+        File.open(pidfile, 'wb') {|f| f << Process.pid}
+        at_exit { File.unlink(pidfile) }
+        File.umask 0000
+        STDIN.reopen "/dev/null"
+        STDOUT.reopen "/dev/null", "a"
+        STDERR.reopen STDOUT
+        block.call
+      end
+    end
+  end
+end

data/lib/async_observer/extend.rb

@@ -0,0 +1,140 @@
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'async_observer/queue'
+
+CLASSES_TO_EXTEND = [
+  ActiveRecord::Base,
+  Array,
+  Hash,
+  Module,
+  Numeric,
+  Range,
+  String,
+  Symbol,
+]
+
+module AsyncObserver::Extensions
+  def async_send(selector, *args)
+    async_send_opts(selector, {}, *args)
+  end
+
+  def async_send_opts(selector, opts, *args)
+    AsyncObserver::Queue.put_call!(self, selector, opts, args)
+  end
+end
+
+CLASSES_TO_EXTEND.each do |c|
+  c.send :include, AsyncObserver::Extensions
+end
+
+class Range
+  DEFAULT_FANOUT_FUZZ = 0
+  DEFAULT_FANOUT_DEGREE = 1000
+
+  def split_to(n)
+    split_by((size + n - 1) / n) { |group| yield(group) }
+  end
+
+  def split_by(n)
+    raise ArgumentError.new('invalid slice size') if n < 1
+    n -= 1 if !exclude_end?
+    i = first
+    while member?(i)
+      j = [i + n, last].min
+      yield(Range.new(i, j, exclude_end?))
+      i = j + (exclude_end? ? 0 : 1)
+    end
+  end
+
+  def size
+    last - first + (exclude_end? ? 0 : 1)
+  end
+
+  def async_each_opts(rcv, selector, opts, *extra)
+    fanout_degree = opts.fetch(:fanout_degree, DEFAULT_FANOUT_DEGREE)
+    if size <= fanout_degree
+      each {|i| rcv.async_send_opts(selector, opts, i, *extra)}
+    else
+      fanout_opts = opts.merge(:fuzz => opts.fetch(:fanout_fuzz,
+                                                   DEFAULT_FANOUT_FUZZ))
+      fanout_opts[:pri] = opts[:fanout_pri] || opts[:pri]
+      fanout_opts = fanout_opts.reject_hash{|k,v| nil.equal?(v)}
+      split_to(fanout_degree) do |subrange|
+        subrange.async_send_opts(:async_each_opts, fanout_opts, rcv, selector,
+                                 opts, *extra)
+      end
+    end
+  end
+
+  def async_each(rcv, selector, *extra)
+    async_each_opts(rcv, selector, {}, *extra)
+  end
+end
+
+HOOKS = [:after_create, :after_update, :after_save]
+
+class << ActiveRecord::Base
+  HOOKS.each do |hook|
+    code = %Q{def async_#{hook}(*methods, &b) add_async_hook(#{hook.inspect}, *methods, &b) end}
+    class_eval(code, __FILE__, __LINE__ - 1)
+  end
+
+  def add_async_hook(hook, *args, &block)
+    if args && args.first.is_a?(Symbol)
+      method = args.shift
+      async_hooks[hook] << lambda{|o| o.send(method)}
+    else
+      async_hooks[hook] << block
+    end
+  end
+
+  def async_hooks
+    @async_hooks ||= Hash.new do |hash, hook|
+      ahook = :"_async_#{hook}"
+
+      # This is for the producer's benefit
+      send(hook){|o| async_send(ahook, o)}
+
+      # This is for the worker's benefit
+      code = "def #{ahook}(o) run_async_hooks(#{hook.inspect}, o) end"
+      instance_eval(code, __FILE__, __LINE__ - 1)
+
+      hash[hook] = []
+    end
+  end
+
+  def run_async_hooks(hook, o)
+    async_hooks[hook].each{|b| b.call(o)}
+  end
+
+  def send_to_instance(id, selector, *args)
+    x = find_by_id(id)
+    x.send(selector, *args) if x
+  end
+
+  def async_each_opts(selector, opts, *args)
+    min = opts.fetch(:min, minimum(:id))
+    max = opts.fetch(:max, maximum(:id))
+
+    (min..max).async_each_opts(self, :send_to_instance, opts, selector, *args)
+  end
+
+  def async_each(selector, *args)
+    async_each_opts(selector, {}, *args)
+  end
+end

data/lib/async_observer/queue.rb

@@ -0,0 +1,182 @@
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+require 'beanstalk-client'
+
+module AsyncObserver; end
+
+class AsyncObserver::Queue; end
+
+class << AsyncObserver::Queue
+  DEFAULT_PRI = 512
+  DEFAULT_FUZZ = 0
+  DEFAULT_DELAY = 0
+  DEFAULT_TTR = 120
+  DEFAULT_TUBE = 'default'
+
+  attr_accessor :queue, :app_version, :after_put
+
+  # This is a fake worker instance for running jobs synchronously.
+  def sync_worker()
+    require 'async_observer/worker'
+    @sync_worker ||= AsyncObserver::Worker.new(binding)
+  end
+
+  # This runs jobs synchronously; it's used when no queue is configured.
+  def sync_run(obj)
+    body = YAML.dump(obj)
+    job = Beanstalk::Job.new(AsyncObserver::FakeConn.new(), 0, body)
+    sync_worker.dispatch(job)
+    sync_worker.do_all_work()
+    return 0, '0.0.0.0'
+  end
+
+  def put!(obj, pri=DEFAULT_PRI, delay=DEFAULT_DELAY, ttr=DEFAULT_TTR,
+           tube=DEFAULT_TUBE)
+    return sync_run(obj) if (:direct.equal?(pri) or !queue)
+    queue.connect()
+    queue.use(tube)
+    info = [queue.yput(obj, pri, delay, ttr), queue.last_server]
+    f = AsyncObserver::Queue.after_put
+    f.call(*info) if f
+    return info
+  end
+
+  SUBMIT_OPTS = [:pri, :fuzz, :delay, :ttr, :tube]
+
+  def put_call!(obj, sel, opts, args=[])
+    pri = opts.fetch(:pri, DEFAULT_PRI)
+    fuzz = opts.fetch(:fuzz, DEFAULT_FUZZ)
+    delay = opts.fetch(:delay, DEFAULT_DELAY)
+    ttr = opts.fetch(:ttr, DEFAULT_TTR)
+    tube = opts.fetch(:tube, (app_version or DEFAULT_TUBE))
+    worker_opts = opts.reject{|k,v| SUBMIT_OPTS.include?(k)}
+
+    pri = pri + rand(fuzz + 1) if !:direct.equal?(pri)
+
+    code = gen(obj, sel, args)
+    RAILS_DEFAULT_LOGGER.info("put #{pri} #{code}")
+    put!(pkg(code, worker_opts), pri, delay, ttr, tube)
+  end
+
+  def pkg(code, opts)
+    opts.merge(:type => :rails, :code => code)
+  end
+
+  # Be careful not to pass in a selector that's not valid ruby source.
+  def gen(obj, selector, args)
+    obj.rrepr + '.' + selector.to_s + '(' + gen_args(args) + ')'
+  end
+
+  def gen_args(args)
+    args.rrepr[1...-1]
+  end
+end
+
+class AsyncObserver::FakeConn
+  def delete(x)
+  end
+
+  def release(x, y, z)
+  end
+
+  def bury(x, y)
+  end
+
+  def addr
+    '<none>'
+  end
+
+  def job_stats(id)
+    {
+      'id' => id,
+      'state' => 'reserved',
+      'age' => 0,
+      'delay' => 0,
+      'time-left' => 5000,
+      'timeouts' => 0,
+      'releases' => 0,
+      'buries' => 0,
+      'kicks' => 0,
+    }
+  end
+end
+
+# This is commented out to workaround (what we think is) a ruby bug in method
+# lookup. Somehow the methods defined here are being used instead of ones in
+# ActiveRecord::Base.
+#class Object
+#  def rrepr()
+#    raise ArgumentError.new('no consistent external repr for ' + self.inspect)
+#  end
+#end
+
+class Symbol
+  def rrepr() inspect end
+end
+
+class Module
+  def rrepr() name end
+end
+
+class NilClass
+  def rrepr() inspect end
+end
+
+class FalseClass
+  def rrepr() inspect end
+end
+
+class TrueClass
+  def rrepr() inspect end
+end
+
+class Numeric
+  def rrepr() inspect end
+end
+
+class String
+  def rrepr() inspect end
+end
+
+class Array
+  def rrepr() '[' + map(&:rrepr).join(', ') + ']' end
+end
+
+class Hash
+  def rrepr() '{' + map{|k,v| k.rrepr + '=>' + v.rrepr}.join(', ') + '}' end
+end
+
+class Range
+  def rrepr() "(#{first.rrepr}#{exclude_end? ? '...' : '..'}#{last.rrepr})" end
+end
+
+class Time
+  def rrepr() "Time.parse('#{self.inspect}')" end
+end
+
+class Date
+  def rrepr() "Date.parse('#{self.inspect}')" end
+end
+
+module AsyncObserver::Extensions
+  def rrepr()
+    method = (respond_to? :get_cache) ? 'get_cache' : 'find'
+    "#{self.class.rrepr}.#{method}(#{id.rrepr})"
+  end
+end