async_observer 0.1.0

data/README

@@ -0,0 +1,7 @@
+This is Async Observer -- a Rails plugin that provides deep integration with
+Beanstalk.
+
+For more information, see http://async-observer.rubyforge.org/.
+
+For more information on Beanstalk, see its home page at
+http://xph.us/software/beanstalkd/.

data/Rakefile

@@ -0,0 +1,13 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "async_observer"
+    gemspec.summary = "Async Observer"
+    gemspec.description = "Async Observer is a Rails plugin that provides deep integration with Beanstalk."
+    gemspec.email = "kristjan@causes.com"
+    gemspec.homepage = "http://async-observer.rubyforge.org/"
+    gemspec.authors = ["Kristján Pétursson"]
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/async_observer.gemspec

@@ -0,0 +1,52 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{async_observer}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kristj\303\241n P\303\251tursson"]
+  s.date = %q{2010-01-07}
+  s.default_executable = %q{worker}
+  s.description = %q{Async Observer is a Rails plugin that provides deep integration with Beanstalk.}
+  s.email = %q{kristjan@causes.com}
+  s.executables = ["worker"]
+  s.extra_rdoc_files = [
+    "README"
+  ]
+  s.files = [
+    ".gitignore",
+     "COPYING",
+     "README",
+     "Rakefile",
+     "VERSION",
+     "async_observer.gemspec",
+     "bin/worker",
+     "init.rb",
+     "lib/async_observer.rb",
+     "lib/async_observer/daemonize.rb",
+     "lib/async_observer/extend.rb",
+     "lib/async_observer/queue.rb",
+     "lib/async_observer/util.rb",
+     "lib/async_observer/worker.rb"
+  ]
+  s.homepage = %q{http://async-observer.rubyforge.org/}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Async Observer}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/bin/worker

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# Use the same pointer (and therefore same buffer) for stdout and stderr.
+$VERBOSE = nil; STDERR = $stderr = STDOUT = $stdout; $VERBOSE = false
+
+require 'time'
+
+def load_rails_and_run
+  # Rails initialization.
+  # We do this here instead of using script/runner because script/runner
+  # breaks __FILE__, which we use below.
+  begin
+    puts "#!load-rails!begin!#{Time.now.utc.xmlschema(6)}"
+    require File.expand_path(File.dirname(__FILE__) + '/../../../../config/boot')
+    puts "RAILS_ROOT=#{RAILS_ROOT.inspect}"
+    require RAILS_ROOT + '/config/environment'
+  ensure
+    puts "#!load-rails!end!#{Time.now.utc.xmlschema(6)}"
+  end
+  require 'async_observer/worker'
+  AsyncObserver::Worker.new(binding).run()
+end
+
+# set environment
+if ARGV.include?('-e') # check rails env
+  env=nil
+  ARGV.each_with_index{|arg,i| env = ARGV[i+1] if arg == '-e' }
+  RAILS_ENV=env unless env.nil?
+end
+
+if ARGV.include?('-d')
+  pidpath = 'log/worker.pid'
+  if ARGV.include?('--pid') # look up the pid path
+    ARGV.each_with_index{|arg,i| pidpath = ARGV[i+1] if arg == '--pid' }
+    STDERR.puts "Missing pid file path!" and exit(1) if pidpath.nil?
+  end
+  unless File.writable?(File.dirname(pidpath))
+    STDERR.puts "#{pidpath} not writable!"
+    exit(1)
+  end
+
+  if File.exist?(pidpath)
+    STDERR.puts "#{pidpath} exits! Make sure the worker isn't still running and try again after rm #{pidpath}"
+    exit(1)
+  end
+
+  require File.dirname(__FILE__) + '/../lib/async_observer/daemonize'
+  AsyncObserver::Daemonize.detach(pidpath) do
+    load_rails_and_run
+  end
+else
+  load_rails_and_run
+end

data/init.rb

@@ -0,0 +1 @@
+require 'async_observer/extend'

data/lib/async_observer.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/../init.rb'

data/lib/async_observer/daemonize.rb

@@ -0,0 +1,35 @@
+# async-observer - Rails plugin for asynchronous job execution
+# Copyright (C) 2009 Todd A. Fisher.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+module AsyncObserver; end
+
+class AsyncObserver::Daemonize
+  def self.detach(pidfile='log/worker.pid',&block)
+    # daemonize, create a pipe to send status to the parent process, after the child has successfully started or failed
+    fork do
+      Process.setsid
+      fork do
+        Process.setsid
+        File.open(pidfile, 'wb') {|f| f << Process.pid}
+        at_exit { File.unlink(pidfile) }
+        File.umask 0000
+        STDIN.reopen "/dev/null"
+        STDOUT.reopen "/dev/null", "a"
+        STDERR.reopen STDOUT
+        block.call
+      end
+    end
+  end
+end

data/lib/async_observer/extend.rb

@@ -0,0 +1,135 @@
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require 'async_observer/queue'
+
+CLASSES_TO_EXTEND = [
+  ActiveRecord::Base,
+  Array,
+  Hash,
+  Module,
+  Numeric,
+  Range,
+  String,
+  Symbol,
+]
+
+module AsyncObserver::Extensions
+  def async_send(selector, *args)
+    async_send_opts(selector, {}, *args)
+  end
+
+  def async_send_opts(selector, opts, *args)
+    AsyncObserver::Queue.put_call!(self, selector, opts, args)
+  end
+end
+
+CLASSES_TO_EXTEND.each do |c|
+  c.send :include, AsyncObserver::Extensions
+end
+
+class Range
+  DEFAULT_FANOUT_FUZZ = 0
+  DEFAULT_FANOUT_DEGREE = 1000
+
+  def split_to(n)
+    split_by((size + n - 1) / n) { |group| yield(group) }
+  end
+
+  def split_by(n)
+    raise ArgumentError.new('invalid slice size') if n < 1
+    n -= 1 if !exclude_end?
+    i = first
+    while member?(i)
+      j = [i + n, last].min
+      yield(Range.new(i, j, exclude_end?))
+      i = j + (exclude_end? ? 0 : 1)
+    end
+  end
+
+  def size
+    last - first + (exclude_end? ? 0 : 1)
+  end
+
+  def async_each_opts(rcv, selector, opts, *extra)
+    fanout_degree = opts.fetch(:fanout_degree, DEFAULT_FANOUT_DEGREE)
+    if size <= fanout_degree
+      each {|i| rcv.async_send_opts(selector, opts, i, *extra)}
+    else
+      fanout_opts = opts.merge(:fuzz => opts.fetch(:fanout_fuzz,
+                                                   DEFAULT_FANOUT_FUZZ))
+      fanout_opts[:pri] = opts[:fanout_pri] || opts[:pri]
+      fanout_opts = fanout_opts.reject_hash{|k,v| nil.equal?(v)}
+      split_to(fanout_degree) do |subrange|
+        subrange.async_send_opts(:async_each_opts, fanout_opts, rcv, selector,
+                                 opts, *extra)
+      end
+    end
+  end
+
+  def async_each(rcv, selector, *extra)
+    async_each_opts(rcv, selector, {}, *extra)
+  end
+end
+
+HOOKS = [:after_create, :after_update, :after_save, :after_destroy]
+
+class << ActiveRecord::Base
+  HOOKS.each do |hook|
+    code = %Q{def async_#{hook}(&b) add_async_hook(#{hook.inspect}, b) end}
+    class_eval(code, __FILE__, __LINE__ - 1)
+  end
+
+  def add_async_hook(hook, block)
+    async_hooks[hook] << block
+  end
+
+  def async_hooks
+    @async_hooks ||= Hash.new do |hash, hook|
+      ahook = :"_async_#{hook}"
+
+      # This is for the producer's benefit
+      send(hook){|o| async_send(ahook, o)}
+
+      # This is for the worker's benefit
+      code = "def #{ahook}(o) run_async_hooks(#{hook.inspect}, o) end"
+      instance_eval(code, __FILE__, __LINE__ - 1)
+
+      hash[hook] = []
+    end
+  end
+
+  def run_async_hooks(hook, o)
+    async_hooks[hook].each{|b| b.call(o)}
+  end
+
+  def send_to_instance(id, selector, *args)
+    x = find_by_id(id)
+    x.send(selector, *args) if x
+  end
+
+  def async_each_opts(selector, opts, *args)
+    min = opts.fetch(:min, minimum(:id))
+    max = opts.fetch(:max, maximum(:id))
+
+    (min..max).async_each_opts(self, :send_to_instance, opts, selector, *args)
+  end
+
+  def async_each(selector, *args)
+    async_each_opts(selector, {}, *args)
+  end
+end

data/lib/async_observer/queue.rb

@@ -0,0 +1,178 @@
+# async-observer - Rails plugin for asynchronous job execution
+
+# Copyright (C) 2007 Philotic Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+require 'beanstalk-client'
+
+module AsyncObserver; end
+
+class AsyncObserver::Queue; end
+
+class << AsyncObserver::Queue
+  DEFAULT_PRI = 512
+  DEFAULT_FUZZ = 0
+  DEFAULT_DELAY = 0
+  DEFAULT_TTR = 120
+  DEFAULT_TUBE = 'default'
+
+  attr_accessor :queue, :app_version, :after_put
+
+  # This is a fake worker instance for running jobs synchronously.
+  def sync_worker()
+    require 'async_observer/worker'
+    @sync_worker ||= AsyncObserver::Worker.new(binding)
+  end
+
+  # This runs jobs synchronously; it's used when no queue is configured.
+  def sync_run(obj)
+    body = YAML.dump(obj)
+    job = Beanstalk::Job.new(AsyncObserver::FakeConn.new(), 0, body)
+    sync_worker.dispatch(job)
+    sync_worker.do_all_work()
+    return 0, '0.0.0.0'
+  end
+
+  def put!(obj, pri=DEFAULT_PRI, delay=DEFAULT_DELAY, ttr=DEFAULT_TTR,
+           tube=DEFAULT_TUBE)
+    return sync_run(obj) if (:direct.equal?(pri) or !queue)
+    queue.connect()
+    queue.use(tube)
+    info = [queue.yput(obj, pri, delay, ttr), queue.last_server]
+    f = AsyncObserver::Queue.after_put
+    f.call(*info) if f
+    return info
+  end
+
+  SUBMIT_OPTS = [:pri, :fuzz, :delay, :ttr, :tube]
+
+  def put_call!(obj, sel, opts, args=[])
+    pri = opts.fetch(:pri, DEFAULT_PRI)
+    fuzz = opts.fetch(:fuzz, DEFAULT_FUZZ)
+    delay = opts.fetch(:delay, DEFAULT_DELAY)
+    ttr = opts.fetch(:ttr, DEFAULT_TTR)
+    tube = opts.fetch(:tube, (app_version or DEFAULT_TUBE))
+    worker_opts = opts.reject{|k,v| SUBMIT_OPTS.include?(k)}
+
+    pri = pri + rand(fuzz + 1) if !:direct.equal?(pri)
+
+    code = gen(obj, sel, args)
+    RAILS_DEFAULT_LOGGER.info("put #{pri} #{code}")
+    put!(pkg(code, worker_opts), pri, delay, ttr, tube)
+  end
+
+  def pkg(code, opts)
+    opts.merge(:type => :rails, :code => code)
+  end
+
+  # Be careful not to pass in a selector that's not valid ruby source.
+  def gen(obj, selector, args)
+    obj.rrepr + '.' + selector.to_s + '(' + gen_args(args) + ')'
+  end
+
+  def gen_args(args)
+    args.rrepr[1...-1]
+  end
+end
+
+class AsyncObserver::FakeConn
+  def delete(x)
+  end
+
+  def release(x, y, z)
+  end
+
+  def bury(x, y)
+  end
+
+  def addr
+    '<none>'
+  end
+
+  def job_stats(id)
+    {
+      'id' => id,
+      'state' => 'reserved',
+      'age' => 0,
+      'delay' => 0,
+      'time-left' => 5000,
+      'timeouts' => 0,
+      'releases' => 0,
+      'buries' => 0,
+      'kicks' => 0,
+    }
+  end
+end
+
+# This is commented out to workaround (what we think is) a ruby bug in method
+# lookup. Somehow the methods defined here are being used instead of ones in
+# ActiveRecord::Base.
+#class Object
+#  def rrepr()
+#    raise ArgumentError.new('no consistent external repr for ' + self.inspect)
+#  end
+#end
+
+class Symbol
+  def rrepr() inspect end
+end
+
+class Module
+  def rrepr() name end
+end
+
+class NilClass
+  def rrepr() inspect end
+end
+
+class FalseClass
+  def rrepr() inspect end
+end
+
+class TrueClass
+  def rrepr() inspect end
+end
+
+class Numeric
+  def rrepr() inspect end
+end
+
+class String
+  def rrepr() inspect end
+end
+
+class Array
+  def rrepr() '[' + map(&:rrepr).join(', ') + ']' end
+end
+
+class Hash
+  def rrepr() '{' + map{|k,v| k.rrepr + '=>' + v.rrepr}.join(', ') + '}' end
+end
+
+class Range
+  def rrepr() "(#{first.rrepr}#{exclude_end? ? '...' : '..'}#{last.rrepr})" end
+end
+
+class Time
+  def rrepr() "Time.parse('#{self.inspect}')" end
+end
+
+module AsyncObserver::Extensions
+  def rrepr()
+    method = (respond_to? :get_cache) ? 'get_cache' : 'find'
+    "#{self.class.rrepr}.#{method}(#{id.rrepr})"
+  end
+end