backgroundrb-rails3 1.1

data/lib/generators/backgroundrb/bdrb_migration/USAGE

@@ -0,0 +1,12 @@
+Description:
+    Creates a migration to add the job queue table used by BackgrounDRb.
+    Pass the migration name, either CamelCased or under_scored, as an argument.
+
+Example:
+    rails generate backgroundrb:bdrb_migration BackgroundrbJob
+
+    Assuming this is run at 09:00:15h on 12 September, 2008, this will create the
+    CreateBackgroundrbQueue migration in:
+
+        db/migrate/20080912090015_backgroundrb_create_backgroundrb_jobs.rb
+

data/lib/generators/backgroundrb/bdrb_migration/bdrb_migration_generator.rb

@@ -0,0 +1,15 @@
+require 'rails/generators/active_record'
+
+module Backgroundrb
+  module Generators
+    class BdrbMigrationGenerator < ActiveRecord::Generators::Base
+      def self.source_root
+        @source_root ||= File.expand_path('../templates', __FILE__)
+      end
+
+      def copy_backgroundrb_migration
+        migration_template "migration.rb", "db/migrate/backgroundrb_create_#{table_name}"
+      end
+    end
+  end
+end

data/lib/generators/backgroundrb/bdrb_migration/templates/migration.rb

@@ -0,0 +1,27 @@
+class BackgroundrbCreate<%= table_name.camelize %> < ActiveRecord::Migration
+  def self.up
+    create_table :<%= table_name %> do |t|
+      t.column :args, :text
+      t.column :worker_name, :string
+      t.column :worker_method, :string
+      t.column :job_key, :string
+      t.column :taken, :int
+      t.column :finished, :int
+      t.column :timeout, :int
+      t.column :priority, :int
+      t.column :submitted_at, :datetime
+      t.column :started_at, :datetime
+      t.column :finished_at, :datetime
+      t.column :archived_at, :datetime
+      t.column :tag, :string
+      t.column :submitter_info, :string
+      t.column :runner_info, :string
+      t.column :worker_key, :string
+      t.column :scheduled_at, :datetime
+    end
+  end
+
+  def self.down
+    drop_table :<%= table_name %>
+  end
+end

data/lib/generators/backgroundrb/worker/USAGE

@@ -0,0 +1,16 @@
+Description:
+    The worker generator creates stubs for a new BackgrounDRb worker.
+
+    The generator takes a worker name as its argument.  The worker name may be
+    given in CamelCase or under_score and should not be suffixed with 'Worker'.
+
+    The generator creates a worker class in lib/workers and a test suite in
+    test/unit.
+
+Example:
+    ./script/generate backgroundrb:worker Tail
+
+    This will create an Tail worker:
+        Model:      lib/workers/tail_worker.rb
+        Test:       test/unit/tail_worker_test.rb
+

data/lib/generators/backgroundrb/worker/templates/unit_test.rb

@@ -0,0 +1,12 @@
+require File.dirname(__FILE__) + '<%= '/..' * class_nesting_depth %>/../test_helper'
+require "#{RAILS_ROOT}/lib/workers/<%= file_name %>_worker"
+require "#{RAILS_ROOT}/vendor/plugins/backgroundrb/lib/backgroundrb.rb"
+require 'drb'
+
+class <%= class_name %>WorkerTest < Test::Unit::TestCase
+
+  # Replace this with your real tests.
+  def test_truth
+    assert <%= class_name %>Worker.included_modules.include?(DRbUndumped)
+  end
+end

data/lib/generators/backgroundrb/worker/templates/worker.rb

@@ -0,0 +1,7 @@
+class <%= file_name.camelize %>Worker < BackgrounDRb::MetaWorker
+  set_worker_name :<%= file_name.underscore %>_worker
+  def create(args = nil)
+    # this method is called, when worker is loaded for the first time
+  end
+end
+

data/lib/generators/backgroundrb/worker/worker_generator.rb

@@ -0,0 +1,14 @@
+module Backgroundrb
+  module Generators
+    class WorkerGenerator < Rails::Generators::NamedBase
+      def self.source_root
+        @source_root ||= File.expand_path('../templates', __FILE__)
+      end
+
+      def copy_backgroundrb_worker
+        template "worker.rb", "lib/workers/#{file_name}_worker.rb"
+        #template "unit_test.rb", "test/unit/#{file_name}_worker_test.rb"
+      end
+    end
+  end
+end

data/lib/tasks/backgroundrb_tasks.rake

@@ -0,0 +1,103 @@
+namespace :backgroundrb do
+  require 'yaml'
+  desc 'Setup backgroundrb in your rails application'
+  task :setup do
+    script_dest = "#{Rails.root}/script/backgroundrb"
+    script_src = File.dirname(__FILE__) + "/../../script/backgroundrb"
+
+    FileUtils.chmod 0774, script_src
+
+    defaults = {:backgroundrb => {:ip => '0.0.0.0',:port => 11006 } }
+
+    config_dest = "#{Rails.root}/config/backgroundrb.yml"
+
+    unless File.exists?(config_dest)
+        puts "Copying backgroundrb.yml config file to #{config_dest}"
+        File.open(config_dest, 'w') { |f| f.write(YAML.dump(defaults)) }
+    end
+
+    unless File.exists?(script_dest)
+        puts "Copying backgroundrb script to #{script_dest}"
+        FileUtils.cp_r(script_src, script_dest)
+    end
+
+    workers_dest = "#{Rails.root}/lib/workers"
+    unless File.exists?(workers_dest)
+      puts "Creating #{workers_dest}"
+      FileUtils.mkdir(workers_dest)
+    end
+
+    test_helper_dest = "#{Rails.root}/test/bdrb_test_helper.rb"
+    test_helper_src = File.dirname(__FILE__) + "/../../script/bdrb_test_helper.rb"
+    unless File.exists?(test_helper_dest)
+      puts "Copying Worker Test helper file #{test_helper_dest}"
+      FileUtils.cp_r(test_helper_src,test_helper_dest)
+    end
+
+    worker_env_loader_dest = "#{Rails.root}/script/load_worker_env.rb"
+    worker_env_loader_src = File.join(File.dirname(__FILE__),"../../","script","load_worker_env.rb")
+    unless File.exists? worker_env_loader_dest
+      puts "Copying Worker envionment loader file #{worker_env_loader_dest}"
+      FileUtils.cp_r(worker_env_loader_src,worker_env_loader_dest)
+    end
+
+    unless File.exists? "#{Rails.root}/tmp/pids"
+      puts "Creating tmp/pids directory"
+      Dir.mkdir "#{Rails.root}/tmp/pids"
+    end
+
+    # Generate the migration
+    Rake::Task['backgroundrb:queue_migration'].invoke
+  end
+
+  desc "Drops and recreate backgroundrb queue table"
+  task :redo_queue => :queue_migration do
+  end
+
+  desc 'update backgroundrb config files from your rails application'
+  task :update do
+    temp_scripts = ["backgroundrb","load_worker_env.rb"].map {|x| "#{Rails.root}/script/#{x}"}
+    temp_scripts.each do |file_name|
+      if File.exists?(file_name)
+        puts "Removing #{file_name} ..."
+        FileUtils.rm(file_name,:force => true)
+      end
+    end
+    new_temp_scripts = ["backgroundrb","load_worker_env.rb"].map {|x| File.dirname(__FILE__) + "/../../script/#{x}" }
+    new_temp_scripts.each do |file_name|
+      puts "Updating file #{File.expand_path(file_name)} ..."
+      FileUtils.cp_r(file_name,"#{Rails.root}/script/")
+    end
+  end
+
+  desc 'Generate a migration for the backgroundrb queue table.  The migration name can be ' +
+    'specified with the MIGRATION environment variable.'
+  task :queue_migration => :environment do
+    raise "Task unavailable to this database (no migration support)" unless ActiveRecord::Base.connection.supports_migrations?
+    require 'rails/generators'
+    Rails::Generators::invoke('backgroundrb:bdrb_migration', ENV['MIGRATION'] || 'BackgroundrbJob')
+  end
+
+  desc 'Remove backgroundrb from your rails application'
+  task :remove do
+    script_src = "#{Rails.root}/script/backgroundrb"
+    temp_scripts = ["backgroundrb","load_worker_env.rb"].map {|x| "#{Rails.root}/script/#{x}"}
+
+    if File.exists?(script_src)
+        puts "Removing #{script_src} ..."
+        FileUtils.rm(script_src, :force => true)
+    end
+
+    test_helper_src = "#{Rails.root}/test/bdrb_test_helper.rb"
+    if File.exists?(test_helper_src)
+      puts "Removing backgroundrb test helper.."
+      FileUtils.rm(test_helper_src,:force => true)
+    end
+
+    workers_dest = "#{Rails.root}/lib/workers"
+    if File.exists?(workers_dest) && Dir.entries("#{workers_dest}").size == 2
+        puts "#{workers_dest} is empty...deleting!"
+        FileUtils.rmdir(workers_dest)
+    end
+  end
+end

data/release_notes.org

@@ -0,0 +1,48 @@
+Hi,
+
+BackgrounDRb 1.2 is being unleashed.
+
+* New features:
+
+** Exceptions/errors are now popped out at the earliest moment
+  in the client side itself. For example:
+
+>> MiddleMan.worker(:foo_worker).async_bar(:args => {:age => 10})
+BackgrounDRb::RemoteWorkerError: BackgrounDRb::RemoteWorkerError
+
+  Above exception is thrown because remote worker doesn't have method
+  "bar" defined on it.
+
+Similarly:
+
+>> MiddleMan.worker(:foo_worker).checksum(:args => {:age => "lolz"})
+BackgrounDRb::RemoteWorkerError: BackgrounDRb::RemoteWorkerError
+
+  Above exception is thrown because remote worker's checksum method
+  expects an integer as an argument.
+
+  For asynchronous method calls, BackgrounDRb doesn't check if method
+  ran successfully, it only checks existence of methods on remote
+  worker. For sync method calls it checks if method ran successfully
+  or not.
+
+** Its possible to have per worker configuration options now.
+
+* Bug Fixes
+
+** Much better error/exception handling. Rogue worker methods shouldn't
+   crash the worker now. All the unhandled exceptions and dispatch
+   errors can be found in debug log file.
+
+** Fixes for postgres db with persistent job queues.
+
+** Switched to lightweight Queue implementation for tasks enqueued
+   to thread pool.
+
+** Fixes for database dropped connections while running tasks from
+   persistent queues.
+
+** Fixes for newer Rails versions.
+
+**
+

data/release_points.org

@@ -0,0 +1,46 @@
+* Before publishing any release, please make sure that following things are working *
+** Scheduling via cron is working
+** Scheduling via unix scheduler is working
+** creating a worker using job_key is working
+** Creating a worker and its scheduling is working
+** ask_work is working => check
+** send_request is working => check
+** all_worker_info is workign
+** status retrieval is working => check
+** sending of large objects around is working
+** sending of rails AR objects is working
+** sending of rails AR objects with plugin is working
+** Environment is getting loaded properly through config file and command line argument.
+** People are unable to call ActionController methods.
+** People should be able to use multiple args while invoking methods.
+** Check if clustering is working.
+** Check if queue is working.
+
+* Problem with new experimental API
+** there is an issue if user invokes multiple tasks in thread pool directly from one of the worker
+   under current settings they are going to end up with same job key.
+
+** Also, new_worker can't have same method invocation conventions because it accepts more parameters.
+
+* Ideas
+** Okay, use:
+   cache[job_key] = result
+   cache[job_key] =>
+     will return the result. worker_name, worker_key shall be taken in to account.
+
+** Write a Rails controller that returns following information:
+   - For each BackgrounDRb server:
+     - Workers running
+     - Memory taken by them
+     - Number of threads in thread pool
+     - Number of Tasks in Queue
+     - If any worker key
+     - Perhaps a count of open file descriptors
+     - Scheduled tasks and next turn
+  - Overall memory taken by entire BackgrounDRb server.
+
+
+
+
+
+

data/script/backgroundrb

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+RAILS_HOME = File.expand_path(File.join(File.dirname(__FILE__),".."))
+
+require "rubygems"
+require "active_support"
+require "active_record"
+
+require RAILS_HOME + "/config/boot"
+require "backgroundrb"
+
+BDRB_HOME = ::BackgrounDRb::BACKGROUNDRB_ROOT
+WORKER_ROOT = File.join(RAILS_HOME,"lib","workers")
+WORKER_LOAD_ENV = File.join(RAILS_HOME,"script","load_worker_env")
+
+["server","server/lib","lib","lib/backgroundrb"].each { |x| $LOAD_PATH.unshift(BDRB_HOME + "/#{x}")}
+$LOAD_PATH.unshift(WORKER_ROOT)
+
+require "bdrb_config"
+
+BDRB_CONFIG = BackgrounDRb::Config.read_config("#{RAILS_HOME}/config/backgroundrb.yml")
+PID_FILE = "#{RAILS_HOME}/tmp/pids/backgroundrb_#{BDRB_CONFIG[:backgroundrb][:port]}.pid"
+SERVER_LOGGER = "#{RAILS_HOME}/log/backgroundrb_debug_#{BDRB_CONFIG[:backgroundrb][:port]}.log"
+
+require "bdrb_start_stop"
+daemon = BackgrounDRb::StartStop.new
+
+case ARGV[0]
+when 'start'; daemon.start
+when 'stop'; daemon.stop
+when 'restart'; daemon.stop;daemon.start
+when 'status'
+  if daemon.running?
+    puts "BackgrounDRb Running"
+    exit
+  else
+    puts "BackgrounDRb Not Running"
+    exit!(daemon.status)
+  end
+else
+  require "logger"
+  require "packet"
+  require "optparse"
+
+  BackgrounDRb::Config.parse_cmd_options ARGV
+
+  require RAILS_HOME + "/config/environment"
+  require "bdrb_job_queue"
+  require "backgroundrb_server"
+  BackgrounDRb::MasterProxy.new()
+end
+

data/script/bdrb_test_helper.rb

@@ -0,0 +1,99 @@
+require File.join(File.dirname(__FILE__) + "/../config/environment")
+WORKER_ROOT = RAILS_ROOT + "/lib/workers"
+$LOAD_PATH.unshift(WORKER_ROOT)
+
+class Object
+  def self.metaclass; class << self; self; end; end
+
+  def self.iattr_accessor *args
+    metaclass.instance_eval do
+      attr_accessor *args
+      args.each do |attr|
+        define_method("set_#{attr}") do |b_value|
+          self.send("#{attr}=",b_value)
+        end
+      end
+    end
+
+    args.each do |attr|
+      class_eval do
+        define_method(attr) do
+          self.class.send(attr)
+        end
+        define_method("#{attr}=") do |b_value|
+          self.class.send("#{attr}=",b_value)
+        end
+      end
+    end
+  end
+end
+
+module BackgrounDRb
+  class WorkerDummyLogger
+    %w(info debug fatal error warn).each do |x|
+      define_method(x) do |log_data|
+      end
+    end
+  end
+
+  class WorkData
+    attr_accessor :args,:block,:job_method,:persistent_job_id,:job_key
+    def initialize(args,job_key,job_method,persistent_job_id)
+      @args = args
+      @job_key = job_key
+      @job_method = job_method
+      @persistent_job_id = persistent_job_id
+    end
+  end
+
+  class ThreadPool
+    attr_accessor :size,:threads,:work_queue,:logger
+    attr_accessor :result_queue,:master
+
+    def initialize(master,size)
+      @master = master
+      @logger = logger
+      @size = size
+      @threads = []
+    end
+
+    def defer(method_name,args = nil)
+      job_key = Thread.current[:job_key]
+      persistent_job_id = Thread.current[:persistent_job_id]
+      t = WorkData.new(args,job_key,method_name,persistent_job_id)
+      result = run_task(t)
+      result
+    end
+
+    # run tasks popped out of queue
+    def run_task task
+      block_arity = master.method(task.job_method).arity
+      begin
+        t_data = task.args
+        result = nil
+        if block_arity != 0
+          result = master.send(task.job_method,task.args)
+        else
+          result = master.send(task.job_method)
+        end
+        return result
+      rescue
+        puts($!.to_s)
+        puts($!.backtrace.join("\n"))
+        return nil
+      end
+    end
+  end #end of class ThreadPool
+
+  class MetaWorker
+    attr_accessor :logger,:thread_pool
+    iattr_accessor :worker_name
+    iattr_accessor :no_auto_load
+
+    def initialize
+      @logger = WorkerDummyLogger.new
+      @thread_pool = ThreadPool.new(self,10)
+    end
+  end
+end
+