josip-backgroundrb_merb 1.0.3

data/LICENSE

@@ -0,0 +1,4 @@
+Copyright (c) 2007 Hemant Kumar ( mail [at] gnufied [dot] org )
+Copyright (c) 2006 Ezra Zygmuntowicz and skaar[at]waste[dot]org
+
+The Library is dual licensed under terms of Ruby License or MIT License.

data/README

@@ -0,0 +1,45 @@
+= Why double fork?
+Well, to original fork didn't work for me...
+
+= Merb fork of BackgrounDRb
+Based on the 1.0.3 version of backgroundrb this is a hacked port to MERB.  Some areas need tyding up but it works for all basic cases.  Contributions welcome!
+
+== Installation
+  (1) Install gem provided in /pkg into your app's gem repository (which has to be named gems)
+      ex. #cd ~/path/to/app
+          #mkdir gems
+          #gem install ~/path/to/bdrb_merb.gem -i gems --no-ri --no-rdoc -l
+  (2) Add to config/init.rb (Merb):
+      require 'backgroundrb_merb'
+  (3) Create configuration file:
+      #rake bdrb:setup
+
+== Usage
+  Workers are created in app/workers
+  To create new worker:
+    merb-gen worker WorkerName
+
+  To start BackgroundRB:
+    rake bdrb:ctl:start
+    Control+C to stop
+  To run in background:
+    rake bdrb:ctl:daemonize
+  And to stop it:
+    rake bdrb:ctl:stop
+
+= BackgrounDRb
+BackgrounDRb is a Ruby job server and scheduler. Its main intent is to be
+used with Ruby on Rails applications for offloading long-running tasks.
+Since a Rails application blocks while serving a request it is best to
+move long-running tasks off into a background process that is divorced
+from http request/response cycle.
+
+This new release of BackgrounDRb is also modular and can be used without Rails so that any Ruby program or framework can use it.
+
+Copyright (c) 2006 Ezra Zygmuntowicz,skaar[at]waste[dot]org,
+Copyright (c) 2007 Hemant Kumar (gethemant [at] gmail.com )
+
+
+== Usage
+
+Please look into http://backgroundrb.rubyforge.org

data/Rakefile

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+
+PLUGIN = "backgroundrb_merb"
+NAME = "backgroundrb_merb"
+VERSION = "1.0.3"
+AUTHOR = "Josip Lisec"
+EMAIL = "josiplisec@gmail.com"
+HOMEPAGE = "not yet"
+SUMMARY = "Merb plugin that provides backgroundrb"
+
+spec = Gem::Specification.new do |s|
+  s.name = NAME
+  s.version = VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README", "LICENSE", 'TODO']
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  s.add_dependency('merb', '>= 0.9.2')
+  s.add_dependency('packet', '>= 0.1.5')
+  s.add_dependency('chronic', '>= 0.2.3')
+  s.require_path = 'lib'
+#  s.autorequire = PLUGIN
+  s.files = %w(LICENSE README Rakefile TODO) + Dir.glob("{config,generators,lib,pids,server,specs}/**/*")
+end
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{NAME}-#{VERSION}}
+end
+
+namespace :jruby do
+  desc "Run :package and install the resulting .gem with jruby"
+  task :install => :package do
+    sh %{#{SUDO} jruby -S gem install pkg/#{NAME}-#{Merb::VERSION}.gem --no-rdoc --no-ri}
+  end
+end

data/TODO

@@ -0,0 +1,8 @@
+* Implement a way to share code between workers.
+* Implement a way to test workers.
+* Implement proper logger stuff so as people can set log level and stuff 
+* Implement a way to disable logger workers.
+* Implement backgroundrb to work across machines.
+* Make the API more simple use to use and less mouthfull.
+
+

data/config/backgroundrb.yml

@@ -0,0 +1,11 @@
+## YAML Template.
+:backgroundrb:
+  :ip: 0.0.0.0
+  :port: 11006
+
+:schedules:
+  :foo_worker:
+    :worker_method: foobar
+    :trigger_args: */5 * * * * * *
+
+

data/generators/worker/USAGE

@@ -0,0 +1,16 @@
+Description:
+    The worker generator creates stubs for a new BackgrounDRb worker.
+
+    The generator takes a worker name as its argument.  The worker name may be
+    given in CamelCase or under_score and should not be suffixed with 'Worker'.
+
+    The generator creates a worker class in app/workers and a test suite in
+    test/unit.
+
+Example:
+    merb-gen worker Tail
+
+    This will create an Tail worker:
+        Model:      app/workers/tail_worker.rb
+        Test:       spec/workers/tail_spec.rb
+

data/generators/worker/templates/app/workers/%worker_file_name%_worker.rb

@@ -0,0 +1,7 @@
+class <%= worker_class_name %>Worker < BackgrounDRbMerb::MetaWorker
+  set_worker_name :<%= worker_file_name %>_worker
+
+  def create(args = nil)
+    # this method is called, when worker is loaded for the first time
+  end
+end

data/generators/worker/templates/spec/workers/%worker_file_name%_spec.rb

@@ -0,0 +1,9 @@
+require File.join(File.dirname(__FILE__), "..", "spec_helper")
+require "backgroundrb_merb"
+require "drb"
+
+describe <%= worker_class_name %> do
+
+  it "should have specs"
+
+end

data/generators/worker/worker_generator.rb

@@ -0,0 +1,36 @@
+class WorkerGenerator < Merb::GeneratorBase
+  attr_reader :worker_class_name, :worker_file_name
+
+  def initialize(args, runtime_args = {})
+    @base =             File.dirname(__FILE__)
+    super
+    @worker_file_name =  args.shift.snake_case
+    @worker_class_name = @worker_file_name.to_const_string
+  end
+
+  def manifest
+    record do |m|
+      @m = m
+
+      @assigns = {
+        :worker_file_name  => worker_file_name,
+        :worker_class_name => worker_class_name
+      }
+
+      copy_dirs
+      copy_files
+
+#      Not sure what is this for, tests?
+#      m.dependency "merb_model_test", [model_file_name], @assigns
+    end
+  end
+
+  protected
+    def banner
+      <<-EOS.split("\n").map{|x| x.strip}.join("\n")
+        Creates a basic BackgroundRB worker in app/workers.
+
+        USAGE: #{spec.name} worker
+      EOS
+    end
+end

data/lib/backgroundrb_merb.rb

@@ -0,0 +1,19 @@
+# FIXME: check if data that we are writing to the socket should end with newline
+require "pathname"
+BACKGROUNDRB_ROOT = Pathname.new(Merb.root).realpath.to_s
+
+if defined?(Merb::Plugins)
+  if File.exists?(Merb.root / "config" / "backgroundrb.yml") then
+    require "packet"
+    require "backgroundrb_merb/bdrb_conn_error"
+    require "backgroundrb_merb/bdrb_config"
+    require "backgroundrb_merb/worker_proxy"
+    require "backgroundrb_merb/merb_worker_proxy"
+
+    Merb::BootLoader.before_app_loads do
+      MiddleMan     = BackgrounDRbMerb::WorkerProxy.init
+    end
+  end
+
+  Merb::Plugins.add_rakefiles "backgroundrb_merb/merbtasks"
+end

data/lib/backgroundrb_merb/bdrb_config.rb

@@ -0,0 +1,47 @@
+require 'erb'
+
+module BackgrounDRbMerb
+  class Config
+    def self.parse_cmd_options(argv)
+      require 'optparse'
+      options = { :environment => (Merb.environment || "development").dup }
+
+      OptionParser.new do |opts|
+        script_name = File.basename($0)
+        opts.banner = "Usage: #{$0} [options]"
+        opts.separator ""
+        opts.on("-e", "--environment=name", String,
+                "Specifies the environment to operate under (test/development/production).",
+                "Default: development") { |v| options[:environment] = v }
+        opts.separator ""
+        opts.on("-h", "--help",
+                "Show this help message.") { $stderr.puts opts; exit }
+        opts.separator ""
+        opts.on("-v","--version",
+                "Show version.") { $stderr.puts "1.0.3"; exit }
+      end.parse!(argv)
+    end
+
+    def self.read_config(config_file)
+      config = YAML.load(ERB.new(IO.read(config_file)).result)
+
+      environment = Merb.environment.to_sym
+      config[:backgroundrb][:environment] = environment.to_s
+
+      if config[environment]
+
+        # block for deep_merging the hashes
+        deep_proc = Proc.new do |key, oldval, newval|
+          if oldval.kind_of?(Hash) && newval.kind_of?(Hash)
+            next oldval.merge(newval,&deep_proc)
+          end
+          next newval
+        end
+
+        config.merge!( config[environment], &deep_proc)
+      end
+
+      config
+    end
+  end
+end

data/lib/backgroundrb_merb/bdrb_conn_error.rb

@@ -0,0 +1,9 @@
+module BackgrounDRbMerb
+  class BdrbConnError < RuntimeError
+    attr_accessor :message
+
+    def initialize(message)
+      @message = message
+    end
+  end
+end

data/lib/backgroundrb_merb/merb_worker_proxy.rb

@@ -0,0 +1,57 @@
+module BackgrounDRbMerb
+  class MerbWorkerProxy
+    attr_accessor :worker_name, :worker_method, :data, :job_key
+
+    def self.worker(p_worker_name, p_job_key = nil)
+      t = new
+      t.worker_name = p_worker_name
+      t.job_key = p_job_key
+      t
+    end
+
+    def method_missing(method_id, *args)
+      worker_method = method_id
+      data = args[0]
+      flag = args[1]
+
+      case worker_method
+      when :ask_status
+        MiddleMan.ask_status(compact(
+          :worker   => worker_name,
+          :job_key  => job_key
+        ))
+      when :worker_info
+        MiddleMan.worker_info(compact(
+          :worker   => worker_name,
+          :job_key  => job_key
+        ))
+      when :delete
+        MiddleMan.delete_worker(compact(
+          :worker   => worker_name,
+          :job_key  => job_key
+        ))
+      else
+        if flag
+          MiddleMan.send_request(compact(
+            :worker         => worker_name,
+            :job_key        => job_key,
+            :worker_method  => worker_method,
+            :data           => data
+          ))
+        else
+          MiddleMan.ask_work(compact(
+            :worker         => worker_name,
+            :job_key        => job_key,
+            :worker_method  => worker_method,
+            :data           => data
+          ))
+        end
+      end
+    end
+
+    def compact(options = { })
+      options.delete_if { |key,value| value.nil? }
+      options
+    end
+  end
+end

data/lib/backgroundrb_merb/merbtasks.rb

@@ -0,0 +1,91 @@
+namespace :bdrb do
+  require 'yaml'
+
+  BDRB_DIR = File.expand_path(File.dirname(__FILE__) + '/../../')
+
+  unless File.exists?(Merb.root/"config"/"backgroundrb.yml") then
+    desc 'Create config file'
+    task :setup do
+      template = BDRB_DIR/"config"/"backgroundrb.yml"
+      FileUtils.cp(template, Merb.root/"config")
+      puts "Created #{Merb.root}/config/backgroundrb.yml"
+    end
+  else
+    desc 'Control BackgrounDRb server'
+    namespace :ctl do
+      $LOAD_PATH.unshift(BDRB_DIR)
+      $LOAD_PATH.unshift(BDRB_DIR / "lib" / "backgroundrb_merb")
+      $LOAD_PATH.unshift(BDRB_DIR / "server" / "lib")
+
+      WORKER_ROOT = Merb.root / "app" / "workers"
+      $LOAD_PATH.unshift(WORKER_ROOT)
+
+      require 'bdrb_config.rb'
+      require 'master_worker.rb'
+      require 'log_worker.rb'
+      require 'meta_worker.rb'
+
+#     BackgrounDRbMerb::Config.parse_cmd_options(ARGV[1..-1])
+      CONFIG_FILE = BackgrounDRbMerb::Config.read_config(Merb.root / "config" / "backgroundrb.yml")
+      bdrb_conf = CONFIG_FILE[:backgroundrb]
+      pid_file = BDRB_DIR / "pids" / "drb_#{bdrb_conf[:port]}.pid"
+      SERVER_LOGGER = Merb.root / "log" / "bdrb_server_#{bdrb_conf[:port]}.log"
+
+
+      desc 'Start and detach from console'
+      task :daemonize do
+        if fork then
+          exit
+        else
+          File.open(pid_file, 'w') {|f| f << Process.pid.to_s}
+          if bdrb_conf[:log].nil? or bdrb_conf[:log] != 'foreground' then
+            log_file = File.open(SERVER_LOGGER, 'w+')
+            [STDIN, STDOUT, STDERR].each {|dev| dev.reopen(log_file)}
+          end
+
+          BackgrounDRbMerb::MasterProxy.new
+        end
+        exit
+      end
+
+      desc 'Stop BackgrounDRb'
+      task :stop do
+        pid = nil
+        File.open(pid_file, 'r') {|f| pid = f.gets.strip.chomp.to_i}
+
+        begin
+          pgid = Process.getpgid(pid)
+          Process.kill('TERM', pid)
+          Process.kill('-TERM', pgid)
+          Process.kill('KILL', pid)
+        rescue Errno::ESRCH => e
+          puts "Deleting pid file"
+        rescue
+          puts $!
+        ensure
+          File.delete(pid_file) if File.exists?(pid_file)
+        end
+        exit
+      end
+
+      desc 'Start BackgrounDRb'
+      task :start do
+        BackgrounDRbMerb::MasterProxy.new
+      end
+    end
+
+    desc 'Remove BackgroundRB from your app'
+    task :remove do
+      [Merb.root/"app"/"workers", Merb.root/"spec"/"workers"].each do |dir|
+        if File.exists?(dir) && Dir.entries(dir).size == 2
+            puts "#{dir} is empty...deleting!"
+            sleep 0.5
+            FileUtils.rmdir(dir)
+        end
+      end
+      FileUtils.rm(Merb.root/"config"/"backgroundrb.yml")
+
+      puts %{Don't forget to remove "require 'backgroundrb_merb'" from your init.rb!}
+    end
+  end
+end

data/lib/backgroundrb_merb/worker_proxy.rb

@@ -0,0 +1,174 @@
+module BackgrounDRbMerb
+  class WorkerProxy
+    include Packet::NbioHelper
+
+    def self.init
+      @config       = BackgrounDRbMerb::Config.read_config("#{BACKGROUNDRB_ROOT}/config/backgroundrb.yml")
+      @server_ip    = @config[:backgroundrb][:ip]
+      @server_port  = @config[:backgroundrb][:port]
+      new
+    end
+
+    def self.server_ip; @server_ip; end
+    def self.server_port; @server_port; end
+
+    def server_ip; self.class.server_ip; end
+    def server_port; self.class.server_port; end
+
+    def self.custom_connection(ip, port)
+      @server_ip = ip
+      @server_port = port
+      new
+    end
+
+    def initialize
+      @mutex = Mutex.new
+      establish_connection
+    end
+
+    def worker(worker_name, job_key = nil)
+      BackgrounDRbMerb::MerbWorkerProxy.worker(worker_name,job_key)
+    end
+
+    def establish_connection
+      puts " ~ Backgroundrb server: #{server_ip}:#{server_port}"
+      begin
+        @connection = TCPSocket.open(server_ip, server_port)
+        @connection.setsockopt(Socket::IPPROTO_TCP,Socket::TCP_NODELAY, 1)
+        @connection_status = true
+#       rescue Timeout::Error
+#         @connection_status = false
+      rescue Exception => e
+        @connection_status = false
+      end
+    end
+
+    def write_data(data)
+      begin
+        flush_in_loop(data)
+      rescue Errno::EAGAIN
+        return
+      rescue Errno::EPIPE
+        establish_connection
+        if @connection_status
+          flush_in_loop(data)
+        else
+          raise BackgrounDRbMerb::BdrbConnError.new("Error while writing")
+        end
+      rescue
+        establish_connection
+        if @connection_status
+          flush_in_loop(data)
+        else
+          raise BackgrounDRbMerb::BdrbConnError.new("Error while writing")
+        end
+      end
+    end
+
+    def flush_in_loop(data)
+      t_length = data.length
+      loop do
+        break if t_length <= 0
+        written_length = @connection.write(data)
+        @connection.flush
+        data = data[written_length..-1]
+        t_length = data.length
+      end
+    end
+
+    def dump_object(data)
+      unless @connection_status
+        establish_connection
+        raise BackgrounDRbMerb::BdrbConnError.new("Error while connecting to the backgroundrb server") unless @connection_status
+      end
+
+      object_dump = Marshal.dump(data)
+      dump_length = object_dump.length.to_s
+      length_str = dump_length.rjust(9,'0')
+      final_data = length_str + object_dump
+      @mutex.synchronize { write_data(final_data) }
+    end
+
+    def ask_work(p_data)
+      p_data[:type] = :do_work
+      dump_object(p_data)
+    end
+
+    def new_worker(p_data)
+      p_data[:type] = :start_worker
+      dump_object(p_data)
+      p_data[:job_key]
+    end
+
+    def worker_info(p_data)
+      p_data[:type] = :worker_info
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      bdrb_response
+    end
+
+    def all_worker_info
+      p_data = { }
+      p_data[:type] = :all_worker_info
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      bdrb_response
+    end
+
+    def delete_worker(p_data)
+      p_data[:type] = :delete_worker
+      dump_object(p_data)
+    end
+
+    def read_object
+      sock_data = ""
+      begin
+        while(sock_data << @connection.read_nonblock(1023)); end
+      rescue Errno::EAGAIN
+        @tokenizer.extract(sock_data) { |b_data| return b_data }
+      rescue
+        raise BackgrounDRbMerb::BdrbConnError.new("Not able to connect")
+      end
+    end
+
+    def query_all_workers
+      p_data = { }
+      p_data[:type] = :all_worker_status
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      bdrb_response
+    end
+
+    def ask_status(p_data)
+      p_data[:type] = :get_status
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      bdrb_response
+    end
+
+    def read_from_bdrb(timeout = 3)
+      @tokenizer = BinParser.new
+      begin
+        ret_val = select([@connection],nil,nil,timeout)
+        return nil unless ret_val
+        raw_response = read_object()
+        master_response = Marshal.load(raw_response)
+        return master_response
+      rescue
+        return nil
+      end
+    end
+
+    def send_request(p_data)
+      p_data[:type] = :get_result
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb(nil) }
+      bdrb_response
+    end
+  end
+end