backgroundrb-rails3 1.1

data/examples/workers/error_worker.rb

@@ -0,0 +1,17 @@
+# Put your code that runs your task inside the do_work method it will be
+# run automatically in a thread. You have access to all of your rails
+# models.  You also get logger and results method inside of this class
+# by default.
+class ErrorWorker < BackgrounDRb::MetaWorker
+  set_worker_name :error_worker
+  set_no_auto_load(true)
+
+  def create(args = nil)
+    logger.info "creating error worker"
+  end
+
+  def hello_world(data)
+    logger.info "invoking #{worker_name} hello world #{data} #{Time.now}"
+  end
+end
+

data/examples/workers/foo_worker.rb

@@ -0,0 +1,38 @@
+# Put your code that runs your task inside the do_work method it will be
+# run automatically in a thread. You have access to all of your rails
+# models.  You also get logger and results method inside of this class
+# by default.
+
+class TimeClient
+  def receive_data(p_data)
+    worker.get_external_data(p_data)
+  end
+
+  def post_init
+    p "***************** : connection completed"
+  end
+end
+
+class FooWorker < BackgrounDRb::MetaWorker
+  set_worker_name :foo_worker
+  def create(args = nil)
+    #register_status("Running")
+    add_periodic_timer(10) { foobar }
+    external_connection = nil
+    connect("localhost",11009,TimeClient) { |conn| external_connection = conn }
+  end
+
+  def get_external_data(p_data)
+    cache[some_key] = p_data
+  end
+
+  def foobar
+    cache[some_key] = "Time is now : #{Time.now}"
+  end
+
+  def barbar(data)
+    logger.info "invoking babrbar on #{Time.now} #{data}"
+  end
+
+end
+

data/examples/workers/god_worker.rb

@@ -0,0 +1,7 @@
+class GodWorker < BackgrounDRb::MetaWorker
+  set_worker_name :god_worker
+  def create(args = nil)
+    logger.info "hello world"
+  end
+end
+

data/examples/workers/model_worker.rb

@@ -0,0 +1,13 @@
+class ModelWorker < BackgrounDRb::MetaWorker
+  set_worker_name :model_worker
+  def create(args = nil)
+    #add_periodic_timer(2) { add_new_user }
+  end
+
+  def add_new_user
+    login,age = "Hemant: #{Time.now}",rand(24)
+    logger.info "creating user #{login} with age #{age}"
+    User.create(:login => login, :age => age)
+  end
+end
+

data/examples/workers/renewal_worker.rb

@@ -0,0 +1,11 @@
+class RenewalWorker < BackgrounDRb::MetaWorker
+  set_worker_name :renewal_worker
+  def create(args = nil)
+
+  end
+  def load_policies(data = nil)
+    logger.info "Loading policies done on #{data}"
+    return "done"
+  end
+end
+

data/examples/workers/rss_worker.rb

@@ -0,0 +1,26 @@
+# this worker would test thread issues that were discussed before
+require "net/http"
+class RssWorker < BackgrounDRb::MetaWorker
+  set_worker_name :rss_worker
+  def create(args = nil)
+    # this method is called, when worker is loaded for the first time
+  end
+
+  # method would fetch supplied urls in a thread
+  def fetch_url(url)
+    puts "fetching url #{url}"
+    thread_pool.defer(:scrap_things,url)
+  end
+
+  def scrap_things url
+    begin
+      data = Net::HTTP.get(url,"/")
+      File.open("#{RAILS_ROOT}/log/pages.txt","w") do |fl|
+        fl.puts(data)
+      end
+    rescue
+      logger.info "Error downloading page"
+    end
+  end
+end
+

data/examples/workers/server_worker.rb

@@ -0,0 +1,31 @@
+class TimeServer
+  # the data send by client would be received here
+  def receive_data(p_data)
+  end
+
+  # would be called when someone connects to the server for the
+  # first time
+  def post_init
+    add_periodic_timer(2) { say_hello_world }
+  end
+
+  # would be called when client connection is complete.
+  def connection_completed
+  end
+
+  def say_hello_world
+    send_data("Hello World\n")
+  end
+end
+
+# this worker is going to act like server.
+class ServerWorker < BackgrounDRb::MetaWorker
+  set_worker_name :server_worker
+  def create(args = nil)
+    # start the server when worker starts
+    start_server("0.0.0.0",11009,TimeServer) do |client_connection|
+      client_connection.say_hello_world
+    end
+  end
+end
+

data/examples/workers/world_worker.rb

@@ -0,0 +1,12 @@
+class WorldWorker < BackgrounDRb::MetaWorker
+  set_worker_name :world_worker
+  def create(args = nil)
+    #logger.info "starting world worker"
+  end
+
+  def hello_world
+    a = lambda { "Hello world" }
+    return a
+  end
+end
+

data/examples/workers/xmpp_worker.rb

@@ -0,0 +1,7 @@
+class XmppWorker < BackgrounDRb::MetaWorker
+  set_worker_name :xmpp_worker
+  def create(args = nil)
+    # this method is called, when worker is loaded for the first time
+  end
+end
+

data/init.rb

@@ -0,0 +1,7 @@
+# Include hook code here
+%w{ controller }.each do |code_dir|
+  $:.unshift File.join(File.dirname(__FILE__),"app",code_dir)
+end
+
+require 'backgroundrb'
+#require "backgroundrb_status_controller"

data/install.rb

@@ -0,0 +1 @@
+# Install hook code here

data/know_issues.org

@@ -0,0 +1,5 @@
+* Some issues that I am aware of *
+- Creation of job queue may fail on other that mysql dbs
+- Database doesn't get created when someone runs the migrations again
+- Delete doesn't seem to be working test it.
+

data/lib/backgroundrb.rb

@@ -0,0 +1 @@
+require "backgroundrb/railtie"

data/lib/backgroundrb/bdrb_client_helper.rb

@@ -0,0 +1,8 @@
+module BackgrounDRb
+  module ClientHelper
+    def gen_worker_key(worker_name,worker_key = nil)
+      return worker_name.to_sym if worker_key.nil?
+      return "#{worker_name}_#{worker_key}".to_sym
+    end
+  end
+end

data/lib/backgroundrb/bdrb_cluster_connection.rb

@@ -0,0 +1,156 @@
+# class stores connections to BackgrounDRb servers in a cluster manner
+module BackgrounDRb
+  class ClusterConnection
+    include ClientHelper
+    attr_accessor :backend_connections,:config,:cache,:bdrb_servers
+    attr_accessor :disconnected_connections
+
+    # initialize cluster connection
+    def initialize
+      @bdrb_servers = []
+      @backend_connections = []
+      @disconnected_connections = {}
+
+      @last_polled_time = Time.now
+      @request_count = 0
+
+      initialize_memcache if BackgrounDRb::BDRB_CONFIG[:backgroundrb][:result_storage] == 'memcache'
+      establish_connections
+      @round_robin = (0...@backend_connections.length).to_a
+    end
+
+    # initialize memache if client is storing results in memcache
+    def initialize_memcache
+      require 'memcache'
+      memcache_options = {
+        :c_threshold => 10_000,
+        :compression => true,
+        :debug => false,
+        :namespace => 'backgroundrb_result_hash',
+        :readonly => false,
+        :urlencode => false
+      }
+      @cache = MemCache.new(memcache_options)
+      @cache.servers = BackgrounDRb::BDRB_CONFIG[:memcache].split(',')
+    end
+
+    # initialize all backend server connections
+    def establish_connections
+      klass = Struct.new(:ip,:port)
+      if t_servers = BackgrounDRb::BDRB_CONFIG[:client]
+        connections = t_servers.split(',')
+        connections.each do |conn_string|
+          ip = conn_string.split(':')[0]
+          port = conn_string.split(':')[1].to_i
+          @bdrb_servers << klass.new(ip,port)
+        end
+      end
+      @bdrb_servers << klass.new(BackgrounDRb::BDRB_CONFIG[:backgroundrb][:ip],BackgrounDRb::BDRB_CONFIG[:backgroundrb][:port].to_i)
+      @bdrb_servers.each_with_index do |connection_info,index|
+        next if @backend_connections.detect { |x| x.server_info == "#{connection_info.ip}:#{connection_info.port}" }
+        @backend_connections << Connection.new(connection_info.ip,connection_info.port,self)
+      end
+    end # end of method establish_connections
+
+    # every 10 request or 10 seconds it will try to reconnect to bdrb servers which were down
+    def discover_server_periodically
+      @disconnected_connections.each do |key,connection|
+        connection.establish_connection
+        if connection.connection_status
+          @backend_connections << connection
+          connection.close_connection
+          @disconnected_connections[key] = nil
+        end
+      end
+      @disconnected_connections.delete_if { |key,value| value.nil? }
+      @round_robin = (0...@backend_connections.length).to_a
+    end
+
+    # Find live connections except those mentioned in array, because they
+    # are already dead.
+    def find_next_except_these connections
+      invalid_connections = @backend_connections.select { |x| connections.include?(x.server_info) }
+      @backend_connections.delete_if { |x| connections.include?(x.server_info) }
+      @round_robin = (0...@backend_connections.length).to_a
+      invalid_connections.each do |x|
+        @disconnected_connections[x.server_info] = x
+      end
+      chosen = @backend_connections.detect { |x| !(connections.include?(x.server_info)) }
+      raise NoServerAvailable.new("No BackgrounDRb server is found running") unless chosen
+      chosen
+    end
+
+    # Fina a connection by host name and port
+    def find_connection host_info
+      conn = @backend_connections.detect { |x| x.server_info == host_info }
+      raise NoServerAvailable.new("BackgrounDRb server is not found running on #{host_info}") unless conn
+      return conn
+    end
+
+    # find the local configured connection
+    def find_local
+      find_connection("#{BackgrounDRb::BDRB_CONFIG[:backgroundrb][:ip]}:#{BackgrounDRb::BDRB_CONFIG[:backgroundrb][:port]}")
+    end
+
+    # return the worker proxy
+    def worker(worker_name,worker_key = nil)
+      update_stats
+      RailsWorkerProxy.new(worker_name,worker_key,self)
+    end
+
+    # Update the stats and discover new nodes if they came up.
+    def update_stats
+      @request_count += 1
+      discover_server_periodically if(time_to_discover? && !@disconnected_connections.empty?)
+    end
+
+    # Check if, we should try to discover new bdrb servers
+    def time_to_discover?
+      if((@request_count%10 == 0) or (Time.now > (@last_polled_time + 10.seconds)))
+        @last_polled_time = Time.now
+        return true
+      else
+        return false
+      end
+    end
+
+    # Send worker information of all currently running workers from all configured bdrb
+    # servers
+    def all_worker_info
+      update_stats
+      info_data = {}
+      @backend_connections.each do |t_connection|
+        info_data[t_connection.server_info] = t_connection.all_worker_info rescue nil
+      end
+      return info_data
+    end
+
+    # one of the backend connections are chosen and worker is started on it
+    def new_worker(options = {})
+      update_stats
+      succeeded = false
+      result = nil
+
+      @backend_connections.each do |connection|
+        begin
+          result = connection.new_worker(options)
+          succeeded = true
+        rescue BdrbConnError; end
+      end
+      raise NoServerAvailable.new("No BackgrounDRb server is found running") unless succeeded
+      return options[:worker_key]
+    end
+
+    # choose a server in round robin manner.
+    def choose_server
+      if @round_robin.empty?
+        @round_robin = (0...@backend_connections.length).to_a
+      end
+      if @round_robin.empty? && @backend_connections.empty?
+        discover_server_periodically
+        raise NoServerAvailable.new("No BackgrounDRb server is found running") if @round_robin.empty? && @backend_connections.empty?
+      end
+      @backend_connections[@round_robin.shift]
+    end
+  end # end of ClusterConnection
+end # end of Module BackgrounDRb

data/lib/backgroundrb/bdrb_config.rb

@@ -0,0 +1,43 @@
+require 'yaml'
+require 'erb'
+module BackgrounDRb
+  class Config
+    def self.parse_cmd_options(argv)
+      options = { }
+
+      OptionParser.new do |opts|
+        script_name = File.basename($0)
+        opts.banner = "Usage: #{$0} [options]"
+        opts.separator ""
+        opts.on("-e", "--environment=name", String,
+                "Specifies the environment to operate under (test/development/production).",
+                "Default: development") { |v| options[:environment] = v }
+        opts.separator ""
+        opts.on("-h", "--help",
+                "Show this help message.") { $stderr.puts opts; exit }
+        opts.separator ""
+        opts.on("-v","--version",
+                "Show version.") { $stderr.puts "1.1"; exit }
+      end.parse!(argv)
+
+      ENV["RAILS_ENV"] = options[:environment] if options[:environment]
+    end
+
+    def self.read_config(config_file)
+      config = YAML.load(ERB.new(IO.read(config_file)).result)
+      environment = ENV["RAILS_ENV"] || config[:backgroundrb][:environment] || "development"
+
+      if respond_to?(:silence_warnings)
+        silence_warnings do
+          Object.const_set("RAILS_ENV",environment)
+        end
+      else
+        Object.const_set("RAILS_ENV",environment)
+      end
+
+      ENV["RAILS_ENV"] = environment
+      config
+    end
+  end
+end
+

data/lib/backgroundrb/bdrb_conn_error.rb

@@ -0,0 +1,29 @@
+# Exception class for BackgrounDRb connection errors
+module BackgrounDRb
+  # raised when connection to a particular server failed
+  class BdrbConnError < RuntimeError
+    attr_accessor :message
+    def initialize(message)
+      @message = message
+    end
+  end
+  # raised when connection to all of the available servers failed
+  class NoServerAvailable < RuntimeError
+    attr_accessor :message
+    def initialize(message)
+      @message = message
+    end
+  end
+
+  # raised, when said task was submitted without a job key, whereas
+  # nature of the task requires a job key
+  class NoJobKey < RuntimeError; end
+
+  # raised if worker throws some error
+  class RemoteWorkerError < RuntimeError
+    attr_accessor :message
+    def initialize message
+      @message = message
+    end
+  end
+end

data/lib/backgroundrb/bdrb_connection.rb

@@ -0,0 +1,179 @@
+module BackgrounDRb
+  class Connection
+    attr_accessor :server_ip,:server_port,:cluster_conn,:connection_status
+
+    def initialize ip,port,cluster_conn
+      @mutex = Mutex.new
+      @server_ip = ip
+      @server_port = port
+      @cluster_conn = cluster_conn
+      @connection_status = true
+    end
+
+
+    def establish_connection
+      begin
+        timeout(3) do
+          @connection = TCPSocket.open(server_ip, server_port)
+          @connection.setsockopt(Socket::IPPROTO_TCP,Socket::TCP_NODELAY,1)
+        end
+        @connection_status = true
+      rescue Timeout::Error
+        @connection_status = false
+      rescue Exception => e
+        @connection_status = false
+      end
+    end
+
+    def write_data data
+      begin
+        flush_in_loop(data)
+      rescue Errno::EAGAIN
+        return
+      rescue Errno::EPIPE
+        establish_connection
+        if @connection_status
+          flush_in_loop(data)
+        else
+          @connection_status = false
+          raise BackgrounDRb::BdrbConnError.new("Error while writing #{server_info}")
+        end
+      rescue
+        establish_connection
+        if @connection_status
+          flush_in_loop(data)
+        else
+          @connection_status = false
+          raise BackgrounDRb::BdrbConnError.new("Error while writing #{server_info}")
+        end
+      end
+    end
+
+    def server_info
+      "#{server_ip}:#{server_port}"
+    end
+
+    def flush_in_loop(data)
+      t_length = data.length
+      loop do
+        break if t_length <= 0
+        written_length = @connection.write(data)
+        raise "Error writing to socket" if written_length <= 0
+        result = @connection.flush
+        data = data[written_length..-1]
+        t_length = data.length
+      end
+    end
+
+    def dump_object data
+      establish_connection
+      raise BackgrounDRb::BdrbConnError.new("Error while connecting to the backgroundrb server #{server_info}") unless @connection_status
+
+      object_dump = Marshal.dump(data)
+      dump_length = object_dump.length.to_s
+      length_str = dump_length.rjust(9,'0')
+      final_data = length_str + object_dump
+      @mutex.synchronize { write_data(final_data) }
+    end
+
+    def close_connection
+      @connection.close
+      @connection = nil
+    end
+
+    def ask_work p_data
+      p_data[:type] = :async_invoke
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      close_connection
+      bdrb_response
+    end
+
+    def new_worker p_data
+      p_data[:type] = :start_worker
+      dump_object(p_data)
+      close_connection
+      # RailsWorkerProxy.worker(p_data[:worker],p_data[:worker_key],self)
+    end
+
+    def worker_info(p_data)
+      p_data[:type] = :worker_info
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      close_connection
+      bdrb_response
+    end
+
+    def all_worker_info
+      p_data = { }
+      p_data[:type] = :all_worker_info
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb() }
+      close_connection
+      bdrb_response
+    end
+
+    def delete_worker p_data
+      p_data[:type] = :delete_worker
+      dump_object(p_data)
+      close_connection
+    end
+
+    def read_object
+      begin
+        message_length_str = @connection.read(9)
+        message_length = message_length_str.to_i
+        message_data = @connection.read(message_length)
+        return message_data
+      rescue
+        raise BackgrounDRb::BdrbConnError.new("Not able to connect #{server_info}")
+      end
+    end
+
+    def gen_key options
+      if BDRB_CONFIG[:backgroundrb][:result_storage] == 'memcache'
+        key = [options[:worker],options[:worker_key],options[:job_key]].compact.join('_')
+        key
+      else
+        options[:job_key]
+      end
+    end
+
+    def ask_result(p_data)
+      if BDRB_CONFIG[:backgroundrb][:result_storage] == 'memcache'
+        return_result_from_memcache(p_data)
+      else
+        p_data[:type] = :get_result
+        dump_object(p_data)
+        bdrb_response = nil
+        @mutex.synchronize { bdrb_response = read_from_bdrb() }
+        close_connection
+        bdrb_response ? bdrb_response[:data] : nil
+      end
+    end
+
+    def read_from_bdrb(timeout = 3)
+      begin
+        ret_val = select([@connection],nil,nil,timeout)
+        return nil unless ret_val
+        raw_response = read_object()
+        master_response = Marshal.load(raw_response)
+        return master_response
+      rescue
+        return nil
+      end
+    end
+
+    def send_request(p_data)
+      p_data[:type] = :sync_invoke
+      dump_object(p_data)
+      bdrb_response = nil
+      @mutex.synchronize { bdrb_response = read_from_bdrb(nil) }
+      close_connection
+      bdrb_response
+    end
+  end
+end