backgroundrb-rails3 1.1

data/server/lib/trigger.rb

@@ -0,0 +1,34 @@
+module BackgrounDRb
+  class Trigger
+
+    attr_accessor :start_time, :end_time, :repeat_interval
+
+    def initialize(opts={})
+      @start_time = Time.parse(opts[:start])
+      @end_time = Time.parse(opts[:end])
+      @repeat_interval = opts[:repeat_interval].to_i
+    end
+
+    def fire_after_time(time)
+      @start_time = time  if not @start_time
+
+      # Support UNIX at-style scheduling, by just specifying a start
+      # time.
+      if @end_time.nil? and @repeat_interval.nil?
+        @end_time = start_time + 1
+        @repeat_interval = 1
+      end
+
+      case
+      when @end_time && time > @end_time
+        nil
+      when time < @start_time
+        @start_time
+      when @repeat_interval != nil && @repeat_interval > 0
+        time + @repeat_interval - ((time - @start_time) % @repeat_interval)
+      end
+    end
+
+  end
+
+end

data/test/bdrb_client_test_helper.rb

@@ -0,0 +1,5 @@
+require "rails_worker_proxy"
+require "bdrb_connection"
+require "bdrb_client_helper"
+require "bdrb_cluster_connection"
+require "bdrb_conn_error"

data/test/bdrb_test_helper.rb

@@ -0,0 +1,35 @@
+require "rubygems"
+require "mocha"
+require "test/spec"
+require "active_record"
+require "active_support"
+require "yaml"
+require "erb"
+
+class BDRB_CONFIG
+  @config_value = {}
+  def self.fuck
+    @config_value
+  end
+  def self.set hash
+    @config_value = hash
+  end
+  def self.[] key
+    @config_value[key]
+  end
+end
+
+RAILS_HOME = File.expand_path(File.join(File.dirname(__FILE__) + "/../../../..")) unless defined?(RAILS_HOME)
+PACKET_APP = RAILS_HOME + "/vendor/plugins/backgroundrb" unless defined?(PACKET_APP)
+WORKER_ROOT = RAILS_HOME + "/vendor/plugins/backgroundrb/test/workers" unless defined?(WORKER_ROOT)
+SERVER_LOGGER = RAILS_HOME + "/log/backgroundrb_debug.log" unless defined?(SERVER_LOGGER)
+
+["server","server/lib","lib","lib/backgroundrb"].each { |x| $LOAD_PATH.unshift(PACKET_APP + "/#{x}")}
+$LOAD_PATH.unshift(WORKER_ROOT)
+
+require "packet"
+require "bdrb_config"
+
+require "backgroundrb_server"
+
+

data/test/client/backgroundrb.yml

@@ -0,0 +1,17 @@
+---
+:backgroundrb:
+  :ip: 0.0.0.0
+  :port: 11008
+  :log: foreground
+  :environment: development
+  :result_storage: memcache
+
+:memcache: "localhost:11211"
+
+# :client: "0.0.0.0:11006,0.0.0.0:11008"
+
+:schedules:
+  :wow_worker:
+    :foo:
+      :trigger_args: */60 * * * * * *
+

data/test/client/test_bdrb_client_helper.rb

@@ -0,0 +1,13 @@
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+
+context "For client helper" do
+  specify "should return correct worker key" do
+    class Foo
+      include BackgrounDRb::ClientHelper
+    end
+    a = Foo.new
+    a.gen_worker_key("hello","world").should == :hello_world
+    a.gen_worker_key("hello").should == :hello
+  end
+end

data/test/client/test_bdrb_cluster_connection.rb

@@ -0,0 +1,162 @@
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+
+context "For Cluster connection" do
+  class BackgrounDRb::Connection
+    attr_accessor :server_ip,:server_port,:cluster_conn,:connection_status
+    def establish_connection
+      @connection_status = true
+    end
+
+    def close_connection
+      @connection_status = false
+    end
+    def server_info; "#{@server_ip}:#{server_port}"; end
+  end
+
+  setup do
+    BDRB_CONFIG.set({:schedules=> {
+        :foo_worker => { :barbar=>{:trigger_args=>"*/5 * * * * * *"}}},
+      :backgroundrb=>{:port=>11008, :ip=>"0.0.0.0", :environment=> "production"},
+      :client => "localhost:11001,localhost:11002,localhost:11003"
+    })
+
+    @cluster_connection = BackgrounDRb::ClusterConnection.new
+    class << @cluster_connection
+      def ivar(var)
+        return instance_variable_get("@#{var}")
+      end
+      def iset(var,value)
+        instance_variable_set("@#{var}",value)
+      end
+    end
+  end
+
+  specify "should read config file and connect to specified servers" do
+    @cluster_connection.backend_connections.length.should == 4
+    @cluster_connection.bdrb_servers.length.should == 4
+    @cluster_connection.ivar(:round_robin).length.should == 4
+    @cluster_connection.backend_connections[0].server_info.should == "localhost:11001"
+  end
+
+  specify "should return worker chosen in round robin manner if nothing specified" do
+    t_conn = @cluster_connection.choose_server
+    t_conn.server_info.should == "localhost:11001"
+    t_conn = @cluster_connection.choose_server
+    t_conn.server_info.should == "localhost:11002"
+  end
+
+  specify "should return connection from chosen host if specified" do
+    t_conn = @cluster_connection.find_connection("localhost:11001")
+    t_conn.server_info.should == "localhost:11001"
+  end
+
+  specify "should return connection from local host if specified" do
+    t_conn = @cluster_connection.find_local
+    t_conn.server_info.should == "0.0.0.0:11008"
+  end
+
+  specify "should not return disconnected connections" do
+    t_conn = @cluster_connection.find_next_except_these(Array("localhost:11001"))
+    @cluster_connection.ivar(:disconnected_connections).size.should == 1
+    @cluster_connection.backend_connections.size.should == 3
+    server_infos = @cluster_connection.backend_connections.map(&:server_info)
+    server_infos.should.include "0.0.0.0:11008"
+    server_infos.should.include "localhost:11002"
+    server_infos.should.include "localhost:11003"
+    server_infos.should.not.include "localhost:11001"
+    t_conn.server_info.should.not == "localhost:11001"
+  end
+
+  specify "should discover new connections when time to discover" do
+    t_conn = @cluster_connection.find_next_except_these(Array("localhost:11001"))
+    @cluster_connection.ivar(:disconnected_connections).size.should == 1
+    @cluster_connection.backend_connections.size.should == 3
+    @cluster_connection.iset(:request_count,9)
+    worker_proxy = @cluster_connection.worker(:hello_worker)
+    @cluster_connection.ivar(:disconnected_connections).size.should == 0
+    @cluster_connection.backend_connections.size.should == 4
+  end
+
+  specify "discover service should run only when connections were dropped" do
+    @cluster_connection.iset(:request_count,9)
+    @cluster_connection.expects(:disover_periodically).never
+    worker_proxy = @cluster_connection.worker(:hello_worker)
+  end
+
+  specify "should work with new_worker method calls" do
+    @cluster_connection.backend_connections.each do |t_conn|
+      t_conn.expects(:new_worker).with(:worker => :hello_worker,:worker_key => "boy",:data => "boy").returns(true)
+    end
+    a = @cluster_connection.new_worker(:worker => :hello_worker,:worker_key => "boy",:data => "boy")
+    a.should == "boy"
+  end
+
+  specify "should work with all worker info methods" do
+    @cluster_connection.backend_connections.each do |t_conn|
+      t_conn.expects(:all_worker_info).returns(:status => :running)
+    end
+    foo = @cluster_connection.all_worker_info
+    foo.should == {"0.0.0.0:11008"=>{:status=>:running}, "localhost:11001"=>{:status=>:running}, "localhost:11002"=>{:status=>:running}, "localhost:11003"=>{:status=>:running}}
+  end
+end
+
+context "For single connections" do
+  class BackgrounDRb::Connection
+    attr_accessor :server_ip,:server_port,:cluster_conn,:connection_status
+  end
+
+  setup do
+    BDRB_CONFIG.set({:schedules=> {
+        :foo_worker => { :barbar=>{:trigger_args=>"*/5 * * * * * *"}}},
+      :backgroundrb=>{:port=>11008, :ip=>"0.0.0.0", :environment=> "production"}
+    })
+
+    @cluster_connection = BackgrounDRb::ClusterConnection.new
+    class << @cluster_connection
+      def ivar(var)
+        return instance_variable_get("@#{var}")
+      end
+    end
+  end
+
+  specify "should read config file and connect to servers" do
+    @cluster_connection.backend_connections.length.should == 1
+    @cluster_connection.bdrb_servers.length.should == 1
+    @cluster_connection.ivar(:round_robin).length.should == 1
+    @cluster_connection.backend_connections[0].server_info.should == "0.0.0.0:11008"
+  end
+end
+
+context "For memcache based result storage" do
+  setup do
+    options = { :schedules =>
+      {
+        :foo_worker => { :barbar=>{:trigger_args=>"*/5 * * * * * *"}}
+      },
+      :backgroundrb =>
+      {
+        :port=>11008, :ip=>"0.0.0.0", :environment=> "production",:result_storage => 'memcache'
+      },
+      :client => "localhost:11001,localhost:11002,localhost:11003",
+      :memcache => "10.0.0.1:11211,10.0.0.2:11211"
+    }
+    BDRB_CONFIG.set(options)
+
+    @cluster_connection = BackgrounDRb::ClusterConnection.new
+    class << @cluster_connection
+      def ivar(var)
+        return instance_variable_get("@#{var}")
+      end
+      def iset(var,value)
+        instance_variable_set("@#{var}",value)
+      end
+    end
+  end
+
+
+  specify "should use memcache based result storage if specified" do
+    @cluster_connection.cache.should.not == nil
+    @cluster_connection.cache.class.should == MemCache
+  end
+end

data/test/client/test_bdrb_config.rb

@@ -0,0 +1,20 @@
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+
+context "For BackgrounDRb config" do
+  conf_file = File.join(File.dirname(__FILE__),"backgroundrb.yml")
+  specify "should setup correct environment from cmd options" do
+    BackgrounDRb::Config.parse_cmd_options(["-e", "production"])
+    BackgrounDRb::Config.read_config(conf_file)
+    ENV["RAILS_ENV"].should == "production"
+    RAILS_ENV.should == "production"
+  end
+
+  specify "should setup correct environment from conf file" do
+    ENV["RAILS_ENV"] = nil
+    BackgrounDRb::Config.parse_cmd_options([])
+    BackgrounDRb::Config.read_config(conf_file)
+    ENV["RAILS_ENV"].should == "development"
+    RAILS_ENV.should == "development"
+  end
+end

data/test/client/test_bdrb_connection.rb

@@ -0,0 +1,29 @@
+require File.join(File.dirname(__FILE__) + "/../socket_mocker")
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+
+context "For Actual BackgrounDRB connection" do
+  setup do
+    options = {:schedules => {
+        :foo_worker => { :barbar=>{:trigger_args=>"*/5 * * * * * *"}}},
+      :backgroundrb=>{:port=>11008, :ip=>"0.0.0.0", :environment=> "production"},
+      :client => "localhost:11001,localhost:11002,localhost:11003"
+    }
+    BDRB_CONFIG.set(options)
+    @cluster = mock()
+    @foo_connection = BackgrounDRb::Connection.new('localhost',1267,"crap")
+    @foo_connection.stubs(:close_connection)
+    class << @foo_connection
+      attr_accessor :outgoing_data
+      def dump_object data
+        @outgoing_data = data
+      end
+    end
+  end
+
+  specify "should return nil if ask_result returns nil" do
+    a = @foo_connection.ask_result(:worker => 'foo_worker',:worker_key => 'bar',:job_key => 10)
+    a.should == nil
+    @foo_connection.outgoing_data.should == {:type=>:get_result, :job_key=>10, :worker=>"foo_worker", :worker_key=>"bar"}
+  end
+end

data/test/client/test_bdrb_job_queue.rb

@@ -0,0 +1,63 @@
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+require "bdrb_job_queue"
+
+context "For BackgrounDRb job Queues" do
+  setup do
+    db_config_file = YAML.load(ERB.new(IO.read("#{RAILS_HOME}/config/database.yml")).result)
+    ActiveRecord::Base.establish_connection(db_config_file["test"])
+    BdrbJobQueue.destroy_all
+  end
+
+  specify "should insert job with proper params" do
+    BdrbJobQueue.insert_job(:worker_name => "hello_world",:worker_method => "foovar",:job_key => "cats",:args => "hello_world",:scheduled_at => Time.now.utc)
+    next_job = BdrbJobQueue.find_next("hello_world")
+    next_job.taken.should == 1
+    next_job.started_at.should.not.be nil
+    next_job.job_key.should == "cats"
+    next_job.worker_name.should == "hello_world"
+    next_job.worker_method.should == "foovar"
+  end
+
+  specify "should respect job priority" do
+
+    BdrbJobQueue.insert_job(:priority => 4, :worker_name => "hello_world",:worker_method => "foovar",:job_key => "4",:args => "priority 4", :scheduled_at => Time.now.utc)
+    BdrbJobQueue.insert_job(:priority => 1, :worker_name => "hello_world",:worker_method => "foovar",:job_key => "1",:args => "priority 1", :scheduled_at => Time.now.utc)
+    BdrbJobQueue.insert_job(:priority => 10, :worker_name => "hello_world",:worker_method => "foovar",:job_key => "10",:args => "priority 10", :scheduled_at => Time.now.utc)
+
+    [10,4,1].each do |priority|      
+      next_job = BdrbJobQueue.find_next("hello_world")
+      next_job.taken.should == 1
+      next_job.started_at.should.not.be nil
+      next_job.job_key.should == priority.to_s
+      next_job.worker_name.should == "hello_world"
+      next_job.worker_method.should == "foovar"
+      next_job.priority.should == priority
+    end
+  end
+
+  specify "release_job should worker properly" do
+    BdrbJobQueue.insert_job(:worker_name => "hello_world",:worker_method => "foovar",:job_key => "cats",:args => "hello_world",:scheduled_at => Time.now.utc)
+    next_job = BdrbJobQueue.find_next("hello_world")
+    next_job.release_job
+    t = BdrbJobQueue.find_by_job_key("cats")
+    t.taken.should == 0
+    t.started_at.should == nil
+  end
+
+  specify "remove job should work properly" do
+    BdrbJobQueue.insert_job(:worker_name => "hello_world",:worker_method => "foovar",:job_key => "cats",:args => "hello_world",:scheduled_at => Time.now.utc)
+    BdrbJobQueue.remove_job(:worker_name => "hello_world",:worker_method => "foovar",:job_key => "cats")
+    t = BdrbJobQueue.find_by_job_key("cats")
+    t.should == nil
+  end
+
+  specify "finish should work properly" do
+    BdrbJobQueue.insert_job(:worker_name => "hello_world",:worker_method => "foovar",:job_key => "cats",:args => "hello_world",:scheduled_at => Time.now.utc)
+    t = BdrbJobQueue.find_next("hello_world")
+    t.finish!
+    t.finished.should == 1
+    t.finished_at.should.not == nil
+    t.job_key.should.match(/finished_\d+_cats/i)
+  end
+end

data/test/client/test_worker_proxy.rb

@@ -0,0 +1,130 @@
+require File.join(File.dirname(__FILE__) + "/../bdrb_test_helper")
+require File.join(File.dirname(__FILE__) + "/../bdrb_client_test_helper")
+
+context "Worker Proxy in general" do
+  setup do
+    BDRB_CONFIG.set({:schedules=> {
+                        :foo_worker => { :barbar=>{:trigger_args=>"*/5 * * * * * *"}}},
+                      :backgroundrb=>{:port=>11008, :ip=>"0.0.0.0", :environment=> "production"}
+                    })
+
+    @cluster_conn = mock
+    @worker_proxy = BackgrounDRb::RailsWorkerProxy.new(:hello_worker,nil,@cluster_conn)
+  end
+
+  specify "should let you fetch results" do
+    @cluster_conn.expects(:backend_connections).returns([])
+    foo = @worker_proxy.ask_result(:foobar)
+    foo.should.be nil
+  end
+
+  specify "should let you invoke sync task  methods" do
+    actual_conn = mock()
+    actual_conn.expects(:server_info).returns("localhost:11008")
+    actual_conn.expects(:send_request).returns({ :data => 20, :result_flag => "ok",:result => true, :type => :response})
+    @cluster_conn.expects(:choose_server).returns(actual_conn)
+    a = @worker_proxy.hello_world(:args => "sucks")
+    a.should == 20
+  end
+
+  specify "should let you invoke delete method" do
+    actual_conn = mock()
+    actual_conn.expects(:delete_worker).with(:worker => :hello_worker).returns(nil)
+    @cluster_conn.expects(:backend_connections).returns(Array(actual_conn))
+    @worker_proxy.delete
+  end
+
+  specify "delete method should run on all nodes" do
+    conn_array = (0..3).map do |i|
+      t = mock()
+      t.expects(:delete_worker).with(:worker => :hello_worker).returns(nil)
+      t
+    end
+    @cluster_conn.expects(:backend_connections).returns(conn_array)
+    @worker_proxy.delete
+  end
+
+  specify "should let you invoke worker_info method" do
+    backend_connections = []
+    2.times { |i|
+      actual_conn = mock()
+      actual_conn.expects(:worker_info).with(:worker => :hello_worker).returns(i)
+      backend_connections << actual_conn
+    }
+    @cluster_conn.expects(:backend_connections).returns(backend_connections)
+    a = @worker_proxy.worker_info
+    a.should == [0,1]
+  end
+
+  specify "should let you run async tasks" do
+    actual_conn = mock()
+    actual_conn.expects(:ask_work).with(:arg => :hello,:worker => :hello_worker,:worker_method => 'foobar',:job_key => 'boy').returns(nil)
+    @cluster_conn.expects(:find_connection).returns(actual_conn)
+    @worker_proxy.async_foobar(:arg => :hello,:job_key => "boy",
+                               :host => "192.168.2.100:100")
+  end
+
+  specify "for enqueued tasks" do
+    BdrbJobQueue = mock() unless Object.const_defined?(:BdrbJobQueue)
+    BdrbJobQueue.expects(:insert_job).with() { |value|
+      value[:worker_name].should == "hello_worker"
+      value[:worker_method].should == "foobar"
+      value[:scheduled_at].should.not == nil
+      value[:job_key] == "catz"
+    }
+    @worker_proxy.enq_foobar(:arg => :hello,:job_key => "catz")
+  end
+
+
+  specify "should run enqueued tasks in order if they have priorites" do
+
+    BdrbJobQueue = mock() unless Object.const_defined?(:BdrbJobQueue)
+
+    [2,4,10].each do |priority|
+      BdrbJobQueue.expects(:insert_job).with() { |value|
+        value[:worker_name].should == "hello_worker"
+        value[:worker_method].should == "foobar"
+        value[:scheduled_at].should.not == nil
+        value[:job_key] == priority.to_s
+        value[:priority] == priority
+      }.once
+    end
+    
+    @worker_proxy.enq_foobar(:job_key => '4', :priority => 4, :arg => :hello)
+    @worker_proxy.enq_foobar(:job_key => '2', :priority => 2, :arg => :hello)
+    @worker_proxy.enq_foobar(:job_key => '10', :priority => 10, :arg => :hello)
+  end
+
+
+  specify "for removing tasks from the queue" do
+    BdrbJobQueue = mock() unless Object.const_defined?(:BdrbJobQueue)
+    BdrbJobQueue.expects(:remove_job).with() do |value|
+      value[:worker_name] == "hello_worker"
+      value[:worker_method] == "foobar"
+      value[:job_key] == "catz"
+    end
+    @worker_proxy.deq_foobar(:job_key => "catz")
+  end
+
+  specify "should run task on all servers if asked" do
+    backend_connections = []
+    2.times { |i|
+      actual_conn = mock()
+      actual_conn.expects(:ask_work).with(:worker => :hello_worker,:worker_method => 'foobar',:job_key => 'hello')
+      backend_connections << actual_conn
+    }
+    @cluster_conn.expects(:backend_connections).returns(backend_connections)
+    a = @worker_proxy.async_foobar(:job_key => "hello",:host => :all)
+  end
+
+  specify "should switch connections if invoke fails on chosen one" do
+  end
+
+  specify "Should allow method with empty params to work" do
+    actual_conn = mock()
+    actual_conn.expects(:server_info).returns("localhost:11211")
+    actual_conn.expects(:ask_work).with(:worker => :hello_worker,:worker_method => 'foobar').returns(nil)
+    @cluster_conn.expects(:choose_server).returns(actual_conn)
+    @worker_proxy.async_foobar
+  end
+end