background_queue 0.2.0

data/spec/background_queue/server_lib/event_connection_spec.rb

@@ -0,0 +1,190 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::EventConnection do
+
+  subject { 
+    s = BackgroundQueue::ServerLib::EventConnection.new(:something)
+    s.server = SimpleServer.new
+    s.post_init
+    s
+  }
+  
+  
+  let(:data) { [1, 20, "12345678901234567890"].pack("SLZ20")}
+  
+  context "#receive_data" do
+    it "can handle a packet in 1 call" do
+      subject.should_receive(:process_data).with("12345678901234567890")
+      subject.receive_data(data)
+    end
+    
+    it "can handle the header in 2 calls" do
+      subject.should_receive(:process_data).with("12345678901234567890")
+      subject.receive_data(data.slice!(0, 2))
+      subject.receive_data(data.slice!(0, 4))
+      subject.receive_data(data)
+    end
+    
+    it "can handle the body in 2 calls" do
+      subject.should_receive(:process_data).with("12345678901234567890")
+      subject.receive_data(data.slice!(0, 6))
+      subject.receive_data(data.slice!(0, 4))
+      subject.receive_data(data)
+    end
+    
+    it "can handle the packets crossing header/body boundry" do
+      subject.should_receive(:process_data).with("12345678901234567890")
+      subject.receive_data(data.slice!(0, 4))
+      subject.receive_data(data.slice!(0, 10))
+      subject.receive_data(data)
+    end
+    
+    it "will error if the version is wrong" do
+      expect { subject.receive_data("ABCDEFG") }.to raise_exception("Invalid header version: 16961")
+    end
+  end
+  
+  context "#process_data" do
+    it "will convert the data to a json object" do
+      BackgroundQueue::Command.should_receive(:from_buf).with("{'a':'b'}").and_return(:command)
+      subject.should_receive(:process_command).with(:command).and_return(:result)
+      subject.should_receive(:send_result).with(:result)
+      subject.process_data("{'a':'b'}")
+    end
+    
+    it "will return an error if the json is invalid" do
+      subject.should_receive(:send_error)
+      subject.process_data("{sdf, sdfsdf}")
+    end
+    
+  end
+  
+  context "#process_command" do
+    it "will call process_add_task_cmd if cmd is add_task" do
+      cmd = double("command", :code=>'add_task')
+      subject.should_receive(:process_add_task_command).with(cmd).and_return(:result)
+      subject.process_command(cmd).should eq(:result)
+    end
+    
+    it "will call process_add_tasks_command if cmd is add_tasks" do
+      cmd = double("command", :code=>'add_tasks')
+      subject.should_receive(:process_add_tasks_command).with(cmd).and_return(:result)
+      subject.process_command(cmd).should eq(:result)
+    end
+    
+    it "will call process_remove_tasks_command if cmd is remove_tasks" do
+      cmd = double("command", :code=>'remove_tasks')
+      subject.should_receive(:process_remove_tasks_command).with(cmd).and_return(:result)
+      subject.process_command(cmd).should eq(:result)
+    end
+    
+    it "will call process_get_status_command if cmd is get_status" do
+      cmd = double("command", :code=>'get_status')
+      subject.should_receive(:process_get_status_command).with(cmd).and_return(:result)
+      subject.process_command(cmd).should eq(:result)
+    end
+    
+    it "will raise an error if unknown task" do
+      cmd = double("command", :code=>'dunno')
+      expect { subject.process_command(cmd).should eq(:result) }.to raise_exception('Unknown command: "dunno"')
+    end
+  end
+  
+  context "#send_result" do
+    it "will serialize the result to a json string and send" do
+      cmd = BackgroundQueue::Command.new('a', {}, {})
+      subject.should_receive(:send).with(cmd.to_buf)
+      subject.send_result(cmd)
+    end
+  end
+  
+  context "#send_error" do
+    it "will wrap the error into a command and call send_result" do
+      BackgroundQueue::Command.should_receive(:new).with(:error, {}, {:message=>"error"}).and_return(:command)
+      subject.should_receive(:send_result).with(:command)
+      subject.send_error("error")
+    end
+  end
+  
+  context "#send" do
+    it "will add a header to the data and call send_data" do
+      data = [1,4,"data"].pack("SLZ4")
+      subject.should_receive(:send_data).with(data)
+      subject.send("data")
+    end
+  end
+  
+  context "#process_add_task_command" do
+    it "builds a task from the command and adds it to the queue" do
+      
+      server = SimpleServer.new(:task_queue=>double('task_queue'))
+      server.task_queue.should_receive(:add_task).with(:task)
+      
+      command = BackgroundQueue::Command.new(:add_task, {}, {'owner_id'=>:owner_id, :job_id=>:job_id, 'task_id'=>:task_id, :priority=>:priority, :params=>:params, :worker=>:worker  } )
+      
+      BackgroundQueue::ServerLib::Task.should_receive(:new).with(:owner_id, :job_id, :task_id, :priority, :worker, :params, command.options).and_return(:task)
+      
+      
+      subject.server = server
+      subject.process_add_task_command(command).code.should eq(:result)
+    end
+    
+  end
+  
+  
+  context "#process_add_tasks_command" do
+    it "builds a set of tasks from the command and adds them to the queue" do
+      
+      server = SimpleServer.new(:task_queue=>double('task_queue'))
+      server.task_queue.should_receive(:add_task).with(:task1)
+      server.task_queue.should_receive(:add_task).with(:task2)
+      
+      command = BackgroundQueue::Command.new(:add_tasks, {}, {'owner_id'=>:owner_id, :job_id=>:job_id, :tasks=>[[:task1_id, {:a=>:b}], [:task2_id, {:e=>:f}]], :priority=>:priority, :params=>:params, :worker=>:worker, :shared_parameters=>{:shared=>:params}  } )
+      
+      BackgroundQueue::ServerLib::Task.should_receive(:new).with(:owner_id, :job_id, :task1_id, :priority, :worker, {:a=>:b, :shared=>:params}, command.options).and_return(:task1)
+      BackgroundQueue::ServerLib::Task.should_receive(:new).with(:owner_id, :job_id, :task2_id, :priority, :worker, {:e=>:f, :shared=>:params}, command.options).and_return(:task2)
+      
+      subject.server = server
+      subject.process_add_tasks_command(command).code.should eq(:result)
+    end
+    
+  end
+  
+  context "#process_get_status_command" do
+    it "will get the status of a registered job" do
+      server = SimpleServer.new(:jobs=>double('jobs'))
+      job = double("job")
+      server.jobs.should_receive(:get_job).with(:job_id).and_return(job)
+      job.should_receive(:get_current_progress).and_return({:percent=>100, :caption=>"done"})
+      subject.server = server
+      
+      command = BackgroundQueue::Command.new(:get_status, {}, {:job_id=>:job_id } )
+      
+      result = subject.process_get_status_command(command)
+      
+      result.args[:percent].should eq(100)
+      result.args[:caption].should eq("done")
+      result.code.should eq(:status)
+    end
+    
+    it "will return job_not_found if job is not registered" do
+      server = SimpleServer.new(:jobs=>double('jobs'))
+      job = double("job")
+      subject.server = server
+      server.jobs.should_receive(:get_job).with(:job_id).and_return(nil)
+      command = BackgroundQueue::Command.new(:get_status, {}, {:job_id=>:job_id } )
+      
+      result = subject.process_get_status_command(command)
+      
+      result.code.should eq(:job_not_found)
+      
+    end
+  end
+  
+  
+  
+  
+  
+end

data/spec/background_queue/server_lib/event_server_spec.rb

@@ -0,0 +1,48 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+#module Rufus
+#  module Scheduler
+#    class EmScheduler
+#    end
+#  end
+#end
+
+describe BackgroundQueue::ServerLib::EventServer do
+
+  let(:server) { SimpleServer.new(:config=>double("config", :address=>double("address", :host=>:host, :port=>:port))) }
+  subject { BackgroundQueue::ServerLib::EventServer.new(server) }
+  
+  
+  context "#start" do
+    it "starts the event machine" do
+      
+      con=double("conection")
+      con.should_receive("server=".intern).with(server)
+      EventMachine.should_receive(:run).and_yield
+      EventMachine.should_receive(:start_server).with(:host, :port, BackgroundQueue::ServerLib::EventConnection).and_yield(con)
+      
+      job = double("job")
+      scheduler = double("scheduler")
+      scheduler.should_receive(:start)
+      server.config.stub(:jobs=>[job])
+      Rufus::Scheduler::EmScheduler.should_receive(:new).and_return(scheduler)
+      job.should_receive(:schedule).with(scheduler, server)
+      
+      subject.start
+    end
+  end
+  
+ # context "#start_jobs" do
+ #   it "schedules jobs" do
+ #     job = double("job")
+ #     server.config.stub(:jobs=>[job])
+ #     Rufus::Scheduler::EmScheduler.should_receive(:start_new).and_return(:scheduler)
+ #     job.should_receive(:schedule).with(:scheduler, server)
+ #     EventMachine.should_receive(:run).and_yield
+ #     subject.start_jobs
+ #   end
+ # end
+
+  
+end

data/spec/background_queue/server_lib/integration/full_test_spec.rb

@@ -0,0 +1,247 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper')
+require 'background_queue_server'
+require 'background_queue'
+require 'background_queue_worker'
+require 'net/http'
+
+require 'rubygems'
+gem "rufus-scheduler"
+
+#unless defined? Rails
+  module Rails; 
+    def self.env
+      "development"
+    end
+  end
+#end
+
+describe "Full Test" do
+
+  context "simple task" do
+  
+    it "will call worker server" do
+
+      config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config.yml')
+      
+      server = BackgroundQueue::ServerLib::Server.new
+
+      thread = Thread.new {
+        server.start(:config=>config_path, :skip_pid=>true, :command=>:run, :log_file=>"/tmp/bq.log", :log_level=>'debug')
+      }
+
+      while server.event_server.nil? || !server.event_server.running
+        sleep(0.1)
+      end
+      
+      begin
+
+        meth_name = nil
+        ss = TestWorkerServer.new(8001)
+        ss.start(Proc.new { |controller|
+          meth_name = controller.request.request_method
+          controller.render :text => lambda { |response,output| 
+            output.write({:percent=>100, :caption=>"Done"}.to_json)
+          }, :type=>"text/text"
+          #controller.render :text=>, :type=>"text/text"
+        })
+        
+        
+        client_config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config-client.yml')
+        
+        client = BackgroundQueue::Client.new(client_config_path)
+        
+  
+        job_handle = client.add_task(:some_worker, :owner_id, :job_id, :task_id, 2, {:something=>:else}, {:domain=>"www.example.com"} )
+        
+        
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(100)
+        result.args[:caption].should eq('Done')
+  
+        meth_name.should eq("POST")
+        ss.stop
+      ensure
+        server.stop
+        
+        thread.join
+      end
+    end
+    
+  end
+  
+  context "init task" do
+    it "will spawn more tasks" do
+      config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config.yml')
+      
+
+      
+      server = BackgroundQueue::ServerLib::Server.new
+      
+
+
+      thread = Thread.new {
+        server.start(:config=>config_path, :skip_pid=>true, :command=>:run, :log_file=>"/tmp/bq.log", :log_level=>'debug')
+      }
+      
+
+
+      while server.event_server.nil? || !server.event_server.running
+        sleep(0.1)
+      end
+      
+      begin
+
+        task_pos = 0
+        
+  
+        
+        client_config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config-client.yml')
+        
+        client = BackgroundQueue::Client.new(client_config_path)
+        
+        meth_name = nil
+        ss = TestWorkerServer.new(8001, true)
+        ss.start(Proc.new { |controller|
+          #puts "worker called"
+          meth_name = controller.request.request_method
+          if task_pos == 0
+            task_pos += 1
+            #puts "Adding more tasks"
+            client2 = BackgroundQueue::Client.new(client_config_path)
+            result2, server_used2 = client.add_tasks(:some_worker, :owner_id, :job_id, [[:task1_id, {:a=>:b}], [:task2_id, {:e=>:f}]], 2, {:something=>:else}, {:domain=>"www.example.com"} )
+          end
+          controller.render :text=>{:percent=>100, :caption=>"Done"}.to_json, :type=>"text/text"
+        })
+        
+        
+        
+        job_handle = client.add_task(:some_worker, :owner_id, :job_id, :task_id, 2, {:something=>:else}, {:domain=>"www.example.com", :weight=>20.0, :exclude=>true, :initial_progress_caption=>"Loading" } )
+        
+        result = client.get_status(job_handle )
+  
+        #puts "test1"
+        result.code.should eq(:status)
+        result.args[:percent].should eq(0.0)
+        result.args[:caption].should eq('Loading')
+        
+        
+        stats = client.get_stats(job_handle.server)
+        stats[:running].should eq(1)
+  
+  
+        
+        ss.allow_to_be_called
+  
+        ss.wait_to_be_called.should be_true
+  
+        
+        result = client.get_status(job_handle )
+  
+        #puts "test1"
+        result.code.should eq(:status)
+        result.args[:percent].should eq(20.0)
+        result.args[:caption].should eq('Done')
+  
+        meth_name.should eq("POST")
+        
+        stats = client.get_stats(job_handle.server)
+        stats[:running].should eq(1)   
+        stats[:run_tasks].should eq(1)
+        
+        #pp server
+        
+        ss.allow_to_be_called
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle )
+  
+        #puts "test2"
+        result.code.should eq(:status)
+        result.args[:percent].should eq(60.0)
+        result.args[:caption].should eq('Done (2/2)')
+        
+        
+        ss.allow_to_be_called
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle )
+  
+        #puts "test3"
+        result.code.should eq(:status)
+        result.args[:percent].should eq(100.0)
+        result.args[:caption].should eq('Done (2/2)')
+        
+        stats = client.get_stats(job_handle.server)
+        stats[:tasks].should eq(0) 
+        stats[:running].should eq(0)   
+        stats[:run_tasks].should eq(3)
+        
+        ss.stop
+      ensure
+        server.stop
+        thread.join
+      end
+    end
+  end
+  
+  
+  context "with worker" do
+  
+    it "will load a worker and call it" do
+
+      config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config.yml')
+      BackgroundQueue::Worker::Config.worker_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/')
+      BackgroundQueue::Worker::Config.secret = "this_is_used_to_make_sure_it_is_secure"
+      
+      server = BackgroundQueue::ServerLib::Server.new
+
+      thread = Thread.new {
+        server.start(:config=>config_path, :skip_pid=>true, :command=>:run, :log_file=>"/tmp/bq.log", :log_level=>'debug')
+      }
+
+      while server.event_server.nil? || !server.event_server.running
+        sleep(0.1)
+      end
+      
+      begin
+
+        meth_name = nil
+        ss = TestWorkerServer.new(8001)
+        ss.start(Proc.new { |controller|
+            #puts controller.params.inspect
+            controller.class.send(:include, BackgroundQueue::Worker::Calling)
+           # puts "B"
+            controller.run_worker
+           # puts "C"
+        })
+        
+        
+        client_config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config-client.yml')
+        
+        client = BackgroundQueue::Client.new(client_config_path)
+        
+  
+        job_handle = client.add_task(:test_worker, :owner_id, :job_id, :task_id, 2, {:something=>:else}, {:domain=>"www.example.com"} )
+  
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(100)
+        result.args[:caption].should eq('Done')
+
+        ss.stop
+      ensure
+        server.stop
+        
+        thread.join
+      end
+    end
+    
+  end
+end