background_queue 0.2.0

data/lib/background_queue_worker.rb

@@ -0,0 +1,5 @@
+require "background_queue/worker/base"
+require "background_queue/worker/environment"
+require "background_queue/worker/worker_loader"
+require "background_queue/worker/calling"
+require "background_queue/worker/config"

data/spec/background_queue/client_lib/command_spec.rb

@@ -0,0 +1,75 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue'
+
+describe "Command" do
+
+  context "add_task" do
+    it "creates valid command" do
+      cmd = BackgroundQueue::ClientLib::Command.add_task_command(:worker, :owner_id, :job_id, :task_id, 1, {:a=>:b}, {:c=>:d} )
+      cmd.code.should eq(:add_task)
+      cmd.options[:c].should eq(:d)
+      cmd.args[:worker].should eq(:worker)
+      cmd.args[:owner_id].should eq(:owner_id)
+      cmd.args[:job_id].should eq(:job_id)
+      cmd.args[:task_id].should eq(:task_id)
+      cmd.args[:params][:a].should eq(:b)
+      
+    end
+  end
+  
+  context "add_tasks" do
+    it "creates valid command" do
+      cmd = BackgroundQueue::ClientLib::Command.add_tasks_command(:worker, :owner_id, :job_id, [[:task1_id, {:a=>:b}], [:task2_id, {:e=>:f}]], 1, {:shared=>:param}, {:c=>:d} )
+      cmd.code.should eq(:add_tasks)
+      cmd.options[:c].should eq(:d)
+      cmd.args[:worker].should eq(:worker)
+      cmd.args[:owner_id].should eq(:owner_id)
+      cmd.args[:job_id].should eq(:job_id)
+      cmd.args[:shared_parameters][:shared].should eq(:param)
+      cmd.args[:tasks].length.should eq(2)
+      cmd.args[:tasks].first.length.should eq(2)
+      cmd.args[:tasks].first[0].should eq(:task1_id)
+      cmd.args[:tasks].first[1][:a].should eq(:b)
+      cmd.args[:tasks].last.length.should eq(2)
+      cmd.args[:tasks].last[0].should eq(:task2_id)
+      cmd.args[:tasks].last[1][:e].should eq(:f)
+    end
+    
+    it "fails if no tasks are defined" do
+      expect {
+       cmd = BackgroundQueue::ClientLib::Command.add_tasks_command(:worker, :owner_id, :job_id, [], 1, {:shared=>:param}, {:c=>:d} )
+      }.to raise_exception(BackgroundQueue::InvalidCommand, "No Tasks In List")
+    end
+  end
+  
+  
+  context "#get_status_command" do
+    it "creates valid command" do
+      cmd = BackgroundQueue::ClientLib::Command.get_status_command(:job_id,  {:c=>:d} )
+      cmd.code.should eq(:get_status)
+      cmd.options[:c].should eq(:d)
+      cmd.args[:job_id].should eq(:job_id)
+    end
+  end
+  
+  
+  #i cant see why this would be needed.... will add back if needed
+  #context "remove_tasks" do
+  #  it "creates valid command" do
+  #    cmd = BackgroundQueue::ClientLib::Command.remove_tasks_command([:task1_id,:task2_id], {:c=>:d} )
+  #    cmd.code.should eq(:remove_tasks)
+  #    cmd.options[:c].should eq(:d)
+  #    cmd.args[:tasks].length.should eq(2)
+  #    cmd.args[:tasks][0].should eq(:task1_id)
+  #    cmd.args[:tasks][1].should eq(:task2_id)
+  #  end
+  #  
+  #  it "fails if no tasks are defined" do
+  #    expect {
+  #     cmd = BackgroundQueue::ClientLib::Command.remove_tasks_command([], {:c=>:d} )
+  #    }.to raise_exception(BackgroundQueue::InvalidCommand, "No Tasks In List")
+  #  end
+  #end
+  
+
+end

data/spec/background_queue/client_lib/config_spec.rb

@@ -0,0 +1,156 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue'
+
+unless defined? Rails
+  module Rails; end
+end
+  
+describe "Client Config" do
+
+  context "using map" do
+    context "loading server entry" do
+      it "creates a server entry if all required string fields exist" do
+        entry = BackgroundQueue::ClientLib::Config::Server.new({
+            'host'=>"127.0.0.1",
+            'port'=>85
+        })
+        entry.host.should eq("127.0.0.1")
+        entry.port.should eq(85)
+      end
+      
+      it "creates a server entry if all required symbol fields exist" do
+        entry = BackgroundQueue::ClientLib::Config::Server.new({
+            :host=>"127.0.0.1",
+            :port=>85
+        })
+        entry.host.should eq("127.0.0.1")
+        entry.port.should eq(85)
+      end
+      
+      it "creates a server entry using default port" do
+        entry = BackgroundQueue::ClientLib::Config::Server.new({
+            :host=>"127.0.0.1"
+        })
+        entry.host.should eq("127.0.0.1")
+        entry.port.should eq(BackgroundQueue::ClientLib::Config::DEFAULT_PORT)
+      end
+      
+       it "errors if missing or invalid hostname on server entry" do
+         expect { 
+          BackgroundQueue::ClientLib::Config::Server.new({})
+         }.to raise_error(
+           BackgroundQueue::LoadError, 
+           "Missing 'host' configuration entry"
+         )
+       end
+       
+       it "errors if invalid type" do
+         expect { 
+          BackgroundQueue::ClientLib::Config::Server.new([])
+         }.to raise_error(
+           BackgroundQueue::LoadError, 
+           "Invalid data type (Array), expecting Hash"
+         )
+       end
+    end
+    
+    context "loading primary server entry" do
+      before do
+        File.stub(:expand_path) { :expanded_path }
+      end
+      
+      it "creates primary server entry" do
+        entry = BackgroundQueue::ClientLib::Config.__prv__build_primary_server_entry({ 'server'=> { :host=>"127.0.0.1" }}, :path_that_exists)
+        entry.host.should eq("127.0.0.1")
+      end
+      
+      it "errors if missing" do
+        expect { 
+          BackgroundQueue::ClientLib::Config.__prv__build_primary_server_entry({}, :path_that_exists)
+        }.to raise_error(
+          BackgroundQueue::LoadError, 
+          "Missing 'server' entry in background queue configuration file expanded_path"
+        )
+      end
+      
+      it "errors if invalid" do
+        expect { 
+          BackgroundQueue::ClientLib::Config.__prv__build_primary_server_entry({:server=> {}}, :path_that_exists)
+        }.to raise_error(
+          BackgroundQueue::LoadError, 
+          "Error loading 'server' entry from background queue configuration file expanded_path: Missing 'host' configuration entry"
+        )
+      end
+    end
+    
+    context "loading failover server entries" do
+    
+      it "creates failover server entries if defined" do
+        entries = BackgroundQueue::ClientLib::Config.__prv__build_failover_server_entries({ :failover=> [{ :host=>"127.0.0.1" }, { :host=>"127.0.0.2" }]}, :path_that_exists)
+        entries.length.should eq(2)
+        entries.first.host.should eq("127.0.0.1")
+        entries.last.host.should eq("127.0.0.2")
+      end
+      
+      it "errors if failover entry is invalid" do
+        File.stub(:expand_path) { :expanded_path }
+        expect { 
+          BackgroundQueue::ClientLib::Config.__prv__build_failover_server_entries({ 'failover'=> [{ :host=>"127.0.0.1" }, { }]}, :path_that_exists)
+        }.to raise_error(
+          BackgroundQueue::LoadError, 
+          "Error loading 'failover' entry (2) from background queue configuration file expanded_path: Missing 'host' configuration entry"
+        )
+      end
+    
+    end
+    
+    
+    
+    context "creating Config entry" do
+      before do
+        File.stub(:expand_path) { :expanded_path }
+      end
+      
+      it "should create Config instance" do
+        config = BackgroundQueue::ClientLib::Config.load_hash({
+            :server=>{:host=>"127.0.0.1"},
+            :failover=>[{:host=>"127.0.0.2"}, {:host=>"127.0.0.3"}],
+            :memcache=> "127.0.0.1:4000"
+        }, :path_that_exists)
+        config.server.host.should eq("127.0.0.1")
+        config.failover.length.should eq(2)
+        config.failover.first.host.should eq("127.0.0.2")
+        config.failover.last.host.should eq("127.0.0.3")
+        config.memcache.length.should eq(1)
+        config.memcache.first.should eq('127.0.0.1:4000')
+      end
+      
+      it "should fail when missing server" do
+        expect { 
+          config = BackgroundQueue::ClientLib::Config.load_hash({ }, :path_that_exists)
+        }.to raise_error(
+          BackgroundQueue::LoadError, 
+          "Missing 'server' entry in background queue configuration file expanded_path"
+        )
+      end
+      
+      it "should fail when missing memcache" do
+        expect { 
+          config = BackgroundQueue::ClientLib::Config.load_hash({ :server=>{:host=>"127.0.0.1"} }, :path_that_exists)
+        }.to raise_error(
+          BackgroundQueue::LoadError, 
+          "Missing 'memcache' entry in configuration file expanded_path"
+        )
+      end
+      
+      it "should not fail when missing failover" do
+        config = BackgroundQueue::ClientLib::Config.load_hash({
+            :server=>{:host=>"127.0.0.1"},
+            :memcache=> "127.0.0.1:4000"
+        }, :path_that_exists)
+        config.failover.length.should eq(0)
+      end
+    end
+  end
+
+end

data/spec/background_queue/client_lib/connection_spec.rb

@@ -0,0 +1,170 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue'
+
+describe "Connection" do
+
+  let(:server) { 
+    svr = double('Server')
+    svr.stub({:host=>:host, :port=>:port})
+    svr
+  }
+  
+  
+
+  context "connecting" do
+    
+    subject { BackgroundQueue::ClientLib::Connection.new(:client, server) }
+    
+    it "can successfully connect" do
+      TCPSocket.should_receive(:open).with(:host, :port) { :socket }
+      subject.__prv__connect.should eq(true)
+    end
+    
+    it "fails when no server to connect to" do
+      TCPSocket.should_receive(:open).with(:host, :port) { raise "Unable to connect" }
+      expect { subject.__prv__connect }.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Error Connecting to host:port: Unable to connect")
+    end
+    
+    it "fails if connection times out" do
+      Timeout.should_receive(:timeout).with(any_args) { raise Timeout::Error }  
+      expect { subject.__prv__connect }.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Timeout Connecting to host:port")
+    end
+  end
+  
+  context "connected" do
+    
+    let(:socket) {  stub() } 
+      
+    subject { 
+      s = BackgroundQueue::ClientLib::Connection.new(:client, server) 
+      TCPSocket.should_receive(:open).with(:host, :port) { socket }
+      s.__prv__connect
+      s
+    }
+  
+    context "sending data" do
+      
+      it "can send data successfully in one go" do
+        buf = "some data"
+        socket.should_receive(:write).with(buf) { buf.length }
+        subject.__prv__send_data(buf).should eq(true)
+      end
+      
+      it "can send data successfully in 3 goes" do
+        buf = "12345678"
+        socket.should_receive(:write).with("12345678") { 2 }
+        socket.should_receive(:write).with("345678") { 3 }
+        socket.should_receive(:write).with("678") { 3 }
+        subject.__prv__send_data(buf).should eq(true)
+      end
+      
+      it "fails when underlying network fails" do
+        buf = "12345678"
+        socket.should_receive(:write).with("12345678") { 2 }
+        socket.should_receive(:write).with("345678") { raise "Socket Disconnected" }
+        
+        expect { subject.__prv__send_data(buf)}.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Error Sending to host:port: Socket Disconnected")
+      end
+      
+      it "fails when command times out" do
+        init_subject = subject #need to init the subject before forcing timeout
+        Timeout.should_receive(:timeout).with(any_args) { raise Timeout::Error }  
+        expect { init_subject.__prv__send_data("data")}.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Timeout Sending to host:port")
+      end
+    end
+    
+    context "sending with header" do
+      
+      let(:data) { "data"}
+      let(:packed_data) { [1,4,data].pack("SLZ4") }
+      
+      it "can pack the header onto the data" do
+        BackgroundQueue::ClientLib::Connection.add_header(data).should eq(packed_data)
+      end
+      
+      it "can send data with header" do
+        subject.should_receive(:send_data).with(packed_data) { true}
+        subject.__prv__send_with_header(data).should eq(true)
+      end
+      
+    end
+    
+    context "receiving data" do
+      it "can receive data successfully in one go" do
+        socket.should_receive(:recvfrom).with(4) { ["data", nil] }
+        subject.__prv__receive_data(4).should eq("data")  
+      end
+      
+      it "can receive data successfully in 3 goes" do
+        socket.should_receive(:recvfrom).with(10) { ["0123", nil] }
+        socket.should_receive(:recvfrom).with(6) { ["45", nil] }
+        socket.should_receive(:recvfrom).with(4) { ["6789", nil] }
+        subject.__prv__receive_data(10).should eq("0123456789")  
+      end
+      
+      it "fails when underlying network fails" do
+        socket.should_receive(:recvfrom).with(10) { ["0123", nil] }
+        socket.should_receive(:recvfrom).with(6) { raise "Socket Disconnected" }
+        expect { subject.__prv__receive_data(10)}.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Error Receiving 10 bytes from host:port: Socket Disconnected")
+      end
+      
+      it "fails when response times out" do
+        init_subject = subject #need to init the subject before forcing timeout
+        Timeout.should_receive(:timeout).with(any_args) { raise Timeout::Error }  
+        expect { init_subject.__prv__receive_data(10)}.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Timeout Receiving 10 bytes from host:port")
+      end
+    end
+    
+    context "receiving with header" do
+      
+      let(:data) { "data"}
+      let(:packed_data) { [1,4,data].pack("SLZ4") }
+      
+
+      it "can receive data with header" do
+        subject.should_receive(:receive_data).with(6) { packed_data[0,6] }
+        subject.should_receive(:receive_data).with(4) { packed_data[6,4] }
+        
+        subject.__prv__receive_with_header().should eq(data)
+      end
+      
+    end
+  end
+  
+  
+  context "sending command" do
+    
+    subject { BackgroundQueue::ClientLib::Connection.new(:client, server) }
+    let(:send_command) { stub({:to_buf=>:data}) }
+
+    it "can successfully send command" do
+      
+      subject.stub(:check_connected) { true }
+      subject.should_receive(:send_with_header).with(:data) { true }
+      subject.should_receive(:receive_with_header).with(no_args) { :receive_response }
+      BackgroundQueue::Command.should_receive(:from_buf).with(:receive_response) { :command_response }
+      
+      subject.send_command(send_command).should eq(:command_response)
+    end
+    
+    it "fails if unable to connect" do
+      TCPSocket.should_receive(:open).with(:host, :port) { raise "Unable to connect" }
+      expect {  subject.send_command(send_command)  }.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Error Connecting to host:port: Unable to connect")
+    end
+    
+    it "fails if unable to send data" do
+      subject.stub(:check_connected) { true }
+      subject.should_receive(:send_with_header).with(:data) { raise BackgroundQueue::ClientLib::ConnectionError, "Send Error" }
+      expect {  subject.send_command(send_command)  }.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Send Error")
+
+    end
+    
+    it "fails if unable to recieve response" do
+      subject.stub(:check_connected) { true }
+      subject.should_receive(:send_with_header).with(:data) { true }
+      subject.should_receive(:receive_with_header).with(no_args) { raise BackgroundQueue::ClientLib::ConnectionError, "Receive Error" }
+      expect {  subject.send_command(send_command)  }.to raise_error(BackgroundQueue::ClientLib::ConnectionError, "Receive Error")
+    end
+  end
+
+end

data/spec/background_queue/client_spec.rb

@@ -0,0 +1,95 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'background_queue'
+
+describe "Client" do
+
+  context "initializing" do
+    it "loads from config path" do
+      BackgroundQueue::ClientLib::Config.stub(:load_file).with(:path_that_exists) { BackgroundQueue::ClientLib::Config.new(:a, :b, :c) }
+      bq_client = BackgroundQueue::Client.new(:path_that_exists)
+      bq_client.config.server.should eq(:a)
+    end
+  end
+  
+  context "initialized" do
+    subject { 
+      BackgroundQueue::ClientLib::Config.stub(:load_file).with(:path_that_exists) { 
+        BackgroundQueue::ClientLib::Config.new(:primary_server, [:failover_1, :failover_2], :memcache_server) 
+      }
+      BackgroundQueue::Client.new(:path_that_exists) 
+    }
+    
+    context "sending a command" do
+      it "can send a command to a server" do
+        BackgroundQueue::ClientLib::Connection.any_instance.should_receive(:send_command).with(:command) { true }
+        subject.__prv__send_command_to_server(:command, :primary_server).should eq([true, :primary_server])
+      end
+      
+      it "can succeed first time" do
+        subject.should_receive(:send_command_to_server).with(:command, :primary_server) { true }
+        subject.__prv__send_command(:command).should eq(true)
+      end
+      
+      it "can use failover server if main server fails" do
+        subject.should_receive(:send_command_to_server).with(:command, :primary_server) { raise BackgroundQueue::ClientLib::ConnectionError, "Primary" }
+        subject.should_receive(:send_command_to_server).with(:command, :failover_1) { true }
+        subject.__prv__send_command(:command).should eq(true)
+      end
+      
+      it "can use second failover server if needed" do
+        subject.should_receive(:send_command_to_server).with(:command, :primary_server) { raise BackgroundQueue::ClientLib::ConnectionError, "Primary" }
+        subject.should_receive(:send_command_to_server).with(:command, :failover_1) { raise BackgroundQueue::ClientLib::ConnectionError, "Failure 1" }
+        subject.should_receive(:send_command_to_server).with(:command, :failover_2) { true }
+        subject.__prv__send_command(:command).should eq(true)
+      end
+      
+      it "throws an exception if all connections fail" do
+        subject.should_receive(:send_command_to_server).with(:command, :primary_server) { raise BackgroundQueue::ClientLib::ConnectionError, "Primary" }
+        subject.should_receive(:send_command_to_server).with(:command, :failover_1) { raise BackgroundQueue::ClientLib::ConnectionError, "Failure 1" }
+        subject.should_receive(:send_command_to_server).with(:command, :failover_2) { raise BackgroundQueue::ClientLib::ConnectionError, "Failure 2" }
+        expect { subject.__prv__send_command(:command) }.to raise_error(BackgroundQueue::ClientException, "Primary, Attempt 2: Failure 1, Attempt 3: Failure 2")
+      end
+    end
+    
+    context "single call commands" do
+      before do
+        subject.should_receive(:send_command).with(any_args) { [true, :server] }
+      end
+      
+      context "#add_task" do
+        it "can build and send an add task command" do
+          job_handle = subject.add_task(:worker, :owner_id, :job_id, :task_id, 1, {}, {} )
+          job_handle.owner_id.should eq(:owner_id)
+          job_handle.job_id.should eq(:job_id)
+          job_handle.server.should eq(:server)
+        end
+        
+        it "will dynamically generate a job id/task id if not passed" do
+          subject.should_receive(:generate_ids).and_return([:jjj, :ttt])
+          job_handle = subject.add_task(:worker, :owner_id, nil, nil, 1, {}, {} )
+          job_handle.job_id.should eq(:jjj)
+        end
+      end
+      
+      it "can build and send an add tasks command" do
+        subject.add_tasks(:worker, :owner_id, :job_id, :tasks, 1, {}, {} ).should eq([true, :server])
+      end
+      
+      
+      #it "can build and send a remove task command" do
+      #  subject.remove_tasks(:tasks, {}).should eq(true)
+      #end
+      
+    end
+    
+    
+    context "#get_status" do
+      it "passes the requested server" do
+        job_handle = double("jh", :server=>:another_server, :job_id=>:job_id)
+        BackgroundQueue::ClientLib::Command.should_receive(:get_status_command).with(:job_id, {} ).and_return(:cmd)
+        subject.should_receive(:send_command).with(:cmd, :another_server) { [true, :another_server] }
+        subject.get_status(job_handle, {}).should eq(true)
+      end
+    end
+  end
+end