background_queue 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -0
- data/.rspec +1 -0
- data/.rvmrc +48 -0
- data/Gemfile +19 -0
- data/LICENSE.txt +20 -0
- data/README.md +69 -0
- data/Rakefile +42 -0
- data/TODO +2 -0
- data/VERSION +1 -0
- data/background_queue.gemspec +158 -0
- data/bin/bg_queue +26 -0
- data/lib/background_queue.rb +8 -0
- data/lib/background_queue/client.rb +96 -0
- data/lib/background_queue/client_lib/command.rb +36 -0
- data/lib/background_queue/client_lib/config.rb +109 -0
- data/lib/background_queue/client_lib/connection.rb +105 -0
- data/lib/background_queue/client_lib/job_handle.rb +19 -0
- data/lib/background_queue/command.rb +49 -0
- data/lib/background_queue/config.rb +118 -0
- data/lib/background_queue/server_lib/balanced_queue.rb +108 -0
- data/lib/background_queue/server_lib/config.rb +339 -0
- data/lib/background_queue/server_lib/event_connection.rb +133 -0
- data/lib/background_queue/server_lib/event_server.rb +35 -0
- data/lib/background_queue/server_lib/job.rb +252 -0
- data/lib/background_queue/server_lib/job_registry.rb +30 -0
- data/lib/background_queue/server_lib/lru.rb +193 -0
- data/lib/background_queue/server_lib/owner.rb +54 -0
- data/lib/background_queue/server_lib/priority_queue.rb +156 -0
- data/lib/background_queue/server_lib/queue_registry.rb +123 -0
- data/lib/background_queue/server_lib/server.rb +314 -0
- data/lib/background_queue/server_lib/sorted_workers.rb +52 -0
- data/lib/background_queue/server_lib/task.rb +79 -0
- data/lib/background_queue/server_lib/task_registry.rb +51 -0
- data/lib/background_queue/server_lib/thread_manager.rb +121 -0
- data/lib/background_queue/server_lib/worker.rb +18 -0
- data/lib/background_queue/server_lib/worker_balancer.rb +97 -0
- data/lib/background_queue/server_lib/worker_client.rb +94 -0
- data/lib/background_queue/server_lib/worker_thread.rb +70 -0
- data/lib/background_queue/utils.rb +40 -0
- data/lib/background_queue/worker/base.rb +46 -0
- data/lib/background_queue/worker/calling.rb +59 -0
- data/lib/background_queue/worker/config.rb +35 -0
- data/lib/background_queue/worker/environment.rb +70 -0
- data/lib/background_queue/worker/worker_loader.rb +94 -0
- data/lib/background_queue_server.rb +21 -0
- data/lib/background_queue_worker.rb +5 -0
- data/spec/background_queue/client_lib/command_spec.rb +75 -0
- data/spec/background_queue/client_lib/config_spec.rb +156 -0
- data/spec/background_queue/client_lib/connection_spec.rb +170 -0
- data/spec/background_queue/client_spec.rb +95 -0
- data/spec/background_queue/command_spec.rb +34 -0
- data/spec/background_queue/config_spec.rb +134 -0
- data/spec/background_queue/server_lib/balanced_queue_spec.rb +122 -0
- data/spec/background_queue/server_lib/config_spec.rb +443 -0
- data/spec/background_queue/server_lib/event_connection_spec.rb +190 -0
- data/spec/background_queue/server_lib/event_server_spec.rb +48 -0
- data/spec/background_queue/server_lib/integration/full_test_spec.rb +247 -0
- data/spec/background_queue/server_lib/integration/queue_integration_spec.rb +98 -0
- data/spec/background_queue/server_lib/integration/serialize_spec.rb +127 -0
- data/spec/background_queue/server_lib/job_registry_spec.rb +65 -0
- data/spec/background_queue/server_lib/job_spec.rb +525 -0
- data/spec/background_queue/server_lib/owner_spec.rb +33 -0
- data/spec/background_queue/server_lib/priority_queue_spec.rb +182 -0
- data/spec/background_queue/server_lib/server_spec.rb +353 -0
- data/spec/background_queue/server_lib/sorted_workers_spec.rb +122 -0
- data/spec/background_queue/server_lib/task_registry_spec.rb +69 -0
- data/spec/background_queue/server_lib/task_spec.rb +20 -0
- data/spec/background_queue/server_lib/thread_manager_spec.rb +106 -0
- data/spec/background_queue/server_lib/worker_balancer_spec.rb +127 -0
- data/spec/background_queue/server_lib/worker_client_spec.rb +196 -0
- data/spec/background_queue/server_lib/worker_thread_spec.rb +125 -0
- data/spec/background_queue/utils_spec.rb +27 -0
- data/spec/background_queue/worker/base_spec.rb +35 -0
- data/spec/background_queue/worker/calling_spec.rb +103 -0
- data/spec/background_queue/worker/environment_spec.rb +67 -0
- data/spec/background_queue/worker/worker_loader_spec.rb +103 -0
- data/spec/background_queue_spec.rb +7 -0
- data/spec/resources/config-client.yml +7 -0
- data/spec/resources/config-serialize.yml +12 -0
- data/spec/resources/config.yml +12 -0
- data/spec/resources/example_worker.rb +4 -0
- data/spec/resources/example_worker_with_error.rb +4 -0
- data/spec/resources/test_worker.rb +8 -0
- data/spec/shared/queue_registry_shared.rb +216 -0
- data/spec/spec_helper.rb +15 -0
- data/spec/support/default_task.rb +9 -0
- data/spec/support/private.rb +23 -0
- data/spec/support/simple_server.rb +28 -0
- data/spec/support/simple_task.rb +58 -0
- data/spec/support/test_worker_server.rb +205 -0
- metadata +254 -0
@@ -0,0 +1,34 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'background_queue'
|
3
|
+
|
4
|
+
describe "Command" do
|
5
|
+
|
6
|
+
|
7
|
+
context "serialization" do
|
8
|
+
it "can reload from serialized format" do
|
9
|
+
cmd = BackgroundQueue::ClientLib::Command.add_task_command(:worker, :owner_id, :job_id, :task_id, 1, {:a=>:b}, {:c=>:d} )
|
10
|
+
serialized = cmd.to_buf
|
11
|
+
|
12
|
+
cmd = BackgroundQueue::Command.from_buf(serialized)
|
13
|
+
|
14
|
+
cmd.code.should eq(:add_task)
|
15
|
+
cmd.options[:c].should eq('d')
|
16
|
+
cmd.args[:worker].should eq('worker')
|
17
|
+
cmd.args[:owner_id].should eq('owner_id')
|
18
|
+
cmd.args[:job_id].should eq('job_id')
|
19
|
+
cmd.args[:task_id].should eq('task_id')
|
20
|
+
cmd.args[:params]['a'].should eq('b')
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
it "fails when loading from invalid json" do
|
25
|
+
expect { BackgroundQueue::Command.from_buf("{sdf, sdfsdf}")}.to raise_exception(BackgroundQueue::InvalidCommand)
|
26
|
+
end
|
27
|
+
|
28
|
+
it "fails when loading from missing data in json" do
|
29
|
+
expect { BackgroundQueue::Command.from_buf('{"a": "b"}')}.to raise_exception(BackgroundQueue::InvalidCommand, "Error loading command from buffer: Missing 'c' (code)")
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
@@ -0,0 +1,134 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
require 'background_queue'
|
3
|
+
|
4
|
+
unless defined? Rails
|
5
|
+
module Rails; end
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "Config" do
|
9
|
+
|
10
|
+
context "loading" do
|
11
|
+
context "from file" do
|
12
|
+
it "gets an io from a file that exists" do
|
13
|
+
File.should_receive(:open).with(:path_that_exists) { :string }
|
14
|
+
File.should_receive(:exist?).with(:path_that_exists) { true }
|
15
|
+
BackgroundQueue::Config.__prv__get_string_from_file(:path_that_exists).should eq(:string)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "calls load using string if file exists" do
|
19
|
+
File.should_receive(:open).with(:path_that_exists) { :string }
|
20
|
+
File.should_receive(:exist?).with(:path_that_exists) { true }
|
21
|
+
BackgroundQueue::Config.should_receive(:load_string).with(:string, :path_that_exists) { true }
|
22
|
+
BackgroundQueue::Config.load_file(:path_that_exists).should eq(true)
|
23
|
+
end
|
24
|
+
|
25
|
+
it "errors if the file is not found" do
|
26
|
+
File.should_receive(:exist?).with(:path_that_does_not_exist) { false }
|
27
|
+
File.should_receive(:expand_path).with(:path_that_does_not_exist) { :expanded_path }
|
28
|
+
expect { BackgroundQueue::Config.__prv__get_string_from_file(:path_that_does_not_exist) }.to raise_error(BackgroundQueue::LoadError, "Failed to open background_queue configuration file at 'expanded_path'")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
context "using string" do
|
33
|
+
it "executes ERB on the string" do
|
34
|
+
BackgroundQueue::Config.__prv__evaluate_erb("TEST <%=1+1%>", :path_that_exists).should eq("TEST 2")
|
35
|
+
end
|
36
|
+
|
37
|
+
it "errors if invalid ERB" do
|
38
|
+
File.should_receive(:expand_path).with(:path_that_exists) { :expanded_path }
|
39
|
+
expect { BackgroundQueue::Config.__prv__evaluate_erb("TEST <%= aa %>", :path_that_exists)}.to raise_error(BackgroundQueue::LoadError, /Error executing ERB for background_queue configuration file at 'expanded_path':/)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "calls load yaml if erb evaluates" do
|
43
|
+
BackgroundQueue::Config.should_receive(:evaluate_erb).with(:string, :path_that_exists) { :loaded_string }
|
44
|
+
BackgroundQueue::Config.should_receive(:load_yaml).with(:loaded_string, :path_that_exists) { true }
|
45
|
+
BackgroundQueue::Config.load_string(:string, :path_that_exists).should eq(true)
|
46
|
+
end
|
47
|
+
|
48
|
+
context "loading as YAML" do
|
49
|
+
it "gets a hash object if the string is valid YAML" do
|
50
|
+
File.stub(:expand_path) { :expanded_path }
|
51
|
+
BackgroundQueue::Config.__prv__convert_yaml_to_hash("a: b", :path_that_exists).should eq( {'a' => 'b'})
|
52
|
+
end
|
53
|
+
|
54
|
+
it "errors if the YAML is not a hash" do
|
55
|
+
File.stub(:expand_path) { :expanded_path }
|
56
|
+
expect { BackgroundQueue::Config.__prv__convert_yaml_to_hash("a", :path_that_exists)}.to raise_error(BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at 'expanded_path': Root of config should be a hash of environment configurations")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
context "extracting the environment entry" do
|
61
|
+
it "gets_the_current environment from env" do
|
62
|
+
ENV.should_receive(:has_key?).with("RAILS_ENV") { true }
|
63
|
+
ENV.should_receive(:[]).with("RAILS_ENV") { :the_env }
|
64
|
+
BackgroundQueue::Config.__prv__current_environment.should eq(:the_env)
|
65
|
+
end
|
66
|
+
|
67
|
+
it "gets_the_current environment from Rails" do
|
68
|
+
ENV.should_receive(:has_key?).with("RAILS_ENV") { false }
|
69
|
+
Rails.should_receive(:env) { :the_env }
|
70
|
+
BackgroundQueue::Config.__prv__current_environment.should eq(:the_env)
|
71
|
+
end
|
72
|
+
|
73
|
+
context "with development environment" do
|
74
|
+
before do
|
75
|
+
BackgroundQueue::Config.stub(:current_environment) { 'development' }
|
76
|
+
end
|
77
|
+
|
78
|
+
it "extracts the correct environment entry from the hash" do
|
79
|
+
BackgroundQueue::Config.__prv__extract_enviroment_entry({:development=>:test}, :path_that_exists).should eq(:test)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "errors if the YAML does not define environment entry" do
|
83
|
+
File.stub(:expand_path) { :expanded_path }
|
84
|
+
expect { BackgroundQueue::Config.__prv__extract_enviroment_entry({:test=>:test}, :path_that_exists).should eq(:test)}.to raise_error(BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at 'expanded_path': missing enviroment root entry: development")
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
context "with extracted environment entry" do
|
90
|
+
before do
|
91
|
+
BackgroundQueue::Config.stub(:current_environment) { 'development' }
|
92
|
+
end
|
93
|
+
it "calls load using map" do
|
94
|
+
BackgroundQueue::Config.should_receive(:load_hash).with({'a'=>'b'}, :path_that_exists) { true }
|
95
|
+
BackgroundQueue::Config.load_yaml("development: { a: b }", :path_that_exists).should eq(true)
|
96
|
+
end
|
97
|
+
|
98
|
+
context "loading memcache server" do
|
99
|
+
it "loads server from comma separated list" do
|
100
|
+
entries = BackgroundQueue::ClientLib::Config.__prv__build_memcache_array({ :memcache=> "127.0.0.1:4000 , 127.0.0.1:4001,127.0.0.1:4002"}, :path_that_exists)
|
101
|
+
entries.length.should eq(3)
|
102
|
+
entries[0].should eq('127.0.0.1:4000')
|
103
|
+
entries[1].should eq('127.0.0.1:4001')
|
104
|
+
entries[2].should eq('127.0.0.1:4002')
|
105
|
+
end
|
106
|
+
|
107
|
+
it "errors if missing memcache entry" do
|
108
|
+
File.stub(:expand_path) { :expanded_path }
|
109
|
+
expect {
|
110
|
+
entries = BackgroundQueue::ClientLib::Config.__prv__build_memcache_array({}, :path_that_exists)
|
111
|
+
}.to raise_error(
|
112
|
+
BackgroundQueue::LoadError,
|
113
|
+
"Missing 'memcache' entry in configuration file expanded_path"
|
114
|
+
)
|
115
|
+
end
|
116
|
+
|
117
|
+
it "errors if memcache entry not String" do
|
118
|
+
File.stub(:expand_path) { :expanded_path }
|
119
|
+
expect {
|
120
|
+
entries = BackgroundQueue::ClientLib::Config.__prv__build_memcache_array({:memcache=>1}, :path_that_exists)
|
121
|
+
}.to raise_error(
|
122
|
+
BackgroundQueue::LoadError,
|
123
|
+
"Error loading 'memcache' entry in configuration file expanded_path: invalid data type (Fixnum), expecting String (comma separated)"
|
124
|
+
)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
|
131
|
+
end
|
132
|
+
|
133
|
+
|
134
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
|
2
|
+
require 'background_queue_server'
|
3
|
+
|
4
|
+
|
5
|
+
describe BackgroundQueue::ServerLib::BalancedQueue do
|
6
|
+
|
7
|
+
it_behaves_like "a queue registry" do
|
8
|
+
let(:server) {
|
9
|
+
SimpleServer.new( :thread_manager=>:thread_manager)
|
10
|
+
}
|
11
|
+
let(:new_instance) { BackgroundQueue::ServerLib::BalancedQueue.new(server) }
|
12
|
+
end
|
13
|
+
|
14
|
+
let(:thread_manager) {
|
15
|
+
tm = double("thread_manager")
|
16
|
+
tm.stub(:protect_access).and_yield
|
17
|
+
tm.stub(:control_access).and_yield
|
18
|
+
tm
|
19
|
+
}
|
20
|
+
let(:server) {
|
21
|
+
SimpleServer.new( :thread_manager=>thread_manager)
|
22
|
+
}
|
23
|
+
subject { BackgroundQueue::ServerLib::BalancedQueue.new(server) }
|
24
|
+
|
25
|
+
context "callbacks" do
|
26
|
+
it "gets the owner_id from tasks" do
|
27
|
+
subject.__prv__get_queue_id_from_item(SimpleTask.new(:owner_id, :job_id, :task_id, 3)).should eq(:owner_id)
|
28
|
+
end
|
29
|
+
|
30
|
+
it "calls add_item to add items to owner" do
|
31
|
+
BackgroundQueue::ServerLib::Owner.any_instance.should_receive(:add_item).with(:item)
|
32
|
+
subject.__prv__add_item_to_queue(BackgroundQueue::ServerLib::Owner.new(1, :parent), :item)
|
33
|
+
end
|
34
|
+
|
35
|
+
it "specifies the owner class as its queue class" do
|
36
|
+
subject.class.queue_class.should eq(BackgroundQueue::ServerLib::Owner)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context "adding tasks" do
|
41
|
+
context "with no existing task running" do
|
42
|
+
before do
|
43
|
+
thread_manager.should_receive(:signal_access)
|
44
|
+
subject.should_receive(:add_item).with(anything) { :does_not_matter }
|
45
|
+
end
|
46
|
+
|
47
|
+
it "adds a new task" do
|
48
|
+
BackgroundQueue::ServerLib::TaskRegistry.any_instance.should_receive(:register).with(anything).and_return([:new, nil])
|
49
|
+
subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3))
|
50
|
+
end
|
51
|
+
|
52
|
+
it "adds a task with existing id thats not running" do
|
53
|
+
BackgroundQueue::ServerLib::TaskRegistry.any_instance.should_receive(:register).with(anything).and_return([:existing, :existing_task])
|
54
|
+
subject.should_receive(:remove_item).with(:existing_task)
|
55
|
+
subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3))
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
it "adds a task with existing id thats running" do
|
60
|
+
BackgroundQueue::ServerLib::TaskRegistry.any_instance.should_receive(:register).with(anything).and_return([:waiting, nil])
|
61
|
+
subject.should_not_receive(:add_item).with(anything)
|
62
|
+
subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3))
|
63
|
+
end
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
context "removing tasks" do
|
68
|
+
it "delegates call to queue_registry" do
|
69
|
+
subject.should_receive(:remove_item).with(anything) { :does_not_matter }
|
70
|
+
subject.remove_task(:task_id)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context "getting tasks" do
|
75
|
+
|
76
|
+
it "will wait on condition if queue is empty" do
|
77
|
+
thread_manager.should_receive(:wait_on_access)
|
78
|
+
bg = BackgroundQueue::ServerLib::BalancedQueue.new(server)
|
79
|
+
bg.next_task.should eq(nil)
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
context "#save_to_file" do
|
85
|
+
it "will write the tasks as json to file" do
|
86
|
+
subject.stub(:register_job)
|
87
|
+
thread_manager.stub(:signal_access)
|
88
|
+
|
89
|
+
t1 = BackgroundQueue::ServerLib::Task.new(:owner_id, :job_id, :task_id, 1, :worker, {:a=>'b'}, {:c=>'d'})
|
90
|
+
t2 = BackgroundQueue::ServerLib::Task.new(:owner2_id, :job2_id, :task2_id, 2, :worker, {}, {})
|
91
|
+
t3 = BackgroundQueue::ServerLib::Task.new(:owner3_id, :job3_id, :task3_id, 3, :worker, {}, {})
|
92
|
+
subject.add_task(t1)
|
93
|
+
subject.add_task(t2)
|
94
|
+
subject.add_task(t3)
|
95
|
+
sio = StringIO.new
|
96
|
+
subject.save_to_file(sio)
|
97
|
+
|
98
|
+
sio.string.should eq([t1.to_json_object(true),t2.to_json_object(true),t3.to_json_object(true)].to_json)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
context "#load_from_file" do
|
103
|
+
it "will load the tasks as json from file" do
|
104
|
+
subject.stub(:register_job)
|
105
|
+
thread_manager.stub(:signal_access)
|
106
|
+
|
107
|
+
t1 = BackgroundQueue::ServerLib::Task.new(:owner_id, :job_id, :task_id, 1, :worker, {:a=>'b'}, {:c=>'d'})
|
108
|
+
t2 = BackgroundQueue::ServerLib::Task.new(:owner2_id, :job2_id, :task2_id, 2, :worker, {}, {})
|
109
|
+
t3 = BackgroundQueue::ServerLib::Task.new(:owner3_id, :job3_id, :task3_id, 3, :worker, {}, {})
|
110
|
+
|
111
|
+
sio = StringIO.new([t1.to_json_object(true),t2.to_json_object(true),t3.to_json_object(true)].to_json)
|
112
|
+
subject.load_from_file(sio)
|
113
|
+
|
114
|
+
sio = StringIO.new
|
115
|
+
subject.save_to_file(sio)
|
116
|
+
|
117
|
+
sio.string.should eq([t1.to_json_object(true),t2.to_json_object(true),t3.to_json_object(true)].to_json)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
|
122
|
+
end
|
@@ -0,0 +1,443 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
|
2
|
+
require 'background_queue_server'
|
3
|
+
|
4
|
+
unless defined? Rails
|
5
|
+
module Rails; end
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "Server Config" do
|
9
|
+
|
10
|
+
context "using map" do
|
11
|
+
context "loading worker entry" do
|
12
|
+
it "creates a worker entry if all required string fields exist" do
|
13
|
+
entry = BackgroundQueue::ServerLib::Config::Worker.new('http://127.0.0.1:801/background_queue')
|
14
|
+
entry.url.should eq('http://127.0.0.1:801/background_queue')
|
15
|
+
entry.uri.port.should eq(801)
|
16
|
+
end
|
17
|
+
|
18
|
+
it "errors if invalid type" do
|
19
|
+
expect {
|
20
|
+
BackgroundQueue::ServerLib::Config::Worker.new([])
|
21
|
+
}.to raise_error(
|
22
|
+
BackgroundQueue::LoadError,
|
23
|
+
"Invalid data type (Array), expecting String (as a url)"
|
24
|
+
)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "errors if missing or nil" do
|
28
|
+
expect {
|
29
|
+
BackgroundQueue::ServerLib::Config::Worker.new(nil)
|
30
|
+
}.to raise_error(
|
31
|
+
BackgroundQueue::LoadError,
|
32
|
+
"Missing worker url"
|
33
|
+
)
|
34
|
+
end
|
35
|
+
|
36
|
+
it "errors if invalid url" do
|
37
|
+
expect {
|
38
|
+
BackgroundQueue::ServerLib::Config::Worker.new("something:/\\asdf dsafdsfdsaf")
|
39
|
+
}.to raise_error(
|
40
|
+
BackgroundQueue::LoadError,
|
41
|
+
"Invalid worker url (something:/\\asdf dsafdsfdsaf)"
|
42
|
+
)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
context "loading worker entries" do
|
47
|
+
it "creates an array of workers" do
|
48
|
+
entries = BackgroundQueue::ServerLib::Config.__prv__build_worker_entries({:workers=>['http://127.0.0.1:801/background_queue', 'http://127.0.0.1:802/background_queue', 'http://127.0.0.1:803/background_queue']}, :path_that_exists)
|
49
|
+
entries.length.should eq(3)
|
50
|
+
entries.first.uri.port.should eq(801)
|
51
|
+
entries.last.uri.port.should eq(803)
|
52
|
+
end
|
53
|
+
|
54
|
+
it "errors if failover entry is invalid" do
|
55
|
+
File.stub(:expand_path) { :expanded_path }
|
56
|
+
expect {
|
57
|
+
BackgroundQueue::ServerLib::Config.__prv__build_worker_entries({ 'workers'=> ['http://127.0.0.1:801/background_queue','']}, :path_that_exists)
|
58
|
+
}.to raise_error(
|
59
|
+
BackgroundQueue::LoadError,
|
60
|
+
"Error loading 'worker' entry (2) from background queue server configuration file expanded_path: Missing worker url"
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
context "loading secret entry" do
|
66
|
+
it "creates an array of workers" do
|
67
|
+
secret = BackgroundQueue::ServerLib::Config.__prv__get_secret_entry({:secret=>'1234567890123456789012345678901234567890'}, :path_that_exists)
|
68
|
+
secret.should eq('1234567890123456789012345678901234567890')
|
69
|
+
end
|
70
|
+
|
71
|
+
it "errors if secret is missing" do
|
72
|
+
File.stub(:expand_path) { :expanded_path }
|
73
|
+
expect {
|
74
|
+
BackgroundQueue::ServerLib::Config.__prv__get_secret_entry({}, :path_that_exists)
|
75
|
+
}.to raise_error(
|
76
|
+
BackgroundQueue::LoadError,
|
77
|
+
"Missing 'secret' entry in background queue server configuration file expanded_path"
|
78
|
+
)
|
79
|
+
end
|
80
|
+
|
81
|
+
it "errors if secret is too short" do
|
82
|
+
File.stub(:expand_path) { :expanded_path }
|
83
|
+
expect {
|
84
|
+
BackgroundQueue::ServerLib::Config.__prv__get_secret_entry({:secret=>'1234567890123456789'}, :path_that_exists)
|
85
|
+
}.to raise_error(
|
86
|
+
BackgroundQueue::LoadError,
|
87
|
+
"Error loading 'secret' entry in background queue server configuration file expanded_path: length too short (must be at least 20 characters long)"
|
88
|
+
)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
context "loading listen entry" do
|
93
|
+
it "defaults to 0.0.0.0:#{BackgroundQueue::Config::DEFAULT_PORT} if missing" do
|
94
|
+
entry = BackgroundQueue::ServerLib::Config::Address.new(nil)
|
95
|
+
entry.host.should eq("0.0.0.0")
|
96
|
+
entry.port.should eq(BackgroundQueue::Config::DEFAULT_PORT)
|
97
|
+
end
|
98
|
+
|
99
|
+
it "defaults to 0.0.0.0 if host missing" do
|
100
|
+
entry = BackgroundQueue::ServerLib::Config::Address.new({:port=>3001})
|
101
|
+
entry.host.should eq("0.0.0.0")
|
102
|
+
entry.port.should eq(3001)
|
103
|
+
end
|
104
|
+
|
105
|
+
it "defaults to port #{BackgroundQueue::Config::DEFAULT_PORT}" do
|
106
|
+
entry = BackgroundQueue::ServerLib::Config::Address.new({:host=>"127.0.0.1"})
|
107
|
+
entry.host.should eq("127.0.0.1")
|
108
|
+
entry.port.should eq(BackgroundQueue::Config::DEFAULT_PORT)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "errors if the host is invalid ap address" do
|
112
|
+
expect { BackgroundQueue::ServerLib::Config::Address.new({:host=>"x.y.z"}) }.to raise_exception("Invalid host: x.y.z")
|
113
|
+
end
|
114
|
+
|
115
|
+
it "errors if the port is an invalid number" do
|
116
|
+
expect { BackgroundQueue::ServerLib::Config::Address.new({:port=>"dsfg"}) }.to raise_exception("Invalid port: dsfg")
|
117
|
+
end
|
118
|
+
|
119
|
+
it "wraps the configuration file path in errors" do
|
120
|
+
File.stub(:expand_path) { :expanded_path }
|
121
|
+
expect {
|
122
|
+
BackgroundQueue::ServerLib::Config.__prv__get_address_entry({:address=>{:host=>"x.y.z"}}, :path_that_exists)
|
123
|
+
}.to raise_error(
|
124
|
+
BackgroundQueue::LoadError,
|
125
|
+
"Error loading 'address' entry in background queue server configuration file expanded_path: Invalid host: x.y.z"
|
126
|
+
)
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
context "#get_connections_per_worker_entry" do
|
132
|
+
it "gets the entry" do
|
133
|
+
entry = BackgroundQueue::ServerLib::Config.__prv__get_connections_per_worker_entry({:connections_per_worker=>10}, :path_that_exists)
|
134
|
+
entry.should eq(10)
|
135
|
+
end
|
136
|
+
|
137
|
+
it "errors if entry is missing" do
|
138
|
+
File.stub(:expand_path) { :expanded_path }
|
139
|
+
expect {
|
140
|
+
BackgroundQueue::ServerLib::Config.__prv__get_connections_per_worker_entry({}, :path_that_exists)
|
141
|
+
}.to raise_error(
|
142
|
+
BackgroundQueue::LoadError,
|
143
|
+
"Missing 'connections_per_worker' entry in background queue server configuration file expanded_path"
|
144
|
+
)
|
145
|
+
end
|
146
|
+
|
147
|
+
it "errors if entry is not an Integer" do
|
148
|
+
File.stub(:expand_path) { :expanded_path }
|
149
|
+
expect {
|
150
|
+
BackgroundQueue::ServerLib::Config.__prv__get_connections_per_worker_entry({:connections_per_worker=>"abc"}, :path_that_exists)
|
151
|
+
}.to raise_error(
|
152
|
+
BackgroundQueue::LoadError,
|
153
|
+
"Error loading 'connections_per_worker' entry in background queue server configuration file expanded_path: invalid data type (String), expecting Integer"
|
154
|
+
)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
context "#get_system_task_options_entry" do
|
159
|
+
it "gets a hash of options" do
|
160
|
+
opts = BackgroundQueue::ServerLib::Config.__prv__get_system_task_options_entry({:system_task_options=>{:a=>:b}}, :path_that_exists)
|
161
|
+
opts.should eq({:a=>:b})
|
162
|
+
end
|
163
|
+
|
164
|
+
it "allows no options" do
|
165
|
+
opts = BackgroundQueue::ServerLib::Config.__prv__get_system_task_options_entry({}, :path_that_exists)
|
166
|
+
opts.should eq({})
|
167
|
+
end
|
168
|
+
|
169
|
+
it "errors if the jobs entry is not a hash" do
|
170
|
+
File.stub(:expand_path) { :expanded_path }
|
171
|
+
expect {
|
172
|
+
BackgroundQueue::ServerLib::Config.__prv__get_system_task_options_entry({:system_task_options=>"abc"}, :path_that_exists)
|
173
|
+
}.to raise_error(
|
174
|
+
BackgroundQueue::LoadError,
|
175
|
+
"Error loading 'system_task_options' entry in background queue server configuration file expanded_path: invalid data type (String), expecting Hash (of options)"
|
176
|
+
)
|
177
|
+
end
|
178
|
+
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
context "#get_jobs_entry" do
|
183
|
+
it "gets an array of entries" do
|
184
|
+
BackgroundQueue::ServerLib::Config::Job.should_receive(:new).twice.and_return(:job)
|
185
|
+
jobs = BackgroundQueue::ServerLib::Config.__prv__get_jobs_entry({:jobs=>[:one, :two]}, :path_that_exists)
|
186
|
+
jobs.should eq([:job, :job])
|
187
|
+
end
|
188
|
+
|
189
|
+
it "allows no jobs" do
|
190
|
+
jobs = BackgroundQueue::ServerLib::Config.__prv__get_jobs_entry({}, :path_that_exists)
|
191
|
+
jobs.should eq([])
|
192
|
+
end
|
193
|
+
|
194
|
+
it "errors if the jobs entry is not an array" do
|
195
|
+
File.stub(:expand_path) { :expanded_path }
|
196
|
+
expect {
|
197
|
+
BackgroundQueue::ServerLib::Config.__prv__get_jobs_entry({:jobs=>"abc"}, :path_that_exists)
|
198
|
+
}.to raise_error(
|
199
|
+
BackgroundQueue::LoadError,
|
200
|
+
"Error loading 'jobs' entry in background queue server configuration file expanded_path: invalid data type (String), expecting Array (of jobs)"
|
201
|
+
)
|
202
|
+
end
|
203
|
+
|
204
|
+
it "passes job loading errors on" do
|
205
|
+
File.stub(:expand_path) { :expanded_path }
|
206
|
+
BackgroundQueue::ServerLib::Config::Job.should_receive(:new).and_raise("blah")
|
207
|
+
expect {
|
208
|
+
BackgroundQueue::ServerLib::Config.__prv__get_jobs_entry({:jobs=>[:a]}, :path_that_exists)
|
209
|
+
}.to raise_error(
|
210
|
+
BackgroundQueue::LoadError,
|
211
|
+
"Error loading 'jobs' entry in background queue server configuration file expanded_path: blah"
|
212
|
+
)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
context "#Job.initialize" do
|
217
|
+
it "will load at" do
|
218
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:at=>"something", :worker=>:abc)
|
219
|
+
job.type.should eq(:at)
|
220
|
+
job.at.should eq("something")
|
221
|
+
end
|
222
|
+
|
223
|
+
it "will load in" do
|
224
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:in=>"something", :worker=>:abc)
|
225
|
+
job.type.should eq(:in)
|
226
|
+
job.in.should eq("something")
|
227
|
+
end
|
228
|
+
|
229
|
+
it "will load cron" do
|
230
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:cron=>"something", :worker=>:abc)
|
231
|
+
job.type.should eq(:cron)
|
232
|
+
job.cron.should eq("something")
|
233
|
+
end
|
234
|
+
|
235
|
+
it "will load the arguments" do
|
236
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:cron=>"something", :worker=>:abc, :args=>{:a=>:b})
|
237
|
+
job.args.should eq({:a=>:b})
|
238
|
+
end
|
239
|
+
|
240
|
+
it "will error if timer not designated" do
|
241
|
+
expect {
|
242
|
+
BackgroundQueue::ServerLib::Config::Job.new(:worker=>:abc)
|
243
|
+
}.to raise_error(
|
244
|
+
"Job is missing timer designation (at, in or cron)"
|
245
|
+
)
|
246
|
+
end
|
247
|
+
|
248
|
+
it "will error if worker not designated" do
|
249
|
+
expect {
|
250
|
+
BackgroundQueue::ServerLib::Config::Job.new(:at=>"abc")
|
251
|
+
}.to raise_error(
|
252
|
+
"Job is missing worker entry"
|
253
|
+
)
|
254
|
+
end
|
255
|
+
|
256
|
+
it "will error if the arguments are not a map" do
|
257
|
+
expect {
|
258
|
+
BackgroundQueue::ServerLib::Config::Job.new(:at=>"abc", :worker=>:abc, :args=>[])
|
259
|
+
}.to raise_error(
|
260
|
+
"Invalid 'args' entry in job: expecting Hash of arguments, got Array"
|
261
|
+
)
|
262
|
+
end
|
263
|
+
|
264
|
+
end
|
265
|
+
|
266
|
+
context "#Job.schedule" do
|
267
|
+
subject {
|
268
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:at=>"something", :worker=>:abc)
|
269
|
+
job.type = nil
|
270
|
+
job.at = nil
|
271
|
+
job
|
272
|
+
}
|
273
|
+
before do
|
274
|
+
subject.should_receive(:run).with(:server)
|
275
|
+
end
|
276
|
+
|
277
|
+
let(:scheduler) { double("scheduler") }
|
278
|
+
|
279
|
+
it "schedules at a time" do
|
280
|
+
subject.type = :at
|
281
|
+
subject.at = :time
|
282
|
+
scheduler.should_receive(:at).with(:time).and_yield
|
283
|
+
subject.schedule(scheduler, :server)
|
284
|
+
end
|
285
|
+
|
286
|
+
it "schedules in a time" do
|
287
|
+
subject.type = :in
|
288
|
+
subject.in = :time
|
289
|
+
scheduler.should_receive(:in).with(:time).and_yield
|
290
|
+
subject.schedule(scheduler, :server)
|
291
|
+
end
|
292
|
+
|
293
|
+
it "schedules by a cron time" do
|
294
|
+
subject.type = :cron
|
295
|
+
subject.cron = :time
|
296
|
+
scheduler.should_receive(:cron).with(:time).and_yield
|
297
|
+
subject.schedule(scheduler, :server)
|
298
|
+
end
|
299
|
+
|
300
|
+
it "schedules every time" do
|
301
|
+
subject.type = :every
|
302
|
+
subject.every = :time
|
303
|
+
scheduler.should_receive(:every).with(:time).and_yield
|
304
|
+
subject.schedule(scheduler, :server)
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
context "#Job.run" do
|
309
|
+
it "adds a task to the queue" do
|
310
|
+
job = BackgroundQueue::ServerLib::Config::Job.new(:at=>"something", :worker=>:abc, :args=>{:a=>:b})
|
311
|
+
server = SimpleServer.new(:config=>double("config", :system_task_options=>{}), :task_queue=>double("task_queue"))
|
312
|
+
server.task_queue.should_receive(:add_task)
|
313
|
+
job.run(server)
|
314
|
+
end
|
315
|
+
|
316
|
+
end
|
317
|
+
|
318
|
+
context "#get_task_file_entry" do
|
319
|
+
before do
|
320
|
+
File.stub(:expand_path) { :expanded_path }
|
321
|
+
end
|
322
|
+
|
323
|
+
it "creates a task_file entry if the file exists" do
|
324
|
+
File.should_receive(:exist?).with('path').and_return(true)
|
325
|
+
entry = BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>'path'}, :path)
|
326
|
+
entry.should eq('path')
|
327
|
+
end
|
328
|
+
|
329
|
+
it "creates a task_file entry if the directory is writable" do
|
330
|
+
File.should_receive(:exist?).with('path').and_return(false)
|
331
|
+
File.should_receive(:dirname).with('path').and_return('dir')
|
332
|
+
File.should_receive(:exist?).with('dir').and_return(true)
|
333
|
+
FileUtils.should_receive(:touch).with('path')
|
334
|
+
FileUtils.should_receive(:rm).with('path')
|
335
|
+
entry = BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>'path'}, :path)
|
336
|
+
entry.should eq('path')
|
337
|
+
end
|
338
|
+
|
339
|
+
it "creates a task_file entry if the directory can be created" do
|
340
|
+
File.should_receive(:exist?).with('path').and_return(false)
|
341
|
+
File.should_receive(:dirname).with('path').and_return('dir')
|
342
|
+
File.should_receive(:exist?).with('dir').and_return(false)
|
343
|
+
FileUtils.should_receive(:mkdir_p).with('dir')
|
344
|
+
entry = BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>'path'}, :path)
|
345
|
+
entry.should eq('path')
|
346
|
+
end
|
347
|
+
|
348
|
+
it "errors if the directory can not be created" do
|
349
|
+
File.should_receive(:exist?).with('path').and_return(false)
|
350
|
+
File.should_receive(:dirname).with('path').and_return('dir')
|
351
|
+
File.should_receive(:exist?).with('dir').and_return(false)
|
352
|
+
FileUtils.should_receive(:mkdir_p).with('dir').and_raise("Permission Denied")
|
353
|
+
expect {
|
354
|
+
BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>'path'}, :path)
|
355
|
+
}.to raise_error(
|
356
|
+
BackgroundQueue::LoadError,
|
357
|
+
"Error loading 'task_file' entry in background queue server configuration file expanded_path: unable to create directory dir (Permission Denied)"
|
358
|
+
)
|
359
|
+
end
|
360
|
+
|
361
|
+
it "errors if the directory can not be written to" do
|
362
|
+
File.should_receive(:exist?).with('path').and_return(false)
|
363
|
+
File.should_receive(:dirname).with('path').and_return('dir')
|
364
|
+
File.should_receive(:exist?).with('dir').and_return(true)
|
365
|
+
FileUtils.should_receive(:touch).with('path').and_raise("Permission Denied")
|
366
|
+
expect {
|
367
|
+
BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>'path'}, :path)
|
368
|
+
}.to raise_error(
|
369
|
+
BackgroundQueue::LoadError,
|
370
|
+
"Error loading 'task_file' entry in background queue server configuration file expanded_path: unable to write to file path (Permission Denied)"
|
371
|
+
)
|
372
|
+
end
|
373
|
+
|
374
|
+
it "errors if invalid type" do
|
375
|
+
expect {
|
376
|
+
BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>123}, :path)
|
377
|
+
}.to raise_error(
|
378
|
+
BackgroundQueue::LoadError,
|
379
|
+
"Error loading 'task_file' entry in background queue server configuration file expanded_path: Invalid data type (Fixnum), expecting String"
|
380
|
+
)
|
381
|
+
end
|
382
|
+
|
383
|
+
it "allows if missing or nil" do
|
384
|
+
|
385
|
+
BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({:task_file=>nil}, :path).should eq(nil)
|
386
|
+
BackgroundQueue::ServerLib::Config.__prv__get_task_file_entry({}, :path).should eq(nil)
|
387
|
+
|
388
|
+
end
|
389
|
+
|
390
|
+
end
|
391
|
+
|
392
|
+
|
393
|
+
context "creating Config entry" do
|
394
|
+
before do
|
395
|
+
File.stub(:expand_path) { :expanded_path }
|
396
|
+
end
|
397
|
+
|
398
|
+
it "should create Config instance" do
|
399
|
+
config = BackgroundQueue::ServerLib::Config.load_hash({
|
400
|
+
:workers=>['http://127.0.0.1:801/background_queue', 'http://127.0.0.1:802/background_queue'],
|
401
|
+
:secret=>'1234567890123456789012345678901234567890',
|
402
|
+
:connections_per_worker=>10,
|
403
|
+
:jobs=>[{:at=>"some time", :worker=>:something}]
|
404
|
+
}, :path_that_exists)
|
405
|
+
config.workers.length.should eq(2)
|
406
|
+
config.workers.first.uri.port.should eq(801)
|
407
|
+
config.workers.last.uri.port.should eq(802)
|
408
|
+
config.jobs.length.should eq(1)
|
409
|
+
config.jobs.first.at.should eq('some time')
|
410
|
+
config.secret.should eq('1234567890123456789012345678901234567890')
|
411
|
+
end
|
412
|
+
|
413
|
+
it "should fail when missing workers" do
|
414
|
+
expect {
|
415
|
+
config = BackgroundQueue::ServerLib::Config.load_hash({ }, :path_that_exists)
|
416
|
+
}.to raise_error(
|
417
|
+
BackgroundQueue::LoadError,
|
418
|
+
"Missing 'workers' in background queue server configuration file expanded_path"
|
419
|
+
)
|
420
|
+
end
|
421
|
+
|
422
|
+
it "should fail when missing secret" do
|
423
|
+
expect {
|
424
|
+
config = BackgroundQueue::ServerLib::Config.load_hash({ :workers=>['http://127.0.0.1:802/background_queue'] }, :path_that_exists)
|
425
|
+
}.to raise_error(
|
426
|
+
BackgroundQueue::LoadError,
|
427
|
+
"Missing 'secret' entry in background queue server configuration file expanded_path"
|
428
|
+
)
|
429
|
+
end
|
430
|
+
|
431
|
+
it "should fail when missing connections_per_worker" do
|
432
|
+
expect {
|
433
|
+
config = BackgroundQueue::ServerLib::Config.load_hash({ :workers=>['http://127.0.0.1:802/background_queue'], :secret=>'1234567890123456789012345678901234567890' }, :path_that_exists)
|
434
|
+
}.to raise_error(
|
435
|
+
BackgroundQueue::LoadError,
|
436
|
+
"Missing 'connections_per_worker' entry in background queue server configuration file expanded_path"
|
437
|
+
)
|
438
|
+
end
|
439
|
+
|
440
|
+
end
|
441
|
+
end
|
442
|
+
|
443
|
+
end
|