background_queue 0.3.0 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- data/TODO +13 -0
- data/VERSION +1 -1
- data/background_queue.gemspec +8 -2
- data/lib/background_queue/client.rb +31 -3
- data/lib/background_queue/client_lib/connection.rb +9 -0
- data/lib/background_queue/client_lib/job_handle.rb +34 -1
- data/lib/background_queue/config.rb +1 -11
- data/lib/background_queue/server_lib/balanced_queue.rb +22 -0
- data/lib/background_queue/server_lib/event_connection.rb +7 -1
- data/lib/background_queue/server_lib/job.rb +46 -7
- data/lib/background_queue/server_lib/priority_queue.rb +7 -0
- data/lib/background_queue/server_lib/queue_registry.rb +20 -5
- data/lib/background_queue/server_lib/task.rb +28 -0
- data/lib/background_queue/server_lib/task_registry.rb +7 -0
- data/lib/background_queue/server_lib/worker_client.rb +5 -2
- data/lib/background_queue/server_lib/worker_thread.rb +1 -1
- data/lib/background_queue/utils.rb +25 -1
- data/lib/background_queue/worker/base.rb +41 -0
- data/lib/background_queue/worker/calling.rb +24 -1
- data/lib/background_queue/worker/config.rb +20 -0
- data/lib/background_queue/worker/environment.rb +25 -1
- data/lib/background_queue/worker/logger.rb +114 -0
- data/lib/background_queue/worker/progress.rb +152 -0
- data/lib/background_queue/worker/worker_loader.rb +1 -1
- data/lib/background_queue_worker.rb +2 -0
- data/spec/background_queue/client_lib/connection_spec.rb +7 -1
- data/spec/background_queue/client_spec.rb +2 -1
- data/spec/background_queue/config_spec.rb +11 -23
- data/spec/background_queue/server_lib/integration/error_handling_spec.rb +85 -0
- data/spec/background_queue/server_lib/integration/full_test_spec.rb +76 -3
- data/spec/background_queue/server_lib/integration/queue_integration_spec.rb +6 -3
- data/spec/background_queue/server_lib/job_spec.rb +44 -3
- data/spec/background_queue/server_lib/worker_thread_spec.rb +2 -2
- data/spec/background_queue/utils_spec.rb +30 -0
- data/spec/background_queue/worker/calling_spec.rb +3 -1
- data/spec/background_queue/worker/environment_spec.rb +3 -1
- data/spec/background_queue/worker/logger_spec.rb +58 -0
- data/spec/background_queue/worker/progress_spec.rb +82 -0
- data/spec/background_queue/worker/worker_loader_spec.rb +1 -1
- data/spec/resources/summary_worker.rb +18 -0
- data/spec/shared/queue_registry_shared.rb +4 -3
- data/spec/support/simple_task.rb +24 -0
- metadata +33 -27
data/TODO
CHANGED
@@ -1,2 +1,15 @@
|
|
1
1
|
Server
|
2
2
|
Raise priority of tasks that have been waiting a long time
|
3
|
+
|
4
|
+
Add shortcut to init and finish tasks
|
5
|
+
Add ability to access "summary" data of the tasks in the finish task.
|
6
|
+
summary data operations:
|
7
|
+
append_summary_data(type, data)
|
8
|
+
set_summary_data(type, key, data)
|
9
|
+
increment_summary_data(type, amount)
|
10
|
+
decrement_sumary_data(type, amount)
|
11
|
+
reset_summary_data(type)
|
12
|
+
Add cache for tasks in a job to share? use memcache?
|
13
|
+
|
14
|
+
Progress Manager
|
15
|
+
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.6.0
|
data/background_queue.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "background_queue"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.6.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["MarkPent"]
|
12
|
-
s.date = "2012-
|
12
|
+
s.date = "2012-11-13"
|
13
13
|
s.description = "Organise background tasks so they will not overload the machine(s) running the tasks, while still giving a fair, balanced allocation of running time to members in the queue"
|
14
14
|
s.email = "mark.pent@gmail.com"
|
15
15
|
s.executables = ["bg_queue"]
|
@@ -63,6 +63,8 @@ Gem::Specification.new do |s|
|
|
63
63
|
"lib/background_queue/worker/calling.rb",
|
64
64
|
"lib/background_queue/worker/config.rb",
|
65
65
|
"lib/background_queue/worker/environment.rb",
|
66
|
+
"lib/background_queue/worker/logger.rb",
|
67
|
+
"lib/background_queue/worker/progress.rb",
|
66
68
|
"lib/background_queue/worker/worker_loader.rb",
|
67
69
|
"lib/background_queue_server.rb",
|
68
70
|
"lib/background_queue_worker.rb",
|
@@ -77,6 +79,7 @@ Gem::Specification.new do |s|
|
|
77
79
|
"spec/background_queue/server_lib/error_task_list_spec.rb",
|
78
80
|
"spec/background_queue/server_lib/event_connection_spec.rb",
|
79
81
|
"spec/background_queue/server_lib/event_server_spec.rb",
|
82
|
+
"spec/background_queue/server_lib/integration/error_handling_spec.rb",
|
80
83
|
"spec/background_queue/server_lib/integration/full_test_spec.rb",
|
81
84
|
"spec/background_queue/server_lib/integration/queue_integration_spec.rb",
|
82
85
|
"spec/background_queue/server_lib/integration/serialize_spec.rb",
|
@@ -96,12 +99,15 @@ Gem::Specification.new do |s|
|
|
96
99
|
"spec/background_queue/worker/base_spec.rb",
|
97
100
|
"spec/background_queue/worker/calling_spec.rb",
|
98
101
|
"spec/background_queue/worker/environment_spec.rb",
|
102
|
+
"spec/background_queue/worker/logger_spec.rb",
|
103
|
+
"spec/background_queue/worker/progress_spec.rb",
|
99
104
|
"spec/background_queue/worker/worker_loader_spec.rb",
|
100
105
|
"spec/resources/config-client.yml",
|
101
106
|
"spec/resources/config-serialize.yml",
|
102
107
|
"spec/resources/config.yml",
|
103
108
|
"spec/resources/example_worker.rb",
|
104
109
|
"spec/resources/example_worker_with_error.rb",
|
110
|
+
"spec/resources/summary_worker.rb",
|
105
111
|
"spec/resources/test_worker.rb",
|
106
112
|
"spec/shared/queue_registry_shared.rb",
|
107
113
|
"spec/spec_helper.rb",
|
@@ -10,7 +10,25 @@ module BackgroundQueue
|
|
10
10
|
@config = BackgroundQueue::ClientLib::Config.load_file(path)
|
11
11
|
end
|
12
12
|
|
13
|
-
#
|
13
|
+
#Add a task to the background
|
14
|
+
#
|
15
|
+
# @param [Symbol] worker name of the worker, ie :some_background_worker
|
16
|
+
# @param [Symbol, String, Number] owner_id something that identified the owner of the task. This will make sure resources are divided between owners equally.
|
17
|
+
# @param [Symbol, String, Number] job_id something to idetify the job. Tracking occurs per job.
|
18
|
+
# @param [Symbol, String, Number] task_id a globally unique id for the task. If the task_id exists elsewhere, it will be removed and added to the owner/job queue specified.
|
19
|
+
# @param [Integer] priority priority for 1 (highest) to 5 (lowest). Used to determine order of jobs.
|
20
|
+
# @param [Hash] task_parameters a hash of parameters passed to the task
|
21
|
+
# @param [Hash] options a hash of options that effect how the task is executed.
|
22
|
+
# @option options [String] :domain the domain to set in the host header when calling the worker.
|
23
|
+
# @option options [Boolean] :exclude if true, will not be included in (x/y) counter of progress caption
|
24
|
+
# @option options [Boolean] :synchronous if true, the task is synchronous, and no other tasks in the job will run until it is finished
|
25
|
+
# @option options [Number] :weight the weight of the task. Usually its weight is the same as other tasks in job
|
26
|
+
# @option options [String] :initial_progress_caption the progress caption to display until the job has started reporting progress
|
27
|
+
# @option options [Boolean] :send_summary if true, the task will receive the summary data
|
28
|
+
# @option options [Symbol] :step the step to run, `:start`, `:run` (Default) or `:finish`
|
29
|
+
#
|
30
|
+
# @return [BackgroundQueue::ClientLib::JobHandle] A handle to the job which can be used in get_status
|
31
|
+
|
14
32
|
def add_task(worker, owner_id, job_id, task_id, priority, task_parameters={}, options={}, server=nil )
|
15
33
|
job_id, task_id = generate_ids(worker, owner_id, job_id, task_id)
|
16
34
|
result, server = send_command(BackgroundQueue::ClientLib::Command.add_task_command(worker, owner_id, job_id, task_id, priority, task_parameters, options ), server)
|
@@ -18,9 +36,19 @@ module BackgroundQueue
|
|
18
36
|
BackgroundQueue::ClientLib::JobHandle.new(owner_id, job_id, server)
|
19
37
|
end
|
20
38
|
|
21
|
-
#
|
39
|
+
#Add multiple tasks to the background, all with the same worker/owner/job
|
40
|
+
#
|
41
|
+
# @param [Symbol] worker name of the worker, ie :some_background_worker
|
42
|
+
# @param [Symbol, String, Number] owner_id something that identified the owner of the task. This will make sure resources are divided between owners equally.
|
43
|
+
# @param [Symbol, String, Number] job_id something to idetify the job. Tracking occurs per job.
|
44
|
+
# @param [Array<Array<String, Hash, Hash>>] tasks an array of arrays in the format [task_id, optional task_params (Hash), optional task_options (Hash)]
|
45
|
+
# @param [Integer] priority priority for 1 (highest) to 5 (lowest). Used to determine order of jobs.
|
46
|
+
# @param [Hash] shared_parameters a hash of parameters passed to the tasks. This is merged with the task_params specified in the tasks param.
|
47
|
+
# @param [Hash] options a hash of options that effect how the tasks are executed. This is merged with the task_options specified in the tasks param. Refer to {#add_task} for options.
|
22
48
|
def add_tasks(worker, owner_id, job_id, tasks, priority, shared_parameters={}, options={}, server=nil )
|
23
|
-
send_command(BackgroundQueue::ClientLib::Command.add_tasks_command(worker, owner_id, job_id, tasks, priority, shared_parameters, options ), server)
|
49
|
+
result, server = send_command(BackgroundQueue::ClientLib::Command.add_tasks_command(worker, owner_id, job_id, tasks, priority, shared_parameters, options ), server)
|
50
|
+
#the server currently either returns :ok or an exception would have been thrown
|
51
|
+
BackgroundQueue::ClientLib::JobHandle.new(owner_id, job_id, server)
|
24
52
|
end
|
25
53
|
|
26
54
|
def get_status(job_handle, options={})
|
@@ -17,6 +17,13 @@ module BackgroundQueue::ClientLib
|
|
17
17
|
send_with_header(command.to_buf)
|
18
18
|
response = receive_with_header
|
19
19
|
BackgroundQueue::Command.from_buf(response)
|
20
|
+
ensure
|
21
|
+
begin
|
22
|
+
@socket.close unless @socket.nil?
|
23
|
+
rescue Exception=>e
|
24
|
+
#dont care...
|
25
|
+
end
|
26
|
+
@socket = nil
|
20
27
|
end
|
21
28
|
|
22
29
|
private
|
@@ -25,6 +32,8 @@ module BackgroundQueue::ClientLib
|
|
25
32
|
begin
|
26
33
|
Timeout::timeout(3) {
|
27
34
|
@socket = TCPSocket.open(@server.host, @server.port)
|
35
|
+
linger = [1,0].pack('ii')
|
36
|
+
@socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, linger) #force close so if called many times it wont clog the available ports
|
28
37
|
true
|
29
38
|
}
|
30
39
|
rescue Timeout::Error
|
@@ -1,6 +1,7 @@
|
|
1
|
+
require 'digest/md5'
|
1
2
|
module BackgroundQueue::ClientLib
|
2
3
|
#returned from add_task to describe what the job/server was added.
|
3
|
-
#this is
|
4
|
+
#this is because you can call add_task without a job_id, and not know what server was used.
|
4
5
|
#this is passed to get_status
|
5
6
|
class JobHandle
|
6
7
|
|
@@ -13,6 +14,38 @@ module BackgroundQueue::ClientLib
|
|
13
14
|
@job_id = job_id
|
14
15
|
@server = server
|
15
16
|
end
|
17
|
+
|
18
|
+
#register this job and return the registered key
|
19
|
+
def register(session_id)
|
20
|
+
md5 = Digest::MD5::new
|
21
|
+
now = Time::now
|
22
|
+
md5.update(now.to_s)
|
23
|
+
md5.update(String(now.usec))
|
24
|
+
md5.update(String(rand(0)))
|
25
|
+
md5.update(String($$))
|
26
|
+
md5.update('foobar')
|
27
|
+
md5.update(owner_id.to_s)
|
28
|
+
md5.update(job_id.to_s)
|
29
|
+
md5.update(server.to_s)
|
30
|
+
key = md5.hexdigest
|
31
|
+
|
32
|
+
Cache.put("#{session_id}_#{key}", self )
|
33
|
+
|
34
|
+
reverse_key = [owner_id, job_id, server].join("_")
|
35
|
+
Cache.put("#{session_id}_#{reverse_key}", key )
|
36
|
+
end
|
37
|
+
|
38
|
+
#look up a job from the key returned from register
|
39
|
+
def self.get_registered_job(session_id, key)
|
40
|
+
Cache.get("#{session_id}_#{key}")
|
41
|
+
end
|
42
|
+
|
43
|
+
#find the key for this job if its already registeed
|
44
|
+
def get_registration_key(session_id)
|
45
|
+
reverse_key = [owner_id, job_id, server].join("_")
|
46
|
+
Cache.get("#{session_id}_#{reverse_key}")
|
47
|
+
end
|
48
|
+
|
16
49
|
|
17
50
|
end
|
18
51
|
|
@@ -67,8 +67,6 @@ module BackgroundQueue
|
|
67
67
|
end
|
68
68
|
end
|
69
69
|
|
70
|
-
|
71
|
-
|
72
70
|
def convert_yaml_to_hash(string, path)
|
73
71
|
begin
|
74
72
|
result = YAML::load(string)
|
@@ -79,16 +77,8 @@ module BackgroundQueue
|
|
79
77
|
end
|
80
78
|
end
|
81
79
|
|
82
|
-
def current_environment
|
83
|
-
if ENV.has_key?('RAILS_ENV')
|
84
|
-
ENV['RAILS_ENV']
|
85
|
-
elsif defined? Rails
|
86
|
-
Rails.env
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
80
|
def extract_enviroment_entry(all_configs, path)
|
91
|
-
env_str = current_environment
|
81
|
+
env_str = BackgroundQueue::Utils.current_environment
|
92
82
|
if all_configs.has_key?(env_str)
|
93
83
|
all_configs[env_str]
|
94
84
|
elsif all_configs.has_key?(env_str.to_s.intern)
|
@@ -17,10 +17,19 @@ module BackgroundQueue::ServerLib
|
|
17
17
|
|
18
18
|
def add_task(task)
|
19
19
|
@thread_manager.protect_access {
|
20
|
+
if task.replaced_while_waiting_to_retry?
|
21
|
+
@server.logger.debug("Not adding task that was replaced while waiting to retry (#{task.id})")
|
22
|
+
return
|
23
|
+
end
|
20
24
|
status, existing_task = @task_registry.register(task)
|
21
25
|
if status != :waiting
|
22
26
|
if status == :existing
|
27
|
+
@server.logger.debug("Removing existing task (#{task.id})")
|
23
28
|
remove_item(existing_task)
|
29
|
+
elsif status == :waiting_to_retry
|
30
|
+
@server.logger.debug("Removing existing task that is waiting to retry (#{task.id})")
|
31
|
+
existing_task.set_error_status(:replaced_while_waiting_to_retry)
|
32
|
+
finish_item(existing_task)
|
24
33
|
end
|
25
34
|
add_item(task)
|
26
35
|
@thread_manager.signal_access #wake anything reading from the queue
|
@@ -36,6 +45,10 @@ module BackgroundQueue::ServerLib
|
|
36
45
|
|
37
46
|
def finish_task(task)
|
38
47
|
@thread_manager.protect_access {
|
48
|
+
if task.replaced_while_waiting_to_retry?
|
49
|
+
@server.logger.debug("Not finishing task that was replaced while waiting to retry (#{task.id})")
|
50
|
+
return
|
51
|
+
end
|
39
52
|
finish_item(task)
|
40
53
|
existing_task = @task_registry.de_register(task.id)
|
41
54
|
if existing_task
|
@@ -44,6 +57,15 @@ module BackgroundQueue::ServerLib
|
|
44
57
|
}
|
45
58
|
end
|
46
59
|
|
60
|
+
#need to synchronise this...
|
61
|
+
def add_task_to_error_list(task)
|
62
|
+
@thread_manager.protect_access {
|
63
|
+
task.running = false
|
64
|
+
task.set_error_status(:waiting_to_retry)
|
65
|
+
@server.error_tasks.add_task(task)
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
47
69
|
def next_task
|
48
70
|
task = nil
|
49
71
|
@thread_manager.control_access {
|
@@ -107,10 +107,16 @@ module BackgroundQueue::ServerLib
|
|
107
107
|
for task_data in command.args[:tasks]
|
108
108
|
if task_data[1].nil?
|
109
109
|
merged_params = shared_params
|
110
|
+
merged_options = command.options
|
110
111
|
else
|
111
112
|
merged_params = shared_params.clone.update(task_data[1])
|
113
|
+
if task_data[2].nil?
|
114
|
+
merged_options = command.options
|
115
|
+
else
|
116
|
+
merged_options = command.options.merge(task_data[2])
|
117
|
+
end
|
112
118
|
end
|
113
|
-
task = BackgroundQueue::ServerLib::Task.new(owner_id, job_id, task_data[0], priority, worker, merged_params,
|
119
|
+
task = BackgroundQueue::ServerLib::Task.new(owner_id, job_id, task_data[0], priority, worker, merged_params, merged_options)
|
114
120
|
server.task_queue.add_task(task)
|
115
121
|
end
|
116
122
|
@server.change_stat(:tasks, command.args[:tasks].length)
|
@@ -19,6 +19,8 @@ module BackgroundQueue::ServerLib
|
|
19
19
|
attr_reader :completed_weighted_tasks
|
20
20
|
attr_reader :running_percent_weighted
|
21
21
|
|
22
|
+
attr_reader :summary
|
23
|
+
|
22
24
|
#attr_reader :current_running_excluded_status
|
23
25
|
|
24
26
|
def initialize(id, owner)
|
@@ -69,7 +71,7 @@ module BackgroundQueue::ServerLib
|
|
69
71
|
@total_weighted_tasks += 1
|
70
72
|
@total_weighted_percent += task.weighted_percent
|
71
73
|
end
|
72
|
-
|
74
|
+
#@synchronous_count+=1 if task.synchronous? #the queue only goes into sync mode once the task is running/about to run
|
73
75
|
unless task.initial_progress_caption.nil? || task.initial_progress_caption.length == 0 || @current_progress[:percent] > 0
|
74
76
|
@current_progress[:caption] = task.initial_progress_caption
|
75
77
|
end
|
@@ -77,7 +79,10 @@ module BackgroundQueue::ServerLib
|
|
77
79
|
end
|
78
80
|
|
79
81
|
def next_item
|
80
|
-
pop
|
82
|
+
item = pop
|
83
|
+
@running_items += 1 if item
|
84
|
+
@synchronous_count+=1 if item && item.synchronous?
|
85
|
+
item
|
81
86
|
end
|
82
87
|
|
83
88
|
def remove_item(item)
|
@@ -85,16 +90,20 @@ module BackgroundQueue::ServerLib
|
|
85
90
|
end
|
86
91
|
|
87
92
|
def finish_item(item)
|
93
|
+
@running_items -= 1
|
88
94
|
@synchronous_count-=1 if item.synchronous?
|
89
95
|
end
|
90
96
|
|
91
97
|
def synchronous?
|
92
|
-
|
98
|
+
next_item = peek
|
99
|
+
@synchronous_count > 0 || (next_item && next_item.synchronous?)
|
93
100
|
end
|
94
101
|
|
95
102
|
def set_worker_status(status)
|
96
103
|
if status[:meta]
|
97
104
|
update_status_meta(status[:meta])
|
105
|
+
elsif status[:summary]
|
106
|
+
update_summary_meta(status)
|
98
107
|
else
|
99
108
|
running_status = get_running_status(status)
|
100
109
|
if status[:percent] >= 100
|
@@ -138,19 +147,49 @@ module BackgroundQueue::ServerLib
|
|
138
147
|
|
139
148
|
|
140
149
|
def update_status_meta(meta)
|
150
|
+
|
141
151
|
[:notice, :warning, :error].each { |key|
|
142
|
-
|
152
|
+
val = BackgroundQueue::Utils.get_hash_entry(meta, key)
|
153
|
+
unless val.nil?
|
143
154
|
@status_meta[key] = [] if @status_meta[key].nil?
|
144
|
-
@status_meta[key] <<
|
155
|
+
@status_meta[key] << val
|
145
156
|
end
|
146
157
|
}
|
147
|
-
|
158
|
+
val = BackgroundQueue::Utils.get_hash_entry(meta, :meta)
|
159
|
+
unless val.nil?
|
148
160
|
@status_meta[:meta] = {} if @status_meta[:meta].nil?
|
149
|
-
@status_meta[:meta] = @status_meta[:meta].update(
|
161
|
+
@status_meta[:meta] = @status_meta[:meta].update(val)
|
150
162
|
end
|
151
163
|
update_current_progress
|
152
164
|
end
|
153
165
|
|
166
|
+
def update_summary_meta(status)
|
167
|
+
@summary ||= {}
|
168
|
+
type = status[:type].intern
|
169
|
+
case status[:summary]
|
170
|
+
when "app"
|
171
|
+
@summary[type] ||= []
|
172
|
+
@summary[type] << status[:data]
|
173
|
+
when "set"
|
174
|
+
@summary[type] ||= {}
|
175
|
+
@summary[type][status[:key]] = status[:data]
|
176
|
+
when "inc"
|
177
|
+
@summary[type] ||= 0
|
178
|
+
@summary[type] += status[:data].to_i
|
179
|
+
when "dec"
|
180
|
+
@summary[type] ||= 0
|
181
|
+
@summary[type] -= status[:data].to_i
|
182
|
+
when "res"
|
183
|
+
if type == :all
|
184
|
+
@summary = {}
|
185
|
+
else
|
186
|
+
@summary.delete(type)
|
187
|
+
end
|
188
|
+
else
|
189
|
+
logger.error("Unknown summary action: #{status[:summary]}")
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
154
193
|
def update_finished_status(status)
|
155
194
|
rstatus = deregister_running_status(status[:task_id])
|
156
195
|
unless rstatus.nil?
|
@@ -8,6 +8,7 @@ module BackgroundQueue::ServerLib
|
|
8
8
|
@items = {}
|
9
9
|
@stalled_items = {}
|
10
10
|
@stalled = false
|
11
|
+
@running_items = 0
|
11
12
|
end
|
12
13
|
|
13
14
|
def pop
|
@@ -53,6 +54,10 @@ module BackgroundQueue::ServerLib
|
|
53
54
|
@queues.empty?
|
54
55
|
end
|
55
56
|
|
57
|
+
def has_running_items?
|
58
|
+
@running_items > 0
|
59
|
+
end
|
60
|
+
|
56
61
|
def number_of_priorities
|
57
62
|
@queues.length
|
58
63
|
end
|
@@ -87,6 +92,7 @@ module BackgroundQueue::ServerLib
|
|
87
92
|
return insert_queue_at_index(priority, idx)
|
88
93
|
end
|
89
94
|
end
|
95
|
+
return nil unless create
|
90
96
|
return insert_queue_at_index(priority, -1)
|
91
97
|
end
|
92
98
|
|
@@ -116,6 +122,7 @@ module BackgroundQueue::ServerLib
|
|
116
122
|
attr_accessor :priority
|
117
123
|
def initialize(priority)
|
118
124
|
@priority = priority
|
125
|
+
raise "Invalid priority" if @priority.nil?
|
119
126
|
super(0)
|
120
127
|
end
|
121
128
|
|
@@ -9,6 +9,8 @@ module BackgroundQueue::ServerLib
|
|
9
9
|
remove(queue, original_priority)
|
10
10
|
end
|
11
11
|
push(queue)
|
12
|
+
elsif queue.stalled? && !(queue.synchronous? && queue.has_running_items?) #it stalled because it was empty...
|
13
|
+
resume_queue(queue)
|
12
14
|
end
|
13
15
|
end
|
14
16
|
|
@@ -26,11 +28,15 @@ module BackgroundQueue::ServerLib
|
|
26
28
|
else
|
27
29
|
push(queue)
|
28
30
|
end
|
31
|
+
@running_items += 1
|
32
|
+
item.running = true unless item.nil?
|
29
33
|
end
|
34
|
+
#server.logger.debug("next item #{item.nil? ? 'nil' : item.id}")
|
30
35
|
item
|
31
36
|
end
|
32
37
|
|
33
38
|
def remove_item(item)
|
39
|
+
item.running = false
|
34
40
|
in_queue, queue = get_queue(get_queue_id_from_item(item), false)
|
35
41
|
raise "Unable to remove task #{item.id} at priority #{item.priority} (no queue at that priority)" if queue.nil?
|
36
42
|
priority_decreased, original_priority, item = remove_item_from_queue(queue, item)
|
@@ -48,7 +54,8 @@ module BackgroundQueue::ServerLib
|
|
48
54
|
in_queue, queue = get_queue(get_queue_id_from_item(item), false)
|
49
55
|
raise "Queue #{get_queue_id_from_item(item)} unavailble when finishing item" if queue.nil?
|
50
56
|
queue.finish_item(item)
|
51
|
-
|
57
|
+
@running_items -= 1
|
58
|
+
resume_queue(queue) unless queue.synchronous? && queue.has_running_items?
|
52
59
|
end
|
53
60
|
|
54
61
|
private
|
@@ -94,21 +101,29 @@ module BackgroundQueue::ServerLib
|
|
94
101
|
|
95
102
|
def stall_queue(queue)
|
96
103
|
queue.stalled = true
|
104
|
+
server.logger.debug("stalling queue #{queue.id} (empty=#{queue.empty?})")
|
97
105
|
#puts "stalling queue #{queue.inspect}"
|
98
106
|
@stalled_items[queue.id] = queue
|
99
107
|
end
|
100
108
|
|
101
109
|
def resume_queue(queue)
|
102
110
|
if queue.stalled?
|
103
|
-
|
104
|
-
|
111
|
+
|
112
|
+
if queue.empty? && !queue.has_running_items?
|
113
|
+
@stalled_items.delete(queue.id)
|
114
|
+
@items.delete(queue.id)
|
115
|
+
server.logger.debug("removed empty queue #{queue.id}")
|
116
|
+
#puts "q empty"
|
117
|
+
elsif !queue.empty?
|
118
|
+
@stalled_items.delete(queue.id)
|
105
119
|
queue.stalled = false
|
106
120
|
push(queue)
|
107
121
|
@items[queue.id] = queue
|
122
|
+
server.logger.debug("resumed queue #{queue.id}")
|
108
123
|
#puts "returned q: #{queue.inspect}"
|
109
124
|
else
|
110
|
-
|
111
|
-
#
|
125
|
+
server.logger.debug("keeping empty queue stalled #{queue.id}")
|
126
|
+
#keep stalled
|
112
127
|
end
|
113
128
|
#else
|
114
129
|
# puts "q not stalled"
|