resque-concurrent-restriction 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,8 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
5
+ .idea
6
+ spec/dump.rdb
7
+ spec/redis-server.log
8
+ spec/redis.pid
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in resque-concurrent-restriction.gemspec
4
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2010 Matt Conway (matt@conwaysplace.com)
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,39 @@
1
+ resque-concurrent-restriction
2
+ ===============
3
+
4
+ Resque Concurrent Restriction is a plugin for the [Resque][0] queueing system (http://github.com/defunkt/resque). It allows one to specify how many of the given job can run concurrently.
5
+
6
+ Resque Concurrent Restriction requires Resque 1.10 and redis 2.2
7
+
8
+ Install
9
+ -------
10
+
11
+ sudo gem install resque-concurrent-restriction
12
+
13
+ To use
14
+ ------
15
+
16
+ It is especially useful when a system has intensive jobs for which you should only run a few at a time. What you should do for the IntensiveJob is to make it extend Resque::Plugins::ConcurrentRestriction and specify the concurrent limit (defaults to 1). For example:
17
+
18
+ class IntensiveJob
19
+ extend Resque::Plugins::ConcurrentRestriction
20
+ concurrent 4
21
+
22
+ #rest of your class here
23
+ end
24
+
25
+ That means the IntensiveJob can not have more than 4 jobs running simultaneously
26
+
27
+ Author
28
+ ------
29
+ Code was originally forkd from the [resque-restriction][1] plugin (Richard Huang :: flyerhzm@gmail.com :: @flyerhzm), but diverged enough that it warranted being its own plugin to keep the code simple.
30
+
31
+ Matt Conway :: matt@conwaysplace.com :: @mattconway
32
+
33
+ Copyright
34
+ ---------
35
+ Copyright (c) 2011 Matt Conway. See LICENSE for details.
36
+
37
+ [0]: http://github.com/defunkt/resque
38
+ [1]: http://github.com/flyerhzm/resque-restriction
39
+
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
@@ -0,0 +1,16 @@
1
+ require 'resque'
2
+ require 'resque/plugins/concurrent_restriction/version'
3
+ require 'resque/plugins/concurrent_restriction/resque_worker_extension'
4
+ require 'resque/plugins/concurrent_restriction/concurrent_restriction_job'
5
+
6
+ Resque::Worker.send(:include, Resque::Plugins::ConcurrentRestriction::Worker)
7
+ Resque::Job.send(:extend, Resque::Plugins::ConcurrentRestriction::Job)
8
+
9
+ unsupported_version = false
10
+ begin
11
+ server_ver = Resque.redis.info["redis_version"].split('.').collect{|x| x.to_i}
12
+ unsupported_version = (server_ver <=> [2, 2, 0]) < 0
13
+ rescue
14
+ end
15
+
16
+ raise "resque-concurrent-restriction requires a redis-server version >= 2.2.0" if unsupported_version
@@ -0,0 +1,382 @@
1
+ module Resque
2
+ module Plugins
3
+ module ConcurrentRestriction
4
+
5
+ # Used by the user in their job class to set the concurrency limit
6
+ def concurrent(limit)
7
+ @concurrent = limit
8
+ end
9
+
10
+ # Allows the user to specify the unique key that identifies a set
11
+ # of jobs that share a concurrency limit. Defaults to the job class name
12
+ def concurrent_identifier(*args)
13
+ end
14
+
15
+ # Used to query what the limit the user has set
16
+ def concurrent_limit
17
+ @concurrent ||= 1
18
+ end
19
+
20
+ # The key used to acquire a lock so we can operate on multiple
21
+ # redis structures (runnables set, running_count) atomically
22
+ def lock_key(tracking_key)
23
+ parts = tracking_key.split(":")
24
+ "concurrent:lock:#{parts[2..-1].join(':')}"
25
+ end
26
+
27
+ # The redis key used to store the number of currently running
28
+ # jobs for the restriction_identifier
29
+ def running_count_key(tracking_key)
30
+ parts = tracking_key.split(":")
31
+ "concurrent:count:#{parts[2..-1].join(':')}"
32
+ end
33
+
34
+ # The key for the redis list where restricted jobs for the given resque queue are stored
35
+ def restriction_queue_key(tracking_key, queue)
36
+ parts = tracking_key.split(":")
37
+ "concurrent:queue:#{queue}:#{parts[2..-1].join(':')}"
38
+ end
39
+
40
+ def restriction_queue_availability_key(tracking_key)
41
+ parts = tracking_key.split(":")
42
+ "concurrent:queue_availability:#{parts[2..-1].join(':')}"
43
+ end
44
+
45
+ # The key that groups all jobs of the same restriction_identifier together
46
+ # so that we can work on any of those jobs if they are runnable
47
+ # Stored in runnables set, and used to build keys for each queue where jobs
48
+ # for those queues are stored
49
+ def tracking_key(*args)
50
+ id = concurrent_identifier(*args)
51
+ id = ":#{id}" if id && id.strip.size > 0
52
+ "concurrent:tracking:#{self.to_s}#{id}"
53
+ end
54
+
55
+ def tracking_class(tracking_key)
56
+ Resque.constantize(tracking_key.split(":")[2])
57
+ end
58
+
59
+ # The key to the redis set where we keep a list of runnable tracking_keys
60
+ def runnables_key(queue=nil)
61
+ key = ":#{queue}" if queue
62
+ "concurrent:runnable#{key}"
63
+ end
64
+
65
+ # Encodes the job intot he restriction queue
66
+ def encode(job)
67
+ item = {:queue => job.queue, :payload => job.payload}
68
+ Resque.encode(item)
69
+ end
70
+
71
+ # Decodes the job from the restriction queue
72
+ def decode(str)
73
+ item = Resque.decode(str)
74
+ Resque::Job.new(item['queue'], item['payload']) if item
75
+ end
76
+
77
+ # The restriction queues that have data for each tracking key
78
+ # Adds/Removes the queue to the list of queues for that tracking key
79
+ # so we can quickly tell in next_runnable_job if a runnable job exists on a
80
+ # specific restriction queue
81
+ def update_queues_available(tracking_key, queue, action)
82
+ availability_key = restriction_queue_availability_key(tracking_key)
83
+ case action
84
+ when :add then Resque.redis.send(:sadd, availability_key, queue)
85
+ when :remove then Resque.redis.send(:srem, availability_key, queue)
86
+ else raise "Invalid action to ConcurrentRestriction.track_queue"
87
+ end
88
+ end
89
+
90
+ def queues_available(tracking_key)
91
+ availability_key = restriction_queue_availability_key(tracking_key)
92
+ Resque.redis.smembers(availability_key)
93
+ end
94
+
95
+ # Pushes the job to the restriction queue
96
+ def push_to_restriction_queue(job, location=:back)
97
+ tracking_key = tracking_key(*job.args)
98
+
99
+ case location
100
+ when :back then Resque.redis.rpush(restriction_queue_key(tracking_key, job.queue), encode(job))
101
+ when :front then Resque.redis.lpush(restriction_queue_key(tracking_key, job.queue), encode(job))
102
+ else raise "Invalid location to ConcurrentRestriction.push_to_restriction_queue"
103
+ end
104
+
105
+ update_queues_available(tracking_key, job.queue, :add)
106
+ mark_runnable(tracking_key, false)
107
+ end
108
+
109
+ # Pops a job from the restriction queue
110
+ def pop_from_restriction_queue(tracking_key, queue)
111
+ queue_key = restriction_queue_key(tracking_key, queue)
112
+ str = Resque.redis.lpop(queue_key)
113
+ post_pop_size = Resque.redis.llen(queue_key)
114
+
115
+ if post_pop_size == 0
116
+ update_queues_available(tracking_key, queue, :remove)
117
+ clear_runnable(tracking_key, queue)
118
+ end
119
+
120
+ # increment by one to indicate that we are running
121
+ # do this before update_queues_available so that the current queue gets cleaned
122
+ increment_running_count(tracking_key) if str
123
+
124
+ decode(str)
125
+ end
126
+
127
+ # Grabs the raw data (undecoded) from the restriction queue
128
+ def restriction_queue_raw(tracking_key, queue)
129
+ Array(Resque.redis.lrange(restriction_queue_key(tracking_key, queue), 0, -1))
130
+ end
131
+
132
+ # Grabs the contents of the restriction queue (decoded)
133
+ def restriction_queue(tracking_key, queue)
134
+ restriction_queue_raw(tracking_key, queue).collect {|s| decode(s) }
135
+ end
136
+
137
+ # Returns the number of jobs currently running
138
+ def running_count(tracking_key)
139
+ Resque.redis.get(running_count_key(tracking_key)).to_i
140
+ end
141
+
142
+ # Returns the number of jobs currently running
143
+ def set_running_count(tracking_key, value)
144
+ count_key = running_count_key(tracking_key)
145
+ Resque.redis.set(count_key, value)
146
+ restricted = (value > concurrent_limit)
147
+ mark_runnable(tracking_key, !restricted)
148
+ return restricted
149
+ end
150
+
151
+ def restricted?(tracking_key)
152
+ count_key = running_count_key(tracking_key)
153
+ value = Resque.redis.get(count_key).to_i
154
+ restricted = (value >= concurrent_limit)
155
+ return restricted
156
+ end
157
+
158
+ def increment_running_count(tracking_key)
159
+ count_key = running_count_key(tracking_key)
160
+ value = Resque.redis.incr(count_key)
161
+ restricted = (value > concurrent_limit)
162
+ mark_runnable(tracking_key, !restricted)
163
+ return restricted
164
+ end
165
+
166
+ def decrement_running_count(tracking_key)
167
+ count_key = running_count_key(tracking_key)
168
+ value = Resque.redis.decr(count_key)
169
+ Resque.redis.set(count_key, 0) if value < 0
170
+ restricted = (value >= concurrent_limit)
171
+ mark_runnable(tracking_key, !restricted)
172
+ return restricted
173
+ end
174
+
175
+ def runnable?(tracking_key, queue)
176
+ Resque.redis.sismember(runnables_key(queue), tracking_key)
177
+ end
178
+
179
+ def get_next_runnable(queue)
180
+ Resque.redis.srandmember(runnables_key(queue))
181
+ end
182
+
183
+ # Returns the list of tracking_keys that have jobs waiting to run (are not over the concurrency limit)
184
+ def runnables(queue=nil)
185
+ Resque.redis.smembers(runnables_key(queue))
186
+ end
187
+
188
+ # Keeps track of which jobs are currently runnable, that is the
189
+ # tracking_key should have jobs on some restriction queue and
190
+ # also have less than concurrency_limit jobs running
191
+ #
192
+ def mark_runnable(tracking_key, runnable)
193
+ queues = queues_available(tracking_key)
194
+ queues.each do |queue|
195
+ runnable_queues_key = runnables_key(queue)
196
+ if runnable
197
+ Resque.redis.sadd(runnable_queues_key, tracking_key)
198
+ else
199
+ Resque.redis.srem(runnable_queues_key, tracking_key)
200
+ end
201
+ end
202
+ if runnable
203
+ Resque.redis.sadd(runnables_key, tracking_key) if queues.size > 0
204
+ else
205
+ Resque.redis.srem(runnables_key, tracking_key)
206
+ end
207
+ end
208
+
209
+ def clear_runnable(tracking_key, queue)
210
+ Resque.redis.srem(runnables_key(queue), tracking_key)
211
+ Resque.redis.srem(runnables_key, tracking_key)
212
+ end
213
+
214
+ # Acquires a lock using the given key and lock expiration time
215
+ def acquire_lock(lock_key, lock_expiration)
216
+ # acquire the lock to work on the restriction queue
217
+ expiration_time = lock_expiration + 1
218
+ acquired_lock = Resque.redis.setnx(lock_key, expiration_time)
219
+
220
+ # If we didn't acquire the lock, check the expiration as described
221
+ # at http://redis.io/commands/setnx
222
+ if ! acquired_lock
223
+ # If expiration time is in the future, then someone else beat us to getting the lock
224
+ old_expiration_time = Resque.redis.get(lock_key)
225
+ return false if old_expiration_time.to_i > Time.now.to_i
226
+
227
+ # if expiration time was in the future when we set it, then someone beat us to it
228
+ old_expiration_time = Resque.redis.getset(lock_key, expiration_time)
229
+ return false if old_expiration_time.to_i > Time.now.to_i
230
+ end
231
+
232
+ # expire the lock eventually so we clean up keys - not needed to timeout
233
+ # lock, just to keep redis clean for locks that aren't being used'
234
+ Resque.redis.expireat(lock_key, expiration_time + 300)
235
+
236
+ return true
237
+ end
238
+
239
+ # Releases the lock acquired by #acquire_lock
240
+ def release_lock(lock_key, lock_expiration)
241
+ # Only delete the lock if the one we created hasn't expired
242
+ expiration_time = lock_expiration + 1
243
+ Resque.redis.del(lock_key) if expiration_time > Time.now.to_i
244
+ end
245
+
246
+
247
+ # Uses a lock to ensure that a sequence of redis operations happen atomically
248
+ # We don't use watch/multi/exec as it doesn't work in a DistributedRedis setup
249
+ def run_atomically(lock_key)
250
+ trying = true
251
+ exp_backoff = 1
252
+
253
+ while trying do
254
+ lock_expiration = Time.now.to_i + 10
255
+ if acquire_lock(lock_key, lock_expiration)
256
+ begin
257
+ yield
258
+ ensure
259
+ release_lock(lock_key, lock_expiration)
260
+ end
261
+ trying = false
262
+ else
263
+ sleep (rand(1000) * 0.0001 * exp_backoff)
264
+ exp_backoff *= 2
265
+ end
266
+ end
267
+
268
+ end
269
+
270
+ # Pushes the job to restriction queue if it is restricted
271
+ # If the job is within the concurrency limit, thus needs to be run, we
272
+ # keep the running count incremented so that other calls don't erroneously
273
+ # see a lower value and run their job. This count gets decremented by call
274
+ # to release_restriction when job completes
275
+ def stash_if_restricted(job)
276
+ restricted = false
277
+ tracking_key = tracking_key(*job.args)
278
+ lock_key = lock_key(tracking_key)
279
+
280
+ run_atomically(lock_key) do
281
+
282
+ restricted = restricted?(tracking_key)
283
+ if restricted
284
+ push_to_restriction_queue(job)
285
+ else
286
+ increment_running_count(tracking_key)
287
+ end
288
+
289
+ end
290
+
291
+ return restricted
292
+ end
293
+
294
+ # Returns the next job that is runnable
295
+ def next_runnable_job(queue)
296
+ tracking_key = get_next_runnable(queue)
297
+ return nil unless tracking_key
298
+
299
+ job = nil
300
+ lock_key = lock_key(tracking_key)
301
+
302
+ run_atomically(lock_key) do
303
+
304
+ # since we don't have a lock when we get the runnable,
305
+ # we need to check it again
306
+ still_runnable = runnable?(tracking_key, queue)
307
+ if still_runnable
308
+ klazz = tracking_class(tracking_key)
309
+ job = klazz.pop_from_restriction_queue(tracking_key, queue)
310
+ end
311
+
312
+ end
313
+
314
+ return job
315
+
316
+ end
317
+
318
+ # Decrements the running_count - to be called at end of job
319
+ def release_restriction(job)
320
+ tracking_key = tracking_key(*job.args)
321
+ lock_key = lock_key(tracking_key)
322
+
323
+ run_atomically(lock_key) do
324
+
325
+ # decrement the count after a job has run
326
+ decrement_running_count(tracking_key)
327
+
328
+ end
329
+ end
330
+
331
+ def stats
332
+ results = {}
333
+
334
+ queue_keys = Resque.redis.keys("concurrent:queue:*")
335
+
336
+ queue_sizes = {}
337
+ ident_sizes = {}
338
+ queue_keys.each do |k|
339
+ parts = k.split(":")
340
+ ident = parts[3..-1].join(":")
341
+ queue_name = parts[2]
342
+ size = Resque.redis.llen(k)
343
+ queue_sizes[queue_name] ||= 0
344
+ queue_sizes[queue_name] += size
345
+ ident_sizes[ident] ||= 0
346
+ ident_sizes[ident] += size
347
+ end
348
+
349
+ count_keys = Resque.redis.keys("concurrent:count:*")
350
+ running_counts = {}
351
+ count_keys.each do |k|
352
+ parts = k.split(":")
353
+ ident = parts[2..-1].join(":")
354
+ running_counts[ident] = Resque.redis.get(k).to_i
355
+ end
356
+
357
+ lock_keys = Resque.redis.keys("concurrent:lock:*")
358
+ lock_count = lock_keys.size
359
+
360
+ runnable_count = Resque.redis.scard(runnables_key)
361
+
362
+ return {
363
+ :queue_totals => {
364
+ :by_queue_name => queue_sizes,
365
+ :by_identifier => ident_sizes
366
+ },
367
+ :running_counts => running_counts,
368
+ :lock_count => lock_count,
369
+ :runnable_count => runnable_count,
370
+ }
371
+
372
+ end
373
+
374
+ end
375
+
376
+ # Allows users to subclass instead of extending in their job classes
377
+ class ConcurrentRestrictionJob
378
+ extend ConcurrentRestriction
379
+ end
380
+
381
+ end
382
+ end