resque-igo 1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (50) hide show
  1. data/HISTORY.md +225 -0
  2. data/LICENSE +20 -0
  3. data/README.markdown +855 -0
  4. data/Rakefile +70 -0
  5. data/bin/resque +57 -0
  6. data/bin/resque-web +23 -0
  7. data/lib/resque.rb +380 -0
  8. data/lib/resque/errors.rb +10 -0
  9. data/lib/resque/failure.rb +66 -0
  10. data/lib/resque/failure/base.rb +61 -0
  11. data/lib/resque/failure/hoptoad.rb +132 -0
  12. data/lib/resque/failure/multiple.rb +50 -0
  13. data/lib/resque/failure/redis.rb +40 -0
  14. data/lib/resque/helpers.rb +71 -0
  15. data/lib/resque/job.rb +209 -0
  16. data/lib/resque/plugin.rb +51 -0
  17. data/lib/resque/server.rb +247 -0
  18. data/lib/resque/server/public/idle.png +0 -0
  19. data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
  20. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  21. data/lib/resque/server/public/poll.png +0 -0
  22. data/lib/resque/server/public/ranger.js +67 -0
  23. data/lib/resque/server/public/reset.css +48 -0
  24. data/lib/resque/server/public/style.css +86 -0
  25. data/lib/resque/server/public/working.png +0 -0
  26. data/lib/resque/server/test_helper.rb +19 -0
  27. data/lib/resque/server/views/error.erb +1 -0
  28. data/lib/resque/server/views/failed.erb +53 -0
  29. data/lib/resque/server/views/key_sets.erb +20 -0
  30. data/lib/resque/server/views/key_string.erb +11 -0
  31. data/lib/resque/server/views/layout.erb +42 -0
  32. data/lib/resque/server/views/next_more.erb +10 -0
  33. data/lib/resque/server/views/overview.erb +4 -0
  34. data/lib/resque/server/views/queues.erb +65 -0
  35. data/lib/resque/server/views/stats.erb +73 -0
  36. data/lib/resque/server/views/workers.erb +109 -0
  37. data/lib/resque/server/views/working.erb +68 -0
  38. data/lib/resque/stat.rb +54 -0
  39. data/lib/resque/tasks.rb +39 -0
  40. data/lib/resque/version.rb +3 -0
  41. data/lib/resque/worker.rb +478 -0
  42. data/tasks/resque.rake +2 -0
  43. data/test/job_hooks_test.rb +323 -0
  44. data/test/job_plugins_test.rb +230 -0
  45. data/test/plugin_test.rb +116 -0
  46. data/test/resque-web_test.rb +54 -0
  47. data/test/resque_test.rb +351 -0
  48. data/test/test_helper.rb +166 -0
  49. data/test/worker_test.rb +302 -0
  50. metadata +180 -0
data/Rakefile ADDED
@@ -0,0 +1,70 @@
1
+ #
2
+ # Setup
3
+ #
4
+ require 'rake/testtask'
5
+
6
+ $LOAD_PATH.unshift 'lib'
7
+ require 'resque/tasks'
8
+
9
+ def command?(command)
10
+ system("type #{command} > /dev/null 2>&1")
11
+ end
12
+
13
+
14
+ #
15
+ # Tests
16
+ #
17
+
18
+ task :default => :test
19
+
20
+ desc "Run the test suite"
21
+ task :test do
22
+ rg = command?(:rg)
23
+ Dir['test/**/*_test.rb'].each do |f|
24
+ #rg ? sh("rg #{f}") : ruby(f)
25
+ ruby(f)
26
+ end
27
+ end
28
+
29
+ if command? :kicker
30
+ desc "Launch Kicker (like autotest)"
31
+ task :kicker do
32
+ puts "Kicking... (ctrl+c to cancel)"
33
+ exec "kicker -e rake test lib examples"
34
+ end
35
+ end
36
+
37
+
38
+ #
39
+ # Install
40
+ #
41
+
42
+ task :install => [ 'dtach:install' ]
43
+
44
+
45
+ #
46
+ # Documentation
47
+ #
48
+
49
+ begin
50
+ require 'sdoc_helpers'
51
+ rescue LoadError
52
+ end
53
+
54
+
55
+ #
56
+ # Publishing
57
+ #
58
+
59
+ desc "Push a new version to Gemcutter"
60
+ task :publish do
61
+ require 'resque/version'
62
+
63
+ sh "gem build resque.gemspec"
64
+ sh "gem push resque-#{Resque::Version}.gem"
65
+ sh "git tag v#{Resque::Version}"
66
+ sh "git push origin v#{Resque::Version}"
67
+ sh "git push origin master"
68
+ sh "git clean -fd"
69
+ exec "rake pages"
70
+ end
data/bin/resque ADDED
@@ -0,0 +1,57 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
4
+ require 'resque'
5
+
6
+ def kill(worker)
7
+ abort "** resque kill WORKER_ID" if worker.nil?
8
+ pid = worker.split(':')[1].to_i
9
+
10
+ begin
11
+ Process.kill("KILL", pid)
12
+ puts "** killed #{worker}"
13
+ rescue Errno::ESRCH
14
+ puts "** worker #{worker} not running"
15
+ end
16
+
17
+ remove worker
18
+ end
19
+
20
+ def remove(worker)
21
+ abort "** resque remove WORKER_ID" if worker.nil?
22
+
23
+ Resque.remove_worker(worker)
24
+ puts "** removed #{worker}"
25
+ end
26
+
27
+ def list
28
+ if Resque.workers.any?
29
+ Resque.workers.each do |worker|
30
+ puts "#{worker} (#{worker.state})"
31
+ end
32
+ else
33
+ puts "None"
34
+ end
35
+ end
36
+
37
+ if (i = ARGV.index('-r')) && ARGV[i+1]
38
+ Resque.redis = ARGV[i+1]
39
+ ARGV.delete_at(i)
40
+ ARGV.delete_at(i+1)
41
+ end
42
+
43
+ case ARGV[0]
44
+ when 'kill'
45
+ kill ARGV[1]
46
+ when 'remove'
47
+ remove ARGV[1]
48
+ when 'list'
49
+ list
50
+ else
51
+ puts "Usage: resque [-r redis_host:redis_port] COMMAND [option]"
52
+ puts
53
+ puts "Commands:"
54
+ puts " remove WORKER Removes a worker"
55
+ puts " kill WORKER Kills a worker"
56
+ puts " list Lists known workers"
57
+ end
data/bin/resque-web ADDED
@@ -0,0 +1,23 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
4
+ begin
5
+ require 'vegas'
6
+ rescue LoadError
7
+ require 'rubygems'
8
+ require 'vegas'
9
+ end
10
+ require 'resque/server'
11
+
12
+
13
+ Vegas::Runner.new(Resque::Server, 'resque-web', {
14
+ :before_run => lambda {|v|
15
+ path = (ENV['RESQUECONFIG'] || v.args.first)
16
+ load path.to_s.strip if path
17
+ }
18
+ }) do |runner, opts, app|
19
+ opts.on('-N NAMESPACE', "--namespace NAMESPACE", "set the Mongo database") {|namespace|
20
+ runner.logger.info "Using Mongo database '#{namespace}'"
21
+ Resque.mongo_db= namespace
22
+ }
23
+ end
data/lib/resque.rb ADDED
@@ -0,0 +1,380 @@
1
+
2
+ begin
3
+ require 'yajl'
4
+ rescue LoadError
5
+ require 'json'
6
+ end
7
+
8
+ require 'mongo'
9
+
10
+ require 'resque/version'
11
+
12
+ require 'resque/errors'
13
+
14
+ require 'resque/failure'
15
+ require 'resque/failure/base'
16
+
17
+ require 'resque/helpers'
18
+ require 'resque/stat'
19
+ require 'resque/job'
20
+ require 'resque/worker'
21
+ require 'resque/plugin'
22
+
23
+ module Resque
24
+ include Helpers
25
+ extend self
26
+ attr_accessor :bypass_queues
27
+ @bypass_queues = false
28
+ @delay_allowed = []
29
+
30
+
31
+ def mongo=(server)
32
+ if server.is_a? String
33
+ opts = server.split(':')
34
+ host = opts[0]
35
+ if opts[1] =~ /\//
36
+ opts = opts[1].split('/')
37
+ port = opts[0]
38
+ queuedb = opts[1]
39
+ else
40
+ port = opts[1]
41
+ end
42
+ conn = Mongo::Connection.new host, port
43
+ elsif server.is_a? Hash
44
+ conn = Mongo::Connection.new(options[:server], options[:port], options)
45
+ queuedb = options[:queuedb] || 'resque'
46
+ elsif server.is_a? Mongo::Connection
47
+ conn = server
48
+ end
49
+ queuedb ||= 'resque'
50
+ @mongo = conn.db queuedb
51
+ initialize_mongo
52
+ end
53
+
54
+ # Returns the current Mongo connection. If none has been created, will
55
+ # create a new one.
56
+ def mongo
57
+ return @mongo if @mongo
58
+ self.mongo = 'localhost:27017/resque'
59
+ self.mongo
60
+ end
61
+
62
+ def mongo_db=(db)
63
+ mongo.conn.db = db
64
+ end
65
+
66
+ def mongo?
67
+ return @mongo
68
+ end
69
+
70
+ def initialize_mongo
71
+ mongo_workers.create_index :worker
72
+ mongo_stats.create_index :stat
73
+ delay_allowed = mongo_stats.find_one({ :stat => 'Delayable Queues'}, { :fields => ['value']})
74
+ @delay_allowed = delay_allowed['value'].map{ |queue| queue.to_sym} if delay_allowed
75
+ end
76
+
77
+ def mongo_workers
78
+ mongo['resque.workers']
79
+ end
80
+
81
+ def mongo_stats
82
+ mongo['resque.metrics']
83
+ end
84
+
85
+ def mongo_failures
86
+ mongo['resque.failures']
87
+ end
88
+
89
+ # The `before_first_fork` hook will be run in the **parent** process
90
+ # only once, before forking to run the first job. Be careful- any
91
+ # changes you make will be permanent for the lifespan of the
92
+ # worker.
93
+ #
94
+ # Call with a block to set the hook.
95
+ # Call with no arguments to return the hook.
96
+ def before_first_fork(&block)
97
+ block ? (@before_first_fork = block) : @before_first_fork
98
+ end
99
+
100
+ # Set a proc that will be called in the parent process before the
101
+ # worker forks for the first time.
102
+ def before_first_fork=(before_first_fork)
103
+ @before_first_fork = before_first_fork
104
+ end
105
+
106
+ # The `before_fork` hook will be run in the **parent** process
107
+ # before every job, so be careful- any changes you make will be
108
+ # permanent for the lifespan of the worker.
109
+ #
110
+ # Call with a block to set the hook.
111
+ # Call with no arguments to return the hook.
112
+ def before_fork(&block)
113
+ block ? (@before_fork = block) : @before_fork
114
+ end
115
+
116
+ # Set the before_fork proc.
117
+ def before_fork=(before_fork)
118
+ @before_fork = before_fork
119
+ end
120
+
121
+ # The `after_fork` hook will be run in the child process and is passed
122
+ # the current job. Any changes you make, therefore, will only live as
123
+ # long as the job currently being processed.
124
+ #
125
+ # Call with a block to set the hook.
126
+ # Call with no arguments to return the hook.
127
+ def after_fork(&block)
128
+ block ? (@after_fork = block) : @after_fork
129
+ end
130
+
131
+ # Set the after_fork proc.
132
+ def after_fork=(after_fork)
133
+ @after_fork = after_fork
134
+ end
135
+
136
+ def to_s
137
+ "Resque Client connected to #{mongo.connection.host}:#{mongo.connection.port}/#{mongo.name}"
138
+ end
139
+
140
+ def allows_unique_jobs(klass)
141
+ klass.instance_variable_get(:@unique_jobs) ||
142
+ (klass.respond_to?(:unique_jobs) and klass.unique_jobs)
143
+ end
144
+
145
+ def allows_delayed_jobs(klass)
146
+ klass.instance_variable_get(:@delayed_jobs) ||
147
+ (klass.respond_to?(:delayed_jobs) and klass.delayed_jobs)
148
+ end
149
+
150
+ def queue_allows_delayed(queue)
151
+ @delay_allowed.include?(queue.to_sym) || @delay_allowed.include?(queue.to_s)
152
+ end
153
+
154
+ def enable_delay(queue)
155
+ unless queue_allows_delayed queue
156
+ @delay_allowed << queue
157
+ mongo_stats.update({:stat => 'Delayable Queues'}, { '$addToSet' => { 'value' => queue}}, { :upsert => true})
158
+ end
159
+ end
160
+
161
+ #
162
+ # queue manipulation
163
+ #
164
+
165
+ # Pushes a job onto a queue. Queue name should be a string and the
166
+ # item should be any JSON-able Ruby object.
167
+ def push(queue, item)
168
+ if item[:unique]
169
+ mongo[queue].update({'_id' => item[:_id]}, item, { :upsert => true})
170
+ else
171
+ mongo[queue] << item
172
+ end
173
+ end
174
+
175
+ # Pops a job off a queue. Queue name should be a string.
176
+ #
177
+ # Returns a Ruby object.
178
+ def pop(queue)
179
+ query = { }
180
+ if queue_allows_delayed queue
181
+ query['delay_until'] = { '$not' => { '$gt' => Time.new}}
182
+ end
183
+ #sorting will result in significant performance penalties for large queues, you have been warned.
184
+ item = mongo[queue].find_and_modify(:query => query, :remove => true )
185
+ rescue Mongo::OperationFailure => e
186
+ return nil if e.message =~ /No matching object/
187
+ raise e
188
+ end
189
+
190
+ # Returns an integer representing the size of a queue.
191
+ # Queue name should be a string.
192
+ def size(queue)
193
+ mongo[queue].count
194
+ end
195
+
196
+ def delayed_size(queue)
197
+ if queue_allows_delayed queue
198
+ mongo[queue].find({'delay_until' => { '$gt' => Time.new}}).count
199
+ else
200
+ mongo[queue].count
201
+ end
202
+ end
203
+
204
+ def ready_size(queue)
205
+ if queue_allows_delayed queue
206
+ mongo[queue].find({'delay_until' => { '$not' => { '$gt' => Time.new}}}).count
207
+ else
208
+ mongo[queue].count
209
+ end
210
+ end
211
+
212
+
213
+ # Returns an array of items currently queued. Queue name should be
214
+ # a string.
215
+ #
216
+ # start and count should be integer and can be used for pagination.
217
+ # start is the item to begin, count is how many items to return.
218
+ #
219
+ # To get the 3rd page of a 30 item, paginatied list one would use:
220
+ # Resque.peek('my_list', 59, 30)
221
+ def peek(queue, start = 0, count = 1, mode = :ready)
222
+ list_range(queue, start, count, mode)
223
+ end
224
+
225
+ # Does the dirty work of fetching a range of items from a Redis list
226
+ # and converting them into Ruby objects.
227
+ def list_range(key, start = 0, count = 1, mode = :ready)
228
+ query = { }
229
+ sort = []
230
+ if queue_allows_delayed(key)
231
+ if mode == :ready
232
+ query['delay_until'] = { '$not' => { '$gt' => Time.new}}
233
+ elsif mode == :delayed
234
+ query['delay_until'] = { '$gt' => Time.new}
235
+ elsif mode == :delayed_sorted
236
+ query['delay_until'] = { '$gt' => Time.new}
237
+ sort << ['delay_until', 1]
238
+ elsif mode == :all_sorted
239
+ query = {}
240
+ sort << ['delay_until', 1]
241
+ end
242
+ end
243
+ items = mongo[key].find(query, { :limit => count, :skip => start, :sort => sort}).to_a.map{ |i| i}
244
+ count > 1 ? items : items.first
245
+ end
246
+
247
+ # Returns an array of all known Resque queues as strings.
248
+ def queues
249
+ names = mongo.collection_names
250
+ names.delete_if{ |name| name == 'system.indexes' || name =~ /resque\./ }
251
+ end
252
+
253
+ # Given a queue name, completely deletes the queue.
254
+ def remove_queue(queue)
255
+ mongo[queue].drop
256
+ end
257
+
258
+ #
259
+ # job shortcuts
260
+ #
261
+
262
+ # This method can be used to conveniently add a job to a queue.
263
+ # It assumes the class you're passing it is a real Ruby class (not
264
+ # a string or reference) which either:
265
+ #
266
+ # a) has a @queue ivar set
267
+ # b) responds to `queue`
268
+ #
269
+ # If either of those conditions are met, it will use the value obtained
270
+ # from performing one of the above operations to determine the queue.
271
+ #
272
+ # If no queue can be inferred this method will raise a `Resque::NoQueueError`
273
+ #
274
+ # This method is considered part of the `stable` API.
275
+ def enqueue(klass, *args)
276
+ if @bypass_queues
277
+ klass.send(:perform, *args)
278
+ else
279
+ Job.create(queue_from_class(klass), klass, *args)
280
+ end
281
+ end
282
+
283
+ # This method can be used to conveniently remove a job from a queue.
284
+ # It assumes the class you're passing it is a real Ruby class (not
285
+ # a string or reference) which either:
286
+ #
287
+ # a) has a @queue ivar set
288
+ # b) responds to `queue`
289
+ #
290
+ # If either of those conditions are met, it will use the value obtained
291
+ # from performing one of the above operations to determine the queue.
292
+ #
293
+ # If no queue can be inferred this method will raise a `Resque::NoQueueError`
294
+ #
295
+ # If no args are given, this method will dequeue *all* jobs matching
296
+ # the provided class. See `Resque::Job.destroy` for more
297
+ # information.
298
+ #
299
+ # Returns the number of jobs destroyed.
300
+ #
301
+ # Example:
302
+ #
303
+ # # Removes all jobs of class `UpdateNetworkGraph`
304
+ # Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph)
305
+ #
306
+ # # Removes all jobs of class `UpdateNetworkGraph` with matching args.
307
+ # Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph, 'repo:135325')
308
+ #
309
+ # This method is considered part of the `stable` API.
310
+ def dequeue(klass, *args)
311
+ Job.destroy(queue_from_class(klass), klass, *args)
312
+ end
313
+
314
+ # Given a class, try to extrapolate an appropriate queue based on a
315
+ # class instance variable or `queue` method.
316
+ def queue_from_class(klass)
317
+ klass.instance_variable_get(:@queue) ||
318
+ (klass.respond_to?(:queue) and klass.queue)
319
+ end
320
+
321
+ # This method will return a `Resque::Job` object or a non-true value
322
+ # depending on whether a job can be obtained. You should pass it the
323
+ # precise name of a queue: case matters.
324
+ #
325
+ # This method is considered part of the `stable` API.
326
+ def reserve(queue)
327
+ Job.reserve(queue)
328
+ end
329
+
330
+
331
+ #
332
+ # worker shortcuts
333
+ #
334
+
335
+ # A shortcut to Worker.all
336
+ def workers
337
+ Worker.all
338
+ end
339
+
340
+ # A shortcut to Worker.working
341
+ def working
342
+ Worker.working
343
+ end
344
+
345
+ # A shortcut to unregister_worker
346
+ # useful for command line tool
347
+ def remove_worker(worker_id)
348
+ worker = Resque::Worker.find(worker_id)
349
+ worker.unregister_worker
350
+ end
351
+
352
+ #
353
+ # stats
354
+ #
355
+
356
+ # Returns a hash, similar to redis-rb's #info, of interesting stats.
357
+ def info
358
+ return {
359
+ :pending => queues.inject(0) { |m,k| m + size(k) },
360
+ :processed => Stat[:processed],
361
+ :queues => queues.size,
362
+ :workers => workers.size.to_i,
363
+ :working => working.count,
364
+ :failed => Stat[:failed],
365
+ :servers => to_s,
366
+ :environment => ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
367
+ }
368
+ end
369
+
370
+ # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
371
+ # is O(N) for the keyspace, so be careful - this can be slow for big databases.
372
+ def keys
373
+ names = mongo.collection_names
374
+ end
375
+
376
+ def drop
377
+ mongo.collections.each{ |collection| collection.drop unless collection.name =~ /^system./ }
378
+ @mongo = nil
379
+ end
380
+ end