grockit-resque 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. data/.gitignore +2 -0
  2. data/.kick +26 -0
  3. data/CONTRIBUTORS +14 -0
  4. data/HISTORY.md +78 -0
  5. data/LICENSE +20 -0
  6. data/README.markdown +762 -0
  7. data/Rakefile +67 -0
  8. data/bin/resque +57 -0
  9. data/bin/resque-web +18 -0
  10. data/config.ru +14 -0
  11. data/deps.rip +6 -0
  12. data/examples/async_helper.rb +31 -0
  13. data/examples/demo/README.markdown +71 -0
  14. data/examples/demo/Rakefile +3 -0
  15. data/examples/demo/app.rb +38 -0
  16. data/examples/demo/config.ru +19 -0
  17. data/examples/demo/job.rb +22 -0
  18. data/examples/god/resque.god +53 -0
  19. data/examples/god/stale.god +26 -0
  20. data/examples/instance.rb +11 -0
  21. data/examples/simple.rb +30 -0
  22. data/init.rb +1 -0
  23. data/lib/resque.rb +247 -0
  24. data/lib/resque/errors.rb +7 -0
  25. data/lib/resque/failure.rb +63 -0
  26. data/lib/resque/failure/base.rb +58 -0
  27. data/lib/resque/failure/hoptoad.rb +122 -0
  28. data/lib/resque/failure/multiple.rb +44 -0
  29. data/lib/resque/failure/redis.rb +33 -0
  30. data/lib/resque/helpers.rb +57 -0
  31. data/lib/resque/job.rb +158 -0
  32. data/lib/resque/server.rb +182 -0
  33. data/lib/resque/server/public/idle.png +0 -0
  34. data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
  35. data/lib/resque/server/public/jquery.relatize_date.js +95 -0
  36. data/lib/resque/server/public/poll.png +0 -0
  37. data/lib/resque/server/public/ranger.js +24 -0
  38. data/lib/resque/server/public/reset.css +48 -0
  39. data/lib/resque/server/public/style.css +75 -0
  40. data/lib/resque/server/public/working.png +0 -0
  41. data/lib/resque/server/views/error.erb +1 -0
  42. data/lib/resque/server/views/failed.erb +35 -0
  43. data/lib/resque/server/views/key.erb +17 -0
  44. data/lib/resque/server/views/layout.erb +38 -0
  45. data/lib/resque/server/views/next_more.erb +10 -0
  46. data/lib/resque/server/views/overview.erb +4 -0
  47. data/lib/resque/server/views/queues.erb +46 -0
  48. data/lib/resque/server/views/stats.erb +62 -0
  49. data/lib/resque/server/views/workers.erb +78 -0
  50. data/lib/resque/server/views/working.erb +69 -0
  51. data/lib/resque/stat.rb +53 -0
  52. data/lib/resque/tasks.rb +39 -0
  53. data/lib/resque/version.rb +3 -0
  54. data/lib/resque/worker.rb +442 -0
  55. data/tasks/redis.rake +135 -0
  56. data/tasks/resque.rake +2 -0
  57. data/test/redis-test.conf +132 -0
  58. data/test/resque_test.rb +227 -0
  59. data/test/test_helper.rb +96 -0
  60. data/test/worker_test.rb +243 -0
  61. metadata +172 -0
@@ -0,0 +1,44 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that uses multiple backends
4
+ # delegates all queries to the first backend
5
+ class Multiple < Base
6
+
7
+ class << self
8
+ attr_accessor :classes
9
+ end
10
+
11
+ def self.configure
12
+ yield self
13
+ Resque::Failure.backend = self
14
+ end
15
+
16
+ def initialize(*args)
17
+ @backends = self.class.classes.map {|klass| klass.new(*args)}
18
+ end
19
+ def save
20
+ @backends.each(&:save)
21
+ end
22
+
23
+ # The number of failures.
24
+ def self.count
25
+ classes.first.count
26
+ end
27
+
28
+ # Returns a paginated array of failure objects.
29
+ def self.all(start = 0, count = 1)
30
+ classes.first.all(start,count)
31
+ end
32
+
33
+ # A URL where someone can go to view failures.
34
+ def self.url
35
+ classes.first.url
36
+ end
37
+
38
+ # Clear all failure objects
39
+ def self.clear
40
+ classes.first.clear
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,33 @@
1
+ module Resque
2
+ module Failure
3
+ # A Failure backend that stores exceptions in Redis. Very simple but
4
+ # works out of the box, along with support in the Resque web app.
5
+ class Redis < Base
6
+ def save
7
+ data = {
8
+ :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S"),
9
+ :payload => payload,
10
+ :error => exception.to_s,
11
+ :backtrace => exception.backtrace,
12
+ :worker => worker.to_s,
13
+ :queue => queue
14
+ }
15
+ data = Resque.encode(data)
16
+ Resque.redis.rpush(:failed, data)
17
+ end
18
+
19
+ def self.count
20
+ Resque.redis.llen(:failed).to_i
21
+ end
22
+
23
+ def self.all(start = 0, count = 1)
24
+ Resque.list_range(:failed, start, count)
25
+ end
26
+
27
+ def self.clear
28
+ Resque.redis.delete('resque:failed')
29
+ end
30
+
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,57 @@
1
+ module Resque
2
+ # Methods used by various classes in Resque.
3
+ module Helpers
4
+ # Direct access to the Redis instance.
5
+ def redis
6
+ Resque.redis
7
+ end
8
+
9
+ # Given a Ruby object, returns a string suitable for storage in a
10
+ # queue.
11
+ def encode(object)
12
+ if defined? Yajl
13
+ Yajl::Encoder.encode(object)
14
+ else
15
+ object.to_json
16
+ end
17
+ end
18
+
19
+ # Given a string, returns a Ruby object.
20
+ def decode(object)
21
+ return unless object
22
+
23
+ if defined? Yajl
24
+ Yajl::Parser.parse(object, :check_utf8 => false)
25
+ else
26
+ JSON.parse(object)
27
+ end
28
+ end
29
+
30
+ # Given a word with dashes, returns a camel cased version of it.
31
+ #
32
+ # classify('job-name') # => 'JobName'
33
+ def classify(dashed_word)
34
+ dashed_word.split('-').each { |part| part[0] = part[0].chr.upcase }.join
35
+ end
36
+
37
+ # Given a camel cased word, returns the constant it represents
38
+ #
39
+ # constantize('JobName') # => JobName
40
+ def constantize(camel_cased_word)
41
+ camel_cased_word = camel_cased_word.to_s
42
+
43
+ if camel_cased_word.include?('-')
44
+ camel_cased_word = classify(camel_cased_word)
45
+ end
46
+
47
+ names = camel_cased_word.split('::')
48
+ names.shift if names.empty? || names.first.empty?
49
+
50
+ constant = Object
51
+ names.each do |name|
52
+ constant = constant.const_get(name) || constant.const_missing(name)
53
+ end
54
+ constant
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,158 @@
1
+ module Resque
2
+ # A Resque::Job represents a unit of work. Each job lives on a
3
+ # single queue and has an associated payload object. The payload
4
+ # is a hash with two attributes: `class` and `args`. The `class` is
5
+ # the name of the Ruby class which should be used to run the
6
+ # job. The `args` are an array of arguments which should be passed
7
+ # to the Ruby class's `perform` class-level method.
8
+ #
9
+ # You can manually run a job using this code:
10
+ #
11
+ # job = Resque::Job.reserve(:high)
12
+ # klass = Resque::Job.constantize(job.payload['class'])
13
+ # klass.perform(*job.payload['args'])
14
+ class Job
15
+ include Helpers
16
+ extend Helpers
17
+
18
+ # The worker object which is currently processing this job.
19
+ attr_accessor :worker
20
+
21
+ # The name of the queue from which this job was pulled (or is to be
22
+ # placed)
23
+ attr_reader :queue
24
+
25
+ # This job's associated payload object.
26
+ attr_reader :payload
27
+
28
+ def initialize(queue, payload)
29
+ @queue = queue
30
+ @payload = payload
31
+ end
32
+
33
+ # Creates a job by placing it on a queue. Expects a string queue
34
+ # name, a string class name, and an optional array of arguments to
35
+ # pass to the class' `perform` method.
36
+ #
37
+ # Raises an exception if no queue or class is given.
38
+ def self.create(queue, klass, *args)
39
+ if !queue
40
+ raise NoQueueError.new("Jobs must be placed onto a queue.")
41
+ end
42
+
43
+ if klass.to_s.empty?
44
+ raise NoClassError.new("Jobs must be given a class.")
45
+ end
46
+
47
+ Resque.push(queue, :class => klass.to_s, :args => args)
48
+ end
49
+
50
+ def self.create_unique(queue, klass, *args)
51
+ if !queue
52
+ raise NoQueueError.new("Jobs must be placed onto a queue.")
53
+ end
54
+
55
+ if klass.to_s.empty?
56
+ raise NoClassError.new("Jobs must be given a class.")
57
+ end
58
+
59
+ Resque.push(queue, :class => klass.to_s, :args => args, :unique => true)
60
+ end
61
+
62
+ # Removes a job from a queue. Expects a string queue name, a
63
+ # string class name, and, optionally, args.
64
+ #
65
+ # Returns the number of jobs destroyed.
66
+ #
67
+ # If no args are provided, it will remove all jobs of the class
68
+ # provided.
69
+ #
70
+ # That is, for these two jobs:
71
+ #
72
+ # { 'class' => 'UpdateGraph', 'args' => ['defunkt'] }
73
+ # { 'class' => 'UpdateGraph', 'args' => ['mojombo'] }
74
+ #
75
+ # The following call will remove both:
76
+ #
77
+ # Resque::Job.destroy(queue, 'UpdateGraph')
78
+ #
79
+ # Whereas specifying args will only remove the 2nd job:
80
+ #
81
+ # Resque::Job.destroy(queue, 'UpdateGraph', 'mojombo')
82
+ #
83
+ # This method can be potentially very slow and memory intensive,
84
+ # depending on the size of your queue, as it loads all jobs into
85
+ # a Ruby array before processing.
86
+ def self.destroy(queue, klass, *args)
87
+ klass = klass.to_s
88
+ queue = "queue:#{queue}"
89
+ destroyed = 0
90
+
91
+ redis.lrange(queue, 0, -1).each do |string|
92
+ json = decode(string)
93
+
94
+ match = json['class'] == klass
95
+ match &= json['args'] == args unless args.empty?
96
+
97
+ if match
98
+ destroyed += redis.lrem(queue, 0, string).to_i
99
+ end
100
+ end
101
+
102
+ destroyed
103
+ end
104
+
105
+ # Given a string queue name, returns an instance of Resque::Job
106
+ # if any jobs are available. If not, returns nil.
107
+ def self.reserve(queue)
108
+ return unless payload = Resque.pop(queue)
109
+ new(queue, payload)
110
+ end
111
+
112
+ # Attempts to perform the work represented by this job instance.
113
+ # Calls #perform on the class given in the payload with the
114
+ # arguments given in the payload.
115
+ def perform
116
+ args ? payload_class.perform(*args) : payload_class.perform
117
+ end
118
+
119
+ # Returns the actual class constant represented in this job's payload.
120
+ def payload_class
121
+ @payload_class ||= constantize(@payload['class'])
122
+ end
123
+
124
+ # Returns an array of args represented in this job's payload.
125
+ def args
126
+ @payload['args']
127
+ end
128
+
129
+ # Given an exception object, hands off the needed parameters to
130
+ # the Failure module.
131
+ def fail(exception)
132
+ Failure.create \
133
+ :payload => payload,
134
+ :exception => exception,
135
+ :worker => worker,
136
+ :queue => queue
137
+ end
138
+
139
+ # Creates an identical job, essentially placing this job back on
140
+ # the queue.
141
+ def recreate
142
+ self.class.create(queue, payload_class, *args)
143
+ end
144
+
145
+ # String representation
146
+ def inspect
147
+ obj = @payload
148
+ "(Job{%s} | %s | %s)" % [ @queue, obj['class'], obj['args'].inspect ]
149
+ end
150
+
151
+ # Equality
152
+ def ==(other)
153
+ queue == other.queue &&
154
+ payload_class == other.payload_class &&
155
+ args == other.args
156
+ end
157
+ end
158
+ end
@@ -0,0 +1,182 @@
1
+ require 'sinatra/base'
2
+ require 'erb'
3
+ require 'resque'
4
+ require 'resque/version'
5
+
6
+ module Resque
7
+ class Server < Sinatra::Base
8
+ dir = File.dirname(File.expand_path(__FILE__))
9
+
10
+ set :views, "#{dir}/server/views"
11
+ set :public, "#{dir}/server/public"
12
+ set :static, true
13
+
14
+ helpers do
15
+ include Rack::Utils
16
+ alias_method :h, :escape_html
17
+
18
+ def current_section
19
+ url request.path_info.sub('/','').split('/')[0].downcase
20
+ end
21
+
22
+ def current_page
23
+ url request.path_info.sub('/','').downcase
24
+ end
25
+
26
+ def url(*path_parts)
27
+ [ path_prefix, path_parts ].join("/").squeeze('/')
28
+ end
29
+ alias_method :u, :url
30
+
31
+ def path_prefix
32
+ request.env['SCRIPT_NAME']
33
+ end
34
+
35
+ def class_if_current(page = '')
36
+ 'class="current"' if current_page.include? page.to_s
37
+ end
38
+
39
+ def tab(name)
40
+ dname = name.to_s.downcase
41
+ "<li #{class_if_current(dname)}><a href='#{url dname}'>#{name}</a></li>"
42
+ end
43
+
44
+ def tabs
45
+ Resque::Server.tabs
46
+ end
47
+
48
+ def redis_get_size(key)
49
+ case Resque.redis.type(key)
50
+ when 'none'
51
+ []
52
+ when 'list'
53
+ Resque.redis.llen(key)
54
+ when 'set'
55
+ Resque.redis.scard(key)
56
+ when 'string'
57
+ Resque.redis.get(key).length
58
+ end
59
+ end
60
+
61
+ def redis_get_value_as_array(key)
62
+ case Resque.redis.type(key)
63
+ when 'none'
64
+ []
65
+ when 'list'
66
+ Resque.redis.lrange(key, 0, 20)
67
+ when 'set'
68
+ Resque.redis.smembers(key)
69
+ when 'string'
70
+ [Resque.redis.get(key)]
71
+ end
72
+ end
73
+
74
+ def show_args(args)
75
+ Array(args).map { |a| a.inspect }.join("\n")
76
+ end
77
+
78
+ def partial?
79
+ @partial
80
+ end
81
+
82
+ def partial(template, local_vars = {})
83
+ @partial = true
84
+ erb(template.to_sym, {:layout => false}, local_vars)
85
+ ensure
86
+ @partial = false
87
+ end
88
+
89
+ def poll
90
+ if @polling
91
+ text = "Last Updated: #{Time.now.strftime("%H:%M:%S")}"
92
+ else
93
+ text = "<a href='#{url(request.path_info)}.poll' rel='poll'>Live Poll</a>"
94
+ end
95
+ "<p class='poll'>#{text}</p>"
96
+ end
97
+
98
+ end
99
+
100
+ def show(page, layout = true)
101
+ begin
102
+ erb page.to_sym, {:layout => layout}, :resque => Resque
103
+ rescue Errno::ECONNREFUSED
104
+ erb :error, {:layout => false}, :error => "Can't connect to Redis! (#{Resque.redis.server})"
105
+ end
106
+ end
107
+
108
+ # to make things easier on ourselves
109
+ get "/?" do
110
+ redirect url(:overview)
111
+ end
112
+
113
+ %w( overview queues working workers key ).each do |page|
114
+ get "/#{page}" do
115
+ show page
116
+ end
117
+
118
+ get "/#{page}/:id" do
119
+ show page
120
+ end
121
+ end
122
+
123
+ %w( overview workers ).each do |page|
124
+ get "/#{page}.poll" do
125
+ content_type "text/plain"
126
+ @polling = true
127
+ show(page.to_sym, false).gsub(/\s{1,}/, ' ')
128
+ end
129
+ end
130
+
131
+ get "/failed" do
132
+ if Resque::Failure.url
133
+ redirect Resque::Failure.url
134
+ else
135
+ show :failed
136
+ end
137
+ end
138
+
139
+ post "/failed/clear" do
140
+ Resque::Failure.clear
141
+ redirect u('failed')
142
+ end
143
+
144
+ get "/stats" do
145
+ redirect url("/stats/resque")
146
+ end
147
+
148
+ get "/stats/:id" do
149
+ show :stats
150
+ end
151
+
152
+ get "/stats/keys/:key" do
153
+ show :stats
154
+ end
155
+
156
+ get "/stats.txt" do
157
+ info = Resque.info
158
+
159
+ stats = []
160
+ stats << "resque.pending=#{info[:pending]}"
161
+ stats << "resque.processed+=#{info[:processed]}"
162
+ stats << "resque.failed+=#{info[:failed]}"
163
+ stats << "resque.workers=#{info[:workers]}"
164
+ stats << "resque.working=#{info[:working]}"
165
+
166
+ Resque.queues.each do |queue|
167
+ stats << "queues.#{queue}=#{Resque.size(queue)}"
168
+ end
169
+
170
+ content_type 'text/plain'
171
+ stats.join "\n"
172
+ end
173
+
174
+ def resque
175
+ Resque
176
+ end
177
+
178
+ def self.tabs
179
+ @tabs ||= ["Overview", "Working", "Failed", "Queues", "Workers", "Stats"]
180
+ end
181
+ end
182
+ end