qless 0.9.2 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +2 -0
- data/README.md +42 -3
- data/Rakefile +26 -2
- data/{bin → exe}/qless-web +3 -2
- data/lib/qless.rb +55 -28
- data/lib/qless/config.rb +1 -3
- data/lib/qless/job.rb +127 -22
- data/lib/qless/job_reservers/round_robin.rb +3 -1
- data/lib/qless/job_reservers/shuffled_round_robin.rb +14 -0
- data/lib/qless/lua_script.rb +42 -0
- data/lib/qless/middleware/redis_reconnect.rb +24 -0
- data/lib/qless/middleware/retry_exceptions.rb +43 -0
- data/lib/qless/middleware/sentry.rb +70 -0
- data/lib/qless/qless-core/cancel.lua +89 -59
- data/lib/qless/qless-core/complete.lua +16 -1
- data/lib/qless/qless-core/config.lua +12 -0
- data/lib/qless/qless-core/deregister_workers.lua +12 -0
- data/lib/qless/qless-core/fail.lua +24 -14
- data/lib/qless/qless-core/heartbeat.lua +2 -1
- data/lib/qless/qless-core/pause.lua +18 -0
- data/lib/qless/qless-core/pop.lua +24 -3
- data/lib/qless/qless-core/put.lua +14 -1
- data/lib/qless/qless-core/qless-lib.lua +2354 -0
- data/lib/qless/qless-core/qless.lua +1862 -0
- data/lib/qless/qless-core/retry.lua +1 -1
- data/lib/qless/qless-core/unfail.lua +54 -0
- data/lib/qless/qless-core/unpause.lua +12 -0
- data/lib/qless/queue.rb +45 -21
- data/lib/qless/server.rb +38 -39
- data/lib/qless/server/static/css/docs.css +21 -1
- data/lib/qless/server/views/_job.erb +5 -5
- data/lib/qless/server/views/overview.erb +14 -9
- data/lib/qless/subscriber.rb +48 -0
- data/lib/qless/version.rb +1 -1
- data/lib/qless/wait_until.rb +19 -0
- data/lib/qless/worker.rb +243 -33
- metadata +49 -30
- data/bin/install_phantomjs +0 -7
- data/bin/qless-campfire +0 -106
- data/bin/qless-growl +0 -99
- data/lib/qless/lua.rb +0 -25
metadata
CHANGED
@@ -1,22 +1,22 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qless
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Dan Lecocq
|
9
9
|
autorequire:
|
10
|
-
bindir:
|
10
|
+
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-05-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: redis
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
|
-
- -
|
19
|
+
- - ! '>='
|
20
20
|
- !ruby/object:Gem::Version
|
21
21
|
version: '2.2'
|
22
22
|
type: :runtime
|
@@ -24,41 +24,41 @@ dependencies:
|
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
none: false
|
26
26
|
requirements:
|
27
|
-
- -
|
27
|
+
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: '2.2'
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
|
-
name:
|
31
|
+
name: vegas
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
35
|
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.
|
38
|
-
type: :
|
37
|
+
version: 0.1.11
|
38
|
+
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.
|
45
|
+
version: 0.1.11
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
|
-
name:
|
47
|
+
name: sinatra
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
none: false
|
50
50
|
requirements:
|
51
51
|
- - ~>
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
54
|
-
type: :
|
53
|
+
version: 1.3.2
|
54
|
+
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
none: false
|
58
58
|
requirements:
|
59
59
|
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
61
|
+
version: 1.3.2
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: rspec
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: 2.
|
69
|
+
version: '2.12'
|
70
70
|
type: :development
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -74,7 +74,7 @@ dependencies:
|
|
74
74
|
requirements:
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: 2.
|
77
|
+
version: '2.12'
|
78
78
|
- !ruby/object:Gem::Dependency
|
79
79
|
name: rspec-fire
|
80
80
|
requirement: !ruby/object:Gem::Requirement
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
requirements:
|
83
83
|
- - ~>
|
84
84
|
- !ruby/object:Gem::Version
|
85
|
-
version: '
|
85
|
+
version: '1.1'
|
86
86
|
type: :development
|
87
87
|
prerelease: false
|
88
88
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -90,7 +90,7 @@ dependencies:
|
|
90
90
|
requirements:
|
91
91
|
- - ~>
|
92
92
|
- !ruby/object:Gem::Version
|
93
|
-
version: '
|
93
|
+
version: '1.1'
|
94
94
|
- !ruby/object:Gem::Dependency
|
95
95
|
name: rake
|
96
96
|
requirement: !ruby/object:Gem::Requirement
|
@@ -98,7 +98,7 @@ dependencies:
|
|
98
98
|
requirements:
|
99
99
|
- - ~>
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: 0
|
101
|
+
version: '10.0'
|
102
102
|
type: :development
|
103
103
|
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -106,7 +106,7 @@ dependencies:
|
|
106
106
|
requirements:
|
107
107
|
- - ~>
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: 0
|
109
|
+
version: '10.0'
|
110
110
|
- !ruby/object:Gem::Dependency
|
111
111
|
name: capybara
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -171,6 +171,22 @@ dependencies:
|
|
171
171
|
- - ~>
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: 0.6.2
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: sentry-raven
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ~>
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0.4'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ~>
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0.4'
|
174
190
|
description: ! "\n `qless` is meant to be a performant alternative to other queueing\n
|
175
191
|
\ systems, with statistics collection, a browser interface, and \n strong guarantees
|
176
192
|
about job losses.\n \n It's written as a collection of Lua scipts that are
|
@@ -194,37 +210,45 @@ files:
|
|
194
210
|
- lib/qless/job.rb
|
195
211
|
- lib/qless/job_reservers/ordered.rb
|
196
212
|
- lib/qless/job_reservers/round_robin.rb
|
197
|
-
- lib/qless/
|
213
|
+
- lib/qless/job_reservers/shuffled_round_robin.rb
|
214
|
+
- lib/qless/lua_script.rb
|
215
|
+
- lib/qless/middleware/redis_reconnect.rb
|
216
|
+
- lib/qless/middleware/retry_exceptions.rb
|
217
|
+
- lib/qless/middleware/sentry.rb
|
198
218
|
- lib/qless/queue.rb
|
199
219
|
- lib/qless/server.rb
|
220
|
+
- lib/qless/subscriber.rb
|
200
221
|
- lib/qless/tasks.rb
|
201
222
|
- lib/qless/version.rb
|
223
|
+
- lib/qless/wait_until.rb
|
202
224
|
- lib/qless/worker.rb
|
203
225
|
- lib/qless.rb
|
204
226
|
- lib/qless/qless-core/cancel.lua
|
205
227
|
- lib/qless/qless-core/complete.lua
|
206
228
|
- lib/qless/qless-core/config.lua
|
207
229
|
- lib/qless/qless-core/depends.lua
|
230
|
+
- lib/qless/qless-core/deregister_workers.lua
|
208
231
|
- lib/qless/qless-core/fail.lua
|
209
232
|
- lib/qless/qless-core/failed.lua
|
210
233
|
- lib/qless/qless-core/get.lua
|
211
234
|
- lib/qless/qless-core/heartbeat.lua
|
212
235
|
- lib/qless/qless-core/jobs.lua
|
236
|
+
- lib/qless/qless-core/pause.lua
|
213
237
|
- lib/qless/qless-core/peek.lua
|
214
238
|
- lib/qless/qless-core/pop.lua
|
215
239
|
- lib/qless/qless-core/priority.lua
|
216
240
|
- lib/qless/qless-core/put.lua
|
241
|
+
- lib/qless/qless-core/qless-lib.lua
|
242
|
+
- lib/qless/qless-core/qless.lua
|
217
243
|
- lib/qless/qless-core/queues.lua
|
218
244
|
- lib/qless/qless-core/recur.lua
|
219
245
|
- lib/qless/qless-core/retry.lua
|
220
246
|
- lib/qless/qless-core/stats.lua
|
221
247
|
- lib/qless/qless-core/tag.lua
|
222
248
|
- lib/qless/qless-core/track.lua
|
249
|
+
- lib/qless/qless-core/unfail.lua
|
250
|
+
- lib/qless/qless-core/unpause.lua
|
223
251
|
- lib/qless/qless-core/workers.lua
|
224
|
-
- bin/install_phantomjs
|
225
|
-
- bin/qless-campfire
|
226
|
-
- bin/qless-growl
|
227
|
-
- bin/qless-web
|
228
252
|
- lib/qless/server/static/css/bootstrap-responsive.css
|
229
253
|
- lib/qless/server/static/css/bootstrap-responsive.min.css
|
230
254
|
- lib/qless/server/static/css/bootstrap.css
|
@@ -272,6 +296,7 @@ files:
|
|
272
296
|
- lib/qless/server/views/track.erb
|
273
297
|
- lib/qless/server/views/worker.erb
|
274
298
|
- lib/qless/server/views/workers.erb
|
299
|
+
- exe/qless-web
|
275
300
|
homepage: http://github.com/seomoz/qless
|
276
301
|
licenses: []
|
277
302
|
post_install_message:
|
@@ -284,18 +309,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
284
309
|
- - ! '>='
|
285
310
|
- !ruby/object:Gem::Version
|
286
311
|
version: '0'
|
287
|
-
segments:
|
288
|
-
- 0
|
289
|
-
hash: -4094860363148682256
|
290
312
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
291
313
|
none: false
|
292
314
|
requirements:
|
293
315
|
- - ! '>='
|
294
316
|
- !ruby/object:Gem::Version
|
295
317
|
version: '0'
|
296
|
-
segments:
|
297
|
-
- 0
|
298
|
-
hash: -4094860363148682256
|
299
318
|
requirements: []
|
300
319
|
rubyforge_project: qless
|
301
320
|
rubygems_version: 1.8.24
|
data/bin/install_phantomjs
DELETED
data/bin/qless-campfire
DELETED
@@ -1,106 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
|
3
|
-
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
|
4
|
-
|
5
|
-
require 'qless'
|
6
|
-
require 'tinder'
|
7
|
-
require 'micro-optparse'
|
8
|
-
|
9
|
-
@options = Parser.new do |p|
|
10
|
-
p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
|
11
|
-
p.option :subdomain, 'campfire subdomain' , :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
|
12
|
-
p.option :token , 'campfire token for bot', :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
|
13
|
-
p.option :room , 'campfire room to talk in (defaults to first room)', :default => ''
|
14
|
-
p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
|
15
|
-
p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
|
16
|
-
end.process!
|
17
|
-
|
18
|
-
# Get a campfire object, and room
|
19
|
-
puts 'Connecting to campfire...'
|
20
|
-
@campfire = Tinder::Campfire.new @options[:subdomain], :token => @options[:token]
|
21
|
-
if not @options[:room]
|
22
|
-
@room = @campfire.rooms.first
|
23
|
-
else
|
24
|
-
@room = @campfire.find_room_by_name(@options[:room])
|
25
|
-
end
|
26
|
-
|
27
|
-
# And now qless stuff
|
28
|
-
puts 'Connecting to qless...'
|
29
|
-
@client = Qless::Client.new(:host => @options[:host])
|
30
|
-
|
31
|
-
def speak(message)
|
32
|
-
@room.speak message
|
33
|
-
puts "Said '#{message}'"
|
34
|
-
end
|
35
|
-
|
36
|
-
def paste(message)
|
37
|
-
@room.paste message
|
38
|
-
puts "Pasted '#{message}'"
|
39
|
-
end
|
40
|
-
|
41
|
-
def event(jid, &block)
|
42
|
-
job = @client.jobs[jid]
|
43
|
-
if job.nil?
|
44
|
-
speak("#{jid[0...8]} #{yield job} | http://#{@options[:host]}/jobs/#{jid}]")
|
45
|
-
else
|
46
|
-
speak("#{jid[0...8]} #{yield job} [#{job.tags.join(', ')}] | http://#{@options[:host]}/jobs/#{jid}")
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
@client.events.listen do |on|
|
51
|
-
on.canceled { |jid| event(jid) { |job| 'canceled' } }
|
52
|
-
on.stalled { |jid| event(jid) { |job| 'stalled' } }
|
53
|
-
on.track { |jid| event(jid) { |job| 'is being tracked' } }
|
54
|
-
on.untrack { |jid| event(jid) { |job| 'not longer tracked' } }
|
55
|
-
|
56
|
-
on.completed do |jid|
|
57
|
-
event(jid) do |job|
|
58
|
-
if job
|
59
|
-
if job.queue_name.empty?
|
60
|
-
'completed'
|
61
|
-
else
|
62
|
-
"advanced to #{job.queue_name}"
|
63
|
-
end
|
64
|
-
else
|
65
|
-
'completed'
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
on.popped do |jid|
|
71
|
-
event(jid) do |job|
|
72
|
-
if job
|
73
|
-
"popped by #{job.worker_name}"
|
74
|
-
else
|
75
|
-
'popped'
|
76
|
-
end
|
77
|
-
end
|
78
|
-
end
|
79
|
-
|
80
|
-
on.put do |jid|
|
81
|
-
event(jid) do |job|
|
82
|
-
if job
|
83
|
-
"moved to #{job.queue_name}"
|
84
|
-
else
|
85
|
-
'moved'
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
on.failed do |jid|
|
91
|
-
job = @client.jobs[jid]
|
92
|
-
if job
|
93
|
-
speak("#{jid} failed by #{job.failure['worker']} | http://#{@options[:web]}/jobs/#{jid}")
|
94
|
-
paste job.failure['message']
|
95
|
-
else
|
96
|
-
speak("#{jid} failed")
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
puts 'Listening...'
|
101
|
-
|
102
|
-
Signal.trap("INT") do
|
103
|
-
puts 'Exiting'
|
104
|
-
Process.exit(0)
|
105
|
-
end
|
106
|
-
end
|
data/bin/qless-growl
DELETED
@@ -1,99 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
|
3
|
-
$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
|
4
|
-
|
5
|
-
require 'qless'
|
6
|
-
require 'ruby-growl'
|
7
|
-
require 'micro-optparse'
|
8
|
-
|
9
|
-
@options = Parser.new do |p|
|
10
|
-
p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
|
11
|
-
p.option :growl , 'host for the growl daemon', :default => 'localhost'
|
12
|
-
p.option :app , 'application name for notifications', :default => 'qless'
|
13
|
-
p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
|
14
|
-
p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
|
15
|
-
end.process!
|
16
|
-
|
17
|
-
# Connect to growl
|
18
|
-
puts 'Connecting to growl...'
|
19
|
-
@growl = Growl::GNTP.new @options[:growl], @options[:app]
|
20
|
-
|
21
|
-
# Qless client
|
22
|
-
puts 'Connecting to qless...'
|
23
|
-
@client = Qless::Client.new(:host => @options[:host])
|
24
|
-
|
25
|
-
%w{canceled completed failed popped stalled put track untrack}.each do |t|
|
26
|
-
@growl.add_notification(t)
|
27
|
-
end
|
28
|
-
@growl.register
|
29
|
-
|
30
|
-
def notify(jid, event, &block)
|
31
|
-
job = @client.jobs[jid]
|
32
|
-
if job.nil?
|
33
|
-
message = yield job
|
34
|
-
puts "#{jid} -> Notifying #{message}"
|
35
|
-
@growl.notify(event, "#{jid[0...8]}", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
|
36
|
-
else
|
37
|
-
message = yield job
|
38
|
-
puts "#{jid} -> Notifying #{message}"
|
39
|
-
@growl.notify(event, "#{jid[0...8]} [#{job.tags.join(', ')}]", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
@client.events.listen do |on|
|
44
|
-
on.canceled { |jid| notify(jid, 'canceled') { |job| 'canceled' } }
|
45
|
-
on.stalled { |jid| notify(jid, 'canceled') { |job| 'stalled' } }
|
46
|
-
on.track { |jid| notify(jid, 'canceled') { |job| 'is being tracked' } }
|
47
|
-
on.untrack { |jid| notify(jid, 'canceled') { |job| 'no longer tracked' } }
|
48
|
-
|
49
|
-
on.completed do |jid|
|
50
|
-
notify(jid, 'completed') do |job|
|
51
|
-
if job
|
52
|
-
if job.queue_name.empty?
|
53
|
-
"completed"
|
54
|
-
else
|
55
|
-
"advanced to #{job.queue_name}"
|
56
|
-
end
|
57
|
-
else
|
58
|
-
"completed"
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
on.failed do |jid|
|
64
|
-
notify(jid, 'failed') do |job|
|
65
|
-
if job
|
66
|
-
"failed by #{job.failure['worker']}"
|
67
|
-
else
|
68
|
-
"failed"
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
73
|
-
on.popped do |jid|
|
74
|
-
notify(jid, 'popped') do |job|
|
75
|
-
if job
|
76
|
-
"popped by #{job.worker_name}"
|
77
|
-
else
|
78
|
-
"popped"
|
79
|
-
end
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
on.put do |jid|
|
84
|
-
notify(jid, 'put') do |job|
|
85
|
-
if job
|
86
|
-
"moved to #{job.queue_name}"
|
87
|
-
else
|
88
|
-
"moved"
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
puts 'Listening...'
|
94
|
-
|
95
|
-
Signal.trap("INT") do
|
96
|
-
puts 'Exiting'
|
97
|
-
Process.exit(0)
|
98
|
-
end
|
99
|
-
end
|
data/lib/qless/lua.rb
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
module Qless
|
2
|
-
class Lua
|
3
|
-
LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
|
4
|
-
|
5
|
-
def initialize(name, redis)
|
6
|
-
@sha = nil
|
7
|
-
@name = name
|
8
|
-
@redis = redis
|
9
|
-
reload()
|
10
|
-
end
|
11
|
-
|
12
|
-
def reload()
|
13
|
-
@sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
|
14
|
-
end
|
15
|
-
|
16
|
-
def call(keys, args)
|
17
|
-
begin
|
18
|
-
return @redis.evalsha(@sha, keys.length, *(keys + args))
|
19
|
-
rescue
|
20
|
-
reload
|
21
|
-
return @redis.evalsha(@sha, keys.length, *(keys + args))
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|