qless 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/Gemfile +2 -0
  2. data/README.md +42 -3
  3. data/Rakefile +26 -2
  4. data/{bin → exe}/qless-web +3 -2
  5. data/lib/qless.rb +55 -28
  6. data/lib/qless/config.rb +1 -3
  7. data/lib/qless/job.rb +127 -22
  8. data/lib/qless/job_reservers/round_robin.rb +3 -1
  9. data/lib/qless/job_reservers/shuffled_round_robin.rb +14 -0
  10. data/lib/qless/lua_script.rb +42 -0
  11. data/lib/qless/middleware/redis_reconnect.rb +24 -0
  12. data/lib/qless/middleware/retry_exceptions.rb +43 -0
  13. data/lib/qless/middleware/sentry.rb +70 -0
  14. data/lib/qless/qless-core/cancel.lua +89 -59
  15. data/lib/qless/qless-core/complete.lua +16 -1
  16. data/lib/qless/qless-core/config.lua +12 -0
  17. data/lib/qless/qless-core/deregister_workers.lua +12 -0
  18. data/lib/qless/qless-core/fail.lua +24 -14
  19. data/lib/qless/qless-core/heartbeat.lua +2 -1
  20. data/lib/qless/qless-core/pause.lua +18 -0
  21. data/lib/qless/qless-core/pop.lua +24 -3
  22. data/lib/qless/qless-core/put.lua +14 -1
  23. data/lib/qless/qless-core/qless-lib.lua +2354 -0
  24. data/lib/qless/qless-core/qless.lua +1862 -0
  25. data/lib/qless/qless-core/retry.lua +1 -1
  26. data/lib/qless/qless-core/unfail.lua +54 -0
  27. data/lib/qless/qless-core/unpause.lua +12 -0
  28. data/lib/qless/queue.rb +45 -21
  29. data/lib/qless/server.rb +38 -39
  30. data/lib/qless/server/static/css/docs.css +21 -1
  31. data/lib/qless/server/views/_job.erb +5 -5
  32. data/lib/qless/server/views/overview.erb +14 -9
  33. data/lib/qless/subscriber.rb +48 -0
  34. data/lib/qless/version.rb +1 -1
  35. data/lib/qless/wait_until.rb +19 -0
  36. data/lib/qless/worker.rb +243 -33
  37. metadata +49 -30
  38. data/bin/install_phantomjs +0 -7
  39. data/bin/qless-campfire +0 -106
  40. data/bin/qless-growl +0 -99
  41. data/lib/qless/lua.rb +0 -25
metadata CHANGED
@@ -1,22 +1,22 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qless
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Dan Lecocq
9
9
  autorequire:
10
- bindir: bin
10
+ bindir: exe
11
11
  cert_chain: []
12
- date: 2012-11-30 00:00:00.000000000 Z
12
+ date: 2013-05-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: redis
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
- - - ~>
19
+ - - ! '>='
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.2'
22
22
  type: :runtime
@@ -24,41 +24,41 @@ dependencies:
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  none: false
26
26
  requirements:
27
- - - ~>
27
+ - - ! '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '2.2'
30
30
  - !ruby/object:Gem::Dependency
31
- name: sinatra
31
+ name: vegas
32
32
  requirement: !ruby/object:Gem::Requirement
33
33
  none: false
34
34
  requirements:
35
35
  - - ~>
36
36
  - !ruby/object:Gem::Version
37
- version: 1.3.2
38
- type: :development
37
+ version: 0.1.11
38
+ type: :runtime
39
39
  prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  none: false
42
42
  requirements:
43
43
  - - ~>
44
44
  - !ruby/object:Gem::Version
45
- version: 1.3.2
45
+ version: 0.1.11
46
46
  - !ruby/object:Gem::Dependency
47
- name: vegas
47
+ name: sinatra
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  none: false
50
50
  requirements:
51
51
  - - ~>
52
52
  - !ruby/object:Gem::Version
53
- version: 0.1.11
54
- type: :development
53
+ version: 1.3.2
54
+ type: :runtime
55
55
  prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  none: false
58
58
  requirements:
59
59
  - - ~>
60
60
  - !ruby/object:Gem::Version
61
- version: 0.1.11
61
+ version: 1.3.2
62
62
  - !ruby/object:Gem::Dependency
63
63
  name: rspec
64
64
  requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
66
66
  requirements:
67
67
  - - ~>
68
68
  - !ruby/object:Gem::Version
69
- version: 2.9.0
69
+ version: '2.12'
70
70
  type: :development
71
71
  prerelease: false
72
72
  version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
74
74
  requirements:
75
75
  - - ~>
76
76
  - !ruby/object:Gem::Version
77
- version: 2.9.0
77
+ version: '2.12'
78
78
  - !ruby/object:Gem::Dependency
79
79
  name: rspec-fire
80
80
  requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
82
82
  requirements:
83
83
  - - ~>
84
84
  - !ruby/object:Gem::Version
85
- version: '0.4'
85
+ version: '1.1'
86
86
  type: :development
87
87
  prerelease: false
88
88
  version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
90
90
  requirements:
91
91
  - - ~>
92
92
  - !ruby/object:Gem::Version
93
- version: '0.4'
93
+ version: '1.1'
94
94
  - !ruby/object:Gem::Dependency
95
95
  name: rake
96
96
  requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
98
98
  requirements:
99
99
  - - ~>
100
100
  - !ruby/object:Gem::Version
101
- version: 0.9.2.2
101
+ version: '10.0'
102
102
  type: :development
103
103
  prerelease: false
104
104
  version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
106
106
  requirements:
107
107
  - - ~>
108
108
  - !ruby/object:Gem::Version
109
- version: 0.9.2.2
109
+ version: '10.0'
110
110
  - !ruby/object:Gem::Dependency
111
111
  name: capybara
112
112
  requirement: !ruby/object:Gem::Requirement
@@ -171,6 +171,22 @@ dependencies:
171
171
  - - ~>
172
172
  - !ruby/object:Gem::Version
173
173
  version: 0.6.2
174
+ - !ruby/object:Gem::Dependency
175
+ name: sentry-raven
176
+ requirement: !ruby/object:Gem::Requirement
177
+ none: false
178
+ requirements:
179
+ - - ~>
180
+ - !ruby/object:Gem::Version
181
+ version: '0.4'
182
+ type: :development
183
+ prerelease: false
184
+ version_requirements: !ruby/object:Gem::Requirement
185
+ none: false
186
+ requirements:
187
+ - - ~>
188
+ - !ruby/object:Gem::Version
189
+ version: '0.4'
174
190
  description: ! "\n `qless` is meant to be a performant alternative to other queueing\n
175
191
  \ systems, with statistics collection, a browser interface, and \n strong guarantees
176
192
  about job losses.\n \n It's written as a collection of Lua scipts that are
@@ -194,37 +210,45 @@ files:
194
210
  - lib/qless/job.rb
195
211
  - lib/qless/job_reservers/ordered.rb
196
212
  - lib/qless/job_reservers/round_robin.rb
197
- - lib/qless/lua.rb
213
+ - lib/qless/job_reservers/shuffled_round_robin.rb
214
+ - lib/qless/lua_script.rb
215
+ - lib/qless/middleware/redis_reconnect.rb
216
+ - lib/qless/middleware/retry_exceptions.rb
217
+ - lib/qless/middleware/sentry.rb
198
218
  - lib/qless/queue.rb
199
219
  - lib/qless/server.rb
220
+ - lib/qless/subscriber.rb
200
221
  - lib/qless/tasks.rb
201
222
  - lib/qless/version.rb
223
+ - lib/qless/wait_until.rb
202
224
  - lib/qless/worker.rb
203
225
  - lib/qless.rb
204
226
  - lib/qless/qless-core/cancel.lua
205
227
  - lib/qless/qless-core/complete.lua
206
228
  - lib/qless/qless-core/config.lua
207
229
  - lib/qless/qless-core/depends.lua
230
+ - lib/qless/qless-core/deregister_workers.lua
208
231
  - lib/qless/qless-core/fail.lua
209
232
  - lib/qless/qless-core/failed.lua
210
233
  - lib/qless/qless-core/get.lua
211
234
  - lib/qless/qless-core/heartbeat.lua
212
235
  - lib/qless/qless-core/jobs.lua
236
+ - lib/qless/qless-core/pause.lua
213
237
  - lib/qless/qless-core/peek.lua
214
238
  - lib/qless/qless-core/pop.lua
215
239
  - lib/qless/qless-core/priority.lua
216
240
  - lib/qless/qless-core/put.lua
241
+ - lib/qless/qless-core/qless-lib.lua
242
+ - lib/qless/qless-core/qless.lua
217
243
  - lib/qless/qless-core/queues.lua
218
244
  - lib/qless/qless-core/recur.lua
219
245
  - lib/qless/qless-core/retry.lua
220
246
  - lib/qless/qless-core/stats.lua
221
247
  - lib/qless/qless-core/tag.lua
222
248
  - lib/qless/qless-core/track.lua
249
+ - lib/qless/qless-core/unfail.lua
250
+ - lib/qless/qless-core/unpause.lua
223
251
  - lib/qless/qless-core/workers.lua
224
- - bin/install_phantomjs
225
- - bin/qless-campfire
226
- - bin/qless-growl
227
- - bin/qless-web
228
252
  - lib/qless/server/static/css/bootstrap-responsive.css
229
253
  - lib/qless/server/static/css/bootstrap-responsive.min.css
230
254
  - lib/qless/server/static/css/bootstrap.css
@@ -272,6 +296,7 @@ files:
272
296
  - lib/qless/server/views/track.erb
273
297
  - lib/qless/server/views/worker.erb
274
298
  - lib/qless/server/views/workers.erb
299
+ - exe/qless-web
275
300
  homepage: http://github.com/seomoz/qless
276
301
  licenses: []
277
302
  post_install_message:
@@ -284,18 +309,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
284
309
  - - ! '>='
285
310
  - !ruby/object:Gem::Version
286
311
  version: '0'
287
- segments:
288
- - 0
289
- hash: -4094860363148682256
290
312
  required_rubygems_version: !ruby/object:Gem::Requirement
291
313
  none: false
292
314
  requirements:
293
315
  - - ! '>='
294
316
  - !ruby/object:Gem::Version
295
317
  version: '0'
296
- segments:
297
- - 0
298
- hash: -4094860363148682256
299
318
  requirements: []
300
319
  rubyforge_project: qless
301
320
  rubygems_version: 1.8.24
@@ -1,7 +0,0 @@
1
- #!/bin/bash
2
-
3
- version=phantomjs-1.7.0-linux-i686
4
- wget http://phantomjs.googlecode.com/files/$version.tar.bz2
5
- tar xjf $version.tar.bz2
6
- mv $version phantomjs
7
-
@@ -1,106 +0,0 @@
1
- #! /usr/bin/env ruby
2
-
3
- $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
4
-
5
- require 'qless'
6
- require 'tinder'
7
- require 'micro-optparse'
8
-
9
- @options = Parser.new do |p|
10
- p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
11
- p.option :subdomain, 'campfire subdomain' , :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
12
- p.option :token , 'campfire token for bot', :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
13
- p.option :room , 'campfire room to talk in (defaults to first room)', :default => ''
14
- p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
15
- p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
16
- end.process!
17
-
18
- # Get a campfire object, and room
19
- puts 'Connecting to campfire...'
20
- @campfire = Tinder::Campfire.new @options[:subdomain], :token => @options[:token]
21
- if not @options[:room]
22
- @room = @campfire.rooms.first
23
- else
24
- @room = @campfire.find_room_by_name(@options[:room])
25
- end
26
-
27
- # And now qless stuff
28
- puts 'Connecting to qless...'
29
- @client = Qless::Client.new(:host => @options[:host])
30
-
31
- def speak(message)
32
- @room.speak message
33
- puts "Said '#{message}'"
34
- end
35
-
36
- def paste(message)
37
- @room.paste message
38
- puts "Pasted '#{message}'"
39
- end
40
-
41
- def event(jid, &block)
42
- job = @client.jobs[jid]
43
- if job.nil?
44
- speak("#{jid[0...8]} #{yield job} | http://#{@options[:host]}/jobs/#{jid}]")
45
- else
46
- speak("#{jid[0...8]} #{yield job} [#{job.tags.join(', ')}] | http://#{@options[:host]}/jobs/#{jid}")
47
- end
48
- end
49
-
50
- @client.events.listen do |on|
51
- on.canceled { |jid| event(jid) { |job| 'canceled' } }
52
- on.stalled { |jid| event(jid) { |job| 'stalled' } }
53
- on.track { |jid| event(jid) { |job| 'is being tracked' } }
54
- on.untrack { |jid| event(jid) { |job| 'not longer tracked' } }
55
-
56
- on.completed do |jid|
57
- event(jid) do |job|
58
- if job
59
- if job.queue_name.empty?
60
- 'completed'
61
- else
62
- "advanced to #{job.queue_name}"
63
- end
64
- else
65
- 'completed'
66
- end
67
- end
68
- end
69
-
70
- on.popped do |jid|
71
- event(jid) do |job|
72
- if job
73
- "popped by #{job.worker_name}"
74
- else
75
- 'popped'
76
- end
77
- end
78
- end
79
-
80
- on.put do |jid|
81
- event(jid) do |job|
82
- if job
83
- "moved to #{job.queue_name}"
84
- else
85
- 'moved'
86
- end
87
- end
88
- end
89
-
90
- on.failed do |jid|
91
- job = @client.jobs[jid]
92
- if job
93
- speak("#{jid} failed by #{job.failure['worker']} | http://#{@options[:web]}/jobs/#{jid}")
94
- paste job.failure['message']
95
- else
96
- speak("#{jid} failed")
97
- end
98
- end
99
-
100
- puts 'Listening...'
101
-
102
- Signal.trap("INT") do
103
- puts 'Exiting'
104
- Process.exit(0)
105
- end
106
- end
@@ -1,99 +0,0 @@
1
- #! /usr/bin/env ruby
2
-
3
- $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
4
-
5
- require 'qless'
6
- require 'ruby-growl'
7
- require 'micro-optparse'
8
-
9
- @options = Parser.new do |p|
10
- p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
11
- p.option :growl , 'host for the growl daemon', :default => 'localhost'
12
- p.option :app , 'application name for notifications', :default => 'qless'
13
- p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
14
- p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
15
- end.process!
16
-
17
- # Connect to growl
18
- puts 'Connecting to growl...'
19
- @growl = Growl::GNTP.new @options[:growl], @options[:app]
20
-
21
- # Qless client
22
- puts 'Connecting to qless...'
23
- @client = Qless::Client.new(:host => @options[:host])
24
-
25
- %w{canceled completed failed popped stalled put track untrack}.each do |t|
26
- @growl.add_notification(t)
27
- end
28
- @growl.register
29
-
30
- def notify(jid, event, &block)
31
- job = @client.jobs[jid]
32
- if job.nil?
33
- message = yield job
34
- puts "#{jid} -> Notifying #{message}"
35
- @growl.notify(event, "#{jid[0...8]}", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
36
- else
37
- message = yield job
38
- puts "#{jid} -> Notifying #{message}"
39
- @growl.notify(event, "#{jid[0...8]} [#{job.tags.join(', ')}]", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
40
- end
41
- end
42
-
43
- @client.events.listen do |on|
44
- on.canceled { |jid| notify(jid, 'canceled') { |job| 'canceled' } }
45
- on.stalled { |jid| notify(jid, 'canceled') { |job| 'stalled' } }
46
- on.track { |jid| notify(jid, 'canceled') { |job| 'is being tracked' } }
47
- on.untrack { |jid| notify(jid, 'canceled') { |job| 'no longer tracked' } }
48
-
49
- on.completed do |jid|
50
- notify(jid, 'completed') do |job|
51
- if job
52
- if job.queue_name.empty?
53
- "completed"
54
- else
55
- "advanced to #{job.queue_name}"
56
- end
57
- else
58
- "completed"
59
- end
60
- end
61
- end
62
-
63
- on.failed do |jid|
64
- notify(jid, 'failed') do |job|
65
- if job
66
- "failed by #{job.failure['worker']}"
67
- else
68
- "failed"
69
- end
70
- end
71
- end
72
-
73
- on.popped do |jid|
74
- notify(jid, 'popped') do |job|
75
- if job
76
- "popped by #{job.worker_name}"
77
- else
78
- "popped"
79
- end
80
- end
81
- end
82
-
83
- on.put do |jid|
84
- notify(jid, 'put') do |job|
85
- if job
86
- "moved to #{job.queue_name}"
87
- else
88
- "moved"
89
- end
90
- end
91
- end
92
-
93
- puts 'Listening...'
94
-
95
- Signal.trap("INT") do
96
- puts 'Exiting'
97
- Process.exit(0)
98
- end
99
- end
@@ -1,25 +0,0 @@
1
- module Qless
2
- class Lua
3
- LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
4
-
5
- def initialize(name, redis)
6
- @sha = nil
7
- @name = name
8
- @redis = redis
9
- reload()
10
- end
11
-
12
- def reload()
13
- @sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
14
- end
15
-
16
- def call(keys, args)
17
- begin
18
- return @redis.evalsha(@sha, keys.length, *(keys + args))
19
- rescue
20
- reload
21
- return @redis.evalsha(@sha, keys.length, *(keys + args))
22
- end
23
- end
24
- end
25
- end