qless 0.9.2 → 0.9.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. data/Gemfile +2 -0
  2. data/README.md +42 -3
  3. data/Rakefile +26 -2
  4. data/{bin → exe}/qless-web +3 -2
  5. data/lib/qless.rb +55 -28
  6. data/lib/qless/config.rb +1 -3
  7. data/lib/qless/job.rb +127 -22
  8. data/lib/qless/job_reservers/round_robin.rb +3 -1
  9. data/lib/qless/job_reservers/shuffled_round_robin.rb +14 -0
  10. data/lib/qless/lua_script.rb +42 -0
  11. data/lib/qless/middleware/redis_reconnect.rb +24 -0
  12. data/lib/qless/middleware/retry_exceptions.rb +43 -0
  13. data/lib/qless/middleware/sentry.rb +70 -0
  14. data/lib/qless/qless-core/cancel.lua +89 -59
  15. data/lib/qless/qless-core/complete.lua +16 -1
  16. data/lib/qless/qless-core/config.lua +12 -0
  17. data/lib/qless/qless-core/deregister_workers.lua +12 -0
  18. data/lib/qless/qless-core/fail.lua +24 -14
  19. data/lib/qless/qless-core/heartbeat.lua +2 -1
  20. data/lib/qless/qless-core/pause.lua +18 -0
  21. data/lib/qless/qless-core/pop.lua +24 -3
  22. data/lib/qless/qless-core/put.lua +14 -1
  23. data/lib/qless/qless-core/qless-lib.lua +2354 -0
  24. data/lib/qless/qless-core/qless.lua +1862 -0
  25. data/lib/qless/qless-core/retry.lua +1 -1
  26. data/lib/qless/qless-core/unfail.lua +54 -0
  27. data/lib/qless/qless-core/unpause.lua +12 -0
  28. data/lib/qless/queue.rb +45 -21
  29. data/lib/qless/server.rb +38 -39
  30. data/lib/qless/server/static/css/docs.css +21 -1
  31. data/lib/qless/server/views/_job.erb +5 -5
  32. data/lib/qless/server/views/overview.erb +14 -9
  33. data/lib/qless/subscriber.rb +48 -0
  34. data/lib/qless/version.rb +1 -1
  35. data/lib/qless/wait_until.rb +19 -0
  36. data/lib/qless/worker.rb +243 -33
  37. metadata +49 -30
  38. data/bin/install_phantomjs +0 -7
  39. data/bin/qless-campfire +0 -106
  40. data/bin/qless-growl +0 -99
  41. data/lib/qless/lua.rb +0 -25
metadata CHANGED
@@ -1,22 +1,22 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qless
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.9.2
4
+ version: 0.9.3
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
8
8
  - Dan Lecocq
9
9
  autorequire:
10
- bindir: bin
10
+ bindir: exe
11
11
  cert_chain: []
12
- date: 2012-11-30 00:00:00.000000000 Z
12
+ date: 2013-05-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: redis
16
16
  requirement: !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
- - - ~>
19
+ - - ! '>='
20
20
  - !ruby/object:Gem::Version
21
21
  version: '2.2'
22
22
  type: :runtime
@@ -24,41 +24,41 @@ dependencies:
24
24
  version_requirements: !ruby/object:Gem::Requirement
25
25
  none: false
26
26
  requirements:
27
- - - ~>
27
+ - - ! '>='
28
28
  - !ruby/object:Gem::Version
29
29
  version: '2.2'
30
30
  - !ruby/object:Gem::Dependency
31
- name: sinatra
31
+ name: vegas
32
32
  requirement: !ruby/object:Gem::Requirement
33
33
  none: false
34
34
  requirements:
35
35
  - - ~>
36
36
  - !ruby/object:Gem::Version
37
- version: 1.3.2
38
- type: :development
37
+ version: 0.1.11
38
+ type: :runtime
39
39
  prerelease: false
40
40
  version_requirements: !ruby/object:Gem::Requirement
41
41
  none: false
42
42
  requirements:
43
43
  - - ~>
44
44
  - !ruby/object:Gem::Version
45
- version: 1.3.2
45
+ version: 0.1.11
46
46
  - !ruby/object:Gem::Dependency
47
- name: vegas
47
+ name: sinatra
48
48
  requirement: !ruby/object:Gem::Requirement
49
49
  none: false
50
50
  requirements:
51
51
  - - ~>
52
52
  - !ruby/object:Gem::Version
53
- version: 0.1.11
54
- type: :development
53
+ version: 1.3.2
54
+ type: :runtime
55
55
  prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  none: false
58
58
  requirements:
59
59
  - - ~>
60
60
  - !ruby/object:Gem::Version
61
- version: 0.1.11
61
+ version: 1.3.2
62
62
  - !ruby/object:Gem::Dependency
63
63
  name: rspec
64
64
  requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
66
66
  requirements:
67
67
  - - ~>
68
68
  - !ruby/object:Gem::Version
69
- version: 2.9.0
69
+ version: '2.12'
70
70
  type: :development
71
71
  prerelease: false
72
72
  version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
74
74
  requirements:
75
75
  - - ~>
76
76
  - !ruby/object:Gem::Version
77
- version: 2.9.0
77
+ version: '2.12'
78
78
  - !ruby/object:Gem::Dependency
79
79
  name: rspec-fire
80
80
  requirement: !ruby/object:Gem::Requirement
@@ -82,7 +82,7 @@ dependencies:
82
82
  requirements:
83
83
  - - ~>
84
84
  - !ruby/object:Gem::Version
85
- version: '0.4'
85
+ version: '1.1'
86
86
  type: :development
87
87
  prerelease: false
88
88
  version_requirements: !ruby/object:Gem::Requirement
@@ -90,7 +90,7 @@ dependencies:
90
90
  requirements:
91
91
  - - ~>
92
92
  - !ruby/object:Gem::Version
93
- version: '0.4'
93
+ version: '1.1'
94
94
  - !ruby/object:Gem::Dependency
95
95
  name: rake
96
96
  requirement: !ruby/object:Gem::Requirement
@@ -98,7 +98,7 @@ dependencies:
98
98
  requirements:
99
99
  - - ~>
100
100
  - !ruby/object:Gem::Version
101
- version: 0.9.2.2
101
+ version: '10.0'
102
102
  type: :development
103
103
  prerelease: false
104
104
  version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
106
106
  requirements:
107
107
  - - ~>
108
108
  - !ruby/object:Gem::Version
109
- version: 0.9.2.2
109
+ version: '10.0'
110
110
  - !ruby/object:Gem::Dependency
111
111
  name: capybara
112
112
  requirement: !ruby/object:Gem::Requirement
@@ -171,6 +171,22 @@ dependencies:
171
171
  - - ~>
172
172
  - !ruby/object:Gem::Version
173
173
  version: 0.6.2
174
+ - !ruby/object:Gem::Dependency
175
+ name: sentry-raven
176
+ requirement: !ruby/object:Gem::Requirement
177
+ none: false
178
+ requirements:
179
+ - - ~>
180
+ - !ruby/object:Gem::Version
181
+ version: '0.4'
182
+ type: :development
183
+ prerelease: false
184
+ version_requirements: !ruby/object:Gem::Requirement
185
+ none: false
186
+ requirements:
187
+ - - ~>
188
+ - !ruby/object:Gem::Version
189
+ version: '0.4'
174
190
  description: ! "\n `qless` is meant to be a performant alternative to other queueing\n
175
191
  \ systems, with statistics collection, a browser interface, and \n strong guarantees
176
192
  about job losses.\n \n It's written as a collection of Lua scipts that are
@@ -194,37 +210,45 @@ files:
194
210
  - lib/qless/job.rb
195
211
  - lib/qless/job_reservers/ordered.rb
196
212
  - lib/qless/job_reservers/round_robin.rb
197
- - lib/qless/lua.rb
213
+ - lib/qless/job_reservers/shuffled_round_robin.rb
214
+ - lib/qless/lua_script.rb
215
+ - lib/qless/middleware/redis_reconnect.rb
216
+ - lib/qless/middleware/retry_exceptions.rb
217
+ - lib/qless/middleware/sentry.rb
198
218
  - lib/qless/queue.rb
199
219
  - lib/qless/server.rb
220
+ - lib/qless/subscriber.rb
200
221
  - lib/qless/tasks.rb
201
222
  - lib/qless/version.rb
223
+ - lib/qless/wait_until.rb
202
224
  - lib/qless/worker.rb
203
225
  - lib/qless.rb
204
226
  - lib/qless/qless-core/cancel.lua
205
227
  - lib/qless/qless-core/complete.lua
206
228
  - lib/qless/qless-core/config.lua
207
229
  - lib/qless/qless-core/depends.lua
230
+ - lib/qless/qless-core/deregister_workers.lua
208
231
  - lib/qless/qless-core/fail.lua
209
232
  - lib/qless/qless-core/failed.lua
210
233
  - lib/qless/qless-core/get.lua
211
234
  - lib/qless/qless-core/heartbeat.lua
212
235
  - lib/qless/qless-core/jobs.lua
236
+ - lib/qless/qless-core/pause.lua
213
237
  - lib/qless/qless-core/peek.lua
214
238
  - lib/qless/qless-core/pop.lua
215
239
  - lib/qless/qless-core/priority.lua
216
240
  - lib/qless/qless-core/put.lua
241
+ - lib/qless/qless-core/qless-lib.lua
242
+ - lib/qless/qless-core/qless.lua
217
243
  - lib/qless/qless-core/queues.lua
218
244
  - lib/qless/qless-core/recur.lua
219
245
  - lib/qless/qless-core/retry.lua
220
246
  - lib/qless/qless-core/stats.lua
221
247
  - lib/qless/qless-core/tag.lua
222
248
  - lib/qless/qless-core/track.lua
249
+ - lib/qless/qless-core/unfail.lua
250
+ - lib/qless/qless-core/unpause.lua
223
251
  - lib/qless/qless-core/workers.lua
224
- - bin/install_phantomjs
225
- - bin/qless-campfire
226
- - bin/qless-growl
227
- - bin/qless-web
228
252
  - lib/qless/server/static/css/bootstrap-responsive.css
229
253
  - lib/qless/server/static/css/bootstrap-responsive.min.css
230
254
  - lib/qless/server/static/css/bootstrap.css
@@ -272,6 +296,7 @@ files:
272
296
  - lib/qless/server/views/track.erb
273
297
  - lib/qless/server/views/worker.erb
274
298
  - lib/qless/server/views/workers.erb
299
+ - exe/qless-web
275
300
  homepage: http://github.com/seomoz/qless
276
301
  licenses: []
277
302
  post_install_message:
@@ -284,18 +309,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
284
309
  - - ! '>='
285
310
  - !ruby/object:Gem::Version
286
311
  version: '0'
287
- segments:
288
- - 0
289
- hash: -4094860363148682256
290
312
  required_rubygems_version: !ruby/object:Gem::Requirement
291
313
  none: false
292
314
  requirements:
293
315
  - - ! '>='
294
316
  - !ruby/object:Gem::Version
295
317
  version: '0'
296
- segments:
297
- - 0
298
- hash: -4094860363148682256
299
318
  requirements: []
300
319
  rubyforge_project: qless
301
320
  rubygems_version: 1.8.24
@@ -1,7 +0,0 @@
1
- #!/bin/bash
2
-
3
- version=phantomjs-1.7.0-linux-i686
4
- wget http://phantomjs.googlecode.com/files/$version.tar.bz2
5
- tar xjf $version.tar.bz2
6
- mv $version phantomjs
7
-
@@ -1,106 +0,0 @@
1
- #! /usr/bin/env ruby
2
-
3
- $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
4
-
5
- require 'qless'
6
- require 'tinder'
7
- require 'micro-optparse'
8
-
9
- @options = Parser.new do |p|
10
- p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
11
- p.option :subdomain, 'campfire subdomain' , :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
12
- p.option :token , 'campfire token for bot', :default => '', :value_satisfies => lambda { |subdomain| subdomain.is_a?(String) }
13
- p.option :room , 'campfire room to talk in (defaults to first room)', :default => ''
14
- p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
15
- p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
16
- end.process!
17
-
18
- # Get a campfire object, and room
19
- puts 'Connecting to campfire...'
20
- @campfire = Tinder::Campfire.new @options[:subdomain], :token => @options[:token]
21
- if not @options[:room]
22
- @room = @campfire.rooms.first
23
- else
24
- @room = @campfire.find_room_by_name(@options[:room])
25
- end
26
-
27
- # And now qless stuff
28
- puts 'Connecting to qless...'
29
- @client = Qless::Client.new(:host => @options[:host])
30
-
31
- def speak(message)
32
- @room.speak message
33
- puts "Said '#{message}'"
34
- end
35
-
36
- def paste(message)
37
- @room.paste message
38
- puts "Pasted '#{message}'"
39
- end
40
-
41
- def event(jid, &block)
42
- job = @client.jobs[jid]
43
- if job.nil?
44
- speak("#{jid[0...8]} #{yield job} | http://#{@options[:host]}/jobs/#{jid}]")
45
- else
46
- speak("#{jid[0...8]} #{yield job} [#{job.tags.join(', ')}] | http://#{@options[:host]}/jobs/#{jid}")
47
- end
48
- end
49
-
50
- @client.events.listen do |on|
51
- on.canceled { |jid| event(jid) { |job| 'canceled' } }
52
- on.stalled { |jid| event(jid) { |job| 'stalled' } }
53
- on.track { |jid| event(jid) { |job| 'is being tracked' } }
54
- on.untrack { |jid| event(jid) { |job| 'not longer tracked' } }
55
-
56
- on.completed do |jid|
57
- event(jid) do |job|
58
- if job
59
- if job.queue_name.empty?
60
- 'completed'
61
- else
62
- "advanced to #{job.queue_name}"
63
- end
64
- else
65
- 'completed'
66
- end
67
- end
68
- end
69
-
70
- on.popped do |jid|
71
- event(jid) do |job|
72
- if job
73
- "popped by #{job.worker_name}"
74
- else
75
- 'popped'
76
- end
77
- end
78
- end
79
-
80
- on.put do |jid|
81
- event(jid) do |job|
82
- if job
83
- "moved to #{job.queue_name}"
84
- else
85
- 'moved'
86
- end
87
- end
88
- end
89
-
90
- on.failed do |jid|
91
- job = @client.jobs[jid]
92
- if job
93
- speak("#{jid} failed by #{job.failure['worker']} | http://#{@options[:web]}/jobs/#{jid}")
94
- paste job.failure['message']
95
- else
96
- speak("#{jid} failed")
97
- end
98
- end
99
-
100
- puts 'Listening...'
101
-
102
- Signal.trap("INT") do
103
- puts 'Exiting'
104
- Process.exit(0)
105
- end
106
- end
@@ -1,99 +0,0 @@
1
- #! /usr/bin/env ruby
2
-
3
- $LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
4
-
5
- require 'qless'
6
- require 'ruby-growl'
7
- require 'micro-optparse'
8
-
9
- @options = Parser.new do |p|
10
- p.banner = 'This agent lets you get campfire notifications for the progress of tracked jobs'
11
- p.option :growl , 'host for the growl daemon', :default => 'localhost'
12
- p.option :app , 'application name for notifications', :default => 'qless'
13
- p.option :host , 'host:port for your qless redis instance', :default => 'localhost:6379'
14
- p.option :web , 'host:port for your qless web ui', :default => 'localhost:5678'
15
- end.process!
16
-
17
- # Connect to growl
18
- puts 'Connecting to growl...'
19
- @growl = Growl::GNTP.new @options[:growl], @options[:app]
20
-
21
- # Qless client
22
- puts 'Connecting to qless...'
23
- @client = Qless::Client.new(:host => @options[:host])
24
-
25
- %w{canceled completed failed popped stalled put track untrack}.each do |t|
26
- @growl.add_notification(t)
27
- end
28
- @growl.register
29
-
30
- def notify(jid, event, &block)
31
- job = @client.jobs[jid]
32
- if job.nil?
33
- message = yield job
34
- puts "#{jid} -> Notifying #{message}"
35
- @growl.notify(event, "#{jid[0...8]}", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
36
- else
37
- message = yield job
38
- puts "#{jid} -> Notifying #{message}"
39
- @growl.notify(event, "#{jid[0...8]} [#{job.tags.join(', ')}]", "#{message}", 0, false, jid, "http://#{@options[:web]}/jobs/#{jid}")
40
- end
41
- end
42
-
43
- @client.events.listen do |on|
44
- on.canceled { |jid| notify(jid, 'canceled') { |job| 'canceled' } }
45
- on.stalled { |jid| notify(jid, 'canceled') { |job| 'stalled' } }
46
- on.track { |jid| notify(jid, 'canceled') { |job| 'is being tracked' } }
47
- on.untrack { |jid| notify(jid, 'canceled') { |job| 'no longer tracked' } }
48
-
49
- on.completed do |jid|
50
- notify(jid, 'completed') do |job|
51
- if job
52
- if job.queue_name.empty?
53
- "completed"
54
- else
55
- "advanced to #{job.queue_name}"
56
- end
57
- else
58
- "completed"
59
- end
60
- end
61
- end
62
-
63
- on.failed do |jid|
64
- notify(jid, 'failed') do |job|
65
- if job
66
- "failed by #{job.failure['worker']}"
67
- else
68
- "failed"
69
- end
70
- end
71
- end
72
-
73
- on.popped do |jid|
74
- notify(jid, 'popped') do |job|
75
- if job
76
- "popped by #{job.worker_name}"
77
- else
78
- "popped"
79
- end
80
- end
81
- end
82
-
83
- on.put do |jid|
84
- notify(jid, 'put') do |job|
85
- if job
86
- "moved to #{job.queue_name}"
87
- else
88
- "moved"
89
- end
90
- end
91
- end
92
-
93
- puts 'Listening...'
94
-
95
- Signal.trap("INT") do
96
- puts 'Exiting'
97
- Process.exit(0)
98
- end
99
- end
@@ -1,25 +0,0 @@
1
- module Qless
2
- class Lua
3
- LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
4
-
5
- def initialize(name, redis)
6
- @sha = nil
7
- @name = name
8
- @redis = redis
9
- reload()
10
- end
11
-
12
- def reload()
13
- @sha = @redis.script(:load, File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua")))
14
- end
15
-
16
- def call(keys, args)
17
- begin
18
- return @redis.evalsha(@sha, keys.length, *(keys + args))
19
- rescue
20
- reload
21
- return @redis.evalsha(@sha, keys.length, *(keys + args))
22
- end
23
- end
24
- end
25
- end