qless 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +9 -3
- data/README.md +70 -25
- data/Rakefile +125 -9
- data/exe/install_phantomjs +21 -0
- data/lib/qless.rb +115 -76
- data/lib/qless/config.rb +11 -9
- data/lib/qless/failure_formatter.rb +43 -0
- data/lib/qless/job.rb +201 -102
- data/lib/qless/job_reservers/ordered.rb +7 -1
- data/lib/qless/job_reservers/round_robin.rb +16 -6
- data/lib/qless/job_reservers/shuffled_round_robin.rb +9 -2
- data/lib/qless/lua/qless-lib.lua +2463 -0
- data/lib/qless/lua/qless.lua +2012 -0
- data/lib/qless/lua_script.rb +63 -12
- data/lib/qless/middleware/memory_usage_monitor.rb +62 -0
- data/lib/qless/middleware/metriks.rb +45 -0
- data/lib/qless/middleware/redis_reconnect.rb +6 -3
- data/lib/qless/middleware/requeue_exceptions.rb +94 -0
- data/lib/qless/middleware/retry_exceptions.rb +38 -9
- data/lib/qless/middleware/sentry.rb +3 -7
- data/lib/qless/middleware/timeout.rb +64 -0
- data/lib/qless/queue.rb +90 -55
- data/lib/qless/server.rb +177 -130
- data/lib/qless/server/views/_job.erb +33 -15
- data/lib/qless/server/views/completed.erb +11 -0
- data/lib/qless/server/views/layout.erb +70 -11
- data/lib/qless/server/views/overview.erb +93 -53
- data/lib/qless/server/views/queue.erb +9 -8
- data/lib/qless/server/views/queues.erb +18 -1
- data/lib/qless/subscriber.rb +37 -22
- data/lib/qless/tasks.rb +5 -10
- data/lib/qless/test_helpers/worker_helpers.rb +55 -0
- data/lib/qless/version.rb +3 -1
- data/lib/qless/worker.rb +4 -413
- data/lib/qless/worker/base.rb +247 -0
- data/lib/qless/worker/forking.rb +245 -0
- data/lib/qless/worker/serial.rb +41 -0
- metadata +135 -52
- data/lib/qless/qless-core/cancel.lua +0 -101
- data/lib/qless/qless-core/complete.lua +0 -233
- data/lib/qless/qless-core/config.lua +0 -56
- data/lib/qless/qless-core/depends.lua +0 -65
- data/lib/qless/qless-core/deregister_workers.lua +0 -12
- data/lib/qless/qless-core/fail.lua +0 -117
- data/lib/qless/qless-core/failed.lua +0 -83
- data/lib/qless/qless-core/get.lua +0 -37
- data/lib/qless/qless-core/heartbeat.lua +0 -51
- data/lib/qless/qless-core/jobs.lua +0 -41
- data/lib/qless/qless-core/pause.lua +0 -18
- data/lib/qless/qless-core/peek.lua +0 -165
- data/lib/qless/qless-core/pop.lua +0 -314
- data/lib/qless/qless-core/priority.lua +0 -32
- data/lib/qless/qless-core/put.lua +0 -169
- data/lib/qless/qless-core/qless-lib.lua +0 -2354
- data/lib/qless/qless-core/qless.lua +0 -1862
- data/lib/qless/qless-core/queues.lua +0 -58
- data/lib/qless/qless-core/recur.lua +0 -190
- data/lib/qless/qless-core/retry.lua +0 -73
- data/lib/qless/qless-core/stats.lua +0 -92
- data/lib/qless/qless-core/tag.lua +0 -100
- data/lib/qless/qless-core/track.lua +0 -79
- data/lib/qless/qless-core/unfail.lua +0 -54
- data/lib/qless/qless-core/unpause.lua +0 -12
- data/lib/qless/qless-core/workers.lua +0 -69
- data/lib/qless/wait_until.rb +0 -19
data/Gemfile
CHANGED
@@ -3,8 +3,14 @@ source "http://rubygems.org"
|
|
3
3
|
# Specify your gem's dependencies in qless.gemspec
|
4
4
|
gemspec
|
5
5
|
|
6
|
-
group :
|
7
|
-
gem 'debugger', :platform => :
|
6
|
+
group :extras do
|
7
|
+
gem 'debugger', :platform => :mri_19
|
8
8
|
end
|
9
9
|
|
10
|
-
|
10
|
+
group :development do
|
11
|
+
gem 'byebug', :platforms => [:ruby_20, :ruby_21]
|
12
|
+
gem 'pry'
|
13
|
+
gem 'pry-byebug', :platforms => [:ruby_20, :ruby_21]
|
14
|
+
gem 'pry-stack_explorer'
|
15
|
+
gem 'cane', :platforms => [:ruby_20, :ruby_21]
|
16
|
+
end
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
qless
|
1
|
+
qless [](https://travis-ci.org/seomoz/qless)
|
2
2
|
=====
|
3
3
|
|
4
4
|
Qless is a powerful `Redis`-based job queueing system inspired by
|
@@ -120,7 +120,7 @@ job.original_retries # => the number of times the job is allowed to be retried
|
|
120
120
|
job.retries_left # => the number of retries left
|
121
121
|
|
122
122
|
# You can also change the job in various ways:
|
123
|
-
job.
|
123
|
+
job.requeue("some_other_queue") # move it to a new queue
|
124
124
|
job.cancel # cancel the job
|
125
125
|
job.tag("foo") # add a tag
|
126
126
|
job.untag("foo") # remove a tag
|
@@ -162,31 +162,41 @@ it is empty, before trying to pop job off the second queue. The
|
|
162
162
|
round-robin reserver will pop a job off the first queue, then the second
|
163
163
|
queue, and so on. You could also easily implement your own.
|
164
164
|
|
165
|
-
To start a worker,
|
166
|
-
|
165
|
+
To start a worker, write a bit of Ruby code that instantiates a
|
166
|
+
worker and runs it. You could write a rake task to do this, for
|
167
|
+
example:
|
167
168
|
|
168
169
|
``` ruby
|
169
|
-
require 'qless/tasks'
|
170
170
|
namespace :qless do
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
# The only required option is QUEUES; the
|
176
|
-
# rest have reasonable defaults.
|
177
|
-
ENV['REDIS_URL'] ||= 'redis://some-host:7000/3'
|
178
|
-
ENV['QUEUES'] ||= 'fizz,buzz'
|
179
|
-
ENV['JOB_RESERVER'] ||= 'Ordered'
|
180
|
-
ENV['INTERVAL'] ||= '10' # 10 seconds
|
181
|
-
ENV['VERBOSE'] ||= 'true'
|
182
|
-
end
|
183
|
-
end
|
184
|
-
```
|
171
|
+
desc "Run a Qless worker"
|
172
|
+
task :work do
|
173
|
+
# Load your application code. All job classes must be loaded.
|
174
|
+
require 'my_app/environment'
|
185
175
|
|
186
|
-
|
176
|
+
# Require the parts of qless you need
|
177
|
+
require 'qless'
|
178
|
+
require 'qless/job_reservers/ordered'
|
179
|
+
require 'qless/worker'
|
187
180
|
|
188
|
-
|
189
|
-
|
181
|
+
# Create a client
|
182
|
+
client = Qless::Client.new(:host => 'foo.bar.com', :port => 1234)
|
183
|
+
|
184
|
+
# Get the queues you use
|
185
|
+
queues = %w[ queue_1 queue_2 ].map do |name|
|
186
|
+
client.queues[name]
|
187
|
+
end
|
188
|
+
|
189
|
+
# Create a job reserver; different reservers use different
|
190
|
+
# strategies for which order jobs are popped off of queues
|
191
|
+
reserver = Qless::JobReservers::Ordered.new(queues)
|
192
|
+
|
193
|
+
# Create a forking worker that uses the given reserver to pop jobs.
|
194
|
+
worker = Qless::Workers::ForkingWorker.new(reserver)
|
195
|
+
|
196
|
+
# Start the worker!
|
197
|
+
worker.run
|
198
|
+
end
|
199
|
+
end
|
190
200
|
```
|
191
201
|
|
192
202
|
The following signals are supported in the parent process:
|
@@ -276,13 +286,13 @@ has a rack-based ruby web app, we recommend you mount Qless's web app
|
|
276
286
|
in it. Here's how you can do that with `Rack::Builder` in your `config.ru`:
|
277
287
|
|
278
288
|
``` ruby
|
279
|
-
|
289
|
+
client = Qless::Client.new(:host => "some-host", :port => 7000)
|
280
290
|
|
281
291
|
Rack::Builder.new do
|
282
292
|
use SomeMiddleware
|
283
293
|
|
284
294
|
map('/some-other-app') { run Apps::Something.new }
|
285
|
-
map('/qless') { run Qless::Server.new }
|
295
|
+
map('/qless') { run Qless::Server.new(client) }
|
286
296
|
end
|
287
297
|
```
|
288
298
|
|
@@ -506,7 +516,7 @@ progress periodically:
|
|
506
516
|
|
507
517
|
``` ruby
|
508
518
|
# Wait until we have 5 minutes left on the heartbeat, and if we find that
|
509
|
-
# we've lost our lock on a job, then
|
519
|
+
# we've lost our lock on a job, then honorably fall on our sword
|
510
520
|
if (job.ttl < 300) && !job.heartbeat
|
511
521
|
return / die / exit
|
512
522
|
end
|
@@ -608,3 +618,38 @@ options a normal job supports. See
|
|
608
618
|
[the source](https://github.com/seomoz/qless/blob/master/lib/qless/job.rb)
|
609
619
|
for a full list.
|
610
620
|
|
621
|
+
Contributing
|
622
|
+
============
|
623
|
+
|
624
|
+
To bootstrap an environment, first [have a redis](https://github.com/seomoz/qless/wiki/Bootstrapping-Qless#a-simple-redis-bootstrap).
|
625
|
+
|
626
|
+
Have `rvm` or `rbenv`. Then to install the dependencies:
|
627
|
+
|
628
|
+
```bash
|
629
|
+
rbenv install # rbenv only. Install bundler if you need it.
|
630
|
+
bundle install
|
631
|
+
./exe/install_phantomjs # Bring in phantomjs 1.7.0 for tests.
|
632
|
+
rbenv rehash # rbenv only
|
633
|
+
git submodule init
|
634
|
+
git submodule update
|
635
|
+
bundle exec rake core:build
|
636
|
+
```
|
637
|
+
|
638
|
+
To run the tests:
|
639
|
+
|
640
|
+
```
|
641
|
+
bundle exec rake spec
|
642
|
+
```
|
643
|
+
|
644
|
+
**The locally installed redis will be flushed before and after each test run.**
|
645
|
+
|
646
|
+
To change the redis instance used in tests, put the connection information into [`./spec/redis.config.yml`](https://github.com/seomoz/qless/blob/92904532aee82aaf1078957ccadfa6fcd27ae408/spec/spec_helper.rb#L26).
|
647
|
+
|
648
|
+
To contribute, fork the repo, use feature branches, run the tests and open PRs.
|
649
|
+
|
650
|
+
Mailing List
|
651
|
+
============
|
652
|
+
|
653
|
+
For questions and general Qless discussion, please join the [Qless
|
654
|
+
Mailing list](https://groups.google.com/forum/?fromgroups#!forum/qless).
|
655
|
+
|
data/Rakefile
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
#!/usr/bin/env rake
|
2
|
-
|
3
|
-
|
4
|
-
Bundler.setup
|
5
|
-
require "bundler/gem_tasks"
|
2
|
+
require 'bundler/gem_helper'
|
3
|
+
Bundler::GemHelper.install_tasks
|
6
4
|
|
7
5
|
require 'rspec/core/rake_task'
|
8
6
|
RSpec::Core::RakeTask.new(:spec) do |t|
|
@@ -11,8 +9,8 @@ RSpec::Core::RakeTask.new(:spec) do |t|
|
|
11
9
|
end
|
12
10
|
|
13
11
|
# TODO: bump this up as test coverage increases. It was 90.29 when I last updated it on 2012-05-21.
|
14
|
-
# On travis where we skip JS tests, it's at
|
15
|
-
min_coverage_threshold =
|
12
|
+
# On travis where we skip JS tests, it's at 90.0 on 2013-10-01
|
13
|
+
min_coverage_threshold = 85.0
|
16
14
|
desc "Checks the spec coverage and fails if it is less than #{min_coverage_threshold}%"
|
17
15
|
task :check_coverage do
|
18
16
|
percent = File.read("./coverage/coverage_percent.txt").to_f
|
@@ -23,17 +21,85 @@ task :check_coverage do
|
|
23
21
|
end
|
24
22
|
end
|
25
23
|
|
26
|
-
task default: [:spec, :check_coverage]
|
24
|
+
task default: [:spec, :check_coverage, :cane]
|
25
|
+
|
26
|
+
namespace :core do
|
27
|
+
qless_core_dir = "./lib/qless/qless-core"
|
28
|
+
|
29
|
+
desc "Builds the qless-core lua scripts"
|
30
|
+
task :build do
|
31
|
+
Dir.chdir(qless_core_dir) do
|
32
|
+
sh "make clean && make"
|
33
|
+
sh "cp qless.lua ../lua"
|
34
|
+
sh "cp qless-lib.lua ../lua"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
task :update_submodule do
|
39
|
+
Dir.chdir(qless_core_dir) do
|
40
|
+
sh "git checkout master"
|
41
|
+
sh "git pull --rebase"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
desc "Updates qless-core and rebuilds it"
|
46
|
+
task update: [:update_submodule, :build]
|
47
|
+
|
48
|
+
namespace :verify do
|
49
|
+
script_files = %w[ lib/qless/lua/qless.lua lib/qless/lua/qless-lib.lua ]
|
50
|
+
|
51
|
+
desc "Verifies the script has no uncommitted changes"
|
52
|
+
task :clean do
|
53
|
+
script_files.each do |file|
|
54
|
+
git_status = `git status -- #{file}`
|
55
|
+
unless /working directory clean/.match(git_status)
|
56
|
+
raise "#{file} is dirty: \n\n#{git_status}\n\n"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
desc "Verifies the script is current"
|
62
|
+
task :current do
|
63
|
+
require 'digest/md5'
|
64
|
+
our_md5s = script_files.map do |file|
|
65
|
+
Digest::MD5.hexdigest(File.read file)
|
66
|
+
end
|
67
|
+
|
68
|
+
canonical_md5s = Dir.chdir(qless_core_dir) do
|
69
|
+
sh "make clean && make"
|
70
|
+
script_files.map do |file|
|
71
|
+
Digest::MD5.hexdigest(File.read(File.basename file))
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
unless our_md5s == canonical_md5s
|
76
|
+
raise "The current scripts are out of date with qless-core"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
desc "Verifies the committed script is current"
|
82
|
+
task verify: %w[ verify:clean verify:current ]
|
83
|
+
end
|
84
|
+
|
85
|
+
desc "Starts a qless console"
|
86
|
+
task :console do
|
87
|
+
ENV['PUBLIC_SEQUEL_API'] = 'true'
|
88
|
+
ENV['NO_NEW_RELIC'] = 'true'
|
89
|
+
exec "bundle exec pry -r./conf/console"
|
90
|
+
end
|
27
91
|
|
28
92
|
require 'qless/tasks'
|
29
93
|
|
30
94
|
namespace :qless do
|
31
|
-
|
95
|
+
desc "Runs a test worker so you can send signals to it for testing"
|
96
|
+
task :run_test_worker do
|
32
97
|
require 'qless'
|
98
|
+
require 'qless/job_reservers/ordered'
|
99
|
+
require 'qless/worker'
|
33
100
|
queue = Qless::Client.new.queues["example"]
|
34
101
|
queue.client.redis.flushdb
|
35
102
|
|
36
|
-
ENV['QUEUES'] = queue.name
|
37
103
|
ENV['VVERBOSE'] = '1'
|
38
104
|
|
39
105
|
class ExampleJob
|
@@ -48,5 +114,55 @@ namespace :qless do
|
|
48
114
|
20.times do |i|
|
49
115
|
queue.put(ExampleJob, sleep: i)
|
50
116
|
end
|
117
|
+
|
118
|
+
reserver = Qless::JobReservers::Ordered.new([queue])
|
119
|
+
Qless::Workers::ForkingWorker.new(reserver, log_level: Logger::INFO).run
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
namespace :cane do
|
125
|
+
begin
|
126
|
+
require 'cane/rake_task'
|
127
|
+
|
128
|
+
libs = [
|
129
|
+
{ name: 'qless', dir: '.', root: '.' },
|
130
|
+
]
|
131
|
+
|
132
|
+
libs.each do |lib|
|
133
|
+
desc "Runs cane code quality checks for #{lib[:name]}"
|
134
|
+
Cane::RakeTask.new(lib[:name]) do |cane|
|
135
|
+
cane.no_doc = true
|
136
|
+
|
137
|
+
cane.abc_glob = "#{lib[:dir]}/{lib,spec}/**/*.rb"
|
138
|
+
cane.abc_max = 15
|
139
|
+
cane.abc_exclude = %w[
|
140
|
+
Middleware::(anon)#expect_job_to_timeout
|
141
|
+
Qless::Job#initialize
|
142
|
+
Qless::Middleware::RequeueExceptions#handle_exception
|
143
|
+
Qless::Middleware::Timeout#initialize
|
144
|
+
Qless::WorkerHelpers#run_jobs
|
145
|
+
Qless::Workers::BaseWorker#initialize
|
146
|
+
Qless::Workers::BaseWorker#register_signal_handlers
|
147
|
+
Qless::Workers::ForkingWorker#register_signal_handlers
|
148
|
+
Qless::Workers::SerialWorker#run
|
149
|
+
]
|
150
|
+
|
151
|
+
cane.style_glob = "#{lib[:dir]}/lib/**/*.rb"
|
152
|
+
cane.style_measure = 100
|
153
|
+
cane.style_exclude = %w[
|
154
|
+
]
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
desc "Runs cane code quality checks for all projects"
|
159
|
+
task all: libs.map { |l| l[:name] }
|
160
|
+
|
161
|
+
rescue LoadError
|
162
|
+
task :all do
|
163
|
+
puts "cane is not supported in ruby #{RUBY_VERSION}"
|
164
|
+
end
|
51
165
|
end
|
52
166
|
end
|
167
|
+
|
168
|
+
task cane: "cane:all"
|
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
os_name=$(uname -s)
|
4
|
+
phantomjs_version=1.7.0
|
5
|
+
|
6
|
+
if [[ "$os_name" == 'Darwin' ]]
|
7
|
+
then
|
8
|
+
# brew versions phantomjs
|
9
|
+
(cd /usr/local/Library && git checkout d37d922 Formula/phantomjs.rb)
|
10
|
+
brew install phantomjs
|
11
|
+
elif [[ "$os_name" == 'Linux' ]]
|
12
|
+
then
|
13
|
+
version=phantomjs-1.7.0-linux-x86_64
|
14
|
+
wget http://phantomjs.googlecode.com/files/$version.tar.bz2
|
15
|
+
tar xjf $version.tar.bz2
|
16
|
+
mv $version phantomjs
|
17
|
+
else
|
18
|
+
echo "Don't know how to install phantom js on $os_name. " \
|
19
|
+
"Your OS is bad! You should feel bad!" >&2
|
20
|
+
exit 1
|
21
|
+
fi
|
data/lib/qless.rb
CHANGED
@@ -1,28 +1,34 @@
|
|
1
|
-
|
2
|
-
require "redis"
|
3
|
-
require "json"
|
4
|
-
require "securerandom"
|
1
|
+
# Encoding: utf-8
|
5
2
|
|
3
|
+
require 'socket'
|
4
|
+
require 'redis'
|
5
|
+
require 'json'
|
6
|
+
require 'securerandom'
|
7
|
+
|
8
|
+
# The top level container for all things qless
|
6
9
|
module Qless
|
7
|
-
# Define our error base class before requiring the other
|
8
|
-
#
|
10
|
+
# Define our error base class before requiring the other files so they can
|
11
|
+
# define subclasses.
|
9
12
|
Error = Class.new(StandardError)
|
10
13
|
|
11
|
-
# to maintain backwards compatibility with v2.x of that gem we need this
|
12
|
-
#
|
13
|
-
#
|
14
|
+
# to maintain backwards compatibility with v2.x of that gem we need this
|
15
|
+
# constant because:
|
16
|
+
# - (lua.rb) the #evalsha method signature changed between v2.x and v3.x of
|
17
|
+
# the redis ruby gem
|
18
|
+
# - (worker.rb) in v3.x you have to reconnect to the redis server after
|
19
|
+
# forking the process
|
14
20
|
USING_LEGACY_REDIS_VERSION = ::Redis::VERSION.to_f < 3.0
|
15
21
|
end
|
16
22
|
|
17
|
-
require
|
18
|
-
require
|
19
|
-
require
|
20
|
-
require
|
21
|
-
require
|
23
|
+
require 'qless/version'
|
24
|
+
require 'qless/config'
|
25
|
+
require 'qless/queue'
|
26
|
+
require 'qless/job'
|
27
|
+
require 'qless/lua_script'
|
28
|
+
require 'qless/failure_formatter'
|
22
29
|
|
30
|
+
# The top level container for all things qless
|
23
31
|
module Qless
|
24
|
-
extend self
|
25
|
-
|
26
32
|
UnsupportedRedisVersionError = Class.new(Error)
|
27
33
|
|
28
34
|
def generate_jid
|
@@ -35,74 +41,93 @@ module Qless
|
|
35
41
|
end
|
36
42
|
end
|
37
43
|
|
38
|
-
|
39
|
-
|
40
|
-
@worker_name ||= [Socket.gethostname, Process.pid.to_s].join('-')
|
44
|
+
def failure_formatter
|
45
|
+
@failure_formatter ||= FailureFormatter.new
|
41
46
|
end
|
42
47
|
|
48
|
+
module_function :generate_jid, :stringify_hash_keys, :failure_formatter
|
49
|
+
|
50
|
+
# A class for interacting with jobs. Not meant to be instantiated directly,
|
51
|
+
# it's accessed through Client#jobs
|
43
52
|
class ClientJobs
|
44
53
|
def initialize(client)
|
45
54
|
@client = client
|
46
55
|
end
|
47
56
|
|
48
|
-
def complete(offset=0, count=25)
|
49
|
-
@client.
|
57
|
+
def complete(offset = 0, count = 25)
|
58
|
+
@client.call('jobs', 'complete', offset, count)
|
50
59
|
end
|
51
60
|
|
52
61
|
def tracked
|
53
|
-
results = JSON.parse(@client.
|
62
|
+
results = JSON.parse(@client.call('track'))
|
54
63
|
results['jobs'] = results['jobs'].map { |j| Job.new(@client, j) }
|
55
64
|
results
|
56
65
|
end
|
57
66
|
|
58
|
-
def tagged(tag, offset=0, count=25)
|
59
|
-
JSON.parse(@client.
|
67
|
+
def tagged(tag, offset = 0, count = 25)
|
68
|
+
results = JSON.parse(@client.call('tag', 'get', tag, offset, count))
|
69
|
+
# Should be an empty array instead of an empty hash
|
70
|
+
results['jobs'] = [] if results['jobs'] == {}
|
71
|
+
results
|
60
72
|
end
|
61
73
|
|
62
|
-
def failed(t=nil, start=0, limit=25)
|
63
|
-
if
|
64
|
-
JSON.parse(@client.
|
74
|
+
def failed(t = nil, start = 0, limit = 25)
|
75
|
+
if !t
|
76
|
+
return JSON.parse(@client.call('failed'))
|
65
77
|
else
|
66
|
-
results = JSON.parse(@client.
|
67
|
-
results['jobs'] = results['jobs']
|
78
|
+
results = JSON.parse(@client.call('failed', t, start, limit))
|
79
|
+
results['jobs'] = multiget(*results['jobs'])
|
68
80
|
results
|
69
81
|
end
|
70
82
|
end
|
71
83
|
|
72
84
|
def [](id)
|
73
|
-
|
85
|
+
get(id)
|
86
|
+
end
|
87
|
+
|
88
|
+
def get(jid)
|
89
|
+
results = @client.call('get', jid)
|
74
90
|
if results.nil?
|
75
|
-
results = @client.
|
76
|
-
if results.nil?
|
77
|
-
return nil
|
78
|
-
end
|
91
|
+
results = @client.call('recur.get', jid)
|
92
|
+
return nil if results.nil?
|
79
93
|
return RecurringJob.new(@client, JSON.parse(results))
|
80
94
|
end
|
81
95
|
Job.new(@client, JSON.parse(results))
|
82
96
|
end
|
97
|
+
|
98
|
+
def multiget(*jids)
|
99
|
+
results = JSON.parse(@client.call('multiget', *jids))
|
100
|
+
results.map do |data|
|
101
|
+
Job.new(@client, data)
|
102
|
+
end
|
103
|
+
end
|
83
104
|
end
|
84
105
|
|
106
|
+
# A class for interacting with workers. Not meant to be instantiated
|
107
|
+
# directly, it's accessed through Client#workers
|
85
108
|
class ClientWorkers
|
86
109
|
def initialize(client)
|
87
110
|
@client = client
|
88
111
|
end
|
89
112
|
|
90
113
|
def counts
|
91
|
-
JSON.parse(@client.
|
114
|
+
JSON.parse(@client.call('workers'))
|
92
115
|
end
|
93
116
|
|
94
117
|
def [](name)
|
95
|
-
JSON.parse(@client.
|
118
|
+
JSON.parse(@client.call('workers', name))
|
96
119
|
end
|
97
120
|
end
|
98
121
|
|
122
|
+
# A class for interacting with queues. Not meant to be instantiated directly,
|
123
|
+
# it's accessed through Client#queues
|
99
124
|
class ClientQueues
|
100
125
|
def initialize(client)
|
101
126
|
@client = client
|
102
127
|
end
|
103
128
|
|
104
129
|
def counts
|
105
|
-
JSON.parse(@client.
|
130
|
+
JSON.parse(@client.call('queues'))
|
106
131
|
end
|
107
132
|
|
108
133
|
def [](name)
|
@@ -110,29 +135,28 @@ module Qless
|
|
110
135
|
end
|
111
136
|
end
|
112
137
|
|
138
|
+
# A class for interacting with events. Not meant to be instantiated directly,
|
139
|
+
# it's accessed through Client#events
|
113
140
|
class ClientEvents
|
141
|
+
EVENTS = %w{canceled completed failed popped stalled put track untrack}
|
142
|
+
EVENTS.each do |method|
|
143
|
+
define_method(method.to_sym) do |&block|
|
144
|
+
@actions[method.to_sym] = block
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
114
148
|
def initialize(redis)
|
115
149
|
@redis = redis
|
116
|
-
@actions =
|
150
|
+
@actions = {}
|
117
151
|
end
|
118
152
|
|
119
|
-
def canceled(&block) ; @actions[:canceled ] = block; end
|
120
|
-
def completed(&block); @actions[:completed] = block; end
|
121
|
-
def failed(&block) ; @actions[:failed ] = block; end
|
122
|
-
def popped(&block) ; @actions[:popped ] = block; end
|
123
|
-
def stalled(&block) ; @actions[:stalled ] = block; end
|
124
|
-
def put(&block) ; @actions[:put ] = block; end
|
125
|
-
def track(&block) ; @actions[:track ] = block; end
|
126
|
-
def untrack(&block) ; @actions[:untrack ] = block; end
|
127
|
-
|
128
153
|
def listen
|
129
154
|
yield(self) if block_given?
|
130
|
-
|
155
|
+
channels = EVENTS.map { |event| "ql:#{event}" }
|
156
|
+
@redis.subscribe(channels) do |on|
|
131
157
|
on.message do |channel, message|
|
132
|
-
callback = @actions[channel.to_sym]
|
133
|
-
|
134
|
-
callback.call(message)
|
135
|
-
end
|
158
|
+
callback = @actions[channel.sub('ql:', '').to_sym]
|
159
|
+
callback.call(message) unless callback.nil?
|
136
160
|
end
|
137
161
|
end
|
138
162
|
end
|
@@ -142,29 +166,25 @@ module Qless
|
|
142
166
|
end
|
143
167
|
end
|
144
168
|
|
169
|
+
# The client for interacting with Qless
|
145
170
|
class Client
|
146
|
-
# Lua
|
147
|
-
attr_reader :
|
148
|
-
|
149
|
-
attr_reader :_pause, :_unpause, :_deregister_workers
|
150
|
-
# A real object
|
151
|
-
attr_reader :config, :redis, :jobs, :queues, :workers
|
171
|
+
# Lua script
|
172
|
+
attr_reader :_qless, :config, :redis, :jobs, :queues, :workers
|
173
|
+
attr_accessor :worker_name
|
152
174
|
|
153
175
|
def initialize(options = {})
|
154
|
-
# This is the redis instance we're connected to
|
155
|
-
|
176
|
+
# This is the redis instance we're connected to. Use connect so REDIS_URL
|
177
|
+
# will be honored
|
178
|
+
@redis = options[:redis] || Redis.connect(options)
|
156
179
|
@options = options
|
157
|
-
assert_minimum_redis_version(
|
180
|
+
assert_minimum_redis_version('2.5.5')
|
158
181
|
@config = Config.new(self)
|
159
|
-
|
160
|
-
'priority', 'put', 'queues', 'recur', 'retry', 'stats', 'tag', 'track', 'workers', 'pause', 'unpause',
|
161
|
-
'deregister_workers'].each do |f|
|
162
|
-
self.instance_variable_set("@_#{f}", Qless::LuaScript.new(f, @redis))
|
163
|
-
end
|
182
|
+
@_qless = Qless::LuaScript.new('qless', @redis)
|
164
183
|
|
165
184
|
@jobs = ClientJobs.new(self)
|
166
185
|
@queues = ClientQueues.new(self)
|
167
186
|
@workers = ClientWorkers.new(self)
|
187
|
+
@worker_name = [Socket.gethostname, Process.pid.to_s].join('-')
|
168
188
|
end
|
169
189
|
|
170
190
|
def inspect
|
@@ -178,39 +198,58 @@ module Qless
|
|
178
198
|
@events ||= ClientEvents.new(Redis.connect(@options))
|
179
199
|
end
|
180
200
|
|
201
|
+
def call(command, *argv)
|
202
|
+
@_qless.call(command, Time.now.to_f, *argv)
|
203
|
+
end
|
204
|
+
|
181
205
|
def track(jid)
|
182
|
-
|
206
|
+
call('track', 'track', jid)
|
183
207
|
end
|
184
208
|
|
185
209
|
def untrack(jid)
|
186
|
-
|
210
|
+
call('track', 'untrack', jid)
|
187
211
|
end
|
188
212
|
|
189
|
-
def tags(offset=0, count=100)
|
190
|
-
JSON.parse(
|
213
|
+
def tags(offset = 0, count = 100)
|
214
|
+
JSON.parse(call('tag', 'top', offset, count))
|
191
215
|
end
|
192
216
|
|
193
217
|
def deregister_workers(*worker_names)
|
194
|
-
|
218
|
+
call('worker.deregister', *worker_names)
|
195
219
|
end
|
196
220
|
|
197
221
|
def bulk_cancel(jids)
|
198
|
-
|
222
|
+
call('cancel', jids)
|
223
|
+
end
|
224
|
+
|
225
|
+
if ::Redis.instance_method(:dup).owner == ::Redis
|
226
|
+
def new_redis_connection
|
227
|
+
redis.dup
|
228
|
+
end
|
229
|
+
else # redis version < 3.0.7
|
230
|
+
def new_redis_connection
|
231
|
+
::Redis.new(@options)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def ==(other)
|
236
|
+
self.class == other.class && redis.id == other.redis.id
|
199
237
|
end
|
238
|
+
alias eql? ==
|
200
239
|
|
201
|
-
def
|
202
|
-
|
240
|
+
def hash
|
241
|
+
self.class.hash ^ redis.id.hash
|
203
242
|
end
|
204
243
|
|
205
244
|
private
|
206
245
|
|
207
246
|
def assert_minimum_redis_version(version)
|
208
247
|
# remove the "-pre2" from "2.6.8-pre2"
|
209
|
-
redis_version = @redis.info.fetch(
|
248
|
+
redis_version = @redis.info.fetch('redis_version').split('-').first
|
210
249
|
return if Gem::Version.new(redis_version) >= Gem::Version.new(version)
|
211
250
|
|
212
251
|
raise UnsupportedRedisVersionError,
|
213
|
-
|
252
|
+
"Qless requires #{version} or better, not #{redis_version}"
|
214
253
|
end
|
215
254
|
end
|
216
255
|
end
|