resque_cmdline 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE +19 -0
- data/README.md +46 -0
- data/Rakefile +103 -0
- data/VERSION +1 -0
- data/bin/rq +327 -0
- data/bin/sq +327 -0
- data/lib/resque_cmdline.rb +5 -0
- data/lib/resque_cmdline/version.rb +3 -0
- data/resque_cmdline.gemspec +28 -0
- metadata +127 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b15e6c5c4c89f45182b04e3e9be21ebe59e5428a
|
4
|
+
data.tar.gz: e7be2715b86572b97996a84064a5996a3e3923a8
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9edc7668525efd1574eb7bcb3ee4109f3a8842d39c93f6a911d7a716011542e43d7e5eda158458d69f63971d4eab230d6bc739fc019b0a68fd9571d27fbd6d65
|
7
|
+
data.tar.gz: 5b1efafa0d7583536eab400feedb7492c71ebec339cd04d69fb0bb35ad7b08cfe369f40712db62c202712e51c6aeaeef875353849ad5979a70191310e5fada00
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
Copyright (c) 2012 Ric Lister
|
3
|
+
|
4
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
5
|
+
this software and associated documentation files (the "Software"), to deal in
|
6
|
+
the Software without restriction, including without limitation the rights to
|
7
|
+
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
8
|
+
of the Software, and to permit persons to whom the Software is furnished to do
|
9
|
+
so, subject to the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be included in all
|
12
|
+
copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
15
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
16
|
+
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
17
|
+
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
18
|
+
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
19
|
+
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# resque_cmdline
|
2
|
+
|
3
|
+
This is a simple command-line client for showing info about queues and
|
4
|
+
workers from
|
5
|
+
[resque](https://github.com/resque/resque) or
|
6
|
+
[sidekiq](https://github.com/mperham/sidekiq).
|
7
|
+
|
8
|
+
resque_cmdline does not use resque or sidekiq gems, but extracts info
|
9
|
+
directly from redis. This should be an order of magnitude faster when
|
10
|
+
querying a redis server over a slow (vpn over internet) connection.
|
11
|
+
|
12
|
+
## Usage
|
13
|
+
|
14
|
+
rq [options] command [-e shell_command]
|
15
|
+
|
16
|
+
where command is one of: queues workers failed running pending stats. Commands
|
17
|
+
may be be shortened to their shortest unique beginning.
|
18
|
+
|
19
|
+
See option list using `-h`.
|
20
|
+
|
21
|
+
## Example
|
22
|
+
|
23
|
+
Show ps listing for all running jobs older than 1 hour:
|
24
|
+
|
25
|
+
rq running -o 1h -e ssh {host} ps -lfp {pid}
|
26
|
+
|
27
|
+
## Config
|
28
|
+
|
29
|
+
Configure your environments and redis servers in `~/.rq.yml`. For example:
|
30
|
+
|
31
|
+
```yaml
|
32
|
+
environments:
|
33
|
+
production: redis-01:6379
|
34
|
+
staging: redis-stg-01:6379
|
35
|
+
development: localhost:6379
|
36
|
+
|
37
|
+
environment: production
|
38
|
+
```
|
39
|
+
|
40
|
+
## License
|
41
|
+
|
42
|
+
See included file LICENSE.
|
43
|
+
|
44
|
+
## Copyright
|
45
|
+
|
46
|
+
Copyright (c) 2012 Richard Lister.
|
data/Rakefile
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
require "bundler/gem_tasks"
|
3
|
+
require "rainbow"
|
4
|
+
|
5
|
+
## begin version management
|
6
|
+
def valid? version
|
7
|
+
pattern = /^\d+\.\d+\.\d+(\-(dev|beta|rc\d+))?$/
|
8
|
+
raise "Tried to set invalid version: #{version}".color(:red) unless version =~ pattern
|
9
|
+
end
|
10
|
+
|
11
|
+
def correct_version version
|
12
|
+
ver, flag = version.split '-'
|
13
|
+
v = ver.split '.'
|
14
|
+
(0..2).each do |n|
|
15
|
+
v[n] = v[n].to_i
|
16
|
+
end
|
17
|
+
[v.join('.'), flag].compact.join '-'
|
18
|
+
end
|
19
|
+
|
20
|
+
def read_version
|
21
|
+
begin
|
22
|
+
File.read 'VERSION'
|
23
|
+
rescue
|
24
|
+
raise "VERSION file not found or unreadable.".color(:red)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def write_version version
|
29
|
+
valid? version
|
30
|
+
begin
|
31
|
+
File.open 'VERSION', 'w' do |file|
|
32
|
+
file.write correct_version(version)
|
33
|
+
end
|
34
|
+
rescue
|
35
|
+
raise "VERSION file not found or unwritable.".color(:red)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def reset current, which
|
40
|
+
version, flag = current.split '-'
|
41
|
+
v = version.split '.'
|
42
|
+
which.each do |part|
|
43
|
+
v[part] = 0
|
44
|
+
end
|
45
|
+
[v.join('.'), flag].compact.join '-'
|
46
|
+
end
|
47
|
+
|
48
|
+
def increment current, which
|
49
|
+
version, flag = current.split '-'
|
50
|
+
v = version.split '.'
|
51
|
+
v[which] = v[which].to_i + 1
|
52
|
+
[v.join('.'), flag].compact.join '-'
|
53
|
+
end
|
54
|
+
|
55
|
+
desc "Prints the current application version"
|
56
|
+
version = read_version
|
57
|
+
task :version do
|
58
|
+
puts <<HELP
|
59
|
+
Available commands are:
|
60
|
+
-----------------------
|
61
|
+
rake version:write[version] # set version explicitly
|
62
|
+
rake version:patch # increment the patch x.x.x+1
|
63
|
+
rake version:minor # increment minor and reset patch x.x+1.0
|
64
|
+
rake version:major # increment major and reset others x+1.0.0
|
65
|
+
|
66
|
+
HELP
|
67
|
+
puts "Current version is: #{version.color(:green)}"
|
68
|
+
puts "NOTE: version should always be in the format of x.x.x".color(:red)
|
69
|
+
end
|
70
|
+
|
71
|
+
namespace :version do
|
72
|
+
|
73
|
+
desc "Write version explicitly by specifying version number as a parameter"
|
74
|
+
task :write, [:version] do |task, args|
|
75
|
+
write_version args[:version].strip
|
76
|
+
puts "Version explicitly written: #{read_version.color(:green)}"
|
77
|
+
end
|
78
|
+
|
79
|
+
desc "Increments the patch version"
|
80
|
+
task :patch do
|
81
|
+
new_version = increment read_version, 2
|
82
|
+
write_version new_version
|
83
|
+
puts "Application patched: #{new_version.color(:green)}"
|
84
|
+
end
|
85
|
+
|
86
|
+
desc "Increments the minor version and resets the patch"
|
87
|
+
task :minor do
|
88
|
+
incremented = increment read_version, 1
|
89
|
+
new_version = reset incremented, [2]
|
90
|
+
write_version new_version
|
91
|
+
puts "New version released: #{new_version.color(:green)}"
|
92
|
+
end
|
93
|
+
|
94
|
+
desc "Increments the major version and resets both minor and patch"
|
95
|
+
task :major do
|
96
|
+
incremented = increment read_version, 0
|
97
|
+
new_version = reset incremented, [1, 2]
|
98
|
+
write_version new_version
|
99
|
+
puts "Major application version change: #{new_version.color(:green)}. Congratulations!"
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
## end version management
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.0.1
|
data/bin/rq
ADDED
@@ -0,0 +1,327 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
##
|
5
|
+
## Fast cmdline client for extracting resque/sidekiq worker info from redis,
|
6
|
+
## displaying info with optional filters for age and class, and optionally
|
7
|
+
## running shell commands on each worker (with hostname and port passed).
|
8
|
+
## Call as 'rq' for resque, or 'sq' for sidekiq (or use --prefix).
|
9
|
+
##
|
10
|
+
## USAGE
|
11
|
+
##
|
12
|
+
## rq [options] command [-e shell_command]
|
13
|
+
##
|
14
|
+
## where command is one of: queues workers failed running pending stats. Commands
|
15
|
+
## may be be shortened to their shortest unique beginning.
|
16
|
+
##
|
17
|
+
## See option list using -h.
|
18
|
+
##
|
19
|
+
## EXAMPLE
|
20
|
+
##
|
21
|
+
## Show ps listing for all running jobs older than 1 hour:
|
22
|
+
##
|
23
|
+
## rq running -o 1h -e ssh {host} ps -lfp {pid}
|
24
|
+
##
|
25
|
+
## CONFIG
|
26
|
+
##
|
27
|
+
## Configure your environments and redis servers in ~/.rq.yml. For example:
|
28
|
+
##
|
29
|
+
## environments:
|
30
|
+
## production: redis-01:6379
|
31
|
+
## staging: redis-stg-01:6379
|
32
|
+
## development: localhost:6379
|
33
|
+
## environment: production
|
34
|
+
##
|
35
|
+
## LICENSE
|
36
|
+
##
|
37
|
+
## See included file LICENSE.
|
38
|
+
##
|
39
|
+
## COPYRIGHT
|
40
|
+
##
|
41
|
+
## Copyright (c) 2012 Richard Lister.
|
42
|
+
|
43
|
+
require "redis"
|
44
|
+
require "json"
|
45
|
+
require "optparse"
|
46
|
+
require 'date'
|
47
|
+
require 'yaml'
|
48
|
+
|
49
|
+
class Object
|
50
|
+
def try(method)
|
51
|
+
send method if respond_to? method
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
## convert array of arrays to array of strings with columns aligned
|
56
|
+
class Array
|
57
|
+
def values
|
58
|
+
self
|
59
|
+
end
|
60
|
+
|
61
|
+
def tabulate
|
62
|
+
widths = self.map do |row|
|
63
|
+
row.values.map { |value| value.to_s.length }
|
64
|
+
end.transpose.map(&:max) # => array of columns widths
|
65
|
+
|
66
|
+
self.map do |row|
|
67
|
+
row.values.each_with_index.map do |cell, index|
|
68
|
+
"%-#{widths[index]}s" % cell
|
69
|
+
end.join(' ')
|
70
|
+
end # => array of formatted table rows as strings
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
## return new hash with all keys recursively turned into symbols
|
75
|
+
class Hash
|
76
|
+
def symbolize_keys
|
77
|
+
Hash[self.map do |key, value|
|
78
|
+
[ key.to_sym, value.respond_to?(:symbolize_keys) ? value.symbolize_keys : value ]
|
79
|
+
end]
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
## human-readable time in past relative to present
|
84
|
+
class String
|
85
|
+
def seconds_ago
|
86
|
+
DateTime.parse(self).to_time.seconds_ago
|
87
|
+
end
|
88
|
+
|
89
|
+
def to_seconds
|
90
|
+
multiplier = { '' => 1, 's' => 1, 'm' => 60, 'h' => 3600, 'd' => 86400, 'w' => 604800 }
|
91
|
+
/^(\d+)\s*([smhdw]?)/.match(self) or raise ArgumentError, "Illegal time period #{self}"
|
92
|
+
Integer($1) * multiplier[$2]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
## assume sidekiq timestamps given as int are epochs
|
97
|
+
class Fixnum
|
98
|
+
def seconds_ago
|
99
|
+
Time.at(self).to_time.seconds_ago
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
class Time
|
104
|
+
def seconds_ago
|
105
|
+
Time.now.utc.to_i - self.utc.to_i
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
## convert time in secs to human-readable units
|
110
|
+
class Fixnum
|
111
|
+
def to_dhms
|
112
|
+
m, s = self.divmod(60)
|
113
|
+
h, m = m.divmod(60)
|
114
|
+
d, h = h.divmod(24)
|
115
|
+
{:d => d, :h => h, :m => m, :s => s}.map do |k,v|
|
116
|
+
v>0 ? "#{v}#{k}" : nil
|
117
|
+
end.compact.join(" ")
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
class ResqueCmdline
|
122
|
+
attr_reader :redis, :options, :prefix
|
123
|
+
|
124
|
+
def initialize(options = nil)
|
125
|
+
@options = options
|
126
|
+
host, port = options[:redis].split(':')
|
127
|
+
@redis = Redis.new(:host => host, :port => port || 6379)
|
128
|
+
@prefix = options[:prefix]
|
129
|
+
end
|
130
|
+
|
131
|
+
def queues
|
132
|
+
redis.smembers("#{prefix}queues").sort.map { |q| [q] }
|
133
|
+
end
|
134
|
+
|
135
|
+
def workers
|
136
|
+
redis.smembers("#{prefix}workers").sort.map { |w| [w] }
|
137
|
+
end
|
138
|
+
|
139
|
+
def failed
|
140
|
+
failed = redis.lrange("#{prefix}failed", 0, -1).map do |f|
|
141
|
+
job = JSON(f)
|
142
|
+
{
|
143
|
+
:worker => job['worker'].match(/^([\w\.-]+:\d+):/).captures.join,
|
144
|
+
:queue => job['queue'],
|
145
|
+
:class => job['payload']['class'],
|
146
|
+
:exception => job['exception'],
|
147
|
+
:date => job['failed_at'].seconds_ago,
|
148
|
+
:retried => job['retried_at'].try(:seconds_ago), # can be nil
|
149
|
+
}
|
150
|
+
end
|
151
|
+
|
152
|
+
if options[:older]
|
153
|
+
failed.select! { |job| job[:date] >= options[:older] }
|
154
|
+
end
|
155
|
+
|
156
|
+
if options[:retried]
|
157
|
+
failed.select! { |job| job[:retried] and job[:retried] >= options[:retried] }
|
158
|
+
end
|
159
|
+
|
160
|
+
failed.sort{ |a,b| b[:date] <=> a[:date] }.each do |job|
|
161
|
+
job[:date] = job[:date].to_dhms
|
162
|
+
job[:retried] = job[:retried].try(:to_dhms) # can be nil
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
def unregister(hostport)
|
167
|
+
redis.smembers("#{prefix}workers").grep(/^#{hostport}/).each do |worker|
|
168
|
+
puts "unregistering: #{worker}"
|
169
|
+
redis.srem("#{prefix}workers", worker)
|
170
|
+
redis.del("#{prefix}worker:#{worker}")
|
171
|
+
redis.del("#{prefix}worker:#{worker}:started")
|
172
|
+
redis.del("#{prefix}stat:processed:#{worker}")
|
173
|
+
redis.del("#{prefix}stat:failed:#{worker}")
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def running
|
178
|
+
keys = redis.smembers("#{prefix}workers").map { |worker| "#{prefix}worker:#{worker}" }
|
179
|
+
return keys if keys.empty?
|
180
|
+
|
181
|
+
running = redis.mapped_mget(*keys).map do |key, value|
|
182
|
+
if value.nil? || value.empty?
|
183
|
+
nil
|
184
|
+
else
|
185
|
+
job = JSON(value)
|
186
|
+
{
|
187
|
+
:worker => key.match(/^#{prefix}worker:([\w\.-]+:[\d\-]+):/).captures.join,
|
188
|
+
:queue => job['queue'],
|
189
|
+
:class => job['payload']['class'],
|
190
|
+
:date => job['run_at'].seconds_ago,
|
191
|
+
}
|
192
|
+
end
|
193
|
+
end.compact
|
194
|
+
|
195
|
+
if options[:older]
|
196
|
+
running.select! { |job| job[:date] >= options[:older] }
|
197
|
+
end
|
198
|
+
|
199
|
+
running.sort { |a,b| b[:date] <=> a[:date] }.each do |job|
|
200
|
+
job[:date] = job[:date].to_dhms
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def pending
|
205
|
+
redis.smembers("#{prefix}queues").map do |queue|
|
206
|
+
{
|
207
|
+
:name => queue,
|
208
|
+
:count => redis.llen("#{prefix}queue:#{queue}")
|
209
|
+
}
|
210
|
+
end.sort { |a,b| b[:count] <=> a[:count] }
|
211
|
+
end
|
212
|
+
|
213
|
+
def stats
|
214
|
+
[
|
215
|
+
[ "total processed", redis.get("#{prefix}stat:processed").to_s ],
|
216
|
+
[ "total failed", redis.get("#{prefix}stat:failed").to_s ],
|
217
|
+
[ "queues", queues.size.to_s ],
|
218
|
+
[ "workers", redis.scard("#{prefix}workers").to_s ],
|
219
|
+
[ "pending", queues.inject(0) { |sum, q| sum + redis.llen("#{prefix}queue:#{q}") }.to_s ],
|
220
|
+
[ "failed", redis.llen("#{prefix}failed").to_s ],
|
221
|
+
]
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
225
|
+
|
226
|
+
## defaults
|
227
|
+
options = {
|
228
|
+
:environments => {
|
229
|
+
:production => 'http://redis:6379',
|
230
|
+
:development => 'http://localhost:6379'
|
231
|
+
},
|
232
|
+
:environment => :production,
|
233
|
+
#:prefix => 'resque:',
|
234
|
+
:prefix => File.basename($0) == 'sq' ? '' : 'resque:', #sidekiq vs resque
|
235
|
+
}
|
236
|
+
|
237
|
+
## merge options from config file
|
238
|
+
cfgfile = File.join(ENV['HOME'], '.rq.yml')
|
239
|
+
options = options.merge(YAML.load_file(cfgfile).symbolize_keys) if File.exists?(cfgfile)
|
240
|
+
|
241
|
+
commands = %w[queues workers failed running pending stats]
|
242
|
+
|
243
|
+
OptionParser.new do |opt|
|
244
|
+
opt.banner = "Usage: #{$0} [options] command [-e shellcmd]\n(use -h for help)."
|
245
|
+
opt.on('-c', '--class REGEX', 'Filter by class name.') do |c|
|
246
|
+
options[:class] = c
|
247
|
+
end
|
248
|
+
opt.on('-e', '--exec CMD', 'Exec cmd with sub for {host} and {pid}, must be last arg.') do |e|
|
249
|
+
options[:exec] = ARGV.shift(ARGV.length).unshift(e).join(' ') # slurp rest of ARGV
|
250
|
+
end
|
251
|
+
opt.on('-E', '--environment REGEX', 'Environment to use.') do |e|
|
252
|
+
options[:environment] = e
|
253
|
+
end
|
254
|
+
opt.on('-j', '--json', 'Dump output as JSON instead of tabulated output.') do |j|
|
255
|
+
options[:json] = j
|
256
|
+
end
|
257
|
+
opt.on('-o', '--older TIME', 'Filter date e.g. 30s, 2d.') do |o|
|
258
|
+
options[:older] = o.to_seconds
|
259
|
+
end
|
260
|
+
opt.on('-p', '--prefix PREFIX', 'Prefix for keys in redis.') do |p|
|
261
|
+
options[:prefix] = p
|
262
|
+
end
|
263
|
+
opt.on('-q', '--queue REGEX', 'Filter by queue name.') do |q|
|
264
|
+
options[:queue] = q
|
265
|
+
end
|
266
|
+
opt.on('-r', '--retried TIME', 'Filter failed jobs retried before time.') do |r|
|
267
|
+
options[:retried] = r.to_seconds
|
268
|
+
end
|
269
|
+
opt.on('-R', '--redis SERVER' 'Set redis server as host:port') do |r|
|
270
|
+
options[:redis] = r
|
271
|
+
end
|
272
|
+
opt.on('-U', '--unregister', 'Unregister matching workers from redis. Be very careful.') do |u|
|
273
|
+
options[:unregister] = u
|
274
|
+
end
|
275
|
+
opt.on_tail("-h", "--help", "Show this message.") do
|
276
|
+
puts opt
|
277
|
+
puts " cmd: #{commands.join(' ')}"
|
278
|
+
exit
|
279
|
+
end
|
280
|
+
end.parse!
|
281
|
+
|
282
|
+
|
283
|
+
## find command that matches arg given
|
284
|
+
matching = commands.grep /^#{ARGV.first}/
|
285
|
+
abort "commands: #{commands.join(' ')}" unless matching.length == 1
|
286
|
+
|
287
|
+
## find first matching environment
|
288
|
+
environment, redis = options[:environments].find do |k, v|
|
289
|
+
k.match(/^#{options[:environment]}/)
|
290
|
+
end
|
291
|
+
abort "environments: #{options[:environments].keys.join(' ')}" unless environment && redis
|
292
|
+
|
293
|
+
## redis server from environment, unless overridden with option
|
294
|
+
options[:redis] ||= redis
|
295
|
+
|
296
|
+
## get data from redis
|
297
|
+
cmd = ResqueCmdline.new(options)
|
298
|
+
output = cmd.send(matching.first)
|
299
|
+
|
300
|
+
## grep output by a specific named column
|
301
|
+
[:class, :queue].each do |field|
|
302
|
+
if options[field]
|
303
|
+
output.select! do |row|
|
304
|
+
c = row.fetch(field) { nil } and c.match(options[field])
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
## handle data for exec or output
|
310
|
+
if options[:exec]
|
311
|
+
output.each do |line|
|
312
|
+
worker = line.fetch(:worker) { false } or next
|
313
|
+
host, pid = worker.split(':')
|
314
|
+
cmd = options[:exec].dup
|
315
|
+
cmd.gsub!(/\{host\}/, host).gsub!(/\{pid\}/, pid)
|
316
|
+
system cmd
|
317
|
+
end
|
318
|
+
elsif options[:unregister]
|
319
|
+
output.each do |line|
|
320
|
+
worker = line.fetch(:worker) { false } or next
|
321
|
+
cmd.unregister(worker)
|
322
|
+
end
|
323
|
+
elsif options[:json]
|
324
|
+
puts JSON.dump(output)
|
325
|
+
else
|
326
|
+
puts output.tabulate
|
327
|
+
end
|
data/bin/sq
ADDED
@@ -0,0 +1,327 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
## SYNOPSIS
|
4
|
+
##
|
5
|
+
## Fast cmdline client for extracting resque/sidekiq worker info from redis,
|
6
|
+
## displaying info with optional filters for age and class, and optionally
|
7
|
+
## running shell commands on each worker (with hostname and port passed).
|
8
|
+
## Call as 'rq' for resque, or 'sq' for sidekiq (or use --prefix).
|
9
|
+
##
|
10
|
+
## USAGE
|
11
|
+
##
|
12
|
+
## rq [options] command [-e shell_command]
|
13
|
+
##
|
14
|
+
## where command is one of: queues workers failed running pending stats. Commands
|
15
|
+
## may be be shortened to their shortest unique beginning.
|
16
|
+
##
|
17
|
+
## See option list using -h.
|
18
|
+
##
|
19
|
+
## EXAMPLE
|
20
|
+
##
|
21
|
+
## Show ps listing for all running jobs older than 1 hour:
|
22
|
+
##
|
23
|
+
## rq running -o 1h -e ssh {host} ps -lfp {pid}
|
24
|
+
##
|
25
|
+
## CONFIG
|
26
|
+
##
|
27
|
+
## Configure your environments and redis servers in ~/.rq.yml. For example:
|
28
|
+
##
|
29
|
+
## environments:
|
30
|
+
## production: redis-01:6379
|
31
|
+
## staging: redis-stg-01:6379
|
32
|
+
## development: localhost:6379
|
33
|
+
## environment: production
|
34
|
+
##
|
35
|
+
## LICENSE
|
36
|
+
##
|
37
|
+
## See included file LICENSE.
|
38
|
+
##
|
39
|
+
## COPYRIGHT
|
40
|
+
##
|
41
|
+
## Copyright (c) 2012 Richard Lister.
|
42
|
+
|
43
|
+
require "redis"
|
44
|
+
require "json"
|
45
|
+
require "optparse"
|
46
|
+
require 'date'
|
47
|
+
require 'yaml'
|
48
|
+
|
49
|
+
class Object
|
50
|
+
def try(method)
|
51
|
+
send method if respond_to? method
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
## convert array of arrays to array of strings with columns aligned
|
56
|
+
class Array
|
57
|
+
def values
|
58
|
+
self
|
59
|
+
end
|
60
|
+
|
61
|
+
def tabulate
|
62
|
+
widths = self.map do |row|
|
63
|
+
row.values.map { |value| value.to_s.length }
|
64
|
+
end.transpose.map(&:max) # => array of columns widths
|
65
|
+
|
66
|
+
self.map do |row|
|
67
|
+
row.values.each_with_index.map do |cell, index|
|
68
|
+
"%-#{widths[index]}s" % cell
|
69
|
+
end.join(' ')
|
70
|
+
end # => array of formatted table rows as strings
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
## return new hash with all keys recursively turned into symbols
|
75
|
+
class Hash
|
76
|
+
def symbolize_keys
|
77
|
+
Hash[self.map do |key, value|
|
78
|
+
[ key.to_sym, value.respond_to?(:symbolize_keys) ? value.symbolize_keys : value ]
|
79
|
+
end]
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
## human-readable time in past relative to present
|
84
|
+
class String
|
85
|
+
def seconds_ago
|
86
|
+
DateTime.parse(self).to_time.seconds_ago
|
87
|
+
end
|
88
|
+
|
89
|
+
def to_seconds
|
90
|
+
multiplier = { '' => 1, 's' => 1, 'm' => 60, 'h' => 3600, 'd' => 86400, 'w' => 604800 }
|
91
|
+
/^(\d+)\s*([smhdw]?)/.match(self) or raise ArgumentError, "Illegal time period #{self}"
|
92
|
+
Integer($1) * multiplier[$2]
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
## assume sidekiq timestamps given as int are epochs
|
97
|
+
class Fixnum
|
98
|
+
def seconds_ago
|
99
|
+
Time.at(self).to_time.seconds_ago
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
class Time
|
104
|
+
def seconds_ago
|
105
|
+
Time.now.utc.to_i - self.utc.to_i
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
## convert time in secs to human-readable units
|
110
|
+
class Fixnum
|
111
|
+
def to_dhms
|
112
|
+
m, s = self.divmod(60)
|
113
|
+
h, m = m.divmod(60)
|
114
|
+
d, h = h.divmod(24)
|
115
|
+
{:d => d, :h => h, :m => m, :s => s}.map do |k,v|
|
116
|
+
v>0 ? "#{v}#{k}" : nil
|
117
|
+
end.compact.join(" ")
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
class ResqueCmdline
|
122
|
+
attr_reader :redis, :options, :prefix
|
123
|
+
|
124
|
+
def initialize(options = nil)
|
125
|
+
@options = options
|
126
|
+
host, port = options[:redis].split(':')
|
127
|
+
@redis = Redis.new(:host => host, :port => port || 6379)
|
128
|
+
@prefix = options[:prefix]
|
129
|
+
end
|
130
|
+
|
131
|
+
def queues
|
132
|
+
redis.smembers("#{prefix}queues").sort.map { |q| [q] }
|
133
|
+
end
|
134
|
+
|
135
|
+
def workers
|
136
|
+
redis.smembers("#{prefix}workers").sort.map { |w| [w] }
|
137
|
+
end
|
138
|
+
|
139
|
+
def failed
|
140
|
+
failed = redis.lrange("#{prefix}failed", 0, -1).map do |f|
|
141
|
+
job = JSON(f)
|
142
|
+
{
|
143
|
+
:worker => job['worker'].match(/^([\w\.-]+:\d+):/).captures.join,
|
144
|
+
:queue => job['queue'],
|
145
|
+
:class => job['payload']['class'],
|
146
|
+
:exception => job['exception'],
|
147
|
+
:date => job['failed_at'].seconds_ago,
|
148
|
+
:retried => job['retried_at'].try(:seconds_ago), # can be nil
|
149
|
+
}
|
150
|
+
end
|
151
|
+
|
152
|
+
if options[:older]
|
153
|
+
failed.select! { |job| job[:date] >= options[:older] }
|
154
|
+
end
|
155
|
+
|
156
|
+
if options[:retried]
|
157
|
+
failed.select! { |job| job[:retried] and job[:retried] >= options[:retried] }
|
158
|
+
end
|
159
|
+
|
160
|
+
failed.sort{ |a,b| b[:date] <=> a[:date] }.each do |job|
|
161
|
+
job[:date] = job[:date].to_dhms
|
162
|
+
job[:retried] = job[:retried].try(:to_dhms) # can be nil
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
def unregister(hostport)
|
167
|
+
redis.smembers("#{prefix}workers").grep(/^#{hostport}/).each do |worker|
|
168
|
+
puts "unregistering: #{worker}"
|
169
|
+
redis.srem("#{prefix}workers", worker)
|
170
|
+
redis.del("#{prefix}worker:#{worker}")
|
171
|
+
redis.del("#{prefix}worker:#{worker}:started")
|
172
|
+
redis.del("#{prefix}stat:processed:#{worker}")
|
173
|
+
redis.del("#{prefix}stat:failed:#{worker}")
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def running
|
178
|
+
keys = redis.smembers("#{prefix}workers").map { |worker| "#{prefix}worker:#{worker}" }
|
179
|
+
return keys if keys.empty?
|
180
|
+
|
181
|
+
running = redis.mapped_mget(*keys).map do |key, value|
|
182
|
+
if value.nil? || value.empty?
|
183
|
+
nil
|
184
|
+
else
|
185
|
+
job = JSON(value)
|
186
|
+
{
|
187
|
+
:worker => key.match(/^#{prefix}worker:([\w\.-]+:[\d\-]+):/).captures.join,
|
188
|
+
:queue => job['queue'],
|
189
|
+
:class => job['payload']['class'],
|
190
|
+
:date => job['run_at'].seconds_ago,
|
191
|
+
}
|
192
|
+
end
|
193
|
+
end.compact
|
194
|
+
|
195
|
+
if options[:older]
|
196
|
+
running.select! { |job| job[:date] >= options[:older] }
|
197
|
+
end
|
198
|
+
|
199
|
+
running.sort { |a,b| b[:date] <=> a[:date] }.each do |job|
|
200
|
+
job[:date] = job[:date].to_dhms
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def pending
|
205
|
+
redis.smembers("#{prefix}queues").map do |queue|
|
206
|
+
{
|
207
|
+
:name => queue,
|
208
|
+
:count => redis.llen("#{prefix}queue:#{queue}")
|
209
|
+
}
|
210
|
+
end.sort { |a,b| b[:count] <=> a[:count] }
|
211
|
+
end
|
212
|
+
|
213
|
+
def stats
|
214
|
+
[
|
215
|
+
[ "total processed", redis.get("#{prefix}stat:processed").to_s ],
|
216
|
+
[ "total failed", redis.get("#{prefix}stat:failed").to_s ],
|
217
|
+
[ "queues", queues.size.to_s ],
|
218
|
+
[ "workers", redis.scard("#{prefix}workers").to_s ],
|
219
|
+
[ "pending", queues.inject(0) { |sum, q| sum + redis.llen("#{prefix}queue:#{q}") }.to_s ],
|
220
|
+
[ "failed", redis.llen("#{prefix}failed").to_s ],
|
221
|
+
]
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
225
|
+
|
226
|
+
## defaults
|
227
|
+
options = {
|
228
|
+
:environments => {
|
229
|
+
:production => 'http://redis:6379',
|
230
|
+
:development => 'http://localhost:6379'
|
231
|
+
},
|
232
|
+
:environment => :production,
|
233
|
+
#:prefix => 'resque:',
|
234
|
+
:prefix => File.basename($0) == 'sq' ? '' : 'resque:', #sidekiq vs resque
|
235
|
+
}
|
236
|
+
|
237
|
+
## merge options from config file
|
238
|
+
cfgfile = File.join(ENV['HOME'], '.rq.yml')
|
239
|
+
options = options.merge(YAML.load_file(cfgfile).symbolize_keys) if File.exists?(cfgfile)
|
240
|
+
|
241
|
+
commands = %w[queues workers failed running pending stats]
|
242
|
+
|
243
|
+
OptionParser.new do |opt|
|
244
|
+
opt.banner = "Usage: #{$0} [options] command [-e shellcmd]\n(use -h for help)."
|
245
|
+
opt.on('-c', '--class REGEX', 'Filter by class name.') do |c|
|
246
|
+
options[:class] = c
|
247
|
+
end
|
248
|
+
opt.on('-e', '--exec CMD', 'Exec cmd with sub for {host} and {pid}, must be last arg.') do |e|
|
249
|
+
options[:exec] = ARGV.shift(ARGV.length).unshift(e).join(' ') # slurp rest of ARGV
|
250
|
+
end
|
251
|
+
opt.on('-E', '--environment REGEX', 'Environment to use.') do |e|
|
252
|
+
options[:environment] = e
|
253
|
+
end
|
254
|
+
opt.on('-j', '--json', 'Dump output as JSON instead of tabulated output.') do |j|
|
255
|
+
options[:json] = j
|
256
|
+
end
|
257
|
+
opt.on('-o', '--older TIME', 'Filter date e.g. 30s, 2d.') do |o|
|
258
|
+
options[:older] = o.to_seconds
|
259
|
+
end
|
260
|
+
opt.on('-p', '--prefix PREFIX', 'Prefix for keys in redis.') do |p|
|
261
|
+
options[:prefix] = p
|
262
|
+
end
|
263
|
+
opt.on('-q', '--queue REGEX', 'Filter by queue name.') do |q|
|
264
|
+
options[:queue] = q
|
265
|
+
end
|
266
|
+
opt.on('-r', '--retried TIME', 'Filter failed jobs retried before time.') do |r|
|
267
|
+
options[:retried] = r.to_seconds
|
268
|
+
end
|
269
|
+
opt.on('-R', '--redis SERVER' 'Set redis server as host:port') do |r|
|
270
|
+
options[:redis] = r
|
271
|
+
end
|
272
|
+
opt.on('-U', '--unregister', 'Unregister matching workers from redis. Be very careful.') do |u|
|
273
|
+
options[:unregister] = u
|
274
|
+
end
|
275
|
+
opt.on_tail("-h", "--help", "Show this message.") do
|
276
|
+
puts opt
|
277
|
+
puts " cmd: #{commands.join(' ')}"
|
278
|
+
exit
|
279
|
+
end
|
280
|
+
end.parse!
|
281
|
+
|
282
|
+
|
283
|
+
## find command that matches arg given
|
284
|
+
matching = commands.grep /^#{ARGV.first}/
|
285
|
+
abort "commands: #{commands.join(' ')}" unless matching.length == 1
|
286
|
+
|
287
|
+
## find first matching environment
|
288
|
+
environment, redis = options[:environments].find do |k, v|
|
289
|
+
k.match(/^#{options[:environment]}/)
|
290
|
+
end
|
291
|
+
abort "environments: #{options[:environments].keys.join(' ')}" unless environment && redis
|
292
|
+
|
293
|
+
## redis server from environment, unless overridden with option
|
294
|
+
options[:redis] ||= redis
|
295
|
+
|
296
|
+
## get data from redis
|
297
|
+
cmd = ResqueCmdline.new(options)
|
298
|
+
output = cmd.send(matching.first)
|
299
|
+
|
300
|
+
## grep output by a specific named column
|
301
|
+
[:class, :queue].each do |field|
|
302
|
+
if options[field]
|
303
|
+
output.select! do |row|
|
304
|
+
c = row.fetch(field) { nil } and c.match(options[field])
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
## handle data for exec or output
|
310
|
+
if options[:exec]
|
311
|
+
output.each do |line|
|
312
|
+
worker = line.fetch(:worker) { false } or next
|
313
|
+
host, pid = worker.split(':')
|
314
|
+
cmd = options[:exec].dup
|
315
|
+
cmd.gsub!(/\{host\}/, host).gsub!(/\{pid\}/, pid)
|
316
|
+
system cmd
|
317
|
+
end
|
318
|
+
elsif options[:unregister]
|
319
|
+
output.each do |line|
|
320
|
+
worker = line.fetch(:worker) { false } or next
|
321
|
+
cmd.unregister(worker)
|
322
|
+
end
|
323
|
+
elsif options[:json]
|
324
|
+
puts JSON.dump(output)
|
325
|
+
else
|
326
|
+
puts output.tabulate
|
327
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'resque_cmdline/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "resque_cmdline"
|
8
|
+
spec.version = ResqueCmdline::VERSION
|
9
|
+
spec.authors = ["Ric Lister"]
|
10
|
+
spec.email = ["rlister@gmail.com"]
|
11
|
+
spec.description = %q{resque_cmdline: simple command-line client for resque workers}
|
12
|
+
spec.summary = %q{Ruby command-line client for resque and sidekiq workers}
|
13
|
+
spec.homepage = "https://github.com/rlister/resque_cmdline"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files`.split($/)
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
# dependencies
|
22
|
+
spec.add_dependency('json' , '>= 1.7.5')
|
23
|
+
spec.add_dependency('redis' , '>= 3.0.2')
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.3"
|
26
|
+
spec.add_development_dependency "rake"
|
27
|
+
spec.add_development_dependency "rainbow"
|
28
|
+
end
|
metadata
ADDED
@@ -0,0 +1,127 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: resque_cmdline
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Ric Lister
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-04-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: json
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.7.5
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.7.5
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: redis
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 3.0.2
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 3.0.2
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ~>
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.3'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.3'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rainbow
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
description: 'resque_cmdline: simple command-line client for resque workers'
|
84
|
+
email:
|
85
|
+
- rlister@gmail.com
|
86
|
+
executables:
|
87
|
+
- rq
|
88
|
+
- sq
|
89
|
+
extensions: []
|
90
|
+
extra_rdoc_files: []
|
91
|
+
files:
|
92
|
+
- .gitignore
|
93
|
+
- Gemfile
|
94
|
+
- LICENSE
|
95
|
+
- README.md
|
96
|
+
- Rakefile
|
97
|
+
- VERSION
|
98
|
+
- bin/rq
|
99
|
+
- bin/sq
|
100
|
+
- lib/resque_cmdline.rb
|
101
|
+
- lib/resque_cmdline/version.rb
|
102
|
+
- resque_cmdline.gemspec
|
103
|
+
homepage: https://github.com/rlister/resque_cmdline
|
104
|
+
licenses:
|
105
|
+
- MIT
|
106
|
+
metadata: {}
|
107
|
+
post_install_message:
|
108
|
+
rdoc_options: []
|
109
|
+
require_paths:
|
110
|
+
- lib
|
111
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
112
|
+
requirements:
|
113
|
+
- - '>='
|
114
|
+
- !ruby/object:Gem::Version
|
115
|
+
version: '0'
|
116
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - '>='
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '0'
|
121
|
+
requirements: []
|
122
|
+
rubyforge_project:
|
123
|
+
rubygems_version: 2.0.0
|
124
|
+
signing_key:
|
125
|
+
specification_version: 4
|
126
|
+
summary: Ruby command-line client for resque and sidekiq workers
|
127
|
+
test_files: []
|