kthxbye 1.0.1 → 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/config.ru +1 -1
- data/lib/kthxbye.rb +86 -19
- data/lib/kthxbye/config.rb +15 -14
- data/lib/kthxbye/exceptions.rb +1 -1
- data/lib/kthxbye/helper.rb +1 -3
- data/lib/kthxbye/job.rb +41 -14
- data/lib/kthxbye/version.rb +1 -1
- data/lib/kthxbye/web_interface.rb +1 -1
- data/lib/kthxbye/worker.rb +45 -23
- metadata +25 -34
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.0.
|
1
|
+
1.0.2
|
data/config.ru
CHANGED
data/lib/kthxbye.rb
CHANGED
@@ -18,15 +18,58 @@ require 'kthxbye/version'
|
|
18
18
|
|
19
19
|
require 'kthxbye/exceptions'
|
20
20
|
|
21
|
+
# Kthxbye is a light-weight, distributed delayed-job processor. It is meant
|
22
|
+
# to take the load of long running processes off of your web-server any place
|
23
|
+
# it in the capable hands of a backend processor. It is unique in that it
|
24
|
+
# provides results delievery post-processing so that your front-end can actually
|
25
|
+
# have its results back.
|
26
|
+
#
|
27
|
+
# Most delayed job processors out there are meant for running processes that we
|
28
|
+
# don't need to get a result from. For example mass-mailings or image processing.
|
29
|
+
# However I've found many a case in which I need the result of a long process but
|
30
|
+
# my web server is bogging down watiting for a response.
|
31
|
+
#
|
32
|
+
# And this is where Kthxbye shines.
|
33
|
+
#
|
34
|
+
# Okay, you say, enough glowing, let's see this thinger work. Simple enough.
|
35
|
+
#
|
36
|
+
# On your front-end (e.g. a Rails app) initialization process, run
|
37
|
+
# Kthxbye::Config.setup(:redis_server => "localhost", :redis_port => 9876)
|
38
|
+
#
|
39
|
+
# Then somewhere accessable to both your app and your workers (maybe a separate
|
40
|
+
# job class file in you /lib dir) create your job class with a class method
|
41
|
+
# perform:
|
42
|
+
# class MyJob
|
43
|
+
# def self.perform(param1, param2)
|
44
|
+
# puts "I'm here with data!"
|
45
|
+
# # perform stuff with data
|
46
|
+
# result = param1 + param2
|
47
|
+
# return result
|
48
|
+
# end
|
49
|
+
# end
|
50
|
+
#
|
51
|
+
# Then all that's left is to queue up the job at the appropriate time:
|
52
|
+
# Kthxbye.enqueue("job-queue", MyJob, 6, 4)
|
53
|
+
#
|
54
|
+
# Your job is now queued for processing! To work this job, its even simpler.
|
55
|
+
# require 'whatever/file/my_job/class/is/in'
|
56
|
+
# worker = Kthxbye::Worker.new("job-queue")
|
57
|
+
# worker.run
|
58
|
+
#
|
59
|
+
# Let the processing commence!
|
21
60
|
module Kthxbye
|
22
61
|
include Helper
|
23
62
|
extend self
|
24
63
|
|
25
|
-
#
|
64
|
+
# This is not necessary to call. Any of the methods that use redis
|
65
|
+
# Will make an inital call to connect to redis.
|
66
|
+
# Useful if you want to connect to an existing redis instance. Othewise, if
|
67
|
+
# called without params, it simply connects a new instance of redis.
|
26
68
|
def connect( redis_instance=nil )
|
27
69
|
@redis = ( redis_instance || Redis.new( :host => Config.options[:redis_server], :port => Config.options[:redis_port] ) )
|
28
70
|
end
|
29
71
|
|
72
|
+
# Returns the Redis instance for direct calls to the Redis db
|
30
73
|
def redis
|
31
74
|
return @redis if @redis
|
32
75
|
Config.setup
|
@@ -34,22 +77,32 @@ module Kthxbye
|
|
34
77
|
self.redis
|
35
78
|
end
|
36
79
|
|
80
|
+
# Returns a hash of all existing Redis keys.
|
37
81
|
def keys
|
38
82
|
redis.keys("*")
|
39
83
|
end
|
40
84
|
|
41
|
-
#
|
85
|
+
# Queues jobs. Takes at minimum two paramters
|
86
|
+
# 1) A string representing a queue
|
87
|
+
# 2) The class of the job being queued.
|
88
|
+
#
|
89
|
+
# You can optionally pass in additional params to the perform method within
|
90
|
+
# the class. You will need to match the number of args in the perform method
|
91
|
+
# when you queue the job. Otherwise this will throw an exception.
|
42
92
|
def enqueue(queue, klass, *args)
|
43
93
|
Job.create(queue, klass, *args)
|
44
94
|
end
|
45
95
|
|
46
|
-
#
|
96
|
+
# Takes a string that represents a job queue.
|
97
|
+
# Returns the size of the given queue.
|
47
98
|
def size(queue)
|
48
99
|
redis.llen("queue:#{queue}").to_i
|
49
100
|
end
|
50
101
|
|
51
|
-
#
|
52
|
-
#
|
102
|
+
# This method is used mostly internally to pop the next job off of the given
|
103
|
+
# queue. It takes in a string representing a queue and will return a
|
104
|
+
# Kthxbye::Job object if a job exists on the queue. This is destructive on
|
105
|
+
# the queue as it will REMOVE the next job off queue and return the job object.
|
53
106
|
def salvage(q)
|
54
107
|
id = redis.lpop( "queue:#{q}" )
|
55
108
|
if id
|
@@ -61,8 +114,12 @@ module Kthxbye
|
|
61
114
|
end
|
62
115
|
end
|
63
116
|
|
64
|
-
#
|
65
|
-
#
|
117
|
+
# This is a generic queue peek method. It isn't used directly but is the basis
|
118
|
+
# for the "ghost" methods "data_peek" and "result_peek". This method takes in
|
119
|
+
# a string representing a redis hash store (only two in kthxbye: "data-store"
|
120
|
+
# and "result-store"), a string representing a queue, and optionally a job id.
|
121
|
+
# If a job id is given, it will return the data for that job only. Otherwise
|
122
|
+
# it returns all the data for all jobs/results.
|
66
123
|
def peek(store, queue, id=nil)
|
67
124
|
if id
|
68
125
|
decode( redis.hget( "#{store}-store:#{queue}", id ) )
|
@@ -74,8 +131,7 @@ module Kthxbye
|
|
74
131
|
end
|
75
132
|
end
|
76
133
|
|
77
|
-
|
78
|
-
def method_missing(name, *args)
|
134
|
+
def method_missing(name, *args) #:nodoc:
|
79
135
|
method_name = name.id2name
|
80
136
|
if method_name =~ /^(data|result)_peek$/
|
81
137
|
Kthxbye.send(:peek, $1, *args)
|
@@ -85,36 +141,45 @@ module Kthxbye
|
|
85
141
|
|
86
142
|
end
|
87
143
|
|
88
|
-
#
|
144
|
+
# Returns all the queues Kthxbye knows about
|
89
145
|
def queues
|
90
146
|
redis.smembers( :queues ).sort
|
91
147
|
end
|
92
148
|
|
93
|
-
# registers
|
149
|
+
# This method takes a string and registers it as a queue in our "known queues"
|
150
|
+
# list
|
94
151
|
def register_queue(queue)
|
95
152
|
redis.sadd(:queues, queue) unless redis.sismember(:queues, queue)
|
96
153
|
end
|
97
154
|
|
98
|
-
# Removes the queue from the active queue listing, does not delete queue
|
99
|
-
# will lead to phantom queues. use delete_queue for complete removal
|
155
|
+
# Removes the queue from the active queue listing, does not delete queue.
|
156
|
+
# This will lead to phantom queues. use delete_queue for complete removal
|
157
|
+
# of queue.
|
100
158
|
def unregister_queue(queue)
|
101
159
|
redis.srem(:queues, queue)
|
102
160
|
end
|
103
161
|
|
104
|
-
# Completely removes queue:
|
105
|
-
#
|
162
|
+
# Completely removes queue: Unregisters it then deletes it should return true
|
163
|
+
# in all cases (including if we try to delete a non-existent queue).
|
106
164
|
def delete_queue(queue)
|
107
165
|
unregister_queue(queue)
|
108
166
|
redis.del( "queue:#{queue}" ) || true
|
109
167
|
end
|
110
168
|
|
111
|
-
#
|
169
|
+
# Returns all workers registered with Kthxbye by the Kthxbye::Worker class.
|
170
|
+
# Special note: Workers are only registered once you call #run on the worker.
|
171
|
+
# You may also run #register_worker on the worker to manually register it, but
|
172
|
+
# this also occurs once the worker is run so there is no need to run this
|
173
|
+
# manually.
|
112
174
|
def workers
|
113
175
|
workers = redis.smembers( :workers )
|
114
176
|
workers.map {|x| Worker.find( x ) }
|
115
177
|
end
|
116
178
|
|
117
|
-
#
|
179
|
+
# Returns all of the workers that are currently working a job.
|
180
|
+
# Also returns the job id and started time of the worker as a hash as follows:
|
181
|
+
#
|
182
|
+
# [worker_id, {:job_id, :started}]
|
118
183
|
def working
|
119
184
|
workers = redis.smembers( :working )
|
120
185
|
data = []
|
@@ -124,8 +189,9 @@ module Kthxbye
|
|
124
189
|
return data
|
125
190
|
end
|
126
191
|
|
127
|
-
#
|
128
|
-
#
|
192
|
+
# Returns either the job results for a specific job (if id specified).
|
193
|
+
# If a job is not specified, it returns all the job results for the given
|
194
|
+
# queue.
|
129
195
|
def job_results(queue, id=nil)
|
130
196
|
if id
|
131
197
|
decode( redis.hget( "result-store:#{queue}", id ) )
|
@@ -134,6 +200,7 @@ module Kthxbye
|
|
134
200
|
end
|
135
201
|
end
|
136
202
|
|
203
|
+
# Returns a pretty inspect message about this instance of Kthxbye.
|
137
204
|
def inspect
|
138
205
|
{
|
139
206
|
:version => Version,
|
data/lib/kthxbye/config.rb
CHANGED
@@ -1,31 +1,32 @@
|
|
1
1
|
module Kthxbye
|
2
|
+
# This module handles the setting and retrival of configuration options for
|
3
|
+
# Kthxbye.
|
2
4
|
module Config
|
3
5
|
|
4
|
-
#
|
5
|
-
#
|
6
|
-
# redis_server = the ip to connect to by defaut
|
7
|
-
#
|
8
|
-
# redis_port = default redis port
|
9
|
-
#
|
10
|
-
# attempts = default number of attempts on a failing job
|
11
|
-
# before moving to the failed job store
|
12
|
-
#
|
13
|
-
# vervose = more output
|
6
|
+
# Default options for Kthxbye
|
14
7
|
#
|
8
|
+
# * redis_server = "127.0.0.1" - the ip to connect to by defaut
|
9
|
+
# * redis_port = 9876 - default redis port
|
10
|
+
# * attempts = 1 -default number of attempts on a failing job before a Failure
|
11
|
+
# is generated
|
12
|
+
# * vervose = false - more output in the worker log
|
15
13
|
DEFAULT = {:redis_server => '127.0.0.1',
|
16
14
|
:redis_port => 9876,
|
17
15
|
:attempts => 1,
|
18
16
|
:verbose => false}.freeze
|
19
17
|
|
20
|
-
#
|
21
|
-
#
|
22
|
-
#
|
18
|
+
# This method takes override args input by the user.
|
19
|
+
# Can pull from a config/kthxbye.yaml file as well.
|
20
|
+
#
|
21
|
+
# NEEDS TO BE REWORKED FOR RAILS APPS
|
23
22
|
def self.setup( args=nil )
|
24
23
|
@options = DEFAULT.dup
|
25
|
-
@options.merge!( YAML.load('config.yaml') ) if File.exist?( 'config.yaml' )
|
24
|
+
@options.merge!( YAML.load('./config/kthxbye.yaml') ) if File.exist?( './config/kthxbye.yaml' )
|
26
25
|
@options.merge!( args ) if args
|
27
26
|
end
|
28
27
|
|
28
|
+
# Quick access to the options hash. Works for setting individual options
|
29
|
+
# during runtime.
|
29
30
|
def self.options
|
30
31
|
return @options if @options
|
31
32
|
Config.setup
|
data/lib/kthxbye/exceptions.rb
CHANGED
data/lib/kthxbye/helper.rb
CHANGED
data/lib/kthxbye/job.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
1
|
module Kthxbye
|
2
|
+
|
3
|
+
# This class is our main job runner. It also handles the instantiation and
|
4
|
+
# the meat of the job queuing and retreival.
|
2
5
|
class Job
|
3
6
|
include Helper
|
4
7
|
extend Helper
|
@@ -6,11 +9,18 @@ module Kthxbye
|
|
6
9
|
attr_accessor :id, :queue, :data, :worker
|
7
10
|
attr_reader :failed_attempts
|
8
11
|
|
9
|
-
|
12
|
+
# Adds a job to the queue from a given job id
|
13
|
+
# and queue. useful for switching a job to another queue or adding a defined
|
14
|
+
# job to multiple queues.
|
15
|
+
def self.add_to_queue(id, queue)
|
10
16
|
redis.rpush( "queue:#{queue}", id )
|
11
17
|
end
|
12
18
|
|
13
|
-
#
|
19
|
+
# The bulk of the job queuing method. Takes a string representing a queue
|
20
|
+
# name, a job class, and arguments to pass to the perform method of the
|
21
|
+
# job class. Returns a unique id for the job based on a redis "uniq_id" key
|
22
|
+
# (int) which is a simple incremented value. Queues the job in the given
|
23
|
+
# queue and the job in the data-store hash.
|
14
24
|
def self.create(queue, klass, *args)
|
15
25
|
raise "Need a queue to store job in" if queue.to_s.empty?
|
16
26
|
raise "No class to reference job type by" if klass.nil?
|
@@ -20,20 +30,23 @@ module Kthxbye
|
|
20
30
|
|
21
31
|
Job.add_to_queue( queue, id )
|
22
32
|
Kthxbye.register_queue( queue )
|
23
|
-
|
33
|
+
|
34
|
+
# mark job as inactive currently. will mark active when job is getting run
|
35
|
+
redis.redis.sadd("jobs:inactive", id)
|
24
36
|
redis.hset( "data-store:#{queue}", id, encode( {:klass => klass, :payload => args} ) )
|
25
37
|
log "Created job in queue #{queue} with an unique key of #{id}"
|
26
38
|
|
27
39
|
return id.to_i
|
28
40
|
end
|
29
41
|
|
42
|
+
# Returns a job object for a given job id off of a given queue.
|
30
43
|
def self.find(id, queue)
|
31
44
|
data = decode( redis.hget( "data-store:#{queue}", id ) )
|
32
45
|
data ? Job.new(id, queue, data) : nil
|
33
46
|
end
|
34
47
|
|
35
|
-
#
|
36
|
-
#
|
48
|
+
# Removes all existence of this job and its data
|
49
|
+
# Returns the last known status of the job
|
37
50
|
def self.destroy(id, queue)
|
38
51
|
ret = Job.find(id, queue).status
|
39
52
|
|
@@ -49,7 +62,9 @@ module Kthxbye
|
|
49
62
|
return ret
|
50
63
|
end
|
51
64
|
|
52
|
-
#
|
65
|
+
# Instantiates a job from a job id, a queue, and the job data.
|
66
|
+
# Most often used in the ::find method and for the worker to recreate
|
67
|
+
# the job for running.
|
53
68
|
def initialize(id, queue, data)
|
54
69
|
@id = id.to_i
|
55
70
|
@queue = queue
|
@@ -57,11 +72,16 @@ module Kthxbye
|
|
57
72
|
@failed_attempts = Failure.fails_for_job(@id) # local tracking only, for rerun purposes
|
58
73
|
end
|
59
74
|
|
60
|
-
#
|
75
|
+
# Simply requeues the job to be rerun.
|
61
76
|
def rerun
|
62
77
|
Job.add_to_queue( @queue, @id )
|
63
78
|
end
|
64
79
|
|
80
|
+
# Returns the job's status. Will be one of 4 things.
|
81
|
+
# 1) :succeeded - the job ran and has a result
|
82
|
+
# 2) :failed - the job failed and reported an error
|
83
|
+
# 3) :active - job is being run.
|
84
|
+
# 4) :inactive - job is waiting to be run.
|
65
85
|
def status
|
66
86
|
if result
|
67
87
|
:succeeded
|
@@ -74,28 +94,34 @@ module Kthxbye
|
|
74
94
|
end
|
75
95
|
end
|
76
96
|
|
97
|
+
# Returns the job's result once it has been run.
|
77
98
|
def result
|
78
99
|
decode( redis.hget("result-store:#{@queue}", @id) )
|
79
100
|
end
|
80
101
|
|
81
|
-
#
|
82
|
-
# on the inactive list.
|
83
|
-
#
|
102
|
+
# Simply removes this job from the active queue and places it
|
103
|
+
# on the inactive list. Does not remove job payload from storage. It just
|
104
|
+
# removes its id from the actively run job queue.
|
84
105
|
def dequeue
|
85
106
|
redis.lrem("queue:#{@queue}", 0, @id)
|
86
107
|
inactive
|
87
108
|
end
|
88
109
|
|
89
|
-
#
|
110
|
+
# Does all the heavy lifting of performing the job and storing the results.
|
111
|
+
# It will get the jobs class, payload and then run the job, storing the
|
112
|
+
# result in the result's store once complete. Also responsible for reporting
|
113
|
+
# errors and storing the job in the failure listing if an exception occurs.
|
90
114
|
def perform
|
91
115
|
begin
|
92
116
|
@klass = Object.const_get(@data['klass'])
|
93
117
|
@payload = @data['payload']
|
118
|
+
#set job active, getting ready to run
|
119
|
+
self.active
|
94
120
|
|
95
121
|
result = @klass.send(:perform, *@payload)
|
96
122
|
redis.hset( "result-store:#{@queue}", @id, encode( result ) )
|
97
123
|
return result
|
98
|
-
rescue
|
124
|
+
rescue Object => ex
|
99
125
|
@failed_attempts += 1
|
100
126
|
log "Error occured: #{ex.message}. Try: #{@failed_attempts}/#{Kthxbye::Config.options[:attempts]}"
|
101
127
|
return Kthxbye::Failure.create( self, ex ) if @failed_attempts >= Kthxbye::Config.options[:attempts]
|
@@ -103,7 +129,7 @@ module Kthxbye
|
|
103
129
|
end
|
104
130
|
end
|
105
131
|
|
106
|
-
#
|
132
|
+
# Removes the job from the inactive queue.
|
107
133
|
def active
|
108
134
|
redis.srem("jobs:inactive", @id)
|
109
135
|
end
|
@@ -112,11 +138,12 @@ module Kthxbye
|
|
112
138
|
!redis.sismember("jobs:inactive", @id)
|
113
139
|
end
|
114
140
|
|
141
|
+
# Places the job on the active queue
|
115
142
|
def inactive
|
116
143
|
redis.sadd("jobs:inactive", @id)
|
117
144
|
end
|
118
145
|
|
119
|
-
def ==(obj)
|
146
|
+
def ==(obj) #:nodoc:
|
120
147
|
return false if obj.nil?
|
121
148
|
@data == obj.data &&
|
122
149
|
@id == obj.id &&
|
data/lib/kthxbye/version.rb
CHANGED
data/lib/kthxbye/worker.rb
CHANGED
@@ -1,16 +1,24 @@
|
|
1
1
|
module Kthxbye
|
2
|
+
# This is the workhorse loop of the gem. Does all the dequeuing and running of jobs.
|
3
|
+
# It mostly makes a bunch of calls to the job methods to run the job. It simply handles
|
4
|
+
# the spawning off of processes to run the job and retry if necessary.
|
2
5
|
class Worker
|
3
6
|
include Helper
|
4
7
|
extend Helper
|
5
8
|
|
6
9
|
attr_accessor :sleep_for, :queues, :current_queue, :id
|
7
10
|
|
11
|
+
# Creates a worker for running jobs off of a given queue.
|
12
|
+
# Takes a queue or queues (csv style, e.g. test,other,bad) or the woldcard
|
13
|
+
# (*) symbol to take in all queues in alphabetical order.
|
14
|
+
# Optionally takes in an interval param on how long it waits to check the
|
15
|
+
# queue for new jobs once it has exhausted the queue(s).
|
8
16
|
def initialize(queues, sleep_for=5)
|
9
17
|
setup_queues(queues)
|
10
18
|
@sleep_for = sleep_for
|
11
19
|
end
|
12
20
|
|
13
|
-
def setup_queues(queues)
|
21
|
+
def setup_queues(queues) # :nodoc:
|
14
22
|
if queues == "*"
|
15
23
|
@queues = Kthxbye.queues.sort
|
16
24
|
elsif queues.include? ?,
|
@@ -20,6 +28,8 @@ module Kthxbye
|
|
20
28
|
end
|
21
29
|
end
|
22
30
|
|
31
|
+
# Allows us to find a worker so that we can look at some of its internal data
|
32
|
+
# later on.
|
23
33
|
def self.find(worker)
|
24
34
|
if exists? worker
|
25
35
|
qs = worker.split(':')[-1].split(",")
|
@@ -31,20 +41,21 @@ module Kthxbye
|
|
31
41
|
end
|
32
42
|
end
|
33
43
|
|
44
|
+
# Checks if a worker is registered.
|
34
45
|
def self.exists?(id)
|
35
46
|
redis.sismember( :workers, id )
|
36
47
|
end
|
37
48
|
|
38
|
-
#
|
39
|
-
#
|
49
|
+
# Gets the job a given worker is working on
|
50
|
+
# Returns a hash with the 'job_id' and the 'started' time
|
40
51
|
def self.working_on(id)
|
41
52
|
decode( redis.get( "worker:#{id}" ) )
|
42
53
|
end
|
43
54
|
|
44
|
-
# major run loop.
|
45
|
-
#
|
46
|
-
#
|
47
|
-
# to
|
55
|
+
# This is the major run loop. Workhorse of a worker... sort of.
|
56
|
+
# In the end, this loop simply runs the jobs in separate processes by forking
|
57
|
+
# out the process then waiting for it to return. we only process one.
|
58
|
+
# Can optionally take in a block to run after the job has run.
|
48
59
|
def run(&block)
|
49
60
|
log "Starting Kthxbye::Worker #{self}"
|
50
61
|
startup
|
@@ -75,23 +86,24 @@ module Kthxbye
|
|
75
86
|
unregister_worker
|
76
87
|
end
|
77
88
|
|
89
|
+
# Returns the queues this worker is attached toin alphabetical order.
|
78
90
|
def queues
|
79
91
|
@queues.sort
|
80
92
|
end
|
81
93
|
|
82
|
-
# startup actions
|
83
|
-
def startup
|
94
|
+
# Run startup actions
|
95
|
+
def startup #:nodoc:
|
84
96
|
register_worker
|
85
97
|
register_signals
|
86
98
|
end
|
87
99
|
|
88
|
-
#
|
100
|
+
# Adds this worker to the worker registry
|
89
101
|
def register_worker
|
90
102
|
log "Registered worker #{self}"
|
91
103
|
redis.sadd( :workers, self ) if !exists?
|
92
104
|
end
|
93
105
|
|
94
|
-
#
|
106
|
+
# Removes the worker from our worker registry
|
95
107
|
def unregister_worker
|
96
108
|
log "Unregistered worker #{self}"
|
97
109
|
if working?
|
@@ -104,14 +116,15 @@ module Kthxbye
|
|
104
116
|
redis.srem :workers, self
|
105
117
|
end
|
106
118
|
|
119
|
+
# Gets the current job this worker is working.
|
107
120
|
def current_job
|
108
121
|
return @current_job if @current_job
|
109
122
|
data = decode( redis.get("worker:#{self}") )
|
110
123
|
@current_job = Job.find( data['job_id'], @current_queue )
|
111
124
|
end
|
112
125
|
|
113
|
-
#
|
114
|
-
def working(job)
|
126
|
+
# Run when the job starts running
|
127
|
+
def working(job) #:nodoc:
|
115
128
|
redis.sadd( :working, self )
|
116
129
|
|
117
130
|
data = encode( {:job_id => job.id, :started => Time.now.to_s} )
|
@@ -122,13 +135,13 @@ module Kthxbye
|
|
122
135
|
job.active
|
123
136
|
end
|
124
137
|
|
125
|
-
#
|
138
|
+
# Is this job working?
|
126
139
|
def working?
|
127
140
|
redis.sismember( :working, self )
|
128
141
|
end
|
129
142
|
|
130
143
|
# job complete actions
|
131
|
-
def done
|
144
|
+
def done #:nodoc:
|
132
145
|
redis.srem( :working, self )
|
133
146
|
redis.del( "worker:#{self}" )
|
134
147
|
log "Completed job #{@current_job}"
|
@@ -138,7 +151,7 @@ module Kthxbye
|
|
138
151
|
#
|
139
152
|
# thanks to http://github.com/defunkt/resque/blob/master/lib/resque/worker.rb for these signals
|
140
153
|
#
|
141
|
-
def register_signals
|
154
|
+
def register_signals #:nordoc:
|
142
155
|
trap('TERM') { shutdown! }
|
143
156
|
trap('INT') { shutdown! }
|
144
157
|
|
@@ -154,17 +167,19 @@ module Kthxbye
|
|
154
167
|
log "Registered signals"
|
155
168
|
end
|
156
169
|
|
157
|
-
|
170
|
+
# Shuts down the worker gracefully (once process has completed
|
171
|
+
def shutdown #:nodoc:
|
158
172
|
log "Shutting down worker #{self}"
|
159
173
|
@terminate = true
|
160
174
|
end
|
161
175
|
|
162
|
-
|
176
|
+
# Hard kills the worker by killing the process.
|
177
|
+
def shutdown! #:nodoc:
|
163
178
|
kill_child
|
164
179
|
shutdown
|
165
180
|
end
|
166
181
|
|
167
|
-
def kill_child
|
182
|
+
def kill_child #:nodoc:
|
168
183
|
if @child
|
169
184
|
log "Killing child at #{@child}"
|
170
185
|
if system("ps -o pid,state -p #{@child}")
|
@@ -176,7 +191,8 @@ module Kthxbye
|
|
176
191
|
end
|
177
192
|
end
|
178
193
|
|
179
|
-
|
194
|
+
# Reserves a job off the queue
|
195
|
+
def grab_job #:nodoc:
|
180
196
|
job = nil
|
181
197
|
@queues.each do |q|
|
182
198
|
@current_queue = q
|
@@ -188,32 +204,38 @@ module Kthxbye
|
|
188
204
|
return job || false
|
189
205
|
end
|
190
206
|
|
191
|
-
|
207
|
+
# Checks if this worker is registered.
|
208
|
+
def exists? #:nodoc:
|
192
209
|
redis.sismember( :workers, self )
|
193
210
|
end
|
194
211
|
|
212
|
+
# Returns the hostname of the machine this worker is running on
|
195
213
|
def hostname
|
196
214
|
@hostname ||= `hostname`.chomp
|
197
215
|
end
|
198
216
|
|
217
|
+
# Returns the process id of this worker.
|
199
218
|
def pid
|
200
219
|
Process.pid
|
201
220
|
end
|
202
221
|
|
222
|
+
# Returns a useful id with the hostname:pid:queues listing
|
223
|
+
# Same return as to_s
|
203
224
|
def id
|
204
225
|
@id ||= "#{hostname}:#{pid}:#{queues.join(",")}"
|
205
226
|
end
|
206
227
|
alias_method :to_s, :id
|
207
228
|
|
229
|
+
# nice inspect for the worker with the same info as #id
|
208
230
|
def inspect
|
209
231
|
"#<Worker: #{@id}>"
|
210
232
|
end
|
211
233
|
|
212
|
-
def ==(other)
|
234
|
+
def ==(other) #:nodoc:
|
213
235
|
to_s == other.to_s
|
214
236
|
end
|
215
237
|
|
216
|
-
def <=>(other)
|
238
|
+
def <=>(other) #:nodoc:
|
217
239
|
to_s <=> other.to_s
|
218
240
|
end
|
219
241
|
|
metadata
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kthxbye
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash: 21
|
5
4
|
prerelease: false
|
6
5
|
segments:
|
7
6
|
- 1
|
8
7
|
- 0
|
9
|
-
-
|
10
|
-
version: 1.0.
|
8
|
+
- 2
|
9
|
+
version: 1.0.2
|
11
10
|
platform: ruby
|
12
11
|
authors:
|
13
12
|
- Luke van der Hoeven
|
@@ -15,112 +14,105 @@ autorequire:
|
|
15
14
|
bindir: bin
|
16
15
|
cert_chain: []
|
17
16
|
|
18
|
-
date: 2010-09-
|
17
|
+
date: 2010-09-24 00:00:00 -04:00
|
19
18
|
default_executable:
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
22
|
-
prerelease: false
|
23
21
|
name: redis
|
24
|
-
|
22
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
23
|
none: false
|
26
24
|
requirements:
|
27
25
|
- - ">="
|
28
26
|
- !ruby/object:Gem::Version
|
29
|
-
hash: 3
|
30
27
|
segments:
|
31
28
|
- 0
|
32
29
|
version: "0"
|
33
|
-
requirement: *id001
|
34
30
|
type: :runtime
|
35
|
-
- !ruby/object:Gem::Dependency
|
36
31
|
prerelease: false
|
32
|
+
version_requirements: *id001
|
33
|
+
- !ruby/object:Gem::Dependency
|
37
34
|
name: yajl-ruby
|
38
|
-
|
35
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
36
|
none: false
|
40
37
|
requirements:
|
41
38
|
- - ">="
|
42
39
|
- !ruby/object:Gem::Version
|
43
|
-
hash: 3
|
44
40
|
segments:
|
45
41
|
- 0
|
46
42
|
version: "0"
|
47
|
-
requirement: *id002
|
48
43
|
type: :runtime
|
49
|
-
- !ruby/object:Gem::Dependency
|
50
44
|
prerelease: false
|
45
|
+
version_requirements: *id002
|
46
|
+
- !ruby/object:Gem::Dependency
|
51
47
|
name: sinatra
|
52
|
-
|
48
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
53
49
|
none: false
|
54
50
|
requirements:
|
55
51
|
- - ">="
|
56
52
|
- !ruby/object:Gem::Version
|
57
|
-
hash: 3
|
58
53
|
segments:
|
59
54
|
- 0
|
60
55
|
version: "0"
|
61
|
-
requirement: *id003
|
62
56
|
type: :runtime
|
63
|
-
- !ruby/object:Gem::Dependency
|
64
57
|
prerelease: false
|
58
|
+
version_requirements: *id003
|
59
|
+
- !ruby/object:Gem::Dependency
|
65
60
|
name: shoulda
|
66
|
-
|
61
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
67
62
|
none: false
|
68
63
|
requirements:
|
69
64
|
- - ">="
|
70
65
|
- !ruby/object:Gem::Version
|
71
|
-
hash: 3
|
72
66
|
segments:
|
73
67
|
- 0
|
74
68
|
version: "0"
|
75
|
-
requirement: *id004
|
76
69
|
type: :development
|
77
|
-
- !ruby/object:Gem::Dependency
|
78
70
|
prerelease: false
|
71
|
+
version_requirements: *id004
|
72
|
+
- !ruby/object:Gem::Dependency
|
79
73
|
name: bundler
|
80
|
-
|
74
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
81
75
|
none: false
|
82
76
|
requirements:
|
83
77
|
- - ~>
|
84
78
|
- !ruby/object:Gem::Version
|
85
|
-
hash: 23
|
86
79
|
segments:
|
87
80
|
- 1
|
88
81
|
- 0
|
89
82
|
- 0
|
90
83
|
version: 1.0.0
|
91
|
-
requirement: *id005
|
92
84
|
type: :development
|
93
|
-
- !ruby/object:Gem::Dependency
|
94
85
|
prerelease: false
|
86
|
+
version_requirements: *id005
|
87
|
+
- !ruby/object:Gem::Dependency
|
95
88
|
name: jeweler
|
96
|
-
|
89
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
97
90
|
none: false
|
98
91
|
requirements:
|
99
92
|
- - ~>
|
100
93
|
- !ruby/object:Gem::Version
|
101
|
-
hash: 270495430
|
102
94
|
segments:
|
103
95
|
- 1
|
104
96
|
- 5
|
105
97
|
- 0
|
106
98
|
- pre3
|
107
99
|
version: 1.5.0.pre3
|
108
|
-
requirement: *id006
|
109
100
|
type: :development
|
110
|
-
- !ruby/object:Gem::Dependency
|
111
101
|
prerelease: false
|
102
|
+
version_requirements: *id006
|
103
|
+
- !ruby/object:Gem::Dependency
|
112
104
|
name: rcov
|
113
|
-
|
105
|
+
requirement: &id007 !ruby/object:Gem::Requirement
|
114
106
|
none: false
|
115
107
|
requirements:
|
116
108
|
- - ">="
|
117
109
|
- !ruby/object:Gem::Version
|
118
|
-
hash: 3
|
119
110
|
segments:
|
120
111
|
- 0
|
121
112
|
version: "0"
|
122
|
-
requirement: *id007
|
123
113
|
type: :development
|
114
|
+
prerelease: false
|
115
|
+
version_requirements: *id007
|
124
116
|
description: "Kthxbye is the answer to a fairly unique-yet-common problem: Background job processing when we care about the result."
|
125
117
|
email: hungerandthirst@gmail.com
|
126
118
|
executables: []
|
@@ -185,7 +177,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
185
177
|
requirements:
|
186
178
|
- - ">="
|
187
179
|
- !ruby/object:Gem::Version
|
188
|
-
hash:
|
180
|
+
hash: 387221811
|
189
181
|
segments:
|
190
182
|
- 0
|
191
183
|
version: "0"
|
@@ -194,7 +186,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
194
186
|
requirements:
|
195
187
|
- - ">="
|
196
188
|
- !ruby/object:Gem::Version
|
197
|
-
hash: 3
|
198
189
|
segments:
|
199
190
|
- 0
|
200
191
|
version: "0"
|