navvy 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/generators/navvy/navvy_generator.rb +2 -2
- data/generators/navvy/templates/script +4 -0
- data/lib/generators/navvy_generator.rb +9 -3
- data/lib/navvy/job.rb +156 -0
- data/lib/navvy/job/active_record.rb +9 -143
- data/lib/navvy/job/data_mapper.rb +18 -143
- data/lib/navvy/job/mongo_mapper.rb +9 -125
- data/lib/navvy/job/sequel.rb +18 -138
- data/lib/navvy/log.rb +3 -3
- data/lib/navvy/worker.rb +33 -1
- data/navvy.gemspec +4 -2
- data/spec/job_spec.rb +44 -20
- data/spec/log_spec.rb +4 -4
- data/spec/setup/rails_default_logger.rb +8 -4
- metadata +4 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.1
|
@@ -4,11 +4,11 @@ class NavvyGenerator < Rails::Generator::Base
|
|
4
4
|
options = {
|
5
5
|
:migration_file_name => 'create_jobs'
|
6
6
|
}
|
7
|
-
|
8
7
|
m.migration_template 'migration.rb', 'db/migrate', options
|
8
|
+
m.file 'script', 'script/navvy', :chmod => 0755
|
9
9
|
end
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
def banner
|
13
13
|
"Usage: #{$0} #{spec.name}"
|
14
14
|
end
|
@@ -1,18 +1,24 @@
|
|
1
1
|
require 'rails/generators/migration'
|
2
2
|
class NavvyGenerator < Rails::Generators::Base
|
3
3
|
include Rails::Generators::Migration
|
4
|
-
|
4
|
+
|
5
5
|
def self.source_root
|
6
6
|
File.join(File.dirname(__FILE__), '..', '..', 'generators', 'navvy', 'templates')
|
7
7
|
end
|
8
|
-
|
8
|
+
|
9
9
|
def install_navvy
|
10
10
|
migration_template(
|
11
11
|
'migration.rb',
|
12
12
|
'db/migrate/create_navvy_table.rb'
|
13
13
|
)
|
14
|
+
|
15
|
+
copy_file(
|
16
|
+
'script',
|
17
|
+
'script/navvy',
|
18
|
+
:chmod => 0755
|
19
|
+
)
|
14
20
|
end
|
15
|
-
|
21
|
+
|
16
22
|
protected
|
17
23
|
def next_migration_number(dirname) #:nodoc:
|
18
24
|
"%.3d" % (current_migration_number(dirname) + 1)
|
data/lib/navvy/job.rb
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
module Navvy
|
2
|
+
class Job
|
3
|
+
class << self
|
4
|
+
attr_writer :limit, :keep, :max_attempts
|
5
|
+
end
|
6
|
+
|
7
|
+
##
|
8
|
+
# Limit of jobs to be fetched at once. Will use the value stored in
|
9
|
+
# Navvy.configuration (defaults to 100), or -- for backwards compatibility
|
10
|
+
# -- Navvy::Job.limit.
|
11
|
+
#
|
12
|
+
# @return [Integer] limit
|
13
|
+
|
14
|
+
def self.limit
|
15
|
+
@limit || Navvy.configuration.job_limit
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# If and how long the jobs should be kept. Will use the value stored in
|
20
|
+
# Navvy.configuration (defaults to false), or -- for backwards
|
21
|
+
# compatibility -- Navvy::Job.keep.
|
22
|
+
#
|
23
|
+
# @return [Fixnum, true, false] keep
|
24
|
+
|
25
|
+
def self.keep
|
26
|
+
@keep || Navvy.configuration.keep_jobs
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# How often should a job be retried? Will use the value stored in
|
31
|
+
# Navvy.configuration (defaults to 24), or -- for backwards compatibility
|
32
|
+
# -- Navvy::Job.max_attempts.
|
33
|
+
#
|
34
|
+
# @return [Fixnum] max_attempts
|
35
|
+
|
36
|
+
def self.max_attempts
|
37
|
+
@max_attempts || Navvy.configuration.max_attempts
|
38
|
+
end
|
39
|
+
|
40
|
+
##
|
41
|
+
# Should the job be kept? Will calculate if the keeptime has passed if
|
42
|
+
# @keep is a Fixnum. Otherwise, it'll just return the @keep value since
|
43
|
+
# it's probably a boolean.
|
44
|
+
#
|
45
|
+
# @return [true, false] keep
|
46
|
+
|
47
|
+
def self.keep?
|
48
|
+
return (Time.now + self.keep) >= Time.now if self.keep.is_a? Fixnum
|
49
|
+
self.keep
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Run the job. Will delete the Navvy::Job record and return its return
|
54
|
+
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
55
|
+
# fails, it'll call Navvy::Job#failed and pass the exception message.
|
56
|
+
# Failed jobs will _not_ get deleted.
|
57
|
+
#
|
58
|
+
# @example
|
59
|
+
# job = Navvy::Job.next # finds the next available job in the queue
|
60
|
+
# job.run # runs the job and returns the job's return value
|
61
|
+
#
|
62
|
+
# @return [String] return value or exception message of the called method.
|
63
|
+
|
64
|
+
def run
|
65
|
+
begin
|
66
|
+
started
|
67
|
+
result = Kernel.const_get(object).send(method_name, *args)
|
68
|
+
Navvy::Job.keep? ? completed : destroy
|
69
|
+
result
|
70
|
+
rescue Exception => exception
|
71
|
+
failed(exception.message)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
##
|
76
|
+
# Retry the current job. Will add self to the queue again, giving the clone
|
77
|
+
# a parend_id equal to self.id. Also, the priority of the new job will be
|
78
|
+
# the same as its parent's and it'll set the :run_at date to N ** 4, where
|
79
|
+
# N is the times_failed count.
|
80
|
+
#
|
81
|
+
# @return [Navvy::Job] job the new job it created.
|
82
|
+
|
83
|
+
def retry
|
84
|
+
self.class.enqueue(
|
85
|
+
object,
|
86
|
+
method_name,
|
87
|
+
*(args << {
|
88
|
+
:job_options => {
|
89
|
+
:parent_id => parent_id || id,
|
90
|
+
:run_at => Time.now + times_failed ** 4,
|
91
|
+
:priority => priority
|
92
|
+
}
|
93
|
+
})
|
94
|
+
)
|
95
|
+
end
|
96
|
+
|
97
|
+
##
|
98
|
+
# Check if the job has been run.
|
99
|
+
#
|
100
|
+
# @return [true, false] ran
|
101
|
+
|
102
|
+
def ran?
|
103
|
+
completed? || failed?
|
104
|
+
end
|
105
|
+
|
106
|
+
##
|
107
|
+
# Check how long it took for a job to complete or fail.
|
108
|
+
#
|
109
|
+
# @return [Time, Integer] time the time it took.
|
110
|
+
|
111
|
+
def duration
|
112
|
+
ran? ? (completed_at || failed_at) - started_at : 0
|
113
|
+
end
|
114
|
+
|
115
|
+
##
|
116
|
+
# Check if completed_at is set.
|
117
|
+
#
|
118
|
+
# @return [true, false] set?
|
119
|
+
|
120
|
+
def completed_at?
|
121
|
+
!completed_at.nil?
|
122
|
+
end
|
123
|
+
|
124
|
+
##
|
125
|
+
# Check if failed_at is set.
|
126
|
+
#
|
127
|
+
# @return [true, false] set?
|
128
|
+
|
129
|
+
def failed_at?
|
130
|
+
!failed_at.nil?
|
131
|
+
end
|
132
|
+
|
133
|
+
##
|
134
|
+
# Get the job arguments as an array.
|
135
|
+
#
|
136
|
+
# @return [array] arguments
|
137
|
+
|
138
|
+
def args
|
139
|
+
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
140
|
+
end
|
141
|
+
|
142
|
+
##
|
143
|
+
# Get the job status
|
144
|
+
#
|
145
|
+
# @return [:pending, :completed, :failed] status
|
146
|
+
|
147
|
+
def status
|
148
|
+
return :completed if completed?
|
149
|
+
return :failed if failed?
|
150
|
+
:pending
|
151
|
+
end
|
152
|
+
|
153
|
+
alias_method :completed?, :completed_at?
|
154
|
+
alias_method :failed?, :failed_at?
|
155
|
+
end
|
156
|
+
end
|
@@ -3,47 +3,6 @@ require 'active_record'
|
|
3
3
|
|
4
4
|
module Navvy
|
5
5
|
class Job < ActiveRecord::Base
|
6
|
-
class << self
|
7
|
-
attr_writer :limit, :keep, :max_attempts
|
8
|
-
end
|
9
|
-
|
10
|
-
##
|
11
|
-
# Default limit of jobs to be fetched.
|
12
|
-
#
|
13
|
-
# @return [Integer] limit
|
14
|
-
|
15
|
-
def self.limit
|
16
|
-
@limit || Navvy.configuration.job_limit
|
17
|
-
end
|
18
|
-
|
19
|
-
##
|
20
|
-
# If and how long the jobs should be kept.
|
21
|
-
#
|
22
|
-
# @return [Fixnum, true, false] keep
|
23
|
-
|
24
|
-
def self.keep
|
25
|
-
@keep || Navvy.configuration.keep_jobs
|
26
|
-
end
|
27
|
-
|
28
|
-
##
|
29
|
-
# How often should a job be retried?
|
30
|
-
#
|
31
|
-
# @return [Fixnum] max_attempts
|
32
|
-
|
33
|
-
def self.max_attempts
|
34
|
-
@max_attempts || Navvy.configuration.max_attempts
|
35
|
-
end
|
36
|
-
|
37
|
-
##
|
38
|
-
# Should the job be kept?
|
39
|
-
#
|
40
|
-
# @return [true, false] keep
|
41
|
-
|
42
|
-
def self.keep?
|
43
|
-
keep = (self.keep || false)
|
44
|
-
return keep.from_now >= Time.now if keep.is_a? Fixnum
|
45
|
-
keep
|
46
|
-
end
|
47
6
|
|
48
7
|
##
|
49
8
|
# Add a job to the job queue.
|
@@ -115,31 +74,15 @@ module Navvy
|
|
115
74
|
end
|
116
75
|
|
117
76
|
##
|
118
|
-
#
|
119
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
120
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
121
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
122
|
-
# deleted.
|
77
|
+
# Mark the job as started. Will set started_at to the current time.
|
123
78
|
#
|
124
|
-
# @
|
125
|
-
#
|
126
|
-
# job.run # runs the job and returns the job's return value
|
127
|
-
#
|
128
|
-
# @return [String] return value of the called method.
|
79
|
+
# @return [true, false] update_attributes the result of the
|
80
|
+
# update_attributes call
|
129
81
|
|
130
|
-
def
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
result = object.constantize.send(method_name)
|
135
|
-
else
|
136
|
-
result = object.constantize.send(method_name, *args)
|
137
|
-
end
|
138
|
-
Navvy::Job.keep? ? completed : destroy
|
139
|
-
result
|
140
|
-
rescue Exception => exception
|
141
|
-
failed(exception.message)
|
142
|
-
end
|
82
|
+
def started
|
83
|
+
update_attributes({
|
84
|
+
:started_at => Time.now
|
85
|
+
})
|
143
86
|
end
|
144
87
|
|
145
88
|
##
|
@@ -176,26 +119,6 @@ module Navvy
|
|
176
119
|
)
|
177
120
|
end
|
178
121
|
|
179
|
-
##
|
180
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
181
|
-
# a parend_id equal to self.id.
|
182
|
-
#
|
183
|
-
# @return [true, false]
|
184
|
-
|
185
|
-
def retry
|
186
|
-
self.class.enqueue(
|
187
|
-
object,
|
188
|
-
method_name,
|
189
|
-
*(args << {
|
190
|
-
:job_options => {
|
191
|
-
:parent_id => parent_id || id,
|
192
|
-
:run_at => Time.now + times_failed ** 4,
|
193
|
-
:priority => priority
|
194
|
-
}
|
195
|
-
})
|
196
|
-
)
|
197
|
-
end
|
198
|
-
|
199
122
|
##
|
200
123
|
# Check how many times the job has failed. Will try to find jobs with a
|
201
124
|
# parent_id that's the same as self.id and count them
|
@@ -208,64 +131,7 @@ module Navvy
|
|
208
131
|
:conditions => "(`id` = '#{i}' OR `parent_id` = '#{i}') AND `failed_at` IS NOT NULL"
|
209
132
|
)
|
210
133
|
end
|
211
|
-
|
212
|
-
##
|
213
|
-
# Check if the job has been run.
|
214
|
-
#
|
215
|
-
# @return [true, false] ran
|
216
|
-
|
217
|
-
def ran?
|
218
|
-
completed? || failed?
|
219
|
-
end
|
220
|
-
|
221
|
-
##
|
222
|
-
# Check how long it took for a job to complete or fail
|
223
|
-
#
|
224
|
-
# @return [Time, Integer] time the time it took
|
225
|
-
|
226
|
-
def duration
|
227
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
228
|
-
end
|
229
|
-
|
230
|
-
##
|
231
|
-
# Check if completed_at is set
|
232
|
-
#
|
233
|
-
# @return [true, false] set?
|
234
|
-
|
235
|
-
def completed_at?
|
236
|
-
!completed_at.nil?
|
237
|
-
end
|
238
|
-
|
239
|
-
##
|
240
|
-
# Check if failed_at is set
|
241
|
-
#
|
242
|
-
# @return [true, false] set?
|
243
|
-
|
244
|
-
def failed_at?
|
245
|
-
!failed_at.nil?
|
246
|
-
end
|
247
|
-
|
248
|
-
##
|
249
|
-
# Get the job arguments as an array
|
250
|
-
#
|
251
|
-
# @return [array] arguments
|
252
|
-
|
253
|
-
def args
|
254
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
255
|
-
end
|
256
|
-
|
257
|
-
##
|
258
|
-
# Get the job status
|
259
|
-
#
|
260
|
-
# @return [:pending, :completed, :failed] status
|
261
|
-
|
262
|
-
def status
|
263
|
-
return :completed if completed?
|
264
|
-
return :failed if failed?
|
265
|
-
:pending
|
266
|
-
end
|
267
|
-
|
268
|
-
alias_method :completed?, :completed_at?
|
269
|
-
alias_method :failed?, :failed_at?
|
270
134
|
end
|
271
135
|
end
|
136
|
+
|
137
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
@@ -4,9 +4,6 @@ require 'dm-core'
|
|
4
4
|
module Navvy
|
5
5
|
class Job
|
6
6
|
include DataMapper::Resource
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
7
|
|
11
8
|
property :id, Serial
|
12
9
|
property :object, String
|
@@ -22,44 +19,6 @@ module Navvy
|
|
22
19
|
property :completed_at, Time
|
23
20
|
property :failed_at, Time
|
24
21
|
|
25
|
-
##
|
26
|
-
# Default limit of jobs to be fetched.
|
27
|
-
#
|
28
|
-
# @return [Integer] limit
|
29
|
-
|
30
|
-
def self.limit
|
31
|
-
@limit || Navvy.configuration.job_limit
|
32
|
-
end
|
33
|
-
|
34
|
-
##
|
35
|
-
# If and how long the jobs should be kept.
|
36
|
-
#
|
37
|
-
# @return [Fixnum, true, false] keep
|
38
|
-
|
39
|
-
def self.keep
|
40
|
-
@keep || Navvy.configuration.keep_jobs
|
41
|
-
end
|
42
|
-
|
43
|
-
##
|
44
|
-
# How often should a job be retried?
|
45
|
-
#
|
46
|
-
# @return [Fixnum] max_attempts
|
47
|
-
|
48
|
-
def self.max_attempts
|
49
|
-
@max_attempts || Navvy.configuration.max_attempts
|
50
|
-
end
|
51
|
-
|
52
|
-
##
|
53
|
-
# Should the job be kept?
|
54
|
-
#
|
55
|
-
# @return [true, false] keep
|
56
|
-
|
57
|
-
def self.keep?
|
58
|
-
keep = (self.keep || false)
|
59
|
-
return (Time.now + keep) >= Time.now if keep.is_a? Fixnum
|
60
|
-
keep
|
61
|
-
end
|
62
|
-
|
63
22
|
##
|
64
23
|
# Add a job to the job queue.
|
65
24
|
#
|
@@ -112,7 +71,7 @@ module Navvy
|
|
112
71
|
)
|
113
72
|
end
|
114
73
|
|
115
|
-
##
|
74
|
+
##
|
116
75
|
# Clean up jobs that we don't need to keep anymore. If Navvy::Job.keep is
|
117
76
|
# false it'll delete every completed job, if it's a timestamp it'll only
|
118
77
|
# delete completed jobs that have passed their keeptime.
|
@@ -128,31 +87,24 @@ module Navvy
|
|
128
87
|
end
|
129
88
|
|
130
89
|
##
|
131
|
-
#
|
132
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
133
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
134
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
135
|
-
# deleted.
|
90
|
+
# Deletes all jobs.
|
136
91
|
#
|
137
|
-
# @
|
138
|
-
|
139
|
-
|
92
|
+
# @return [true, false] deleted?
|
93
|
+
|
94
|
+
def self.delete_all
|
95
|
+
Navvy::Job.all.destroy
|
96
|
+
end
|
97
|
+
|
98
|
+
##
|
99
|
+
# Mark the job as started. Will set started_at to the current time.
|
140
100
|
#
|
141
|
-
# @return [
|
101
|
+
# @return [true, false] update_attributes the result of the
|
102
|
+
# update_attributes call
|
142
103
|
|
143
|
-
def
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result = Kernel.const_get(object).send(method_name)
|
148
|
-
else
|
149
|
-
result = Kernel.const_get(object).send(method_name, *args)
|
150
|
-
end
|
151
|
-
Navvy::Job.keep? ? completed : destroy
|
152
|
-
result
|
153
|
-
rescue Exception => exception
|
154
|
-
failed(exception.message)
|
155
|
-
end
|
104
|
+
def started
|
105
|
+
update({
|
106
|
+
:started_at => Time.now
|
107
|
+
})
|
156
108
|
end
|
157
109
|
|
158
110
|
##
|
@@ -190,26 +142,6 @@ module Navvy
|
|
190
142
|
)
|
191
143
|
end
|
192
144
|
|
193
|
-
##
|
194
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
195
|
-
# a parend_id equal to self.id.
|
196
|
-
#
|
197
|
-
# @return [true, false]
|
198
|
-
|
199
|
-
def retry
|
200
|
-
self.class.enqueue(
|
201
|
-
object,
|
202
|
-
method_name,
|
203
|
-
*(args << {
|
204
|
-
:job_options => {
|
205
|
-
:parent_id => parent_id || id,
|
206
|
-
:run_at => Time.now + times_failed ** 4,
|
207
|
-
:priority => priority
|
208
|
-
}
|
209
|
-
})
|
210
|
-
)
|
211
|
-
end
|
212
|
-
|
213
145
|
##
|
214
146
|
# Check how many times the job has failed. Will try to find jobs with a
|
215
147
|
# parent_id that's the same as self.id and count them
|
@@ -222,64 +154,7 @@ module Navvy
|
|
222
154
|
:conditions => ["(`id` = ? OR `parent_id` = ?) AND `failed_at` IS NOT NULL", i, i]
|
223
155
|
).count
|
224
156
|
end
|
225
|
-
|
226
|
-
##
|
227
|
-
# Check if the job has been run.
|
228
|
-
#
|
229
|
-
# @return [true, false] ran
|
230
|
-
|
231
|
-
def ran?
|
232
|
-
completed? || failed?
|
233
|
-
end
|
234
|
-
|
235
|
-
##
|
236
|
-
# Check if completed_at is set
|
237
|
-
#
|
238
|
-
# @return [true, false] set?
|
239
|
-
|
240
|
-
def completed_at?
|
241
|
-
!completed_at.nil?
|
242
|
-
end
|
243
|
-
|
244
|
-
##
|
245
|
-
# Check if failed_at is set
|
246
|
-
#
|
247
|
-
# @return [true, false] set?
|
248
|
-
|
249
|
-
def failed_at?
|
250
|
-
!failed_at.nil?
|
251
|
-
end
|
252
|
-
|
253
|
-
##
|
254
|
-
# Check how long it took for a job to complete or fail
|
255
|
-
#
|
256
|
-
# @return [Time, Integer] time the time it took
|
257
|
-
|
258
|
-
def duration
|
259
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
260
|
-
end
|
261
|
-
|
262
|
-
##
|
263
|
-
# Get the job arguments as an array
|
264
|
-
#
|
265
|
-
# @return [array] arguments
|
266
|
-
|
267
|
-
def args
|
268
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
269
|
-
end
|
270
|
-
|
271
|
-
##
|
272
|
-
# Get the job status
|
273
|
-
#
|
274
|
-
# @return [:pending, :completed, :failed] status
|
275
|
-
|
276
|
-
def status
|
277
|
-
return :completed if completed?
|
278
|
-
return :failed if failed?
|
279
|
-
:pending
|
280
|
-
end
|
281
|
-
|
282
|
-
alias_method :completed?, :completed_at?
|
283
|
-
alias_method :failed?, :failed_at?
|
284
157
|
end
|
285
158
|
end
|
159
|
+
|
160
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
@@ -4,9 +4,6 @@ require 'mongo_mapper'
|
|
4
4
|
module Navvy
|
5
5
|
class Job
|
6
6
|
include MongoMapper::Document
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
7
|
|
11
8
|
key :object, String
|
12
9
|
key :method_name, Symbol
|
@@ -21,44 +18,6 @@ module Navvy
|
|
21
18
|
key :completed_at, Time
|
22
19
|
key :failed_at, Time
|
23
20
|
|
24
|
-
##
|
25
|
-
# Default limit of jobs to be fetched.
|
26
|
-
#
|
27
|
-
# @return [Integer] limit
|
28
|
-
|
29
|
-
def self.limit
|
30
|
-
@limit || Navvy.configuration.job_limit
|
31
|
-
end
|
32
|
-
|
33
|
-
##
|
34
|
-
# If and how long the jobs should be kept.
|
35
|
-
#
|
36
|
-
# @return [Fixnum, true, false] keep
|
37
|
-
|
38
|
-
def self.keep
|
39
|
-
@keep || Navvy.configuration.keep_jobs
|
40
|
-
end
|
41
|
-
|
42
|
-
##
|
43
|
-
# How often should a job be retried?
|
44
|
-
#
|
45
|
-
# @return [Fixnum] max_attempts
|
46
|
-
|
47
|
-
def self.max_attempts
|
48
|
-
@max_attempts || Navvy.configuration.max_attempts
|
49
|
-
end
|
50
|
-
|
51
|
-
##
|
52
|
-
# Should the job be kept?
|
53
|
-
#
|
54
|
-
# @return [true, false] keep
|
55
|
-
|
56
|
-
def self.keep?
|
57
|
-
keep = (self.keep || false)
|
58
|
-
return keep.from_now >= Time.now if keep.is_a? Fixnum
|
59
|
-
keep
|
60
|
-
end
|
61
|
-
|
62
21
|
##
|
63
22
|
# Add a job to the job queue.
|
64
23
|
#
|
@@ -128,31 +87,15 @@ module Navvy
|
|
128
87
|
end
|
129
88
|
|
130
89
|
##
|
131
|
-
#
|
132
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
133
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
134
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
135
|
-
# deleted.
|
90
|
+
# Mark the job as started. Will set started_at to the current time.
|
136
91
|
#
|
137
|
-
# @
|
138
|
-
#
|
139
|
-
# job.run # runs the job and returns the job's return value
|
140
|
-
#
|
141
|
-
# @return [String] return value of the called method.
|
92
|
+
# @return [true, false] update_attributes the result of the
|
93
|
+
# update_attributes call
|
142
94
|
|
143
|
-
def
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result = object.constantize.send(method_name)
|
148
|
-
else
|
149
|
-
result = object.constantize.send(method_name, *args)
|
150
|
-
end
|
151
|
-
Navvy::Job.keep? ? completed : destroy
|
152
|
-
result
|
153
|
-
rescue Exception => exception
|
154
|
-
failed(exception.message)
|
155
|
-
end
|
95
|
+
def started
|
96
|
+
update_attributes({
|
97
|
+
:started_at => Time.now
|
98
|
+
})
|
156
99
|
end
|
157
100
|
|
158
101
|
##
|
@@ -189,26 +132,6 @@ module Navvy
|
|
189
132
|
)
|
190
133
|
end
|
191
134
|
|
192
|
-
##
|
193
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
194
|
-
# a parend_id equal to self.id.
|
195
|
-
#
|
196
|
-
# @return [true, false]
|
197
|
-
|
198
|
-
def retry
|
199
|
-
self.class.enqueue(
|
200
|
-
object,
|
201
|
-
method_name,
|
202
|
-
*(args << {
|
203
|
-
:job_options => {
|
204
|
-
:parent_id => parent_id || id,
|
205
|
-
:run_at => Time.now + times_failed ** 4,
|
206
|
-
:priority => priority
|
207
|
-
}
|
208
|
-
})
|
209
|
-
)
|
210
|
-
end
|
211
|
-
|
212
135
|
##
|
213
136
|
# Check how many times the job has failed. Will try to find jobs with a
|
214
137
|
# parent_id that's the same as self.id and count them
|
@@ -222,46 +145,7 @@ module Navvy
|
|
222
145
|
'$where' => "this._id == '#{i}' || this.parent_id == '#{i}'"
|
223
146
|
)
|
224
147
|
end
|
225
|
-
|
226
|
-
##
|
227
|
-
# Check if the job has been run.
|
228
|
-
#
|
229
|
-
# @return [true, false] ran
|
230
|
-
|
231
|
-
def ran?
|
232
|
-
completed? || failed?
|
233
|
-
end
|
234
|
-
|
235
|
-
##
|
236
|
-
# Check how long it took for a job to complete or fail
|
237
|
-
#
|
238
|
-
# @return [Time, Integer] time the time it took
|
239
|
-
|
240
|
-
def duration
|
241
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
242
|
-
end
|
243
|
-
|
244
|
-
##
|
245
|
-
# Get the job arguments as an array
|
246
|
-
#
|
247
|
-
# @return [array] arguments
|
248
|
-
|
249
|
-
def args
|
250
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
251
|
-
end
|
252
|
-
|
253
|
-
##
|
254
|
-
# Get the job status
|
255
|
-
#
|
256
|
-
# @return [:pending, :completed, :failed] status
|
257
|
-
|
258
|
-
def status
|
259
|
-
return :completed if completed?
|
260
|
-
return :failed if failed?
|
261
|
-
:pending
|
262
|
-
end
|
263
|
-
|
264
|
-
alias_method :completed?, :completed_at?
|
265
|
-
alias_method :failed?, :failed_at?
|
266
148
|
end
|
267
149
|
end
|
150
|
+
|
151
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
data/lib/navvy/job/sequel.rb
CHANGED
@@ -4,47 +4,6 @@ require 'yaml'
|
|
4
4
|
|
5
5
|
module Navvy
|
6
6
|
class Job < Sequel::Model
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
|
-
|
11
|
-
##
|
12
|
-
# Default limit of jobs to be fetched.
|
13
|
-
#
|
14
|
-
# @return [Integer] limit
|
15
|
-
|
16
|
-
def self.limit
|
17
|
-
@limit || Navvy.configuration.job_limit
|
18
|
-
end
|
19
|
-
|
20
|
-
##
|
21
|
-
# If and how long the jobs should be kept.
|
22
|
-
#
|
23
|
-
# @return [Fixnum, true, false] keep
|
24
|
-
|
25
|
-
def self.keep
|
26
|
-
@keep || Navvy.configuration.keep_jobs
|
27
|
-
end
|
28
|
-
|
29
|
-
##
|
30
|
-
# How often should a job be retried?
|
31
|
-
#
|
32
|
-
# @return [Fixnum] max_attempts
|
33
|
-
|
34
|
-
def self.max_attempts
|
35
|
-
@max_attempts || Navvy.configuration.max_attempts
|
36
|
-
end
|
37
|
-
|
38
|
-
##
|
39
|
-
# Should the job be kept?
|
40
|
-
#
|
41
|
-
# @return [true, false] keep
|
42
|
-
|
43
|
-
def self.keep?
|
44
|
-
keep = (self.keep || false)
|
45
|
-
return (Time.now + keep) >= Time.now if keep.is_a? Fixnum
|
46
|
-
keep
|
47
|
-
end
|
48
7
|
|
49
8
|
##
|
50
9
|
# Add a job to the job queue.
|
@@ -108,27 +67,25 @@ module Navvy
|
|
108
67
|
end
|
109
68
|
|
110
69
|
##
|
111
|
-
#
|
112
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
113
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
114
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
115
|
-
# deleted.
|
70
|
+
# Deletes all jobs.
|
116
71
|
#
|
117
|
-
# @
|
118
|
-
|
119
|
-
|
72
|
+
# @return [Integer] amount the amount of jobs that were deleted
|
73
|
+
|
74
|
+
def self.delete_all
|
75
|
+
Navvy::Job.destroy
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
##
|
80
|
+
# Mark the job as started. Will set started_at to the current time.
|
120
81
|
#
|
121
|
-
# @return [
|
82
|
+
# @return [true, false] update_attributes the result of the
|
83
|
+
# update_attributes call
|
122
84
|
|
123
|
-
def
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
Navvy::Job.keep? ? completed : destroy
|
128
|
-
result
|
129
|
-
rescue Exception => exception
|
130
|
-
failed(exception.message)
|
131
|
-
end
|
85
|
+
def started
|
86
|
+
update({
|
87
|
+
:started_at => Time.now
|
88
|
+
})
|
132
89
|
end
|
133
90
|
|
134
91
|
##
|
@@ -165,26 +122,6 @@ module Navvy
|
|
165
122
|
)
|
166
123
|
end
|
167
124
|
|
168
|
-
##
|
169
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
170
|
-
# a parend_id equal to self.id.
|
171
|
-
#
|
172
|
-
# @return [true, false]
|
173
|
-
|
174
|
-
def retry
|
175
|
-
self.class.enqueue(
|
176
|
-
object,
|
177
|
-
method_name,
|
178
|
-
*(args << {
|
179
|
-
:job_options => {
|
180
|
-
:parent_id => parent_id || id,
|
181
|
-
:run_at => Time.now + times_failed ** 4,
|
182
|
-
:priority => priority
|
183
|
-
}
|
184
|
-
})
|
185
|
-
)
|
186
|
-
end
|
187
|
-
|
188
125
|
##
|
189
126
|
# Check how many times the job has failed. Will try to find jobs with a
|
190
127
|
# parent_id that's the same as self.id and count them
|
@@ -197,64 +134,7 @@ module Navvy
|
|
197
134
|
"(`id` == '#{i}' OR `parent_id` == '#{i}') AND `failed_at` IS NOT NULL"
|
198
135
|
).count
|
199
136
|
end
|
200
|
-
|
201
|
-
##
|
202
|
-
# Check if the job has been run.
|
203
|
-
#
|
204
|
-
# @return [true, false] ran
|
205
|
-
|
206
|
-
def ran?
|
207
|
-
completed? || failed?
|
208
|
-
end
|
209
|
-
|
210
|
-
##
|
211
|
-
# Check how long it took for a job to complete or fail
|
212
|
-
#
|
213
|
-
# @return [Time, Integer] time the time it took
|
214
|
-
|
215
|
-
def duration
|
216
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
217
|
-
end
|
218
|
-
|
219
|
-
##
|
220
|
-
# Check if completed_at is set
|
221
|
-
#
|
222
|
-
# @return [true, false] set?
|
223
|
-
|
224
|
-
def completed_at?
|
225
|
-
!completed_at.nil?
|
226
|
-
end
|
227
|
-
|
228
|
-
##
|
229
|
-
# Check if failed_at is set
|
230
|
-
#
|
231
|
-
# @return [true, false] set?
|
232
|
-
|
233
|
-
def failed_at?
|
234
|
-
!failed_at.nil?
|
235
|
-
end
|
236
|
-
|
237
|
-
##
|
238
|
-
# Get the job arguments as an array
|
239
|
-
#
|
240
|
-
# @return [array] arguments
|
241
|
-
|
242
|
-
def args
|
243
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
244
|
-
end
|
245
|
-
|
246
|
-
##
|
247
|
-
# Get the job status
|
248
|
-
#
|
249
|
-
# @return [:pending, :completed, :failed] status
|
250
|
-
|
251
|
-
def status
|
252
|
-
return :completed if completed?
|
253
|
-
return :failed if failed?
|
254
|
-
:pending
|
255
|
-
end
|
256
|
-
|
257
|
-
alias_method :completed?, :completed_at?
|
258
|
-
alias_method :failed?, :failed_at?
|
259
137
|
end
|
260
138
|
end
|
139
|
+
|
140
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
data/lib/navvy/log.rb
CHANGED
@@ -65,9 +65,9 @@ module Navvy
|
|
65
65
|
when :rails
|
66
66
|
raise(
|
67
67
|
LoggerNotFound,
|
68
|
-
'
|
69
|
-
) unless defined?
|
70
|
-
|
68
|
+
'Rails.logger could not be found. No logs were created.'
|
69
|
+
) unless defined? Rails.logger.info
|
70
|
+
Rails.logger.info(message)
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
data/lib/navvy/worker.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'daemons'
|
3
|
+
|
1
4
|
module Navvy
|
2
5
|
class Worker
|
3
6
|
class << self
|
@@ -12,7 +15,7 @@ module Navvy
|
|
12
15
|
def self.sleep_time
|
13
16
|
@sleep_time || Navvy.configuration.sleep_time
|
14
17
|
end
|
15
|
-
|
18
|
+
|
16
19
|
##
|
17
20
|
# Start the worker.
|
18
21
|
|
@@ -46,5 +49,34 @@ module Navvy
|
|
46
49
|
)
|
47
50
|
end
|
48
51
|
end
|
52
|
+
|
53
|
+
##
|
54
|
+
# Daemonize the worker
|
55
|
+
|
56
|
+
def self.daemonize(*args)
|
57
|
+
if defined?(ActiveRecord)
|
58
|
+
# Sets ActiveRecord's logger to Navvy a new Logger instance
|
59
|
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
60
|
+
end
|
61
|
+
|
62
|
+
# If #daemonize does not receive any arguments, the options variable will
|
63
|
+
# contain an empty hash, and the ARGV of the environment will be used instead
|
64
|
+
# of the :ARGV options from Daemons#run_proc. However, if the *args param has been set
|
65
|
+
# this will be used instead of the environment's ARGV for the Daemons.
|
66
|
+
options = args.empty? ? {} : {:ARGV => args}
|
67
|
+
|
68
|
+
# Finally, the directory store mode will be set to normal and the Daemons PID file
|
69
|
+
# will be stored inside tmp/pids of the application.
|
70
|
+
options.merge!({:dir_mode => :normal, :dir => 'tmp/pids'})
|
71
|
+
|
72
|
+
# Ensures that the tmp/pids folder exists so that the process id file can properly be stored
|
73
|
+
%x(mkdir -p tmp/pids)
|
74
|
+
|
75
|
+
# Runs the Navvy Worker inside a Daemon
|
76
|
+
Daemons.run_proc('navvy', options) do
|
77
|
+
Navvy::Worker.start
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
49
81
|
end
|
50
82
|
end
|
data/navvy.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{navvy}
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jeff Kreeftmeijer"]
|
12
|
-
s.date = %q{2010-02-
|
12
|
+
s.date = %q{2010-02-10}
|
13
13
|
s.description = %q{Simple background job processor inspired by delayed_job, but aiming for database agnosticism.}
|
14
14
|
s.email = %q{jeff@kreeftmeijer.nl}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -25,9 +25,11 @@ Gem::Specification.new do |s|
|
|
25
25
|
"VERSION",
|
26
26
|
"generators/navvy/navvy_generator.rb",
|
27
27
|
"generators/navvy/templates/migration.rb",
|
28
|
+
"generators/navvy/templates/script",
|
28
29
|
"lib/generators/navvy_generator.rb",
|
29
30
|
"lib/navvy.rb",
|
30
31
|
"lib/navvy/configuration.rb",
|
32
|
+
"lib/navvy/job.rb",
|
31
33
|
"lib/navvy/job/active_record.rb",
|
32
34
|
"lib/navvy/job/data_mapper.rb",
|
33
35
|
"lib/navvy/job/mongo_mapper.rb",
|
data/spec/job_spec.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
2
|
|
3
3
|
describe 'Navvy::Job' do
|
4
|
-
|
5
4
|
describe '.keep?' do
|
6
5
|
after(:each) do
|
7
6
|
Navvy::Job.keep = false
|
@@ -9,26 +8,26 @@ describe 'Navvy::Job' do
|
|
9
8
|
config.keep_jobs = false
|
10
9
|
end
|
11
10
|
end
|
12
|
-
|
11
|
+
|
13
12
|
describe 'when configured using Navvy::Job.keep=' do
|
14
13
|
it 'should return false' do
|
15
14
|
Navvy::Job.keep = false
|
16
15
|
Navvy::Job.keep?.should == false
|
17
|
-
end
|
16
|
+
end
|
18
17
|
|
19
18
|
it 'should return true' do
|
20
19
|
Navvy::Job.keep = true
|
21
20
|
Navvy::Job.keep?.should == true
|
22
21
|
end
|
23
|
-
end
|
24
|
-
|
22
|
+
end
|
23
|
+
|
25
24
|
describe 'when configured with Navvy.configure' do
|
26
25
|
it 'should return false' do
|
27
26
|
Navvy.configure do |config|
|
28
27
|
config.keep_jobs = false
|
29
28
|
end
|
30
29
|
Navvy::Job.keep?.should == false
|
31
|
-
end
|
30
|
+
end
|
32
31
|
|
33
32
|
it 'should return true' do
|
34
33
|
Navvy.configure do |config|
|
@@ -38,7 +37,7 @@ describe 'Navvy::Job' do
|
|
38
37
|
end
|
39
38
|
end
|
40
39
|
end
|
41
|
-
|
40
|
+
|
42
41
|
describe '.enqueue' do
|
43
42
|
before(:each) do
|
44
43
|
delete_all_jobs
|
@@ -177,6 +176,14 @@ describe 'Navvy::Job' do
|
|
177
176
|
end
|
178
177
|
end
|
179
178
|
|
179
|
+
describe '.delete_all' do
|
180
|
+
it 'should delete all jobs' do
|
181
|
+
3.times do; Navvy::Job.create; end
|
182
|
+
Navvy::Job.delete_all
|
183
|
+
job_count.should == 0
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
180
187
|
describe '#run' do
|
181
188
|
it 'should pass the arguments' do
|
182
189
|
delete_all_jobs
|
@@ -242,6 +249,19 @@ describe 'Navvy::Job' do
|
|
242
249
|
end
|
243
250
|
end
|
244
251
|
|
252
|
+
describe '#started' do
|
253
|
+
before(:each) do
|
254
|
+
delete_all_jobs
|
255
|
+
Navvy::Job.enqueue(Cow, :speak)
|
256
|
+
end
|
257
|
+
|
258
|
+
it 'should update the jobs started_at date' do
|
259
|
+
jobs = Navvy::Job.next
|
260
|
+
jobs.first.started
|
261
|
+
jobs.first.started_at.should_not be_nil
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
245
265
|
describe '#completed' do
|
246
266
|
before(:each) do
|
247
267
|
delete_all_jobs
|
@@ -388,19 +408,23 @@ describe 'Navvy::Job' do
|
|
388
408
|
|
389
409
|
@failed_job.times_failed.should == 3
|
390
410
|
end
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
411
|
+
|
412
|
+
it 'should return 2 when having 1 failed and one pending child' do
|
413
|
+
Navvy::Job.create(
|
414
|
+
:failed_at => Time.now,
|
415
|
+
:parent_id => @failed_job.id
|
416
|
+
)
|
417
|
+
|
418
|
+
Navvy::Job.create(
|
419
|
+
:parent_id => @failed_job.id
|
420
|
+
)
|
421
|
+
|
422
|
+
Navvy::Job.create(
|
423
|
+
:parent_id => @failed_job.id
|
424
|
+
)
|
425
|
+
|
426
|
+
@failed_job.times_failed.should == 2
|
427
|
+
end
|
404
428
|
|
405
429
|
it 'should return 2 when having failed and having a failed parent' do
|
406
430
|
failed_child = Navvy::Job.create(
|
data/spec/log_spec.rb
CHANGED
@@ -13,9 +13,9 @@ describe Navvy::Log do
|
|
13
13
|
lambda { Navvy::Log.info('123') }.should raise_error
|
14
14
|
end
|
15
15
|
|
16
|
-
it 'should pass the log to
|
16
|
+
it 'should pass the log to Rails.logger' do
|
17
17
|
require File.expand_path(File.dirname(__FILE__) + '/setup/rails_default_logger')
|
18
|
-
|
18
|
+
Rails.logger.should_receive(:info).with('123')
|
19
19
|
Navvy::Log.info('123')
|
20
20
|
end
|
21
21
|
end
|
@@ -42,7 +42,7 @@ describe Navvy::Log do
|
|
42
42
|
end
|
43
43
|
|
44
44
|
it 'should pass the log to justlogging' do
|
45
|
-
|
45
|
+
Rails.logger.should_receive(:info).with('123')
|
46
46
|
Justlogging.should_receive(:log).with('123')
|
47
47
|
Navvy::Log.info('123')
|
48
48
|
end
|
@@ -54,7 +54,7 @@ describe Navvy::Log do
|
|
54
54
|
end
|
55
55
|
|
56
56
|
it 'should not log' do
|
57
|
-
|
57
|
+
Rails.logger.should_not_receive(:info)
|
58
58
|
Justlogging.should_not_receive(:log)
|
59
59
|
Navvy::Log.info('123')
|
60
60
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: navvy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeff Kreeftmeijer
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-02-
|
12
|
+
date: 2010-02-10 00:00:00 +01:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -80,9 +80,11 @@ files:
|
|
80
80
|
- VERSION
|
81
81
|
- generators/navvy/navvy_generator.rb
|
82
82
|
- generators/navvy/templates/migration.rb
|
83
|
+
- generators/navvy/templates/script
|
83
84
|
- lib/generators/navvy_generator.rb
|
84
85
|
- lib/navvy.rb
|
85
86
|
- lib/navvy/configuration.rb
|
87
|
+
- lib/navvy/job.rb
|
86
88
|
- lib/navvy/job/active_record.rb
|
87
89
|
- lib/navvy/job/data_mapper.rb
|
88
90
|
- lib/navvy/job/mongo_mapper.rb
|