navvy 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/generators/navvy/navvy_generator.rb +2 -2
- data/generators/navvy/templates/script +4 -0
- data/lib/generators/navvy_generator.rb +9 -3
- data/lib/navvy/job.rb +156 -0
- data/lib/navvy/job/active_record.rb +9 -143
- data/lib/navvy/job/data_mapper.rb +18 -143
- data/lib/navvy/job/mongo_mapper.rb +9 -125
- data/lib/navvy/job/sequel.rb +18 -138
- data/lib/navvy/log.rb +3 -3
- data/lib/navvy/worker.rb +33 -1
- data/navvy.gemspec +4 -2
- data/spec/job_spec.rb +44 -20
- data/spec/log_spec.rb +4 -4
- data/spec/setup/rails_default_logger.rb +8 -4
- metadata +4 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.1
|
@@ -4,11 +4,11 @@ class NavvyGenerator < Rails::Generator::Base
|
|
4
4
|
options = {
|
5
5
|
:migration_file_name => 'create_jobs'
|
6
6
|
}
|
7
|
-
|
8
7
|
m.migration_template 'migration.rb', 'db/migrate', options
|
8
|
+
m.file 'script', 'script/navvy', :chmod => 0755
|
9
9
|
end
|
10
10
|
end
|
11
|
-
|
11
|
+
|
12
12
|
def banner
|
13
13
|
"Usage: #{$0} #{spec.name}"
|
14
14
|
end
|
@@ -1,18 +1,24 @@
|
|
1
1
|
require 'rails/generators/migration'
|
2
2
|
class NavvyGenerator < Rails::Generators::Base
|
3
3
|
include Rails::Generators::Migration
|
4
|
-
|
4
|
+
|
5
5
|
def self.source_root
|
6
6
|
File.join(File.dirname(__FILE__), '..', '..', 'generators', 'navvy', 'templates')
|
7
7
|
end
|
8
|
-
|
8
|
+
|
9
9
|
def install_navvy
|
10
10
|
migration_template(
|
11
11
|
'migration.rb',
|
12
12
|
'db/migrate/create_navvy_table.rb'
|
13
13
|
)
|
14
|
+
|
15
|
+
copy_file(
|
16
|
+
'script',
|
17
|
+
'script/navvy',
|
18
|
+
:chmod => 0755
|
19
|
+
)
|
14
20
|
end
|
15
|
-
|
21
|
+
|
16
22
|
protected
|
17
23
|
def next_migration_number(dirname) #:nodoc:
|
18
24
|
"%.3d" % (current_migration_number(dirname) + 1)
|
data/lib/navvy/job.rb
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
module Navvy
|
2
|
+
class Job
|
3
|
+
class << self
|
4
|
+
attr_writer :limit, :keep, :max_attempts
|
5
|
+
end
|
6
|
+
|
7
|
+
##
|
8
|
+
# Limit of jobs to be fetched at once. Will use the value stored in
|
9
|
+
# Navvy.configuration (defaults to 100), or -- for backwards compatibility
|
10
|
+
# -- Navvy::Job.limit.
|
11
|
+
#
|
12
|
+
# @return [Integer] limit
|
13
|
+
|
14
|
+
def self.limit
|
15
|
+
@limit || Navvy.configuration.job_limit
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# If and how long the jobs should be kept. Will use the value stored in
|
20
|
+
# Navvy.configuration (defaults to false), or -- for backwards
|
21
|
+
# compatibility -- Navvy::Job.keep.
|
22
|
+
#
|
23
|
+
# @return [Fixnum, true, false] keep
|
24
|
+
|
25
|
+
def self.keep
|
26
|
+
@keep || Navvy.configuration.keep_jobs
|
27
|
+
end
|
28
|
+
|
29
|
+
##
|
30
|
+
# How often should a job be retried? Will use the value stored in
|
31
|
+
# Navvy.configuration (defaults to 24), or -- for backwards compatibility
|
32
|
+
# -- Navvy::Job.max_attempts.
|
33
|
+
#
|
34
|
+
# @return [Fixnum] max_attempts
|
35
|
+
|
36
|
+
def self.max_attempts
|
37
|
+
@max_attempts || Navvy.configuration.max_attempts
|
38
|
+
end
|
39
|
+
|
40
|
+
##
|
41
|
+
# Should the job be kept? Will calculate if the keeptime has passed if
|
42
|
+
# @keep is a Fixnum. Otherwise, it'll just return the @keep value since
|
43
|
+
# it's probably a boolean.
|
44
|
+
#
|
45
|
+
# @return [true, false] keep
|
46
|
+
|
47
|
+
def self.keep?
|
48
|
+
return (Time.now + self.keep) >= Time.now if self.keep.is_a? Fixnum
|
49
|
+
self.keep
|
50
|
+
end
|
51
|
+
|
52
|
+
##
|
53
|
+
# Run the job. Will delete the Navvy::Job record and return its return
|
54
|
+
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
55
|
+
# fails, it'll call Navvy::Job#failed and pass the exception message.
|
56
|
+
# Failed jobs will _not_ get deleted.
|
57
|
+
#
|
58
|
+
# @example
|
59
|
+
# job = Navvy::Job.next # finds the next available job in the queue
|
60
|
+
# job.run # runs the job and returns the job's return value
|
61
|
+
#
|
62
|
+
# @return [String] return value or exception message of the called method.
|
63
|
+
|
64
|
+
def run
|
65
|
+
begin
|
66
|
+
started
|
67
|
+
result = Kernel.const_get(object).send(method_name, *args)
|
68
|
+
Navvy::Job.keep? ? completed : destroy
|
69
|
+
result
|
70
|
+
rescue Exception => exception
|
71
|
+
failed(exception.message)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
##
|
76
|
+
# Retry the current job. Will add self to the queue again, giving the clone
|
77
|
+
# a parend_id equal to self.id. Also, the priority of the new job will be
|
78
|
+
# the same as its parent's and it'll set the :run_at date to N ** 4, where
|
79
|
+
# N is the times_failed count.
|
80
|
+
#
|
81
|
+
# @return [Navvy::Job] job the new job it created.
|
82
|
+
|
83
|
+
def retry
|
84
|
+
self.class.enqueue(
|
85
|
+
object,
|
86
|
+
method_name,
|
87
|
+
*(args << {
|
88
|
+
:job_options => {
|
89
|
+
:parent_id => parent_id || id,
|
90
|
+
:run_at => Time.now + times_failed ** 4,
|
91
|
+
:priority => priority
|
92
|
+
}
|
93
|
+
})
|
94
|
+
)
|
95
|
+
end
|
96
|
+
|
97
|
+
##
|
98
|
+
# Check if the job has been run.
|
99
|
+
#
|
100
|
+
# @return [true, false] ran
|
101
|
+
|
102
|
+
def ran?
|
103
|
+
completed? || failed?
|
104
|
+
end
|
105
|
+
|
106
|
+
##
|
107
|
+
# Check how long it took for a job to complete or fail.
|
108
|
+
#
|
109
|
+
# @return [Time, Integer] time the time it took.
|
110
|
+
|
111
|
+
def duration
|
112
|
+
ran? ? (completed_at || failed_at) - started_at : 0
|
113
|
+
end
|
114
|
+
|
115
|
+
##
|
116
|
+
# Check if completed_at is set.
|
117
|
+
#
|
118
|
+
# @return [true, false] set?
|
119
|
+
|
120
|
+
def completed_at?
|
121
|
+
!completed_at.nil?
|
122
|
+
end
|
123
|
+
|
124
|
+
##
|
125
|
+
# Check if failed_at is set.
|
126
|
+
#
|
127
|
+
# @return [true, false] set?
|
128
|
+
|
129
|
+
def failed_at?
|
130
|
+
!failed_at.nil?
|
131
|
+
end
|
132
|
+
|
133
|
+
##
|
134
|
+
# Get the job arguments as an array.
|
135
|
+
#
|
136
|
+
# @return [array] arguments
|
137
|
+
|
138
|
+
def args
|
139
|
+
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
140
|
+
end
|
141
|
+
|
142
|
+
##
|
143
|
+
# Get the job status
|
144
|
+
#
|
145
|
+
# @return [:pending, :completed, :failed] status
|
146
|
+
|
147
|
+
def status
|
148
|
+
return :completed if completed?
|
149
|
+
return :failed if failed?
|
150
|
+
:pending
|
151
|
+
end
|
152
|
+
|
153
|
+
alias_method :completed?, :completed_at?
|
154
|
+
alias_method :failed?, :failed_at?
|
155
|
+
end
|
156
|
+
end
|
@@ -3,47 +3,6 @@ require 'active_record'
|
|
3
3
|
|
4
4
|
module Navvy
|
5
5
|
class Job < ActiveRecord::Base
|
6
|
-
class << self
|
7
|
-
attr_writer :limit, :keep, :max_attempts
|
8
|
-
end
|
9
|
-
|
10
|
-
##
|
11
|
-
# Default limit of jobs to be fetched.
|
12
|
-
#
|
13
|
-
# @return [Integer] limit
|
14
|
-
|
15
|
-
def self.limit
|
16
|
-
@limit || Navvy.configuration.job_limit
|
17
|
-
end
|
18
|
-
|
19
|
-
##
|
20
|
-
# If and how long the jobs should be kept.
|
21
|
-
#
|
22
|
-
# @return [Fixnum, true, false] keep
|
23
|
-
|
24
|
-
def self.keep
|
25
|
-
@keep || Navvy.configuration.keep_jobs
|
26
|
-
end
|
27
|
-
|
28
|
-
##
|
29
|
-
# How often should a job be retried?
|
30
|
-
#
|
31
|
-
# @return [Fixnum] max_attempts
|
32
|
-
|
33
|
-
def self.max_attempts
|
34
|
-
@max_attempts || Navvy.configuration.max_attempts
|
35
|
-
end
|
36
|
-
|
37
|
-
##
|
38
|
-
# Should the job be kept?
|
39
|
-
#
|
40
|
-
# @return [true, false] keep
|
41
|
-
|
42
|
-
def self.keep?
|
43
|
-
keep = (self.keep || false)
|
44
|
-
return keep.from_now >= Time.now if keep.is_a? Fixnum
|
45
|
-
keep
|
46
|
-
end
|
47
6
|
|
48
7
|
##
|
49
8
|
# Add a job to the job queue.
|
@@ -115,31 +74,15 @@ module Navvy
|
|
115
74
|
end
|
116
75
|
|
117
76
|
##
|
118
|
-
#
|
119
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
120
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
121
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
122
|
-
# deleted.
|
77
|
+
# Mark the job as started. Will set started_at to the current time.
|
123
78
|
#
|
124
|
-
# @
|
125
|
-
#
|
126
|
-
# job.run # runs the job and returns the job's return value
|
127
|
-
#
|
128
|
-
# @return [String] return value of the called method.
|
79
|
+
# @return [true, false] update_attributes the result of the
|
80
|
+
# update_attributes call
|
129
81
|
|
130
|
-
def
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
result = object.constantize.send(method_name)
|
135
|
-
else
|
136
|
-
result = object.constantize.send(method_name, *args)
|
137
|
-
end
|
138
|
-
Navvy::Job.keep? ? completed : destroy
|
139
|
-
result
|
140
|
-
rescue Exception => exception
|
141
|
-
failed(exception.message)
|
142
|
-
end
|
82
|
+
def started
|
83
|
+
update_attributes({
|
84
|
+
:started_at => Time.now
|
85
|
+
})
|
143
86
|
end
|
144
87
|
|
145
88
|
##
|
@@ -176,26 +119,6 @@ module Navvy
|
|
176
119
|
)
|
177
120
|
end
|
178
121
|
|
179
|
-
##
|
180
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
181
|
-
# a parend_id equal to self.id.
|
182
|
-
#
|
183
|
-
# @return [true, false]
|
184
|
-
|
185
|
-
def retry
|
186
|
-
self.class.enqueue(
|
187
|
-
object,
|
188
|
-
method_name,
|
189
|
-
*(args << {
|
190
|
-
:job_options => {
|
191
|
-
:parent_id => parent_id || id,
|
192
|
-
:run_at => Time.now + times_failed ** 4,
|
193
|
-
:priority => priority
|
194
|
-
}
|
195
|
-
})
|
196
|
-
)
|
197
|
-
end
|
198
|
-
|
199
122
|
##
|
200
123
|
# Check how many times the job has failed. Will try to find jobs with a
|
201
124
|
# parent_id that's the same as self.id and count them
|
@@ -208,64 +131,7 @@ module Navvy
|
|
208
131
|
:conditions => "(`id` = '#{i}' OR `parent_id` = '#{i}') AND `failed_at` IS NOT NULL"
|
209
132
|
)
|
210
133
|
end
|
211
|
-
|
212
|
-
##
|
213
|
-
# Check if the job has been run.
|
214
|
-
#
|
215
|
-
# @return [true, false] ran
|
216
|
-
|
217
|
-
def ran?
|
218
|
-
completed? || failed?
|
219
|
-
end
|
220
|
-
|
221
|
-
##
|
222
|
-
# Check how long it took for a job to complete or fail
|
223
|
-
#
|
224
|
-
# @return [Time, Integer] time the time it took
|
225
|
-
|
226
|
-
def duration
|
227
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
228
|
-
end
|
229
|
-
|
230
|
-
##
|
231
|
-
# Check if completed_at is set
|
232
|
-
#
|
233
|
-
# @return [true, false] set?
|
234
|
-
|
235
|
-
def completed_at?
|
236
|
-
!completed_at.nil?
|
237
|
-
end
|
238
|
-
|
239
|
-
##
|
240
|
-
# Check if failed_at is set
|
241
|
-
#
|
242
|
-
# @return [true, false] set?
|
243
|
-
|
244
|
-
def failed_at?
|
245
|
-
!failed_at.nil?
|
246
|
-
end
|
247
|
-
|
248
|
-
##
|
249
|
-
# Get the job arguments as an array
|
250
|
-
#
|
251
|
-
# @return [array] arguments
|
252
|
-
|
253
|
-
def args
|
254
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
255
|
-
end
|
256
|
-
|
257
|
-
##
|
258
|
-
# Get the job status
|
259
|
-
#
|
260
|
-
# @return [:pending, :completed, :failed] status
|
261
|
-
|
262
|
-
def status
|
263
|
-
return :completed if completed?
|
264
|
-
return :failed if failed?
|
265
|
-
:pending
|
266
|
-
end
|
267
|
-
|
268
|
-
alias_method :completed?, :completed_at?
|
269
|
-
alias_method :failed?, :failed_at?
|
270
134
|
end
|
271
135
|
end
|
136
|
+
|
137
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
@@ -4,9 +4,6 @@ require 'dm-core'
|
|
4
4
|
module Navvy
|
5
5
|
class Job
|
6
6
|
include DataMapper::Resource
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
7
|
|
11
8
|
property :id, Serial
|
12
9
|
property :object, String
|
@@ -22,44 +19,6 @@ module Navvy
|
|
22
19
|
property :completed_at, Time
|
23
20
|
property :failed_at, Time
|
24
21
|
|
25
|
-
##
|
26
|
-
# Default limit of jobs to be fetched.
|
27
|
-
#
|
28
|
-
# @return [Integer] limit
|
29
|
-
|
30
|
-
def self.limit
|
31
|
-
@limit || Navvy.configuration.job_limit
|
32
|
-
end
|
33
|
-
|
34
|
-
##
|
35
|
-
# If and how long the jobs should be kept.
|
36
|
-
#
|
37
|
-
# @return [Fixnum, true, false] keep
|
38
|
-
|
39
|
-
def self.keep
|
40
|
-
@keep || Navvy.configuration.keep_jobs
|
41
|
-
end
|
42
|
-
|
43
|
-
##
|
44
|
-
# How often should a job be retried?
|
45
|
-
#
|
46
|
-
# @return [Fixnum] max_attempts
|
47
|
-
|
48
|
-
def self.max_attempts
|
49
|
-
@max_attempts || Navvy.configuration.max_attempts
|
50
|
-
end
|
51
|
-
|
52
|
-
##
|
53
|
-
# Should the job be kept?
|
54
|
-
#
|
55
|
-
# @return [true, false] keep
|
56
|
-
|
57
|
-
def self.keep?
|
58
|
-
keep = (self.keep || false)
|
59
|
-
return (Time.now + keep) >= Time.now if keep.is_a? Fixnum
|
60
|
-
keep
|
61
|
-
end
|
62
|
-
|
63
22
|
##
|
64
23
|
# Add a job to the job queue.
|
65
24
|
#
|
@@ -112,7 +71,7 @@ module Navvy
|
|
112
71
|
)
|
113
72
|
end
|
114
73
|
|
115
|
-
##
|
74
|
+
##
|
116
75
|
# Clean up jobs that we don't need to keep anymore. If Navvy::Job.keep is
|
117
76
|
# false it'll delete every completed job, if it's a timestamp it'll only
|
118
77
|
# delete completed jobs that have passed their keeptime.
|
@@ -128,31 +87,24 @@ module Navvy
|
|
128
87
|
end
|
129
88
|
|
130
89
|
##
|
131
|
-
#
|
132
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
133
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
134
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
135
|
-
# deleted.
|
90
|
+
# Deletes all jobs.
|
136
91
|
#
|
137
|
-
# @
|
138
|
-
|
139
|
-
|
92
|
+
# @return [true, false] deleted?
|
93
|
+
|
94
|
+
def self.delete_all
|
95
|
+
Navvy::Job.all.destroy
|
96
|
+
end
|
97
|
+
|
98
|
+
##
|
99
|
+
# Mark the job as started. Will set started_at to the current time.
|
140
100
|
#
|
141
|
-
# @return [
|
101
|
+
# @return [true, false] update_attributes the result of the
|
102
|
+
# update_attributes call
|
142
103
|
|
143
|
-
def
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result = Kernel.const_get(object).send(method_name)
|
148
|
-
else
|
149
|
-
result = Kernel.const_get(object).send(method_name, *args)
|
150
|
-
end
|
151
|
-
Navvy::Job.keep? ? completed : destroy
|
152
|
-
result
|
153
|
-
rescue Exception => exception
|
154
|
-
failed(exception.message)
|
155
|
-
end
|
104
|
+
def started
|
105
|
+
update({
|
106
|
+
:started_at => Time.now
|
107
|
+
})
|
156
108
|
end
|
157
109
|
|
158
110
|
##
|
@@ -190,26 +142,6 @@ module Navvy
|
|
190
142
|
)
|
191
143
|
end
|
192
144
|
|
193
|
-
##
|
194
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
195
|
-
# a parend_id equal to self.id.
|
196
|
-
#
|
197
|
-
# @return [true, false]
|
198
|
-
|
199
|
-
def retry
|
200
|
-
self.class.enqueue(
|
201
|
-
object,
|
202
|
-
method_name,
|
203
|
-
*(args << {
|
204
|
-
:job_options => {
|
205
|
-
:parent_id => parent_id || id,
|
206
|
-
:run_at => Time.now + times_failed ** 4,
|
207
|
-
:priority => priority
|
208
|
-
}
|
209
|
-
})
|
210
|
-
)
|
211
|
-
end
|
212
|
-
|
213
145
|
##
|
214
146
|
# Check how many times the job has failed. Will try to find jobs with a
|
215
147
|
# parent_id that's the same as self.id and count them
|
@@ -222,64 +154,7 @@ module Navvy
|
|
222
154
|
:conditions => ["(`id` = ? OR `parent_id` = ?) AND `failed_at` IS NOT NULL", i, i]
|
223
155
|
).count
|
224
156
|
end
|
225
|
-
|
226
|
-
##
|
227
|
-
# Check if the job has been run.
|
228
|
-
#
|
229
|
-
# @return [true, false] ran
|
230
|
-
|
231
|
-
def ran?
|
232
|
-
completed? || failed?
|
233
|
-
end
|
234
|
-
|
235
|
-
##
|
236
|
-
# Check if completed_at is set
|
237
|
-
#
|
238
|
-
# @return [true, false] set?
|
239
|
-
|
240
|
-
def completed_at?
|
241
|
-
!completed_at.nil?
|
242
|
-
end
|
243
|
-
|
244
|
-
##
|
245
|
-
# Check if failed_at is set
|
246
|
-
#
|
247
|
-
# @return [true, false] set?
|
248
|
-
|
249
|
-
def failed_at?
|
250
|
-
!failed_at.nil?
|
251
|
-
end
|
252
|
-
|
253
|
-
##
|
254
|
-
# Check how long it took for a job to complete or fail
|
255
|
-
#
|
256
|
-
# @return [Time, Integer] time the time it took
|
257
|
-
|
258
|
-
def duration
|
259
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
260
|
-
end
|
261
|
-
|
262
|
-
##
|
263
|
-
# Get the job arguments as an array
|
264
|
-
#
|
265
|
-
# @return [array] arguments
|
266
|
-
|
267
|
-
def args
|
268
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
269
|
-
end
|
270
|
-
|
271
|
-
##
|
272
|
-
# Get the job status
|
273
|
-
#
|
274
|
-
# @return [:pending, :completed, :failed] status
|
275
|
-
|
276
|
-
def status
|
277
|
-
return :completed if completed?
|
278
|
-
return :failed if failed?
|
279
|
-
:pending
|
280
|
-
end
|
281
|
-
|
282
|
-
alias_method :completed?, :completed_at?
|
283
|
-
alias_method :failed?, :failed_at?
|
284
157
|
end
|
285
158
|
end
|
159
|
+
|
160
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
@@ -4,9 +4,6 @@ require 'mongo_mapper'
|
|
4
4
|
module Navvy
|
5
5
|
class Job
|
6
6
|
include MongoMapper::Document
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
7
|
|
11
8
|
key :object, String
|
12
9
|
key :method_name, Symbol
|
@@ -21,44 +18,6 @@ module Navvy
|
|
21
18
|
key :completed_at, Time
|
22
19
|
key :failed_at, Time
|
23
20
|
|
24
|
-
##
|
25
|
-
# Default limit of jobs to be fetched.
|
26
|
-
#
|
27
|
-
# @return [Integer] limit
|
28
|
-
|
29
|
-
def self.limit
|
30
|
-
@limit || Navvy.configuration.job_limit
|
31
|
-
end
|
32
|
-
|
33
|
-
##
|
34
|
-
# If and how long the jobs should be kept.
|
35
|
-
#
|
36
|
-
# @return [Fixnum, true, false] keep
|
37
|
-
|
38
|
-
def self.keep
|
39
|
-
@keep || Navvy.configuration.keep_jobs
|
40
|
-
end
|
41
|
-
|
42
|
-
##
|
43
|
-
# How often should a job be retried?
|
44
|
-
#
|
45
|
-
# @return [Fixnum] max_attempts
|
46
|
-
|
47
|
-
def self.max_attempts
|
48
|
-
@max_attempts || Navvy.configuration.max_attempts
|
49
|
-
end
|
50
|
-
|
51
|
-
##
|
52
|
-
# Should the job be kept?
|
53
|
-
#
|
54
|
-
# @return [true, false] keep
|
55
|
-
|
56
|
-
def self.keep?
|
57
|
-
keep = (self.keep || false)
|
58
|
-
return keep.from_now >= Time.now if keep.is_a? Fixnum
|
59
|
-
keep
|
60
|
-
end
|
61
|
-
|
62
21
|
##
|
63
22
|
# Add a job to the job queue.
|
64
23
|
#
|
@@ -128,31 +87,15 @@ module Navvy
|
|
128
87
|
end
|
129
88
|
|
130
89
|
##
|
131
|
-
#
|
132
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
133
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
134
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
135
|
-
# deleted.
|
90
|
+
# Mark the job as started. Will set started_at to the current time.
|
136
91
|
#
|
137
|
-
# @
|
138
|
-
#
|
139
|
-
# job.run # runs the job and returns the job's return value
|
140
|
-
#
|
141
|
-
# @return [String] return value of the called method.
|
92
|
+
# @return [true, false] update_attributes the result of the
|
93
|
+
# update_attributes call
|
142
94
|
|
143
|
-
def
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
result = object.constantize.send(method_name)
|
148
|
-
else
|
149
|
-
result = object.constantize.send(method_name, *args)
|
150
|
-
end
|
151
|
-
Navvy::Job.keep? ? completed : destroy
|
152
|
-
result
|
153
|
-
rescue Exception => exception
|
154
|
-
failed(exception.message)
|
155
|
-
end
|
95
|
+
def started
|
96
|
+
update_attributes({
|
97
|
+
:started_at => Time.now
|
98
|
+
})
|
156
99
|
end
|
157
100
|
|
158
101
|
##
|
@@ -189,26 +132,6 @@ module Navvy
|
|
189
132
|
)
|
190
133
|
end
|
191
134
|
|
192
|
-
##
|
193
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
194
|
-
# a parend_id equal to self.id.
|
195
|
-
#
|
196
|
-
# @return [true, false]
|
197
|
-
|
198
|
-
def retry
|
199
|
-
self.class.enqueue(
|
200
|
-
object,
|
201
|
-
method_name,
|
202
|
-
*(args << {
|
203
|
-
:job_options => {
|
204
|
-
:parent_id => parent_id || id,
|
205
|
-
:run_at => Time.now + times_failed ** 4,
|
206
|
-
:priority => priority
|
207
|
-
}
|
208
|
-
})
|
209
|
-
)
|
210
|
-
end
|
211
|
-
|
212
135
|
##
|
213
136
|
# Check how many times the job has failed. Will try to find jobs with a
|
214
137
|
# parent_id that's the same as self.id and count them
|
@@ -222,46 +145,7 @@ module Navvy
|
|
222
145
|
'$where' => "this._id == '#{i}' || this.parent_id == '#{i}'"
|
223
146
|
)
|
224
147
|
end
|
225
|
-
|
226
|
-
##
|
227
|
-
# Check if the job has been run.
|
228
|
-
#
|
229
|
-
# @return [true, false] ran
|
230
|
-
|
231
|
-
def ran?
|
232
|
-
completed? || failed?
|
233
|
-
end
|
234
|
-
|
235
|
-
##
|
236
|
-
# Check how long it took for a job to complete or fail
|
237
|
-
#
|
238
|
-
# @return [Time, Integer] time the time it took
|
239
|
-
|
240
|
-
def duration
|
241
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
242
|
-
end
|
243
|
-
|
244
|
-
##
|
245
|
-
# Get the job arguments as an array
|
246
|
-
#
|
247
|
-
# @return [array] arguments
|
248
|
-
|
249
|
-
def args
|
250
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
251
|
-
end
|
252
|
-
|
253
|
-
##
|
254
|
-
# Get the job status
|
255
|
-
#
|
256
|
-
# @return [:pending, :completed, :failed] status
|
257
|
-
|
258
|
-
def status
|
259
|
-
return :completed if completed?
|
260
|
-
return :failed if failed?
|
261
|
-
:pending
|
262
|
-
end
|
263
|
-
|
264
|
-
alias_method :completed?, :completed_at?
|
265
|
-
alias_method :failed?, :failed_at?
|
266
148
|
end
|
267
149
|
end
|
150
|
+
|
151
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
data/lib/navvy/job/sequel.rb
CHANGED
@@ -4,47 +4,6 @@ require 'yaml'
|
|
4
4
|
|
5
5
|
module Navvy
|
6
6
|
class Job < Sequel::Model
|
7
|
-
class << self
|
8
|
-
attr_writer :limit, :keep, :max_attempts
|
9
|
-
end
|
10
|
-
|
11
|
-
##
|
12
|
-
# Default limit of jobs to be fetched.
|
13
|
-
#
|
14
|
-
# @return [Integer] limit
|
15
|
-
|
16
|
-
def self.limit
|
17
|
-
@limit || Navvy.configuration.job_limit
|
18
|
-
end
|
19
|
-
|
20
|
-
##
|
21
|
-
# If and how long the jobs should be kept.
|
22
|
-
#
|
23
|
-
# @return [Fixnum, true, false] keep
|
24
|
-
|
25
|
-
def self.keep
|
26
|
-
@keep || Navvy.configuration.keep_jobs
|
27
|
-
end
|
28
|
-
|
29
|
-
##
|
30
|
-
# How often should a job be retried?
|
31
|
-
#
|
32
|
-
# @return [Fixnum] max_attempts
|
33
|
-
|
34
|
-
def self.max_attempts
|
35
|
-
@max_attempts || Navvy.configuration.max_attempts
|
36
|
-
end
|
37
|
-
|
38
|
-
##
|
39
|
-
# Should the job be kept?
|
40
|
-
#
|
41
|
-
# @return [true, false] keep
|
42
|
-
|
43
|
-
def self.keep?
|
44
|
-
keep = (self.keep || false)
|
45
|
-
return (Time.now + keep) >= Time.now if keep.is_a? Fixnum
|
46
|
-
keep
|
47
|
-
end
|
48
7
|
|
49
8
|
##
|
50
9
|
# Add a job to the job queue.
|
@@ -108,27 +67,25 @@ module Navvy
|
|
108
67
|
end
|
109
68
|
|
110
69
|
##
|
111
|
-
#
|
112
|
-
# value if it runs successfully unless Navvy::Job.keep is set. If a job
|
113
|
-
# fails, it'll update the Navvy::Job record to include the exception
|
114
|
-
# message it sent back and set the :failed_at date. Failed jobs never get
|
115
|
-
# deleted.
|
70
|
+
# Deletes all jobs.
|
116
71
|
#
|
117
|
-
# @
|
118
|
-
|
119
|
-
|
72
|
+
# @return [Integer] amount the amount of jobs that were deleted
|
73
|
+
|
74
|
+
def self.delete_all
|
75
|
+
Navvy::Job.destroy
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
##
|
80
|
+
# Mark the job as started. Will set started_at to the current time.
|
120
81
|
#
|
121
|
-
# @return [
|
82
|
+
# @return [true, false] update_attributes the result of the
|
83
|
+
# update_attributes call
|
122
84
|
|
123
|
-
def
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
Navvy::Job.keep? ? completed : destroy
|
128
|
-
result
|
129
|
-
rescue Exception => exception
|
130
|
-
failed(exception.message)
|
131
|
-
end
|
85
|
+
def started
|
86
|
+
update({
|
87
|
+
:started_at => Time.now
|
88
|
+
})
|
132
89
|
end
|
133
90
|
|
134
91
|
##
|
@@ -165,26 +122,6 @@ module Navvy
|
|
165
122
|
)
|
166
123
|
end
|
167
124
|
|
168
|
-
##
|
169
|
-
# Retry the current job. Will add self to the queue again, giving the clone
|
170
|
-
# a parend_id equal to self.id.
|
171
|
-
#
|
172
|
-
# @return [true, false]
|
173
|
-
|
174
|
-
def retry
|
175
|
-
self.class.enqueue(
|
176
|
-
object,
|
177
|
-
method_name,
|
178
|
-
*(args << {
|
179
|
-
:job_options => {
|
180
|
-
:parent_id => parent_id || id,
|
181
|
-
:run_at => Time.now + times_failed ** 4,
|
182
|
-
:priority => priority
|
183
|
-
}
|
184
|
-
})
|
185
|
-
)
|
186
|
-
end
|
187
|
-
|
188
125
|
##
|
189
126
|
# Check how many times the job has failed. Will try to find jobs with a
|
190
127
|
# parent_id that's the same as self.id and count them
|
@@ -197,64 +134,7 @@ module Navvy
|
|
197
134
|
"(`id` == '#{i}' OR `parent_id` == '#{i}') AND `failed_at` IS NOT NULL"
|
198
135
|
).count
|
199
136
|
end
|
200
|
-
|
201
|
-
##
|
202
|
-
# Check if the job has been run.
|
203
|
-
#
|
204
|
-
# @return [true, false] ran
|
205
|
-
|
206
|
-
def ran?
|
207
|
-
completed? || failed?
|
208
|
-
end
|
209
|
-
|
210
|
-
##
|
211
|
-
# Check how long it took for a job to complete or fail
|
212
|
-
#
|
213
|
-
# @return [Time, Integer] time the time it took
|
214
|
-
|
215
|
-
def duration
|
216
|
-
ran? ? (completed_at || failed_at) - started_at : 0
|
217
|
-
end
|
218
|
-
|
219
|
-
##
|
220
|
-
# Check if completed_at is set
|
221
|
-
#
|
222
|
-
# @return [true, false] set?
|
223
|
-
|
224
|
-
def completed_at?
|
225
|
-
!completed_at.nil?
|
226
|
-
end
|
227
|
-
|
228
|
-
##
|
229
|
-
# Check if failed_at is set
|
230
|
-
#
|
231
|
-
# @return [true, false] set?
|
232
|
-
|
233
|
-
def failed_at?
|
234
|
-
!failed_at.nil?
|
235
|
-
end
|
236
|
-
|
237
|
-
##
|
238
|
-
# Get the job arguments as an array
|
239
|
-
#
|
240
|
-
# @return [array] arguments
|
241
|
-
|
242
|
-
def args
|
243
|
-
arguments.is_a?(Array) ? arguments : YAML.load(arguments)
|
244
|
-
end
|
245
|
-
|
246
|
-
##
|
247
|
-
# Get the job status
|
248
|
-
#
|
249
|
-
# @return [:pending, :completed, :failed] status
|
250
|
-
|
251
|
-
def status
|
252
|
-
return :completed if completed?
|
253
|
-
return :failed if failed?
|
254
|
-
:pending
|
255
|
-
end
|
256
|
-
|
257
|
-
alias_method :completed?, :completed_at?
|
258
|
-
alias_method :failed?, :failed_at?
|
259
137
|
end
|
260
138
|
end
|
139
|
+
|
140
|
+
require File.expand_path(File.dirname(__FILE__) + '/../job')
|
data/lib/navvy/log.rb
CHANGED
@@ -65,9 +65,9 @@ module Navvy
|
|
65
65
|
when :rails
|
66
66
|
raise(
|
67
67
|
LoggerNotFound,
|
68
|
-
'
|
69
|
-
) unless defined?
|
70
|
-
|
68
|
+
'Rails.logger could not be found. No logs were created.'
|
69
|
+
) unless defined? Rails.logger.info
|
70
|
+
Rails.logger.info(message)
|
71
71
|
end
|
72
72
|
end
|
73
73
|
end
|
data/lib/navvy/worker.rb
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'daemons'
|
3
|
+
|
1
4
|
module Navvy
|
2
5
|
class Worker
|
3
6
|
class << self
|
@@ -12,7 +15,7 @@ module Navvy
|
|
12
15
|
def self.sleep_time
|
13
16
|
@sleep_time || Navvy.configuration.sleep_time
|
14
17
|
end
|
15
|
-
|
18
|
+
|
16
19
|
##
|
17
20
|
# Start the worker.
|
18
21
|
|
@@ -46,5 +49,34 @@ module Navvy
|
|
46
49
|
)
|
47
50
|
end
|
48
51
|
end
|
52
|
+
|
53
|
+
##
|
54
|
+
# Daemonize the worker
|
55
|
+
|
56
|
+
def self.daemonize(*args)
|
57
|
+
if defined?(ActiveRecord)
|
58
|
+
# Sets ActiveRecord's logger to Navvy a new Logger instance
|
59
|
+
ActiveRecord::Base.logger = Logger.new(STDOUT)
|
60
|
+
end
|
61
|
+
|
62
|
+
# If #daemonize does not receive any arguments, the options variable will
|
63
|
+
# contain an empty hash, and the ARGV of the environment will be used instead
|
64
|
+
# of the :ARGV options from Daemons#run_proc. However, if the *args param has been set
|
65
|
+
# this will be used instead of the environment's ARGV for the Daemons.
|
66
|
+
options = args.empty? ? {} : {:ARGV => args}
|
67
|
+
|
68
|
+
# Finally, the directory store mode will be set to normal and the Daemons PID file
|
69
|
+
# will be stored inside tmp/pids of the application.
|
70
|
+
options.merge!({:dir_mode => :normal, :dir => 'tmp/pids'})
|
71
|
+
|
72
|
+
# Ensures that the tmp/pids folder exists so that the process id file can properly be stored
|
73
|
+
%x(mkdir -p tmp/pids)
|
74
|
+
|
75
|
+
# Runs the Navvy Worker inside a Daemon
|
76
|
+
Daemons.run_proc('navvy', options) do
|
77
|
+
Navvy::Worker.start
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
49
81
|
end
|
50
82
|
end
|
data/navvy.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{navvy}
|
8
|
-
s.version = "0.2.
|
8
|
+
s.version = "0.2.1"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jeff Kreeftmeijer"]
|
12
|
-
s.date = %q{2010-02-
|
12
|
+
s.date = %q{2010-02-10}
|
13
13
|
s.description = %q{Simple background job processor inspired by delayed_job, but aiming for database agnosticism.}
|
14
14
|
s.email = %q{jeff@kreeftmeijer.nl}
|
15
15
|
s.extra_rdoc_files = [
|
@@ -25,9 +25,11 @@ Gem::Specification.new do |s|
|
|
25
25
|
"VERSION",
|
26
26
|
"generators/navvy/navvy_generator.rb",
|
27
27
|
"generators/navvy/templates/migration.rb",
|
28
|
+
"generators/navvy/templates/script",
|
28
29
|
"lib/generators/navvy_generator.rb",
|
29
30
|
"lib/navvy.rb",
|
30
31
|
"lib/navvy/configuration.rb",
|
32
|
+
"lib/navvy/job.rb",
|
31
33
|
"lib/navvy/job/active_record.rb",
|
32
34
|
"lib/navvy/job/data_mapper.rb",
|
33
35
|
"lib/navvy/job/mongo_mapper.rb",
|
data/spec/job_spec.rb
CHANGED
@@ -1,7 +1,6 @@
|
|
1
1
|
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
2
|
|
3
3
|
describe 'Navvy::Job' do
|
4
|
-
|
5
4
|
describe '.keep?' do
|
6
5
|
after(:each) do
|
7
6
|
Navvy::Job.keep = false
|
@@ -9,26 +8,26 @@ describe 'Navvy::Job' do
|
|
9
8
|
config.keep_jobs = false
|
10
9
|
end
|
11
10
|
end
|
12
|
-
|
11
|
+
|
13
12
|
describe 'when configured using Navvy::Job.keep=' do
|
14
13
|
it 'should return false' do
|
15
14
|
Navvy::Job.keep = false
|
16
15
|
Navvy::Job.keep?.should == false
|
17
|
-
end
|
16
|
+
end
|
18
17
|
|
19
18
|
it 'should return true' do
|
20
19
|
Navvy::Job.keep = true
|
21
20
|
Navvy::Job.keep?.should == true
|
22
21
|
end
|
23
|
-
end
|
24
|
-
|
22
|
+
end
|
23
|
+
|
25
24
|
describe 'when configured with Navvy.configure' do
|
26
25
|
it 'should return false' do
|
27
26
|
Navvy.configure do |config|
|
28
27
|
config.keep_jobs = false
|
29
28
|
end
|
30
29
|
Navvy::Job.keep?.should == false
|
31
|
-
end
|
30
|
+
end
|
32
31
|
|
33
32
|
it 'should return true' do
|
34
33
|
Navvy.configure do |config|
|
@@ -38,7 +37,7 @@ describe 'Navvy::Job' do
|
|
38
37
|
end
|
39
38
|
end
|
40
39
|
end
|
41
|
-
|
40
|
+
|
42
41
|
describe '.enqueue' do
|
43
42
|
before(:each) do
|
44
43
|
delete_all_jobs
|
@@ -177,6 +176,14 @@ describe 'Navvy::Job' do
|
|
177
176
|
end
|
178
177
|
end
|
179
178
|
|
179
|
+
describe '.delete_all' do
|
180
|
+
it 'should delete all jobs' do
|
181
|
+
3.times do; Navvy::Job.create; end
|
182
|
+
Navvy::Job.delete_all
|
183
|
+
job_count.should == 0
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
180
187
|
describe '#run' do
|
181
188
|
it 'should pass the arguments' do
|
182
189
|
delete_all_jobs
|
@@ -242,6 +249,19 @@ describe 'Navvy::Job' do
|
|
242
249
|
end
|
243
250
|
end
|
244
251
|
|
252
|
+
describe '#started' do
|
253
|
+
before(:each) do
|
254
|
+
delete_all_jobs
|
255
|
+
Navvy::Job.enqueue(Cow, :speak)
|
256
|
+
end
|
257
|
+
|
258
|
+
it 'should update the jobs started_at date' do
|
259
|
+
jobs = Navvy::Job.next
|
260
|
+
jobs.first.started
|
261
|
+
jobs.first.started_at.should_not be_nil
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
245
265
|
describe '#completed' do
|
246
266
|
before(:each) do
|
247
267
|
delete_all_jobs
|
@@ -388,19 +408,23 @@ describe 'Navvy::Job' do
|
|
388
408
|
|
389
409
|
@failed_job.times_failed.should == 3
|
390
410
|
end
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
411
|
+
|
412
|
+
it 'should return 2 when having 1 failed and one pending child' do
|
413
|
+
Navvy::Job.create(
|
414
|
+
:failed_at => Time.now,
|
415
|
+
:parent_id => @failed_job.id
|
416
|
+
)
|
417
|
+
|
418
|
+
Navvy::Job.create(
|
419
|
+
:parent_id => @failed_job.id
|
420
|
+
)
|
421
|
+
|
422
|
+
Navvy::Job.create(
|
423
|
+
:parent_id => @failed_job.id
|
424
|
+
)
|
425
|
+
|
426
|
+
@failed_job.times_failed.should == 2
|
427
|
+
end
|
404
428
|
|
405
429
|
it 'should return 2 when having failed and having a failed parent' do
|
406
430
|
failed_child = Navvy::Job.create(
|
data/spec/log_spec.rb
CHANGED
@@ -13,9 +13,9 @@ describe Navvy::Log do
|
|
13
13
|
lambda { Navvy::Log.info('123') }.should raise_error
|
14
14
|
end
|
15
15
|
|
16
|
-
it 'should pass the log to
|
16
|
+
it 'should pass the log to Rails.logger' do
|
17
17
|
require File.expand_path(File.dirname(__FILE__) + '/setup/rails_default_logger')
|
18
|
-
|
18
|
+
Rails.logger.should_receive(:info).with('123')
|
19
19
|
Navvy::Log.info('123')
|
20
20
|
end
|
21
21
|
end
|
@@ -42,7 +42,7 @@ describe Navvy::Log do
|
|
42
42
|
end
|
43
43
|
|
44
44
|
it 'should pass the log to justlogging' do
|
45
|
-
|
45
|
+
Rails.logger.should_receive(:info).with('123')
|
46
46
|
Justlogging.should_receive(:log).with('123')
|
47
47
|
Navvy::Log.info('123')
|
48
48
|
end
|
@@ -54,7 +54,7 @@ describe Navvy::Log do
|
|
54
54
|
end
|
55
55
|
|
56
56
|
it 'should not log' do
|
57
|
-
|
57
|
+
Rails.logger.should_not_receive(:info)
|
58
58
|
Justlogging.should_not_receive(:log)
|
59
59
|
Navvy::Log.info('123')
|
60
60
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: navvy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeff Kreeftmeijer
|
@@ -9,7 +9,7 @@ autorequire:
|
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
11
|
|
12
|
-
date: 2010-02-
|
12
|
+
date: 2010-02-10 00:00:00 +01:00
|
13
13
|
default_executable:
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
@@ -80,9 +80,11 @@ files:
|
|
80
80
|
- VERSION
|
81
81
|
- generators/navvy/navvy_generator.rb
|
82
82
|
- generators/navvy/templates/migration.rb
|
83
|
+
- generators/navvy/templates/script
|
83
84
|
- lib/generators/navvy_generator.rb
|
84
85
|
- lib/navvy.rb
|
85
86
|
- lib/navvy/configuration.rb
|
87
|
+
- lib/navvy/job.rb
|
86
88
|
- lib/navvy/job/active_record.rb
|
87
89
|
- lib/navvy/job/data_mapper.rb
|
88
90
|
- lib/navvy/job/mongo_mapper.rb
|