rocketjob 1.1.3 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rocket_job/cli.rb +62 -5
- data/lib/rocket_job/concerns/worker.rb +7 -11
- data/lib/rocket_job/job.rb +28 -10
- data/lib/rocket_job/version.rb +1 -1
- data/lib/rocket_job/worker.rb +3 -1
- data/test/dirmon_entry_test.rb +2 -2
- data/test/dirmon_job_test.rb +21 -21
- data/test/job_test.rb +4 -2
- data/test/job_worker_test.rb +2 -2
- data/test/jobs/test_job.rb +5 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c742a1c093d122fbb2d6d9b194607d1ebd6743f3
|
4
|
+
data.tar.gz: 2998a334bf98a4624ac3c42b69c70c1a4a4dcaf8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b032a72a38db2aeb1f121cd65e5835fce438d77c1f6248560c40086fe4fb47f69815c83b8ac491188819ad0b4fea2a93e661015d7e38e0ccb1474e9b3e6c9505
|
7
|
+
data.tar.gz: f152e0cbd1af97f02674a5b47e8f9854f7cfac1c5fc509489b5ab30cd3c464ba11fc6ba41809687720a2caa18fd8dc2b7eaebc5a33fac2c6df7f411777089960
|
data/lib/rocket_job/cli.rb
CHANGED
@@ -1,23 +1,27 @@
|
|
1
1
|
require 'optparse'
|
2
|
+
require 'yaml'
|
2
3
|
module RocketJob
|
3
4
|
# Command Line Interface parser for RocketJob
|
4
5
|
class CLI
|
5
|
-
|
6
|
+
include SemanticLogger::Loggable
|
7
|
+
attr_accessor :name, :threads, :environment, :pidfile, :directory, :quiet, :log_level
|
6
8
|
|
7
9
|
def initialize(argv)
|
8
10
|
@name = nil
|
9
11
|
@threads = nil
|
10
12
|
@quiet = false
|
11
|
-
@environment =
|
13
|
+
@environment = nil
|
12
14
|
@pidfile = nil
|
13
15
|
@directory = '.'
|
16
|
+
@log_level = nil
|
14
17
|
parse(argv)
|
15
18
|
end
|
16
19
|
|
17
20
|
# Run a RocketJob::Worker from the command line
|
18
21
|
def run
|
19
|
-
|
20
|
-
|
22
|
+
setup_environment
|
23
|
+
setup_logger
|
24
|
+
boot_standalone unless boot_rails
|
21
25
|
write_pidfile
|
22
26
|
|
23
27
|
opts = {}
|
@@ -27,14 +31,28 @@ module RocketJob
|
|
27
31
|
end
|
28
32
|
|
29
33
|
# Initialize the Rails environment
|
34
|
+
# Returns [true|false] whether Rails is present
|
30
35
|
def boot_rails
|
31
|
-
|
36
|
+
boot_file = Pathname.new(directory).join('config/environment.rb').expand_path
|
37
|
+
return false unless boot_file.file?
|
38
|
+
|
39
|
+
logger.info 'Booting Rails'
|
40
|
+
require boot_file.to_s
|
32
41
|
if Rails.configuration.eager_load
|
33
42
|
RocketJob::Worker.logger.benchmark_info('Eager loaded Rails and all Engines') do
|
34
43
|
Rails.application.eager_load!
|
35
44
|
Rails::Engine.subclasses.each(&:eager_load!)
|
36
45
|
end
|
37
46
|
end
|
47
|
+
|
48
|
+
self.class.load_config(Rails.env)
|
49
|
+
true
|
50
|
+
end
|
51
|
+
|
52
|
+
def boot_standalone
|
53
|
+
logger.info 'Rails not detected. Running standalone.'
|
54
|
+
self.class.load_config(environment)
|
55
|
+
self.class.eager_load_jobs
|
38
56
|
end
|
39
57
|
|
40
58
|
# Create a PID file if requested
|
@@ -49,6 +67,44 @@ module RocketJob
|
|
49
67
|
end
|
50
68
|
end
|
51
69
|
|
70
|
+
def setup_environment
|
71
|
+
# Override Env vars when environment is supplied
|
72
|
+
if environment
|
73
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
|
74
|
+
else
|
75
|
+
self.environment = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def setup_logger
|
80
|
+
SemanticLogger.add_appender(STDOUT, &SemanticLogger::Appender::Base.colorized_formatter) unless quiet
|
81
|
+
SemanticLogger.default_level = log_level.to_sym if log_level
|
82
|
+
end
|
83
|
+
|
84
|
+
# Configure MongoMapper if it has not already been configured
|
85
|
+
def self.load_config(environment='development', file_name=nil)
|
86
|
+
return false if MongoMapper.config
|
87
|
+
|
88
|
+
config_file = file_name ? Pathname.new(file_name) : Pathname.pwd.join('config/mongo.yml')
|
89
|
+
if config_file.file?
|
90
|
+
config = YAML.load(ERB.new(config_file.read).result)
|
91
|
+
log = SemanticLogger::DebugAsTraceLogger.new('Mongo')
|
92
|
+
MongoMapper.setup(config, environment, logger: log)
|
93
|
+
true
|
94
|
+
else
|
95
|
+
raise(ArgumentError, "Mongo Configuration file: #{config_file.to_s} not found")
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# Eager load files in jobs folder
|
100
|
+
def self.eager_load_jobs(path = 'jobs')
|
101
|
+
Pathname.glob("#{path}/**/*.rb").each do |path|
|
102
|
+
next if path.directory?
|
103
|
+
logger.debug "Loading #{path.to_s}"
|
104
|
+
load path.expand_path.to_s
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
52
108
|
# Parse command line options placing results in the corresponding instance variables
|
53
109
|
def parse(argv)
|
54
110
|
parser = OptionParser.new do |o|
|
@@ -57,6 +113,7 @@ module RocketJob
|
|
57
113
|
o.on('-q', '--quiet', 'Do not write to stdout, only to logfile. Necessary when running as a daemon') { @quiet = true }
|
58
114
|
o.on('-d', '--dir DIR', 'Directory containing Rails app, if not current directory') { |arg| @directory = arg }
|
59
115
|
o.on('-e', '--environment ENVIRONMENT', 'The environment to run the app on (Default: RAILS_ENV || RACK_ENV || development)') { |arg| @environment = arg }
|
116
|
+
o.on('-l', '--log_level trace|debug|info|warn|error|fatal', 'The log level to use') { |arg| @log_level = arg }
|
60
117
|
o.on('--pidfile PATH', 'Use PATH as a pidfile') { |arg| @pidfile = arg }
|
61
118
|
o.on('-v', '--version', 'Print the version information') do
|
62
119
|
puts "Rocket Job v#{RocketJob::VERSION}"
|
@@ -6,9 +6,6 @@ module RocketJob
|
|
6
6
|
module Worker
|
7
7
|
def self.included(base)
|
8
8
|
base.extend ClassMethods
|
9
|
-
base.class_eval do
|
10
|
-
@rocket_job_defaults = nil
|
11
|
-
end
|
12
9
|
end
|
13
10
|
|
14
11
|
module ClassMethods
|
@@ -25,7 +22,13 @@ module RocketJob
|
|
25
22
|
|
26
23
|
# Create a job and process it immediately in-line by this thread
|
27
24
|
def now(method, *args, &block)
|
28
|
-
job
|
25
|
+
job = build(method, *args, &block)
|
26
|
+
# Call validations
|
27
|
+
if job.respond_to?(:validate!)
|
28
|
+
job.validate!
|
29
|
+
elsif job.invalid?
|
30
|
+
raise(MongoMapper::DocumentNotValid, "Validation failed: #{job.errors.messages.join(', ')}")
|
31
|
+
end
|
29
32
|
worker = RocketJob::Worker.new(name: 'inline')
|
30
33
|
worker.started
|
31
34
|
job.start
|
@@ -43,7 +46,6 @@ module RocketJob
|
|
43
46
|
# discarded, call #cleanup! to clear out any partially uploaded data
|
44
47
|
def build(method, *args, &block)
|
45
48
|
job = new(arguments: args, perform_method: method.to_sym)
|
46
|
-
@rocket_job_defaults.call(job) if @rocket_job_defaults
|
47
49
|
block.call(job) if block
|
48
50
|
job
|
49
51
|
end
|
@@ -63,12 +65,6 @@ module RocketJob
|
|
63
65
|
now(:perform, *args, &block)
|
64
66
|
end
|
65
67
|
|
66
|
-
# Define job defaults
|
67
|
-
def rocket_job(&block)
|
68
|
-
@rocket_job_defaults = block
|
69
|
-
self
|
70
|
-
end
|
71
|
-
|
72
68
|
# Returns the next job to work on in priority based order
|
73
69
|
# Returns nil if there are currently no queued jobs, or processing batch jobs
|
74
70
|
# with records that require processing
|
data/lib/rocket_job/job.rb
CHANGED
@@ -223,6 +223,12 @@ module RocketJob
|
|
223
223
|
instance_method(method).arity
|
224
224
|
end
|
225
225
|
|
226
|
+
# Override parent defaults
|
227
|
+
def self.rocket_job(&block)
|
228
|
+
@rocket_job_defaults = block
|
229
|
+
self
|
230
|
+
end
|
231
|
+
|
226
232
|
# Returns [true|false] whether to collect the results from running this batch
|
227
233
|
def collect_output?
|
228
234
|
collect_output == true
|
@@ -313,14 +319,6 @@ module RocketJob
|
|
313
319
|
end
|
314
320
|
end
|
315
321
|
|
316
|
-
# After this model is read, convert any hashes in the arguments list to HashWithIndifferentAccess
|
317
|
-
def load_from_database(*args)
|
318
|
-
super
|
319
|
-
if arguments.present?
|
320
|
-
self.arguments = arguments.collect { |i| i.is_a?(BSON::OrderedHash) ? i.with_indifferent_access : i }
|
321
|
-
end
|
322
|
-
end
|
323
|
-
|
324
322
|
# Set exception information for this job and fail it
|
325
323
|
def fail(worker_name='user', exc_or_message='Job failed through user action')
|
326
324
|
if exc_or_message.is_a?(Exception)
|
@@ -360,7 +358,6 @@ module RocketJob
|
|
360
358
|
aasm_fire_event(:requeue, persist: false)
|
361
359
|
end
|
362
360
|
|
363
|
-
############################################################################
|
364
361
|
protected
|
365
362
|
|
366
363
|
# Before events that can be overridden by child classes
|
@@ -382,7 +379,7 @@ module RocketJob
|
|
382
379
|
|
383
380
|
def before_retry
|
384
381
|
self.completed_at = nil
|
385
|
-
self.exception
|
382
|
+
self.exception = nil
|
386
383
|
end
|
387
384
|
|
388
385
|
def before_pause
|
@@ -404,5 +401,26 @@ module RocketJob
|
|
404
401
|
self.worker_name = nil
|
405
402
|
end
|
406
403
|
|
404
|
+
private
|
405
|
+
|
406
|
+
# After this model is loaded, convert any hashes in the arguments list to HashWithIndifferentAccess
|
407
|
+
def load_from_database(*args)
|
408
|
+
super
|
409
|
+
if arguments.present?
|
410
|
+
self.arguments = arguments.collect { |i| i.is_a?(BSON::OrderedHash) ? i.with_indifferent_access : i }
|
411
|
+
end
|
412
|
+
end
|
413
|
+
|
414
|
+
def self.apply_defaults(job)
|
415
|
+
@rocket_job_defaults.call(job) if @rocket_job_defaults
|
416
|
+
end
|
417
|
+
|
418
|
+
# Apply RocketJob defaults after initializing default values
|
419
|
+
# but before setting attributes
|
420
|
+
def initialize_default_values(except = {})
|
421
|
+
super
|
422
|
+
self.class.apply_defaults(self)
|
423
|
+
end
|
424
|
+
|
407
425
|
end
|
408
426
|
end
|
data/lib/rocket_job/version.rb
CHANGED
data/lib/rocket_job/worker.rb
CHANGED
@@ -102,7 +102,9 @@ module RocketJob
|
|
102
102
|
worker.save!
|
103
103
|
create_indexes
|
104
104
|
register_signal_handlers
|
105
|
-
|
105
|
+
if defined?(RocketJobPro) && (RocketJob::Job.database.name != RocketJob::SlicedJob.database.name)
|
106
|
+
raise 'The RocketJob configuration is being applied after the system has been initialized'
|
107
|
+
end
|
106
108
|
logger.info "Using MongoDB Database: #{RocketJob::Job.database.name}"
|
107
109
|
worker.run
|
108
110
|
end
|
data/test/dirmon_entry_test.rb
CHANGED
@@ -245,9 +245,9 @@ class DirmonEntryTest < Minitest::Test
|
|
245
245
|
|
246
246
|
describe '#later' do
|
247
247
|
it 'enqueue job' do
|
248
|
-
@entry.arguments
|
248
|
+
@entry.arguments = [{}]
|
249
249
|
@entry.perform_method = :event
|
250
|
-
job
|
250
|
+
job = @entry.later(@pathname)
|
251
251
|
assert_equal File.join(@archive_directory, "#{job.id}_#{File.basename(@file_name)}"), job.arguments.first[:full_file_name]
|
252
252
|
assert job.queued?
|
253
253
|
end
|
data/test/dirmon_job_test.rb
CHANGED
@@ -117,7 +117,7 @@ class DirmonJobTest < Minitest::Test
|
|
117
117
|
|
118
118
|
it 'skip files in archive directory' do
|
119
119
|
@entry.archive_directory = nil
|
120
|
-
@entry.pattern
|
120
|
+
@entry.pattern = "#{@directory}/abc/**/*"
|
121
121
|
|
122
122
|
create_file("#{@directory}/abc/file1", 5)
|
123
123
|
create_file("#{@directory}/abc/file2", 10)
|
@@ -139,7 +139,7 @@ class DirmonJobTest < Minitest::Test
|
|
139
139
|
"#{@directory}/abc/file1" => 5,
|
140
140
|
"#{@directory}/abc/file2" => 10,
|
141
141
|
}
|
142
|
-
new_file_names
|
142
|
+
new_file_names = {
|
143
143
|
"#{@directory}/abc/file1" => 10,
|
144
144
|
"#{@directory}/abc/file2" => 10,
|
145
145
|
}
|
@@ -165,28 +165,28 @@ class DirmonJobTest < Minitest::Test
|
|
165
165
|
new_dirmon_job.destroy
|
166
166
|
end
|
167
167
|
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
end
|
168
|
+
it 'check directories and reschedule even on exception' do
|
169
|
+
dirmon_job = nil
|
170
|
+
RocketJob::Jobs::DirmonJob.destroy_all
|
171
|
+
RocketJob::Jobs::DirmonJob.stub_any_instance(:check_directories, -> previous { raise RuntimeError.new("Oh no") }) do
|
172
|
+
# perform_now does not save the job, just runs it
|
173
|
+
dirmon_job = RocketJob::Jobs::DirmonJob.perform_now do |job|
|
174
|
+
job.priority = 11
|
175
|
+
job.check_seconds = 30
|
177
176
|
end
|
178
|
-
|
177
|
+
end
|
178
|
+
assert dirmon_job.failed?, dirmon_job.status.inspect
|
179
179
|
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
180
|
+
# It it have enqueued another instance to run in the future
|
181
|
+
assert_equal 2, RocketJob::Jobs::DirmonJob.count
|
182
|
+
assert new_dirmon_job = RocketJob::Jobs::DirmonJob.last
|
183
|
+
assert new_dirmon_job.run_at
|
184
|
+
assert_equal 11, new_dirmon_job.priority
|
185
|
+
assert_equal 30, new_dirmon_job.check_seconds
|
186
|
+
assert new_dirmon_job.queued?
|
187
187
|
|
188
|
-
|
189
|
-
|
188
|
+
new_dirmon_job.destroy
|
189
|
+
end
|
190
190
|
end
|
191
191
|
|
192
192
|
def create_file(file_name, size)
|
data/test/job_test.rb
CHANGED
@@ -17,7 +17,8 @@ class JobTest < Minitest::Test
|
|
17
17
|
@job2 = Jobs::TestJob.new(
|
18
18
|
description: "#{@description} 2",
|
19
19
|
arguments: @arguments,
|
20
|
-
destroy_on_complete: false
|
20
|
+
destroy_on_complete: false,
|
21
|
+
priority: 52
|
21
22
|
)
|
22
23
|
end
|
23
24
|
|
@@ -64,7 +65,7 @@ class JobTest < Minitest::Test
|
|
64
65
|
assert_nil @job.expires_at
|
65
66
|
assert_equal @arguments, @job.arguments
|
66
67
|
assert_equal 0, @job.percent_complete
|
67
|
-
assert_equal
|
68
|
+
assert_equal 51, @job.priority
|
68
69
|
assert_equal 0, @job.failure_count
|
69
70
|
assert_nil @job.run_at
|
70
71
|
assert_nil @job.started_at
|
@@ -268,6 +269,7 @@ class JobTest < Minitest::Test
|
|
268
269
|
|
269
270
|
describe '.requeue_dead_worker' do
|
270
271
|
it 'requeue jobs from dead workers' do
|
272
|
+
assert_equal 52, @job2.priority
|
271
273
|
worker_name = 'server:12345'
|
272
274
|
@job.worker_name = worker_name
|
273
275
|
@job.start!
|
data/test/job_worker_test.rb
CHANGED
@@ -29,7 +29,7 @@ class WorkerTest < Minitest::Test
|
|
29
29
|
assert_equal false, @job.destroy_on_complete
|
30
30
|
assert_nil @job.expires_at
|
31
31
|
assert_equal 0, @job.percent_complete
|
32
|
-
assert_equal
|
32
|
+
assert_equal 51, @job.priority
|
33
33
|
assert_equal 0, @job.failure_count
|
34
34
|
assert_nil @job.run_at
|
35
35
|
assert_nil @job.started_at
|
@@ -48,7 +48,7 @@ class WorkerTest < Minitest::Test
|
|
48
48
|
assert_equal false, @job.destroy_on_complete
|
49
49
|
assert_nil @job.expires_at
|
50
50
|
assert_equal 100, @job.percent_complete
|
51
|
-
assert_equal
|
51
|
+
assert_equal 51, @job.priority
|
52
52
|
assert_equal 0, @job.failure_count
|
53
53
|
assert_nil @job.run_at
|
54
54
|
assert @job.started_at
|
data/test/jobs/test_job.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rocketjob
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Reid Morrison
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-09-
|
11
|
+
date: 2015-09-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aasm
|