steve 2.0.3 → 2.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/active_job/queue_adapters/steve_adapter.rb +1 -1
- data/lib/steve.rb +16 -16
- data/lib/steve/interface.rb +16 -13
- data/lib/steve/interface/views/completed.haml +0 -2
- data/lib/steve/interface/views/failed.haml +2 -5
- data/lib/steve/job.rb +8 -8
- data/lib/steve/queued_job.rb +51 -28
- data/lib/steve/worker.rb +7 -7
- metadata +4 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff4ed0070df9be5041ccf923f6241cd8c484fbd1
|
4
|
+
data.tar.gz: fe44158d942b5ef3bd69f7acafd276bd36d07f42
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4db62a04d2c0c42669cbabeb60716adf504b3943dbc3a8fe36dc6e6e587b06b299c08f5487fb244705e594b9f4f7f06fdcdbd6980c1c901faada4fe331eb8a5b
|
7
|
+
data.tar.gz: 25d9f7681af9ec48b87035baa2a3df3ede024ca806d1f2193197bea1bf22027885cb15c4b5ae6538fef5a9de9583eb96ecccfc5fad5c193c2ba117789149e870
|
data/lib/steve.rb
CHANGED
@@ -1,58 +1,58 @@
|
|
1
1
|
module Steve
|
2
|
-
|
2
|
+
|
3
3
|
class << self
|
4
|
-
|
4
|
+
|
5
5
|
## The default queue for new jobs
|
6
6
|
attr_accessor :default_job_queue
|
7
|
-
|
7
|
+
|
8
8
|
## The default priority for new jobs
|
9
|
-
attr_accessor :default_job_priority
|
10
|
-
|
9
|
+
attr_accessor :default_job_priority
|
10
|
+
|
11
11
|
## The name of the table where jobs are stored (default 'jobs')
|
12
12
|
attr_accessor :jobs_table_name
|
13
13
|
|
14
14
|
## The name of the table where completed jobs are stored
|
15
15
|
attr_accessor :archived_jobs_table_name
|
16
|
-
|
16
|
+
|
17
17
|
## The period of time to wait before looking for new jobs
|
18
18
|
attr_accessor :worker_sleep_time
|
19
|
-
|
19
|
+
|
20
20
|
## The logger object for all output from steve
|
21
21
|
attr_accessor :logger
|
22
|
-
|
22
|
+
|
23
23
|
## The maximum number of times to retry a job
|
24
24
|
attr_accessor :max_job_retries
|
25
|
-
|
25
|
+
|
26
26
|
## Whether or not jobs should be queued or run in the background
|
27
27
|
attr_accessor :run_jobs_in_foreground
|
28
|
-
|
28
|
+
|
29
29
|
## Proc to run after forking
|
30
30
|
attr_accessor :after_job_fork
|
31
|
-
|
31
|
+
|
32
32
|
## Allow parent process name to be kept
|
33
33
|
attr_accessor :keep_parent_process_name
|
34
34
|
|
35
35
|
## Delete jobs after successful completion
|
36
36
|
attr_accessor :delete_successful_jobs
|
37
|
-
|
37
|
+
|
38
38
|
## Return the worker name for this current process/host
|
39
39
|
def worker_name
|
40
40
|
"host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
## Set/return the logger object
|
44
44
|
def logger
|
45
45
|
@logger ||= Logger.new(File.join(Rails.root, 'log', 'jobs.log'))
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
## Log a new message
|
49
49
|
def log(message)
|
50
50
|
message.gsub!(/(\[\d+\])/) { "\e[33m#{$1}\e[0m" }
|
51
51
|
logger.info "\e[37m#{Time.now.utc.to_s(:db)}\e[0m #{message}"
|
52
52
|
end
|
53
|
-
|
53
|
+
|
54
54
|
end
|
55
|
-
|
55
|
+
|
56
56
|
end
|
57
57
|
|
58
58
|
require 'steve/job'
|
data/lib/steve/interface.rb
CHANGED
@@ -1,21 +1,17 @@
|
|
1
|
-
require 'haml'
|
2
|
-
require 'will_paginate/view_helpers'
|
3
|
-
require 'will_paginate/view_helpers/link_renderer'
|
1
|
+
# require 'haml'
|
4
2
|
|
5
3
|
module Steve
|
6
4
|
class Interface
|
7
|
-
include WillPaginate::ViewHelpers
|
8
|
-
|
9
5
|
def call(env)
|
10
6
|
@req = Rack::Request.new(env)
|
11
7
|
case env['PATH_INFO'].to_s
|
12
8
|
when '/', ''
|
13
9
|
[200, {'Content-type' => 'text/html'}, [haml(:index)]]
|
14
10
|
when /failed/
|
15
|
-
@jobs = Steve::ArchivedJob.failed.asc
|
11
|
+
@jobs = paginate(Steve::ArchivedJob.failed.asc, :page => @req.params['page'], :per_page => 50)
|
16
12
|
[200, {'Content-type' => 'text/html'}, [haml(:failed)]]
|
17
13
|
when /completed/
|
18
|
-
@jobs = Steve::ArchivedJob.completed.asc
|
14
|
+
@jobs = paginate(Steve::ArchivedJob.completed.asc, :page => @req.params['page'], :per_page => 50)
|
19
15
|
[200, {'Content-type' => 'text/html'}, [haml(:completed)]]
|
20
16
|
when /archived\/object/
|
21
17
|
@jobs = Steve::ArchivedJob.where(:associated_object_type => @req.params['type'], :associated_object_id => @req.params['id']).order('created_at desc').limit(25)
|
@@ -44,23 +40,30 @@ module Steve
|
|
44
40
|
else
|
45
41
|
[404, {'Content-type' => 'text/plain'}, ["Not found"]]
|
46
42
|
end
|
47
|
-
|
43
|
+
|
48
44
|
end
|
49
45
|
end
|
50
|
-
|
46
|
+
|
51
47
|
private
|
52
|
-
|
48
|
+
|
49
|
+
def paginate(collection, options = {})
|
50
|
+
options[:per_page] ||= 50
|
51
|
+
options[:page] ||= 1
|
52
|
+
|
53
|
+
collection.limit(options[:per_page]).offset((options[:page].to_i - 1) * options[:per_page])
|
54
|
+
end
|
55
|
+
|
53
56
|
def haml(view_name)
|
54
57
|
Haml::Engine.new(File.read(view_path(view_name))).render(self)
|
55
58
|
end
|
56
|
-
|
59
|
+
|
57
60
|
def view_path(name)
|
58
61
|
File.expand_path(File.join('..', 'interface', 'views', name.to_s + '.haml'), __FILE__)
|
59
62
|
end
|
60
|
-
|
63
|
+
|
61
64
|
def static_path(name)
|
62
65
|
File.expand_path(File.join('..', 'interface', 'public', name.to_s), __FILE__).gsub(/\.\./, '')
|
63
66
|
end
|
64
|
-
|
67
|
+
|
65
68
|
end
|
66
69
|
end
|
@@ -12,7 +12,7 @@
|
|
12
12
|
%p
|
13
13
|
The jobs listed below have failed to execute recently. You can clear all the jobs using the button below, this will
|
14
14
|
remove them from the database and you will not be able to retry once removed.
|
15
|
-
|
15
|
+
|
16
16
|
%table.data
|
17
17
|
%thead
|
18
18
|
%tr
|
@@ -37,13 +37,10 @@
|
|
37
37
|
%code{:style => 'font-size:90%', :title => job.error}= ERB::Util.html_escape(job.error.to_s.split("\n").first)\
|
38
38
|
%td
|
39
39
|
%form{:action => "/jobs/retry?id=#{job.id}", :method => :post}
|
40
|
-
%input{:type => 'submit', :value => 'Retry'}
|
40
|
+
%input{:type => 'submit', :value => 'Retry'}
|
41
41
|
- if @jobs.empty?
|
42
42
|
%tr
|
43
43
|
%td.none{:colspan => 6} There are no jobs currently running
|
44
|
-
|
45
|
-
.pagination-details
|
46
|
-
= page_entries_info @jobs
|
47
44
|
|
48
45
|
%p
|
49
46
|
%a{:href => "/jobs"} Back to overview
|
data/lib/steve/job.rb
CHANGED
@@ -3,19 +3,19 @@
|
|
3
3
|
|
4
4
|
module Steve
|
5
5
|
class Job
|
6
|
-
|
6
|
+
|
7
7
|
class Delay < StandardError; end
|
8
8
|
class Error < StandardError; end
|
9
9
|
class Aborted < StandardError; end
|
10
|
-
|
10
|
+
|
11
11
|
attr_reader :job
|
12
12
|
attr_reader :params
|
13
|
-
|
13
|
+
|
14
14
|
def initialize(job, params = {})
|
15
15
|
@job = job
|
16
16
|
@params = job.params
|
17
17
|
end
|
18
|
-
|
18
|
+
|
19
19
|
## Queue the job
|
20
20
|
def self.queue(params = {}, &block)
|
21
21
|
if Steve.run_jobs_in_foreground
|
@@ -27,22 +27,22 @@ module Steve
|
|
27
27
|
QueuedJob.queue(self, params, &block)
|
28
28
|
end
|
29
29
|
end
|
30
|
-
|
30
|
+
|
31
31
|
## Live run a job after passing the params
|
32
32
|
def self.run(params = {})
|
33
33
|
self.new(FakeQueuedJob.new(params)).perform
|
34
34
|
end
|
35
|
-
|
35
|
+
|
36
36
|
class FakeQueuedJob
|
37
37
|
attr_reader :params
|
38
38
|
def initialize(params)
|
39
39
|
@params = params
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
def method_missing(*_)
|
43
43
|
true
|
44
44
|
end
|
45
45
|
end
|
46
|
-
|
46
|
+
|
47
47
|
end
|
48
48
|
end
|
data/lib/steve/queued_job.rb
CHANGED
@@ -2,20 +2,20 @@ require 'stringio'
|
|
2
2
|
|
3
3
|
module Steve
|
4
4
|
class QueuedJob < ActiveRecord::Base
|
5
|
-
|
5
|
+
|
6
6
|
## Set the table name - a migration should be created for this
|
7
7
|
self.table_name = Steve.jobs_table_name || 'jobs'
|
8
|
-
|
8
|
+
|
9
9
|
## Serialize the options
|
10
10
|
serialize :params
|
11
|
-
|
11
|
+
|
12
12
|
## Can belong to another active record object?
|
13
13
|
belongs_to :associated_object, :polymorphic => true
|
14
|
-
|
14
|
+
|
15
15
|
## Scopes
|
16
16
|
scope :pending, lambda { where(:status => ['pending', 'delayed']) }
|
17
17
|
scope :running, lambda { where(:status => 'running') }
|
18
|
-
|
18
|
+
|
19
19
|
## Automatically set defaults for new jobs
|
20
20
|
before_create do
|
21
21
|
self.priority ||= (Steve.default_job_priority || 10)
|
@@ -23,7 +23,7 @@ module Steve
|
|
23
23
|
self.status ||= 'pending'
|
24
24
|
self.run_at ||= Time.now.utc
|
25
25
|
end
|
26
|
-
|
26
|
+
|
27
27
|
after_save do
|
28
28
|
Steve::ArchivedJob.archive_job(self) if self.archive_job?
|
29
29
|
true
|
@@ -40,8 +40,8 @@ module Steve
|
|
40
40
|
block.call(job) if block_given?
|
41
41
|
job.save
|
42
42
|
end
|
43
|
-
|
44
|
-
## Execute a new job from the queue. Returns true if a job was executed, or false if a
|
43
|
+
|
44
|
+
## Execute a new job from the queue. Returns true if a job was executed, or false if a
|
45
45
|
## job was not found or we couldn't obtain locks for them.
|
46
46
|
def self.execute_jobs(queue = '*', limit = 5)
|
47
47
|
jobs = self.where(:status => ['pending', 'delayed'], :worker => nil).where(["run_at <= ?", Time.now.utc]).order("priority asc").limit(5)
|
@@ -51,7 +51,7 @@ module Steve
|
|
51
51
|
jobs_executed = Array.new
|
52
52
|
for job in pending_jobs.sort_by { rand() }
|
53
53
|
Steve.log "[#{job.id}] Attempt to aquire lock"
|
54
|
-
if job.lock
|
54
|
+
if job.lock
|
55
55
|
Steve.log "[#{job.id}] Lock acquired"
|
56
56
|
if @child = fork
|
57
57
|
rand
|
@@ -63,7 +63,7 @@ module Steve
|
|
63
63
|
$0 = "sj: executing job ##{job.id} since #{Time.now.utc.to_s(:db)}"
|
64
64
|
Steve.after_job_fork.call if Steve.after_job_fork.is_a?(Proc)
|
65
65
|
job.execute
|
66
|
-
exit
|
66
|
+
exit!
|
67
67
|
end
|
68
68
|
jobs_executed << job
|
69
69
|
else
|
@@ -72,7 +72,7 @@ module Steve
|
|
72
72
|
end
|
73
73
|
jobs_executed
|
74
74
|
end
|
75
|
-
|
75
|
+
|
76
76
|
## Execute this job, catching any errors if they occur and ensuring the
|
77
77
|
## job is started & finished as appropriate.
|
78
78
|
def execute
|
@@ -92,17 +92,37 @@ module Steve
|
|
92
92
|
else
|
93
93
|
self.error = e.message
|
94
94
|
delay!
|
95
|
-
Steve.log "[#{self.id}] Delayed ('#{e.message}')"
|
95
|
+
Steve.log "[#{self.id}] Delayed ('#{e.message}')"
|
96
96
|
end
|
97
97
|
rescue Timeout::Error
|
98
|
+
reconnect_database!
|
98
99
|
fail!('Timed out')
|
99
100
|
Steve.log "[#{self.id}] Timed out: #{e.to_s}"
|
100
101
|
rescue => e
|
101
102
|
if e.is_a?(Steve::Job::Error)
|
102
103
|
fail!(e.message)
|
103
104
|
else
|
105
|
+
fail!([e.to_s, e.backtrace].join("\n"))
|
104
106
|
if defined?(Airbrake)
|
105
|
-
|
107
|
+
begin
|
108
|
+
Timeout.timeout(5) do
|
109
|
+
Airbrake.notify(e, :component => self.job.to_s, :action => self.id.to_s, :parameters => self.params)
|
110
|
+
end
|
111
|
+
rescue Timeout::Error
|
112
|
+
reconnect_database!
|
113
|
+
Steve.log "[#{self.id}] Timed out sending exception to Airbrake"
|
114
|
+
end
|
115
|
+
end
|
116
|
+
if defined?(Raven)
|
117
|
+
Raven.capture_exception(e, :tags => {:component => 'worker'}, :extra => {:steve => {
|
118
|
+
:id => self.id,
|
119
|
+
:priority => self.priority,
|
120
|
+
:attempts => self.retries,
|
121
|
+
:run_at => self.run_at,
|
122
|
+
:queue => self.queue,
|
123
|
+
:created_at => self.created_at,
|
124
|
+
:params => self.params
|
125
|
+
}})
|
106
126
|
end
|
107
127
|
fail!([e.to_s, e.backtrace].join("\n"))
|
108
128
|
end
|
@@ -114,12 +134,11 @@ module Steve
|
|
114
134
|
return false
|
115
135
|
end
|
116
136
|
ensure
|
117
|
-
STDOUT.flush && STDERR.flush
|
118
|
-
if File.exist?(output_file)
|
119
|
-
self.output = File.read(output_file)
|
120
|
-
FileUtils.rm(output_file)
|
121
|
-
end
|
122
137
|
self.save(:validate => false)
|
138
|
+
STDOUT.flush
|
139
|
+
STDERR.flush
|
140
|
+
|
141
|
+
FileUtils.rm(output_file) if File.exist?(output_file)
|
123
142
|
end
|
124
143
|
|
125
144
|
## Get the output of a job, even if the job is in progress,
|
@@ -129,10 +148,10 @@ module Steve
|
|
129
148
|
if read_attribute(:output)
|
130
149
|
read_attribute(:output)
|
131
150
|
elsif File.exist?(output_file)
|
132
|
-
File.read(output_file)
|
151
|
+
File.read(output_file)
|
133
152
|
end
|
134
153
|
end
|
135
|
-
|
154
|
+
|
136
155
|
## Get a lock on this job. Returns true if the lock was successful
|
137
156
|
## otherwise it returns false.
|
138
157
|
def lock
|
@@ -144,25 +163,25 @@ module Steve
|
|
144
163
|
return false
|
145
164
|
end
|
146
165
|
end
|
147
|
-
|
166
|
+
|
148
167
|
## Mark this job as succeeded successfully.
|
149
168
|
def success!
|
150
169
|
self.status = 'completed'
|
151
170
|
finish!
|
152
171
|
end
|
153
|
-
|
172
|
+
|
154
173
|
## Mark this job as failed.
|
155
174
|
def fail!(message)
|
156
175
|
self.error = message
|
157
176
|
self.status = 'failed'
|
158
177
|
finish!
|
159
178
|
end
|
160
|
-
|
179
|
+
|
161
180
|
## Mark this job as finished
|
162
181
|
def finish!
|
163
182
|
self.finished_at = Time.now.utc
|
164
183
|
end
|
165
|
-
|
184
|
+
|
166
185
|
## Mark this job as started
|
167
186
|
def start!
|
168
187
|
self.error = nil
|
@@ -172,7 +191,7 @@ module Steve
|
|
172
191
|
self.job_pid = Process.pid
|
173
192
|
self.save(:validate => false)
|
174
193
|
end
|
175
|
-
|
194
|
+
|
176
195
|
## Delay this job by the time specified
|
177
196
|
def delay!(delay_time = 30.seconds)
|
178
197
|
self.status = 'delayed'
|
@@ -182,13 +201,13 @@ module Steve
|
|
182
201
|
self.retries += 1
|
183
202
|
self.save(:validate => false)
|
184
203
|
end
|
185
|
-
|
204
|
+
|
186
205
|
## Associate this job with the pased active record object
|
187
206
|
def associate_with(object)
|
188
207
|
self.associated_object = object
|
189
208
|
self.save(:validate => false)
|
190
209
|
end
|
191
|
-
|
210
|
+
|
192
211
|
## In a state appropriate to be archived?
|
193
212
|
def archive_job?
|
194
213
|
['failed', 'completed'].include?(self.status)
|
@@ -202,6 +221,10 @@ module Steve
|
|
202
221
|
def self.cleanup(age = 5.days.ago)
|
203
222
|
self.delete_all(["status = 'completed' and run_at < ?", age])
|
204
223
|
end
|
205
|
-
|
224
|
+
|
225
|
+
def reconnect_database!
|
226
|
+
self.class.connection.reconnect!
|
227
|
+
end
|
228
|
+
|
206
229
|
end
|
207
230
|
end
|
data/lib/steve/worker.rb
CHANGED
@@ -1,20 +1,20 @@
|
|
1
1
|
module Steve
|
2
2
|
class Worker
|
3
|
-
|
3
|
+
|
4
4
|
def initialize(queue)
|
5
5
|
@queue = queue
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
def start
|
9
9
|
Steve.log "*** Starting job worker #{Steve.worker_name} (queue: #{@queue})"
|
10
|
-
|
10
|
+
|
11
11
|
trap("TERM") { Steve.log("*** Exiting..."); $exit = true }
|
12
12
|
trap("INT") { Steve.log("*** Exiting..."); $exit = true }
|
13
|
-
|
13
|
+
|
14
14
|
loop do
|
15
15
|
jobs = Steve::QueuedJob.execute_jobs(@queue)
|
16
16
|
count = jobs.size
|
17
|
-
|
17
|
+
|
18
18
|
unless count == 0
|
19
19
|
Steve.log "*** #{count} jobs processed"
|
20
20
|
else
|
@@ -22,10 +22,10 @@ module Steve
|
|
22
22
|
$0 = "sj: waiting for jobs on #{@queue}"
|
23
23
|
sleep(Steve.worker_sleep_time || 5)
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
break if $exit
|
27
27
|
end
|
28
|
-
|
28
|
+
|
29
29
|
end
|
30
30
|
end
|
31
31
|
end
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: steve
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- aTech Media
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
12
|
-
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: will_paginate
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 3.0.0
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: 3.0.0
|
11
|
+
date: 2019-09-30 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
27
13
|
description:
|
28
14
|
email: support@atechmedia.com
|
29
15
|
executables: []
|
@@ -62,7 +48,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
62
48
|
version: '0'
|
63
49
|
requirements: []
|
64
50
|
rubyforge_project:
|
65
|
-
rubygems_version: 2.
|
51
|
+
rubygems_version: 2.6.14.3
|
66
52
|
signing_key:
|
67
53
|
specification_version: 4
|
68
54
|
summary: Steve Jobs background job runner
|