steve 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/steve/interface/public/style.css +45 -0
- data/lib/steve/interface/views/completed.haml +41 -0
- data/lib/steve/interface/views/failed.haml +43 -0
- data/lib/steve/interface/views/index.haml +74 -0
- data/lib/steve/interface/views/object.haml +38 -0
- data/lib/steve/interface/views/view.haml +52 -0
- data/lib/steve/interface.rb +48 -0
- data/lib/steve/job.rb +47 -0
- data/lib/steve/queued_job.rb +186 -0
- data/lib/steve/worker.rb +31 -0
- data/lib/steve.rb +53 -0
- metadata +55 -0
@@ -0,0 +1,45 @@
|
|
1
|
+
html { color: #000; background: #FFF; }
|
2
|
+
body,div,dl,dt,dd,ul,ol,li,h1,h2,h3,h4,h5,h6,pre,code,form,fieldset,legend,input,textarea,p,blockquote,th,td { margin: 0; padding: 0; }
|
3
|
+
li { list-style: none; }
|
4
|
+
h1, h2, h3, h4, h5, h6 { font-size: 100%; font-weight: normal; }
|
5
|
+
pre, form { font-style: normal; font-weight: normal; }
|
6
|
+
fieldset { border: 0; }
|
7
|
+
legend { color: #000; }
|
8
|
+
input, textarea { margin: 0; padding: 0; font-family: inherit; font-size: inherit; font-weight: inherit; *font-size: 100%; }
|
9
|
+
p, blockquote { margin: 0; padding: 0; }
|
10
|
+
th { margin: 0; padding: 0; font-style: normal; font-weight: normal; text-align: left; }
|
11
|
+
table { border-collapse: collapse; border-spacing: 0; }
|
12
|
+
img { border: 0; }
|
13
|
+
address { font-style: normal; font-weight: normal; }
|
14
|
+
caption { font-style: normal; font-weight: normal; text-align: left; }
|
15
|
+
cite, dfn, em, strong, var { font-style: normal; font-weight: normal; }
|
16
|
+
q:before, q:after { content: ''; }
|
17
|
+
abbr, acronym { border: 0; font-variant: normal; }
|
18
|
+
sup { vertical-align: text-top; }
|
19
|
+
sub { vertical-align: text-bottom; }
|
20
|
+
select { font-family: inherit; font-size: inherit; font-weight: inherit; *font-size: 100%; }
|
21
|
+
|
22
|
+
|
23
|
+
html { font-family:"Helvetica Neue", Arial, sans-serif; font-size:13px; margin:0; }
|
24
|
+
body { margin:0;}
|
25
|
+
#header { background:#000; color:#fff; padding:10px 2%;}
|
26
|
+
#header h1 { color:#999;}
|
27
|
+
#header h1 span { font-weight:bold; font-size:90%; color:#fff;}
|
28
|
+
#header h1 em { color:yellow; font-weight:bold;}
|
29
|
+
|
30
|
+
#content { margin:15px 2%;}
|
31
|
+
#content h2 { font-size:150%; font-weight:bold; margin:15px 0;}
|
32
|
+
#content table.data {width:100%;}
|
33
|
+
#content table.data thead td { background:#efefef; font-weight:bold; font-size:90%;}
|
34
|
+
#content table.data td { padding:5px; border:1px solid #ccc;}
|
35
|
+
#content table.data td code span { display:block; font-size:70%;}
|
36
|
+
#content table.data td.none { color:#999; font-style:italic; font-size:80%;}
|
37
|
+
#content ul { line-height:1.5; }
|
38
|
+
#content ul li { list-style:square; margin-left:25px;}
|
39
|
+
#content p { margin:10px 0; line-height:1.5;}
|
40
|
+
#content a { color:#333;}
|
41
|
+
|
42
|
+
#content dl {margin:40px 0; font-size:110%;}
|
43
|
+
#content dl dt { font-weight:bold; float:left; width:150px; text-align:right;}
|
44
|
+
#content dl dd { margin-left:180px; margin-bottom:10px;}
|
45
|
+
#content dl dd pre { background:#000; color:#fff; line-height:1.3; padding:6px;}
|
@@ -0,0 +1,41 @@
|
|
1
|
+
!!!
|
2
|
+
%html
|
3
|
+
%head
|
4
|
+
%title Steve - Job Queue Monitor
|
5
|
+
%link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
|
6
|
+
%body
|
7
|
+
#header
|
8
|
+
%h1 <span>Steve</span>—Job Queue Monitor
|
9
|
+
|
10
|
+
#content
|
11
|
+
#recent
|
12
|
+
%h2 Recently completed jobs
|
13
|
+
%table.data
|
14
|
+
%thead
|
15
|
+
%tr
|
16
|
+
%td{:width => '35%'} Job
|
17
|
+
%td{:width => '25%'} Object
|
18
|
+
%td{:width => '20%'} Queued at
|
19
|
+
%td{:width => '15%'} Run at
|
20
|
+
%td{:width => '5%'} Retries
|
21
|
+
%tbody
|
22
|
+
- jobs = Steve::QueuedJob.completed.limit(15).order('run_at desc')
|
23
|
+
- for job in jobs
|
24
|
+
%tr
|
25
|
+
%td
|
26
|
+
%code
|
27
|
+
%a{:href => "view?id=#{job.id}"}= job.job
|
28
|
+
%span= job.params.inspect
|
29
|
+
%td
|
30
|
+
- if job.associated_object_type.blank?
|
31
|
+
—
|
32
|
+
- else
|
33
|
+
== #{job.associated_object_type}##{job.associated_object_id}
|
34
|
+
%td= job.created_at.to_s(:db)
|
35
|
+
%td= job.run_at.to_s(:db)
|
36
|
+
%td= job.retries
|
37
|
+
- if jobs.empty?
|
38
|
+
%tr
|
39
|
+
%td.none{:colspan => 7} There are no jobs recently completed
|
40
|
+
%p
|
41
|
+
%a{:href => "/jobs"} Back to overview
|
@@ -0,0 +1,43 @@
|
|
1
|
+
!!!
|
2
|
+
%html
|
3
|
+
%head
|
4
|
+
%title Steve - Job Queue Monitor
|
5
|
+
%link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
|
6
|
+
%body
|
7
|
+
#header
|
8
|
+
%h1 <span>Steve</span>—Job Queue Monitor
|
9
|
+
|
10
|
+
#content
|
11
|
+
%h2 Failed jobs
|
12
|
+
%p
|
13
|
+
The jobs listed below have failed to execute recently. You can clear all the jobs using the button below, this will
|
14
|
+
remove them from the database and you will not be able to retry once removed.
|
15
|
+
%table.data
|
16
|
+
%thead
|
17
|
+
%tr
|
18
|
+
%td{:width => '15%'} Date
|
19
|
+
%td{:width => '5%'} ID
|
20
|
+
%td{:width => '35%'} Job
|
21
|
+
%td{:width => '10%'} Queue
|
22
|
+
%td{:width => '35%'} Error
|
23
|
+
%td{:width => '10%'}
|
24
|
+
%tbody
|
25
|
+
- jobs = Steve::QueuedJob.failed
|
26
|
+
- for job in jobs
|
27
|
+
%tr
|
28
|
+
%td= job.finished_at.to_s(:short)
|
29
|
+
%td= job.id
|
30
|
+
%td
|
31
|
+
%code
|
32
|
+
%a{:href => "view?id=#{job.id}"}= job.job
|
33
|
+
%span= job.params.inspect
|
34
|
+
%td= job.queue
|
35
|
+
%td
|
36
|
+
%code{:style => 'font-size:90%', :title => job.error}= ERB::Util.html_escape(job.error.split("\n").first)\
|
37
|
+
%td
|
38
|
+
%a{:href => '#'} Retry
|
39
|
+
- if jobs.empty?
|
40
|
+
%tr
|
41
|
+
%td.none{:colspan => 6} There are no jobs currently running
|
42
|
+
%p
|
43
|
+
%a{:href => "/jobs"} Back to overview
|
@@ -0,0 +1,74 @@
|
|
1
|
+
!!!
|
2
|
+
%html
|
3
|
+
%head
|
4
|
+
%title Steve - Job Queue Monitor
|
5
|
+
%link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
|
6
|
+
%body
|
7
|
+
#header
|
8
|
+
%h1 <span>Steve</span>—Job Queue Monitor
|
9
|
+
|
10
|
+
#content
|
11
|
+
#active
|
12
|
+
%h2 Jobs being executed at the moment
|
13
|
+
%table.data
|
14
|
+
%thead
|
15
|
+
%tr
|
16
|
+
%td{:width => '25%'} Job
|
17
|
+
%td{:width => '25%'} Queue
|
18
|
+
%td{:width => '50%'} Worker
|
19
|
+
%tbody
|
20
|
+
- jobs = Steve::QueuedJob.running
|
21
|
+
- for job in jobs
|
22
|
+
%tr
|
23
|
+
%td
|
24
|
+
%code
|
25
|
+
%b= job.job
|
26
|
+
%span= job.params.inspect
|
27
|
+
%td= job.queue
|
28
|
+
%td= job.worker
|
29
|
+
- if jobs.empty?
|
30
|
+
%tr
|
31
|
+
%td.none{:colspan => 3} There are no jobs currently running
|
32
|
+
|
33
|
+
#queues
|
34
|
+
%h2 Jobs which are pending execution
|
35
|
+
%table.data
|
36
|
+
%thead
|
37
|
+
%tr
|
38
|
+
%td{:width => '25%'} Job
|
39
|
+
%td{:width => '15%'} Queue
|
40
|
+
%td{:width => '10%'} Status
|
41
|
+
%td{:width => '10%'} Priority
|
42
|
+
%td{:width => '20%'} Queued at
|
43
|
+
%td{:width => '15%'} Run at
|
44
|
+
%td{:width => '5%'} Retries
|
45
|
+
%tbody
|
46
|
+
- jobs = Steve::QueuedJob.pending
|
47
|
+
- for job in jobs
|
48
|
+
%tr
|
49
|
+
%td
|
50
|
+
%code
|
51
|
+
%b= job.job
|
52
|
+
%span= job.params.inspect
|
53
|
+
%td= job.queue
|
54
|
+
%td= job.status
|
55
|
+
%td= job.priority
|
56
|
+
%td= job.created_at.to_s(:db)
|
57
|
+
%td= job.run_at.to_s(:db)
|
58
|
+
%td= job.retries
|
59
|
+
- if jobs.empty?
|
60
|
+
%tr
|
61
|
+
%td.none{:colspan => 7} There are no jobs pending
|
62
|
+
|
63
|
+
|
64
|
+
|
65
|
+
#stats
|
66
|
+
%h2 Statistics
|
67
|
+
%ul
|
68
|
+
%li== #{Steve::QueuedJob.count} jobs stored
|
69
|
+
%li== #{Steve::QueuedJob.pending.count} pending execution
|
70
|
+
%li== #{Steve::QueuedJob.running.count} running now
|
71
|
+
%li
|
72
|
+
%a{:href => 'jobs/completed'}== #{Steve::QueuedJob.completed.count} completed successfully
|
73
|
+
%li
|
74
|
+
%a{:href => 'jobs/failed'}== #{Steve::QueuedJob.failed.count} failed to execute
|
@@ -0,0 +1,38 @@
|
|
1
|
+
!!!
|
2
|
+
%html
|
3
|
+
%head
|
4
|
+
%title Steve - Job Queue Monitor
|
5
|
+
%link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
|
6
|
+
%body
|
7
|
+
#header
|
8
|
+
%h1 <span>Steve</span>—Job Queue Monitor
|
9
|
+
|
10
|
+
#content
|
11
|
+
#recent
|
12
|
+
%h2 Jobs for #{@req.params['type']}##{@req.params['id']}
|
13
|
+
%table.data
|
14
|
+
%thead
|
15
|
+
%tr
|
16
|
+
%td{:width => '35%'} Job
|
17
|
+
%td{:width => '10%'} Status
|
18
|
+
%td{:width => '15%'} Queue
|
19
|
+
%td{:width => '20%'} Queued at
|
20
|
+
%td{:width => '15%'} Run at
|
21
|
+
%td{:width => '5%'} Retries
|
22
|
+
%tbody
|
23
|
+
- for job in @jobs
|
24
|
+
%tr
|
25
|
+
%td
|
26
|
+
%code
|
27
|
+
%a{:href => "view?id=#{job.id}"}= job.job
|
28
|
+
%span= job.params.inspect
|
29
|
+
%td= job.status
|
30
|
+
%td= job.queue
|
31
|
+
%td= job.created_at.to_s(:db)
|
32
|
+
%td= job.run_at.to_s(:db)
|
33
|
+
%td= job.retries
|
34
|
+
- if @jobs.empty?
|
35
|
+
%tr
|
36
|
+
%td.none{:colspan => 7} There are no jobs recently completed
|
37
|
+
%p
|
38
|
+
%a{:href => "/jobs"} Back to overview
|
@@ -0,0 +1,52 @@
|
|
1
|
+
!!!
|
2
|
+
%html
|
3
|
+
%head
|
4
|
+
%title Steve - Job Queue Monitor
|
5
|
+
%link{:href => "/jobs/style.css", :media => 'screen', :type => 'text/css', :rel => 'stylesheet'}
|
6
|
+
%body
|
7
|
+
#header
|
8
|
+
%h1 <span>Steve</span>—Job Queue Monitor
|
9
|
+
|
10
|
+
#content
|
11
|
+
%dl
|
12
|
+
%dt Job
|
13
|
+
%dd= @job.job
|
14
|
+
|
15
|
+
%dt Queue
|
16
|
+
%dd= @job.queue
|
17
|
+
|
18
|
+
%dt Status
|
19
|
+
%dd= @job.status
|
20
|
+
|
21
|
+
%dt Parameters
|
22
|
+
%dd= @job.params.inspect
|
23
|
+
|
24
|
+
%dt Association?
|
25
|
+
%dd== #{@job.associated_object_type}##{@job.associated_object_id}
|
26
|
+
|
27
|
+
%dt Run at
|
28
|
+
%dd= @job.run_at
|
29
|
+
|
30
|
+
%dt Queued at
|
31
|
+
%dd= @job.created_at
|
32
|
+
|
33
|
+
%dt Started at
|
34
|
+
%dd= @job.started_at
|
35
|
+
|
36
|
+
%dt Finished at
|
37
|
+
%dd= @job.finished_at
|
38
|
+
|
39
|
+
- unless @job.output.blank?
|
40
|
+
%dt Output
|
41
|
+
%dd
|
42
|
+
%pre~ preserve(@job.output)
|
43
|
+
|
44
|
+
- unless @job.error.blank?
|
45
|
+
%dt Error
|
46
|
+
%dd
|
47
|
+
%pre~ preserve(@job.error || " ")
|
48
|
+
|
49
|
+
%dt Worker
|
50
|
+
%dd= @job.worker
|
51
|
+
%p
|
52
|
+
%a{:href => "/jobs"} Back to overview
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'haml'
|
2
|
+
|
3
|
+
module Steve
|
4
|
+
class Interface
|
5
|
+
|
6
|
+
def call(env)
|
7
|
+
@req = Rack::Request.new(env)
|
8
|
+
case env['PATH_INFO'].to_s
|
9
|
+
when ''
|
10
|
+
[200, {'Content-type' => 'text/html'}, [haml(:index)]]
|
11
|
+
when /failed/
|
12
|
+
[200, {'Content-type' => 'text/html'}, [haml(:failed)]]
|
13
|
+
when /completed/
|
14
|
+
[200, {'Content-type' => 'text/html'}, [haml(:completed)]]
|
15
|
+
when /object/
|
16
|
+
@jobs = Steve::QueuedJob.where(:associated_object_type => @req.params['type'], :associated_object_id => @req.params['id']).order('created_at desc').limit(25)
|
17
|
+
[200, {'Content-type' => 'text/html'}, [haml(:object)]]
|
18
|
+
when /view/
|
19
|
+
@job = Steve::QueuedJob.find(@req.params['id'])
|
20
|
+
[200, {'Content-type' => 'text/html'}, [haml(:view)]]
|
21
|
+
|
22
|
+
else
|
23
|
+
path = static_path(env['PATH_INFO'])
|
24
|
+
if File.exist?(path)
|
25
|
+
[200, {}, [File.read(path)]]
|
26
|
+
else
|
27
|
+
[404, {'Content-type' => 'text/plain'}, ["Not found"]]
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def haml(view_name)
|
36
|
+
Haml::Engine.new(File.read(view_path(view_name))).render(self)
|
37
|
+
end
|
38
|
+
|
39
|
+
def view_path(name)
|
40
|
+
File.expand_path(File.join('..', 'interface', 'views', name.to_s + '.haml'), __FILE__)
|
41
|
+
end
|
42
|
+
|
43
|
+
def static_path(name)
|
44
|
+
File.expand_path(File.join('..', 'interface', 'public', name.to_s), __FILE__).gsub(/\.\./, '')
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
data/lib/steve/job.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
## Jobs can inherit from this class to add useful functionality and avoid
|
2
|
+
## needing to specify an initializer for all your jobs.
|
3
|
+
|
4
|
+
module Steve
|
5
|
+
class Job
|
6
|
+
|
7
|
+
class Delay < StandardError; end
|
8
|
+
class Error < StandardError; end
|
9
|
+
|
10
|
+
attr_reader :job
|
11
|
+
attr_reader :params
|
12
|
+
|
13
|
+
def initialize(job, params = {})
|
14
|
+
@job = job
|
15
|
+
@params = job.params
|
16
|
+
end
|
17
|
+
|
18
|
+
## Queue the job
|
19
|
+
def self.queue(params = {}, &block)
|
20
|
+
if Steve.run_jobs_in_foreground
|
21
|
+
Rails.logger.info "\e[33m -> Started to exectute #{self.to_s} job in foreground\e[0m"
|
22
|
+
run(params)
|
23
|
+
Rails.logger.info "\e[33m -> Finished executing #{self.to_s} job in foreground\e[0m"
|
24
|
+
else
|
25
|
+
Rails.logger.info "\e[33m -> Queued #{self.to_s} job\e[0m"
|
26
|
+
QueuedJob.queue(self, params, &block)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
## Live run a job after passing the params
|
31
|
+
def self.run(params = {})
|
32
|
+
self.new(FakeQueuedJob.new(params)).perform
|
33
|
+
end
|
34
|
+
|
35
|
+
class FakeQueuedJob
|
36
|
+
attr_reader :params
|
37
|
+
def initialize(params)
|
38
|
+
@params = params
|
39
|
+
end
|
40
|
+
|
41
|
+
def method_missing(*_)
|
42
|
+
true
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,186 @@
|
|
1
|
+
require 'stringio'
|
2
|
+
|
3
|
+
module Steve
|
4
|
+
class QueuedJob < ActiveRecord::Base
|
5
|
+
|
6
|
+
## Set the table name - a migration should be created for this
|
7
|
+
set_table_name(Steve.jobs_table_name || 'jobs')
|
8
|
+
|
9
|
+
## Serialize the options
|
10
|
+
serialize :params
|
11
|
+
|
12
|
+
## Can belong to another active record object?
|
13
|
+
belongs_to :associated_object, :polymorphic => true
|
14
|
+
|
15
|
+
## Scopes
|
16
|
+
scope :pending, where(:status => ['pending', 'delayed'])
|
17
|
+
scope :running, where(:status => 'running')
|
18
|
+
scope :completed, where(:status => 'completed')
|
19
|
+
scope :failed, where(:status => 'failed')
|
20
|
+
|
21
|
+
## Automatically set defaults for new jobs
|
22
|
+
before_create do
|
23
|
+
self.priority ||= (Steve.default_job_priority || 10)
|
24
|
+
self.queue ||= (Steve.default_job_queue || 'normal')
|
25
|
+
self.status ||= 'pending'
|
26
|
+
self.run_at ||= Time.now.utc
|
27
|
+
end
|
28
|
+
|
29
|
+
## Queue a new job for processing. Returns true or false depending whether
|
30
|
+
## the job has been queued or not.
|
31
|
+
def self.queue(klass, params = {}, &block)
|
32
|
+
job = self.new
|
33
|
+
job.job = klass.to_s
|
34
|
+
job.params = params
|
35
|
+
job.queue = klass.instance_variable_get('@queue')
|
36
|
+
block.call(job) if block_given?
|
37
|
+
job.save
|
38
|
+
end
|
39
|
+
|
40
|
+
## Execute a new job from the queue. Returns true if a job was executed, or false if a
|
41
|
+
## job was not found or we couldn't obtain locks for them.
|
42
|
+
def self.execute_jobs(queue = '*', limit = 5)
|
43
|
+
pending_jobs = ActiveRecord::Base.silence do
|
44
|
+
jobs = self.where(:status => ['pending', 'delayed'], :worker => nil).where(["run_at <= ?", Time.now.utc]).order("priority asc").limit(5)
|
45
|
+
jobs = jobs.where(:queue => queue) unless queue.nil? or queue == '*'
|
46
|
+
jobs.all
|
47
|
+
end
|
48
|
+
|
49
|
+
jobs_executed = Array.new
|
50
|
+
for job in pending_jobs.sort_by { rand() }
|
51
|
+
Steve.log "[#{job.id}] Attempt to aquire lock"
|
52
|
+
if job.lock
|
53
|
+
Steve.log "[#{job.id}] Lock acquired"
|
54
|
+
ActiveRecord::Base.remove_connection
|
55
|
+
if @child = fork
|
56
|
+
rand
|
57
|
+
Steve.log "[#{job.id}] Forked to #{@child}"
|
58
|
+
$0 = "sj: forked to #{@child} at #{Time.now.utc.to_s(:db)}"
|
59
|
+
ActiveRecord::Base.establish_connection
|
60
|
+
Process.wait
|
61
|
+
else
|
62
|
+
Steve.log "[#{job.id}] Executing"
|
63
|
+
$0 = "sj: executing job ##{job.id} since #{Time.now.utc.to_s(:db)}"
|
64
|
+
ActiveRecord::Base.establish_connection
|
65
|
+
Steve.after_job_fork.call if Steve.after_job_fork.is_a?(Proc)
|
66
|
+
job.execute
|
67
|
+
exit
|
68
|
+
end
|
69
|
+
jobs_executed << job
|
70
|
+
else
|
71
|
+
Steve.log "[#{job.id}] Lock could not be acquired. Moving on."
|
72
|
+
end
|
73
|
+
end
|
74
|
+
jobs_executed
|
75
|
+
end
|
76
|
+
|
77
|
+
## Execute this job, catching any errors if they occur and ensuring the
|
78
|
+
## job is started & finished as appropriate.
|
79
|
+
def execute
|
80
|
+
@output_file = File.join('', 'tmp', "steve-job-#{self.id}")
|
81
|
+
job = self.job.constantize.new(self)
|
82
|
+
if job.respond_to?(:perform)
|
83
|
+
start!
|
84
|
+
STDOUT.reopen(@output_file) && STDERR.reopen(@output_file)
|
85
|
+
begin
|
86
|
+
job.perform
|
87
|
+
success!
|
88
|
+
Steve.log "[#{self.id}] Succeeded"
|
89
|
+
rescue Steve::Job::Delay => e
|
90
|
+
max_attempts = (Steve.max_job_retries || 5)
|
91
|
+
if self.retries >= max_attempts
|
92
|
+
fail!("#{e.message} after #{max_attempts} attempt(s)")
|
93
|
+
Steve.log "[#{self.id}] Failed after #{max_attempts} attempt(s): #{e.message}"
|
94
|
+
else
|
95
|
+
self.error = e.message
|
96
|
+
delay!
|
97
|
+
Steve.log "[#{self.id}] Delayed ('#{e.message}')"
|
98
|
+
end
|
99
|
+
rescue Timeout::Error
|
100
|
+
fail!('Timed out')
|
101
|
+
Steve.log "[#{self.id}] Timed out: #{e.to_s}"
|
102
|
+
rescue => e
|
103
|
+
if e.is_a?(Steve::Job::Error)
|
104
|
+
fail!(e.message)
|
105
|
+
else
|
106
|
+
if defined?(Airbrake)
|
107
|
+
Airbrake.notify(e, :component => self.job.to_s, :action => self.id.to_s, :parameters => self.params)
|
108
|
+
end
|
109
|
+
fail!([e.to_s, e.backtrace].join("\n"))
|
110
|
+
end
|
111
|
+
Steve.log "[#{self.id}] Failed: #{e.to_s}"
|
112
|
+
end
|
113
|
+
else
|
114
|
+
fail! "#{self.id} did not respond to 'perform'"
|
115
|
+
Steve.log "[#{self.id}] Failed: does not respond to 'perform'"
|
116
|
+
return false
|
117
|
+
end
|
118
|
+
ensure
|
119
|
+
STDOUT.flush && STDERR.flush
|
120
|
+
self.output = File.read(@output_file)
|
121
|
+
self.save(:validate => false)
|
122
|
+
FileUtils.rm(@output_file) if File.exist?(@output_file)
|
123
|
+
end
|
124
|
+
|
125
|
+
## Get a lock on this job. Returns true if the lock was successful
|
126
|
+
## otherwise it returns false.
|
127
|
+
def lock
|
128
|
+
rows = self.class.update_all({:worker => Steve.worker_name}, {:id => self.id, :worker => nil})
|
129
|
+
if rows == 1
|
130
|
+
self.worker = Steve.worker_name
|
131
|
+
return true
|
132
|
+
else
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
## Mark this job as succeeded successfully.
|
138
|
+
def success!
|
139
|
+
self.status = 'completed'
|
140
|
+
finish!
|
141
|
+
end
|
142
|
+
|
143
|
+
## Mark this job as failed.
|
144
|
+
def fail!(message)
|
145
|
+
self.error = message
|
146
|
+
self.status = 'failed'
|
147
|
+
finish!
|
148
|
+
end
|
149
|
+
|
150
|
+
## Mark this job as finished
|
151
|
+
def finish!
|
152
|
+
self.finished_at = Time.now.utc
|
153
|
+
end
|
154
|
+
|
155
|
+
## Mark this job as started
|
156
|
+
def start!
|
157
|
+
self.error = nil
|
158
|
+
self.finished_at = nil
|
159
|
+
self.started_at = Time.now.utc
|
160
|
+
self.status = 'running'
|
161
|
+
self.save(:validate => false)
|
162
|
+
end
|
163
|
+
|
164
|
+
## Delay this job by the time specified
|
165
|
+
def delay!(delay_time = 30.seconds)
|
166
|
+
self.status = 'delayed'
|
167
|
+
self.run_at = Time.now.utc + delay_time
|
168
|
+
self.started_at = nil
|
169
|
+
self.worker = nil
|
170
|
+
self.retries += 1
|
171
|
+
self.save(:validate => false)
|
172
|
+
end
|
173
|
+
|
174
|
+
## Associate this job with the pased active record object
|
175
|
+
def associate_with(object)
|
176
|
+
self.associated_object = object
|
177
|
+
self.save(:validate => false)
|
178
|
+
end
|
179
|
+
|
180
|
+
## Remove old completed jobs from the database
|
181
|
+
def self.cleanup
|
182
|
+
self.delete_all(["status = 'completed' and run_at < ?", 5.days.ago])
|
183
|
+
end
|
184
|
+
|
185
|
+
end
|
186
|
+
end
|
data/lib/steve/worker.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
module Steve
|
2
|
+
class Worker
|
3
|
+
|
4
|
+
def initialize(queue)
|
5
|
+
@queue = queue
|
6
|
+
end
|
7
|
+
|
8
|
+
def start
|
9
|
+
Steve.log "*** Starting job worker #{Steve.worker_name} (queue: #{@queue})"
|
10
|
+
|
11
|
+
trap("TERM") { Steve.log("*** Exiting..."); $exit = true }
|
12
|
+
trap("INT") { Steve.log("*** Exiting..."); $exit = true }
|
13
|
+
|
14
|
+
loop do
|
15
|
+
jobs = Steve::QueuedJob.execute_jobs(@queue)
|
16
|
+
count = jobs.size
|
17
|
+
|
18
|
+
unless count == 0
|
19
|
+
Steve.log "*** #{count} jobs processed"
|
20
|
+
else
|
21
|
+
break if $exit
|
22
|
+
$0 = "sj: waiting for jobs on #{@queue}"
|
23
|
+
sleep(Steve.worker_sleep_time || 5)
|
24
|
+
end
|
25
|
+
|
26
|
+
break if $exit
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/lib/steve.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
module Steve
|
2
|
+
|
3
|
+
class << self
|
4
|
+
|
5
|
+
## The default queue for new jobs
|
6
|
+
attr_accessor :default_job_queue
|
7
|
+
|
8
|
+
## The default priority for new jobs
|
9
|
+
attr_accessor :default_job_priority
|
10
|
+
|
11
|
+
## The name of the table where jobs are stored (default 'jobs')
|
12
|
+
attr_accessor :jobs_table_name
|
13
|
+
|
14
|
+
## The period of time to wait before looking for new jobs
|
15
|
+
attr_accessor :worker_sleep_time
|
16
|
+
|
17
|
+
## The logger object for all output from steve
|
18
|
+
attr_accessor :logger
|
19
|
+
|
20
|
+
## The maximum number of times to retry a job
|
21
|
+
attr_accessor :max_job_retries
|
22
|
+
|
23
|
+
## Whether or not jobs should be queued or run in the background
|
24
|
+
attr_accessor :run_jobs_in_foreground
|
25
|
+
|
26
|
+
## Proc to run after forking
|
27
|
+
attr_accessor :after_job_fork
|
28
|
+
|
29
|
+
## Return the worker name for this current process/host
|
30
|
+
def worker_name
|
31
|
+
"host:#{Socket.gethostname} pid:#{Process.pid}" rescue "pid:#{Process.pid}"
|
32
|
+
end
|
33
|
+
|
34
|
+
## Set/return the logger object
|
35
|
+
def logger
|
36
|
+
@logger ||= Logger.new(File.join(Rails.root, 'log', 'jobs.log'))
|
37
|
+
end
|
38
|
+
|
39
|
+
## Log a new message
|
40
|
+
def log(message)
|
41
|
+
message.gsub!(/(\[\d+\])/) { "\e[33m#{$1}\e[0m" }
|
42
|
+
logger.info "\e[37m#{Time.now.utc.to_s(:db)}\e[0m #{message}"
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
require 'steve/job'
|
50
|
+
require 'steve/queued_job'
|
51
|
+
require 'steve/worker'
|
52
|
+
require 'steve/interface'
|
53
|
+
|
metadata
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: steve
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- aTech Media
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-03-09 00:00:00.000000000Z
|
13
|
+
dependencies: []
|
14
|
+
description:
|
15
|
+
email: support@atechmedia.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- lib/steve/interface/public/style.css
|
21
|
+
- lib/steve/interface/views/completed.haml
|
22
|
+
- lib/steve/interface/views/failed.haml
|
23
|
+
- lib/steve/interface/views/index.haml
|
24
|
+
- lib/steve/interface/views/object.haml
|
25
|
+
- lib/steve/interface/views/view.haml
|
26
|
+
- lib/steve/interface.rb
|
27
|
+
- lib/steve/job.rb
|
28
|
+
- lib/steve/queued_job.rb
|
29
|
+
- lib/steve/worker.rb
|
30
|
+
- lib/steve.rb
|
31
|
+
homepage: http://atechmedia.com
|
32
|
+
licenses: []
|
33
|
+
post_install_message:
|
34
|
+
rdoc_options: []
|
35
|
+
require_paths:
|
36
|
+
- lib
|
37
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ! '>='
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '0'
|
43
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
44
|
+
none: false
|
45
|
+
requirements:
|
46
|
+
- - ! '>='
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
requirements: []
|
50
|
+
rubyforge_project:
|
51
|
+
rubygems_version: 1.8.10
|
52
|
+
signing_key:
|
53
|
+
specification_version: 3
|
54
|
+
summary: Deployment Recipes for Appli
|
55
|
+
test_files: []
|