mobilize-base 1.3 → 1.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +11 -10
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +2 -7
- data/lib/mobilize-base/extensions/string.rb +4 -12
- data/lib/mobilize-base/extensions/yaml.rb +7 -10
- data/lib/mobilize-base/handlers/gbook.rb +31 -24
- data/lib/mobilize-base/handlers/gfile.rb +3 -5
- data/lib/mobilize-base/handlers/gridfs.rb +24 -19
- data/lib/mobilize-base/handlers/gsheet.rb +20 -25
- data/lib/mobilize-base/handlers/resque.rb +4 -16
- data/lib/mobilize-base/jobtracker.rb +5 -13
- data/lib/mobilize-base/models/job.rb +48 -36
- data/lib/mobilize-base/models/runner.rb +123 -24
- data/lib/mobilize-base/models/stage.rb +43 -14
- data/lib/mobilize-base/tasks.rb +3 -16
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +1 -5
- data/lib/samples/gridfs.yml +3 -0
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +5 -4
- data/test/mobilize-base_test.rb +0 -1
- metadata +31 -20
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +0 -59
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +0 -85
- data/lib/mobilize-base/helpers/job_helper.rb +0 -54
- data/lib/mobilize-base/helpers/runner_helper.rb +0 -83
- data/lib/mobilize-base/helpers/stage_helper.rb +0 -38
@@ -1,85 +0,0 @@
|
|
1
|
-
<% if params[:id] && (worker = Resque::Worker.find(params[:id])) && worker.job %>
|
2
|
-
<h1><%= worker %>'s job</h1>
|
3
|
-
|
4
|
-
<table>
|
5
|
-
<tr>
|
6
|
-
<th> </th>
|
7
|
-
<th>Where</th>
|
8
|
-
<th>Queue</th>
|
9
|
-
<th>Started</th>
|
10
|
-
<th>Class</th>
|
11
|
-
<th>Args</th>
|
12
|
-
</tr>
|
13
|
-
<tr>
|
14
|
-
<td><img src="<%=u 'working.png' %>" alt="working" title="working"></td>
|
15
|
-
<% host, pid, _ = worker.to_s.split(':') %>
|
16
|
-
<td><a href="<%=u "/workers/#{worker}" %>"><%= host %>:<%= pid %></a></td>
|
17
|
-
<% data = worker.job %>
|
18
|
-
<% queue = data['queue'] %>
|
19
|
-
<td><a class="queue" href="<%=u "/queues/#{queue}" %>"><%= queue %></a></td>
|
20
|
-
<td><span class="time"><%= data['run_at'] %></span></td>
|
21
|
-
<td>
|
22
|
-
<code><%= data['payload']['class'] %></code>
|
23
|
-
</td>
|
24
|
-
<td><%=h data['payload']['args'].inspect %></td>
|
25
|
-
</tr>
|
26
|
-
</table>
|
27
|
-
|
28
|
-
<% else %>
|
29
|
-
|
30
|
-
<%
|
31
|
-
workers = resque.working
|
32
|
-
jobs = workers.collect {|w| w.job }
|
33
|
-
worker_jobs = workers.zip(jobs)
|
34
|
-
worker_jobs = worker_jobs.reject { |w, j| w.idle? }
|
35
|
-
%>
|
36
|
-
|
37
|
-
<h1 class='wi'><%= worker_jobs.size %> of <%= resque.workers.size %> Workers Working</h1>
|
38
|
-
<p class='intro'>The list below contains all workers which are currently running a job.</p>
|
39
|
-
<table class='workers'>
|
40
|
-
<tr>
|
41
|
-
<th> </th>
|
42
|
-
<th>Where</th>
|
43
|
-
<th>Queue</th>
|
44
|
-
<th>Processing</th>
|
45
|
-
</tr>
|
46
|
-
<% if worker_jobs.empty? %>
|
47
|
-
<tr>
|
48
|
-
<td colspan="4" class='no-data'>Nothing is happening right now...</td>
|
49
|
-
</tr>
|
50
|
-
<% end %>
|
51
|
-
|
52
|
-
<% worker_jobs.sort_by {|w, j| j['run_at'] ? j['run_at'] : '' }.each do |worker, job| %>
|
53
|
-
<tr>
|
54
|
-
<td class='icon'><img src="<%=u state = worker.state %>.png" alt="<%= state %>" title="<%= state %>"></td>
|
55
|
-
<% host, pid, queues = worker.to_s.split(':') %>
|
56
|
-
<td class='where'><a href="<%=u "/workers/#{worker}" %>"><%= host %>:<%= pid %></a></td>
|
57
|
-
<td class='queues queue'>
|
58
|
-
<a class="queue-tag" href="<%=u "/queues/#{job['queue']}" %>"><%= job['queue'] %></a>
|
59
|
-
</td>
|
60
|
-
<td class='process'>
|
61
|
-
<% if job['queue']
|
62
|
-
job_stats = begin
|
63
|
-
j = job
|
64
|
-
args_hash = j['payload']['args'][1]
|
65
|
-
args_array = args_hash.map{|k,v| "#{k} : #{v}" }.join("</code><br><code>") if args_hash.class==Hash
|
66
|
-
args = [args_array].compact.join("")
|
67
|
-
path = j['payload']['args'].first
|
68
|
-
[path,args].join("</code><br><code>")
|
69
|
-
rescue => exc
|
70
|
-
[exc.to_s,exc.backtrace.join("<br>")].join("<br>")
|
71
|
-
end
|
72
|
-
%>
|
73
|
-
<%=job_stats%>
|
74
|
-
</code>
|
75
|
-
<br>
|
76
|
-
<small><a class="queue time" href="<%=u "/working/#{worker}" %>"><%= job['run_at'] %></a></small>
|
77
|
-
<% else %>
|
78
|
-
<span class='waiting'>Waiting for a job...</span>
|
79
|
-
<% end %>
|
80
|
-
</td>
|
81
|
-
</tr>
|
82
|
-
<% end %>
|
83
|
-
</table>
|
84
|
-
|
85
|
-
<% end %>
|
@@ -1,54 +0,0 @@
|
|
1
|
-
#this module adds convenience methods to the Job model
|
2
|
-
module Mobilize
|
3
|
-
module JobHelper
|
4
|
-
def name
|
5
|
-
j = self
|
6
|
-
j.path.split("/").last
|
7
|
-
end
|
8
|
-
|
9
|
-
def stages
|
10
|
-
j = self
|
11
|
-
#starts with the job path, followed by a slash
|
12
|
-
Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
|
13
|
-
end
|
14
|
-
|
15
|
-
def status
|
16
|
-
#last stage status
|
17
|
-
j = self
|
18
|
-
j.active_stage.status if j.active_stage
|
19
|
-
end
|
20
|
-
|
21
|
-
def active_stage
|
22
|
-
j = self
|
23
|
-
#latest started at or first
|
24
|
-
j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
|
25
|
-
end
|
26
|
-
|
27
|
-
def completed_at
|
28
|
-
j = self
|
29
|
-
j.stages.last.completed_at if j.stages.last
|
30
|
-
end
|
31
|
-
|
32
|
-
def failed_at
|
33
|
-
j = self
|
34
|
-
j.active_stage.failed_at if j.active_stage
|
35
|
-
end
|
36
|
-
|
37
|
-
def status_at
|
38
|
-
j = self
|
39
|
-
j.active_stage.status_at if j.active_stage
|
40
|
-
end
|
41
|
-
|
42
|
-
#convenience methods
|
43
|
-
def runner
|
44
|
-
j = self
|
45
|
-
runner_path = j.path.split("/")[0..-2].join("/")
|
46
|
-
return Runner.where(:path=>runner_path).first
|
47
|
-
end
|
48
|
-
|
49
|
-
def is_working?
|
50
|
-
j = self
|
51
|
-
j.stages.select{|s| s.is_working?}.compact.length>0
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
@@ -1,83 +0,0 @@
|
|
1
|
-
#this module adds convenience methods to the Runner model
|
2
|
-
module Mobilize
|
3
|
-
module RunnerHelper
|
4
|
-
def headers
|
5
|
-
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
6
|
-
end
|
7
|
-
|
8
|
-
def title
|
9
|
-
r = self
|
10
|
-
r.path.split("/").first
|
11
|
-
end
|
12
|
-
|
13
|
-
def worker
|
14
|
-
r = self
|
15
|
-
Mobilize::Resque.find_worker_by_path(r.path)
|
16
|
-
end
|
17
|
-
|
18
|
-
def dataset
|
19
|
-
r = self
|
20
|
-
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
21
|
-
end
|
22
|
-
|
23
|
-
def gbook(gdrive_slot)
|
24
|
-
r = self
|
25
|
-
title = r.path.split("/").first
|
26
|
-
Gbook.find_by_path(title,gdrive_slot)
|
27
|
-
end
|
28
|
-
|
29
|
-
def gsheet(gdrive_slot)
|
30
|
-
r = self
|
31
|
-
u = r.user
|
32
|
-
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
33
|
-
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
34
|
-
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
35
|
-
#only give the user edit permissions if they're the ones
|
36
|
-
#creating it
|
37
|
-
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
38
|
-
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
39
|
-
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
40
|
-
end
|
41
|
-
jobs_sheet.add_headers(r.headers)
|
42
|
-
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
43
|
-
end
|
44
|
-
return jobs_sheet
|
45
|
-
end
|
46
|
-
|
47
|
-
def jobs(jname=nil)
|
48
|
-
r = self
|
49
|
-
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
50
|
-
if jname
|
51
|
-
return js.sel{|j| j.name == jname}.first
|
52
|
-
else
|
53
|
-
return js
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
def user
|
58
|
-
r = self
|
59
|
-
user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
|
60
|
-
User.where(:name=>user_name).first
|
61
|
-
end
|
62
|
-
|
63
|
-
def update_status(msg)
|
64
|
-
r = self
|
65
|
-
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
66
|
-
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
67
|
-
return true
|
68
|
-
end
|
69
|
-
|
70
|
-
def is_working?
|
71
|
-
r = self
|
72
|
-
Mobilize::Resque.active_paths.include?(r.path)
|
73
|
-
end
|
74
|
-
|
75
|
-
def is_due?
|
76
|
-
r = self.reload
|
77
|
-
return false if r.is_working?
|
78
|
-
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
79
|
-
return true if r.started_at.nil? or r.started_at < prev_due_time
|
80
|
-
end
|
81
|
-
|
82
|
-
end
|
83
|
-
end
|
@@ -1,38 +0,0 @@
|
|
1
|
-
#this module adds convenience methods to the Stage model
|
2
|
-
module Mobilize
|
3
|
-
module StageHelper
|
4
|
-
def idx
|
5
|
-
s = self
|
6
|
-
s.path.split("/").last.gsub("stage","").to_i
|
7
|
-
end
|
8
|
-
|
9
|
-
def out_dst
|
10
|
-
#this gives a dataset that points to the output
|
11
|
-
#allowing you to determine its size
|
12
|
-
#before committing to a read or write
|
13
|
-
s = self
|
14
|
-
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
15
|
-
end
|
16
|
-
|
17
|
-
def err_dst
|
18
|
-
#this gives a dataset that points to the output
|
19
|
-
#allowing you to determine its size
|
20
|
-
#before committing to a read or write
|
21
|
-
s = self
|
22
|
-
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
23
|
-
end
|
24
|
-
|
25
|
-
def params
|
26
|
-
s = self
|
27
|
-
p = YAML.easy_load(s.param_string)
|
28
|
-
raise "Must resolve to Hash" unless p.class==Hash
|
29
|
-
return p
|
30
|
-
end
|
31
|
-
|
32
|
-
def job
|
33
|
-
s = self
|
34
|
-
job_path = s.path.split("/")[0..-2].join("/")
|
35
|
-
Job.where(:path=>job_path).first
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|