mobilize-base 1.2 → 1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +10 -11
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +7 -2
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +59 -0
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +85 -0
- data/lib/mobilize-base/extensions/string.rb +12 -4
- data/lib/mobilize-base/extensions/yaml.rb +11 -7
- data/lib/mobilize-base/handlers/gbook.rb +24 -31
- data/lib/mobilize-base/handlers/gfile.rb +5 -3
- data/lib/mobilize-base/handlers/gridfs.rb +19 -24
- data/lib/mobilize-base/handlers/gsheet.rb +25 -20
- data/lib/mobilize-base/handlers/resque.rb +16 -4
- data/lib/mobilize-base/helpers/job_helper.rb +54 -0
- data/lib/mobilize-base/helpers/runner_helper.rb +83 -0
- data/lib/mobilize-base/helpers/stage_helper.rb +38 -0
- data/lib/mobilize-base/jobtracker.rb +13 -5
- data/lib/mobilize-base/models/job.rb +36 -48
- data/lib/mobilize-base/models/runner.rb +24 -123
- data/lib/mobilize-base/models/stage.rb +14 -43
- data/lib/mobilize-base/tasks.rb +16 -3
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +5 -1
- data/lib/samples/gridfs.yml +0 -3
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +4 -5
- data/test/mobilize-base_test.rb +1 -0
- metadata +21 -32
@@ -0,0 +1,54 @@
|
|
1
|
+
#this module adds convenience methods to the Job model
|
2
|
+
module Mobilize
|
3
|
+
module JobHelper
|
4
|
+
def name
|
5
|
+
j = self
|
6
|
+
j.path.split("/").last
|
7
|
+
end
|
8
|
+
|
9
|
+
def stages
|
10
|
+
j = self
|
11
|
+
#starts with the job path, followed by a slash
|
12
|
+
Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
|
13
|
+
end
|
14
|
+
|
15
|
+
def status
|
16
|
+
#last stage status
|
17
|
+
j = self
|
18
|
+
j.active_stage.status if j.active_stage
|
19
|
+
end
|
20
|
+
|
21
|
+
def active_stage
|
22
|
+
j = self
|
23
|
+
#latest started at or first
|
24
|
+
j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
|
25
|
+
end
|
26
|
+
|
27
|
+
def completed_at
|
28
|
+
j = self
|
29
|
+
j.stages.last.completed_at if j.stages.last
|
30
|
+
end
|
31
|
+
|
32
|
+
def failed_at
|
33
|
+
j = self
|
34
|
+
j.active_stage.failed_at if j.active_stage
|
35
|
+
end
|
36
|
+
|
37
|
+
def status_at
|
38
|
+
j = self
|
39
|
+
j.active_stage.status_at if j.active_stage
|
40
|
+
end
|
41
|
+
|
42
|
+
#convenience methods
|
43
|
+
def runner
|
44
|
+
j = self
|
45
|
+
runner_path = j.path.split("/")[0..-2].join("/")
|
46
|
+
return Runner.where(:path=>runner_path).first
|
47
|
+
end
|
48
|
+
|
49
|
+
def is_working?
|
50
|
+
j = self
|
51
|
+
j.stages.select{|s| s.is_working?}.compact.length>0
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
#this module adds convenience methods to the Runner model
|
2
|
+
module Mobilize
|
3
|
+
module RunnerHelper
|
4
|
+
def headers
|
5
|
+
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
6
|
+
end
|
7
|
+
|
8
|
+
def title
|
9
|
+
r = self
|
10
|
+
r.path.split("/").first
|
11
|
+
end
|
12
|
+
|
13
|
+
def worker
|
14
|
+
r = self
|
15
|
+
Mobilize::Resque.find_worker_by_path(r.path)
|
16
|
+
end
|
17
|
+
|
18
|
+
def dataset
|
19
|
+
r = self
|
20
|
+
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
21
|
+
end
|
22
|
+
|
23
|
+
def gbook(gdrive_slot)
|
24
|
+
r = self
|
25
|
+
title = r.path.split("/").first
|
26
|
+
Gbook.find_by_path(title,gdrive_slot)
|
27
|
+
end
|
28
|
+
|
29
|
+
def gsheet(gdrive_slot)
|
30
|
+
r = self
|
31
|
+
u = r.user
|
32
|
+
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
33
|
+
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
34
|
+
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
35
|
+
#only give the user edit permissions if they're the ones
|
36
|
+
#creating it
|
37
|
+
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
38
|
+
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
39
|
+
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
40
|
+
end
|
41
|
+
jobs_sheet.add_headers(r.headers)
|
42
|
+
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
43
|
+
end
|
44
|
+
return jobs_sheet
|
45
|
+
end
|
46
|
+
|
47
|
+
def jobs(jname=nil)
|
48
|
+
r = self
|
49
|
+
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
50
|
+
if jname
|
51
|
+
return js.sel{|j| j.name == jname}.first
|
52
|
+
else
|
53
|
+
return js
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def user
|
58
|
+
r = self
|
59
|
+
user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
|
60
|
+
User.where(:name=>user_name).first
|
61
|
+
end
|
62
|
+
|
63
|
+
def update_status(msg)
|
64
|
+
r = self
|
65
|
+
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
66
|
+
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
67
|
+
return true
|
68
|
+
end
|
69
|
+
|
70
|
+
def is_working?
|
71
|
+
r = self
|
72
|
+
Mobilize::Resque.active_paths.include?(r.path)
|
73
|
+
end
|
74
|
+
|
75
|
+
def is_due?
|
76
|
+
r = self.reload
|
77
|
+
return false if r.is_working?
|
78
|
+
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
79
|
+
return true if r.started_at.nil? or r.started_at < prev_due_time
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
#this module adds convenience methods to the Stage model
|
2
|
+
module Mobilize
|
3
|
+
module StageHelper
|
4
|
+
def idx
|
5
|
+
s = self
|
6
|
+
s.path.split("/").last.gsub("stage","").to_i
|
7
|
+
end
|
8
|
+
|
9
|
+
def out_dst
|
10
|
+
#this gives a dataset that points to the output
|
11
|
+
#allowing you to determine its size
|
12
|
+
#before committing to a read or write
|
13
|
+
s = self
|
14
|
+
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
15
|
+
end
|
16
|
+
|
17
|
+
def err_dst
|
18
|
+
#this gives a dataset that points to the output
|
19
|
+
#allowing you to determine its size
|
20
|
+
#before committing to a read or write
|
21
|
+
s = self
|
22
|
+
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
23
|
+
end
|
24
|
+
|
25
|
+
def params
|
26
|
+
s = self
|
27
|
+
p = YAML.easy_load(s.param_string)
|
28
|
+
raise "Must resolve to Hash" unless p.class==Hash
|
29
|
+
return p
|
30
|
+
end
|
31
|
+
|
32
|
+
def job
|
33
|
+
s = self
|
34
|
+
job_path = s.path.split("/")[0..-2].join("/")
|
35
|
+
Job.where(:path=>job_path).first
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -48,6 +48,8 @@ module Mobilize
|
|
48
48
|
end
|
49
49
|
|
50
50
|
def Jobtracker.update_status(msg)
|
51
|
+
#this is to keep jobtracker from resisting stop commands
|
52
|
+
return false if Jobtracker.status=="stopping"
|
51
53
|
#Jobtracker has no persistent database state
|
52
54
|
Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
|
53
55
|
return true
|
@@ -122,7 +124,7 @@ module Mobilize
|
|
122
124
|
sleep 5
|
123
125
|
i=0
|
124
126
|
while Jobtracker.status=='stopping'
|
125
|
-
|
127
|
+
puts "#{Jobtracker.to_s} still on queue, waiting"
|
126
128
|
sleep 5
|
127
129
|
i+=1
|
128
130
|
end
|
@@ -145,8 +147,8 @@ module Mobilize
|
|
145
147
|
def Jobtracker.max_run_time_workers
|
146
148
|
#return workers who have been cranking away for 6+ hours
|
147
149
|
workers = Jobtracker.workers('working').select do |w|
|
148
|
-
w.job['
|
149
|
-
(Time.now.utc - Time.parse(w.job['
|
150
|
+
w.job['run_at'].to_s.length>0 and
|
151
|
+
(Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
|
150
152
|
end
|
151
153
|
return workers
|
152
154
|
end
|
@@ -185,13 +187,18 @@ module Mobilize
|
|
185
187
|
if lws.length>0
|
186
188
|
n = {}
|
187
189
|
n['subject'] = "#{lws.length.to_s} max run time jobs"
|
188
|
-
n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']}
|
190
|
+
n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
|
189
191
|
n['to'] = Jobtracker.admin_emails.join(",")
|
190
192
|
notifs << n
|
191
193
|
end
|
192
194
|
#deliver each email generated
|
193
195
|
notifs.each do |notif|
|
194
|
-
|
196
|
+
begin
|
197
|
+
Email.write(notif).deliver
|
198
|
+
rescue
|
199
|
+
#log email on failure
|
200
|
+
Jobtracker.update_status("Failed to deliver #{notif.to_s}")
|
201
|
+
end
|
195
202
|
end
|
196
203
|
#update notification time so JT knows to wait a while
|
197
204
|
Jobtracker.last_notification = Time.now.utc.to_s
|
@@ -286,6 +293,7 @@ module Mobilize
|
|
286
293
|
# delete any old runner from previous test runs
|
287
294
|
gdrive_slot = Gdrive.owner_email
|
288
295
|
u.runner.gsheet(gdrive_slot).spreadsheet.delete
|
296
|
+
Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
|
289
297
|
Jobtracker.update_status("enqueue jobtracker, wait 45s")
|
290
298
|
Mobilize::Jobtracker.start
|
291
299
|
sleep 45
|
@@ -2,66 +2,54 @@ module Mobilize
|
|
2
2
|
class Job
|
3
3
|
include Mongoid::Document
|
4
4
|
include Mongoid::Timestamps
|
5
|
+
include Mobilize::JobHelper
|
5
6
|
field :path, type: String
|
6
7
|
field :active, type: Boolean
|
7
8
|
field :trigger, type: String
|
8
9
|
|
9
10
|
index({ path: 1})
|
10
11
|
|
11
|
-
def name
|
12
|
-
j = self
|
13
|
-
j.path.split("/").last
|
14
|
-
end
|
15
|
-
|
16
|
-
def stages
|
17
|
-
j = self
|
18
|
-
#starts with the job path, followed by a slash
|
19
|
-
Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
|
20
|
-
end
|
21
|
-
|
22
12
|
def Job.find_or_create_by_path(path)
|
23
13
|
j = Job.where(:path=>path).first
|
24
14
|
j = Job.create(:path=>path) unless j
|
25
15
|
return j
|
26
16
|
end
|
27
17
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
j
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
j = self
|
64
|
-
j.stages.select{|s| s.is_working?}.compact.length>0
|
18
|
+
#takes a hash of job parameters (name, active, trigger, stages)
|
19
|
+
#and creates/updates a job with it
|
20
|
+
def Job.update_by_user_name_and_hash(user_name,hash)
|
21
|
+
u = User.where(name: user_name).first
|
22
|
+
r = u.runner
|
23
|
+
j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
|
24
|
+
#update top line params
|
25
|
+
j.update_attributes(:active => hash['active'],
|
26
|
+
:trigger => hash['trigger'])
|
27
|
+
(1..5).to_a.each do |s_idx|
|
28
|
+
stage_string = hash["stage#{s_idx.to_s}"]
|
29
|
+
s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
|
30
|
+
if stage_string.to_s.length==0
|
31
|
+
#delete this stage and all stages after
|
32
|
+
if s
|
33
|
+
j = s.job
|
34
|
+
j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
|
35
|
+
#just in case
|
36
|
+
s.delete
|
37
|
+
end
|
38
|
+
break
|
39
|
+
elsif s.nil?
|
40
|
+
#create this stage
|
41
|
+
s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
|
42
|
+
end
|
43
|
+
#parse command string, update stage with it
|
44
|
+
s_handler, call, param_string = [""*3]
|
45
|
+
stage_string.split(" ").ie do |spls|
|
46
|
+
s_handler = spls.first.split(".").first
|
47
|
+
call = spls.first.split(".").last
|
48
|
+
param_string = spls[1..-1].join(" ").strip
|
49
|
+
end
|
50
|
+
s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
|
51
|
+
end
|
52
|
+
return j.reload
|
65
53
|
end
|
66
54
|
|
67
55
|
def is_due?
|
@@ -1,5 +1,6 @@
|
|
1
1
|
module Mobilize
|
2
2
|
class Runner
|
3
|
+
include Mobilize::RunnerHelper
|
3
4
|
include Mongoid::Document
|
4
5
|
include Mongoid::Timestamps
|
5
6
|
field :path, type: String
|
@@ -11,20 +12,6 @@ module Mobilize
|
|
11
12
|
|
12
13
|
index({ path: 1})
|
13
14
|
|
14
|
-
def headers
|
15
|
-
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
16
|
-
end
|
17
|
-
|
18
|
-
def title
|
19
|
-
r = self
|
20
|
-
r.path.split("/").first
|
21
|
-
end
|
22
|
-
|
23
|
-
def worker
|
24
|
-
r = self
|
25
|
-
Mobilize::Resque.find_worker_by_path(r.path)
|
26
|
-
end
|
27
|
-
|
28
15
|
def Runner.find_by_path(path)
|
29
16
|
Runner.where(:path=>path).first
|
30
17
|
end
|
@@ -32,6 +19,7 @@ module Mobilize
|
|
32
19
|
def Runner.find_by_title(title)
|
33
20
|
Runner.where(:path=>"#{title}/jobs").first
|
34
21
|
end
|
22
|
+
|
35
23
|
def Runner.perform(id,*args)
|
36
24
|
r = Runner.find_by_path(id)
|
37
25
|
#get gdrive slot for read
|
@@ -41,12 +29,18 @@ module Mobilize
|
|
41
29
|
return false
|
42
30
|
end
|
43
31
|
r.update_attributes(:started_at=>Time.now.utc)
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
32
|
+
begin
|
33
|
+
#make sure any updates to activity are processed first
|
34
|
+
#as in when someone runs a "once" job that has completed
|
35
|
+
r.update_gsheet(gdrive_slot)
|
36
|
+
#read the jobs in the gsheet and update models with news
|
37
|
+
r.read_gsheet(gdrive_slot)
|
38
|
+
#queue up the jobs that are due and active
|
39
|
+
rescue => exc
|
40
|
+
#log the exception, but continue w job processing
|
41
|
+
#This ensures jobs are still processed if google drive goes down
|
42
|
+
r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
|
43
|
+
end
|
50
44
|
r.jobs.each do |j|
|
51
45
|
begin
|
52
46
|
if j.is_due?
|
@@ -64,90 +58,32 @@ module Mobilize
|
|
64
58
|
r.update_attributes(:completed_at=>Time.now.utc)
|
65
59
|
end
|
66
60
|
|
67
|
-
def dataset
|
68
|
-
r = self
|
69
|
-
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
70
|
-
end
|
71
|
-
|
72
61
|
def Runner.find_or_create_by_path(path)
|
73
62
|
Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
|
74
63
|
end
|
75
64
|
|
76
|
-
def gbook(gdrive_slot)
|
77
|
-
r = self
|
78
|
-
title = r.path.split("/").first
|
79
|
-
Gbook.find_all_by_path(title,gdrive_slot).first
|
80
|
-
end
|
81
|
-
|
82
|
-
def gsheet(gdrive_slot)
|
83
|
-
r = self
|
84
|
-
u = r.user
|
85
|
-
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
86
|
-
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
87
|
-
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
88
|
-
#only give the user edit permissions if they're the ones
|
89
|
-
#creating it
|
90
|
-
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
91
|
-
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
92
|
-
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
93
|
-
end
|
94
|
-
end
|
95
|
-
jobs_sheet.add_headers(r.headers)
|
96
|
-
#add url to dataset
|
97
|
-
Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
|
98
|
-
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
99
|
-
return jobs_sheet
|
100
|
-
end
|
101
|
-
|
102
65
|
def read_gsheet(gdrive_slot)
|
103
66
|
r = self
|
104
|
-
|
67
|
+
#argument converts line breaks in cells to spaces
|
68
|
+
gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
|
105
69
|
#turn it into a hash array
|
106
|
-
|
70
|
+
gsheet_hashes = gsheet_tsv.tsv_to_hash_array
|
107
71
|
#go through each job, update relevant job with its params
|
108
72
|
done_jobs = []
|
109
73
|
#parse out the jobs and update the Job collection
|
110
|
-
|
74
|
+
gsheet_hashes.each do |gsheet_hash|
|
111
75
|
#skip non-jobs or jobs without required values
|
112
|
-
next if (
|
113
|
-
j = Job.
|
114
|
-
#update top line params
|
115
|
-
j.update_attributes(:active => rj['active'],
|
116
|
-
:trigger => rj['trigger'])
|
117
|
-
(1..5).to_a.each do |s_idx|
|
118
|
-
stage_string = rj["stage#{s_idx.to_s}"]
|
119
|
-
s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
|
120
|
-
if stage_string.to_s.length==0
|
121
|
-
#delete this stage and all stages after
|
122
|
-
if s
|
123
|
-
j = s.job
|
124
|
-
j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
|
125
|
-
#just in case
|
126
|
-
s.delete
|
127
|
-
end
|
128
|
-
break
|
129
|
-
elsif s.nil?
|
130
|
-
#create this stage
|
131
|
-
s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
|
132
|
-
end
|
133
|
-
#parse command string, update stage with it
|
134
|
-
s_handler, call, param_string = [""*3]
|
135
|
-
stage_string.split(" ").ie do |spls|
|
136
|
-
s_handler = spls.first.split(".").first
|
137
|
-
call = spls.first.split(".").last
|
138
|
-
param_string = spls[1..-1].join(" ").strip
|
139
|
-
end
|
140
|
-
s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
|
141
|
-
end
|
76
|
+
next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
|
77
|
+
j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
|
142
78
|
r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
|
143
79
|
#add this job to list of read ones
|
144
80
|
done_jobs << j
|
145
81
|
end
|
146
82
|
#delete user jobs that are not included in Runner
|
147
|
-
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |
|
148
|
-
j = Job.where(:path=>
|
83
|
+
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
|
84
|
+
j = Job.where(:path=>gsheet_hash_path).first
|
149
85
|
j.delete if j
|
150
|
-
r.update_status("Deleted job:#{
|
86
|
+
r.update_status("Deleted job:#{gsheet_hash_path}")
|
151
87
|
end
|
152
88
|
r.update_status("jobs read at #{Time.now.utc}")
|
153
89
|
return true
|
@@ -158,48 +94,13 @@ module Mobilize
|
|
158
94
|
#there's nothing to update if runner has never had a completed at
|
159
95
|
return false unless r.completed_at
|
160
96
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
161
|
-
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
|
97
|
+
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
|
162
98
|
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
163
99
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
164
100
|
r.update_status("gsheet updated")
|
165
101
|
return true
|
166
102
|
end
|
167
103
|
|
168
|
-
def jobs(jname=nil)
|
169
|
-
r = self
|
170
|
-
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
171
|
-
if jname
|
172
|
-
return js.sel{|j| j.name == jname}.first
|
173
|
-
else
|
174
|
-
return js
|
175
|
-
end
|
176
|
-
end
|
177
|
-
|
178
|
-
def user
|
179
|
-
r = self
|
180
|
-
user_name = r.path.split("_").second.split("(").first.split("/").first
|
181
|
-
User.where(:name=>user_name).first
|
182
|
-
end
|
183
|
-
|
184
|
-
def update_status(msg)
|
185
|
-
r = self
|
186
|
-
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
187
|
-
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
188
|
-
return true
|
189
|
-
end
|
190
|
-
|
191
|
-
def is_working?
|
192
|
-
r = self
|
193
|
-
Mobilize::Resque.active_paths.include?(r.path)
|
194
|
-
end
|
195
|
-
|
196
|
-
def is_due?
|
197
|
-
r = self.reload
|
198
|
-
return false if r.is_working?
|
199
|
-
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
200
|
-
return true if r.started_at.nil? or r.started_at < prev_due_time
|
201
|
-
end
|
202
|
-
|
203
104
|
def enqueue!
|
204
105
|
r = self
|
205
106
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
@@ -2,6 +2,7 @@ module Mobilize
|
|
2
2
|
class Stage
|
3
3
|
include Mongoid::Document
|
4
4
|
include Mongoid::Timestamps
|
5
|
+
include Mobilize::StageHelper
|
5
6
|
field :path, type: String
|
6
7
|
field :handler, type: String
|
7
8
|
field :call, type: String
|
@@ -16,40 +17,6 @@ module Mobilize
|
|
16
17
|
|
17
18
|
index({ path: 1})
|
18
19
|
|
19
|
-
def idx
|
20
|
-
s = self
|
21
|
-
s.path.split("/").last.gsub("stage","").to_i
|
22
|
-
end
|
23
|
-
|
24
|
-
def out_dst
|
25
|
-
#this gives a dataset that points to the output
|
26
|
-
#allowing you to determine its size
|
27
|
-
#before committing to a read or write
|
28
|
-
s = self
|
29
|
-
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
30
|
-
end
|
31
|
-
|
32
|
-
def err_dst
|
33
|
-
#this gives a dataset that points to the output
|
34
|
-
#allowing you to determine its size
|
35
|
-
#before committing to a read or write
|
36
|
-
s = self
|
37
|
-
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
38
|
-
end
|
39
|
-
|
40
|
-
def params
|
41
|
-
s = self
|
42
|
-
p = YAML.easy_load(s.param_string)
|
43
|
-
raise "Must resolve to Hash" unless p.class==Hash
|
44
|
-
return p
|
45
|
-
end
|
46
|
-
|
47
|
-
def job
|
48
|
-
s = self
|
49
|
-
job_path = s.path.split("/")[0..-2].join("/")
|
50
|
-
Job.where(:path=>job_path).first
|
51
|
-
end
|
52
|
-
|
53
20
|
def Stage.find_or_create_by_path(path)
|
54
21
|
s = Stage.where(:path=>path).first
|
55
22
|
s = Stage.create(:path=>path) unless s
|
@@ -80,7 +47,11 @@ module Mobilize
|
|
80
47
|
s.update_attributes(:started_at=>Time.now.utc)
|
81
48
|
s.update_status(%{Starting at #{Time.now.utc}})
|
82
49
|
#get response by running method
|
83
|
-
response =
|
50
|
+
response = begin
|
51
|
+
"Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
|
52
|
+
rescue => exc
|
53
|
+
{'err_str'=>"#{exc.to_s}\n#{exc.backtrace.to_a.join("\n")}", 'signal'=>500}
|
54
|
+
end
|
84
55
|
unless response
|
85
56
|
#re-queue self if no response
|
86
57
|
s.enqueue!
|
@@ -92,10 +63,10 @@ module Mobilize
|
|
92
63
|
#retry
|
93
64
|
s.update_attributes(:retries_done => s.retries_done.to_i + 1, :response => response)
|
94
65
|
s.update_status(%{Retry #{s.retries_done.to_s} at #{Time.now.utc}})
|
66
|
+
sleep s['delay'].to_i
|
95
67
|
s.enqueue!
|
96
68
|
else
|
97
69
|
#sleep as much as user specifies
|
98
|
-
sleep s['delay'].to_i
|
99
70
|
s.fail(response)
|
100
71
|
end
|
101
72
|
return true
|
@@ -111,7 +82,7 @@ module Mobilize
|
|
111
82
|
r = j.runner
|
112
83
|
dep_jobs = r.jobs.select do |dj|
|
113
84
|
dj.active==true and
|
114
|
-
dj.trigger.strip.downcase == "after #{j.name}"
|
85
|
+
dj.trigger.strip.downcase == "after #{j.name.downcase}"
|
115
86
|
end
|
116
87
|
#put begin/rescue so all dependencies run
|
117
88
|
dep_jobs.each do |dj|
|
@@ -141,7 +112,7 @@ module Mobilize
|
|
141
112
|
j = s.job
|
142
113
|
r = j.runner
|
143
114
|
u = r.user
|
144
|
-
j.update_attributes(:active=>false)
|
115
|
+
j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
|
145
116
|
s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
|
146
117
|
stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
|
147
118
|
target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
|
@@ -158,10 +129,9 @@ module Mobilize
|
|
158
129
|
err_txt = ["response","\n",err_txt].join
|
159
130
|
err_sheet.write(err_txt,u.name)
|
160
131
|
#exception will be first row below "response" header
|
161
|
-
exc_to_s,backtrace = err_txt.split("\n").ie{|ea| [ea[1], ea[2..-1]]}
|
162
132
|
s.update_status(status_msg)
|
163
133
|
#raise the exception so it bubbles up to resque
|
164
|
-
raise Exception,
|
134
|
+
raise Exception,err_txt
|
165
135
|
end
|
166
136
|
|
167
137
|
def enqueue!
|
@@ -208,14 +178,15 @@ module Mobilize
|
|
208
178
|
raise "incompatible target handler #{handler} for #{s.handler} stage"
|
209
179
|
else
|
210
180
|
begin
|
211
|
-
|
181
|
+
#nil gdrive_slot for targets since there is no verification
|
182
|
+
return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path,nil)
|
212
183
|
rescue => exc
|
213
184
|
raise "Could not get #{target_path} with error: #{exc.to_s}"
|
214
185
|
end
|
215
186
|
end
|
216
187
|
end
|
217
188
|
|
218
|
-
def sources
|
189
|
+
def sources(gdrive_slot)
|
219
190
|
#returns an array of Datasets corresponding to
|
220
191
|
#items listed as sources in the stage params
|
221
192
|
s = self
|
@@ -244,7 +215,7 @@ module Mobilize
|
|
244
215
|
end
|
245
216
|
begin
|
246
217
|
stage_path = s.path
|
247
|
-
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path)
|
218
|
+
dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path,gdrive_slot)
|
248
219
|
rescue => exc
|
249
220
|
raise "Could not get #{source_path} with error: #{exc.to_s}"
|
250
221
|
end
|