mobilize-base 1.2 → 1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,54 @@
1
+ #this module adds convenience methods to the Job model
2
+ module Mobilize
3
+ module JobHelper
4
+ def name
5
+ j = self
6
+ j.path.split("/").last
7
+ end
8
+
9
+ def stages
10
+ j = self
11
+ #starts with the job path, followed by a slash
12
+ Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
13
+ end
14
+
15
+ def status
16
+ #last stage status
17
+ j = self
18
+ j.active_stage.status if j.active_stage
19
+ end
20
+
21
+ def active_stage
22
+ j = self
23
+ #latest started at or first
24
+ j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
25
+ end
26
+
27
+ def completed_at
28
+ j = self
29
+ j.stages.last.completed_at if j.stages.last
30
+ end
31
+
32
+ def failed_at
33
+ j = self
34
+ j.active_stage.failed_at if j.active_stage
35
+ end
36
+
37
+ def status_at
38
+ j = self
39
+ j.active_stage.status_at if j.active_stage
40
+ end
41
+
42
+ #convenience methods
43
+ def runner
44
+ j = self
45
+ runner_path = j.path.split("/")[0..-2].join("/")
46
+ return Runner.where(:path=>runner_path).first
47
+ end
48
+
49
+ def is_working?
50
+ j = self
51
+ j.stages.select{|s| s.is_working?}.compact.length>0
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,83 @@
1
+ #this module adds convenience methods to the Runner model
2
+ module Mobilize
3
+ module RunnerHelper
4
+ def headers
5
+ %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
6
+ end
7
+
8
+ def title
9
+ r = self
10
+ r.path.split("/").first
11
+ end
12
+
13
+ def worker
14
+ r = self
15
+ Mobilize::Resque.find_worker_by_path(r.path)
16
+ end
17
+
18
+ def dataset
19
+ r = self
20
+ Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
21
+ end
22
+
23
+ def gbook(gdrive_slot)
24
+ r = self
25
+ title = r.path.split("/").first
26
+ Gbook.find_by_path(title,gdrive_slot)
27
+ end
28
+
29
+ def gsheet(gdrive_slot)
30
+ r = self
31
+ u = r.user
32
+ jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
33
+ #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
34
+ unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
35
+ #only give the user edit permissions if they're the ones
36
+ #creating it
37
+ jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
38
+ unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
39
+ jobs_sheet.spreadsheet.update_acl(u.email,"writer")
40
+ end
41
+ jobs_sheet.add_headers(r.headers)
42
+ begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
43
+ end
44
+ return jobs_sheet
45
+ end
46
+
47
+ def jobs(jname=nil)
48
+ r = self
49
+ js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
50
+ if jname
51
+ return js.sel{|j| j.name == jname}.first
52
+ else
53
+ return js
54
+ end
55
+ end
56
+
57
+ def user
58
+ r = self
59
+ user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
60
+ User.where(:name=>user_name).first
61
+ end
62
+
63
+ def update_status(msg)
64
+ r = self
65
+ r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
66
+ Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
67
+ return true
68
+ end
69
+
70
+ def is_working?
71
+ r = self
72
+ Mobilize::Resque.active_paths.include?(r.path)
73
+ end
74
+
75
+ def is_due?
76
+ r = self.reload
77
+ return false if r.is_working?
78
+ prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
79
+ return true if r.started_at.nil? or r.started_at < prev_due_time
80
+ end
81
+
82
+ end
83
+ end
@@ -0,0 +1,38 @@
1
+ #this module adds convenience methods to the Stage model
2
+ module Mobilize
3
+ module StageHelper
4
+ def idx
5
+ s = self
6
+ s.path.split("/").last.gsub("stage","").to_i
7
+ end
8
+
9
+ def out_dst
10
+ #this gives a dataset that points to the output
11
+ #allowing you to determine its size
12
+ #before committing to a read or write
13
+ s = self
14
+ Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
15
+ end
16
+
17
+ def err_dst
18
+ #this gives a dataset that points to the output
19
+ #allowing you to determine its size
20
+ #before committing to a read or write
21
+ s = self
22
+ Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
23
+ end
24
+
25
+ def params
26
+ s = self
27
+ p = YAML.easy_load(s.param_string)
28
+ raise "Must resolve to Hash" unless p.class==Hash
29
+ return p
30
+ end
31
+
32
+ def job
33
+ s = self
34
+ job_path = s.path.split("/")[0..-2].join("/")
35
+ Job.where(:path=>job_path).first
36
+ end
37
+ end
38
+ end
@@ -48,6 +48,8 @@ module Mobilize
48
48
  end
49
49
 
50
50
  def Jobtracker.update_status(msg)
51
+ #this is to keep jobtracker from resisting stop commands
52
+ return false if Jobtracker.status=="stopping"
51
53
  #Jobtracker has no persistent database state
52
54
  Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
53
55
  return true
@@ -122,7 +124,7 @@ module Mobilize
122
124
  sleep 5
123
125
  i=0
124
126
  while Jobtracker.status=='stopping'
125
- Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
127
+ puts "#{Jobtracker.to_s} still on queue, waiting"
126
128
  sleep 5
127
129
  i+=1
128
130
  end
@@ -145,8 +147,8 @@ module Mobilize
145
147
  def Jobtracker.max_run_time_workers
146
148
  #return workers who have been cranking away for 6+ hours
147
149
  workers = Jobtracker.workers('working').select do |w|
148
- w.job['runat'].to_s.length>0 and
149
- (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
150
+ w.job['run_at'].to_s.length>0 and
151
+ (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
150
152
  end
151
153
  return workers
152
154
  end
@@ -185,13 +187,18 @@ module Mobilize
185
187
  if lws.length>0
186
188
  n = {}
187
189
  n['subject'] = "#{lws.length.to_s} max run time jobs"
188
- n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
190
+ n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
189
191
  n['to'] = Jobtracker.admin_emails.join(",")
190
192
  notifs << n
191
193
  end
192
194
  #deliver each email generated
193
195
  notifs.each do |notif|
194
- Email.write(notif).deliver
196
+ begin
197
+ Email.write(notif).deliver
198
+ rescue
199
+ #log email on failure
200
+ Jobtracker.update_status("Failed to deliver #{notif.to_s}")
201
+ end
195
202
  end
196
203
  #update notification time so JT knows to wait a while
197
204
  Jobtracker.last_notification = Time.now.utc.to_s
@@ -286,6 +293,7 @@ module Mobilize
286
293
  # delete any old runner from previous test runs
287
294
  gdrive_slot = Gdrive.owner_email
288
295
  u.runner.gsheet(gdrive_slot).spreadsheet.delete
296
+ Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
289
297
  Jobtracker.update_status("enqueue jobtracker, wait 45s")
290
298
  Mobilize::Jobtracker.start
291
299
  sleep 45
@@ -2,66 +2,54 @@ module Mobilize
2
2
  class Job
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::JobHelper
5
6
  field :path, type: String
6
7
  field :active, type: Boolean
7
8
  field :trigger, type: String
8
9
 
9
10
  index({ path: 1})
10
11
 
11
- def name
12
- j = self
13
- j.path.split("/").last
14
- end
15
-
16
- def stages
17
- j = self
18
- #starts with the job path, followed by a slash
19
- Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
20
- end
21
-
22
12
  def Job.find_or_create_by_path(path)
23
13
  j = Job.where(:path=>path).first
24
14
  j = Job.create(:path=>path) unless j
25
15
  return j
26
16
  end
27
17
 
28
- def status
29
- #last stage status
30
- j = self
31
- j.active_stage.status if j.active_stage
32
- end
33
-
34
- def active_stage
35
- j = self
36
- #latest started at or first
37
- j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
38
- end
39
-
40
- def completed_at
41
- j = self
42
- j.stages.last.completed_at if j.stages.last
43
- end
44
-
45
- def failed_at
46
- j = self
47
- j.active_stage.failed_at if j.active_stage
48
- end
49
-
50
- def status_at
51
- j = self
52
- j.active_stage.status_at if j.active_stage
53
- end
54
-
55
- #convenience methods
56
- def runner
57
- j = self
58
- runner_path = j.path.split("/")[0..-2].join("/")
59
- return Runner.where(:path=>runner_path).first
60
- end
61
-
62
- def is_working?
63
- j = self
64
- j.stages.select{|s| s.is_working?}.compact.length>0
18
+ #takes a hash of job parameters (name, active, trigger, stages)
19
+ #and creates/updates a job with it
20
+ def Job.update_by_user_name_and_hash(user_name,hash)
21
+ u = User.where(name: user_name).first
22
+ r = u.runner
23
+ j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
24
+ #update top line params
25
+ j.update_attributes(:active => hash['active'],
26
+ :trigger => hash['trigger'])
27
+ (1..5).to_a.each do |s_idx|
28
+ stage_string = hash["stage#{s_idx.to_s}"]
29
+ s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
30
+ if stage_string.to_s.length==0
31
+ #delete this stage and all stages after
32
+ if s
33
+ j = s.job
34
+ j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
35
+ #just in case
36
+ s.delete
37
+ end
38
+ break
39
+ elsif s.nil?
40
+ #create this stage
41
+ s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
42
+ end
43
+ #parse command string, update stage with it
44
+ s_handler, call, param_string = [""*3]
45
+ stage_string.split(" ").ie do |spls|
46
+ s_handler = spls.first.split(".").first
47
+ call = spls.first.split(".").last
48
+ param_string = spls[1..-1].join(" ").strip
49
+ end
50
+ s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
51
+ end
52
+ return j.reload
65
53
  end
66
54
 
67
55
  def is_due?
@@ -1,5 +1,6 @@
1
1
  module Mobilize
2
2
  class Runner
3
+ include Mobilize::RunnerHelper
3
4
  include Mongoid::Document
4
5
  include Mongoid::Timestamps
5
6
  field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
11
12
 
12
13
  index({ path: 1})
13
14
 
14
- def headers
15
- %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
16
- end
17
-
18
- def title
19
- r = self
20
- r.path.split("/").first
21
- end
22
-
23
- def worker
24
- r = self
25
- Mobilize::Resque.find_worker_by_path(r.path)
26
- end
27
-
28
15
  def Runner.find_by_path(path)
29
16
  Runner.where(:path=>path).first
30
17
  end
@@ -32,6 +19,7 @@ module Mobilize
32
19
  def Runner.find_by_title(title)
33
20
  Runner.where(:path=>"#{title}/jobs").first
34
21
  end
22
+
35
23
  def Runner.perform(id,*args)
36
24
  r = Runner.find_by_path(id)
37
25
  #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
41
29
  return false
42
30
  end
43
31
  r.update_attributes(:started_at=>Time.now.utc)
44
- #make sure any updates to activity are processed first
45
- #as in when someone runs a "once" job that has completed
46
- r.update_gsheet(gdrive_slot)
47
- #read the jobs in the gsheet and update models with news
48
- r.read_gsheet(gdrive_slot)
49
- #queue up the jobs that are due and active
32
+ begin
33
+ #make sure any updates to activity are processed first
34
+ #as in when someone runs a "once" job that has completed
35
+ r.update_gsheet(gdrive_slot)
36
+ #read the jobs in the gsheet and update models with news
37
+ r.read_gsheet(gdrive_slot)
38
+ #queue up the jobs that are due and active
39
+ rescue => exc
40
+ #log the exception, but continue w job processing
41
+ #This ensures jobs are still processed if google drive goes down
42
+ r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
43
+ end
50
44
  r.jobs.each do |j|
51
45
  begin
52
46
  if j.is_due?
@@ -64,90 +58,32 @@ module Mobilize
64
58
  r.update_attributes(:completed_at=>Time.now.utc)
65
59
  end
66
60
 
67
- def dataset
68
- r = self
69
- Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
70
- end
71
-
72
61
  def Runner.find_or_create_by_path(path)
73
62
  Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
74
63
  end
75
64
 
76
- def gbook(gdrive_slot)
77
- r = self
78
- title = r.path.split("/").first
79
- Gbook.find_all_by_path(title,gdrive_slot).first
80
- end
81
-
82
- def gsheet(gdrive_slot)
83
- r = self
84
- u = r.user
85
- jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
86
- #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
87
- unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
88
- #only give the user edit permissions if they're the ones
89
- #creating it
90
- jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
91
- unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
92
- jobs_sheet.spreadsheet.update_acl(u.email,"writer")
93
- end
94
- end
95
- jobs_sheet.add_headers(r.headers)
96
- #add url to dataset
97
- Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
98
- begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
99
- return jobs_sheet
100
- end
101
-
102
65
  def read_gsheet(gdrive_slot)
103
66
  r = self
104
- gsheet_tsv = r.gsheet(gdrive_slot).read(Gdrive.owner_name)
67
+ #argument converts line breaks in cells to spaces
68
+ gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
105
69
  #turn it into a hash array
106
- gsheet_jobs = gsheet_tsv.tsv_to_hash_array
70
+ gsheet_hashes = gsheet_tsv.tsv_to_hash_array
107
71
  #go through each job, update relevant job with its params
108
72
  done_jobs = []
109
73
  #parse out the jobs and update the Job collection
110
- gsheet_jobs.each_with_index do |rj,rj_i|
74
+ gsheet_hashes.each do |gsheet_hash|
111
75
  #skip non-jobs or jobs without required values
112
- next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
113
- j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
114
- #update top line params
115
- j.update_attributes(:active => rj['active'],
116
- :trigger => rj['trigger'])
117
- (1..5).to_a.each do |s_idx|
118
- stage_string = rj["stage#{s_idx.to_s}"]
119
- s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
120
- if stage_string.to_s.length==0
121
- #delete this stage and all stages after
122
- if s
123
- j = s.job
124
- j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
125
- #just in case
126
- s.delete
127
- end
128
- break
129
- elsif s.nil?
130
- #create this stage
131
- s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
132
- end
133
- #parse command string, update stage with it
134
- s_handler, call, param_string = [""*3]
135
- stage_string.split(" ").ie do |spls|
136
- s_handler = spls.first.split(".").first
137
- call = spls.first.split(".").last
138
- param_string = spls[1..-1].join(" ").strip
139
- end
140
- s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
141
- end
76
+ next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
77
+ j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
142
78
  r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
143
79
  #add this job to list of read ones
144
80
  done_jobs << j
145
81
  end
146
82
  #delete user jobs that are not included in Runner
147
- (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
148
- j = Job.where(:path=>rj_path).first
83
+ (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
84
+ j = Job.where(:path=>gsheet_hash_path).first
149
85
  j.delete if j
150
- r.update_status("Deleted job:#{rj_path}")
86
+ r.update_status("Deleted job:#{gsheet_hash_path}")
151
87
  end
152
88
  r.update_status("jobs read at #{Time.now.utc}")
153
89
  return true
@@ -158,48 +94,13 @@ module Mobilize
158
94
  #there's nothing to update if runner has never had a completed at
159
95
  return false unless r.completed_at
160
96
  jobs_gsheet = r.gsheet(gdrive_slot)
161
- upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
97
+ upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
162
98
  upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
163
99
  jobs_gsheet.add_or_update_rows(upd_rows)
164
100
  r.update_status("gsheet updated")
165
101
  return true
166
102
  end
167
103
 
168
- def jobs(jname=nil)
169
- r = self
170
- js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
171
- if jname
172
- return js.sel{|j| j.name == jname}.first
173
- else
174
- return js
175
- end
176
- end
177
-
178
- def user
179
- r = self
180
- user_name = r.path.split("_").second.split("(").first.split("/").first
181
- User.where(:name=>user_name).first
182
- end
183
-
184
- def update_status(msg)
185
- r = self
186
- r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
187
- Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
188
- return true
189
- end
190
-
191
- def is_working?
192
- r = self
193
- Mobilize::Resque.active_paths.include?(r.path)
194
- end
195
-
196
- def is_due?
197
- r = self.reload
198
- return false if r.is_working?
199
- prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
200
- return true if r.started_at.nil? or r.started_at < prev_due_time
201
- end
202
-
203
104
  def enqueue!
204
105
  r = self
205
106
  ::Resque::Job.create("mobilize",Runner,r.path,{})
@@ -2,6 +2,7 @@ module Mobilize
2
2
  class Stage
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::StageHelper
5
6
  field :path, type: String
6
7
  field :handler, type: String
7
8
  field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
16
17
 
17
18
  index({ path: 1})
18
19
 
19
- def idx
20
- s = self
21
- s.path.split("/").last.gsub("stage","").to_i
22
- end
23
-
24
- def out_dst
25
- #this gives a dataset that points to the output
26
- #allowing you to determine its size
27
- #before committing to a read or write
28
- s = self
29
- Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
30
- end
31
-
32
- def err_dst
33
- #this gives a dataset that points to the output
34
- #allowing you to determine its size
35
- #before committing to a read or write
36
- s = self
37
- Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
38
- end
39
-
40
- def params
41
- s = self
42
- p = YAML.easy_load(s.param_string)
43
- raise "Must resolve to Hash" unless p.class==Hash
44
- return p
45
- end
46
-
47
- def job
48
- s = self
49
- job_path = s.path.split("/")[0..-2].join("/")
50
- Job.where(:path=>job_path).first
51
- end
52
-
53
20
  def Stage.find_or_create_by_path(path)
54
21
  s = Stage.where(:path=>path).first
55
22
  s = Stage.create(:path=>path) unless s
@@ -80,7 +47,11 @@ module Mobilize
80
47
  s.update_attributes(:started_at=>Time.now.utc)
81
48
  s.update_status(%{Starting at #{Time.now.utc}})
82
49
  #get response by running method
83
- response = "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
50
+ response = begin
51
+ "Mobilize::#{s.handler.humanize}".constantize.send("#{s.call}_by_stage_path",s.path)
52
+ rescue => exc
53
+ {'err_str'=>"#{exc.to_s}\n#{exc.backtrace.to_a.join("\n")}", 'signal'=>500}
54
+ end
84
55
  unless response
85
56
  #re-queue self if no response
86
57
  s.enqueue!
@@ -92,10 +63,10 @@ module Mobilize
92
63
  #retry
93
64
  s.update_attributes(:retries_done => s.retries_done.to_i + 1, :response => response)
94
65
  s.update_status(%{Retry #{s.retries_done.to_s} at #{Time.now.utc}})
66
+ sleep s['delay'].to_i
95
67
  s.enqueue!
96
68
  else
97
69
  #sleep as much as user specifies
98
- sleep s['delay'].to_i
99
70
  s.fail(response)
100
71
  end
101
72
  return true
@@ -111,7 +82,7 @@ module Mobilize
111
82
  r = j.runner
112
83
  dep_jobs = r.jobs.select do |dj|
113
84
  dj.active==true and
114
- dj.trigger.strip.downcase == "after #{j.name}"
85
+ dj.trigger.strip.downcase == "after #{j.name.downcase}"
115
86
  end
116
87
  #put begin/rescue so all dependencies run
117
88
  dep_jobs.each do |dj|
@@ -141,7 +112,7 @@ module Mobilize
141
112
  j = s.job
142
113
  r = j.runner
143
114
  u = r.user
144
- j.update_attributes(:active=>false)
115
+ j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
145
116
  s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
146
117
  stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
147
118
  target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
@@ -158,10 +129,9 @@ module Mobilize
158
129
  err_txt = ["response","\n",err_txt].join
159
130
  err_sheet.write(err_txt,u.name)
160
131
  #exception will be first row below "response" header
161
- exc_to_s,backtrace = err_txt.split("\n").ie{|ea| [ea[1], ea[2..-1]]}
162
132
  s.update_status(status_msg)
163
133
  #raise the exception so it bubbles up to resque
164
- raise Exception,exc_to_s,backtrace
134
+ raise Exception,err_txt
165
135
  end
166
136
 
167
137
  def enqueue!
@@ -208,14 +178,15 @@ module Mobilize
208
178
  raise "incompatible target handler #{handler} for #{s.handler} stage"
209
179
  else
210
180
  begin
211
- return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path)
181
+ #nil gdrive_slot for targets since there is no verification
182
+ return "Mobilize::#{s.handler.downcase.capitalize}".constantize.path_to_dst(target_path,s.path,nil)
212
183
  rescue => exc
213
184
  raise "Could not get #{target_path} with error: #{exc.to_s}"
214
185
  end
215
186
  end
216
187
  end
217
188
 
218
- def sources
189
+ def sources(gdrive_slot)
219
190
  #returns an array of Datasets corresponding to
220
191
  #items listed as sources in the stage params
221
192
  s = self
@@ -244,7 +215,7 @@ module Mobilize
244
215
  end
245
216
  begin
246
217
  stage_path = s.path
247
- dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path)
218
+ dsts << "Mobilize::#{handler.downcase.capitalize}".constantize.path_to_dst(source_path,stage_path,gdrive_slot)
248
219
  rescue => exc
249
220
  raise "Could not get #{source_path} with error: #{exc.to_s}"
250
221
  end