mobilize-base 1.297 → 1.298

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -560,7 +560,11 @@ from the referenced sheet.
560
560
  * All stages accept retry parameters:
561
561
  * retries: an integer specifying the number of times that the system will try it again before giving up.
562
562
  * delay: an integer specifying the number of seconds between retries.
563
- * always_on: if true, keeps the job on regardless of stage failures. The job will retry from the beginning with the same frequency as the Runner refresh rate.
563
+ * always_on: if false, turns the job off on stage failures.
564
+ Otherwise the job will retry from the beginning with the same frequency as the Runner refresh rate.
565
+ * notify: by default, the stage owner will be notified on failure.
566
+ * if false, will not notify the stage owner in the event of a failure.
567
+ * If it's an email address, will email the specified person.
564
568
  * If a stage fails after all retries, it will output its standard error to a tab in the Runner with the name of the job, the name of the stage, and a ".err" extension
565
569
  * The tab will be headed "response" and will contain the exception and backtrace for the error.
566
570
  * The test uses "Requestor_mobilize(test)/base1.out" and
data/lib/mobilize-base.rb CHANGED
@@ -65,9 +65,13 @@ if File.exists?(mongoid_config_path)
65
65
  Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
66
66
  require "mobilize-base/models/dataset"
67
67
  require "mobilize-base/models/user"
68
+ require "mobilize-base/helpers/runner_helper"
68
69
  require "mobilize-base/models/runner"
70
+ require "mobilize-base/helpers/job_helper"
69
71
  require "mobilize-base/models/job"
72
+ require "mobilize-base/helpers/stage_helper"
70
73
  require "mobilize-base/models/stage"
74
+
71
75
  end
72
76
  require 'google_drive'
73
77
  require 'resque'
@@ -16,11 +16,13 @@ class String
16
16
  end
17
17
  def bash(except=true)
18
18
  str = self
19
- pid,stdin,stdout,stderr = Open4.popen4(str)
20
- pid,stdin = [nil,nil]
21
- err_str = stderr.read if stderr
22
- out_str = stdout.read if stdout
23
- raise err_str if (err_str.length>0 and except==true)
19
+ out_str,err_str = []
20
+ status = Open4.popen4(str) do |pid,stdin,stdout,stderr|
21
+ out_str = stdout.read
22
+ err_str = stderr.read
23
+ end
24
+ exit_status = status.exitstatus
25
+ raise err_str if (exit_status !=0 and except==true)
24
26
  return out_str
25
27
  end
26
28
  def escape_regex
@@ -25,7 +25,7 @@ module Mobilize
25
25
  return idle_workers if state == 'idle'
26
26
  stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
27
27
  return stale_workers if state == 'stale'
28
- timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
28
+ timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)}
29
29
  return timeout_workers if state == 'timeout'
30
30
  raise "invalid state #{state}"
31
31
  end
@@ -113,7 +113,13 @@ module Mobilize
113
113
  stage_path = f['payload']['args'].first
114
114
  email = begin
115
115
  s = Stage.where(:path=>stage_path).first
116
- s.job.runner.user.email
116
+ if s.params['notify'].to_s=="false"
117
+ next
118
+ elsif s.params['notify'].index("@")
119
+ s.params['notify']
120
+ else
121
+ s.job.runner.user.email
122
+ end
117
123
  rescue
118
124
  #jobs without stages are sent to first admin
119
125
  Jobtracker.admin_emails.first
@@ -0,0 +1,54 @@
1
+ #this module adds convenience methods to the Job model
2
+ module Mobilize
3
+ module JobHelper
4
+ def name
5
+ j = self
6
+ j.path.split("/").last
7
+ end
8
+
9
+ def stages
10
+ j = self
11
+ #starts with the job path, followed by a slash
12
+ Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
13
+ end
14
+
15
+ def status
16
+ #last stage status
17
+ j = self
18
+ j.active_stage.status if j.active_stage
19
+ end
20
+
21
+ def active_stage
22
+ j = self
23
+ #latest started at or first
24
+ j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
25
+ end
26
+
27
+ def completed_at
28
+ j = self
29
+ j.stages.last.completed_at if j.stages.last
30
+ end
31
+
32
+ def failed_at
33
+ j = self
34
+ j.active_stage.failed_at if j.active_stage
35
+ end
36
+
37
+ def status_at
38
+ j = self
39
+ j.active_stage.status_at if j.active_stage
40
+ end
41
+
42
+ #convenience methods
43
+ def runner
44
+ j = self
45
+ runner_path = j.path.split("/")[0..-2].join("/")
46
+ return Runner.where(:path=>runner_path).first
47
+ end
48
+
49
+ def is_working?
50
+ j = self
51
+ j.stages.select{|s| s.is_working?}.compact.length>0
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,83 @@
1
+ #this module adds convenience methods to the Runner model
2
+ module Mobilize
3
+ module RunnerHelper
4
+ def headers
5
+ %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
6
+ end
7
+
8
+ def title
9
+ r = self
10
+ r.path.split("/").first
11
+ end
12
+
13
+ def worker
14
+ r = self
15
+ Mobilize::Resque.find_worker_by_path(r.path)
16
+ end
17
+
18
+ def dataset
19
+ r = self
20
+ Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
21
+ end
22
+
23
+ def gbook(gdrive_slot)
24
+ r = self
25
+ title = r.path.split("/").first
26
+ Gbook.find_by_path(title,gdrive_slot)
27
+ end
28
+
29
+ def gsheet(gdrive_slot)
30
+ r = self
31
+ u = r.user
32
+ jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
33
+ #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
34
+ unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
35
+ #only give the user edit permissions if they're the ones
36
+ #creating it
37
+ jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
38
+ unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
39
+ jobs_sheet.spreadsheet.update_acl(u.email,"writer")
40
+ end
41
+ jobs_sheet.add_headers(r.headers)
42
+ begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
43
+ end
44
+ return jobs_sheet
45
+ end
46
+
47
+ def jobs(jname=nil)
48
+ r = self
49
+ js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
50
+ if jname
51
+ return js.sel{|j| j.name == jname}.first
52
+ else
53
+ return js
54
+ end
55
+ end
56
+
57
+ def user
58
+ r = self
59
+ user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
60
+ User.where(:name=>user_name).first
61
+ end
62
+
63
+ def update_status(msg)
64
+ r = self
65
+ r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
66
+ Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
67
+ return true
68
+ end
69
+
70
+ def is_working?
71
+ r = self
72
+ Mobilize::Resque.active_paths.include?(r.path)
73
+ end
74
+
75
+ def is_due?
76
+ r = self.reload
77
+ return false if r.is_working?
78
+ prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
79
+ return true if r.started_at.nil? or r.started_at < prev_due_time
80
+ end
81
+
82
+ end
83
+ end
@@ -0,0 +1,38 @@
1
+ #this module adds convenience methods to the Stage model
2
+ module Mobilize
3
+ module StageHelper
4
+ def idx
5
+ s = self
6
+ s.path.split("/").last.gsub("stage","").to_i
7
+ end
8
+
9
+ def out_dst
10
+ #this gives a dataset that points to the output
11
+ #allowing you to determine its size
12
+ #before committing to a read or write
13
+ s = self
14
+ Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
15
+ end
16
+
17
+ def err_dst
18
+ #this gives a dataset that points to the output
19
+ #allowing you to determine its size
20
+ #before committing to a read or write
21
+ s = self
22
+ Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
23
+ end
24
+
25
+ def params
26
+ s = self
27
+ p = YAML.easy_load(s.param_string)
28
+ raise "Must resolve to Hash" unless p.class==Hash
29
+ return p
30
+ end
31
+
32
+ def job
33
+ s = self
34
+ job_path = s.path.split("/")[0..-2].join("/")
35
+ Job.where(:path=>job_path).first
36
+ end
37
+ end
38
+ end
@@ -48,6 +48,8 @@ module Mobilize
48
48
  end
49
49
 
50
50
  def Jobtracker.update_status(msg)
51
+ #this is to keep jobtracker from resisting stop commands
52
+ return false if Jobtracker.status=="stopping"
51
53
  #Jobtracker has no persistent database state
52
54
  Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
53
55
  return true
@@ -122,7 +124,7 @@ module Mobilize
122
124
  sleep 5
123
125
  i=0
124
126
  while Jobtracker.status=='stopping'
125
- Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
127
+ puts "#{Jobtracker.to_s} still on queue, waiting"
126
128
  sleep 5
127
129
  i+=1
128
130
  end
@@ -145,8 +147,8 @@ module Mobilize
145
147
  def Jobtracker.max_run_time_workers
146
148
  #return workers who have been cranking away for 6+ hours
147
149
  workers = Jobtracker.workers('working').select do |w|
148
- w.job['runat'].to_s.length>0 and
149
- (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
150
+ w.job['run_at'].to_s.length>0 and
151
+ (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
150
152
  end
151
153
  return workers
152
154
  end
@@ -185,13 +187,18 @@ module Mobilize
185
187
  if lws.length>0
186
188
  n = {}
187
189
  n['subject'] = "#{lws.length.to_s} max run time jobs"
188
- n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
190
+ n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
189
191
  n['to'] = Jobtracker.admin_emails.join(",")
190
192
  notifs << n
191
193
  end
192
194
  #deliver each email generated
193
195
  notifs.each do |notif|
194
- Email.write(notif).deliver
196
+ begin
197
+ Email.write(notif).deliver
198
+ rescue
199
+ #log email on failure
200
+ Jobtracker.update_status("Failed to deliver #{notif.to_s}")
201
+ end
195
202
  end
196
203
  #update notification time so JT knows to wait a while
197
204
  Jobtracker.last_notification = Time.now.utc.to_s
@@ -2,66 +2,54 @@ module Mobilize
2
2
  class Job
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::JobHelper
5
6
  field :path, type: String
6
7
  field :active, type: Boolean
7
8
  field :trigger, type: String
8
9
 
9
10
  index({ path: 1})
10
11
 
11
- def name
12
- j = self
13
- j.path.split("/").last
14
- end
15
-
16
- def stages
17
- j = self
18
- #starts with the job path, followed by a slash
19
- Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
20
- end
21
-
22
12
  def Job.find_or_create_by_path(path)
23
13
  j = Job.where(:path=>path).first
24
14
  j = Job.create(:path=>path) unless j
25
15
  return j
26
16
  end
27
17
 
28
- def status
29
- #last stage status
30
- j = self
31
- j.active_stage.status if j.active_stage
32
- end
33
-
34
- def active_stage
35
- j = self
36
- #latest started at or first
37
- j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
38
- end
39
-
40
- def completed_at
41
- j = self
42
- j.stages.last.completed_at if j.stages.last
43
- end
44
-
45
- def failed_at
46
- j = self
47
- j.active_stage.failed_at if j.active_stage
48
- end
49
-
50
- def status_at
51
- j = self
52
- j.active_stage.status_at if j.active_stage
53
- end
54
-
55
- #convenience methods
56
- def runner
57
- j = self
58
- runner_path = j.path.split("/")[0..-2].join("/")
59
- return Runner.where(:path=>runner_path).first
60
- end
61
-
62
- def is_working?
63
- j = self
64
- j.stages.select{|s| s.is_working?}.compact.length>0
18
+ #takes a hash of job parameters (name, active, trigger, stages)
19
+ #and creates/updates a job with it
20
+ def Job.update_by_user_name_and_hash(user_name,hash)
21
+ u = User.where(name: user_name).first
22
+ r = u.runner
23
+ j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
24
+ #update top line params
25
+ j.update_attributes(:active => hash['active'],
26
+ :trigger => hash['trigger'])
27
+ (1..5).to_a.each do |s_idx|
28
+ stage_string = hash["stage#{s_idx.to_s}"]
29
+ s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
30
+ if stage_string.to_s.length==0
31
+ #delete this stage and all stages after
32
+ if s
33
+ j = s.job
34
+ j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
35
+ #just in case
36
+ s.delete
37
+ end
38
+ break
39
+ elsif s.nil?
40
+ #create this stage
41
+ s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
42
+ end
43
+ #parse command string, update stage with it
44
+ s_handler, call, param_string = [""*3]
45
+ stage_string.split(" ").ie do |spls|
46
+ s_handler = spls.first.split(".").first
47
+ call = spls.first.split(".").last
48
+ param_string = spls[1..-1].join(" ").strip
49
+ end
50
+ s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
51
+ end
52
+ return j.reload
65
53
  end
66
54
 
67
55
  def is_due?
@@ -1,5 +1,6 @@
1
1
  module Mobilize
2
2
  class Runner
3
+ include Mobilize::RunnerHelper
3
4
  include Mongoid::Document
4
5
  include Mongoid::Timestamps
5
6
  field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
11
12
 
12
13
  index({ path: 1})
13
14
 
14
- def headers
15
- %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
16
- end
17
-
18
- def title
19
- r = self
20
- r.path.split("/").first
21
- end
22
-
23
- def worker
24
- r = self
25
- Mobilize::Resque.find_worker_by_path(r.path)
26
- end
27
-
28
15
  def Runner.find_by_path(path)
29
16
  Runner.where(:path=>path).first
30
17
  end
@@ -32,6 +19,7 @@ module Mobilize
32
19
  def Runner.find_by_title(title)
33
20
  Runner.where(:path=>"#{title}/jobs").first
34
21
  end
22
+
35
23
  def Runner.perform(id,*args)
36
24
  r = Runner.find_by_path(id)
37
25
  #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
41
29
  return false
42
30
  end
43
31
  r.update_attributes(:started_at=>Time.now.utc)
44
- #make sure any updates to activity are processed first
45
- #as in when someone runs a "once" job that has completed
46
- r.update_gsheet(gdrive_slot)
47
- #read the jobs in the gsheet and update models with news
48
- r.read_gsheet(gdrive_slot)
49
- #queue up the jobs that are due and active
32
+ begin
33
+ #make sure any updates to activity are processed first
34
+ #as in when someone runs a "once" job that has completed
35
+ r.update_gsheet(gdrive_slot)
36
+ #read the jobs in the gsheet and update models with news
37
+ r.read_gsheet(gdrive_slot)
38
+ #queue up the jobs that are due and active
39
+ rescue => exc
40
+ #log the exception, but continue w job processing
41
+ #This ensures jobs are still processed if google drive goes down
42
+ r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
43
+ end
50
44
  r.jobs.each do |j|
51
45
  begin
52
46
  if j.is_due?
@@ -64,89 +58,32 @@ module Mobilize
64
58
  r.update_attributes(:completed_at=>Time.now.utc)
65
59
  end
66
60
 
67
- def dataset
68
- r = self
69
- Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
70
- end
71
-
72
61
  def Runner.find_or_create_by_path(path)
73
62
  Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
74
63
  end
75
64
 
76
- def gbook(gdrive_slot)
77
- r = self
78
- title = r.path.split("/").first
79
- Gbook.find_by_path(title,gdrive_slot)
80
- end
81
-
82
- def gsheet(gdrive_slot)
83
- r = self
84
- u = r.user
85
- jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
86
- #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
87
- unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
88
- #only give the user edit permissions if they're the ones
89
- #creating it
90
- jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
91
- unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
92
- jobs_sheet.spreadsheet.update_acl(u.email,"writer")
93
- end
94
- end
95
- jobs_sheet.add_headers(r.headers)
96
- begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
97
- return jobs_sheet
98
- end
99
-
100
65
  def read_gsheet(gdrive_slot)
101
66
  r = self
102
67
  #argument converts line breaks in cells to spaces
103
68
  gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
104
69
  #turn it into a hash array
105
- gsheet_jobs = gsheet_tsv.tsv_to_hash_array
70
+ gsheet_hashes = gsheet_tsv.tsv_to_hash_array
106
71
  #go through each job, update relevant job with its params
107
72
  done_jobs = []
108
73
  #parse out the jobs and update the Job collection
109
- gsheet_jobs.each_with_index do |rj,rj_i|
74
+ gsheet_hashes.each do |gsheet_hash|
110
75
  #skip non-jobs or jobs without required values
111
- next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
112
- j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
113
- #update top line params
114
- j.update_attributes(:active => rj['active'],
115
- :trigger => rj['trigger'])
116
- (1..5).to_a.each do |s_idx|
117
- stage_string = rj["stage#{s_idx.to_s}"]
118
- s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
119
- if stage_string.to_s.length==0
120
- #delete this stage and all stages after
121
- if s
122
- j = s.job
123
- j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
124
- #just in case
125
- s.delete
126
- end
127
- break
128
- elsif s.nil?
129
- #create this stage
130
- s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
131
- end
132
- #parse command string, update stage with it
133
- s_handler, call, param_string = [""*3]
134
- stage_string.split(" ").ie do |spls|
135
- s_handler = spls.first.split(".").first
136
- call = spls.first.split(".").last
137
- param_string = spls[1..-1].join(" ").strip
138
- end
139
- s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
140
- end
76
+ next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
77
+ j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
141
78
  r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
142
79
  #add this job to list of read ones
143
80
  done_jobs << j
144
81
  end
145
82
  #delete user jobs that are not included in Runner
146
- (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
147
- j = Job.where(:path=>rj_path).first
83
+ (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
84
+ j = Job.where(:path=>gsheet_hash_path).first
148
85
  j.delete if j
149
- r.update_status("Deleted job:#{rj_path}")
86
+ r.update_status("Deleted job:#{gsheet_hash_path}")
150
87
  end
151
88
  r.update_status("jobs read at #{Time.now.utc}")
152
89
  return true
@@ -164,41 +101,6 @@ module Mobilize
164
101
  return true
165
102
  end
166
103
 
167
- def jobs(jname=nil)
168
- r = self
169
- js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
170
- if jname
171
- return js.sel{|j| j.name == jname}.first
172
- else
173
- return js
174
- end
175
- end
176
-
177
- def user
178
- r = self
179
- user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
180
- User.where(:name=>user_name).first
181
- end
182
-
183
- def update_status(msg)
184
- r = self
185
- r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
186
- Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
187
- return true
188
- end
189
-
190
- def is_working?
191
- r = self
192
- Mobilize::Resque.active_paths.include?(r.path)
193
- end
194
-
195
- def is_due?
196
- r = self.reload
197
- return false if r.is_working?
198
- prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
199
- return true if r.started_at.nil? or r.started_at < prev_due_time
200
- end
201
-
202
104
  def enqueue!
203
105
  r = self
204
106
  ::Resque::Job.create("mobilize",Runner,r.path,{})
@@ -2,6 +2,7 @@ module Mobilize
2
2
  class Stage
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::StageHelper
5
6
  field :path, type: String
6
7
  field :handler, type: String
7
8
  field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
16
17
 
17
18
  index({ path: 1})
18
19
 
19
- def idx
20
- s = self
21
- s.path.split("/").last.gsub("stage","").to_i
22
- end
23
-
24
- def out_dst
25
- #this gives a dataset that points to the output
26
- #allowing you to determine its size
27
- #before committing to a read or write
28
- s = self
29
- Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
30
- end
31
-
32
- def err_dst
33
- #this gives a dataset that points to the output
34
- #allowing you to determine its size
35
- #before committing to a read or write
36
- s = self
37
- Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
38
- end
39
-
40
- def params
41
- s = self
42
- p = YAML.easy_load(s.param_string)
43
- raise "Must resolve to Hash" unless p.class==Hash
44
- return p
45
- end
46
-
47
- def job
48
- s = self
49
- job_path = s.path.split("/")[0..-2].join("/")
50
- Job.where(:path=>job_path).first
51
- end
52
-
53
20
  def Stage.find_or_create_by_path(path)
54
21
  s = Stage.where(:path=>path).first
55
22
  s = Stage.create(:path=>path) unless s
@@ -145,7 +112,7 @@ module Mobilize
145
112
  j = s.job
146
113
  r = j.runner
147
114
  u = r.user
148
- j.update_attributes(:active=>false) unless s.params['always_on']
115
+ j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
149
116
  s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
150
117
  stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
151
118
  target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
@@ -1,5 +1,5 @@
1
1
  module Mobilize
2
2
  module Base
3
- VERSION = "1.297"
3
+ VERSION = "1.298"
4
4
  end
5
5
  end
@@ -64,6 +64,7 @@ describe "Mobilize" do
64
64
  test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
65
65
  test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
66
66
 
67
+ assert test_1_tsv.to_s.length>0
67
68
  assert test_1_tsv == test_2_tsv
68
69
 
69
70
  puts "change first job to fail, wait for stages"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mobilize-base
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.297'
4
+ version: '1.298'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-04-06 00:00:00.000000000 Z
12
+ date: 2013-04-17 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -190,6 +190,9 @@ files:
190
190
  - lib/mobilize-base/handlers/gridfs.rb
191
191
  - lib/mobilize-base/handlers/gsheet.rb
192
192
  - lib/mobilize-base/handlers/resque.rb
193
+ - lib/mobilize-base/helpers/job_helper.rb
194
+ - lib/mobilize-base/helpers/runner_helper.rb
195
+ - lib/mobilize-base/helpers/stage_helper.rb
193
196
  - lib/mobilize-base/jobtracker.rb
194
197
  - lib/mobilize-base/models/dataset.rb
195
198
  - lib/mobilize-base/models/job.rb
@@ -225,7 +228,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
225
228
  version: '0'
226
229
  segments:
227
230
  - 0
228
- hash: -414933210981921919
231
+ hash: 2488340168623489502
229
232
  required_rubygems_version: !ruby/object:Gem::Requirement
230
233
  none: false
231
234
  requirements:
@@ -234,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
234
237
  version: '0'
235
238
  segments:
236
239
  - 0
237
- hash: -414933210981921919
240
+ hash: 2488340168623489502
238
241
  requirements: []
239
242
  rubyforge_project: mobilize-base
240
243
  rubygems_version: 1.8.25