mobilize-base 1.297 → 1.298

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.md CHANGED
@@ -560,7 +560,11 @@ from the referenced sheet.
560
560
  * All stages accept retry parameters:
561
561
  * retries: an integer specifying the number of times that the system will try it again before giving up.
562
562
  * delay: an integer specifying the number of seconds between retries.
563
- * always_on: if true, keeps the job on regardless of stage failures. The job will retry from the beginning with the same frequency as the Runner refresh rate.
563
+ * always_on: if false, turns the job off on stage failures.
564
+ Otherwise the job will retry from the beginning with the same frequency as the Runner refresh rate.
565
+ * notify: by default, the stage owner will be notified on failure.
566
+ * if false, will not notify the stage owner in the event of a failure.
567
+ * If it's an email address, will email the specified person.
564
568
  * If a stage fails after all retries, it will output its standard error to a tab in the Runner with the name of the job, the name of the stage, and a ".err" extension
565
569
  * The tab will be headed "response" and will contain the exception and backtrace for the error.
566
570
  * The test uses "Requestor_mobilize(test)/base1.out" and
data/lib/mobilize-base.rb CHANGED
@@ -65,9 +65,13 @@ if File.exists?(mongoid_config_path)
65
65
  Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
66
66
  require "mobilize-base/models/dataset"
67
67
  require "mobilize-base/models/user"
68
+ require "mobilize-base/helpers/runner_helper"
68
69
  require "mobilize-base/models/runner"
70
+ require "mobilize-base/helpers/job_helper"
69
71
  require "mobilize-base/models/job"
72
+ require "mobilize-base/helpers/stage_helper"
70
73
  require "mobilize-base/models/stage"
74
+
71
75
  end
72
76
  require 'google_drive'
73
77
  require 'resque'
@@ -16,11 +16,13 @@ class String
16
16
  end
17
17
  def bash(except=true)
18
18
  str = self
19
- pid,stdin,stdout,stderr = Open4.popen4(str)
20
- pid,stdin = [nil,nil]
21
- err_str = stderr.read if stderr
22
- out_str = stdout.read if stdout
23
- raise err_str if (err_str.length>0 and except==true)
19
+ out_str,err_str = []
20
+ status = Open4.popen4(str) do |pid,stdin,stdout,stderr|
21
+ out_str = stdout.read
22
+ err_str = stderr.read
23
+ end
24
+ exit_status = status.exitstatus
25
+ raise err_str if (exit_status !=0 and except==true)
24
26
  return out_str
25
27
  end
26
28
  def escape_regex
@@ -25,7 +25,7 @@ module Mobilize
25
25
  return idle_workers if state == 'idle'
26
26
  stale_workers = workers.select{|w| Time.parse(w.started) < Jobtracker.deployed_at}
27
27
  return stale_workers if state == 'stale'
28
- timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['runat'] < (Time.now.utc - Jobtracker.max_run_time)}
28
+ timeout_workers = workers.select{|w| w.job['payload'] and w.job['payload']['class']!='Jobtracker' and w.job['run_at'] < (Time.now.utc - Jobtracker.max_run_time)}
29
29
  return timeout_workers if state == 'timeout'
30
30
  raise "invalid state #{state}"
31
31
  end
@@ -113,7 +113,13 @@ module Mobilize
113
113
  stage_path = f['payload']['args'].first
114
114
  email = begin
115
115
  s = Stage.where(:path=>stage_path).first
116
- s.job.runner.user.email
116
+ if s.params['notify'].to_s=="false"
117
+ next
118
+ elsif s.params['notify'].index("@")
119
+ s.params['notify']
120
+ else
121
+ s.job.runner.user.email
122
+ end
117
123
  rescue
118
124
  #jobs without stages are sent to first admin
119
125
  Jobtracker.admin_emails.first
@@ -0,0 +1,54 @@
1
+ #this module adds convenience methods to the Job model
2
+ module Mobilize
3
+ module JobHelper
4
+ def name
5
+ j = self
6
+ j.path.split("/").last
7
+ end
8
+
9
+ def stages
10
+ j = self
11
+ #starts with the job path, followed by a slash
12
+ Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
13
+ end
14
+
15
+ def status
16
+ #last stage status
17
+ j = self
18
+ j.active_stage.status if j.active_stage
19
+ end
20
+
21
+ def active_stage
22
+ j = self
23
+ #latest started at or first
24
+ j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
25
+ end
26
+
27
+ def completed_at
28
+ j = self
29
+ j.stages.last.completed_at if j.stages.last
30
+ end
31
+
32
+ def failed_at
33
+ j = self
34
+ j.active_stage.failed_at if j.active_stage
35
+ end
36
+
37
+ def status_at
38
+ j = self
39
+ j.active_stage.status_at if j.active_stage
40
+ end
41
+
42
+ #convenience methods
43
+ def runner
44
+ j = self
45
+ runner_path = j.path.split("/")[0..-2].join("/")
46
+ return Runner.where(:path=>runner_path).first
47
+ end
48
+
49
+ def is_working?
50
+ j = self
51
+ j.stages.select{|s| s.is_working?}.compact.length>0
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,83 @@
1
+ #this module adds convenience methods to the Runner model
2
+ module Mobilize
3
+ module RunnerHelper
4
+ def headers
5
+ %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
6
+ end
7
+
8
+ def title
9
+ r = self
10
+ r.path.split("/").first
11
+ end
12
+
13
+ def worker
14
+ r = self
15
+ Mobilize::Resque.find_worker_by_path(r.path)
16
+ end
17
+
18
+ def dataset
19
+ r = self
20
+ Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
21
+ end
22
+
23
+ def gbook(gdrive_slot)
24
+ r = self
25
+ title = r.path.split("/").first
26
+ Gbook.find_by_path(title,gdrive_slot)
27
+ end
28
+
29
+ def gsheet(gdrive_slot)
30
+ r = self
31
+ u = r.user
32
+ jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
33
+ #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
34
+ unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
35
+ #only give the user edit permissions if they're the ones
36
+ #creating it
37
+ jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
38
+ unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
39
+ jobs_sheet.spreadsheet.update_acl(u.email,"writer")
40
+ end
41
+ jobs_sheet.add_headers(r.headers)
42
+ begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
43
+ end
44
+ return jobs_sheet
45
+ end
46
+
47
+ def jobs(jname=nil)
48
+ r = self
49
+ js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
50
+ if jname
51
+ return js.sel{|j| j.name == jname}.first
52
+ else
53
+ return js
54
+ end
55
+ end
56
+
57
+ def user
58
+ r = self
59
+ user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
60
+ User.where(:name=>user_name).first
61
+ end
62
+
63
+ def update_status(msg)
64
+ r = self
65
+ r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
66
+ Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
67
+ return true
68
+ end
69
+
70
+ def is_working?
71
+ r = self
72
+ Mobilize::Resque.active_paths.include?(r.path)
73
+ end
74
+
75
+ def is_due?
76
+ r = self.reload
77
+ return false if r.is_working?
78
+ prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
79
+ return true if r.started_at.nil? or r.started_at < prev_due_time
80
+ end
81
+
82
+ end
83
+ end
@@ -0,0 +1,38 @@
1
+ #this module adds convenience methods to the Stage model
2
+ module Mobilize
3
+ module StageHelper
4
+ def idx
5
+ s = self
6
+ s.path.split("/").last.gsub("stage","").to_i
7
+ end
8
+
9
+ def out_dst
10
+ #this gives a dataset that points to the output
11
+ #allowing you to determine its size
12
+ #before committing to a read or write
13
+ s = self
14
+ Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
15
+ end
16
+
17
+ def err_dst
18
+ #this gives a dataset that points to the output
19
+ #allowing you to determine its size
20
+ #before committing to a read or write
21
+ s = self
22
+ Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
23
+ end
24
+
25
+ def params
26
+ s = self
27
+ p = YAML.easy_load(s.param_string)
28
+ raise "Must resolve to Hash" unless p.class==Hash
29
+ return p
30
+ end
31
+
32
+ def job
33
+ s = self
34
+ job_path = s.path.split("/")[0..-2].join("/")
35
+ Job.where(:path=>job_path).first
36
+ end
37
+ end
38
+ end
@@ -48,6 +48,8 @@ module Mobilize
48
48
  end
49
49
 
50
50
  def Jobtracker.update_status(msg)
51
+ #this is to keep jobtracker from resisting stop commands
52
+ return false if Jobtracker.status=="stopping"
51
53
  #Jobtracker has no persistent database state
52
54
  Resque.set_worker_args_by_path("jobtracker",{'status'=>msg})
53
55
  return true
@@ -122,7 +124,7 @@ module Mobilize
122
124
  sleep 5
123
125
  i=0
124
126
  while Jobtracker.status=='stopping'
125
- Jobtracker.update_status("#{Jobtracker.to_s} still on queue, waiting")
127
+ puts "#{Jobtracker.to_s} still on queue, waiting"
126
128
  sleep 5
127
129
  i+=1
128
130
  end
@@ -145,8 +147,8 @@ module Mobilize
145
147
  def Jobtracker.max_run_time_workers
146
148
  #return workers who have been cranking away for 6+ hours
147
149
  workers = Jobtracker.workers('working').select do |w|
148
- w.job['runat'].to_s.length>0 and
149
- (Time.now.utc - Time.parse(w.job['runat'])) > Jobtracker.max_run_time
150
+ w.job['run_at'].to_s.length>0 and
151
+ (Time.now.utc - Time.parse(w.job['run_at'])) > Jobtracker.max_run_time
150
152
  end
151
153
  return workers
152
154
  end
@@ -185,13 +187,18 @@ module Mobilize
185
187
  if lws.length>0
186
188
  n = {}
187
189
  n['subject'] = "#{lws.length.to_s} max run time jobs"
188
- n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} runat:#{w['runat'].to_s}}}.join("\n\n")
190
+ n['body'] = lws.map{|w| %{spec:#{w['spec']} stg:#{w['stg']} run_at:#{w['run_at'].to_s}}}.join("\n\n")
189
191
  n['to'] = Jobtracker.admin_emails.join(",")
190
192
  notifs << n
191
193
  end
192
194
  #deliver each email generated
193
195
  notifs.each do |notif|
194
- Email.write(notif).deliver
196
+ begin
197
+ Email.write(notif).deliver
198
+ rescue
199
+ #log email on failure
200
+ Jobtracker.update_status("Failed to deliver #{notif.to_s}")
201
+ end
195
202
  end
196
203
  #update notification time so JT knows to wait a while
197
204
  Jobtracker.last_notification = Time.now.utc.to_s
@@ -2,66 +2,54 @@ module Mobilize
2
2
  class Job
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::JobHelper
5
6
  field :path, type: String
6
7
  field :active, type: Boolean
7
8
  field :trigger, type: String
8
9
 
9
10
  index({ path: 1})
10
11
 
11
- def name
12
- j = self
13
- j.path.split("/").last
14
- end
15
-
16
- def stages
17
- j = self
18
- #starts with the job path, followed by a slash
19
- Stage.where(:path=>/^#{j.path.escape_regex}\//).to_a.sort_by{|s| s.path}
20
- end
21
-
22
12
  def Job.find_or_create_by_path(path)
23
13
  j = Job.where(:path=>path).first
24
14
  j = Job.create(:path=>path) unless j
25
15
  return j
26
16
  end
27
17
 
28
- def status
29
- #last stage status
30
- j = self
31
- j.active_stage.status if j.active_stage
32
- end
33
-
34
- def active_stage
35
- j = self
36
- #latest started at or first
37
- j.stages.select{|s| s.started_at}.sort_by{|s| s.started_at}.last || j.stages.first
38
- end
39
-
40
- def completed_at
41
- j = self
42
- j.stages.last.completed_at if j.stages.last
43
- end
44
-
45
- def failed_at
46
- j = self
47
- j.active_stage.failed_at if j.active_stage
48
- end
49
-
50
- def status_at
51
- j = self
52
- j.active_stage.status_at if j.active_stage
53
- end
54
-
55
- #convenience methods
56
- def runner
57
- j = self
58
- runner_path = j.path.split("/")[0..-2].join("/")
59
- return Runner.where(:path=>runner_path).first
60
- end
61
-
62
- def is_working?
63
- j = self
64
- j.stages.select{|s| s.is_working?}.compact.length>0
18
+ #takes a hash of job parameters (name, active, trigger, stages)
19
+ #and creates/updates a job with it
20
+ def Job.update_by_user_name_and_hash(user_name,hash)
21
+ u = User.where(name: user_name).first
22
+ r = u.runner
23
+ j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
24
+ #update top line params
25
+ j.update_attributes(:active => hash['active'],
26
+ :trigger => hash['trigger'])
27
+ (1..5).to_a.each do |s_idx|
28
+ stage_string = hash["stage#{s_idx.to_s}"]
29
+ s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
30
+ if stage_string.to_s.length==0
31
+ #delete this stage and all stages after
32
+ if s
33
+ j = s.job
34
+ j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
35
+ #just in case
36
+ s.delete
37
+ end
38
+ break
39
+ elsif s.nil?
40
+ #create this stage
41
+ s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
42
+ end
43
+ #parse command string, update stage with it
44
+ s_handler, call, param_string = [""*3]
45
+ stage_string.split(" ").ie do |spls|
46
+ s_handler = spls.first.split(".").first
47
+ call = spls.first.split(".").last
48
+ param_string = spls[1..-1].join(" ").strip
49
+ end
50
+ s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
51
+ end
52
+ return j.reload
65
53
  end
66
54
 
67
55
  def is_due?
@@ -1,5 +1,6 @@
1
1
  module Mobilize
2
2
  class Runner
3
+ include Mobilize::RunnerHelper
3
4
  include Mongoid::Document
4
5
  include Mongoid::Timestamps
5
6
  field :path, type: String
@@ -11,20 +12,6 @@ module Mobilize
11
12
 
12
13
  index({ path: 1})
13
14
 
14
- def headers
15
- %w{name active trigger status stage1 stage2 stage3 stage4 stage5}
16
- end
17
-
18
- def title
19
- r = self
20
- r.path.split("/").first
21
- end
22
-
23
- def worker
24
- r = self
25
- Mobilize::Resque.find_worker_by_path(r.path)
26
- end
27
-
28
15
  def Runner.find_by_path(path)
29
16
  Runner.where(:path=>path).first
30
17
  end
@@ -32,6 +19,7 @@ module Mobilize
32
19
  def Runner.find_by_title(title)
33
20
  Runner.where(:path=>"#{title}/jobs").first
34
21
  end
22
+
35
23
  def Runner.perform(id,*args)
36
24
  r = Runner.find_by_path(id)
37
25
  #get gdrive slot for read
@@ -41,12 +29,18 @@ module Mobilize
41
29
  return false
42
30
  end
43
31
  r.update_attributes(:started_at=>Time.now.utc)
44
- #make sure any updates to activity are processed first
45
- #as in when someone runs a "once" job that has completed
46
- r.update_gsheet(gdrive_slot)
47
- #read the jobs in the gsheet and update models with news
48
- r.read_gsheet(gdrive_slot)
49
- #queue up the jobs that are due and active
32
+ begin
33
+ #make sure any updates to activity are processed first
34
+ #as in when someone runs a "once" job that has completed
35
+ r.update_gsheet(gdrive_slot)
36
+ #read the jobs in the gsheet and update models with news
37
+ r.read_gsheet(gdrive_slot)
38
+ #queue up the jobs that are due and active
39
+ rescue => exc
40
+ #log the exception, but continue w job processing
41
+ #This ensures jobs are still processed if google drive goes down
42
+ r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
43
+ end
50
44
  r.jobs.each do |j|
51
45
  begin
52
46
  if j.is_due?
@@ -64,89 +58,32 @@ module Mobilize
64
58
  r.update_attributes(:completed_at=>Time.now.utc)
65
59
  end
66
60
 
67
- def dataset
68
- r = self
69
- Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
70
- end
71
-
72
61
  def Runner.find_or_create_by_path(path)
73
62
  Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
74
63
  end
75
64
 
76
- def gbook(gdrive_slot)
77
- r = self
78
- title = r.path.split("/").first
79
- Gbook.find_by_path(title,gdrive_slot)
80
- end
81
-
82
- def gsheet(gdrive_slot)
83
- r = self
84
- u = r.user
85
- jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
86
- #make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
87
- unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
88
- #only give the user edit permissions if they're the ones
89
- #creating it
90
- jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
91
- unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
92
- jobs_sheet.spreadsheet.update_acl(u.email,"writer")
93
- end
94
- end
95
- jobs_sheet.add_headers(r.headers)
96
- begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
97
- return jobs_sheet
98
- end
99
-
100
65
  def read_gsheet(gdrive_slot)
101
66
  r = self
102
67
  #argument converts line breaks in cells to spaces
103
68
  gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
104
69
  #turn it into a hash array
105
- gsheet_jobs = gsheet_tsv.tsv_to_hash_array
70
+ gsheet_hashes = gsheet_tsv.tsv_to_hash_array
106
71
  #go through each job, update relevant job with its params
107
72
  done_jobs = []
108
73
  #parse out the jobs and update the Job collection
109
- gsheet_jobs.each_with_index do |rj,rj_i|
74
+ gsheet_hashes.each do |gsheet_hash|
110
75
  #skip non-jobs or jobs without required values
111
- next if (rj['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| rj[c].to_s.strip==""}.length>0)
112
- j = Job.find_or_create_by_path("#{r.path}/#{rj['name']}")
113
- #update top line params
114
- j.update_attributes(:active => rj['active'],
115
- :trigger => rj['trigger'])
116
- (1..5).to_a.each do |s_idx|
117
- stage_string = rj["stage#{s_idx.to_s}"]
118
- s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
119
- if stage_string.to_s.length==0
120
- #delete this stage and all stages after
121
- if s
122
- j = s.job
123
- j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
124
- #just in case
125
- s.delete
126
- end
127
- break
128
- elsif s.nil?
129
- #create this stage
130
- s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
131
- end
132
- #parse command string, update stage with it
133
- s_handler, call, param_string = [""*3]
134
- stage_string.split(" ").ie do |spls|
135
- s_handler = spls.first.split(".").first
136
- call = spls.first.split(".").last
137
- param_string = spls[1..-1].join(" ").strip
138
- end
139
- s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
140
- end
76
+ next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
77
+ j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
141
78
  r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
142
79
  #add this job to list of read ones
143
80
  done_jobs << j
144
81
  end
145
82
  #delete user jobs that are not included in Runner
146
- (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |rj_path|
147
- j = Job.where(:path=>rj_path).first
83
+ (r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
84
+ j = Job.where(:path=>gsheet_hash_path).first
148
85
  j.delete if j
149
- r.update_status("Deleted job:#{rj_path}")
86
+ r.update_status("Deleted job:#{gsheet_hash_path}")
150
87
  end
151
88
  r.update_status("jobs read at #{Time.now.utc}")
152
89
  return true
@@ -164,41 +101,6 @@ module Mobilize
164
101
  return true
165
102
  end
166
103
 
167
- def jobs(jname=nil)
168
- r = self
169
- js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
170
- if jname
171
- return js.sel{|j| j.name == jname}.first
172
- else
173
- return js
174
- end
175
- end
176
-
177
- def user
178
- r = self
179
- user_name = r.path.split("_")[1..-1].join("_").split("(").first.split("/").first
180
- User.where(:name=>user_name).first
181
- end
182
-
183
- def update_status(msg)
184
- r = self
185
- r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
186
- Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
187
- return true
188
- end
189
-
190
- def is_working?
191
- r = self
192
- Mobilize::Resque.active_paths.include?(r.path)
193
- end
194
-
195
- def is_due?
196
- r = self.reload
197
- return false if r.is_working?
198
- prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
199
- return true if r.started_at.nil? or r.started_at < prev_due_time
200
- end
201
-
202
104
  def enqueue!
203
105
  r = self
204
106
  ::Resque::Job.create("mobilize",Runner,r.path,{})
@@ -2,6 +2,7 @@ module Mobilize
2
2
  class Stage
3
3
  include Mongoid::Document
4
4
  include Mongoid::Timestamps
5
+ include Mobilize::StageHelper
5
6
  field :path, type: String
6
7
  field :handler, type: String
7
8
  field :call, type: String
@@ -16,40 +17,6 @@ module Mobilize
16
17
 
17
18
  index({ path: 1})
18
19
 
19
- def idx
20
- s = self
21
- s.path.split("/").last.gsub("stage","").to_i
22
- end
23
-
24
- def out_dst
25
- #this gives a dataset that points to the output
26
- #allowing you to determine its size
27
- #before committing to a read or write
28
- s = self
29
- Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
30
- end
31
-
32
- def err_dst
33
- #this gives a dataset that points to the output
34
- #allowing you to determine its size
35
- #before committing to a read or write
36
- s = self
37
- Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
38
- end
39
-
40
- def params
41
- s = self
42
- p = YAML.easy_load(s.param_string)
43
- raise "Must resolve to Hash" unless p.class==Hash
44
- return p
45
- end
46
-
47
- def job
48
- s = self
49
- job_path = s.path.split("/")[0..-2].join("/")
50
- Job.where(:path=>job_path).first
51
- end
52
-
53
20
  def Stage.find_or_create_by_path(path)
54
21
  s = Stage.where(:path=>path).first
55
22
  s = Stage.create(:path=>path) unless s
@@ -145,7 +112,7 @@ module Mobilize
145
112
  j = s.job
146
113
  r = j.runner
147
114
  u = r.user
148
- j.update_attributes(:active=>false) unless s.params['always_on']
115
+ j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
149
116
  s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
150
117
  stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
151
118
  target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
@@ -1,5 +1,5 @@
1
1
  module Mobilize
2
2
  module Base
3
- VERSION = "1.297"
3
+ VERSION = "1.298"
4
4
  end
5
5
  end
@@ -64,6 +64,7 @@ describe "Mobilize" do
64
64
  test_1_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
65
65
  test_2_tsv = Mobilize::Dataset.read_by_url(test_target_sheet_1_url,user_name,gdrive_slot)
66
66
 
67
+ assert test_1_tsv.to_s.length>0
67
68
  assert test_1_tsv == test_2_tsv
68
69
 
69
70
  puts "change first job to fail, wait for stages"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: mobilize-base
3
3
  version: !ruby/object:Gem::Version
4
- version: '1.297'
4
+ version: '1.298'
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-04-06 00:00:00.000000000 Z
12
+ date: 2013-04-17 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rake
@@ -190,6 +190,9 @@ files:
190
190
  - lib/mobilize-base/handlers/gridfs.rb
191
191
  - lib/mobilize-base/handlers/gsheet.rb
192
192
  - lib/mobilize-base/handlers/resque.rb
193
+ - lib/mobilize-base/helpers/job_helper.rb
194
+ - lib/mobilize-base/helpers/runner_helper.rb
195
+ - lib/mobilize-base/helpers/stage_helper.rb
193
196
  - lib/mobilize-base/jobtracker.rb
194
197
  - lib/mobilize-base/models/dataset.rb
195
198
  - lib/mobilize-base/models/job.rb
@@ -225,7 +228,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
225
228
  version: '0'
226
229
  segments:
227
230
  - 0
228
- hash: -414933210981921919
231
+ hash: 2488340168623489502
229
232
  required_rubygems_version: !ruby/object:Gem::Requirement
230
233
  none: false
231
234
  requirements:
@@ -234,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
234
237
  version: '0'
235
238
  segments:
236
239
  - 0
237
- hash: -414933210981921919
240
+ hash: 2488340168623489502
238
241
  requirements: []
239
242
  rubyforge_project: mobilize-base
240
243
  rubygems_version: 1.8.25