mobilize-base 1.351 → 1.361

Sign up to get free protection for your applications and to get access to all the features.
@@ -8,7 +8,7 @@ module GoogleDrive
8
8
  attempts = 0
9
9
  sleep_time = nil
10
10
  #try 5 times to make the call
11
- while (response.nil? or response.code.starts_with?("5")) and attempts < 20
11
+ while (response.nil? or response.code.starts_with?("5")) and attempts < Mobilize::Gdrive.max_api_retries
12
12
  #instantiate http object, set params
13
13
  http = @proxy.new(uri.host, uri.port)
14
14
  http.use_ssl = true
@@ -16,6 +16,7 @@ module GoogleDrive
16
16
  #set 600 to allow for large downloads
17
17
  http.read_timeout = 600
18
18
  response = begin
19
+ puts "#{Time.now.utc} Gdrive API #{method.to_s}: #{uri.to_s} #{extra_header.to_s}"
19
20
  clf.http_call(http, method, uri, data, extra_header, auth)
20
21
  rescue
21
22
  #timeouts etc.
@@ -43,7 +43,7 @@ class String
43
43
  return v if v.to_s.strip==""
44
44
  #normalize numbers by removing '$', '%', ',', ' '
45
45
  vnorm = v.to_s.norm_num
46
- vdigits = vnorm.split(".").last.length
46
+ vdigits = vnorm.split(".").last.to_s.length
47
47
  if vnorm.to_f.to_s=="Infinity"
48
48
  #do nothing
49
49
  elsif ("%.#{vdigits}f" % vnorm.to_f.to_s)==vnorm
@@ -0,0 +1,20 @@
1
+ class Time
2
+ def Time.at_marks_ago(number=1, unit='day', mark='0000')
3
+ curr_time = Time.now.utc
4
+ #strip out non-numerical characters from mark, add colon
5
+ mark = mark.gsub(/[^0-9]/i,"").rjust(4,'0').ie{|m| [m[0..1],":",m[-2..-1]].join}
6
+ #if user passes in 0 for the number, make it 1
7
+ number = (number.to_i <= 0 ? 1 : number.to_i)
8
+ if unit == 'day'
9
+ curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d #{mark} UTC"))
10
+ elsif unit == 'hour'
11
+ if curr_time.strftime("%H%M").to_i > mark.to_i
12
+ curr_mark_time = Time.parse(curr_time.strftime("%Y-%m-%d %H:#{mark[-2..-1]} UTC"))
13
+ end
14
+ end
15
+ #last mark time is
16
+ mark_ago_increment = (curr_time > curr_mark_time ? (number-1).send(unit) : number.send(unit))
17
+ last_mark_time = curr_mark_time - mark_ago_increment
18
+ return last_mark_time
19
+ end
20
+ end
@@ -25,6 +25,18 @@ module Mobilize
25
25
  end
26
26
  end
27
27
 
28
+ def Gdrive.max_api_retries
29
+ Gdrive.config['max_api_retries']
30
+ end
31
+
32
+ def Gdrive.max_file_write_retries
33
+ Gdrive.config['max_file_write_retries']
34
+ end
35
+
36
+ def Gdrive.file_write_retry_delay
37
+ Gdrive.config['file_write_retry_delay']
38
+ end
39
+
28
40
  def Gdrive.admins
29
41
  Gdrive.config['admins']
30
42
  end
@@ -1,9 +1,23 @@
1
1
  module Mobilize
2
2
  module Gfile
3
+
4
+ def Gfile.config
5
+ Base.config('gfile')
6
+ end
7
+
8
+ def Gfile.max_length
9
+ Gfile.config['max_length']
10
+ end
11
+
3
12
  def Gfile.path_to_dst(path,stage_path,gdrive_slot)
13
+ s = Stage.where(:path=>stage_path).first
14
+ params = s.params
15
+ target_path = params['target']
16
+ #if this is the target, it doesn't have to exist already
17
+ is_target = true if path == target_path
4
18
  #don't need the ://
5
19
  path = path.split("://").last if path.index("://")
6
- if Gfile.find_by_path(path)
20
+ if is_target or Gfile.find_by_path(path)
7
21
  handler = "gfile"
8
22
  Dataset.find_or_create_by_url("#{handler}://#{path}")
9
23
  else
@@ -30,12 +44,6 @@ module Mobilize
30
44
  :content_type=>"test/plain",
31
45
  :convert=>false)
32
46
  file.add_admin_acl
33
- #make sure user is owner or can edit
34
- u = User.where(:name=>user_name).first
35
- entry = file.acl_entry(u.email)
36
- unless entry and ['writer','owner'].include?(entry.role)
37
- file.update_acl(u.email)
38
- end
39
47
  #update http url for file
40
48
  dst = Dataset.find_by_handler_and_path("gfile",dst_path)
41
49
  api_url = file.human_url.split("&").first
@@ -93,5 +101,49 @@ module Mobilize
93
101
  end
94
102
  return file
95
103
  end
104
+
105
+ def Gfile.write_by_stage_path(stage_path)
106
+ gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
107
+ #return blank response if there are no slots available
108
+ return nil unless gdrive_slot
109
+ s = Stage.where(:path=>stage_path).first
110
+ u = s.job.runner.user
111
+ retries = 0
112
+ stdout,stderr = []
113
+ while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
114
+ begin
115
+ #get tsv to write from stage
116
+ source = s.sources(gdrive_slot).first
117
+ raise "Need source for gfile write" unless source
118
+ tsv = source.read(u.name,gdrive_slot)
119
+ raise "No data source found for #{source.url}" unless tsv.to_s.length>0
120
+ if tsv.length > Gfile.max_length
121
+ raise "Too much data; you have #{tsv.length.to_s}, max is #{Gfile.max_length.to_s}"
122
+ end
123
+ stdout = if tsv.length == 0
124
+ #soft error; no data to write. Stage will complete.
125
+ "Write skipped for #{s.target.url}"
126
+ else
127
+ Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot)
128
+ #update status
129
+ "Write successful for #{s.target.url}"
130
+ end
131
+ Gdrive.unslot_worker_by_path(stage_path)
132
+ stderr = nil
133
+ s.update_status(stdout)
134
+ signal = 0
135
+ rescue => exc
136
+ if retries < Gdrive.max_file_write_retries
137
+ retries +=1
138
+ sleep Gdrive.file_write_retry_delay
139
+ else
140
+ stdout = nil
141
+ stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
142
+ signal = 500
143
+ end
144
+ end
145
+ end
146
+ return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
147
+ end
96
148
  end
97
149
  end
@@ -127,33 +127,43 @@ module Mobilize
127
127
  s = Stage.where(:path=>stage_path).first
128
128
  u = s.job.runner.user
129
129
  crop = s.params['crop'] || true
130
- begin
131
- #get tsv to write from stage
132
- source = s.sources(gdrive_slot).first
133
- raise "Need source for gsheet write" unless source
134
- tsv = source.read(u.name,gdrive_slot)
135
- raise "No data source found for #{source.url}" unless tsv
136
- tsv_row_count = tsv.to_s.split("\n").length
137
- tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
138
- tsv_cell_count = tsv_row_count * tsv_col_count
139
- stdout = if tsv_row_count == 0
140
- #soft error; no data to write. Stage will complete.
141
- "Write skipped for #{s.target.url}"
142
- elsif tsv_cell_count > Gsheet.max_cells
143
- raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
144
- else
145
- Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
146
- #update status
147
- "Write successful for #{s.target.url}"
148
- end
149
- Gdrive.unslot_worker_by_path(stage_path)
150
- stderr = nil
151
- s.update_status(stdout)
152
- signal = 0
153
- rescue => exc
154
- stdout = nil
155
- stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
156
- signal = 500
130
+ retries = 0
131
+ stdout,stderr = []
132
+ while stdout.nil? and stderr.nil? and retries < Gdrive.max_file_write_retries
133
+ begin
134
+ #get tsv to write from stage
135
+ source = s.sources(gdrive_slot).first
136
+ raise "Need source for gsheet write" unless source
137
+ tsv = source.read(u.name,gdrive_slot)
138
+ raise "No data source found for #{source.url}" unless tsv
139
+ tsv_row_count = tsv.to_s.split("\n").length
140
+ tsv_col_count = tsv.to_s.split("\n").first.to_s.split("\t").length
141
+ tsv_cell_count = tsv_row_count * tsv_col_count
142
+ if tsv_cell_count > Gsheet.max_cells
143
+ raise "Too many datapoints; you have #{tsv_cell_count.to_s}, max is #{Gsheet.max_cells.to_s}"
144
+ end
145
+ stdout = if tsv_row_count == 0
146
+ #soft error; no data to write. Stage will complete.
147
+ "Write skipped for #{s.target.url}"
148
+ else
149
+ Dataset.write_by_url(s.target.url,tsv,u.name,gdrive_slot,crop)
150
+ #update status
151
+ "Write successful for #{s.target.url}"
152
+ end
153
+ Gdrive.unslot_worker_by_path(stage_path)
154
+ stderr = nil
155
+ s.update_status(stdout)
156
+ signal = 0
157
+ rescue => exc
158
+ if retries < Gdrive.max_file_write_retries
159
+ retries +=1
160
+ sleep Gdrive.file_write_retry_delay
161
+ else
162
+ stdout = nil
163
+ stderr = [exc.to_s,"\n",exc.backtrace.join("\n")].join
164
+ signal = 500
165
+ end
166
+ end
157
167
  end
158
168
  return {'out_str'=>stdout, 'err_str'=>stderr, 'signal' => signal}
159
169
  end
@@ -148,10 +148,7 @@ module Mobilize
148
148
 
149
149
  def Resque.start_workers(count=1)
150
150
  count.times do
151
- dir_envs = "MOBILIZE_ENV=#{Base.env} " +
152
- "MOBILIZE_CONFIG_DIR=#{Base.config_dir} " +
153
- "MOBILIZE_LOG_DIR=#{Base.log_dir}"
154
- "(cd #{Base.root};rake #{dir_envs} mobilize_base:work) >> #{Resque.log_path} 2>&1 &".bash
151
+ "(cd #{Base.root};rake mobilize:work[#{Base.env}]) >> #{Resque.log_path} 2>&1 &".bash
155
152
  end
156
153
  end
157
154
 
@@ -15,12 +15,32 @@ module Mobilize
15
15
  return j
16
16
  end
17
17
 
18
+ def parent
19
+ j = self
20
+ u = j.runner.user
21
+ if j.trigger.strip[0..4].downcase == "after"
22
+ parent_name = j.trigger[5..-1].to_s.strip
23
+ parent_j = u.jobs.select{|job| job.name == parent_name}.first
24
+ return parent_j
25
+ else
26
+ return nil
27
+ end
28
+ end
29
+
30
+ def children
31
+ j = self
32
+ u = j.runner.user
33
+ u.jobs.select do |job|
34
+ parent_name = job.trigger[5..-1].to_s.strip
35
+ job.trigger.strip[0..4].downcase == "after" and
36
+ parent_name == j.name
37
+ end
38
+ end
39
+
18
40
  #takes a hash of job parameters (name, active, trigger, stages)
19
41
  #and creates/updates a job with it
20
- def Job.update_by_user_name_and_hash(user_name,hash)
21
- u = User.where(name: user_name).first
22
- r = u.runner
23
- j = Job.find_or_create_by_path("#{r.path}/#{hash['name']}")
42
+ def update_from_hash(hash)
43
+ j = self
24
44
  #update top line params
25
45
  j.update_attributes(:active => hash['active'],
26
46
  :trigger => hash['trigger'])
@@ -54,59 +74,82 @@ module Mobilize
54
74
 
55
75
  def is_due?
56
76
  j = self
57
- return false if j.is_working? or j.active == false or j.trigger.to_s.starts_with?("after")
58
- last_run = j.completed_at
59
- #check trigger
60
- trigger = j.trigger
61
- return true if trigger == 'once'
62
- #strip the "every" from the front if present
63
- trigger = trigger.gsub("every","").gsub("."," ").strip
64
- value,unit,operator,job_utctime = trigger.split(" ")
65
- curr_utctime = Time.now.utc
66
- curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
67
- if job_utctime
68
- job_utctime = job_utctime.split(" ").first
69
- job_utctime = Time.parse([curr_utcdate,job_utctime,"UTC"].join(" "))
77
+ #working or inactive jobs are not due
78
+ if j.is_working? or j.active == false
79
+ return false
80
+ end
81
+
82
+ #if job contains handlers not loaded by jobtracker, not due
83
+ loaded_handlers = Jobtracker.config['extensions'].map{|m| m.split("-").last}
84
+ job_handlers = j.stages.map{|s| s.handler}.uniq
85
+ #base handlers are the ones in mobilize-base/handlers
86
+ if (job_handlers - loaded_handlers - Base.handlers).length>0
87
+ return false
88
+ end
89
+
90
+ #once
91
+ if j.trigger.strip.downcase=='once'
92
+ #active and once means due
93
+ return true
70
94
  end
71
- #after is the only operator
72
- raise "Unknown #{operator.to_s} operator" if operator and operator != "after"
73
- if ["hour","hours"].include?(unit)
74
- #if it's later than the last run + hour tolerance, is due
75
- if last_run.nil? or curr_utctime > (last_run + value.to_i.hour)
95
+
96
+ #depedencies
97
+ if j.parent
98
+ #if parent is not working and completed more recently than self, is due
99
+ if !j.parent.is_working? and
100
+ j.parent.completed_at and (j.completed_at.nil? or j.parent.completed_at > j.completed_at)
76
101
  return true
102
+ else
103
+ return false
77
104
  end
78
- elsif ["day","days"].include?(unit)
79
- if last_run.nil? or curr_utctime.to_date >= (last_run.to_date + value.to_i.day)
80
- if operator and job_utctime
81
- if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
82
- return true
83
- end
84
- elsif operator || job_utctime
85
- raise "Please specify both an operator and a time in UTC, or neither"
86
- else
105
+ end
106
+
107
+ #time based
108
+ last_comp_time = j.completed_at
109
+ #check trigger; strip the "every" from the front if present, change dot to space
110
+ trigger = j.trigger.strip.gsub("every","").gsub("."," ").strip
111
+ number, unit, operator, mark = trigger.split(" ").map{|t_node| t_node.downcase}
112
+ #operator is not used
113
+ operator = nil
114
+ #get time for time-based evaluations
115
+ curr_time = Time.now.utc
116
+ if ["hour","hours","day","days"].include?(unit)
117
+ if mark
118
+ last_mark_time = Time.at_marks_ago(number,unit,mark)
119
+ if last_comp_time.nil? or last_comp_time < last_mark_time
87
120
  return true
121
+ else
122
+ return false
88
123
  end
124
+ elsif last_comp_time.nil? or last_comp_time < (curr_time - number.to_i.send(unit))
125
+ return true
126
+ else
127
+ return false
89
128
  end
90
129
  elsif unit == "day_of_week"
91
- if curr_utctime.wday==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
92
- if operator and job_utctime
93
- if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
130
+ if curr_time.wday==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
131
+ if mark
132
+ #check if it already ran today
133
+ last_mark_time = Time.at_marks_ago(1,"day",mark)
134
+ if last_comp_time < last_mark_time
94
135
  return true
136
+ else
137
+ return false
95
138
  end
96
- elsif operator || job_utctime
97
- raise "Please specify both an operator and a time in UTC, or neither"
98
139
  else
99
140
  return true
100
141
  end
101
142
  end
102
143
  elsif unit == "day_of_month"
103
- if curr_utctime.day==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
104
- if operator and job_utctime
105
- if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
144
+ if curr_time.day==number and (last_comp_time.nil? or last_comp_time.to_date != curr_time.to_date)
145
+ if mark
146
+ #check if it already ran today
147
+ last_mark_time = Time.at_marks_ago(1,"day",mark)
148
+ if last_comp_time < last_mark_time
106
149
  return true
150
+ else
151
+ return false
107
152
  end
108
- elsif operator || job_utctime
109
- raise "Please specify both an operator and a time in UTC, or neither"
110
153
  else
111
154
  return true
112
155
  end
@@ -50,8 +50,7 @@ module Mobilize
50
50
  s.enqueue!
51
51
  end
52
52
  rescue ScriptError, StandardError => exc
53
- r.update_status("Failed to enqueue #{j.path} with #{exc.to_s}")
54
- j.update_attributes(:active=>false)
53
+ r.update_status("Failed to enqueue #{j.path}")
55
54
  end
56
55
  end
57
56
  r.update_gsheet(gdrive_slot)
@@ -74,7 +73,9 @@ module Mobilize
74
73
  gsheet_hashes.each do |gsheet_hash|
75
74
  #skip non-jobs or jobs without required values
76
75
  next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
77
- j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
76
+ #find job w this name, or make one
77
+ j = r.jobs.select{|rj| rj.name == gsheet_hash['name']}.first || Job.find_or_create_by_path("#{r.path}/#{gsheet_hash['name']}")
78
+ j.update_from_hash(gsheet_hash)
78
79
  r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
79
80
  #add this job to list of read ones
80
81
  done_jobs << j
@@ -95,7 +96,15 @@ module Mobilize
95
96
  return false unless r.completed_at
96
97
  jobs_gsheet = r.gsheet(gdrive_slot)
97
98
  upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
98
- upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
99
+ upd_rows = upd_rows = upd_jobs.map do |j|
100
+ uj = {'name'=>j.name, 'status'=>j.status}
101
+ #jobs can only be turned off
102
+ #automatically, not back on
103
+ if j.active==false
104
+ uj['active'] = false
105
+ end
106
+ uj
107
+ end
99
108
  jobs_gsheet.add_or_update_rows(upd_rows)
100
109
  r.update_status("gsheet updated")
101
110
  return true
@@ -82,8 +82,6 @@ module Mobilize
82
82
 
83
83
  def complete(response)
84
84
  s = self
85
- s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
86
- s.update_status("Completed at #{Time.now.utc.to_s}")
87
85
  j = s.job
88
86
  if s.idx == j.stages.length
89
87
  #check for any dependent jobs, if there are, enqueue them
@@ -110,6 +108,10 @@ module Mobilize
110
108
  s.next.update_attributes(:retries_done=>0)
111
109
  s.next.enqueue!
112
110
  end
111
+ #complete after dependencies are processed
112
+ #to make sure it doesn't enqueue due to runner check
113
+ s.update_attributes(:completed_at=>Time.now.utc,:response=>response)
114
+ s.update_status("Completed at #{Time.now.utc.to_s}")
113
115
  true
114
116
  end
115
117
 
@@ -1,6 +1,7 @@
1
- namespace :mobilize_base do
1
+ namespace :mobilize do
2
2
  desc "Start a Resque worker"
3
- task :work do
3
+ task :work, :env do |t,args|
4
+ ENV['MOBILIZE_ENV']=args.env
4
5
  require 'mobilize-base'
5
6
  Mobilize::Base.config('jobtracker')['extensions'].each do |e|
6
7
  begin
@@ -20,42 +21,77 @@ namespace :mobilize_base do
20
21
  worker.work(ENV['INTERVAL'] || 5) # interval, will block
21
22
  end
22
23
  desc "Kill all Resque workers"
23
- task :kill_workers do
24
+ task :kill_workers, :env do |t,args|
25
+ ENV['MOBILIZE_ENV']=args.env
24
26
  require 'mobilize-base'
25
27
  Mobilize::Jobtracker.kill_workers
26
28
  end
27
29
  desc "Kill idle workers not in sync with repo"
28
- task :kill_idle_and_stale_workers do
30
+ task :kill_idle_and_stale_workers, :env do |t,args|
31
+ ENV['MOBILIZE_ENV']=args.env
29
32
  require 'mobilize-base'
30
33
  Mobilize::Jobtracker.kill_idle_and_stale_workers
31
34
  end
32
- desc "Kill idle workers"
33
- task :kill_idle_workers do
35
+ desc "Kill idle workers"
36
+ task :kill_idle_workers, :env do |t,args|
37
+ ENV['MOBILIZE_ENV']=args.env
34
38
  require 'mobilize-base'
35
39
  Mobilize::Jobtracker.kill_idle_workers
36
40
  end
37
41
  desc "Make sure there are the correct # of workers, kill if too many"
38
- task :prep_workers do
42
+ task :prep_workers, :env do |t,args|
43
+ ENV['MOBILIZE_ENV']=args.env
39
44
  require 'mobilize-base'
40
45
  Mobilize::Jobtracker.prep_workers
41
46
  end
47
+ desc "Restart Resque workers"
48
+ task :restart_workers, :env do |t,args|
49
+ ENV['MOBILIZE_ENV']=args.env
50
+ require 'mobilize-base'
51
+ Mobilize::Jobtracker.kill_workers
52
+ sleep 5
53
+ Mobilize::Jobtracker.prep_workers
54
+ end
42
55
  desc "Stop Jobtracker"
43
- task :stop_jobtracker do
56
+ task :stop_jobtracker, :env do |t,args|
57
+ ENV['MOBILIZE_ENV']=args.env
44
58
  require 'mobilize-base'
45
59
  Mobilize::Jobtracker.stop!
46
60
  end
47
61
  desc "Start Jobtracker"
48
- task :start_jobtracker do
62
+ task :start_jobtracker, :env do |t,args|
63
+ ENV['MOBILIZE_ENV']=args.env
49
64
  require 'mobilize-base'
50
65
  Mobilize::Jobtracker.start
51
66
  end
52
67
  desc "Restart Jobtracker"
53
- task :restart_jobtracker do
68
+ task :restart_jobtracker, :env do |t,args|
69
+ ENV['MOBILIZE_ENV']=args.env
54
70
  require 'mobilize-base'
55
71
  Mobilize::Jobtracker.restart!
56
72
  end
73
+ desc "Add a user"
74
+ task :add_user, :name, :env do |t,args|
75
+ ENV['MOBILIZE_ENV']=args.env
76
+ require 'mobilize-base'
77
+ Mobilize::User.find_or_create_by_name(args.name)
78
+ end
79
+ desc "Enqueue a user's runner"
80
+ task :enqueue_user, :name, :env do |t,args|
81
+ ENV['MOBILIZE_ENV']=args.env
82
+ require 'mobilize-base'
83
+ Mobilize::User.where(name: args.name).first.runner.enqueue!
84
+ end
85
+ desc "Enqueue a stage"
86
+ task :enqueue_stage, :path, :env do |t,args|
87
+ ENV['MOBILIZE_ENV']=args.env
88
+ require 'mobilize-base'
89
+ user,job,stage = args.path.split("/")
90
+ Mobilize::Stage.where(path: "Runner_#{user}/jobs/#{job}/#{stage}").first.en
91
+ end
57
92
  desc "kill all old resque web processes, start new one with resque_web.rb extension file"
58
- task :resque_web do
93
+ task :resque_web, :env do |t,args|
94
+ ENV['MOBILIZE_ENV']=args.env
59
95
  require 'mobilize-base'
60
96
  port = Mobilize::Base.config('resque')['web_port']
61
97
  config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
@@ -83,14 +119,15 @@ namespace :mobilize_base do
83
119
  `#{command}`
84
120
  end
85
121
  desc "create indexes for all base models in mongodb"
86
- task :create_indexes do
122
+ task :create_indexes, :env do |t,args|
123
+ ENV['MOBILIZE_ENV']=args.env
87
124
  require 'mobilize-base'
88
125
  ["Dataset","Job","Runner","Task","User"].each do |m|
89
126
  "Mobilize::#{m}".constantize.create_indexes
90
127
  end
91
128
  end
92
129
  desc "Set up config and log folders and files, populate from samples"
93
- task :setup do
130
+ task :setup_base do
94
131
  config_dir = (ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/")
95
132
  log_dir = (ENV['MOBILIZE_LOG_DIR'] ||= "log/")
96
133
  sample_dir = File.dirname(__FILE__) + '/../samples/'
@@ -1,5 +1,5 @@
1
1
  module Mobilize
2
2
  module Base
3
- VERSION = "1.351"
3
+ VERSION = "1.361"
4
4
  end
5
5
  end
data/lib/mobilize-base.rb CHANGED
@@ -3,6 +3,7 @@ require "mobilize-base/extensions/array"
3
3
  require "mobilize-base/extensions/hash"
4
4
  require "mobilize-base/extensions/object"
5
5
  require "mobilize-base/extensions/string"
6
+ require "mobilize-base/extensions/time"
6
7
  require "mobilize-base/extensions/yaml"
7
8
  #this is the base of the mobilize object, any methods that should be
8
9
  #made available application-wide go over here
@@ -16,6 +17,9 @@ module Mobilize
16
17
  ENV['PWD']
17
18
  end
18
19
  end
20
+ def Base.home_dir
21
+ File.expand_path('..',File.dirname(__FILE__))
22
+ end
19
23
  def Base.config_dir
20
24
  ENV['MOBILIZE_CONFIG_DIR'] ||= "config/mobilize/"
21
25
  end
@@ -56,6 +60,9 @@ module Mobilize
56
60
  raise "Could not find #{log_dir} folder for logs"
57
61
  end
58
62
  end
63
+ def Base.handlers
64
+ Dir.entries(File.dirname(__FILE__) + "/mobilize-base/handlers").select{|e| e.ends_with?(".rb")}.map{|e| e.split(".").first}
65
+ end
59
66
  end
60
67
  end
61
68
  mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
@@ -1,6 +1,9 @@
1
1
  ---
2
2
  development:
3
3
  domain: host.com
4
+ max_api_retries: 10
5
+ file_write_retries: 5
6
+ file_write_retry_delay: 30
4
7
  owner:
5
8
  name: owner_development
6
9
  pw: google_drive_password
@@ -15,6 +18,9 @@ development:
15
18
  pw: worker002_google_drive_password
16
19
  test:
17
20
  domain: host.com
21
+ max_api_retries: 10
22
+ file_write_retries: 5
23
+ file_write_retry_delay: 30
18
24
  owner:
19
25
  name: owner_test
20
26
  pw: google_drive_password
@@ -29,6 +35,9 @@ test:
29
35
  pw: worker002_google_drive_password
30
36
  production:
31
37
  domain: host.com
38
+ max_api_retries: 10
39
+ file_write_retries: 5
40
+ file_write_retry_delay: 30
32
41
  owner:
33
42
  name: owner_production
34
43
  pw: google_drive_password
@@ -0,0 +1,9 @@
1
+ ---
2
+ development:
3
+ max_length: 100000000 #about 100MB
4
+ test:
5
+ max_length: 100000000 #about 100MB
6
+ staging:
7
+ max_length: 100000000 #about 100MB
8
+ production:
9
+ max_length: 100000000 #about 100MB
@@ -0,0 +1,10 @@
1
+ ---
2
+ - date: 2013-01-01
3
+ snake_date: 2013-02-01
4
+ camelDate: 2013-03-01
5
+ - date: 2013-01-01
6
+ snake_date: 2013-02-02
7
+ camelDate: 2013-03-02
8
+ - date: 2013-01-01
9
+ snake_date: 2013-02-03
10
+ camelDate: 2013-03-03