mobilize-base 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +20 -0
- data/README.md +509 -0
- data/Rakefile +34 -0
- data/lib/mobilize-base/extensions/array.rb +22 -0
- data/lib/mobilize-base/extensions/google_drive.rb +296 -0
- data/lib/mobilize-base/extensions/hash.rb +86 -0
- data/lib/mobilize-base/extensions/object.rb +6 -0
- data/lib/mobilize-base/extensions/resque.rb +180 -0
- data/lib/mobilize-base/extensions/string.rb +94 -0
- data/lib/mobilize-base/handlers/emailer.rb +24 -0
- data/lib/mobilize-base/handlers/gdriver.rb +309 -0
- data/lib/mobilize-base/handlers/mongoer.rb +32 -0
- data/lib/mobilize-base/jobtracker.rb +208 -0
- data/lib/mobilize-base/models/dataset.rb +70 -0
- data/lib/mobilize-base/models/job.rb +253 -0
- data/lib/mobilize-base/models/requestor.rb +223 -0
- data/lib/mobilize-base/tasks/mobilize-base.rake +2 -0
- data/lib/mobilize-base/tasks.rb +43 -0
- data/lib/mobilize-base/version.rb +5 -0
- data/lib/mobilize-base.rb +76 -0
- data/lib/samples/gdrive.yml +27 -0
- data/lib/samples/jobtracker.yml +24 -0
- data/lib/samples/mongoid.yml +21 -0
- data/lib/samples/resque.yml +12 -0
- data/mobilize-base.gemspec +35 -0
- data/test/mobilize_test.rb +125 -0
- data/test/redis-test.conf +540 -0
- data/test/test_helper.rb +23 -0
- metadata +260 -0
@@ -0,0 +1,253 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class Job
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :requestor_id, type: String
|
6
|
+
field :name, type: String
|
7
|
+
field :active, type: Boolean #active, inactive
|
8
|
+
field :schedule, type: String
|
9
|
+
field :active_task, type: String
|
10
|
+
field :tasks, type: Hash
|
11
|
+
field :status, type: String
|
12
|
+
field :last_error, type: String
|
13
|
+
field :last_trace, type: String
|
14
|
+
field :last_completed_at, type: Time
|
15
|
+
field :read_handler, type: String
|
16
|
+
field :write_handler, type: String
|
17
|
+
field :param_source, type: String #name of sheet on doc
|
18
|
+
field :param_hash, type: String #JSON
|
19
|
+
field :destination, type: String #output destination - could be file, could be sheet
|
20
|
+
|
21
|
+
index({ requestor_id: 1})
|
22
|
+
index({ name: 1})
|
23
|
+
|
24
|
+
before_destroy :destroy_output_dst_ids
|
25
|
+
|
26
|
+
def worker
|
27
|
+
j = self
|
28
|
+
Mobilize::Resque.find_worker_by_mongo_id(j.id.to_s)
|
29
|
+
end
|
30
|
+
|
31
|
+
def Job.find_by_name(name)
|
32
|
+
Job.where(:name=>name).first
|
33
|
+
end
|
34
|
+
|
35
|
+
def Job.find_all_by_requestor_id(requestor_id)
|
36
|
+
Job.where(:requestor_id=>requestor_id).to_a
|
37
|
+
end
|
38
|
+
|
39
|
+
def Job.find_or_create_by_requestor_id_and_name(requestor_id,name)
|
40
|
+
j = Job.where(:requestor_id=>requestor_id, :name=>name).first
|
41
|
+
j = Job.create(:requestor_id=>requestor_id, :name=>name) unless j
|
42
|
+
return j
|
43
|
+
end
|
44
|
+
|
45
|
+
#called by Resque
|
46
|
+
def Job.perform(id,*args)
|
47
|
+
j = Job.find(id)
|
48
|
+
task_params = j.tasks[j.active_task]
|
49
|
+
begin
|
50
|
+
j.update_status(%{Starting #{j.active_task} task at #{Time.now.utc}})
|
51
|
+
task_output = "Mobilize::#{task_params['handler'].humanize}".constantize.send(j.active_task,id)
|
52
|
+
#this allows user to return false if the stage didn't go as expected and needs to retry
|
53
|
+
#e.g. tried to write to Google but all the accounts were in use
|
54
|
+
return false if task_output == false
|
55
|
+
task_output_dst_id = if task_output.to_s.length==24 and Dataset.find(task_output.to_s)
|
56
|
+
#user has returned dst as output from task
|
57
|
+
task_output
|
58
|
+
else
|
59
|
+
#store the output in a cache
|
60
|
+
dst = Dataset.find_or_create_by_requestor_id_and_handler_and_name(j.requestor.id.to_s,'mongoer',"#{j.id.to_s}/#{j.active_task}")
|
61
|
+
dst.write_cache(task_output.to_s)
|
62
|
+
dst.id.to_s
|
63
|
+
end
|
64
|
+
j.tasks[j.active_task]['output_dst_id'] = task_output_dst_id
|
65
|
+
if j.active_task == j.tasks.keys.last
|
66
|
+
j.active_task = nil
|
67
|
+
j.last_error = ""
|
68
|
+
j.last_trace = ""
|
69
|
+
j.last_completed_at = Time.now.utc
|
70
|
+
j.status = %{Completed all tasks at #{Time.now.utc}}
|
71
|
+
j.save!
|
72
|
+
#check for any dependent jobs, if there are, enqueue them
|
73
|
+
r = j.requestor
|
74
|
+
dep_jobs = Job.where(:active=>true, :requestor_id=>r.id.to_s, :schedule=>"after #{j.name}").to_a
|
75
|
+
dep_jobs += Job.where(:active=>true, :schedule=>"after #{r.name}/#{j.name}").to_a
|
76
|
+
#put begin/rescue so all dependencies run
|
77
|
+
dep_jobs.each{|dj| begin;dj.enqueue! unless dj.is_working?;rescue;end}
|
78
|
+
else
|
79
|
+
task_names = j.tasks.keys
|
80
|
+
stage_idx = task_names.index(j.active_task) + 1
|
81
|
+
j.active_task = j.tasks.keys[stage_idx]
|
82
|
+
j.save!
|
83
|
+
#queue up next task
|
84
|
+
j.enqueue!
|
85
|
+
end
|
86
|
+
rescue ScriptError,StandardError => exc
|
87
|
+
#record the failure in Job so it appears on spec sheets
|
88
|
+
j.status='failed'
|
89
|
+
j.save!
|
90
|
+
j.update_status("Failed at #{Time.now.utc.to_s}")
|
91
|
+
j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
|
92
|
+
[exc.to_s,exc.backtrace.to_s].join("=>").oputs
|
93
|
+
#raising here will cause the failure to show on the Resque UI
|
94
|
+
raise exc
|
95
|
+
end
|
96
|
+
return true
|
97
|
+
end
|
98
|
+
|
99
|
+
def enqueue!
|
100
|
+
j = self
|
101
|
+
#assign first task if none assigned
|
102
|
+
if j.tasks.blank?
|
103
|
+
#make a hash with the read/write tasks
|
104
|
+
j.update_attributes(:tasks=>{"read_by_job_id"=>{'handler'=>j.read_handler},
|
105
|
+
"write_by_job_id"=>{'handler'=>j.write_handler}})
|
106
|
+
end
|
107
|
+
j.update_attributes(:active_task=>"read_by_job_id") if j.active_task.blank?
|
108
|
+
::Resque::Job.create("mobilize",Job,j.id.to_s,%{#{j.requestor.name}=>#{j.name}})
|
109
|
+
return true
|
110
|
+
end
|
111
|
+
|
112
|
+
#convenience methods
|
113
|
+
def requestor
|
114
|
+
j = self
|
115
|
+
return Requestor.find(j.requestor_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def restart
|
119
|
+
j = self
|
120
|
+
j.update_attributes(:last_completed_at=>nil)
|
121
|
+
return true
|
122
|
+
end
|
123
|
+
|
124
|
+
def prior_task
|
125
|
+
j = self
|
126
|
+
return nil if j.active_task.nil?
|
127
|
+
task_idx = j.tasks.keys.index(j.active_task)
|
128
|
+
return nil if task_idx==0
|
129
|
+
return j.tasks.keys[task_idx-1]
|
130
|
+
end
|
131
|
+
|
132
|
+
def destination_url
|
133
|
+
j = self
|
134
|
+
return nil if j.destination.nil?
|
135
|
+
destination = j.destination
|
136
|
+
dst = if j.write_handler == 'gsheet'
|
137
|
+
destination = [j.requestor.jobspec_title,j.destination].join("/") if destination.split("/").length==1
|
138
|
+
Dataset.find_by_handler_and_name('gsheeter',destination)
|
139
|
+
elsif j.write_handler == 'gtxt'
|
140
|
+
#all gtxt files must end in gz
|
141
|
+
destination += ".gz" unless destination.ends_with?(".gz")
|
142
|
+
destination = [s.requestor.name,"_"].join + destination unless destination.starts_with?([s.requestor.name,"_"].join)
|
143
|
+
Dataset.find_by_handler_and_name('gtxter',destination)
|
144
|
+
end
|
145
|
+
return dst.url if dst
|
146
|
+
end
|
147
|
+
|
148
|
+
def worker_args
|
149
|
+
j = self
|
150
|
+
Jobtracker.get_worker_args(j.worker)
|
151
|
+
end
|
152
|
+
|
153
|
+
def set_worker_args(args)
|
154
|
+
j = self
|
155
|
+
Jobtracker.set_worker_args(j.worker,args)
|
156
|
+
end
|
157
|
+
|
158
|
+
def cache_params
|
159
|
+
j = self
|
160
|
+
#go to paramsheet and read
|
161
|
+
param_path = if j.paramsheet.split("/").length==1
|
162
|
+
[j.requestor.jobspec_title,j.paramsheet].join("/")
|
163
|
+
else
|
164
|
+
j.paramsheet
|
165
|
+
end
|
166
|
+
param_sheet = j.requestor.find_or_create_gsheet_by_path(param_path)
|
167
|
+
param_tsv = param_sheet.to_tsv
|
168
|
+
param_dst = j.requestor.gsheets.select{|s| s.path == param_sheet.path}
|
169
|
+
param_dst.cache.write(param_tsv)
|
170
|
+
s.update_attributes(:param_dst_id=>paramdst.id.to_s)
|
171
|
+
(s.requestor.name + "'s #{s.name} params cached").oputs
|
172
|
+
return true
|
173
|
+
end
|
174
|
+
|
175
|
+
def update_status(msg)
|
176
|
+
j = self
|
177
|
+
j.update_attributes(:status=>msg)
|
178
|
+
Mobilize::Resque.update_job_status(j.id.to_s,msg)
|
179
|
+
return true
|
180
|
+
end
|
181
|
+
|
182
|
+
def is_working?
|
183
|
+
j = self
|
184
|
+
Mobilize::Resque.active_mongo_ids.include?(j.id.to_s)
|
185
|
+
end
|
186
|
+
|
187
|
+
def is_due?
|
188
|
+
j = self
|
189
|
+
return false if j.is_working? or j.schedule.to_s.starts_with?("after")
|
190
|
+
last_run = j.last_completed_at
|
191
|
+
#check schedule
|
192
|
+
schedule = j.schedule
|
193
|
+
return true if schedule == 'once' and j.active
|
194
|
+
#strip the "every" from the front if present
|
195
|
+
schedule = schedule.gsub("every","").gsub("."," ").strip
|
196
|
+
value,unit,operator,job_utctime = schedule.split(" ")
|
197
|
+
curr_utctime = Time.now.utc
|
198
|
+
curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
|
199
|
+
if job_utctime
|
200
|
+
job_utctime = job_utctime.split(" ").first
|
201
|
+
job_utctime = Time.parse([curr_utcdate,job_utctime,"UTC"].join(" "))
|
202
|
+
end
|
203
|
+
#after is the only operator
|
204
|
+
raise "Unknown #{operator.to_s} operator" if operator and operator != "after"
|
205
|
+
if ["hour","hours"].include?(unit)
|
206
|
+
#if it's later than the last run + hour tolerance, is due
|
207
|
+
if last_run.nil? or curr_utctime > (last_run + value.to_i.hour)
|
208
|
+
return true
|
209
|
+
end
|
210
|
+
elsif ["day","days"].include?(unit)
|
211
|
+
if last_run.nil? or curr_utctime.to_date >= (last_run.to_date + value.to_i.day)
|
212
|
+
if operator and job_utctime
|
213
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
214
|
+
return true
|
215
|
+
end
|
216
|
+
elsif operator || job_utctime
|
217
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
218
|
+
else
|
219
|
+
return true
|
220
|
+
end
|
221
|
+
end
|
222
|
+
elsif unit == "day_of_week"
|
223
|
+
if curr_utctime.wday==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
|
224
|
+
if operator and job_utctime
|
225
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
226
|
+
return true
|
227
|
+
end
|
228
|
+
elsif operator || job_utctime
|
229
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
230
|
+
else
|
231
|
+
return true
|
232
|
+
end
|
233
|
+
end
|
234
|
+
elsif unit == "day_of_month"
|
235
|
+
if curr_utctime.day==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
|
236
|
+
if operator and job_utctime
|
237
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
238
|
+
return true
|
239
|
+
end
|
240
|
+
elsif operator || job_utctime
|
241
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
242
|
+
else
|
243
|
+
return true
|
244
|
+
end
|
245
|
+
end
|
246
|
+
else
|
247
|
+
raise "Unknown #{unit.to_s} time unit"
|
248
|
+
end
|
249
|
+
#if nothing happens, return false
|
250
|
+
return false
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
@@ -0,0 +1,223 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class Requestor
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :email, type: String
|
6
|
+
field :oauth, type: String
|
7
|
+
field :name, type: String
|
8
|
+
field :first_name, type: String
|
9
|
+
field :last_name, type: String
|
10
|
+
field :admin_role, type: String
|
11
|
+
field :last_run, type: Time
|
12
|
+
field :status, type: String
|
13
|
+
|
14
|
+
validates_presence_of :name, :message => ' cannot be blank.'
|
15
|
+
validates_uniqueness_of :name, :message => ' has already been used.'
|
16
|
+
|
17
|
+
before_destroy :destroy_jobs
|
18
|
+
|
19
|
+
def Requestor.find_or_create_by_name(name)
|
20
|
+
r = Requestor.where(:name => name).first
|
21
|
+
r = Requestor.create(:name => name) unless r
|
22
|
+
return r
|
23
|
+
end
|
24
|
+
|
25
|
+
def Requestor.find_or_create_by_email(email)
|
26
|
+
r = Requestor.where(:email => email).first
|
27
|
+
r = Requestor.create(:email => email) unless r
|
28
|
+
user_name = email.split("@").first
|
29
|
+
r.update_attributes(:name => user_name) unless r.name.to_s.length>0
|
30
|
+
return r
|
31
|
+
end
|
32
|
+
|
33
|
+
def Requestor.jobs_sheet_headers
|
34
|
+
%w{name active schedule status last_error destination_url read_handler write_handler param_source params destination}
|
35
|
+
end
|
36
|
+
|
37
|
+
def Requestor.perform(id,*args)
|
38
|
+
r = Requestor.find(id.to_s)
|
39
|
+
#reserve email account for read
|
40
|
+
gdrive_email = Gdriver.get_worker_email_by_mongo_id(id)
|
41
|
+
return false unless gdrive_email
|
42
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
43
|
+
#write headers to sheet
|
44
|
+
Requestor.jobs_sheet_headers.each_with_index do |h,h_i|
|
45
|
+
jobs_sheet[1,h_i+1] = h
|
46
|
+
end
|
47
|
+
jobs_sheet.save
|
48
|
+
#read the jobs sheet
|
49
|
+
#record jobs in DB
|
50
|
+
#deactivate jobs not in sheet
|
51
|
+
r.read_jobs(gdrive_email)
|
52
|
+
#queue up the jobs that are due and active
|
53
|
+
r.jobs.each do |j|
|
54
|
+
begin
|
55
|
+
j.enqueue! if j.active and j.is_due?
|
56
|
+
rescue ScriptError,StandardError => exc
|
57
|
+
#update errors
|
58
|
+
j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
#write any updates to status, error, datasource_url etc.
|
62
|
+
r.write_jobs(gdrive_email)
|
63
|
+
r.update_attributes(:last_run=>Time.now.utc)
|
64
|
+
end
|
65
|
+
|
66
|
+
def jobs_sheet(gdrive_email)#gdrive_email to read with
|
67
|
+
r = self
|
68
|
+
r.find_or_create_gbook_by_title(r.jobspec_title,gdrive_email)
|
69
|
+
jobs_name = [r.jobspec_title,"Jobs"].join("/")
|
70
|
+
r.find_or_create_gsheet_by_name(jobs_name,gdrive_email)
|
71
|
+
end
|
72
|
+
|
73
|
+
def read_jobs(gdrive_email)
|
74
|
+
r = self
|
75
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
76
|
+
rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
|
77
|
+
#go through each job, update relevant job with its params
|
78
|
+
loc_jobs = []
|
79
|
+
rem_jobs.each_with_index do |rj,rj_i|
|
80
|
+
#skip bad rows
|
81
|
+
next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
|
82
|
+
j = Job.find_or_create_by_requestor_id_and_name(r.id.to_s,rj['name'])
|
83
|
+
#update top line params
|
84
|
+
j.update_attributes(:active => rj['active'],
|
85
|
+
:schedule => rj['schedule'],
|
86
|
+
:read_handler => rj['read_handler'],
|
87
|
+
:write_handler => rj['write_handler'],
|
88
|
+
:param_source => rj['param_source'],
|
89
|
+
:params => rj['params'],
|
90
|
+
:destination => rj['destination'])
|
91
|
+
#update laststatus with "Created job for" if job is due
|
92
|
+
j.update_status("Due and active at #{Time.now.utc}") if j.is_due? and j.active
|
93
|
+
#add this job to list of local ones
|
94
|
+
loc_jobs << j
|
95
|
+
end
|
96
|
+
#deactivate requestor jobs that are not included in sheet;
|
97
|
+
#this makes sure we don't run obsolete jobs
|
98
|
+
(r.jobs.map{|j| j.id.to_s} - loc_jobs.map{|j| j.id.to_s}).each do |rjid|
|
99
|
+
j = Job.find(rjid)
|
100
|
+
if j.active
|
101
|
+
j.update_attributes(:active=>false)
|
102
|
+
r.update_status("Deactivated job:#{r.name}=>#{j.name}")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
r.update_status(r.name + " jobs read at #{Time.now.utc}")
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
|
109
|
+
def write_jobs(gdrive_email) #gdrive_email to update with
|
110
|
+
r = self
|
111
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
112
|
+
rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
|
113
|
+
#go through each job, update relevant job with its params
|
114
|
+
headers = Requestor.jobs_sheet_headers
|
115
|
+
#write headers
|
116
|
+
headers.each_with_index do |h,h_i|
|
117
|
+
jobs_sheet[1,h_i+1] = h
|
118
|
+
end
|
119
|
+
#write rows
|
120
|
+
rem_jobs.each_with_index do |rj,rj_i|
|
121
|
+
#skip bad rows
|
122
|
+
next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
|
123
|
+
j = r.jobs(rj['name'])
|
124
|
+
#update active to false if this was a run once
|
125
|
+
j.update_attributes(:active=>false) if j.schedule.to_s == 'once'
|
126
|
+
jobs_sheet[rj_i+2,headers.index('active')+1] = j.active.to_s
|
127
|
+
jobs_sheet[rj_i+2,headers.index('status')+1] = j.status.to_s.gsub("\n",";").gsub("\t"," ")
|
128
|
+
jobs_sheet[rj_i+2,headers.index('last_error')+1] = j.last_error.to_s.gsub("\n",";").gsub("\t"," ")
|
129
|
+
jobs_sheet[rj_i+2,headers.index('destination_url')+1] = j.destination_url.to_s
|
130
|
+
end
|
131
|
+
jobs_sheet.save
|
132
|
+
r.update_status(r.name + " jobs written")
|
133
|
+
return true
|
134
|
+
end
|
135
|
+
|
136
|
+
def jobspec_title
|
137
|
+
r = self
|
138
|
+
prefix = "Jobspec_"
|
139
|
+
suffix = ""
|
140
|
+
if Mobilize::Base.env == 'development'
|
141
|
+
suffix = "_dev"
|
142
|
+
elsif Mobilize::Base.env == 'test' or Mobilize::Base.env == 'pry_dev'
|
143
|
+
suffix = "_test"
|
144
|
+
elsif Mobilize::Base.env == 'production' or Mobilize::Base.env == 'integration'
|
145
|
+
suffix = ""
|
146
|
+
else
|
147
|
+
raise "Invalid environment"
|
148
|
+
end
|
149
|
+
title = prefix + r.name + suffix
|
150
|
+
return title
|
151
|
+
end
|
152
|
+
|
153
|
+
#Google doc helper methods
|
154
|
+
|
155
|
+
def find_or_create_gbook_by_title(title,gdrive_email)
|
156
|
+
r = self
|
157
|
+
book_dst = Dataset.find_or_create_by_handler_and_name('gbooker',title)
|
158
|
+
#give dst this requestor if none
|
159
|
+
book_dst.update_attributes(:requestor_id=>r.id.to_s) if book_dst.requestor_id.nil?
|
160
|
+
book = Gbooker.find_or_create_by_dst_id(book_dst.id.to_s,gdrive_email)
|
161
|
+
return book
|
162
|
+
end
|
163
|
+
|
164
|
+
def find_or_create_gsheet_by_name(name,gdrive_email)
|
165
|
+
r = self
|
166
|
+
sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',name)
|
167
|
+
sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
|
168
|
+
sheet = Gsheeter.find_or_create_by_dst_id(sheet_dst.id.to_s,gdrive_email)
|
169
|
+
return sheet
|
170
|
+
end
|
171
|
+
|
172
|
+
def jobs(jname=nil)
|
173
|
+
r = self
|
174
|
+
js = Job.find_all_by_requestor_id(r.id.to_s)
|
175
|
+
if jname
|
176
|
+
return js.sel{|j| j.name == jname}.first
|
177
|
+
else
|
178
|
+
return js
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def destroy_jobs
|
183
|
+
r = self
|
184
|
+
r.jobs.each{|s| s.delete}
|
185
|
+
end
|
186
|
+
|
187
|
+
def gsheets
|
188
|
+
r = self
|
189
|
+
Dataset.find_all_by_handler_and_requestor_id('gsheet',r.id.to_s)
|
190
|
+
end
|
191
|
+
|
192
|
+
def worker
|
193
|
+
r = self
|
194
|
+
Mobilize::Resque.find_worker_by_mongo_id(r.id.to_s)
|
195
|
+
end
|
196
|
+
|
197
|
+
def update_status(msg)
|
198
|
+
r = self
|
199
|
+
r.update_attributes(:status=>msg)
|
200
|
+
Mobilize::Resque.update_job_status(r.id.to_s,msg)
|
201
|
+
return true
|
202
|
+
end
|
203
|
+
|
204
|
+
def is_working?
|
205
|
+
r = self
|
206
|
+
Mobilize::Resque.active_mongo_ids.include?(r.id.to_s)
|
207
|
+
end
|
208
|
+
|
209
|
+
def is_due?
|
210
|
+
r = self
|
211
|
+
return false if r.is_working?
|
212
|
+
last_due_time = Time.now.utc - Jobtracker.requestor_refresh_freq
|
213
|
+
return true if r.last_run.nil? or r.last_run < last_due_time
|
214
|
+
end
|
215
|
+
|
216
|
+
def enqueue!
|
217
|
+
r = self
|
218
|
+
::Resque::Job.create("mobilize",Requestor,r.id.to_s,{"name"=>r.name})
|
219
|
+
return true
|
220
|
+
end
|
221
|
+
|
222
|
+
end
|
223
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# require 'resque/tasks'
|
2
|
+
# will give you the resque tasks
|
3
|
+
|
4
|
+
namespace :mobilize do
|
5
|
+
|
6
|
+
desc "Start a Resque worker"
|
7
|
+
task :work do
|
8
|
+
require 'resque'
|
9
|
+
require 'mobilize-base'
|
10
|
+
|
11
|
+
begin
|
12
|
+
worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
|
13
|
+
rescue Resque::NoQueueError
|
14
|
+
abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
|
15
|
+
end
|
16
|
+
|
17
|
+
puts "Starting worker #{worker}"
|
18
|
+
|
19
|
+
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Set up config and log folders and files"
|
23
|
+
task :setup do
|
24
|
+
sample_dir = File.dirname(__FILE__) + '/../samples/'
|
25
|
+
sample_files = Dir.entries(sample_dir)
|
26
|
+
config_dir = "#{ENV['PWD']}/config/"
|
27
|
+
log_dir = "#{ENV['PWD']}/log/"
|
28
|
+
unless File.exists?(config_dir)
|
29
|
+
puts "creating config dir"
|
30
|
+
`mkdir #{config_dir}`
|
31
|
+
end
|
32
|
+
unless File.exists?(log_dir)
|
33
|
+
puts "creating log dir"
|
34
|
+
`mkdir #{log_dir}`
|
35
|
+
end
|
36
|
+
sample_files.each do |fname|
|
37
|
+
unless File.exists?("#{config_dir}#{fname}")
|
38
|
+
puts "creating config/#{fname}"
|
39
|
+
`cp #{sample_dir}#{fname} #{config_dir}#{fname}`
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "mobilize-base/version"
|
2
|
+
require "mobilize-base/extensions/array"
|
3
|
+
require "mobilize-base/extensions/hash"
|
4
|
+
require "mobilize-base/extensions/object"
|
5
|
+
require "mobilize-base/extensions/string"
|
6
|
+
#this is the base of the mobilize object, any methods that should be
|
7
|
+
#made available application-wide go over here
|
8
|
+
#these also define base variables for Rails
|
9
|
+
module Mobilize
|
10
|
+
module Base
|
11
|
+
def Base.root
|
12
|
+
begin
|
13
|
+
::Rails.root
|
14
|
+
rescue
|
15
|
+
ENV['PWD']
|
16
|
+
end
|
17
|
+
end
|
18
|
+
def Base.config(config_name)
|
19
|
+
config_dir = begin
|
20
|
+
"#{::Rails.root}/config/"
|
21
|
+
rescue
|
22
|
+
"#{Base.root}/config/"
|
23
|
+
end
|
24
|
+
yaml_path = "#{config_dir}#{config_name}.yml"
|
25
|
+
if ::File.exists?(yaml_path)
|
26
|
+
return ::YAML.load_file(yaml_path)
|
27
|
+
else
|
28
|
+
raise "Could not find #{config_name}.yml in #{config_dir}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
def Base.env
|
32
|
+
begin
|
33
|
+
::Rails.env
|
34
|
+
rescue
|
35
|
+
#use MOBILIZE_ENV to manually set your environment when you start your app
|
36
|
+
ENV['MOBILIZE_ENV'] || "development"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
def Base.log_path(log_name)
|
40
|
+
log_dir = begin
|
41
|
+
"#{::Rails.root}/log/"
|
42
|
+
rescue
|
43
|
+
"#{Base.root}/log/"
|
44
|
+
end
|
45
|
+
log_path = "#{log_dir}#{log_name}.log"
|
46
|
+
if ::File.exists?(log_dir)
|
47
|
+
return log_path
|
48
|
+
else
|
49
|
+
raise "Could not find #{log_dir} folder for logs"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
|
55
|
+
if File.exists?(mongoid_config_path)
|
56
|
+
require 'mongo'
|
57
|
+
require 'mongoid'
|
58
|
+
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
59
|
+
require "mobilize-base/models/dataset"
|
60
|
+
require "mobilize-base/models/requestor"
|
61
|
+
require "mobilize-base/models/job"
|
62
|
+
end
|
63
|
+
require 'google_drive'
|
64
|
+
require 'resque'
|
65
|
+
require "mobilize-base/extensions/resque"
|
66
|
+
#specify appropriate redis port per resque.yml
|
67
|
+
Resque.redis = "127.0.0.1:#{Mobilize::Resque.config['redis_port']}"
|
68
|
+
require 'popen4'
|
69
|
+
require "mobilize-base/jobtracker"
|
70
|
+
require "mobilize-base/handlers/gdriver"
|
71
|
+
require "mobilize-base/extensions/google_drive.rb"
|
72
|
+
require "mobilize-base/handlers/mongoer"
|
73
|
+
require "mobilize-base/handlers/emailer"
|
74
|
+
|
75
|
+
#require "mobilize-base/handlers/*"
|
76
|
+
#require "mobilize-base/models/*"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
development:
|
2
|
+
owner:
|
3
|
+
email: 'owner_development@host.com'
|
4
|
+
pw: "google_drive_password"
|
5
|
+
admins:
|
6
|
+
- {email: 'admin@host.com'}
|
7
|
+
workers:
|
8
|
+
- {email: 'worker_development001@host.com', pw: "worker001_google_drive_password"}
|
9
|
+
- {email: 'worker_development002@host.com', pw: "worker002_google_drive_password"}
|
10
|
+
test:
|
11
|
+
owner:
|
12
|
+
email: 'owner_test@host.com'
|
13
|
+
pw: "google_drive_password"
|
14
|
+
admins:
|
15
|
+
- {email: 'admin@host.com'}
|
16
|
+
workers:
|
17
|
+
- {email: 'worker_test001@host.com', pw: "worker001_google_drive_password"}
|
18
|
+
- {email: 'worker_test002@host.com', pw: "worker002_google_drive_password"}
|
19
|
+
production:
|
20
|
+
owner:
|
21
|
+
email: 'owner_production@host.com'
|
22
|
+
pw: "google_drive_password"
|
23
|
+
admins:
|
24
|
+
- {email: 'admin@host.com'}
|
25
|
+
workers:
|
26
|
+
- {email: 'worker_production001@host.com', pw: "worker001_google_drive_password"}
|
27
|
+
- {email: 'worker_production002@host.com', pw: "worker002_google_drive_password"}
|
@@ -0,0 +1,24 @@
|
|
1
|
+
development:
|
2
|
+
#time between Jobtracker sweeps
|
3
|
+
cycle_freq: 10
|
4
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
5
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
6
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
7
|
+
admins: #emails to send notifications to
|
8
|
+
- {'email': 'admin@host.com'}
|
9
|
+
test:
|
10
|
+
#time between Jobtracker sweeps
|
11
|
+
cycle_freq: 10
|
12
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
13
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
14
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
15
|
+
admins: #emails to send notifications to
|
16
|
+
- {'email': 'admin@host.com'}
|
17
|
+
production:
|
18
|
+
#time between Jobtracker sweeps
|
19
|
+
cycle_freq: 10
|
20
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
21
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
22
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
23
|
+
admins: #emails to send notifications to
|
24
|
+
- {'email': 'admin@host.com'}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
development:
|
2
|
+
sessions:
|
3
|
+
default:
|
4
|
+
database: mobilize-development
|
5
|
+
persist_in_safe_mode: true
|
6
|
+
hosts:
|
7
|
+
- 127.0.0.1:27017
|
8
|
+
test:
|
9
|
+
sessions:
|
10
|
+
default:
|
11
|
+
database: mobilize-test
|
12
|
+
persist_in_safe_mode: true
|
13
|
+
hosts:
|
14
|
+
- 127.0.0.1:27017
|
15
|
+
production:
|
16
|
+
sessions:
|
17
|
+
default:
|
18
|
+
database: mobilize-production
|
19
|
+
persist_in_safe_mode: true
|
20
|
+
hosts:
|
21
|
+
- 127.0.0.1:27017
|