mobilize-base 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +20 -0
- data/README.md +509 -0
- data/Rakefile +34 -0
- data/lib/mobilize-base/extensions/array.rb +22 -0
- data/lib/mobilize-base/extensions/google_drive.rb +296 -0
- data/lib/mobilize-base/extensions/hash.rb +86 -0
- data/lib/mobilize-base/extensions/object.rb +6 -0
- data/lib/mobilize-base/extensions/resque.rb +180 -0
- data/lib/mobilize-base/extensions/string.rb +94 -0
- data/lib/mobilize-base/handlers/emailer.rb +24 -0
- data/lib/mobilize-base/handlers/gdriver.rb +309 -0
- data/lib/mobilize-base/handlers/mongoer.rb +32 -0
- data/lib/mobilize-base/jobtracker.rb +208 -0
- data/lib/mobilize-base/models/dataset.rb +70 -0
- data/lib/mobilize-base/models/job.rb +253 -0
- data/lib/mobilize-base/models/requestor.rb +223 -0
- data/lib/mobilize-base/tasks/mobilize-base.rake +2 -0
- data/lib/mobilize-base/tasks.rb +43 -0
- data/lib/mobilize-base/version.rb +5 -0
- data/lib/mobilize-base.rb +76 -0
- data/lib/samples/gdrive.yml +27 -0
- data/lib/samples/jobtracker.yml +24 -0
- data/lib/samples/mongoid.yml +21 -0
- data/lib/samples/resque.yml +12 -0
- data/mobilize-base.gemspec +35 -0
- data/test/mobilize_test.rb +125 -0
- data/test/redis-test.conf +540 -0
- data/test/test_helper.rb +23 -0
- metadata +260 -0
@@ -0,0 +1,253 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class Job
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :requestor_id, type: String
|
6
|
+
field :name, type: String
|
7
|
+
field :active, type: Boolean #active, inactive
|
8
|
+
field :schedule, type: String
|
9
|
+
field :active_task, type: String
|
10
|
+
field :tasks, type: Hash
|
11
|
+
field :status, type: String
|
12
|
+
field :last_error, type: String
|
13
|
+
field :last_trace, type: String
|
14
|
+
field :last_completed_at, type: Time
|
15
|
+
field :read_handler, type: String
|
16
|
+
field :write_handler, type: String
|
17
|
+
field :param_source, type: String #name of sheet on doc
|
18
|
+
field :param_hash, type: String #JSON
|
19
|
+
field :destination, type: String #output destination - could be file, could be sheet
|
20
|
+
|
21
|
+
index({ requestor_id: 1})
|
22
|
+
index({ name: 1})
|
23
|
+
|
24
|
+
before_destroy :destroy_output_dst_ids
|
25
|
+
|
26
|
+
def worker
|
27
|
+
j = self
|
28
|
+
Mobilize::Resque.find_worker_by_mongo_id(j.id.to_s)
|
29
|
+
end
|
30
|
+
|
31
|
+
def Job.find_by_name(name)
|
32
|
+
Job.where(:name=>name).first
|
33
|
+
end
|
34
|
+
|
35
|
+
def Job.find_all_by_requestor_id(requestor_id)
|
36
|
+
Job.where(:requestor_id=>requestor_id).to_a
|
37
|
+
end
|
38
|
+
|
39
|
+
def Job.find_or_create_by_requestor_id_and_name(requestor_id,name)
|
40
|
+
j = Job.where(:requestor_id=>requestor_id, :name=>name).first
|
41
|
+
j = Job.create(:requestor_id=>requestor_id, :name=>name) unless j
|
42
|
+
return j
|
43
|
+
end
|
44
|
+
|
45
|
+
#called by Resque
|
46
|
+
def Job.perform(id,*args)
|
47
|
+
j = Job.find(id)
|
48
|
+
task_params = j.tasks[j.active_task]
|
49
|
+
begin
|
50
|
+
j.update_status(%{Starting #{j.active_task} task at #{Time.now.utc}})
|
51
|
+
task_output = "Mobilize::#{task_params['handler'].humanize}".constantize.send(j.active_task,id)
|
52
|
+
#this allows user to return false if the stage didn't go as expected and needs to retry
|
53
|
+
#e.g. tried to write to Google but all the accounts were in use
|
54
|
+
return false if task_output == false
|
55
|
+
task_output_dst_id = if task_output.to_s.length==24 and Dataset.find(task_output.to_s)
|
56
|
+
#user has returned dst as output from task
|
57
|
+
task_output
|
58
|
+
else
|
59
|
+
#store the output in a cache
|
60
|
+
dst = Dataset.find_or_create_by_requestor_id_and_handler_and_name(j.requestor.id.to_s,'mongoer',"#{j.id.to_s}/#{j.active_task}")
|
61
|
+
dst.write_cache(task_output.to_s)
|
62
|
+
dst.id.to_s
|
63
|
+
end
|
64
|
+
j.tasks[j.active_task]['output_dst_id'] = task_output_dst_id
|
65
|
+
if j.active_task == j.tasks.keys.last
|
66
|
+
j.active_task = nil
|
67
|
+
j.last_error = ""
|
68
|
+
j.last_trace = ""
|
69
|
+
j.last_completed_at = Time.now.utc
|
70
|
+
j.status = %{Completed all tasks at #{Time.now.utc}}
|
71
|
+
j.save!
|
72
|
+
#check for any dependent jobs, if there are, enqueue them
|
73
|
+
r = j.requestor
|
74
|
+
dep_jobs = Job.where(:active=>true, :requestor_id=>r.id.to_s, :schedule=>"after #{j.name}").to_a
|
75
|
+
dep_jobs += Job.where(:active=>true, :schedule=>"after #{r.name}/#{j.name}").to_a
|
76
|
+
#put begin/rescue so all dependencies run
|
77
|
+
dep_jobs.each{|dj| begin;dj.enqueue! unless dj.is_working?;rescue;end}
|
78
|
+
else
|
79
|
+
task_names = j.tasks.keys
|
80
|
+
stage_idx = task_names.index(j.active_task) + 1
|
81
|
+
j.active_task = j.tasks.keys[stage_idx]
|
82
|
+
j.save!
|
83
|
+
#queue up next task
|
84
|
+
j.enqueue!
|
85
|
+
end
|
86
|
+
rescue ScriptError,StandardError => exc
|
87
|
+
#record the failure in Job so it appears on spec sheets
|
88
|
+
j.status='failed'
|
89
|
+
j.save!
|
90
|
+
j.update_status("Failed at #{Time.now.utc.to_s}")
|
91
|
+
j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
|
92
|
+
[exc.to_s,exc.backtrace.to_s].join("=>").oputs
|
93
|
+
#raising here will cause the failure to show on the Resque UI
|
94
|
+
raise exc
|
95
|
+
end
|
96
|
+
return true
|
97
|
+
end
|
98
|
+
|
99
|
+
def enqueue!
|
100
|
+
j = self
|
101
|
+
#assign first task if none assigned
|
102
|
+
if j.tasks.blank?
|
103
|
+
#make a hash with the read/write tasks
|
104
|
+
j.update_attributes(:tasks=>{"read_by_job_id"=>{'handler'=>j.read_handler},
|
105
|
+
"write_by_job_id"=>{'handler'=>j.write_handler}})
|
106
|
+
end
|
107
|
+
j.update_attributes(:active_task=>"read_by_job_id") if j.active_task.blank?
|
108
|
+
::Resque::Job.create("mobilize",Job,j.id.to_s,%{#{j.requestor.name}=>#{j.name}})
|
109
|
+
return true
|
110
|
+
end
|
111
|
+
|
112
|
+
#convenience methods
|
113
|
+
def requestor
|
114
|
+
j = self
|
115
|
+
return Requestor.find(j.requestor_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def restart
|
119
|
+
j = self
|
120
|
+
j.update_attributes(:last_completed_at=>nil)
|
121
|
+
return true
|
122
|
+
end
|
123
|
+
|
124
|
+
def prior_task
|
125
|
+
j = self
|
126
|
+
return nil if j.active_task.nil?
|
127
|
+
task_idx = j.tasks.keys.index(j.active_task)
|
128
|
+
return nil if task_idx==0
|
129
|
+
return j.tasks.keys[task_idx-1]
|
130
|
+
end
|
131
|
+
|
132
|
+
def destination_url
|
133
|
+
j = self
|
134
|
+
return nil if j.destination.nil?
|
135
|
+
destination = j.destination
|
136
|
+
dst = if j.write_handler == 'gsheet'
|
137
|
+
destination = [j.requestor.jobspec_title,j.destination].join("/") if destination.split("/").length==1
|
138
|
+
Dataset.find_by_handler_and_name('gsheeter',destination)
|
139
|
+
elsif j.write_handler == 'gtxt'
|
140
|
+
#all gtxt files must end in gz
|
141
|
+
destination += ".gz" unless destination.ends_with?(".gz")
|
142
|
+
destination = [s.requestor.name,"_"].join + destination unless destination.starts_with?([s.requestor.name,"_"].join)
|
143
|
+
Dataset.find_by_handler_and_name('gtxter',destination)
|
144
|
+
end
|
145
|
+
return dst.url if dst
|
146
|
+
end
|
147
|
+
|
148
|
+
def worker_args
|
149
|
+
j = self
|
150
|
+
Jobtracker.get_worker_args(j.worker)
|
151
|
+
end
|
152
|
+
|
153
|
+
def set_worker_args(args)
|
154
|
+
j = self
|
155
|
+
Jobtracker.set_worker_args(j.worker,args)
|
156
|
+
end
|
157
|
+
|
158
|
+
def cache_params
|
159
|
+
j = self
|
160
|
+
#go to paramsheet and read
|
161
|
+
param_path = if j.paramsheet.split("/").length==1
|
162
|
+
[j.requestor.jobspec_title,j.paramsheet].join("/")
|
163
|
+
else
|
164
|
+
j.paramsheet
|
165
|
+
end
|
166
|
+
param_sheet = j.requestor.find_or_create_gsheet_by_path(param_path)
|
167
|
+
param_tsv = param_sheet.to_tsv
|
168
|
+
param_dst = j.requestor.gsheets.select{|s| s.path == param_sheet.path}
|
169
|
+
param_dst.cache.write(param_tsv)
|
170
|
+
s.update_attributes(:param_dst_id=>paramdst.id.to_s)
|
171
|
+
(s.requestor.name + "'s #{s.name} params cached").oputs
|
172
|
+
return true
|
173
|
+
end
|
174
|
+
|
175
|
+
def update_status(msg)
|
176
|
+
j = self
|
177
|
+
j.update_attributes(:status=>msg)
|
178
|
+
Mobilize::Resque.update_job_status(j.id.to_s,msg)
|
179
|
+
return true
|
180
|
+
end
|
181
|
+
|
182
|
+
def is_working?
|
183
|
+
j = self
|
184
|
+
Mobilize::Resque.active_mongo_ids.include?(j.id.to_s)
|
185
|
+
end
|
186
|
+
|
187
|
+
def is_due?
|
188
|
+
j = self
|
189
|
+
return false if j.is_working? or j.schedule.to_s.starts_with?("after")
|
190
|
+
last_run = j.last_completed_at
|
191
|
+
#check schedule
|
192
|
+
schedule = j.schedule
|
193
|
+
return true if schedule == 'once' and j.active
|
194
|
+
#strip the "every" from the front if present
|
195
|
+
schedule = schedule.gsub("every","").gsub("."," ").strip
|
196
|
+
value,unit,operator,job_utctime = schedule.split(" ")
|
197
|
+
curr_utctime = Time.now.utc
|
198
|
+
curr_utcdate = curr_utctime.to_date.strftime("%Y-%m-%d")
|
199
|
+
if job_utctime
|
200
|
+
job_utctime = job_utctime.split(" ").first
|
201
|
+
job_utctime = Time.parse([curr_utcdate,job_utctime,"UTC"].join(" "))
|
202
|
+
end
|
203
|
+
#after is the only operator
|
204
|
+
raise "Unknown #{operator.to_s} operator" if operator and operator != "after"
|
205
|
+
if ["hour","hours"].include?(unit)
|
206
|
+
#if it's later than the last run + hour tolerance, is due
|
207
|
+
if last_run.nil? or curr_utctime > (last_run + value.to_i.hour)
|
208
|
+
return true
|
209
|
+
end
|
210
|
+
elsif ["day","days"].include?(unit)
|
211
|
+
if last_run.nil? or curr_utctime.to_date >= (last_run.to_date + value.to_i.day)
|
212
|
+
if operator and job_utctime
|
213
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
214
|
+
return true
|
215
|
+
end
|
216
|
+
elsif operator || job_utctime
|
217
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
218
|
+
else
|
219
|
+
return true
|
220
|
+
end
|
221
|
+
end
|
222
|
+
elsif unit == "day_of_week"
|
223
|
+
if curr_utctime.wday==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
|
224
|
+
if operator and job_utctime
|
225
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
226
|
+
return true
|
227
|
+
end
|
228
|
+
elsif operator || job_utctime
|
229
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
230
|
+
else
|
231
|
+
return true
|
232
|
+
end
|
233
|
+
end
|
234
|
+
elsif unit == "day_of_month"
|
235
|
+
if curr_utctime.day==value and (last_run.nil? or last_run.to_date != curr_utctime.to_date)
|
236
|
+
if operator and job_utctime
|
237
|
+
if curr_utctime>job_utctime and (job_utctime - curr_utctime).abs < 1.hour
|
238
|
+
return true
|
239
|
+
end
|
240
|
+
elsif operator || job_utctime
|
241
|
+
raise "Please specify both an operator and a time in UTC, or neither"
|
242
|
+
else
|
243
|
+
return true
|
244
|
+
end
|
245
|
+
end
|
246
|
+
else
|
247
|
+
raise "Unknown #{unit.to_s} time unit"
|
248
|
+
end
|
249
|
+
#if nothing happens, return false
|
250
|
+
return false
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
@@ -0,0 +1,223 @@
|
|
1
|
+
module Mobilize
|
2
|
+
class Requestor
|
3
|
+
include Mongoid::Document
|
4
|
+
include Mongoid::Timestamps
|
5
|
+
field :email, type: String
|
6
|
+
field :oauth, type: String
|
7
|
+
field :name, type: String
|
8
|
+
field :first_name, type: String
|
9
|
+
field :last_name, type: String
|
10
|
+
field :admin_role, type: String
|
11
|
+
field :last_run, type: Time
|
12
|
+
field :status, type: String
|
13
|
+
|
14
|
+
validates_presence_of :name, :message => ' cannot be blank.'
|
15
|
+
validates_uniqueness_of :name, :message => ' has already been used.'
|
16
|
+
|
17
|
+
before_destroy :destroy_jobs
|
18
|
+
|
19
|
+
def Requestor.find_or_create_by_name(name)
|
20
|
+
r = Requestor.where(:name => name).first
|
21
|
+
r = Requestor.create(:name => name) unless r
|
22
|
+
return r
|
23
|
+
end
|
24
|
+
|
25
|
+
def Requestor.find_or_create_by_email(email)
|
26
|
+
r = Requestor.where(:email => email).first
|
27
|
+
r = Requestor.create(:email => email) unless r
|
28
|
+
user_name = email.split("@").first
|
29
|
+
r.update_attributes(:name => user_name) unless r.name.to_s.length>0
|
30
|
+
return r
|
31
|
+
end
|
32
|
+
|
33
|
+
def Requestor.jobs_sheet_headers
|
34
|
+
%w{name active schedule status last_error destination_url read_handler write_handler param_source params destination}
|
35
|
+
end
|
36
|
+
|
37
|
+
def Requestor.perform(id,*args)
|
38
|
+
r = Requestor.find(id.to_s)
|
39
|
+
#reserve email account for read
|
40
|
+
gdrive_email = Gdriver.get_worker_email_by_mongo_id(id)
|
41
|
+
return false unless gdrive_email
|
42
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
43
|
+
#write headers to sheet
|
44
|
+
Requestor.jobs_sheet_headers.each_with_index do |h,h_i|
|
45
|
+
jobs_sheet[1,h_i+1] = h
|
46
|
+
end
|
47
|
+
jobs_sheet.save
|
48
|
+
#read the jobs sheet
|
49
|
+
#record jobs in DB
|
50
|
+
#deactivate jobs not in sheet
|
51
|
+
r.read_jobs(gdrive_email)
|
52
|
+
#queue up the jobs that are due and active
|
53
|
+
r.jobs.each do |j|
|
54
|
+
begin
|
55
|
+
j.enqueue! if j.active and j.is_due?
|
56
|
+
rescue ScriptError,StandardError => exc
|
57
|
+
#update errors
|
58
|
+
j.update_attributes(:last_error=>exc.to_s,:last_trace=>exc.backtrace.to_s)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
#write any updates to status, error, datasource_url etc.
|
62
|
+
r.write_jobs(gdrive_email)
|
63
|
+
r.update_attributes(:last_run=>Time.now.utc)
|
64
|
+
end
|
65
|
+
|
66
|
+
def jobs_sheet(gdrive_email)#gdrive_email to read with
|
67
|
+
r = self
|
68
|
+
r.find_or_create_gbook_by_title(r.jobspec_title,gdrive_email)
|
69
|
+
jobs_name = [r.jobspec_title,"Jobs"].join("/")
|
70
|
+
r.find_or_create_gsheet_by_name(jobs_name,gdrive_email)
|
71
|
+
end
|
72
|
+
|
73
|
+
def read_jobs(gdrive_email)
|
74
|
+
r = self
|
75
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
76
|
+
rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
|
77
|
+
#go through each job, update relevant job with its params
|
78
|
+
loc_jobs = []
|
79
|
+
rem_jobs.each_with_index do |rj,rj_i|
|
80
|
+
#skip bad rows
|
81
|
+
next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
|
82
|
+
j = Job.find_or_create_by_requestor_id_and_name(r.id.to_s,rj['name'])
|
83
|
+
#update top line params
|
84
|
+
j.update_attributes(:active => rj['active'],
|
85
|
+
:schedule => rj['schedule'],
|
86
|
+
:read_handler => rj['read_handler'],
|
87
|
+
:write_handler => rj['write_handler'],
|
88
|
+
:param_source => rj['param_source'],
|
89
|
+
:params => rj['params'],
|
90
|
+
:destination => rj['destination'])
|
91
|
+
#update laststatus with "Created job for" if job is due
|
92
|
+
j.update_status("Due and active at #{Time.now.utc}") if j.is_due? and j.active
|
93
|
+
#add this job to list of local ones
|
94
|
+
loc_jobs << j
|
95
|
+
end
|
96
|
+
#deactivate requestor jobs that are not included in sheet;
|
97
|
+
#this makes sure we don't run obsolete jobs
|
98
|
+
(r.jobs.map{|j| j.id.to_s} - loc_jobs.map{|j| j.id.to_s}).each do |rjid|
|
99
|
+
j = Job.find(rjid)
|
100
|
+
if j.active
|
101
|
+
j.update_attributes(:active=>false)
|
102
|
+
r.update_status("Deactivated job:#{r.name}=>#{j.name}")
|
103
|
+
end
|
104
|
+
end
|
105
|
+
r.update_status(r.name + " jobs read at #{Time.now.utc}")
|
106
|
+
return true
|
107
|
+
end
|
108
|
+
|
109
|
+
def write_jobs(gdrive_email) #gdrive_email to update with
|
110
|
+
r = self
|
111
|
+
jobs_sheet = r.jobs_sheet(gdrive_email)
|
112
|
+
rem_jobs = jobs_sheet.to_tsv.tsv_to_hash_array
|
113
|
+
#go through each job, update relevant job with its params
|
114
|
+
headers = Requestor.jobs_sheet_headers
|
115
|
+
#write headers
|
116
|
+
headers.each_with_index do |h,h_i|
|
117
|
+
jobs_sheet[1,h_i+1] = h
|
118
|
+
end
|
119
|
+
#write rows
|
120
|
+
rem_jobs.each_with_index do |rj,rj_i|
|
121
|
+
#skip bad rows
|
122
|
+
next if (rj['name'].to_s.first == "#" or ['name','schedule','read_handler','write_handler','active'].select{|c| rj[c].to_s.strip==""}.length>0)
|
123
|
+
j = r.jobs(rj['name'])
|
124
|
+
#update active to false if this was a run once
|
125
|
+
j.update_attributes(:active=>false) if j.schedule.to_s == 'once'
|
126
|
+
jobs_sheet[rj_i+2,headers.index('active')+1] = j.active.to_s
|
127
|
+
jobs_sheet[rj_i+2,headers.index('status')+1] = j.status.to_s.gsub("\n",";").gsub("\t"," ")
|
128
|
+
jobs_sheet[rj_i+2,headers.index('last_error')+1] = j.last_error.to_s.gsub("\n",";").gsub("\t"," ")
|
129
|
+
jobs_sheet[rj_i+2,headers.index('destination_url')+1] = j.destination_url.to_s
|
130
|
+
end
|
131
|
+
jobs_sheet.save
|
132
|
+
r.update_status(r.name + " jobs written")
|
133
|
+
return true
|
134
|
+
end
|
135
|
+
|
136
|
+
def jobspec_title
|
137
|
+
r = self
|
138
|
+
prefix = "Jobspec_"
|
139
|
+
suffix = ""
|
140
|
+
if Mobilize::Base.env == 'development'
|
141
|
+
suffix = "_dev"
|
142
|
+
elsif Mobilize::Base.env == 'test' or Mobilize::Base.env == 'pry_dev'
|
143
|
+
suffix = "_test"
|
144
|
+
elsif Mobilize::Base.env == 'production' or Mobilize::Base.env == 'integration'
|
145
|
+
suffix = ""
|
146
|
+
else
|
147
|
+
raise "Invalid environment"
|
148
|
+
end
|
149
|
+
title = prefix + r.name + suffix
|
150
|
+
return title
|
151
|
+
end
|
152
|
+
|
153
|
+
#Google doc helper methods
|
154
|
+
|
155
|
+
def find_or_create_gbook_by_title(title,gdrive_email)
|
156
|
+
r = self
|
157
|
+
book_dst = Dataset.find_or_create_by_handler_and_name('gbooker',title)
|
158
|
+
#give dst this requestor if none
|
159
|
+
book_dst.update_attributes(:requestor_id=>r.id.to_s) if book_dst.requestor_id.nil?
|
160
|
+
book = Gbooker.find_or_create_by_dst_id(book_dst.id.to_s,gdrive_email)
|
161
|
+
return book
|
162
|
+
end
|
163
|
+
|
164
|
+
def find_or_create_gsheet_by_name(name,gdrive_email)
|
165
|
+
r = self
|
166
|
+
sheet_dst = Dataset.find_or_create_by_handler_and_name('gsheeter',name)
|
167
|
+
sheet_dst.update_attributes(:requestor_id=>r.id.to_s) if sheet_dst.requestor_id.nil?
|
168
|
+
sheet = Gsheeter.find_or_create_by_dst_id(sheet_dst.id.to_s,gdrive_email)
|
169
|
+
return sheet
|
170
|
+
end
|
171
|
+
|
172
|
+
def jobs(jname=nil)
|
173
|
+
r = self
|
174
|
+
js = Job.find_all_by_requestor_id(r.id.to_s)
|
175
|
+
if jname
|
176
|
+
return js.sel{|j| j.name == jname}.first
|
177
|
+
else
|
178
|
+
return js
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def destroy_jobs
|
183
|
+
r = self
|
184
|
+
r.jobs.each{|s| s.delete}
|
185
|
+
end
|
186
|
+
|
187
|
+
def gsheets
|
188
|
+
r = self
|
189
|
+
Dataset.find_all_by_handler_and_requestor_id('gsheet',r.id.to_s)
|
190
|
+
end
|
191
|
+
|
192
|
+
def worker
|
193
|
+
r = self
|
194
|
+
Mobilize::Resque.find_worker_by_mongo_id(r.id.to_s)
|
195
|
+
end
|
196
|
+
|
197
|
+
def update_status(msg)
|
198
|
+
r = self
|
199
|
+
r.update_attributes(:status=>msg)
|
200
|
+
Mobilize::Resque.update_job_status(r.id.to_s,msg)
|
201
|
+
return true
|
202
|
+
end
|
203
|
+
|
204
|
+
def is_working?
|
205
|
+
r = self
|
206
|
+
Mobilize::Resque.active_mongo_ids.include?(r.id.to_s)
|
207
|
+
end
|
208
|
+
|
209
|
+
def is_due?
|
210
|
+
r = self
|
211
|
+
return false if r.is_working?
|
212
|
+
last_due_time = Time.now.utc - Jobtracker.requestor_refresh_freq
|
213
|
+
return true if r.last_run.nil? or r.last_run < last_due_time
|
214
|
+
end
|
215
|
+
|
216
|
+
def enqueue!
|
217
|
+
r = self
|
218
|
+
::Resque::Job.create("mobilize",Requestor,r.id.to_s,{"name"=>r.name})
|
219
|
+
return true
|
220
|
+
end
|
221
|
+
|
222
|
+
end
|
223
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
# require 'resque/tasks'
|
2
|
+
# will give you the resque tasks
|
3
|
+
|
4
|
+
namespace :mobilize do
|
5
|
+
|
6
|
+
desc "Start a Resque worker"
|
7
|
+
task :work do
|
8
|
+
require 'resque'
|
9
|
+
require 'mobilize-base'
|
10
|
+
|
11
|
+
begin
|
12
|
+
worker = Resque::Worker.new(Mobilize::Resque.config['queue_name'])
|
13
|
+
rescue Resque::NoQueueError
|
14
|
+
abort "set QUEUE env var, e.g. $ QUEUE=critical,high rake resque:work"
|
15
|
+
end
|
16
|
+
|
17
|
+
puts "Starting worker #{worker}"
|
18
|
+
|
19
|
+
worker.work(ENV['INTERVAL'] || 5) # interval, will block
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Set up config and log folders and files"
|
23
|
+
task :setup do
|
24
|
+
sample_dir = File.dirname(__FILE__) + '/../samples/'
|
25
|
+
sample_files = Dir.entries(sample_dir)
|
26
|
+
config_dir = "#{ENV['PWD']}/config/"
|
27
|
+
log_dir = "#{ENV['PWD']}/log/"
|
28
|
+
unless File.exists?(config_dir)
|
29
|
+
puts "creating config dir"
|
30
|
+
`mkdir #{config_dir}`
|
31
|
+
end
|
32
|
+
unless File.exists?(log_dir)
|
33
|
+
puts "creating log dir"
|
34
|
+
`mkdir #{log_dir}`
|
35
|
+
end
|
36
|
+
sample_files.each do |fname|
|
37
|
+
unless File.exists?("#{config_dir}#{fname}")
|
38
|
+
puts "creating config/#{fname}"
|
39
|
+
`cp #{sample_dir}#{fname} #{config_dir}#{fname}`
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require "mobilize-base/version"
|
2
|
+
require "mobilize-base/extensions/array"
|
3
|
+
require "mobilize-base/extensions/hash"
|
4
|
+
require "mobilize-base/extensions/object"
|
5
|
+
require "mobilize-base/extensions/string"
|
6
|
+
#this is the base of the mobilize object, any methods that should be
|
7
|
+
#made available application-wide go over here
|
8
|
+
#these also define base variables for Rails
|
9
|
+
module Mobilize
|
10
|
+
module Base
|
11
|
+
def Base.root
|
12
|
+
begin
|
13
|
+
::Rails.root
|
14
|
+
rescue
|
15
|
+
ENV['PWD']
|
16
|
+
end
|
17
|
+
end
|
18
|
+
def Base.config(config_name)
|
19
|
+
config_dir = begin
|
20
|
+
"#{::Rails.root}/config/"
|
21
|
+
rescue
|
22
|
+
"#{Base.root}/config/"
|
23
|
+
end
|
24
|
+
yaml_path = "#{config_dir}#{config_name}.yml"
|
25
|
+
if ::File.exists?(yaml_path)
|
26
|
+
return ::YAML.load_file(yaml_path)
|
27
|
+
else
|
28
|
+
raise "Could not find #{config_name}.yml in #{config_dir}"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
def Base.env
|
32
|
+
begin
|
33
|
+
::Rails.env
|
34
|
+
rescue
|
35
|
+
#use MOBILIZE_ENV to manually set your environment when you start your app
|
36
|
+
ENV['MOBILIZE_ENV'] || "development"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
def Base.log_path(log_name)
|
40
|
+
log_dir = begin
|
41
|
+
"#{::Rails.root}/log/"
|
42
|
+
rescue
|
43
|
+
"#{Base.root}/log/"
|
44
|
+
end
|
45
|
+
log_path = "#{log_dir}#{log_name}.log"
|
46
|
+
if ::File.exists?(log_dir)
|
47
|
+
return log_path
|
48
|
+
else
|
49
|
+
raise "Could not find #{log_dir} folder for logs"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
mongoid_config_path = "#{Mobilize::Base.root}/config/mongoid.yml"
|
55
|
+
if File.exists?(mongoid_config_path)
|
56
|
+
require 'mongo'
|
57
|
+
require 'mongoid'
|
58
|
+
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
59
|
+
require "mobilize-base/models/dataset"
|
60
|
+
require "mobilize-base/models/requestor"
|
61
|
+
require "mobilize-base/models/job"
|
62
|
+
end
|
63
|
+
require 'google_drive'
|
64
|
+
require 'resque'
|
65
|
+
require "mobilize-base/extensions/resque"
|
66
|
+
#specify appropriate redis port per resque.yml
|
67
|
+
Resque.redis = "127.0.0.1:#{Mobilize::Resque.config['redis_port']}"
|
68
|
+
require 'popen4'
|
69
|
+
require "mobilize-base/jobtracker"
|
70
|
+
require "mobilize-base/handlers/gdriver"
|
71
|
+
require "mobilize-base/extensions/google_drive.rb"
|
72
|
+
require "mobilize-base/handlers/mongoer"
|
73
|
+
require "mobilize-base/handlers/emailer"
|
74
|
+
|
75
|
+
#require "mobilize-base/handlers/*"
|
76
|
+
#require "mobilize-base/models/*"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
development:
|
2
|
+
owner:
|
3
|
+
email: 'owner_development@host.com'
|
4
|
+
pw: "google_drive_password"
|
5
|
+
admins:
|
6
|
+
- {email: 'admin@host.com'}
|
7
|
+
workers:
|
8
|
+
- {email: 'worker_development001@host.com', pw: "worker001_google_drive_password"}
|
9
|
+
- {email: 'worker_development002@host.com', pw: "worker002_google_drive_password"}
|
10
|
+
test:
|
11
|
+
owner:
|
12
|
+
email: 'owner_test@host.com'
|
13
|
+
pw: "google_drive_password"
|
14
|
+
admins:
|
15
|
+
- {email: 'admin@host.com'}
|
16
|
+
workers:
|
17
|
+
- {email: 'worker_test001@host.com', pw: "worker001_google_drive_password"}
|
18
|
+
- {email: 'worker_test002@host.com', pw: "worker002_google_drive_password"}
|
19
|
+
production:
|
20
|
+
owner:
|
21
|
+
email: 'owner_production@host.com'
|
22
|
+
pw: "google_drive_password"
|
23
|
+
admins:
|
24
|
+
- {email: 'admin@host.com'}
|
25
|
+
workers:
|
26
|
+
- {email: 'worker_production001@host.com', pw: "worker001_google_drive_password"}
|
27
|
+
- {email: 'worker_production002@host.com', pw: "worker002_google_drive_password"}
|
@@ -0,0 +1,24 @@
|
|
1
|
+
development:
|
2
|
+
#time between Jobtracker sweeps
|
3
|
+
cycle_freq: 10
|
4
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
5
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
6
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
7
|
+
admins: #emails to send notifications to
|
8
|
+
- {'email': 'admin@host.com'}
|
9
|
+
test:
|
10
|
+
#time between Jobtracker sweeps
|
11
|
+
cycle_freq: 10
|
12
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
13
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
14
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
15
|
+
admins: #emails to send notifications to
|
16
|
+
- {'email': 'admin@host.com'}
|
17
|
+
production:
|
18
|
+
#time between Jobtracker sweeps
|
19
|
+
cycle_freq: 10
|
20
|
+
notification_freq: 3600 #1 hour between failure/timeout notifications
|
21
|
+
requestor_refresh_freq: 300 #5 min between requestor checks
|
22
|
+
max_run_time: 14400 # if a job runs for 4h+, notification will be sent
|
23
|
+
admins: #emails to send notifications to
|
24
|
+
- {'email': 'admin@host.com'}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
development:
|
2
|
+
sessions:
|
3
|
+
default:
|
4
|
+
database: mobilize-development
|
5
|
+
persist_in_safe_mode: true
|
6
|
+
hosts:
|
7
|
+
- 127.0.0.1:27017
|
8
|
+
test:
|
9
|
+
sessions:
|
10
|
+
default:
|
11
|
+
database: mobilize-test
|
12
|
+
persist_in_safe_mode: true
|
13
|
+
hosts:
|
14
|
+
- 127.0.0.1:27017
|
15
|
+
production:
|
16
|
+
sessions:
|
17
|
+
default:
|
18
|
+
database: mobilize-production
|
19
|
+
persist_in_safe_mode: true
|
20
|
+
hosts:
|
21
|
+
- 127.0.0.1:27017
|