mobilize-base 1.29 → 1.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +19 -10
- data/lib/mobilize-base/extensions/google_drive/file.rb +7 -7
- data/lib/mobilize-base/extensions/resque-server/views/queues.erb +59 -0
- data/lib/mobilize-base/extensions/resque-server/views/working.erb +85 -0
- data/lib/mobilize-base/extensions/string.rb +12 -4
- data/lib/mobilize-base/extensions/yaml.rb +10 -7
- data/lib/mobilize-base/handlers/gbook.rb +25 -38
- data/lib/mobilize-base/handlers/gdrive.rb +8 -0
- data/lib/mobilize-base/handlers/gfile.rb +4 -2
- data/lib/mobilize-base/handlers/gridfs.rb +19 -24
- data/lib/mobilize-base/handlers/gsheet.rb +13 -7
- data/lib/mobilize-base/handlers/resque.rb +9 -3
- data/lib/mobilize-base/helpers/job_helper.rb +54 -0
- data/lib/mobilize-base/helpers/jobtracker_helper.rb +147 -0
- data/lib/mobilize-base/helpers/runner_helper.rb +83 -0
- data/lib/mobilize-base/helpers/stage_helper.rb +38 -0
- data/lib/mobilize-base/jobtracker.rb +16 -192
- data/lib/mobilize-base/models/job.rb +36 -48
- data/lib/mobilize-base/models/runner.rb +22 -122
- data/lib/mobilize-base/models/stage.rb +14 -35
- data/lib/mobilize-base/tasks.rb +16 -3
- data/lib/mobilize-base/version.rb +1 -1
- data/lib/mobilize-base.rb +5 -1
- data/lib/samples/gdrive.yml +6 -0
- data/lib/samples/gridfs.yml +0 -3
- data/lib/samples/gsheet.yml +4 -4
- data/mobilize-base.gemspec +3 -3
- data/test/base_job_rows.yml +5 -1
- data/test/mobilize-base_test.rb +20 -61
- data/test/test_helper.rb +79 -0
- metadata +57 -25
- checksums.yaml +0 -7
@@ -1,5 +1,6 @@
|
|
1
1
|
module Mobilize
|
2
2
|
class Runner
|
3
|
+
include Mobilize::RunnerHelper
|
3
4
|
include Mongoid::Document
|
4
5
|
include Mongoid::Timestamps
|
5
6
|
field :path, type: String
|
@@ -11,20 +12,6 @@ module Mobilize
|
|
11
12
|
|
12
13
|
index({ path: 1})
|
13
14
|
|
14
|
-
def headers
|
15
|
-
%w{name active trigger status stage1 stage2 stage3 stage4 stage5}
|
16
|
-
end
|
17
|
-
|
18
|
-
def title
|
19
|
-
r = self
|
20
|
-
r.path.split("/").first
|
21
|
-
end
|
22
|
-
|
23
|
-
def worker
|
24
|
-
r = self
|
25
|
-
Mobilize::Resque.find_worker_by_path(r.path)
|
26
|
-
end
|
27
|
-
|
28
15
|
def Runner.find_by_path(path)
|
29
16
|
Runner.where(:path=>path).first
|
30
17
|
end
|
@@ -32,6 +19,7 @@ module Mobilize
|
|
32
19
|
def Runner.find_by_title(title)
|
33
20
|
Runner.where(:path=>"#{title}/jobs").first
|
34
21
|
end
|
22
|
+
|
35
23
|
def Runner.perform(id,*args)
|
36
24
|
r = Runner.find_by_path(id)
|
37
25
|
#get gdrive slot for read
|
@@ -41,12 +29,18 @@ module Mobilize
|
|
41
29
|
return false
|
42
30
|
end
|
43
31
|
r.update_attributes(:started_at=>Time.now.utc)
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
32
|
+
begin
|
33
|
+
#make sure any updates to activity are processed first
|
34
|
+
#as in when someone runs a "once" job that has completed
|
35
|
+
r.update_gsheet(gdrive_slot)
|
36
|
+
#read the jobs in the gsheet and update models with news
|
37
|
+
r.read_gsheet(gdrive_slot)
|
38
|
+
#queue up the jobs that are due and active
|
39
|
+
rescue => exc
|
40
|
+
#log the exception, but continue w job processing
|
41
|
+
#This ensures jobs are still processed if google drive goes down
|
42
|
+
r.update_status("Failed to read or update gsheet with #{exc.to_s} #{exc.backtrace.join(";")}")
|
43
|
+
end
|
50
44
|
r.jobs.each do |j|
|
51
45
|
begin
|
52
46
|
if j.is_due?
|
@@ -64,91 +58,32 @@ module Mobilize
|
|
64
58
|
r.update_attributes(:completed_at=>Time.now.utc)
|
65
59
|
end
|
66
60
|
|
67
|
-
def dataset
|
68
|
-
r = self
|
69
|
-
Dataset.find_or_create_by_handler_and_path("gsheet",r.path)
|
70
|
-
end
|
71
|
-
|
72
61
|
def Runner.find_or_create_by_path(path)
|
73
62
|
Runner.where(:path=>path).first || Runner.create(:path=>path,:active=>true)
|
74
63
|
end
|
75
64
|
|
76
|
-
def gbook(gdrive_slot)
|
77
|
-
r = self
|
78
|
-
title = r.path.split("/").first
|
79
|
-
Gbook.find_by_path(title,gdrive_slot)
|
80
|
-
end
|
81
|
-
|
82
|
-
def gsheet(gdrive_slot)
|
83
|
-
r = self
|
84
|
-
u = r.user
|
85
|
-
jobs_sheet = Gsheet.find_by_path(r.path,gdrive_slot)
|
86
|
-
#make sure the user has a runner with a jobs sheet and has write privileges on the spreadsheet
|
87
|
-
unless (jobs_sheet and jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="writer"})
|
88
|
-
#only give the user edit permissions if they're the ones
|
89
|
-
#creating it
|
90
|
-
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
91
|
-
unless jobs_sheet.spreadsheet.acl_entry(u.email).ie{|e| e and e.role=="owner"}
|
92
|
-
jobs_sheet.spreadsheet.update_acl(u.email,"writer")
|
93
|
-
end
|
94
|
-
end
|
95
|
-
jobs_sheet.add_headers(r.headers)
|
96
|
-
#add url to dataset
|
97
|
-
Dataset.find_or_create_by_url("gsheet://#{r.path}").update_attributes(:http_url=>jobs_sheet.spreadsheet.human_url)
|
98
|
-
begin;jobs_sheet.delete_sheet1;rescue;end #don't care if sheet1 deletion fails
|
99
|
-
return jobs_sheet
|
100
|
-
end
|
101
|
-
|
102
65
|
def read_gsheet(gdrive_slot)
|
103
66
|
r = self
|
104
67
|
#argument converts line breaks in cells to spaces
|
105
68
|
gsheet_tsv = r.gsheet(gdrive_slot).to_tsv(" ")
|
106
69
|
#turn it into a hash array
|
107
|
-
|
70
|
+
gsheet_hashes = gsheet_tsv.tsv_to_hash_array
|
108
71
|
#go through each job, update relevant job with its params
|
109
72
|
done_jobs = []
|
110
73
|
#parse out the jobs and update the Job collection
|
111
|
-
|
74
|
+
gsheet_hashes.each do |gsheet_hash|
|
112
75
|
#skip non-jobs or jobs without required values
|
113
|
-
next if (
|
114
|
-
j = Job.
|
115
|
-
#update top line params
|
116
|
-
j.update_attributes(:active => rj['active'],
|
117
|
-
:trigger => rj['trigger'])
|
118
|
-
(1..5).to_a.each do |s_idx|
|
119
|
-
stage_string = rj["stage#{s_idx.to_s}"]
|
120
|
-
s = Stage.find_by_path("#{j.path}/stage#{s_idx.to_s}")
|
121
|
-
if stage_string.to_s.length==0
|
122
|
-
#delete this stage and all stages after
|
123
|
-
if s
|
124
|
-
j = s.job
|
125
|
-
j.stages[(s.idx-1)..-1].each{|ps| ps.delete}
|
126
|
-
#just in case
|
127
|
-
s.delete
|
128
|
-
end
|
129
|
-
break
|
130
|
-
elsif s.nil?
|
131
|
-
#create this stage
|
132
|
-
s = Stage.find_or_create_by_path("#{j.path}/stage#{s_idx.to_s}")
|
133
|
-
end
|
134
|
-
#parse command string, update stage with it
|
135
|
-
s_handler, call, param_string = [""*3]
|
136
|
-
stage_string.split(" ").ie do |spls|
|
137
|
-
s_handler = spls.first.split(".").first
|
138
|
-
call = spls.first.split(".").last
|
139
|
-
param_string = spls[1..-1].join(" ").strip
|
140
|
-
end
|
141
|
-
s.update_attributes(:call=>call, :handler=>s_handler, :param_string=>param_string)
|
142
|
-
end
|
76
|
+
next if (gsheet_hash['name'].to_s.first == "#" or ['name','active','trigger','stage1'].select{|c| gsheet_hash[c].to_s.strip==""}.length>0)
|
77
|
+
j = Job.update_by_user_name_and_hash(r.user.name,gsheet_hash)
|
143
78
|
r.update_status("Updated #{j.path} stages at #{Time.now.utc}")
|
144
79
|
#add this job to list of read ones
|
145
80
|
done_jobs << j
|
146
81
|
end
|
147
82
|
#delete user jobs that are not included in Runner
|
148
|
-
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |
|
149
|
-
j = Job.where(:path=>
|
83
|
+
(r.jobs.map{|j| j.path} - done_jobs.map{|j| j.path}).each do |gsheet_hash_path|
|
84
|
+
j = Job.where(:path=>gsheet_hash_path).first
|
150
85
|
j.delete if j
|
151
|
-
r.update_status("Deleted job:#{
|
86
|
+
r.update_status("Deleted job:#{gsheet_hash_path}")
|
152
87
|
end
|
153
88
|
r.update_status("jobs read at #{Time.now.utc}")
|
154
89
|
return true
|
@@ -159,48 +94,13 @@ module Mobilize
|
|
159
94
|
#there's nothing to update if runner has never had a completed at
|
160
95
|
return false unless r.completed_at
|
161
96
|
jobs_gsheet = r.gsheet(gdrive_slot)
|
162
|
-
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at > j.runner.completed_at}
|
97
|
+
upd_jobs = r.jobs.select{|j| j.status_at and j.status_at.to_f > j.runner.completed_at.to_f}
|
163
98
|
upd_rows = upd_jobs.map{|j| {'name'=>j.name, 'active'=>j.active, 'status'=>j.status}}
|
164
99
|
jobs_gsheet.add_or_update_rows(upd_rows)
|
165
100
|
r.update_status("gsheet updated")
|
166
101
|
return true
|
167
102
|
end
|
168
103
|
|
169
|
-
def jobs(jname=nil)
|
170
|
-
r = self
|
171
|
-
js = Job.where(:path=>/^#{r.path.escape_regex}/).to_a
|
172
|
-
if jname
|
173
|
-
return js.sel{|j| j.name == jname}.first
|
174
|
-
else
|
175
|
-
return js
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
|
-
def user
|
180
|
-
r = self
|
181
|
-
user_name = r.path.split("_").second.split("(").first.split("/").first
|
182
|
-
User.where(:name=>user_name).first
|
183
|
-
end
|
184
|
-
|
185
|
-
def update_status(msg)
|
186
|
-
r = self
|
187
|
-
r.update_attributes(:status=>msg, :status_at=>Time.now.utc)
|
188
|
-
Mobilize::Resque.set_worker_args_by_path(r.path,{'status'=>msg})
|
189
|
-
return true
|
190
|
-
end
|
191
|
-
|
192
|
-
def is_working?
|
193
|
-
r = self
|
194
|
-
Mobilize::Resque.active_paths.include?(r.path)
|
195
|
-
end
|
196
|
-
|
197
|
-
def is_due?
|
198
|
-
r = self.reload
|
199
|
-
return false if r.is_working?
|
200
|
-
prev_due_time = Time.now.utc - Jobtracker.runner_read_freq
|
201
|
-
return true if r.started_at.nil? or r.started_at < prev_due_time
|
202
|
-
end
|
203
|
-
|
204
104
|
def enqueue!
|
205
105
|
r = self
|
206
106
|
::Resque::Job.create("mobilize",Runner,r.path,{})
|
@@ -2,6 +2,7 @@ module Mobilize
|
|
2
2
|
class Stage
|
3
3
|
include Mongoid::Document
|
4
4
|
include Mongoid::Timestamps
|
5
|
+
include Mobilize::StageHelper
|
5
6
|
field :path, type: String
|
6
7
|
field :handler, type: String
|
7
8
|
field :call, type: String
|
@@ -16,40 +17,6 @@ module Mobilize
|
|
16
17
|
|
17
18
|
index({ path: 1})
|
18
19
|
|
19
|
-
def idx
|
20
|
-
s = self
|
21
|
-
s.path.split("/").last.gsub("stage","").to_i
|
22
|
-
end
|
23
|
-
|
24
|
-
def out_dst
|
25
|
-
#this gives a dataset that points to the output
|
26
|
-
#allowing you to determine its size
|
27
|
-
#before committing to a read or write
|
28
|
-
s = self
|
29
|
-
Dataset.find_by_url(s.response['out_url']) if s.response and s.response['out_url']
|
30
|
-
end
|
31
|
-
|
32
|
-
def err_dst
|
33
|
-
#this gives a dataset that points to the output
|
34
|
-
#allowing you to determine its size
|
35
|
-
#before committing to a read or write
|
36
|
-
s = self
|
37
|
-
Dataset.find_by_url(s.response['err_url']) if s.response and s.response['err_url']
|
38
|
-
end
|
39
|
-
|
40
|
-
def params
|
41
|
-
s = self
|
42
|
-
p = YAML.easy_load(s.param_string)
|
43
|
-
raise "Must resolve to Hash" unless p.class==Hash
|
44
|
-
return p
|
45
|
-
end
|
46
|
-
|
47
|
-
def job
|
48
|
-
s = self
|
49
|
-
job_path = s.path.split("/")[0..-2].join("/")
|
50
|
-
Job.where(:path=>job_path).first
|
51
|
-
end
|
52
|
-
|
53
20
|
def Stage.find_or_create_by_path(path)
|
54
21
|
s = Stage.where(:path=>path).first
|
55
22
|
s = Stage.create(:path=>path) unless s
|
@@ -77,6 +44,14 @@ module Mobilize
|
|
77
44
|
|
78
45
|
def Stage.perform(id,*args)
|
79
46
|
s = Stage.where(:path=>id).first
|
47
|
+
#check to make sure params are parsable
|
48
|
+
begin
|
49
|
+
param_hash = s.params
|
50
|
+
raise ScriptError if param_hash.class!=Hash
|
51
|
+
rescue StandardError, ScriptError
|
52
|
+
s.fail({'signal'=>500,
|
53
|
+
'err_str'=>"Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."})
|
54
|
+
end
|
80
55
|
s.update_attributes(:started_at=>Time.now.utc)
|
81
56
|
s.update_status(%{Starting at #{Time.now.utc}})
|
82
57
|
#get response by running method
|
@@ -145,7 +120,11 @@ module Mobilize
|
|
145
120
|
j = s.job
|
146
121
|
r = j.runner
|
147
122
|
u = r.user
|
148
|
-
|
123
|
+
begin
|
124
|
+
j.update_attributes(:active=>false) if s.params['always_on'].to_s=="false"
|
125
|
+
rescue StandardError, ScriptError
|
126
|
+
#skip due to parse error on params
|
127
|
+
end
|
149
128
|
s.update_attributes(:failed_at=>Time.now.utc,:response=>response)
|
150
129
|
stage_name = "#{j.name}_stage#{s.idx.to_s}.err"
|
151
130
|
target_path = (r.path.split("/")[0..-2] + [stage_name]).join("/")
|
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -40,17 +40,17 @@ namespace :mobilize_base do
|
|
40
40
|
Mobilize::Jobtracker.prep_workers
|
41
41
|
end
|
42
42
|
desc "Stop Jobtracker"
|
43
|
-
task :
|
43
|
+
task :stop_jobtracker do
|
44
44
|
require 'mobilize-base'
|
45
45
|
Mobilize::Jobtracker.stop!
|
46
46
|
end
|
47
47
|
desc "Start Jobtracker"
|
48
|
-
task :
|
48
|
+
task :start_jobtracker do
|
49
49
|
require 'mobilize-base'
|
50
50
|
Mobilize::Jobtracker.start
|
51
51
|
end
|
52
52
|
desc "Restart Jobtracker"
|
53
|
-
task :
|
53
|
+
task :restart_jobtracker do
|
54
54
|
require 'mobilize-base'
|
55
55
|
Mobilize::Jobtracker.restart!
|
56
56
|
end
|
@@ -66,6 +66,19 @@ namespace :mobilize_base do
|
|
66
66
|
resque_redis_port_args = if Mobilize::Base.env == 'test'
|
67
67
|
" -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
|
68
68
|
end.to_s
|
69
|
+
#determine view folder and override queues and working erbs
|
70
|
+
require 'resque/server'
|
71
|
+
view_dir = ::Resque::Server.views + "/"
|
72
|
+
old_queues_erb_path = view_dir + "queues.erb"
|
73
|
+
old_working_erb_path = view_dir + "working.erb"
|
74
|
+
gem_dir = Gem::Specification.find_by_name("mobilize-base").gem_dir
|
75
|
+
new_queues_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/queues.erb"
|
76
|
+
new_working_erb_path = gem_dir + "/lib/mobilize-base/extensions/resque-server/views/working.erb"
|
77
|
+
[old_queues_erb_path,old_working_erb_path].each{|p| File.delete(p) if File.exists?(p)}
|
78
|
+
require 'fileutils'
|
79
|
+
FileUtils.copy(new_queues_erb_path,old_queues_erb_path)
|
80
|
+
FileUtils.copy(new_working_erb_path,old_working_erb_path)
|
81
|
+
sleep 5 #give them time to die
|
69
82
|
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
|
70
83
|
`#{command}`
|
71
84
|
end
|
data/lib/mobilize-base.rb
CHANGED
@@ -60,14 +60,18 @@ module Mobilize
|
|
60
60
|
end
|
61
61
|
mongoid_config_path = "#{Mobilize::Base.root}/#{Mobilize::Base.config_dir}mongoid.yml"
|
62
62
|
if File.exists?(mongoid_config_path)
|
63
|
-
require 'mongo'
|
64
63
|
require 'mongoid'
|
64
|
+
require 'mongoid-grid_fs'
|
65
65
|
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
66
66
|
require "mobilize-base/models/dataset"
|
67
67
|
require "mobilize-base/models/user"
|
68
|
+
require "mobilize-base/helpers/runner_helper"
|
68
69
|
require "mobilize-base/models/runner"
|
70
|
+
require "mobilize-base/helpers/job_helper"
|
69
71
|
require "mobilize-base/models/job"
|
72
|
+
require "mobilize-base/helpers/stage_helper"
|
70
73
|
require "mobilize-base/models/stage"
|
74
|
+
|
71
75
|
end
|
72
76
|
require 'google_drive'
|
73
77
|
require 'resque'
|
data/lib/samples/gdrive.yml
CHANGED
@@ -4,8 +4,10 @@ development:
|
|
4
4
|
owner:
|
5
5
|
name: owner_development
|
6
6
|
pw: google_drive_password
|
7
|
+
admin_group_name: admins_development
|
7
8
|
admins:
|
8
9
|
- name: admin
|
10
|
+
worker_group_name: workers_development
|
9
11
|
workers:
|
10
12
|
- name: worker_development001
|
11
13
|
pw: worker001_google_drive_password
|
@@ -16,8 +18,10 @@ test:
|
|
16
18
|
owner:
|
17
19
|
name: owner_test
|
18
20
|
pw: google_drive_password
|
21
|
+
admin_group_name: admins_test
|
19
22
|
admins:
|
20
23
|
- name: admin
|
24
|
+
worker_group_name: workers_test
|
21
25
|
workers:
|
22
26
|
- name: worker_test001
|
23
27
|
pw: worker001_google_drive_password
|
@@ -28,8 +32,10 @@ production:
|
|
28
32
|
owner:
|
29
33
|
name: owner_production
|
30
34
|
pw: google_drive_password
|
35
|
+
admin_group_name: admins_production
|
31
36
|
admins:
|
32
37
|
- name: admin
|
38
|
+
worker_group_name: workers_production
|
33
39
|
workers:
|
34
40
|
- name: worker_production001
|
35
41
|
pw: worker001_google_drive_password
|
data/lib/samples/gridfs.yml
CHANGED
@@ -1,10 +1,7 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
4
3
|
max_compressed_write_size: 1000000000 #~1GB
|
5
4
|
test:
|
6
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
7
5
|
max_compressed_write_size: 1000000000 #~1GB
|
8
6
|
production:
|
9
|
-
max_versions: 10 #number of versions of cache to keep in gridfs
|
10
7
|
max_compressed_write_size: 1000000000 #~1GB
|
data/lib/samples/gsheet.yml
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
---
|
2
2
|
development:
|
3
|
-
max_cells:
|
3
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
4
4
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
5
5
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
6
6
|
test:
|
7
|
-
max_cells:
|
7
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
8
8
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
9
9
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
10
10
|
staging:
|
11
|
-
max_cells:
|
11
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
12
12
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
13
13
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
14
14
|
production:
|
15
|
-
max_cells:
|
15
|
+
max_cells: 50000 #current google limit is 400k cells per book but fails earlier
|
16
16
|
read_date_format: "%Y-%m-%d" #format to record when reading sheets
|
17
17
|
sheet_date_format: "%m/%d/%Y" #format to use to parse sheets
|
data/mobilize-base.gemspec
CHANGED
@@ -22,10 +22,10 @@ Gem::Specification.new do |s|
|
|
22
22
|
s.require_paths = ["lib"]
|
23
23
|
|
24
24
|
s.add_runtime_dependency 'rake'
|
25
|
-
s.add_runtime_dependency 'bson','1.
|
26
|
-
s.add_runtime_dependency 'bson_ext','1.
|
27
|
-
s.add_runtime_dependency 'mongo', '1.6.1'
|
25
|
+
s.add_runtime_dependency 'bson','1.8.4'
|
26
|
+
s.add_runtime_dependency 'bson_ext','1.8.4'
|
28
27
|
s.add_runtime_dependency "mongoid", "~>3.0.0"
|
28
|
+
s.add_runtime_dependency 'mongoid-grid_fs'
|
29
29
|
s.add_runtime_dependency 'resque', '1.24.0'
|
30
30
|
s.add_runtime_dependency 'google_drive','0.3.2'
|
31
31
|
s.add_runtime_dependency 'popen4','0.1.2'
|
data/test/base_job_rows.yml
CHANGED
@@ -3,9 +3,13 @@
|
|
3
3
|
trigger: once
|
4
4
|
status: ""
|
5
5
|
stage1: gsheet.write source:"gfile://test_base_1.tsv", target:base1.out
|
6
|
-
|
7
6
|
- name: base2
|
8
7
|
active: true
|
9
8
|
trigger: after base1
|
10
9
|
status: ""
|
11
10
|
stage1: gsheet.write source:base1.out, target:base2.out
|
11
|
+
- name: base3
|
12
|
+
active: true
|
13
|
+
trigger: after base2
|
14
|
+
status: ""
|
15
|
+
stage1: gsheet.write source:"base1."_"", target:base2.out
|
data/test/mobilize-base_test.rb
CHANGED
@@ -1,43 +1,34 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
|
3
3
|
describe "Mobilize" do
|
4
|
-
|
5
|
-
def before
|
6
|
-
puts 'nothing before'
|
7
|
-
end
|
8
|
-
|
9
|
-
# enqueues 4 workers on Resque
|
10
4
|
it "runs integration test" do
|
11
5
|
|
12
6
|
puts "restart test redis"
|
13
|
-
|
7
|
+
TestHelper.restart_test_redis
|
14
8
|
|
15
9
|
puts "clear out test db"
|
16
|
-
|
10
|
+
TestHelper.drop_test_db
|
17
11
|
|
18
12
|
puts "restart workers"
|
19
13
|
Mobilize::Jobtracker.restart_workers!
|
20
14
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
user_name = gdrive_slot.split("@").first
|
25
|
-
u = Mobilize::User.find_or_create_by_name(user_name)
|
26
|
-
assert u.email == gdrive_slot
|
15
|
+
u = TestHelper.owner_user
|
16
|
+
user_name = u.name
|
17
|
+
gdrive_slot = u.email
|
27
18
|
|
28
|
-
|
19
|
+
puts "build test runner"
|
20
|
+
TestHelper.build_test_runner(user_name)
|
29
21
|
assert Mobilize::Jobtracker.workers.length == Mobilize::Resque.config['max_workers'].to_i
|
30
22
|
|
31
23
|
puts "Jobtracker created runner with 'jobs' sheet?"
|
24
|
+
|
32
25
|
r = u.runner
|
33
26
|
jobs_sheet_url = "gsheet://#{r.path}"
|
34
27
|
jobs_sheet = Mobilize::Gsheet.find_by_path(r.path,gdrive_slot)
|
35
28
|
jobs_sheet_dst = Mobilize::Dataset.find_or_create_by_url(jobs_sheet_url)
|
36
29
|
jobs_sheet_tsv = jobs_sheet_dst.read(user_name,gdrive_slot)
|
37
|
-
assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
|
38
30
|
|
39
|
-
|
40
|
-
#Mobilize::Jobtracker.stop!
|
31
|
+
assert jobs_sheet_tsv.tsv_header_array.join.length == 53 #total header length
|
41
32
|
|
42
33
|
puts "add base1 input file"
|
43
34
|
test_filename = "test_base_1"
|
@@ -53,54 +44,22 @@ describe "Mobilize" do
|
|
53
44
|
jobs_sheet.reload
|
54
45
|
jobs_sheet.add_or_update_rows(test_job_rows)
|
55
46
|
#wait for stages to complete
|
56
|
-
|
57
|
-
wait_for_stages
|
47
|
+
TestHelper.wait_for_stages
|
58
48
|
|
59
49
|
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
50
|
+
tsv_hash = {}
|
51
|
+
["base1.out", "base2.out", "base3_stage1.err"].each do |sheet_name|
|
52
|
+
url = "gsheet://#{r.title}/#{sheet_name}"
|
53
|
+
data = Mobilize::Dataset.read_by_url(url,user_name,gdrive_slot)
|
54
|
+
tsv_hash[sheet_name] = data
|
55
|
+
end
|
66
56
|
|
67
|
-
assert
|
57
|
+
assert tsv_hash["base1.out"].to_s.length>0
|
58
|
+
assert tsv_hash["base2.out"] == tsv_hash["base1.out"]
|
68
59
|
|
69
|
-
|
70
|
-
|
71
|
-
Mobilize::Dataset.write_by_url(test_error_sheet_url," ",user_name,gdrive_slot)
|
72
|
-
jobs_sheet.add_or_update_rows(test_job_rows)
|
60
|
+
base3_response = tsv_hash["base3_stage1.err"].tsv_to_hash_array.first['response']
|
61
|
+
assert base3_response == "Unable to parse stage params, make sure you don't have issues with your quotes, commas, or colons."
|
73
62
|
|
74
|
-
#wait for stages to complete
|
75
|
-
wait_for_stages
|
76
|
-
|
77
|
-
test_error_sheet = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1_stage1.err",gdrive_slot)
|
78
|
-
puts "jobtracker posted failing test error to sheet "
|
79
|
-
error_rows = test_error_sheet.read(user_name).tsv_to_hash_array
|
80
|
-
assert error_rows.first['response'] == "Could not get gfile://test_base_1.fail with error: unable to find test_base_1.fail"
|
81
63
|
Mobilize::Jobtracker.stop!
|
82
64
|
end
|
83
|
-
|
84
|
-
def wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
|
85
|
-
time = 0
|
86
|
-
time_since_stage = 0
|
87
|
-
#check for 10 min
|
88
|
-
while time < time_limit and time_since_stage < stage_limit
|
89
|
-
sleep wait_length
|
90
|
-
job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
|
91
|
-
if job_classes.include?("Mobilize::Stage")
|
92
|
-
time_since_stage = 0
|
93
|
-
puts "saw stage at #{time.to_s} seconds"
|
94
|
-
else
|
95
|
-
time_since_stage += wait_length
|
96
|
-
puts "#{time_since_stage.to_s} seconds since stage seen"
|
97
|
-
end
|
98
|
-
time += wait_length
|
99
|
-
puts "total wait time #{time.to_s} seconds"
|
100
|
-
end
|
101
|
-
|
102
|
-
if time >= time_limit
|
103
|
-
raise "Timed out before stage completion"
|
104
|
-
end
|
105
|
-
end
|
106
65
|
end
|
data/test/test_helper.rb
CHANGED
@@ -8,3 +8,82 @@ $dir = File.dirname(File.expand_path(__FILE__))
|
|
8
8
|
ENV['MOBILIZE_ENV'] = 'test'
|
9
9
|
require 'mobilize-base'
|
10
10
|
$TESTING = true
|
11
|
+
module TestHelper
|
12
|
+
def TestHelper.wait_for_stages(time_limit=600,stage_limit=120,wait_length=10)
|
13
|
+
time = 0
|
14
|
+
time_since_stage = 0
|
15
|
+
#check for 10 min
|
16
|
+
while time < time_limit and time_since_stage < stage_limit
|
17
|
+
sleep wait_length
|
18
|
+
job_classes = Mobilize::Resque.jobs.map{|j| j['class']}
|
19
|
+
if job_classes.include?("Mobilize::Stage")
|
20
|
+
time_since_stage = 0
|
21
|
+
puts "saw stage at #{time.to_s} seconds"
|
22
|
+
else
|
23
|
+
time_since_stage += wait_length
|
24
|
+
puts "#{time_since_stage.to_s} seconds since stage seen"
|
25
|
+
end
|
26
|
+
time += wait_length
|
27
|
+
puts "total wait time #{time.to_s} seconds"
|
28
|
+
end
|
29
|
+
|
30
|
+
if time >= time_limit
|
31
|
+
raise "Timed out before stage completion"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
#test methods
|
36
|
+
def TestHelper.restart_test_redis
|
37
|
+
TestHelper.stop_test_redis
|
38
|
+
if !system("which redis-server")
|
39
|
+
raise "** can't find `redis-server` in your path, you need redis to run Resque and Mobilize"
|
40
|
+
end
|
41
|
+
"redis-server #{Mobilize::Base.root}/test/redis-test.conf".bash
|
42
|
+
end
|
43
|
+
|
44
|
+
def TestHelper.stop_test_redis
|
45
|
+
processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
|
46
|
+
pids = processes.map { |process| process.split(" ")[0] }
|
47
|
+
puts "Killing test redis server..."
|
48
|
+
pids.each { |pid| Process.kill("TERM", pid.to_i) }
|
49
|
+
puts "removing redis db dump file"
|
50
|
+
sleep 5
|
51
|
+
`rm -f #{Mobilize::Base.root}/test/dump.rdb #{Mobilize::Base.root}/test/dump-cluster.rdb`
|
52
|
+
end
|
53
|
+
|
54
|
+
def TestHelper.set_test_env
|
55
|
+
ENV['MOBILIZE_ENV']='test'
|
56
|
+
::Resque.redis="localhost:9736"
|
57
|
+
mongoid_config_path = "#{Mobilize::Base.root}/config/mobilize/mongoid.yml"
|
58
|
+
Mongoid.load!(mongoid_config_path, Mobilize::Base.env)
|
59
|
+
end
|
60
|
+
|
61
|
+
def TestHelper.drop_test_db
|
62
|
+
TestHelper.set_test_env
|
63
|
+
Mongoid.session(:default).collections.each do |collection|
|
64
|
+
unless collection.name =~ /^system\./
|
65
|
+
collection.drop
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def TestHelper.build_test_runner(user_name)
|
71
|
+
TestHelper.set_test_env
|
72
|
+
u = Mobilize::User.where(:name=>user_name).first
|
73
|
+
Mobilize::Jobtracker.update_status("delete old books and datasets")
|
74
|
+
# delete any old runner from previous test runs
|
75
|
+
gdrive_slot = Mobilize::Gdrive.owner_email
|
76
|
+
u.runner.gsheet(gdrive_slot).spreadsheet.delete
|
77
|
+
Mobilize::Dataset.find_by_handler_and_path('gbook',u.runner.title).delete
|
78
|
+
Mobilize::Jobtracker.update_status("enqueue jobtracker, wait 45s")
|
79
|
+
Mobilize::Jobtracker.start
|
80
|
+
sleep 45
|
81
|
+
end
|
82
|
+
|
83
|
+
def TestHelper.owner_user
|
84
|
+
gdrive_slot = Mobilize::Gdrive.owner_email
|
85
|
+
puts "create user 'mobilize'"
|
86
|
+
user_name = gdrive_slot.split("@").first
|
87
|
+
return Mobilize::User.find_or_create_by_name(user_name)
|
88
|
+
end
|
89
|
+
end
|