mobilize-base 1.0.94 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +7 -7
- data/lib/mobilize-base/extensions/google_drive/file.rb +12 -6
- data/lib/mobilize-base/extensions/google_drive/worksheet.rb +20 -2
- data/lib/mobilize-base/extensions/hash.rb +36 -42
- data/lib/mobilize-base/handlers/gdrive.rb +4 -0
- data/lib/mobilize-base/handlers/gfile.rb +31 -5
- data/lib/mobilize-base/handlers/gridfs.rb +3 -3
- data/lib/mobilize-base/handlers/gsheet.rb +9 -8
- data/lib/mobilize-base/models/dataset.rb +6 -12
- data/lib/mobilize-base/models/runner.rb +8 -2
- data/lib/mobilize-base/models/stage.rb +47 -13
- data/lib/mobilize-base/tasks.rb +4 -1
- data/lib/mobilize-base/version.rb +1 -1
- data/test/base_job_rows.yml +6 -7
- data/test/mobilize-base_test.rb +3 -3
- metadata +4 -4
data/README.md
CHANGED
@@ -501,7 +501,7 @@ name>))` and enter values under each header:
|
|
501
501
|
* status Mobilize writes this field with the last status returned by the job
|
502
502
|
|
503
503
|
* stage1..stage5 List of stages to be performed by the job.
|
504
|
-
* Stages have this syntax:
|
504
|
+
* Stages have this syntax: `<handler>.<call> <params>`.
|
505
505
|
* handler specifies the file that should receive the stage
|
506
506
|
* the call specifies the method within the file. The method should
|
507
507
|
be called `"<handler>.<call>_by_stage_path"`
|
@@ -514,14 +514,14 @@ curly braces).
|
|
514
514
|
* gsheet.read `source: <input_gsheet_full_path>`, which reads the sheet.
|
515
515
|
* The gsheet_full_path should be of the form `<gbook_name>/<gsheet_name>`. The test uses
|
516
516
|
"Requestor_mobilize(test)/base1_stage1.in".
|
517
|
-
* gsheet.write `source: <
|
517
|
+
* gsheet.write `source: <stage_name>`,`target: <target_gsheet_path>`,
|
518
518
|
which writes the specified stage output to the target_gsheet.
|
519
|
-
* The
|
520
|
-
|
521
|
-
|
522
|
-
from the
|
519
|
+
* The stage_name should be of the form `<stage_column>`. The test uses "stage1" for the first test
|
520
|
+
and "Runner_mobilize(test)/base1.out" for the second test. The first
|
521
|
+
takes the output from the first stage and the second reads it straight
|
522
|
+
from the referenced sheet.
|
523
523
|
* The test uses "Requestor_mobilize(test)/base1.out" and
|
524
|
-
"
|
524
|
+
"Runner_mobilize(test)/base2.out" for target sheets.
|
525
525
|
|
526
526
|
<a name='section_Start_Run_Test'></a>
|
527
527
|
### Run Test
|
@@ -40,6 +40,16 @@ module GoogleDrive
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
+
def read(user)
|
44
|
+
f = self
|
45
|
+
entry = f.acl_entry("#{user}@#{Mobilize::Gdrive.domain}")
|
46
|
+
if entry and ['reader','writer','owner'].include?(entry.role)
|
47
|
+
f.download_to_string
|
48
|
+
else
|
49
|
+
raise "User #{user} is not allowed to read #{f.title}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
43
53
|
def update_acl(email,role="writer")
|
44
54
|
f = self
|
45
55
|
#need these flags for HTTP retries
|
@@ -70,15 +80,11 @@ module GoogleDrive
|
|
70
80
|
f = self
|
71
81
|
f.acls.select{|a| ['group','user'].include?(a.scope_type) and a.scope == email}.first
|
72
82
|
end
|
73
|
-
|
74
83
|
def entry_hash
|
75
84
|
f = self
|
76
85
|
dfe_xml = f.document_feed_entry.to_xml
|
77
|
-
|
78
|
-
|
79
|
-
rescue
|
80
|
-
{}
|
81
|
-
end
|
86
|
+
result = Nokogiri::XML(dfe_xml)
|
87
|
+
{ result.root.name.to_sym => Hash.xml_node_to_hash(result.root)}[:entry]
|
82
88
|
end
|
83
89
|
end
|
84
90
|
end
|
@@ -44,11 +44,15 @@ module GoogleDrive
|
|
44
44
|
sheet.save
|
45
45
|
end
|
46
46
|
|
47
|
-
def merge(merge_sheet)
|
47
|
+
def merge(merge_sheet,user)
|
48
48
|
#write the top left of sheet
|
49
49
|
#with the contents of merge_sheet
|
50
50
|
sheet = self
|
51
51
|
sheet.reload
|
52
|
+
entry = merge_sheet.spreadsheet.acl_entry("#{user}@#{Mobilize::Gdrive.domain}")
|
53
|
+
unless entry and ['writer','owner'].include?(entry.role)
|
54
|
+
raise "User #{user} is not allowed to write to #{merge_sheet.spreadsheet.title}"
|
55
|
+
end
|
52
56
|
merge_sheet.reload
|
53
57
|
curr_rows = sheet.num_rows
|
54
58
|
curr_cols = sheet.num_cols
|
@@ -77,8 +81,22 @@ module GoogleDrive
|
|
77
81
|
sheet.save
|
78
82
|
end
|
79
83
|
|
80
|
-
def
|
84
|
+
def read(user)
|
81
85
|
sheet = self
|
86
|
+
entry = sheet.spreadsheet.acl_entry("#{user}@#{Mobilize::Gdrive.domain}")
|
87
|
+
if entry and ['reader','writer','owner'].include?(entry.role)
|
88
|
+
sheet.to_tsv
|
89
|
+
else
|
90
|
+
raise "User #{user} is not allowed to read #{sheet.spreadsheet.title}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def write(tsv,user)
|
95
|
+
sheet = self
|
96
|
+
entry = sheet.spreadsheet.acl_entry("#{user}@#{Mobilize::Gdrive.domain}")
|
97
|
+
unless entry and ['writer','owner'].include?(entry.role)
|
98
|
+
raise "User #{user} is not allowed to write to #{sheet.spreadsheet.title}"
|
99
|
+
end
|
82
100
|
tsvrows = tsv.split("\n")
|
83
101
|
#no rows, no write
|
84
102
|
return true if tsvrows.length==0
|
@@ -29,55 +29,49 @@ class Hash
|
|
29
29
|
return self
|
30
30
|
end
|
31
31
|
# BEGIN methods to create hash from XML
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
result_hash = {}
|
45
|
-
if node.attributes != {}
|
46
|
-
result_hash[:attributes] = {}
|
47
|
-
node.attributes.keys.each do |key|
|
48
|
-
result_hash[:attributes][node.attributes[key].name.to_sym] = prepare(node.attributes[key].value)
|
49
|
-
end
|
32
|
+
def Hash.from_xml(xml_io)
|
33
|
+
result = Nokogiri::XML(xml_io)
|
34
|
+
return { result.root.name.to_sym => Hash.xml_node_to_hash(result.root)}
|
35
|
+
end
|
36
|
+
def Hash.xml_node_to_hash(node)
|
37
|
+
# If we are at the root of the document, start the hash
|
38
|
+
if node.element?
|
39
|
+
result_hash = {}
|
40
|
+
if node.attributes != {}
|
41
|
+
result_hash[:attributes] = {}
|
42
|
+
node.attributes.keys.each do |key|
|
43
|
+
result_hash[:attributes][node.attributes[key].name.to_sym] = prepare(node.attributes[key].value)
|
50
44
|
end
|
51
|
-
|
52
|
-
|
53
|
-
|
45
|
+
end
|
46
|
+
if node.children.size > 0
|
47
|
+
node.children.each do |child|
|
48
|
+
result = xml_node_to_hash(child)
|
54
49
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
end
|
59
|
-
elsif result_hash[child.name.to_sym]
|
60
|
-
if result_hash[child.name.to_sym].is_a?(Object::Array)
|
61
|
-
result_hash[child.name.to_sym] << prepare(result)
|
62
|
-
else
|
63
|
-
result_hash[child.name.to_sym] = [result_hash[child.name.to_sym]] << prepare(result)
|
64
|
-
end
|
65
|
-
else
|
66
|
-
result_hash[child.name.to_sym] = prepare(result)
|
50
|
+
if child.name == "text"
|
51
|
+
unless child.next_sibling || child.previous_sibling
|
52
|
+
return prepare(result)
|
67
53
|
end
|
54
|
+
elsif result_hash[child.name.to_sym]
|
55
|
+
if result_hash[child.name.to_sym].is_a?(Object::Array)
|
56
|
+
result_hash[child.name.to_sym] << prepare(result)
|
57
|
+
else
|
58
|
+
result_hash[child.name.to_sym] = [result_hash[child.name.to_sym]] << prepare(result)
|
59
|
+
end
|
60
|
+
else
|
61
|
+
result_hash[child.name.to_sym] = prepare(result)
|
68
62
|
end
|
63
|
+
end
|
69
64
|
|
70
|
-
|
71
|
-
else
|
72
|
-
return result_hash
|
73
|
-
end
|
65
|
+
return result_hash
|
74
66
|
else
|
75
|
-
return
|
67
|
+
return result_hash
|
76
68
|
end
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
69
|
+
else
|
70
|
+
return prepare(node.content.to_s)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
def Hash.prepare(data)
|
74
|
+
(data.class == String && data.to_i.to_s == data) ? data.to_i : data
|
81
75
|
end
|
82
76
|
def to_struct(struct_name)
|
83
77
|
Struct.new(struct_name,*keys).new(*values)
|
@@ -18,10 +18,6 @@ module Mobilize
|
|
18
18
|
file.update_acl(gdrive_slot,role)
|
19
19
|
end
|
20
20
|
|
21
|
-
def Gfile.find_by_path(path,gdrive_slot)
|
22
|
-
Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"}).first
|
23
|
-
end
|
24
|
-
|
25
21
|
def Gfile.read_by_stage_path(stage_path)
|
26
22
|
#reserve gdrive_slot account for read
|
27
23
|
gdrive_slot = Gdrive.slot_worker_by_path(s.path)
|
@@ -31,7 +27,37 @@ module Mobilize
|
|
31
27
|
out_tsv = Gfile.find_by_path(gfile_path,gdrive_slot).read
|
32
28
|
#use Gridfs to cache result
|
33
29
|
out_url = "gridfs://#{s.path}/out"
|
34
|
-
Dataset.
|
30
|
+
Dataset.write_by_url(out_url,out_tsv,s.job.runner.user.name)
|
31
|
+
end
|
32
|
+
|
33
|
+
def Gfile.find_by_path(path)
|
34
|
+
#file must be owned by owner
|
35
|
+
gdrive_slot = Gdrive.owner_email
|
36
|
+
files = Gdrive.files(gdrive_slot,{"title"=>path,"title-exact"=>"true"})
|
37
|
+
dst = Dataset.find_or_create_by_handler_and_path('gfile',path)
|
38
|
+
#there should only be one file with each path, otherwise we have fail
|
39
|
+
file = nil
|
40
|
+
if files.length>1
|
41
|
+
#keep most recent file, delete the rest
|
42
|
+
files.sort_by do |f|
|
43
|
+
(f.entry_hash[:published] || Time.now).to_time
|
44
|
+
end.reverse.each_with_index do |f,f_i|
|
45
|
+
if f_i == 0
|
46
|
+
file = f
|
47
|
+
else
|
48
|
+
#delete the old file
|
49
|
+
f.delete
|
50
|
+
("Deleted duplicate file #{path}").oputs
|
51
|
+
end
|
52
|
+
end
|
53
|
+
else
|
54
|
+
file = files.first
|
55
|
+
end
|
56
|
+
#always make sure dataset http URL is up to date
|
57
|
+
#and that it has admin acl
|
58
|
+
dst.update_attributes(:http_url=>file.human_url)
|
59
|
+
file.add_admin_acl
|
60
|
+
return file
|
35
61
|
end
|
36
62
|
end
|
37
63
|
end
|
@@ -11,7 +11,7 @@ module Mobilize
|
|
11
11
|
return ::Mongo::GridFileSystem.new(::Mongo::Connection.new(host,port).db(database_name))
|
12
12
|
end
|
13
13
|
|
14
|
-
def Gridfs.read_by_dataset_path(dst_path)
|
14
|
+
def Gridfs.read_by_dataset_path(dst_path,user)
|
15
15
|
begin
|
16
16
|
zs=Gridfs.grid.open(dst_path,'r').read
|
17
17
|
return ::Zlib::Inflate.inflate(zs)
|
@@ -20,10 +20,10 @@ module Mobilize
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def Gridfs.write_by_dataset_path(dst_path,string)
|
23
|
+
def Gridfs.write_by_dataset_path(dst_path,string,user)
|
24
24
|
zs = ::Zlib::Deflate.deflate(string)
|
25
25
|
raise "compressed string too large for Gridfs write" if zs.length > Gridfs.config['max_compressed_write_size']
|
26
|
-
curr_zs = Gridfs.read_by_dataset_path(dst_path).to_s
|
26
|
+
curr_zs = Gridfs.read_by_dataset_path(dst_path,user).to_s
|
27
27
|
#write a new version when there is a change
|
28
28
|
if curr_zs != zs
|
29
29
|
Gridfs.grid.open(dst_path,'w',:versions => Gridfs.config['max_versions']){|f| f.write(zs)}
|
@@ -37,11 +37,12 @@ module Mobilize
|
|
37
37
|
gdrive_slot = Gdrive.slot_worker_by_path(stage_path)
|
38
38
|
return false unless gdrive_slot
|
39
39
|
s = Stage.where(:path=>stage_path).first
|
40
|
+
user = s.job.runner.user.name
|
40
41
|
gsheet_path = s.params['source']
|
41
|
-
out_tsv = Gsheet.find_by_path(gsheet_path,gdrive_slot).
|
42
|
+
out_tsv = Gsheet.find_by_path(gsheet_path,gdrive_slot).read(user)
|
42
43
|
#use Gridfs to cache result
|
43
44
|
out_url = "gridfs://#{s.path}/out"
|
44
|
-
Dataset.
|
45
|
+
Dataset.write_by_url(out_url,out_tsv,Gdrive.owner_name)
|
45
46
|
end
|
46
47
|
|
47
48
|
def Gsheet.write_by_stage_path(stage_path)
|
@@ -49,24 +50,24 @@ module Mobilize
|
|
49
50
|
#return blank response if there are no slots available
|
50
51
|
return nil unless gdrive_slot
|
51
52
|
s = Stage.where(:path=>stage_path).first
|
52
|
-
|
53
|
+
user = s.job.runner.user.name
|
53
54
|
target_path = s.params['target']
|
54
|
-
source_dst = s.
|
55
|
-
tsv = source_dst.read
|
55
|
+
source_dst = s.source_dsts(gdrive_slot).first
|
56
|
+
tsv = source_dst.read(user)
|
56
57
|
sheet_name = target_path.split("/").last
|
57
58
|
temp_path = [stage_path.gridsafe,sheet_name].join("/")
|
58
59
|
temp_sheet = Gsheet.find_or_create_by_path(temp_path,gdrive_slot)
|
59
|
-
temp_sheet.write(tsv)
|
60
|
+
temp_sheet.write(tsv,Gdrive.owner_name)
|
60
61
|
temp_sheet.check_and_fix(tsv)
|
61
62
|
target_sheet = Gsheet.find_or_create_by_path(target_path,gdrive_slot)
|
62
|
-
target_sheet.merge(temp_sheet)
|
63
|
+
target_sheet.merge(temp_sheet,user)
|
63
64
|
#delete the temp sheet's book
|
64
65
|
temp_sheet.spreadsheet.delete
|
65
66
|
status = "Write successful for #{target_path}"
|
66
67
|
s.update_status(status)
|
67
68
|
#use Gridfs to cache result
|
68
69
|
out_url = "gridfs://#{s.path}/out"
|
69
|
-
Dataset.
|
70
|
+
Dataset.write_by_url(out_url,status,Gdrive.owner_name)
|
70
71
|
end
|
71
72
|
end
|
72
73
|
end
|
@@ -38,30 +38,24 @@ module Mobilize
|
|
38
38
|
return dst
|
39
39
|
end
|
40
40
|
|
41
|
-
def Dataset.
|
41
|
+
def Dataset.write_by_url(url,string,user)
|
42
42
|
dst = Dataset.find_or_create_by_url(url)
|
43
|
-
dst.write(string)
|
43
|
+
dst.write(string,user)
|
44
44
|
url
|
45
45
|
end
|
46
46
|
|
47
|
-
def read
|
47
|
+
def read(user)
|
48
48
|
dst = self
|
49
49
|
dst.update_attributes(:last_read_at=>Time.now.utc)
|
50
|
-
"Mobilize::#{dst.handler.humanize}".constantize.read_by_dataset_path(dst.path)
|
50
|
+
"Mobilize::#{dst.handler.humanize}".constantize.read_by_dataset_path(dst.path,user)
|
51
51
|
end
|
52
52
|
|
53
|
-
def write(string)
|
53
|
+
def write(string,user)
|
54
54
|
dst = self
|
55
|
-
"Mobilize::#{dst.handler.humanize}".constantize.write_by_dataset_path(dst.path,string)
|
55
|
+
"Mobilize::#{dst.handler.humanize}".constantize.write_by_dataset_path(dst.path,string,user)
|
56
56
|
dst.raw_size = string.length
|
57
57
|
dst.save!
|
58
58
|
return true
|
59
59
|
end
|
60
|
-
|
61
|
-
def delete
|
62
|
-
dst = self
|
63
|
-
"Mobilize::#{dst.handler.humanize}".constantize.delete_by_dataset_path(dst.path)
|
64
|
-
return true
|
65
|
-
end
|
66
60
|
end
|
67
61
|
end
|
@@ -72,6 +72,12 @@ module Mobilize
|
|
72
72
|
Dataset.find_or_create_by_url("gridfs://#{r.path}")
|
73
73
|
end
|
74
74
|
|
75
|
+
def gbook(gdrive_slot)
|
76
|
+
r = self
|
77
|
+
title = r.path.split("/").first
|
78
|
+
Gbook.find_all_by_path(title,gdrive_slot).first
|
79
|
+
end
|
80
|
+
|
75
81
|
def gsheet(gdrive_slot)
|
76
82
|
r = self
|
77
83
|
jobs_sheet = Gsheet.find_or_create_by_path(r.path,gdrive_slot)
|
@@ -82,9 +88,9 @@ module Mobilize
|
|
82
88
|
|
83
89
|
def read_gsheet(gdrive_slot)
|
84
90
|
r = self
|
85
|
-
gsheet_tsv = r.gsheet(gdrive_slot).
|
91
|
+
gsheet_tsv = r.gsheet(gdrive_slot).read(Gdrive.owner_name)
|
86
92
|
#cache in DB
|
87
|
-
r.cache.write(gsheet_tsv)
|
93
|
+
r.cache.write(gsheet_tsv,Gdrive.owner_name)
|
88
94
|
#turn it into a hash array
|
89
95
|
gsheet_jobs = gsheet_tsv.tsv_to_hash_array
|
90
96
|
#go through each job, update relevant job with its params
|
@@ -68,19 +68,6 @@ module Mobilize
|
|
68
68
|
return j.stages[s.idx]
|
69
69
|
end
|
70
70
|
|
71
|
-
def source_dst(source_path)
|
72
|
-
#gets dataset based on path given in source parameter
|
73
|
-
s = self
|
74
|
-
source_job_name, source_stage_name = if source_path.index("/")
|
75
|
-
source_path.split("/")
|
76
|
-
else
|
77
|
-
[nil, source_path]
|
78
|
-
end
|
79
|
-
source_stage_path = "#{s.job.runner.path}/#{source_job_name || s.job.name}/#{source_stage_name}"
|
80
|
-
source_stage = Stage.where(:path=>source_stage_path).first
|
81
|
-
source_stage.out_dst
|
82
|
-
end
|
83
|
-
|
84
71
|
def Stage.perform(id,*args)
|
85
72
|
s = Stage.where(:path=>id).first
|
86
73
|
j = s.job
|
@@ -118,6 +105,53 @@ module Mobilize
|
|
118
105
|
return true
|
119
106
|
end
|
120
107
|
|
108
|
+
def source_dsts(gdrive_slot)
|
109
|
+
#returns an array of Datasets corresponding to
|
110
|
+
#gridfs caches for stage outputs, gsheets and gfiles
|
111
|
+
#or dataset pointers for other handlers
|
112
|
+
s = self
|
113
|
+
params = s.params
|
114
|
+
source_paths = if params['sources']
|
115
|
+
params['sources']
|
116
|
+
elsif params['source']
|
117
|
+
[params['source']]
|
118
|
+
end
|
119
|
+
user = s.job.runner.user.name
|
120
|
+
return [] if (source_paths.class!=Array or source_paths.length==0)
|
121
|
+
dsts = []
|
122
|
+
source_paths.each do |source_path|
|
123
|
+
if source_path.index(/^stage[1-5]$/)
|
124
|
+
source_stage_path = "#{s.job.runner.path}/#{s.job.name}/#{source_path}"
|
125
|
+
source_stage = Stage.where(:path=>source_stage_path).first
|
126
|
+
dsts << source_stage.out_dst
|
127
|
+
elsif source_path.index("://")
|
128
|
+
#find or create by url
|
129
|
+
dsts << Dataset.find_or_create_by_url(source_path)
|
130
|
+
else
|
131
|
+
if source_path.index("/")
|
132
|
+
#slashes mean sheets
|
133
|
+
out_tsv = Gsheet.find_by_path(source_path,gdrive_slot).read(user)
|
134
|
+
else
|
135
|
+
#check sheets in runner
|
136
|
+
r = s.job.runner
|
137
|
+
runner_sheet = r.gbook.worksheet_by_title(source_path)
|
138
|
+
out_tsv = if runner_sheet
|
139
|
+
runner_sheet.read(user)
|
140
|
+
else
|
141
|
+
#check for gfile. will fail if there isn't one.
|
142
|
+
Gfile.find_by_path(source_path).read(user)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
#use Gridfs to cache gdrive results
|
146
|
+
file_name = source_path.split("/").last
|
147
|
+
out_url = "gridfs://#{s.path}/#{file_name}"
|
148
|
+
Dataset.write_by_url(out_url,out_tsv,user)
|
149
|
+
dsts << Dataset.find_by_url(out_url)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
return dsts
|
153
|
+
end
|
154
|
+
|
121
155
|
def enqueue!
|
122
156
|
s = self
|
123
157
|
::Resque::Job.create("mobilize",Stage,s.path,{})
|
data/lib/mobilize-base/tasks.rb
CHANGED
@@ -65,7 +65,10 @@ namespace :mobilize_base do
|
|
65
65
|
resque_web_extension_path = "#{full_config_dir}resque_web.rb"
|
66
66
|
#kill any resque-web for now
|
67
67
|
`ps aux | grep resque-web | awk '{print $2}' | xargs kill`
|
68
|
-
|
68
|
+
resque_redis_port_args = if Mobilize::Base.env == 'test'
|
69
|
+
" -r localhost:#{Mobilize::Base.config('resque')['redis_port']}"
|
70
|
+
end.to_s
|
71
|
+
command = "bundle exec resque-web -p #{port.to_s} #{resque_web_extension_path} #{resque_redis_port_args}"
|
69
72
|
`#{command}`
|
70
73
|
end
|
71
74
|
desc "create indexes for all base models in mongodb"
|
data/test/base_job_rows.yml
CHANGED
@@ -1,13 +1,12 @@
|
|
1
|
-
- name:
|
1
|
+
- name: base1
|
2
2
|
active: true
|
3
3
|
trigger: once
|
4
4
|
status: ""
|
5
|
-
stage1:
|
6
|
-
stage2:
|
5
|
+
stage1: gsheet.read source:"Runner_mobilize(test)/base1_stage1.in"
|
6
|
+
stage2: gsheet.write source:"stage1", target:"Runner_mobilize(test)/base1.out"
|
7
7
|
|
8
|
-
- name:
|
8
|
+
- name: base2
|
9
9
|
active: true
|
10
|
-
trigger:
|
10
|
+
trigger: after base1
|
11
11
|
status: ""
|
12
|
-
stage1:
|
13
|
-
stage2: 'gsheet.write source:"stage1", target:"Runner_mobilize(test)/base2.out"'
|
12
|
+
stage1: gsheet.write source:"Runner_mobilize(test)/base1.out", target:"Runner_mobilize(test)/base2.out"
|
data/test/mobilize-base_test.rb
CHANGED
@@ -31,7 +31,7 @@ describe "Mobilize" do
|
|
31
31
|
puts "Jobtracker created runner with 'jobs' sheet?"
|
32
32
|
r = u.runner
|
33
33
|
jobs_sheet = r.gsheet(gdrive_slot)
|
34
|
-
tsv = jobs_sheet.
|
34
|
+
tsv = jobs_sheet.read(Mobilize::Gdrive.owner_name)
|
35
35
|
assert tsv.length == 61 #headers only
|
36
36
|
|
37
37
|
puts "add base1_stage1 input sheet"
|
@@ -39,7 +39,7 @@ describe "Mobilize" do
|
|
39
39
|
|
40
40
|
test_source_ha = ::YAML.load_file("#{Mobilize::Base.root}/test/base1_stage1.yml")*40
|
41
41
|
test_source_tsv = test_source_ha.hash_array_to_tsv
|
42
|
-
test_source_sheet.write(test_source_tsv)
|
42
|
+
test_source_sheet.write(test_source_tsv,Mobilize::Gdrive.owner_name)
|
43
43
|
|
44
44
|
puts "add row to jobs sheet, wait 150s"
|
45
45
|
test_job_rows = ::YAML.load_file("#{Mobilize::Base.root}/test/base_job_rows.yml")
|
@@ -57,7 +57,7 @@ describe "Mobilize" do
|
|
57
57
|
|
58
58
|
jobs_sheet.add_or_update_rows([{'name'=>'base1','active'=>true}])
|
59
59
|
sleep 120
|
60
|
-
|
60
|
+
|
61
61
|
test_target_sheet_2 = Mobilize::Gsheet.find_by_path("#{r.path.split("/")[0..-2].join("/")}/base1.out",gdrive_slot)
|
62
62
|
puts "jobtracker posted test sheet data to test destination, and checksum succeeded?"
|
63
63
|
assert test_target_sheet_2.to_tsv == test_source_sheet.to_tsv
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mobilize-base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-01-
|
12
|
+
date: 2013-01-18 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|
@@ -238,7 +238,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
238
238
|
version: '0'
|
239
239
|
segments:
|
240
240
|
- 0
|
241
|
-
hash: -
|
241
|
+
hash: -781501523970053172
|
242
242
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
243
243
|
none: false
|
244
244
|
requirements:
|
@@ -247,7 +247,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
247
247
|
version: '0'
|
248
248
|
segments:
|
249
249
|
- 0
|
250
|
-
hash: -
|
250
|
+
hash: -781501523970053172
|
251
251
|
requirements: []
|
252
252
|
rubyforge_project: mobilize-base
|
253
253
|
rubygems_version: 1.8.24
|