rbbt-util 5.32.22 → 5.32.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/rbbt +2 -3
- data/bin/rbbt_find.rb +74 -0
- data/lib/rbbt/tsv/excel.rb +16 -8
- data/lib/rbbt/tsv/manipulate.rb +10 -0
- data/lib/rbbt/tsv/parser.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +0 -44
- data/lib/rbbt/workflow/dependencies.rb +45 -0
- data/lib/rbbt/workflow/examples.rb +1 -1
- data/lib/rbbt/workflow/step.rb +4 -0
- data/lib/rbbt/workflow/util/data.rb +141 -11
- data/lib/rbbt/workflow.rb +20 -14
- data/share/rbbt_commands/workflow/task +5 -3
- data/test/rbbt/tsv/test_excel.rb +38 -4
- data/test/rbbt/workflow/util/test_data.rb +21 -2
- metadata +4 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 072aa8b010c3f395051fd4e29f6ac22647944eec529bdf05be77ac6ad5260a37
|
|
4
|
+
data.tar.gz: d68843b2fe56019dd63165a34c97ec23d9e60933e4493bdb61f29985548bca69
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9d572a394b5736d8780021bc8ec65ba4381dc8cb84f111704854ce77ceca8da0ca72626e0144a04ab5c9b552a5566f4bdd553cad2a70050216fae231179beb95
|
|
7
|
+
data.tar.gz: ce51f8a4f0f5b4028148deb7fd3e57a4fbf5bb531f366378a2bae31354b799a66c6ba2768d9ae25578b3a0b09ee4e29f817f6ab768166bed39e4245b35a4a7e4
|
data/bin/rbbt
CHANGED
|
@@ -262,9 +262,8 @@ rescue ParameterException
|
|
|
262
262
|
puts
|
|
263
263
|
exit_status = -1
|
|
264
264
|
exit exit_status
|
|
265
|
-
rescue SystemExit
|
|
266
|
-
|
|
267
|
-
exit_status = $!.exit_status
|
|
265
|
+
rescue SystemExit,CmdStop
|
|
266
|
+
exit_status = $!.status
|
|
268
267
|
exit exit_status
|
|
269
268
|
rescue Exception
|
|
270
269
|
Log.exception $!
|
data/bin/rbbt_find.rb
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'rbbt-util'
|
|
4
|
+
require 'rbbt/util/simpleopt'
|
|
5
|
+
|
|
6
|
+
$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
|
|
7
|
+
|
|
8
|
+
options = SOPT.setup <<EOF
|
|
9
|
+
|
|
10
|
+
Find a path
|
|
11
|
+
|
|
12
|
+
$ #{$0} [options] [<subpath>] <path>
|
|
13
|
+
|
|
14
|
+
Use - to read from STDIN
|
|
15
|
+
|
|
16
|
+
-h--help Print this help
|
|
17
|
+
-w--workflows Workflow to load
|
|
18
|
+
-s--search_path* Workflow to load
|
|
19
|
+
-l--list List contents of resolved directories
|
|
20
|
+
-n--nocolor Don't color output
|
|
21
|
+
EOF
|
|
22
|
+
if options[:help]
|
|
23
|
+
if defined? rbbt_usage
|
|
24
|
+
rbbt_usage
|
|
25
|
+
else
|
|
26
|
+
puts SOPT.doc
|
|
27
|
+
end
|
|
28
|
+
exit 0
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
subpath, path = ARGV
|
|
32
|
+
path, subpath = subpath, nil if path.nil?
|
|
33
|
+
|
|
34
|
+
begin
|
|
35
|
+
require 'rbbt/workflow'
|
|
36
|
+
workflow = Workflow.require_workflow subpath
|
|
37
|
+
subpath = workflow.libdir
|
|
38
|
+
rescue
|
|
39
|
+
Log.exception $!
|
|
40
|
+
end if subpath && subpath =~ /^[A-Z][a-zA-Z]+$/
|
|
41
|
+
|
|
42
|
+
path = subpath ? Path.setup(subpath)[path] : Path.setup(path)
|
|
43
|
+
|
|
44
|
+
search_path = options[:search_path].to_sym if options.include? :search_path
|
|
45
|
+
nocolor = options[:nocolor]
|
|
46
|
+
|
|
47
|
+
found = if search_path
|
|
48
|
+
[path.find(search_path)]
|
|
49
|
+
else
|
|
50
|
+
path.find_all
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
found.each do |path|
|
|
54
|
+
if options[:list] && File.directory?(path)
|
|
55
|
+
puts Log.color :blue, path
|
|
56
|
+
path.glob("*").each do |subpath|
|
|
57
|
+
if nocolor
|
|
58
|
+
puts subpath
|
|
59
|
+
else
|
|
60
|
+
color = File.directory?(subpath) ? :blue : nil
|
|
61
|
+
puts " " << Log.color(color, subpath)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
else
|
|
65
|
+
if nocolor
|
|
66
|
+
puts path
|
|
67
|
+
else
|
|
68
|
+
color = File.exists?(path) ? (File.directory?(path) ? :blue : nil) : :red
|
|
69
|
+
puts Log.color color, path
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
data/lib/rbbt/tsv/excel.rb
CHANGED
|
@@ -177,7 +177,7 @@ module TSV
|
|
|
177
177
|
|
|
178
178
|
sheet ||= "0"
|
|
179
179
|
workbook = RubyXL::Parser.parse file
|
|
180
|
-
if sheet &&
|
|
180
|
+
if sheet && sheet =~ /^\d+$/
|
|
181
181
|
sheet = workbook.worksheets.collect{|s| s.sheet_name }[sheet.to_i]
|
|
182
182
|
end
|
|
183
183
|
sheet_name = sheet
|
|
@@ -185,7 +185,9 @@ module TSV
|
|
|
185
185
|
|
|
186
186
|
TmpFile.with_file :extension => Misc.sanitize_filename(sheet_name.to_s) do |filename|
|
|
187
187
|
|
|
188
|
-
sheet =
|
|
188
|
+
sheet = sheet_name ? workbook[sheet_name] : workbook.worksheets.first
|
|
189
|
+
|
|
190
|
+
raise "No sheet #{sheet_name} found" if sheet.nil?
|
|
189
191
|
|
|
190
192
|
rows = []
|
|
191
193
|
|
|
@@ -217,21 +219,27 @@ module TSV
|
|
|
217
219
|
end
|
|
218
220
|
|
|
219
221
|
def self.write(tsv, file, options = {})
|
|
220
|
-
sheet = Misc.process_options options, :sheet
|
|
222
|
+
sheet, add_sheet = Misc.process_options options, :sheet, :add_sheet
|
|
221
223
|
|
|
222
224
|
fields, rows = TSV._excel_data(tsv, options)
|
|
223
225
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
226
|
+
if Open.exists?(file) && add_sheet
|
|
227
|
+
book = RubyXL::Parser.parse file
|
|
228
|
+
sheet1 = book.add_worksheet(sheet)
|
|
229
|
+
else
|
|
230
|
+
book = RubyXL::Workbook.new
|
|
231
|
+
sheet1 = book.worksheets.first
|
|
232
|
+
sheet1.sheet_name = sheet if sheet
|
|
233
|
+
end
|
|
227
234
|
|
|
228
235
|
fields.each_with_index do |e,i|
|
|
229
236
|
sheet1.add_cell(0, i, e)
|
|
230
|
-
end
|
|
237
|
+
end if fields
|
|
231
238
|
|
|
232
239
|
rows.each_with_index do |cells,i|
|
|
240
|
+
i += 1 if fields
|
|
233
241
|
cells.each_with_index do |e,j|
|
|
234
|
-
sheet1.add_cell(i
|
|
242
|
+
sheet1.add_cell(i, j, e)
|
|
235
243
|
end
|
|
236
244
|
end
|
|
237
245
|
|
data/lib/rbbt/tsv/manipulate.rb
CHANGED
|
@@ -356,6 +356,16 @@ module TSV
|
|
|
356
356
|
elems.sort_by{|k,v| v}.collect{|k,v| k}
|
|
357
357
|
end
|
|
358
358
|
|
|
359
|
+
def subset(keys)
|
|
360
|
+
new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
|
|
361
|
+
self.with_unnamed do
|
|
362
|
+
keys.each do |k|
|
|
363
|
+
new[k] = self[k]
|
|
364
|
+
end
|
|
365
|
+
end
|
|
366
|
+
new
|
|
367
|
+
end
|
|
368
|
+
|
|
359
369
|
def select(method = nil, invert = false, &block)
|
|
360
370
|
new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
|
|
361
371
|
|
data/lib/rbbt/tsv/parser.rb
CHANGED
|
@@ -172,50 +172,6 @@ module Workflow
|
|
|
172
172
|
}.tap{|h| IndiferentHash.setup(h)}
|
|
173
173
|
end
|
|
174
174
|
|
|
175
|
-
def assign_dep_inputs(_inputs, options, all_d, task_info)
|
|
176
|
-
IndiferentHash.setup(_inputs)
|
|
177
|
-
|
|
178
|
-
options.each{|i,v|
|
|
179
|
-
next if i == :compute or i == "compute"
|
|
180
|
-
case v
|
|
181
|
-
when :compute
|
|
182
|
-
compute = v
|
|
183
|
-
when Symbol
|
|
184
|
-
rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
|
|
185
|
-
|
|
186
|
-
if rec_dependency.nil?
|
|
187
|
-
if _inputs.include?(v)
|
|
188
|
-
#_inputs[i] = _inputs.delete(v)
|
|
189
|
-
_inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
|
|
190
|
-
else
|
|
191
|
-
_inputs[i] = v unless _inputs.include? i
|
|
192
|
-
end
|
|
193
|
-
else
|
|
194
|
-
input_options = task_info[:input_options][i] || {}
|
|
195
|
-
|
|
196
|
-
#ToDo why was this always true?
|
|
197
|
-
if input_options[:stream] or true
|
|
198
|
-
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
|
199
|
-
_inputs[i] = rec_dependency
|
|
200
|
-
else
|
|
201
|
-
rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
|
|
202
|
-
rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
|
|
203
|
-
if rec_dependency.streaming? and rec_dependency.running?
|
|
204
|
-
_inputs[i] = rec_dependency.join.load
|
|
205
|
-
else
|
|
206
|
-
rec_dependency.run(true)
|
|
207
|
-
rec_dependency.join
|
|
208
|
-
_inputs[i] = rec_dependency.load
|
|
209
|
-
end
|
|
210
|
-
end
|
|
211
|
-
end
|
|
212
|
-
else
|
|
213
|
-
_inputs[i] = v
|
|
214
|
-
end
|
|
215
|
-
} if options
|
|
216
|
-
|
|
217
|
-
_inputs
|
|
218
|
-
end
|
|
219
175
|
|
|
220
176
|
def task_from_dep(dep)
|
|
221
177
|
task = case dep
|
|
@@ -92,6 +92,51 @@ module Workflow
|
|
|
92
92
|
dep
|
|
93
93
|
end
|
|
94
94
|
|
|
95
|
+
def assign_dep_inputs(_inputs, options, all_d, task_info)
|
|
96
|
+
IndiferentHash.setup(_inputs)
|
|
97
|
+
|
|
98
|
+
options.each{|i,v|
|
|
99
|
+
next if i == :compute or i == "compute"
|
|
100
|
+
case v
|
|
101
|
+
when :compute
|
|
102
|
+
compute = v
|
|
103
|
+
when Symbol
|
|
104
|
+
rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
|
|
105
|
+
|
|
106
|
+
if rec_dependency.nil?
|
|
107
|
+
if _inputs.include?(v)
|
|
108
|
+
#_inputs[i] = _inputs.delete(v)
|
|
109
|
+
_inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
|
|
110
|
+
else
|
|
111
|
+
_inputs[i] = v unless _inputs.include? i
|
|
112
|
+
end
|
|
113
|
+
else
|
|
114
|
+
input_options = task_info[:input_options][i] || {}
|
|
115
|
+
|
|
116
|
+
#ToDo why was this always true?
|
|
117
|
+
if input_options[:stream] or true
|
|
118
|
+
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
|
119
|
+
_inputs[i] = rec_dependency
|
|
120
|
+
else
|
|
121
|
+
rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
|
|
122
|
+
rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
|
|
123
|
+
if rec_dependency.streaming? and rec_dependency.running?
|
|
124
|
+
_inputs[i] = rec_dependency.join.load
|
|
125
|
+
else
|
|
126
|
+
rec_dependency.run(true)
|
|
127
|
+
rec_dependency.join
|
|
128
|
+
_inputs[i] = rec_dependency.load
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
else
|
|
133
|
+
_inputs[i] = v
|
|
134
|
+
end
|
|
135
|
+
} if options
|
|
136
|
+
|
|
137
|
+
_inputs
|
|
138
|
+
end
|
|
139
|
+
|
|
95
140
|
def real_dependencies(task, orig_jobname, inputs, dependencies)
|
|
96
141
|
real_dependencies = []
|
|
97
142
|
path_deps = {}
|
data/lib/rbbt/workflow/step.rb
CHANGED
|
@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
|
|
|
3
3
|
|
|
4
4
|
module Workflow
|
|
5
5
|
module Data
|
|
6
|
-
|
|
6
|
+
class DataNotFound < RbbtException; end
|
|
7
|
+
|
|
8
|
+
def data_dir(directory)
|
|
7
9
|
@@data_directory = directory
|
|
8
10
|
end
|
|
9
11
|
|
|
10
12
|
def get_datadir(clean_name)
|
|
11
13
|
data_dir = File.join(@@data_directory, clean_name)
|
|
12
|
-
raise "Data dir not found #{
|
|
14
|
+
raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
|
|
13
15
|
if Path === @@data_directory
|
|
14
16
|
@@data_directory.annotate data_dir
|
|
15
17
|
else
|
|
@@ -17,19 +19,147 @@ module Workflow
|
|
|
17
19
|
end
|
|
18
20
|
end
|
|
19
21
|
|
|
22
|
+
def get_file(clean_name, type = nil, extension = nil)
|
|
23
|
+
begin
|
|
24
|
+
extension = nil if String === extension && extension.empty?
|
|
25
|
+
file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
|
|
26
|
+
file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
|
|
27
|
+
if Open.exists?(file1)
|
|
28
|
+
data_file = file1
|
|
29
|
+
elsif Open.exists?(file2)
|
|
30
|
+
data_file = file2
|
|
31
|
+
else
|
|
32
|
+
raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
if Path === @@data_directory
|
|
37
|
+
@@data_directory.annotate data_file
|
|
38
|
+
else
|
|
39
|
+
Path.setup(data_file)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
20
43
|
def data_task(name, workflow, oname, *rest, &block)
|
|
21
|
-
dep_task(name, workflow, oname, *rest) do |jobname, options|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
44
|
+
dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
|
|
45
|
+
begin
|
|
46
|
+
task_info = workflow.nil? ? nil : workflow.task_info(oname)
|
|
47
|
+
options[:extension] ||= task_info[:extension]
|
|
48
|
+
path = get_file jobname, name, options[:extension]
|
|
49
|
+
job = Step.new path
|
|
50
|
+
job.task_name = name
|
|
51
|
+
job.result_type = options[:result_type] || options[:type] || options[:extension]
|
|
52
|
+
job
|
|
53
|
+
rescue DataNotFound
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
next job if job
|
|
57
|
+
|
|
58
|
+
data_options = {}
|
|
59
|
+
data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
|
|
60
|
+
|
|
61
|
+
begin
|
|
62
|
+
data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
|
|
63
|
+
rescue DataNotFound
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
begin
|
|
67
|
+
data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
|
|
68
|
+
rescue DataNotFound
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
begin
|
|
72
|
+
task_info = workflow.nil? ? nil : workflow.task_info(oname)
|
|
73
|
+
data_options = block.call get_datadir(jobname), data_options, task_info
|
|
74
|
+
rescue
|
|
75
|
+
Log.exception $!
|
|
76
|
+
end if block_given?
|
|
77
|
+
|
|
78
|
+
case data_options
|
|
79
|
+
when Step
|
|
80
|
+
next data_options
|
|
81
|
+
when Hash
|
|
82
|
+
if data_options.include?(:inputs)
|
|
83
|
+
data_options = data_options.merge(options)
|
|
84
|
+
workflow = data_options[:workflow] if data_options[:workflow]
|
|
85
|
+
oname = data_options[:task] if data_options[:task]
|
|
86
|
+
inputs = options.merge(data_options[:inputs])
|
|
87
|
+
else
|
|
88
|
+
inputs = options.merge(data_options)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
{:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
|
|
28
92
|
else
|
|
29
|
-
|
|
93
|
+
raise "Cannot understand data_options: #{data_options}"
|
|
30
94
|
end
|
|
95
|
+
|
|
31
96
|
end
|
|
32
97
|
end
|
|
33
|
-
end
|
|
34
98
|
|
|
99
|
+
def data(name, options = {}, &block)
|
|
100
|
+
dep do |jobname, job_options, dependencies|
|
|
101
|
+
if block_given?
|
|
102
|
+
block.call jobname, job_options.merge(options), dependencies
|
|
103
|
+
else
|
|
104
|
+
begin
|
|
105
|
+
path = get_file jobname, name, options[:extension]
|
|
106
|
+
job = Step.new path
|
|
107
|
+
job.task_name = name
|
|
108
|
+
job.result_type = options[:result_type] || options[:type] || options[:extension]
|
|
109
|
+
job
|
|
110
|
+
rescue DataNotFound
|
|
111
|
+
{:task => name, :options => options, :jobname => jobname}
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
35
117
|
end
|
|
118
|
+
#module Workflow
|
|
119
|
+
# module Data
|
|
120
|
+
# def data(directory)
|
|
121
|
+
# @@data_directory = directory
|
|
122
|
+
# end
|
|
123
|
+
#
|
|
124
|
+
# def get_datadir(clean_name)
|
|
125
|
+
# data_dir = File.join(@@data_directory, clean_name)
|
|
126
|
+
# raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
|
|
127
|
+
# if Path === @@data_directory
|
|
128
|
+
# @@data_directory.annotate data_dir
|
|
129
|
+
# else
|
|
130
|
+
# Path.setup(data_dir)
|
|
131
|
+
# end
|
|
132
|
+
# end
|
|
133
|
+
#
|
|
134
|
+
# def data_task(name, workflow, oname, *rest, &block)
|
|
135
|
+
# dep_task(name, workflow, oname, *rest) do |jobname, options|
|
|
136
|
+
# data_dir = self.get_datadir(jobname)
|
|
137
|
+
# task_info = workflow.task_info(oname)
|
|
138
|
+
#
|
|
139
|
+
# dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
|
|
140
|
+
# data_options = block.call data_dir, dir_options, task_info
|
|
141
|
+
#
|
|
142
|
+
# case data_options
|
|
143
|
+
# when Step
|
|
144
|
+
# job = data_options
|
|
145
|
+
# when Hash
|
|
146
|
+
# if data_options.include?(:inputs)
|
|
147
|
+
# data_options = data_options.merge(options)
|
|
148
|
+
# workflow = data_options[:workflow] if data_options[:workflow]
|
|
149
|
+
# oname = data_options[:task] if data_options[:task]
|
|
150
|
+
# inputs = options.merge(data_options[:inputs])
|
|
151
|
+
# else
|
|
152
|
+
# inputs = options.merge(data_options)
|
|
153
|
+
# end
|
|
154
|
+
#
|
|
155
|
+
# job = workflow.job(oname, jobname, inputs)
|
|
156
|
+
# else
|
|
157
|
+
# raise "Cannot understand data_options: #{data_options}"
|
|
158
|
+
# end
|
|
159
|
+
#
|
|
160
|
+
# job
|
|
161
|
+
# end
|
|
162
|
+
# end
|
|
163
|
+
# end
|
|
164
|
+
#
|
|
165
|
+
#end
|
data/lib/rbbt/workflow.rb
CHANGED
|
@@ -45,14 +45,10 @@ module Workflow
|
|
|
45
45
|
load_remote_tasks(Rbbt.root.etc.remote_tasks.find) if Rbbt.root.etc.remote_tasks.exists?
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
-
def self.require_remote_workflow(wf_name, url)
|
|
49
|
-
require 'rbbt/workflow/remote_workflow'
|
|
50
|
-
eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
|
|
51
|
-
end
|
|
52
48
|
|
|
53
49
|
def self.require_remote_workflow(wf_name, url)
|
|
54
50
|
require 'rbbt/workflow/remote_workflow'
|
|
55
|
-
eval "Object::#{wf_name} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
|
|
51
|
+
eval "Object::#{wf_name.split("+").first} = RemoteWorkflow.new '#{ url }', '#{wf_name}'"
|
|
56
52
|
end
|
|
57
53
|
|
|
58
54
|
def self.load_workflow_libdir(filename)
|
|
@@ -134,9 +130,10 @@ module Workflow
|
|
|
134
130
|
end
|
|
135
131
|
|
|
136
132
|
def self.require_local_workflow(wf_name)
|
|
133
|
+
|
|
137
134
|
filename = local_workflow_filename(wf_name)
|
|
138
135
|
|
|
139
|
-
if filename and File.exist?
|
|
136
|
+
if filename and File.exist?(filename)
|
|
140
137
|
load_workflow_file filename
|
|
141
138
|
else
|
|
142
139
|
return false
|
|
@@ -194,14 +191,23 @@ module Workflow
|
|
|
194
191
|
end
|
|
195
192
|
|
|
196
193
|
Log.high{"Loading workflow #{wf_name}"}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
194
|
+
|
|
195
|
+
first = nil
|
|
196
|
+
wf_name.split("+").each do |wf_name|
|
|
197
|
+
require_local_workflow(wf_name) or
|
|
198
|
+
(Workflow.autoinstall and `rbbt workflow install #{Misc.snake_case(wf_name)} || rbbt workflow install #{wf_name}` and require_local_workflow(wf_name)) or raise("Workflow not found or could not be loaded: #{ wf_name }")
|
|
199
|
+
|
|
200
|
+
workflow = begin
|
|
201
|
+
Misc.string2const Misc.camel_case(wf_name.split("+").first)
|
|
202
|
+
rescue
|
|
203
|
+
Workflow.workflows.last || true
|
|
204
|
+
end
|
|
205
|
+
workflow.load_documentation
|
|
206
|
+
|
|
207
|
+
first ||= workflow
|
|
208
|
+
end
|
|
209
|
+
return first
|
|
210
|
+
|
|
205
211
|
workflow
|
|
206
212
|
end
|
|
207
213
|
|
|
@@ -287,8 +287,8 @@ else
|
|
|
287
287
|
puts
|
|
288
288
|
puts $!.message
|
|
289
289
|
puts
|
|
290
|
-
|
|
291
|
-
exit
|
|
290
|
+
|
|
291
|
+
exit -1
|
|
292
292
|
end
|
|
293
293
|
end
|
|
294
294
|
|
|
@@ -572,7 +572,9 @@ when Step
|
|
|
572
572
|
exit! 0
|
|
573
573
|
else
|
|
574
574
|
res.join
|
|
575
|
-
|
|
575
|
+
Open.open(res.path, :mode => 'rb') do |io|
|
|
576
|
+
Misc.consume_stream(io, false, out)
|
|
577
|
+
end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
|
|
576
578
|
end
|
|
577
579
|
else
|
|
578
580
|
if Array === res
|
data/test/rbbt/tsv/test_excel.rb
CHANGED
|
@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
|
|
|
2
2
|
require 'rbbt/tsv/excel'
|
|
3
3
|
|
|
4
4
|
class TestExcel < Test::Unit::TestCase
|
|
5
|
-
def
|
|
5
|
+
def test_xls
|
|
6
6
|
content =<<-EOF
|
|
7
7
|
#Id ValueA ValueB OtherID
|
|
8
8
|
row1 a|aa|aaa b Id1|Id2
|
|
@@ -19,7 +19,7 @@ row2 A B Id3
|
|
|
19
19
|
end
|
|
20
20
|
end
|
|
21
21
|
|
|
22
|
-
def
|
|
22
|
+
def test_xlsx
|
|
23
23
|
content =<<-EOF
|
|
24
24
|
#Id ValueA ValueB OtherID
|
|
25
25
|
row1 a|aa|aaa b Id1|Id2
|
|
@@ -36,7 +36,7 @@ row2 A B Id3
|
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
-
def
|
|
39
|
+
def test_excel
|
|
40
40
|
content =<<-EOF
|
|
41
41
|
#Id ValueA ValueB OtherID
|
|
42
42
|
row1 a|aa|aaa b Id1|Id2
|
|
@@ -63,7 +63,7 @@ row2 A B Id3
|
|
|
63
63
|
end
|
|
64
64
|
end
|
|
65
65
|
|
|
66
|
-
def
|
|
66
|
+
def test_excel_sheets
|
|
67
67
|
content =<<-EOF
|
|
68
68
|
#Id ValueA ValueB OtherID
|
|
69
69
|
row1 a|aa|aaa b Id1|Id2
|
|
@@ -133,5 +133,39 @@ row2 A B Id3
|
|
|
133
133
|
end
|
|
134
134
|
end
|
|
135
135
|
end
|
|
136
|
+
|
|
137
|
+
def test_excel_multi_sheets
|
|
138
|
+
content =<<-EOF
|
|
139
|
+
#Id ValueA ValueB OtherID
|
|
140
|
+
row1 a|aa|aaa b Id1|Id2
|
|
141
|
+
row2 A B Id3
|
|
142
|
+
EOF
|
|
143
|
+
|
|
144
|
+
TmpFile.with_file(content) do |filename|
|
|
145
|
+
tsv1 = TSV.open(filename, :sep => /\s+/)
|
|
146
|
+
tsv2 = tsv1.annotate(tsv1.dup)
|
|
147
|
+
tsv3 = tsv1.annotate(tsv1.dup)
|
|
148
|
+
|
|
149
|
+
tsv2["row2"] = [["AA"], ["BB"], ["Id4"]]
|
|
150
|
+
tsv3["row2"] = [["AAA"], ["BBB"], ["Id5"]]
|
|
151
|
+
|
|
152
|
+
TmpFile.with_file(nil, false, :extension => 'xlsx') do |excelfile|
|
|
153
|
+
tsv1.xlsx(excelfile, :sheet => "S1")
|
|
154
|
+
tsv2.xlsx(excelfile, :sheet => "S2", :add_sheet => true)
|
|
155
|
+
workbook = RubyXL::Parser.parse excelfile
|
|
156
|
+
|
|
157
|
+
assert_equal %w(S1 S2), workbook.worksheets.collect{|s| s.sheet_name}
|
|
158
|
+
|
|
159
|
+
new = TSV.excel(excelfile, :sheet => "S1")
|
|
160
|
+
assert_equal %w(row1 row2), new.keys.sort
|
|
161
|
+
assert_equal %w(A), new["row2"]["ValueA"]
|
|
162
|
+
|
|
163
|
+
new = TSV.excel(excelfile, :sheet => "S2")
|
|
164
|
+
assert_equal %w(row1 row2), new.keys.sort
|
|
165
|
+
assert_equal %w(AA), new["row2"]["ValueA"]
|
|
166
|
+
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
end
|
|
136
170
|
end
|
|
137
171
|
|
|
@@ -1,6 +1,16 @@
|
|
|
1
1
|
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
|
2
2
|
require 'rbbt/workflow/util/data'
|
|
3
3
|
|
|
4
|
+
module TestDataWF2
|
|
5
|
+
extend Workflow
|
|
6
|
+
extend Workflow::Data
|
|
7
|
+
|
|
8
|
+
input :salutation, :string
|
|
9
|
+
task :salute_antonio => :string do |name,salutation|
|
|
10
|
+
"Hi Antonio: #{salutation}"
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
4
14
|
module TestDataWF
|
|
5
15
|
extend Workflow
|
|
6
16
|
extend Workflow::Data
|
|
@@ -23,6 +33,11 @@ module TestDataWF
|
|
|
23
33
|
data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
|
24
34
|
{:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
|
|
25
35
|
end
|
|
36
|
+
|
|
37
|
+
data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
|
38
|
+
{:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
|
|
39
|
+
end
|
|
40
|
+
|
|
26
41
|
end
|
|
27
42
|
|
|
28
43
|
class TestWorkflowData < Test::Unit::TestCase
|
|
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
|
|
|
33
48
|
Open.write(tmpdir.TestDir.options.name, "Miguel")
|
|
34
49
|
Open.write(tmpdir.TestDir.salutation, "My salutations")
|
|
35
50
|
|
|
36
|
-
TestDataWF.
|
|
51
|
+
TestDataWF.data_dir tmpdir
|
|
37
52
|
|
|
38
53
|
job = TestDataWF.job(:salute_data, "TestDir")
|
|
39
54
|
job.recursive_clean.run
|
|
40
55
|
assert job.run.include? "Miguel"
|
|
41
56
|
|
|
42
57
|
job = TestDataWF.job(:salute_data2, "TestDir")
|
|
43
|
-
job.recursive_clean.run
|
|
58
|
+
puts job.recursive_clean.run
|
|
44
59
|
assert job.run.include? "Luis"
|
|
60
|
+
|
|
61
|
+
job = TestDataWF.job(:salute_data3, "TestDir")
|
|
62
|
+
job.recursive_clean.run
|
|
63
|
+
assert job.run.include? "Antonio"
|
|
45
64
|
end
|
|
46
65
|
end
|
|
47
66
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rbbt-util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.32.
|
|
4
|
+
version: 5.32.26
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Miguel Vazquez
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2021-
|
|
11
|
+
date: 2021-11-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -172,6 +172,7 @@ executables:
|
|
|
172
172
|
- rbbt_Rutil.rb
|
|
173
173
|
- rbbt
|
|
174
174
|
- rbbt_dangling_locks.rb
|
|
175
|
+
- rbbt_find.rb
|
|
175
176
|
extensions: []
|
|
176
177
|
extra_rdoc_files:
|
|
177
178
|
- LICENSE
|
|
@@ -183,6 +184,7 @@ files:
|
|
|
183
184
|
- bin/rbbt_Rutil.rb
|
|
184
185
|
- bin/rbbt_dangling_locks.rb
|
|
185
186
|
- bin/rbbt_exec.rb
|
|
187
|
+
- bin/rbbt_find.rb
|
|
186
188
|
- bin/rbbt_query.rb
|
|
187
189
|
- etc/app.d/base.rb
|
|
188
190
|
- etc/app.d/entities.rb
|