rbbt-util 5.32.22 → 5.32.23
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/tsv/parser.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +0 -44
- data/lib/rbbt/workflow/dependencies.rb +45 -0
- data/lib/rbbt/workflow/step.rb +4 -0
- data/lib/rbbt/workflow/util/data.rb +141 -11
- data/test/rbbt/workflow/util/test_data.rb +21 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 900a7061d40ec2cbffd38886ef9629aa0c0ed6d6ee1aef6e0c95c51656230ef0
|
4
|
+
data.tar.gz: 13b965a0ec134a932c6ae0ae6a109fbc2bc83b7ffd855a1ef05ed4e1e51b9d88
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3b890e2b6fb3675df4319d0541d9057a54b0cb37286a814822915f5b5b061d4f7e02deab0afdd7cd0833cb741aeded7454a9585f93ff3486e595b99db4da00f6
|
7
|
+
data.tar.gz: 105072ca6c7b5c43c6a60455eaa9c7167feeb93f505c61c6aeffe6305de3166a6cb440c262abf025407306ba82c2fe3167f5f9e743e086d99fa936e44ea8e448
|
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -172,50 +172,6 @@ module Workflow
|
|
172
172
|
}.tap{|h| IndiferentHash.setup(h)}
|
173
173
|
end
|
174
174
|
|
175
|
-
def assign_dep_inputs(_inputs, options, all_d, task_info)
|
176
|
-
IndiferentHash.setup(_inputs)
|
177
|
-
|
178
|
-
options.each{|i,v|
|
179
|
-
next if i == :compute or i == "compute"
|
180
|
-
case v
|
181
|
-
when :compute
|
182
|
-
compute = v
|
183
|
-
when Symbol
|
184
|
-
rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
|
185
|
-
|
186
|
-
if rec_dependency.nil?
|
187
|
-
if _inputs.include?(v)
|
188
|
-
#_inputs[i] = _inputs.delete(v)
|
189
|
-
_inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
|
190
|
-
else
|
191
|
-
_inputs[i] = v unless _inputs.include? i
|
192
|
-
end
|
193
|
-
else
|
194
|
-
input_options = task_info[:input_options][i] || {}
|
195
|
-
|
196
|
-
#ToDo why was this always true?
|
197
|
-
if input_options[:stream] or true
|
198
|
-
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
199
|
-
_inputs[i] = rec_dependency
|
200
|
-
else
|
201
|
-
rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
|
202
|
-
rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
|
203
|
-
if rec_dependency.streaming? and rec_dependency.running?
|
204
|
-
_inputs[i] = rec_dependency.join.load
|
205
|
-
else
|
206
|
-
rec_dependency.run(true)
|
207
|
-
rec_dependency.join
|
208
|
-
_inputs[i] = rec_dependency.load
|
209
|
-
end
|
210
|
-
end
|
211
|
-
end
|
212
|
-
else
|
213
|
-
_inputs[i] = v
|
214
|
-
end
|
215
|
-
} if options
|
216
|
-
|
217
|
-
_inputs
|
218
|
-
end
|
219
175
|
|
220
176
|
def task_from_dep(dep)
|
221
177
|
task = case dep
|
@@ -92,6 +92,51 @@ module Workflow
|
|
92
92
|
dep
|
93
93
|
end
|
94
94
|
|
95
|
+
def assign_dep_inputs(_inputs, options, all_d, task_info)
|
96
|
+
IndiferentHash.setup(_inputs)
|
97
|
+
|
98
|
+
options.each{|i,v|
|
99
|
+
next if i == :compute or i == "compute"
|
100
|
+
case v
|
101
|
+
when :compute
|
102
|
+
compute = v
|
103
|
+
when Symbol
|
104
|
+
rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
|
105
|
+
|
106
|
+
if rec_dependency.nil?
|
107
|
+
if _inputs.include?(v)
|
108
|
+
#_inputs[i] = _inputs.delete(v)
|
109
|
+
_inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
|
110
|
+
else
|
111
|
+
_inputs[i] = v unless _inputs.include? i
|
112
|
+
end
|
113
|
+
else
|
114
|
+
input_options = task_info[:input_options][i] || {}
|
115
|
+
|
116
|
+
#ToDo why was this always true?
|
117
|
+
if input_options[:stream] or true
|
118
|
+
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
119
|
+
_inputs[i] = rec_dependency
|
120
|
+
else
|
121
|
+
rec_dependency.abort if rec_dependency.streaming? and not rec_dependency.running?
|
122
|
+
rec_dependency.clean if rec_dependency.error? or rec_dependency.aborted?
|
123
|
+
if rec_dependency.streaming? and rec_dependency.running?
|
124
|
+
_inputs[i] = rec_dependency.join.load
|
125
|
+
else
|
126
|
+
rec_dependency.run(true)
|
127
|
+
rec_dependency.join
|
128
|
+
_inputs[i] = rec_dependency.load
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
else
|
133
|
+
_inputs[i] = v
|
134
|
+
end
|
135
|
+
} if options
|
136
|
+
|
137
|
+
_inputs
|
138
|
+
end
|
139
|
+
|
95
140
|
def real_dependencies(task, orig_jobname, inputs, dependencies)
|
96
141
|
real_dependencies = []
|
97
142
|
path_deps = {}
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -3,13 +3,15 @@ require 'rbbt/workflow/examples'
|
|
3
3
|
|
4
4
|
module Workflow
|
5
5
|
module Data
|
6
|
-
|
6
|
+
class DataNotFound < RbbtException; end
|
7
|
+
|
8
|
+
def data_dir(directory)
|
7
9
|
@@data_directory = directory
|
8
10
|
end
|
9
11
|
|
10
12
|
def get_datadir(clean_name)
|
11
13
|
data_dir = File.join(@@data_directory, clean_name)
|
12
|
-
raise "Data dir not found #{
|
14
|
+
raise DataNotFound, "Data dir not found for #{clean_name} in #{@@data_directory}" unless File.directory?(data_dir)
|
13
15
|
if Path === @@data_directory
|
14
16
|
@@data_directory.annotate data_dir
|
15
17
|
else
|
@@ -17,19 +19,147 @@ module Workflow
|
|
17
19
|
end
|
18
20
|
end
|
19
21
|
|
22
|
+
def get_file(clean_name, type = nil, extension = nil)
|
23
|
+
begin
|
24
|
+
extension = nil if String === extension && extension.empty?
|
25
|
+
file1 = File.join(@@data_directory, type.to_s, (extension.nil? ? clean_name.to_s : clean_name.to_s + ".#{extension}"))
|
26
|
+
file2 = File.join(@@data_directory, clean_name.to_s, (extension.nil? ? type.to_s : type.to_s + ".#{extension}"))
|
27
|
+
if Open.exists?(file1)
|
28
|
+
data_file = file1
|
29
|
+
elsif Open.exists?(file2)
|
30
|
+
data_file = file2
|
31
|
+
else
|
32
|
+
raise DataNotFound, "Data type #{type} not found for #{clean_name} in #{@@data_directory}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
if Path === @@data_directory
|
37
|
+
@@data_directory.annotate data_file
|
38
|
+
else
|
39
|
+
Path.setup(data_file)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
20
43
|
def data_task(name, workflow, oname, *rest, &block)
|
21
|
-
dep_task(name, workflow, oname, *rest) do |jobname, options|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
44
|
+
dep_task(name, workflow, oname, *rest) do |jobname, options,dependencies|
|
45
|
+
begin
|
46
|
+
task_info = workflow.nil? ? nil : workflow.task_info(oname)
|
47
|
+
options[:extension] ||= task_info[:extension]
|
48
|
+
path = get_file jobname, name, options[:extension]
|
49
|
+
job = Step.new path
|
50
|
+
job.task_name = name
|
51
|
+
job.result_type = options[:result_type] || options[:type] || options[:extension]
|
52
|
+
job
|
53
|
+
rescue DataNotFound
|
54
|
+
end
|
55
|
+
|
56
|
+
next job if job
|
57
|
+
|
58
|
+
data_options = {}
|
59
|
+
data_options = data_options.merge(Workflow.load_inputs(File.join(@@data_directory,"options"), task_info[:inputs], task_info[:input_types]))
|
60
|
+
|
61
|
+
begin
|
62
|
+
data_options = data_options.merge(Workflow.load_inputs(get_file(jobname, :options), task_info[:inputs], task_info[:input_types]))
|
63
|
+
rescue DataNotFound
|
64
|
+
end
|
65
|
+
|
66
|
+
begin
|
67
|
+
data_options = data_options.merge(Workflow.load_inputs(get_datadir(jobname).options, task_info[:inputs], task_info[:input_types]))
|
68
|
+
rescue DataNotFound
|
69
|
+
end
|
70
|
+
|
71
|
+
begin
|
72
|
+
task_info = workflow.nil? ? nil : workflow.task_info(oname)
|
73
|
+
data_options = block.call get_datadir(jobname), data_options, task_info
|
74
|
+
rescue
|
75
|
+
Log.exception $!
|
76
|
+
end if block_given?
|
77
|
+
|
78
|
+
case data_options
|
79
|
+
when Step
|
80
|
+
next data_options
|
81
|
+
when Hash
|
82
|
+
if data_options.include?(:inputs)
|
83
|
+
data_options = data_options.merge(options)
|
84
|
+
workflow = data_options[:workflow] if data_options[:workflow]
|
85
|
+
oname = data_options[:task] if data_options[:task]
|
86
|
+
inputs = options.merge(data_options[:inputs])
|
87
|
+
else
|
88
|
+
inputs = options.merge(data_options)
|
89
|
+
end
|
90
|
+
|
91
|
+
{:workflow => workflow, :task => oname, :jobname => jobname, :inputs => inputs}
|
28
92
|
else
|
29
|
-
|
93
|
+
raise "Cannot understand data_options: #{data_options}"
|
30
94
|
end
|
95
|
+
|
31
96
|
end
|
32
97
|
end
|
33
|
-
end
|
34
98
|
|
99
|
+
def data(name, options = {}, &block)
|
100
|
+
dep do |jobname, job_options, dependencies|
|
101
|
+
if block_given?
|
102
|
+
block.call jobname, job_options.merge(options), dependencies
|
103
|
+
else
|
104
|
+
begin
|
105
|
+
path = get_file jobname, name, options[:extension]
|
106
|
+
job = Step.new path
|
107
|
+
job.task_name = name
|
108
|
+
job.result_type = options[:result_type] || options[:type] || options[:extension]
|
109
|
+
job
|
110
|
+
rescue DataNotFound
|
111
|
+
{:task => name, :options => options, :jobname => jobname}
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
35
117
|
end
|
118
|
+
#module Workflow
|
119
|
+
# module Data
|
120
|
+
# def data(directory)
|
121
|
+
# @@data_directory = directory
|
122
|
+
# end
|
123
|
+
#
|
124
|
+
# def get_datadir(clean_name)
|
125
|
+
# data_dir = File.join(@@data_directory, clean_name)
|
126
|
+
# raise "Data dir not found #{data_dir}" unless File.directory?(data_dir)
|
127
|
+
# if Path === @@data_directory
|
128
|
+
# @@data_directory.annotate data_dir
|
129
|
+
# else
|
130
|
+
# Path.setup(data_dir)
|
131
|
+
# end
|
132
|
+
# end
|
133
|
+
#
|
134
|
+
# def data_task(name, workflow, oname, *rest, &block)
|
135
|
+
# dep_task(name, workflow, oname, *rest) do |jobname, options|
|
136
|
+
# data_dir = self.get_datadir(jobname)
|
137
|
+
# task_info = workflow.task_info(oname)
|
138
|
+
#
|
139
|
+
# dir_options = Workflow.load_inputs(data_dir.options, task_info[:inputs], task_info[:input_types])
|
140
|
+
# data_options = block.call data_dir, dir_options, task_info
|
141
|
+
#
|
142
|
+
# case data_options
|
143
|
+
# when Step
|
144
|
+
# job = data_options
|
145
|
+
# when Hash
|
146
|
+
# if data_options.include?(:inputs)
|
147
|
+
# data_options = data_options.merge(options)
|
148
|
+
# workflow = data_options[:workflow] if data_options[:workflow]
|
149
|
+
# oname = data_options[:task] if data_options[:task]
|
150
|
+
# inputs = options.merge(data_options[:inputs])
|
151
|
+
# else
|
152
|
+
# inputs = options.merge(data_options)
|
153
|
+
# end
|
154
|
+
#
|
155
|
+
# job = workflow.job(oname, jobname, inputs)
|
156
|
+
# else
|
157
|
+
# raise "Cannot understand data_options: #{data_options}"
|
158
|
+
# end
|
159
|
+
#
|
160
|
+
# job
|
161
|
+
# end
|
162
|
+
# end
|
163
|
+
# end
|
164
|
+
#
|
165
|
+
#end
|
@@ -1,6 +1,16 @@
|
|
1
1
|
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
2
|
require 'rbbt/workflow/util/data'
|
3
3
|
|
4
|
+
module TestDataWF2
|
5
|
+
extend Workflow
|
6
|
+
extend Workflow::Data
|
7
|
+
|
8
|
+
input :salutation, :string
|
9
|
+
task :salute_antonio => :string do |name,salutation|
|
10
|
+
"Hi Antonio: #{salutation}"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
4
14
|
module TestDataWF
|
5
15
|
extend Workflow
|
6
16
|
extend Workflow::Data
|
@@ -23,6 +33,11 @@ module TestDataWF
|
|
23
33
|
data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
24
34
|
{:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
|
25
35
|
end
|
36
|
+
|
37
|
+
data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
|
38
|
+
{:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
|
39
|
+
end
|
40
|
+
|
26
41
|
end
|
27
42
|
|
28
43
|
class TestWorkflowData < Test::Unit::TestCase
|
@@ -33,15 +48,19 @@ class TestWorkflowData < Test::Unit::TestCase
|
|
33
48
|
Open.write(tmpdir.TestDir.options.name, "Miguel")
|
34
49
|
Open.write(tmpdir.TestDir.salutation, "My salutations")
|
35
50
|
|
36
|
-
TestDataWF.
|
51
|
+
TestDataWF.data_dir tmpdir
|
37
52
|
|
38
53
|
job = TestDataWF.job(:salute_data, "TestDir")
|
39
54
|
job.recursive_clean.run
|
40
55
|
assert job.run.include? "Miguel"
|
41
56
|
|
42
57
|
job = TestDataWF.job(:salute_data2, "TestDir")
|
43
|
-
job.recursive_clean.run
|
58
|
+
puts job.recursive_clean.run
|
44
59
|
assert job.run.include? "Luis"
|
60
|
+
|
61
|
+
job = TestDataWF.job(:salute_data3, "TestDir")
|
62
|
+
job.recursive_clean.run
|
63
|
+
assert job.run.include? "Antonio"
|
45
64
|
end
|
46
65
|
end
|
47
66
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.32.
|
4
|
+
version: 5.32.23
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-09-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|