rbbt-util 5.28.5 → 5.28.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -0
- data/lib/rbbt/util/misc/inspect.rb +1 -1
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +94 -93
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +5 -1
- data/lib/rbbt/workflow/step/accessor.rb +5 -3
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/share/rbbt_commands/workflow/server +2 -0
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 900106fb1799c857a482187c7cb7d18a40c35dbeb5dd231bf8e1851235e28c63
|
|
4
|
+
data.tar.gz: e5ec2271b3ada0b282963f4ffc7e2b623d8bb84fcb72867a6848b93a80052463
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1a418d5b4fe4369f7c25dd063c60a65c2b6c0bd7e3dbf9fd156fe875f85ef1b2eeb4d2eb1a03472e191de98bbfbae20abdb86877d381193f86fe4c8325d2f391
|
|
7
|
+
data.tar.gz: 5225d6dbdbab08d0ed5239eba91bbe3d2a8dd16c92c24fe177b66ed3b9d4baef4e24818057de8bf694edf715aabd72b481926a2466c76492c27b75be25634b27
|
data/lib/rbbt/workflow.rb
CHANGED
|
@@ -364,6 +364,7 @@ module Workflow
|
|
|
364
364
|
raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
|
|
365
365
|
end
|
|
366
366
|
|
|
367
|
+
# jobname => true sets the value of the input to the name of the job
|
|
367
368
|
if task.input_options
|
|
368
369
|
jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
|
|
369
370
|
else
|
|
@@ -26,105 +26,106 @@ module Workflow
|
|
|
26
26
|
raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
|
|
27
27
|
id = File.join(self.to_s, name.to_s)
|
|
28
28
|
@task_info ||= {}
|
|
29
|
-
@task_info[id] ||= begin
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
29
|
+
@task_info[id] ||= begin
|
|
30
|
+
description = task.description
|
|
31
|
+
result_description = task.result_description
|
|
32
|
+
result_type = task.result_type
|
|
33
|
+
inputs = rec_inputs(name).uniq
|
|
34
|
+
input_types = rec_input_types(name)
|
|
35
|
+
input_descriptions = rec_input_descriptions(name)
|
|
36
|
+
input_use = rec_input_use(name)
|
|
37
|
+
input_defaults = rec_input_defaults(name)
|
|
38
|
+
input_options = rec_input_options(name)
|
|
39
|
+
extension = task.extension
|
|
40
|
+
export = case
|
|
41
|
+
when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
|
|
42
|
+
:synchronous
|
|
43
|
+
when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
|
|
44
|
+
:asynchronous
|
|
45
|
+
when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
|
|
46
|
+
:exec
|
|
47
|
+
when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
|
|
48
|
+
:stream
|
|
49
|
+
else
|
|
50
|
+
:none
|
|
51
|
+
end
|
|
51
52
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
53
|
+
dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
|
|
54
|
+
{ :id => id,
|
|
55
|
+
:description => description,
|
|
56
|
+
:export => export,
|
|
57
|
+
:inputs => inputs,
|
|
58
|
+
:input_types => input_types,
|
|
59
|
+
:input_descriptions => input_descriptions,
|
|
60
|
+
:input_defaults => input_defaults,
|
|
61
|
+
:input_options => input_options,
|
|
62
|
+
:input_use => input_use,
|
|
63
|
+
:result_type => result_type,
|
|
64
|
+
:result_description => result_description,
|
|
65
|
+
:dependencies => dependencies,
|
|
66
|
+
:extension => extension
|
|
67
|
+
}
|
|
68
|
+
end
|
|
68
69
|
end
|
|
69
70
|
|
|
70
71
|
def rec_dependencies(taskname)
|
|
71
72
|
@rec_dependencies ||= {}
|
|
72
73
|
@rec_dependencies[taskname] ||= begin
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
74
|
+
if task_dependencies.include? taskname
|
|
75
|
+
|
|
76
|
+
deps = task_dependencies[taskname]
|
|
77
|
+
|
|
78
|
+
#all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
|
|
79
|
+
|
|
80
|
+
all_deps = []
|
|
81
|
+
deps.each do |dep|
|
|
82
|
+
if DependencyBlock === dep
|
|
83
|
+
all_deps << dep.dependency if dep.dependency
|
|
84
|
+
else
|
|
85
|
+
all_deps << dep unless Proc === dep
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
begin
|
|
89
|
+
case dep
|
|
90
|
+
when Array
|
|
91
|
+
wf, t, o = dep
|
|
92
|
+
|
|
93
|
+
wf.rec_dependencies(t.to_sym).each do |d|
|
|
94
|
+
if Array === d
|
|
95
|
+
new = d.dup
|
|
96
|
+
else
|
|
97
|
+
new = [dep.first, d]
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
if Hash === o and not o.empty?
|
|
101
|
+
if Hash === new.last
|
|
102
|
+
hash = new.last.dup
|
|
103
|
+
o.each{|k,v| hash[k] ||= v}
|
|
104
|
+
new[new.length-1] = hash
|
|
105
|
+
else
|
|
106
|
+
new.push o.dup
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
all_deps << new
|
|
111
|
+
end if wf && t
|
|
112
|
+
|
|
113
|
+
when String, Symbol
|
|
114
|
+
rec_deps = rec_dependencies(dep.to_sym)
|
|
115
|
+
all_deps.concat rec_deps
|
|
116
|
+
when DependencyBlock
|
|
117
|
+
dep = dep.dependency
|
|
118
|
+
raise TryAgain
|
|
119
|
+
end
|
|
120
|
+
rescue TryAgain
|
|
121
|
+
retry
|
|
122
|
+
end
|
|
106
123
|
end
|
|
124
|
+
all_deps.uniq
|
|
125
|
+
else
|
|
126
|
+
[]
|
|
107
127
|
end
|
|
108
|
-
|
|
109
|
-
all_deps << new
|
|
110
|
-
end if wf && t
|
|
111
|
-
|
|
112
|
-
when String, Symbol
|
|
113
|
-
rec_deps = rec_dependencies(dep.to_sym)
|
|
114
|
-
all_deps.concat rec_deps
|
|
115
|
-
when DependencyBlock
|
|
116
|
-
dep = dep.dependency
|
|
117
|
-
raise TryAgain
|
|
118
|
-
end
|
|
119
|
-
rescue TryAgain
|
|
120
|
-
retry
|
|
121
|
-
end
|
|
122
|
-
end
|
|
123
|
-
all_deps.uniq
|
|
124
|
-
else
|
|
125
|
-
[]
|
|
126
|
-
end
|
|
127
|
-
end
|
|
128
|
+
end
|
|
128
129
|
end
|
|
129
130
|
|
|
130
131
|
def task_from_dep(dep)
|
|
@@ -182,7 +183,7 @@ module Workflow
|
|
|
182
183
|
acc
|
|
183
184
|
}.tap{|h| IndiferentHash.setup(h)}
|
|
184
185
|
end
|
|
185
|
-
|
|
186
|
+
|
|
186
187
|
def rec_input_use(taskname)
|
|
187
188
|
task = task_from_dep(taskname)
|
|
188
189
|
deps = rec_dependencies(taskname)
|
|
@@ -474,7 +475,7 @@ module Workflow
|
|
|
474
475
|
else
|
|
475
476
|
workdir_find = workdir
|
|
476
477
|
end
|
|
477
|
-
|
|
478
|
+
|
|
478
479
|
workdir_find = File.expand_path(workdir_find)
|
|
479
480
|
path = File.expand_path(path)
|
|
480
481
|
dir = File.dirname(path)
|
|
@@ -85,7 +85,11 @@ class RemoteWorkflow
|
|
|
85
85
|
|
|
86
86
|
RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
|
|
87
87
|
name = RemoteWorkflow.capture_exception do
|
|
88
|
-
|
|
88
|
+
begin
|
|
89
|
+
RestClient.post(self.encode(url), params)
|
|
90
|
+
rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
|
|
91
|
+
raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
|
|
92
|
+
end
|
|
89
93
|
end
|
|
90
94
|
|
|
91
95
|
Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
|
|
@@ -93,8 +93,8 @@ class Step
|
|
|
93
93
|
else
|
|
94
94
|
Open.write(path + '.read', value.to_s)
|
|
95
95
|
end
|
|
96
|
-
when Step ===
|
|
97
|
-
|
|
96
|
+
when Step === value
|
|
97
|
+
value = value.produce.load
|
|
98
98
|
else
|
|
99
99
|
Open.write(path, value.to_s)
|
|
100
100
|
end
|
|
@@ -110,12 +110,14 @@ class Step
|
|
|
110
110
|
task_info = workflow.task_info(task_name)
|
|
111
111
|
input_types = task_info[:input_types]
|
|
112
112
|
task_inputs = task_info[:inputs]
|
|
113
|
+
input_defaults = task_info[:input_defaults]
|
|
113
114
|
|
|
114
115
|
inputs = {}
|
|
115
116
|
job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
|
|
116
117
|
next unless task_inputs.include? name.to_sym
|
|
117
118
|
next if options and ! options.include?(name)
|
|
118
119
|
next if value.nil?
|
|
120
|
+
next if input_defaults[name] == value
|
|
119
121
|
inputs[name] = value
|
|
120
122
|
end
|
|
121
123
|
|
|
@@ -125,7 +127,7 @@ class Step
|
|
|
125
127
|
end
|
|
126
128
|
save_inputs(inputs, input_types, dir)
|
|
127
129
|
|
|
128
|
-
inputs.
|
|
130
|
+
inputs.keys
|
|
129
131
|
end
|
|
130
132
|
|
|
131
133
|
def name
|
data/lib/rbbt/workflow/usage.rb
CHANGED
|
@@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow>
|
|
|
25
25
|
--stream Activate streaming of workflow tasks
|
|
26
26
|
-fs--file_server Activate file serving for resources
|
|
27
27
|
-mj--monitor_jobs Monitor jobs (UNSAFE)
|
|
28
|
+
-a--app_dir* Application execution directory
|
|
28
29
|
--export_all Export all workflow tasks (use with caution!)
|
|
29
30
|
--export* Export workflow tasks (asynchronous)
|
|
30
31
|
--export_asynchronous* Export workflow tasks as asynchronous
|
|
@@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn
|
|
|
74
75
|
exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
|
|
75
76
|
|
|
76
77
|
TmpFile.with_file do |app_dir|
|
|
78
|
+
app_dir = options[:app_dir] if options[:app_dir]
|
|
77
79
|
Misc.in_dir(app_dir) do
|
|
78
80
|
app_dir = Path.setup(app_dir.dup)
|
|
79
81
|
Open.write(app_dir.etc.target_workflow.find, workflow)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rbbt-util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.28.
|
|
4
|
+
version: 5.28.6
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Miguel Vazquez
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-10-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|