rbbt-util 5.32.25 → 5.32.30
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/annotated_array.rb +4 -0
- data/lib/rbbt/annotations/util.rb +29 -0
- data/lib/rbbt/entity.rb +3 -1
- data/lib/rbbt/hpc/orchestrate/batches.rb +152 -0
- data/lib/rbbt/hpc/orchestrate/chains.rb +173 -0
- data/lib/rbbt/hpc/orchestrate/rules.rb +70 -0
- data/lib/rbbt/hpc/orchestrate.old.rb +220 -0
- data/lib/rbbt/hpc/orchestrate.rb +24 -200
- data/lib/rbbt/hpc/slurm.rb +1 -0
- data/lib/rbbt/persist/tsv.rb +1 -1
- data/lib/rbbt/tsv/excel.rb +16 -8
- data/lib/rbbt/util/log.rb +6 -2
- data/lib/rbbt/util/migrate.rb +6 -1
- data/lib/rbbt/util/misc/inspect.rb +4 -1
- data/lib/rbbt/util/misc.rb +5 -0
- data/lib/rbbt/util/python.rb +1 -1
- data/lib/rbbt/workflow/definition.rb +1 -1
- data/lib/rbbt/workflow/examples.rb +0 -65
- data/lib/rbbt/workflow/integration/nextflow.rb +74 -14
- data/lib/rbbt/workflow/step/accessor.rb +0 -70
- data/lib/rbbt/workflow/step/dependencies.rb +8 -2
- data/lib/rbbt/workflow/step/run.rb +1 -1
- data/lib/rbbt/workflow/step/save_load_inputs.rb +175 -0
- data/lib/rbbt/workflow/step.rb +2 -1
- data/lib/rbbt/workflow/task.rb +2 -2
- data/lib/rbbt/workflow.rb +9 -2
- data/share/rbbt_commands/hpc/tail +0 -13
- data/share/rbbt_commands/lsf/tail +0 -13
- data/share/rbbt_commands/slurm/tail +0 -13
- data/share/rbbt_commands/tsv/keys +14 -15
- data/share/rbbt_commands/tsv/read_excel +2 -2
- data/share/rbbt_commands/workflow/task +11 -5
- data/test/rbbt/annotations/test_util.rb +11 -0
- data/test/rbbt/hpc/orchestrate/test_batches.rb +113 -0
- data/test/rbbt/hpc/orchestrate/test_chains.rb +139 -0
- data/test/rbbt/hpc/orchestrate/test_rules.rb +92 -0
- data/test/rbbt/hpc/test_orchestrate.rb +144 -0
- data/test/rbbt/tsv/test_excel.rb +38 -4
- data/test/rbbt/util/test_misc.rb +4 -0
- data/test/rbbt/workflow/step/test_dependencies.rb +14 -13
- data/test/rbbt/workflow/step/test_save_load_inputs.rb +46 -0
- metadata +17 -2
@@ -0,0 +1,139 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/workflow'
|
3
|
+
require 'rbbt/hpc/orchestrate/chains'
|
4
|
+
|
5
|
+
module TestWFA
|
6
|
+
extend Workflow
|
7
|
+
|
8
|
+
task :a1 => :string do self.task_name.to_s end
|
9
|
+
|
10
|
+
dep :a1
|
11
|
+
task :a2 => :string do self.task_name.to_s end
|
12
|
+
|
13
|
+
dep :a2
|
14
|
+
task :a3 => :string do self.task_name.to_s end
|
15
|
+
end
|
16
|
+
|
17
|
+
module TestWFB
|
18
|
+
extend Workflow
|
19
|
+
|
20
|
+
dep TestWFA, :a2
|
21
|
+
task :b1 => :string do self.task_name.to_s end
|
22
|
+
|
23
|
+
dep :b1
|
24
|
+
task :b2 => :string do self.task_name.to_s end
|
25
|
+
end
|
26
|
+
|
27
|
+
module TestWFC
|
28
|
+
extend Workflow
|
29
|
+
|
30
|
+
dep TestWFA, :a1
|
31
|
+
dep_task :c1, TestWFB, :b2
|
32
|
+
|
33
|
+
task :c2 => :string do self.task_name.to_s end
|
34
|
+
|
35
|
+
dep :c1
|
36
|
+
dep :c2
|
37
|
+
task :c3 => :string do self.task_name.to_s end
|
38
|
+
end
|
39
|
+
|
40
|
+
module TestWFD
|
41
|
+
extend Workflow
|
42
|
+
|
43
|
+
dep TestWFC, :c3, :jobname => "First c3"
|
44
|
+
dep TestWFC, :c3, :jobname => "Second c3"
|
45
|
+
task :d1 => :string do self.task_name.to_s end
|
46
|
+
end
|
47
|
+
|
48
|
+
class TestOrchestrate < Test::Unit::TestCase
|
49
|
+
|
50
|
+
RULES = IndiferentHash.setup(YAML.load(<<-EOF))
|
51
|
+
---
|
52
|
+
defaults:
|
53
|
+
queue: first_queue
|
54
|
+
time: 1h
|
55
|
+
log: 2
|
56
|
+
config_keys: key1 value1 token1
|
57
|
+
chains:
|
58
|
+
chain_a_b:
|
59
|
+
tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
|
60
|
+
comment: Should not include TestWFA#a1
|
61
|
+
chain_a:
|
62
|
+
workflow: TestWFA
|
63
|
+
tasks: a1, a2, a3
|
64
|
+
config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
|
65
|
+
chain_b:
|
66
|
+
workflow: TestWFB
|
67
|
+
tasks: b1, b2
|
68
|
+
chain_b2:
|
69
|
+
tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
|
70
|
+
chain_d:
|
71
|
+
tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
|
72
|
+
TestWFA:
|
73
|
+
defaults:
|
74
|
+
log: 4
|
75
|
+
config_keys: key4 value4 token4
|
76
|
+
a1:
|
77
|
+
cpus: 10
|
78
|
+
config_keys: key5 value5 token5
|
79
|
+
TestWFC:
|
80
|
+
defaults:
|
81
|
+
skip: true
|
82
|
+
log: 4
|
83
|
+
EOF
|
84
|
+
|
85
|
+
def test_parse_chains
|
86
|
+
chains = HPC::Orchestration.parse_chains RULES
|
87
|
+
assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_job_chain_a
|
91
|
+
job = TestWFA.job(:a3, nil)
|
92
|
+
job.recursive_clean
|
93
|
+
|
94
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
95
|
+
job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
|
96
|
+
|
97
|
+
assert_equal %w(chain_a_b chain_a), job_chains.fields
|
98
|
+
assert_equal 3, job_chains["chain_a"].last[:jobs].length
|
99
|
+
assert_equal job, job_chains["chain_a"].last[:top_level]
|
100
|
+
end
|
101
|
+
|
102
|
+
def test_job_chain_b
|
103
|
+
job = TestWFB.job(:b2, nil)
|
104
|
+
job.recursive_clean
|
105
|
+
|
106
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
107
|
+
job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
|
108
|
+
|
109
|
+
assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
|
110
|
+
|
111
|
+
assert_equal 2, job_chains["chain_a"].last[:jobs].length
|
112
|
+
assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
|
113
|
+
|
114
|
+
assert_equal 2, job_chains["chain_b"].last[:jobs].length
|
115
|
+
assert_equal job, job_chains["chain_b"].last[:top_level]
|
116
|
+
|
117
|
+
assert_equal 2, job_chains["chain_b2"].last[:jobs].length
|
118
|
+
assert_equal job, job_chains["chain_b2"].last[:top_level]
|
119
|
+
|
120
|
+
assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
|
121
|
+
assert_equal job, job_chains["chain_a_b"].last[:top_level]
|
122
|
+
end
|
123
|
+
|
124
|
+
def test_job_chains_double
|
125
|
+
job = TestWFD.job(:d1, nil)
|
126
|
+
job.recursive_clean
|
127
|
+
|
128
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
129
|
+
job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
|
130
|
+
|
131
|
+
assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
|
132
|
+
assert_equal ["First c3", "Second c3"].sort, job_chains.
|
133
|
+
select{|n,i| n == 'chain_a' }.
|
134
|
+
collect{|n,i| i[:top_level].name }.sort
|
135
|
+
|
136
|
+
assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/workflow'
|
3
|
+
require 'rbbt/hpc/orchestrate/rules'
|
4
|
+
|
5
|
+
module TestWFA
|
6
|
+
extend Workflow
|
7
|
+
|
8
|
+
task :a1 => :string do self.task_name.to_s end
|
9
|
+
|
10
|
+
dep :a1
|
11
|
+
task :a2 => :string do self.task_name.to_s end
|
12
|
+
|
13
|
+
dep :a2
|
14
|
+
task :a3 => :string do self.task_name.to_s end
|
15
|
+
end
|
16
|
+
|
17
|
+
module TestWFB
|
18
|
+
extend Workflow
|
19
|
+
|
20
|
+
dep TestWFA, :a2
|
21
|
+
task :b1 => :string do self.task_name.to_s end
|
22
|
+
|
23
|
+
dep :b1
|
24
|
+
task :b2 => :string do self.task_name.to_s end
|
25
|
+
end
|
26
|
+
|
27
|
+
module TestWFC
|
28
|
+
extend Workflow
|
29
|
+
|
30
|
+
dep TestWFA, :a1
|
31
|
+
dep_task :c1, TestWFB, :b2
|
32
|
+
|
33
|
+
task :c2 => :string do self.task_name.to_s end
|
34
|
+
|
35
|
+
dep :c1
|
36
|
+
dep :c2
|
37
|
+
task :c3 => :string do self.task_name.to_s end
|
38
|
+
end
|
39
|
+
|
40
|
+
class TestOrchestrate < Test::Unit::TestCase
|
41
|
+
|
42
|
+
RULES = IndiferentHash.setup(YAML.load(<<-EOF))
|
43
|
+
---
|
44
|
+
defaults:
|
45
|
+
queue: first_queue
|
46
|
+
time: 1h
|
47
|
+
log: 2
|
48
|
+
config_keys: key1 value1 token1
|
49
|
+
chains:
|
50
|
+
chain_a:
|
51
|
+
workflow: TestWFA
|
52
|
+
tasks: a1, a2, a3
|
53
|
+
config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
|
54
|
+
chain_b:
|
55
|
+
workflow: TestWFB
|
56
|
+
tasks: b1, b2
|
57
|
+
chain_b2:
|
58
|
+
comment: This chain is not valid, it is missing a2
|
59
|
+
tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
|
60
|
+
TestWFA:
|
61
|
+
defaults:
|
62
|
+
log: 4
|
63
|
+
config_keys: key4 value4 token4
|
64
|
+
a1:
|
65
|
+
cpus: 10
|
66
|
+
config_keys: key5 value5 token5
|
67
|
+
TestWFC:
|
68
|
+
defaults:
|
69
|
+
skip: true
|
70
|
+
log: 4
|
71
|
+
EOF
|
72
|
+
|
73
|
+
def test_defaults
|
74
|
+
rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
|
75
|
+
assert_equal "first_queue" , rules[:queue]
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def test_task_options
|
80
|
+
rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
|
81
|
+
assert_equal 10, rules[:cpus]
|
82
|
+
assert_equal 4, rules[:log]
|
83
|
+
end
|
84
|
+
|
85
|
+
def test_skip
|
86
|
+
rules = HPC::Orchestration.task_specific_rules RULES, "TestWFC", :c1
|
87
|
+
assert rules[:skip]
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
end
|
92
|
+
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/workflow'
|
3
|
+
require 'rbbt/hpc/orchestrate'
|
4
|
+
|
5
|
+
module TestWFA
|
6
|
+
extend Workflow
|
7
|
+
|
8
|
+
task :a1 => :string do self.task_name.to_s end
|
9
|
+
|
10
|
+
dep :a1
|
11
|
+
task :a2 => :string do self.task_name.to_s end
|
12
|
+
|
13
|
+
dep :a2
|
14
|
+
task :a3 => :string do self.task_name.to_s end
|
15
|
+
end
|
16
|
+
|
17
|
+
module TestWFB
|
18
|
+
extend Workflow
|
19
|
+
|
20
|
+
dep TestWFA, :a2
|
21
|
+
task :b1 => :string do self.task_name.to_s end
|
22
|
+
|
23
|
+
dep :b1
|
24
|
+
task :b2 => :string do self.task_name.to_s end
|
25
|
+
end
|
26
|
+
|
27
|
+
module TestWFC
|
28
|
+
extend Workflow
|
29
|
+
|
30
|
+
dep TestWFA, :a1
|
31
|
+
dep_task :c1, TestWFB, :b2
|
32
|
+
|
33
|
+
task :c2 => :string do self.task_name.to_s end
|
34
|
+
|
35
|
+
dep :c1
|
36
|
+
dep :c2
|
37
|
+
task :c3 => :string do self.task_name.to_s end
|
38
|
+
end
|
39
|
+
|
40
|
+
class TestOrchestrate < Test::Unit::TestCase
|
41
|
+
|
42
|
+
RULES = IndiferentHash.setup(YAML.load(<<-EOF))
|
43
|
+
---
|
44
|
+
defaults:
|
45
|
+
queue: first_queue
|
46
|
+
time: 1h
|
47
|
+
log: 2
|
48
|
+
config_keys: key1 value1 token1
|
49
|
+
chains:
|
50
|
+
chain_a:
|
51
|
+
workflow: TestWFA
|
52
|
+
tasks: a1, a2, a3
|
53
|
+
config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
|
54
|
+
chain_b:
|
55
|
+
workflow: TestWFB
|
56
|
+
tasks: b1, b2
|
57
|
+
chain_b2:
|
58
|
+
comment: This chain is not valid, it is missing a2
|
59
|
+
tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
|
60
|
+
TestWFA:
|
61
|
+
defaults:
|
62
|
+
log: 4
|
63
|
+
config_keys: key4 value4 token4
|
64
|
+
a1:
|
65
|
+
cpus: 10
|
66
|
+
config_keys: key5 value5 token5
|
67
|
+
TestWFC:
|
68
|
+
defaults:
|
69
|
+
skip: true
|
70
|
+
log: 4
|
71
|
+
EOF
|
72
|
+
|
73
|
+
def test_parse_chains
|
74
|
+
chains = HPC::Orchestration.parse_chains RULES
|
75
|
+
assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_task_options
|
79
|
+
rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
|
80
|
+
assert_equal 10, rules[:cpus]
|
81
|
+
assert_equal 4, rules[:log]
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_job_workload
|
85
|
+
job = TestWFB.job(:b2, nil)
|
86
|
+
job.recursive_clean
|
87
|
+
|
88
|
+
workload = HPC::Orchestration.job_workload(job)
|
89
|
+
|
90
|
+
assert workload.include? job
|
91
|
+
end
|
92
|
+
|
93
|
+
def test_job_chains
|
94
|
+
job = TestWFB.job(:b2, nil)
|
95
|
+
|
96
|
+
job.recursive_clean
|
97
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
98
|
+
assert_equal 3, job_chains.length
|
99
|
+
|
100
|
+
TestWFB.job(:b1).run
|
101
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
102
|
+
assert_equal 0, job_chains.length
|
103
|
+
|
104
|
+
job.recursive_clean
|
105
|
+
TestWFA.job(:a1).run
|
106
|
+
job_chains = HPC::Orchestration.job_chains(RULES, job)
|
107
|
+
assert_equal 2, job_chains.length
|
108
|
+
end
|
109
|
+
|
110
|
+
def test_job_batches
|
111
|
+
job = TestWFB.job(:b2, nil)
|
112
|
+
|
113
|
+
job.recursive_clean
|
114
|
+
batches = HPC::Orchestration.job_batches(RULES, job)
|
115
|
+
assert_equal 2, batches.length
|
116
|
+
|
117
|
+
assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
|
118
|
+
end
|
119
|
+
|
120
|
+
def test_job_batches_skip
|
121
|
+
job = TestWFC.job(:c1, nil)
|
122
|
+
|
123
|
+
job.recursive_clean
|
124
|
+
batches = HPC::Orchestration.job_batches(RULES, job)
|
125
|
+
assert_equal 2, batches.length
|
126
|
+
|
127
|
+
assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
|
128
|
+
end
|
129
|
+
|
130
|
+
def test_job_batches_skip_2
|
131
|
+
job = TestWFC.job(:c3, nil)
|
132
|
+
|
133
|
+
job.recursive_clean
|
134
|
+
batches = HPC::Orchestration.job_batches(RULES, job)
|
135
|
+
assert_equal 4, batches.length
|
136
|
+
|
137
|
+
assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
|
138
|
+
|
139
|
+
batches
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
end
|
144
|
+
|
data/test/rbbt/tsv/test_excel.rb
CHANGED
@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
|
|
2
2
|
require 'rbbt/tsv/excel'
|
3
3
|
|
4
4
|
class TestExcel < Test::Unit::TestCase
|
5
|
-
def
|
5
|
+
def test_xls
|
6
6
|
content =<<-EOF
|
7
7
|
#Id ValueA ValueB OtherID
|
8
8
|
row1 a|aa|aaa b Id1|Id2
|
@@ -19,7 +19,7 @@ row2 A B Id3
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def test_xlsx
|
23
23
|
content =<<-EOF
|
24
24
|
#Id ValueA ValueB OtherID
|
25
25
|
row1 a|aa|aaa b Id1|Id2
|
@@ -36,7 +36,7 @@ row2 A B Id3
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
def
|
39
|
+
def test_excel
|
40
40
|
content =<<-EOF
|
41
41
|
#Id ValueA ValueB OtherID
|
42
42
|
row1 a|aa|aaa b Id1|Id2
|
@@ -63,7 +63,7 @@ row2 A B Id3
|
|
63
63
|
end
|
64
64
|
end
|
65
65
|
|
66
|
-
def
|
66
|
+
def test_excel_sheets
|
67
67
|
content =<<-EOF
|
68
68
|
#Id ValueA ValueB OtherID
|
69
69
|
row1 a|aa|aaa b Id1|Id2
|
@@ -133,5 +133,39 @@ row2 A B Id3
|
|
133
133
|
end
|
134
134
|
end
|
135
135
|
end
|
136
|
+
|
137
|
+
def test_excel_multi_sheets
|
138
|
+
content =<<-EOF
|
139
|
+
#Id ValueA ValueB OtherID
|
140
|
+
row1 a|aa|aaa b Id1|Id2
|
141
|
+
row2 A B Id3
|
142
|
+
EOF
|
143
|
+
|
144
|
+
TmpFile.with_file(content) do |filename|
|
145
|
+
tsv1 = TSV.open(filename, :sep => /\s+/)
|
146
|
+
tsv2 = tsv1.annotate(tsv1.dup)
|
147
|
+
tsv3 = tsv1.annotate(tsv1.dup)
|
148
|
+
|
149
|
+
tsv2["row2"] = [["AA"], ["BB"], ["Id4"]]
|
150
|
+
tsv3["row2"] = [["AAA"], ["BBB"], ["Id5"]]
|
151
|
+
|
152
|
+
TmpFile.with_file(nil, false, :extension => 'xlsx') do |excelfile|
|
153
|
+
tsv1.xlsx(excelfile, :sheet => "S1")
|
154
|
+
tsv2.xlsx(excelfile, :sheet => "S2", :add_sheet => true)
|
155
|
+
workbook = RubyXL::Parser.parse excelfile
|
156
|
+
|
157
|
+
assert_equal %w(S1 S2), workbook.worksheets.collect{|s| s.sheet_name}
|
158
|
+
|
159
|
+
new = TSV.excel(excelfile, :sheet => "S1")
|
160
|
+
assert_equal %w(row1 row2), new.keys.sort
|
161
|
+
assert_equal %w(A), new["row2"]["ValueA"]
|
162
|
+
|
163
|
+
new = TSV.excel(excelfile, :sheet => "S2")
|
164
|
+
assert_equal %w(row1 row2), new.keys.sort
|
165
|
+
assert_equal %w(AA), new["row2"]["ValueA"]
|
166
|
+
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
136
170
|
end
|
137
171
|
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -604,8 +604,12 @@ EOF
|
|
604
604
|
assert_equal 10, Misc.timespan('10')
|
605
605
|
assert_equal 10, Misc.timespan('10s')
|
606
606
|
assert_equal 60, Misc.timespan('1min')
|
607
|
+
assert_equal 60, Misc.timespan('00:01:00')
|
608
|
+
assert_equal 60, Misc.timespan('01:00')
|
609
|
+
assert_equal 60, Misc.timespan('0:1:00')
|
607
610
|
assert_equal 60*60*24, Misc.timespan('1d')
|
608
611
|
assert_equal 60*60*24, Misc.timespan('1d')
|
612
|
+
assert_equal 60*60*24, Misc.timespan('24:00:00')
|
609
613
|
end
|
610
614
|
|
611
615
|
def test_remove_long_items
|
@@ -203,20 +203,21 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
203
203
|
size = 100000
|
204
204
|
content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
205
205
|
last_line = nil
|
206
|
-
Log.
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
206
|
+
Log.with_severity 0 do
|
207
|
+
TmpFile.with_file(content) do |input_file|
|
208
|
+
begin
|
209
|
+
job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
|
210
|
+
job.recursive_clean
|
211
|
+
job.run(:stream)
|
212
|
+
io = TSV.get_stream job
|
213
|
+
while line = io.gets
|
214
|
+
last_line = line.strip
|
215
|
+
end
|
216
|
+
io.join if io.respond_to? :join
|
217
|
+
rescue Exception
|
218
|
+
job.abort
|
219
|
+
raise $!
|
215
220
|
end
|
216
|
-
io.join if io.respond_to? :join
|
217
|
-
rescue Exception
|
218
|
-
job.abort
|
219
|
-
raise $!
|
220
221
|
end
|
221
222
|
end
|
222
223
|
assert last_line.include? "Line #{size}"
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/workflow/step/save_load_inputs'
|
3
|
+
|
4
|
+
ENV["RBBT_DEBUG_JOB_HASH"] = true.to_s
|
5
|
+
require 'rbbt/workflow'
|
6
|
+
module TestSaveLoadWF
|
7
|
+
extend Workflow
|
8
|
+
|
9
|
+
task :number => :integer do
|
10
|
+
10
|
11
|
+
end
|
12
|
+
|
13
|
+
task :list => :array do
|
14
|
+
(0..10).to_a.collect{|e| e.to_s}
|
15
|
+
end
|
16
|
+
|
17
|
+
input :list, :array
|
18
|
+
input :number, :integer
|
19
|
+
task :reverse => :array do |list|
|
20
|
+
list.reverse
|
21
|
+
end
|
22
|
+
|
23
|
+
dep :list
|
24
|
+
dep :number
|
25
|
+
dep :reverse, :list => :list, :number => :number
|
26
|
+
task :prefix => :array do
|
27
|
+
step(:reverse).run.collect{|e| "A-#{e}" }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class TestSaveLoad < Test::Unit::TestCase
|
32
|
+
def test_save
|
33
|
+
Log.with_severity 0 do
|
34
|
+
job = TestSaveLoadWF.job(:prefix)
|
35
|
+
job.recursive_clean
|
36
|
+
job = TestSaveLoadWF.job(:prefix)
|
37
|
+
TmpFile.with_file do |directory|
|
38
|
+
Step.save_job_inputs(job.step(:reverse), directory)
|
39
|
+
job.produce
|
40
|
+
newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
|
41
|
+
assert_equal job.rec_dependencies.last.path, newjob.path
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.32.
|
4
|
+
version: 5.32.30
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-02-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -215,7 +215,11 @@ files:
|
|
215
215
|
- lib/rbbt/hpc.rb
|
216
216
|
- lib/rbbt/hpc/batch.rb
|
217
217
|
- lib/rbbt/hpc/lsf.rb
|
218
|
+
- lib/rbbt/hpc/orchestrate.old.rb
|
218
219
|
- lib/rbbt/hpc/orchestrate.rb
|
220
|
+
- lib/rbbt/hpc/orchestrate/batches.rb
|
221
|
+
- lib/rbbt/hpc/orchestrate/chains.rb
|
222
|
+
- lib/rbbt/hpc/orchestrate/rules.rb
|
219
223
|
- lib/rbbt/hpc/slurm.rb
|
220
224
|
- lib/rbbt/knowledge_base.rb
|
221
225
|
- lib/rbbt/knowledge_base/enrichment.rb
|
@@ -345,6 +349,7 @@ files:
|
|
345
349
|
- lib/rbbt/workflow/step/info.rb
|
346
350
|
- lib/rbbt/workflow/step/prepare.rb
|
347
351
|
- lib/rbbt/workflow/step/run.rb
|
352
|
+
- lib/rbbt/workflow/step/save_load_inputs.rb
|
348
353
|
- lib/rbbt/workflow/step/status.rb
|
349
354
|
- lib/rbbt/workflow/task.rb
|
350
355
|
- lib/rbbt/workflow/usage.rb
|
@@ -470,7 +475,11 @@ files:
|
|
470
475
|
- test/rbbt/association/test_open.rb
|
471
476
|
- test/rbbt/association/test_util.rb
|
472
477
|
- test/rbbt/entity/test_identifiers.rb
|
478
|
+
- test/rbbt/hpc/orchestrate/test_batches.rb
|
479
|
+
- test/rbbt/hpc/orchestrate/test_chains.rb
|
480
|
+
- test/rbbt/hpc/orchestrate/test_rules.rb
|
473
481
|
- test/rbbt/hpc/test_batch.rb
|
482
|
+
- test/rbbt/hpc/test_orchestrate.rb
|
474
483
|
- test/rbbt/hpc/test_slurm.rb
|
475
484
|
- test/rbbt/knowledge_base/test_enrichment.rb
|
476
485
|
- test/rbbt/knowledge_base/test_entity.rb
|
@@ -550,6 +559,7 @@ files:
|
|
550
559
|
- test/rbbt/util/test_simpleopt.rb
|
551
560
|
- test/rbbt/util/test_tmpfile.rb
|
552
561
|
- test/rbbt/workflow/step/test_dependencies.rb
|
562
|
+
- test/rbbt/workflow/step/test_save_load_inputs.rb
|
553
563
|
- test/rbbt/workflow/test_doc.rb
|
554
564
|
- test/rbbt/workflow/test_remote_workflow.rb
|
555
565
|
- test/rbbt/workflow/test_schedule.rb
|
@@ -592,6 +602,7 @@ test_files:
|
|
592
602
|
- test/rbbt/workflow/test_schedule.rb
|
593
603
|
- test/rbbt/workflow/test_step.rb
|
594
604
|
- test/rbbt/workflow/step/test_dependencies.rb
|
605
|
+
- test/rbbt/workflow/step/test_save_load_inputs.rb
|
595
606
|
- test/rbbt/workflow/test_task.rb
|
596
607
|
- test/rbbt/resource/test_path.rb
|
597
608
|
- test/rbbt/util/test_colorize.rb
|
@@ -649,7 +660,11 @@ test_files:
|
|
649
660
|
- test/rbbt/tsv/test_stream.rb
|
650
661
|
- test/rbbt/test_association.rb
|
651
662
|
- test/rbbt/hpc/test_batch.rb
|
663
|
+
- test/rbbt/hpc/orchestrate/test_chains.rb
|
664
|
+
- test/rbbt/hpc/orchestrate/test_rules.rb
|
665
|
+
- test/rbbt/hpc/orchestrate/test_batches.rb
|
652
666
|
- test/rbbt/hpc/test_slurm.rb
|
667
|
+
- test/rbbt/hpc/test_orchestrate.rb
|
653
668
|
- test/rbbt/association/test_database.rb
|
654
669
|
- test/rbbt/association/test_item.rb
|
655
670
|
- test/rbbt/association/test_open.rb
|