rbbt-util 5.43.0 → 5.44.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/association/util.rb +1 -1
- data/lib/rbbt/hpc/batch.rb +19 -17
- data/lib/rbbt/tsv/accessor.rb +6 -11
- data/lib/rbbt/tsv/dumper.rb +21 -10
- data/lib/rbbt/tsv/index.rb +2 -1
- data/lib/rbbt/util/misc/math.rb +0 -1
- data/lib/rbbt/util/open.rb +1 -1
- data/lib/rbbt/workflow/step/info.rb +1 -0
- data/lib/rbbt/workflow/step.rb +2 -1
- data/python/rbbt/__init__.py +2 -2
- data/share/rbbt_commands/workflow/task +1 -1
- metadata +4 -220
- data/test/rbbt/annotations/test_util.rb +0 -43
- data/test/rbbt/association/test_database.rb +0 -87
- data/test/rbbt/association/test_index.rb +0 -127
- data/test/rbbt/association/test_item.rb +0 -15
- data/test/rbbt/association/test_open.rb +0 -63
- data/test/rbbt/association/test_util.rb +0 -108
- data/test/rbbt/entity/test_identifiers.rb +0 -34
- data/test/rbbt/hpc/orchestrate/test_batches.rb +0 -70
- data/test/rbbt/hpc/orchestrate/test_chains.rb +0 -108
- data/test/rbbt/hpc/orchestrate/test_rules.rb +0 -59
- data/test/rbbt/hpc/test_batch.rb +0 -64
- data/test/rbbt/hpc/test_hpc_test_workflows.rb +0 -0
- data/test/rbbt/hpc/test_orchestrate.rb +0 -144
- data/test/rbbt/hpc/test_pbs.rb +0 -43
- data/test/rbbt/hpc/test_slurm.rb +0 -28
- data/test/rbbt/knowledge_base/test_enrichment.rb +0 -50
- data/test/rbbt/knowledge_base/test_entity.rb +0 -62
- data/test/rbbt/knowledge_base/test_query.rb +0 -46
- data/test/rbbt/knowledge_base/test_registry.rb +0 -74
- data/test/rbbt/knowledge_base/test_syndicate.rb +0 -48
- data/test/rbbt/knowledge_base/test_traverse.rb +0 -133
- data/test/rbbt/persist/test_tsv.rb +0 -88
- data/test/rbbt/persist/tsv/test_cdb.rb +0 -18
- data/test/rbbt/persist/tsv/test_kyotocabinet.rb +0 -27
- data/test/rbbt/persist/tsv/test_leveldb.rb +0 -18
- data/test/rbbt/persist/tsv/test_lmdb.rb +0 -20
- data/test/rbbt/persist/tsv/test_sharder.rb +0 -164
- data/test/rbbt/persist/tsv/test_tokyocabinet.rb +0 -262
- data/test/rbbt/resource/test_path.rb +0 -49
- data/test/rbbt/test_annotations.rb +0 -167
- data/test/rbbt/test_association.rb +0 -103
- data/test/rbbt/test_entity.rb +0 -252
- data/test/rbbt/test_fix_width_table.rb +0 -135
- data/test/rbbt/test_knowledge_base.rb +0 -226
- data/test/rbbt/test_monitor.rb +0 -11
- data/test/rbbt/test_packed_index.rb +0 -68
- data/test/rbbt/test_persist.rb +0 -85
- data/test/rbbt/test_resource.rb +0 -110
- data/test/rbbt/test_tsv.rb +0 -669
- data/test/rbbt/test_workflow.rb +0 -609
- data/test/rbbt/tsv/parallel/test_through.rb +0 -40
- data/test/rbbt/tsv/parallel/test_traverse.rb +0 -456
- data/test/rbbt/tsv/test_accessor.rb +0 -319
- data/test/rbbt/tsv/test_attach.rb +0 -715
- data/test/rbbt/tsv/test_change_id.rb +0 -61
- data/test/rbbt/tsv/test_csv.rb +0 -49
- data/test/rbbt/tsv/test_excel.rb +0 -171
- data/test/rbbt/tsv/test_field_index.rb +0 -19
- data/test/rbbt/tsv/test_filter.rb +0 -187
- data/test/rbbt/tsv/test_index.rb +0 -308
- data/test/rbbt/tsv/test_manipulate.rb +0 -334
- data/test/rbbt/tsv/test_marshal.rb +0 -24
- data/test/rbbt/tsv/test_matrix.rb +0 -20
- data/test/rbbt/tsv/test_parallel.rb +0 -7
- data/test/rbbt/tsv/test_parser.rb +0 -101
- data/test/rbbt/tsv/test_stream.rb +0 -253
- data/test/rbbt/tsv/test_util.rb +0 -52
- data/test/rbbt/util/R/test_eval.rb +0 -43
- data/test/rbbt/util/R/test_model.rb +0 -128
- data/test/rbbt/util/R/test_plot.rb +0 -38
- data/test/rbbt/util/concurrency/processes/test_socket.rb +0 -70
- data/test/rbbt/util/concurrency/test_processes.rb +0 -192
- data/test/rbbt/util/concurrency/test_threads.rb +0 -40
- data/test/rbbt/util/log/test_progress.rb +0 -111
- data/test/rbbt/util/misc/test_bgzf.rb +0 -48
- data/test/rbbt/util/misc/test_communication.rb +0 -13
- data/test/rbbt/util/misc/test_development.rb +0 -26
- data/test/rbbt/util/misc/test_format.rb +0 -10
- data/test/rbbt/util/misc/test_indiferent_hash.rb +0 -14
- data/test/rbbt/util/misc/test_lock.rb +0 -77
- data/test/rbbt/util/misc/test_multipart_payload.rb +0 -202
- data/test/rbbt/util/misc/test_omics.rb +0 -116
- data/test/rbbt/util/misc/test_pipes.rb +0 -343
- data/test/rbbt/util/misc/test_serialize.rb +0 -24
- data/test/rbbt/util/python/test_util.rb +0 -25
- data/test/rbbt/util/simpleopt/test_get.rb +0 -12
- data/test/rbbt/util/simpleopt/test_parse.rb +0 -10
- data/test/rbbt/util/simpleopt/test_setup.rb +0 -76
- data/test/rbbt/util/test_R.rb +0 -37
- data/test/rbbt/util/test_chain_methods.rb +0 -22
- data/test/rbbt/util/test_cmd.rb +0 -87
- data/test/rbbt/util/test_colorize.rb +0 -22
- data/test/rbbt/util/test_concurrency.rb +0 -6
- data/test/rbbt/util/test_config.rb +0 -69
- data/test/rbbt/util/test_excel2tsv.rb +0 -10
- data/test/rbbt/util/test_filecache.rb +0 -36
- data/test/rbbt/util/test_log.rb +0 -52
- data/test/rbbt/util/test_migrate.rb +0 -34
- data/test/rbbt/util/test_misc.rb +0 -728
- data/test/rbbt/util/test_open.rb +0 -200
- data/test/rbbt/util/test_procpath.rb +0 -23
- data/test/rbbt/util/test_python.rb +0 -144
- data/test/rbbt/util/test_semaphore.rb +0 -36
- data/test/rbbt/util/test_simpleDSL.rb +0 -55
- data/test/rbbt/util/test_simpleopt.rb +0 -11
- data/test/rbbt/util/test_ssh.rb +0 -10
- data/test/rbbt/util/test_tmpfile.rb +0 -32
- data/test/rbbt/workflow/step/test_dependencies.rb +0 -295
- data/test/rbbt/workflow/step/test_save_load_inputs.rb +0 -136
- data/test/rbbt/workflow/test_doc.rb +0 -30
- data/test/rbbt/workflow/test_remote_workflow.rb +0 -99
- data/test/rbbt/workflow/test_schedule.rb +0 -0
- data/test/rbbt/workflow/test_step.rb +0 -231
- data/test/rbbt/workflow/test_task.rb +0 -85
- data/test/rbbt/workflow/util/test_archive.rb +0 -27
- data/test/rbbt/workflow/util/test_data.rb +0 -67
- data/test/rbbt/workflow/util/test_orchestrator.rb +0 -263
- data/test/test_helper.rb +0 -114
data/test/rbbt/test_workflow.rb
DELETED
@@ -1,609 +0,0 @@
|
|
1
|
-
require File.expand_path(File.dirname(__FILE__) + '/../test_helper')
|
2
|
-
require 'rbbt/workflow'
|
3
|
-
require 'rbbt/util/tmpfile'
|
4
|
-
require 'test/unit'
|
5
|
-
|
6
|
-
module TestWF
|
7
|
-
extend Workflow
|
8
|
-
|
9
|
-
helper :user do
|
10
|
-
"User"
|
11
|
-
end
|
12
|
-
|
13
|
-
task :user => :string do
|
14
|
-
user
|
15
|
-
end
|
16
|
-
|
17
|
-
str = "TEST"
|
18
|
-
task :str => :string do
|
19
|
-
str
|
20
|
-
end
|
21
|
-
|
22
|
-
dep :str
|
23
|
-
task :reverse => :string do
|
24
|
-
step(:str).load.reverse
|
25
|
-
end
|
26
|
-
|
27
|
-
dep :str
|
28
|
-
task :downcase => :string do
|
29
|
-
step(:str).load.downcase
|
30
|
-
end
|
31
|
-
|
32
|
-
dep :str
|
33
|
-
input :times, :integer, "Times to repeat"
|
34
|
-
task :repeat => :string do |times|
|
35
|
-
[step(:str).load] * times * "\n"
|
36
|
-
end
|
37
|
-
|
38
|
-
input :number, :float, "Number to doble"
|
39
|
-
def self.double(number)
|
40
|
-
2 * number
|
41
|
-
end
|
42
|
-
task :double => :float
|
43
|
-
|
44
|
-
desc <<-EOT
|
45
|
-
Returns numer * 2 lines containing TEST
|
46
|
-
EOT
|
47
|
-
dep :str
|
48
|
-
dep :double
|
49
|
-
task :repeat2 => :string do
|
50
|
-
[step(:str).load] * step(:double).load * "\n"
|
51
|
-
end
|
52
|
-
|
53
|
-
dep :str, :repeat, :repeat2
|
54
|
-
task :double_dep => :array do
|
55
|
-
[] << step(:str).load << step(:repeat).load << step(:repeat2).load
|
56
|
-
end
|
57
|
-
|
58
|
-
export_synchronous :double
|
59
|
-
export_asynchronous :repeat2
|
60
|
-
|
61
|
-
input :letter, :string, "Letter", "D"
|
62
|
-
task :letter => :string do |l|
|
63
|
-
l
|
64
|
-
end
|
65
|
-
|
66
|
-
dep :letter
|
67
|
-
task :letter_repeat => :string do |l|
|
68
|
-
([step(:letter).load] * 2) * ""
|
69
|
-
end
|
70
|
-
|
71
|
-
dep :letter
|
72
|
-
dep :letter_repeat, :letter => "A"
|
73
|
-
task :two_letters => :string do
|
74
|
-
dependencies.collect{|d| d.load } * ":"
|
75
|
-
end
|
76
|
-
|
77
|
-
task :stream => :array do
|
78
|
-
Misc.open_pipe do |sin|
|
79
|
-
5.times do |i|
|
80
|
-
sin.puts "line #{ i }"
|
81
|
-
sleep 1
|
82
|
-
end
|
83
|
-
end
|
84
|
-
end
|
85
|
-
|
86
|
-
dep :stream
|
87
|
-
task :stream2 => :array do
|
88
|
-
TSV.get_stream step(:stream)
|
89
|
-
end
|
90
|
-
|
91
|
-
input :name, :string, "Name"
|
92
|
-
task :input_dep => :text do |name|
|
93
|
-
<<-EOF
|
94
|
-
Hi #{name}:
|
95
|
-
This is the input text
|
96
|
-
for this dependency
|
97
|
-
EOF
|
98
|
-
end
|
99
|
-
|
100
|
-
input :text, :text, "Input text"
|
101
|
-
task :reverse_input_text => :text do |text|
|
102
|
-
text.reverse
|
103
|
-
end
|
104
|
-
|
105
|
-
dep :input_dep
|
106
|
-
dep :reverse_input_text, :text => :input_dep
|
107
|
-
task :send_input_dep_to_reverse => :text do
|
108
|
-
TSV.get_stream step(:reverse_input_text)
|
109
|
-
end
|
110
|
-
|
111
|
-
input :i, :string, "Input", "A"
|
112
|
-
task :t1 => :string do |i|
|
113
|
-
i
|
114
|
-
end
|
115
|
-
|
116
|
-
input :i, :string, "Input", "B"
|
117
|
-
task :t2 => :string do |i|
|
118
|
-
i
|
119
|
-
end
|
120
|
-
|
121
|
-
dep :t1, :i => "C"
|
122
|
-
dep :t2
|
123
|
-
task :t3 => :string do |i|
|
124
|
-
step(:t1).load + step(:t2).load
|
125
|
-
end
|
126
|
-
|
127
|
-
input :name, :string, "Name", nil, :jobname => true
|
128
|
-
task :call_name => :string do |name|
|
129
|
-
"Hi #{name}"
|
130
|
-
end
|
131
|
-
|
132
|
-
input :num, :integer
|
133
|
-
task :odd => :integer do |num|
|
134
|
-
raise ParameterException, "Not odd" if num % 2 == 0
|
135
|
-
num
|
136
|
-
end
|
137
|
-
|
138
|
-
dep :odd, :num => 10, :compute => :canfail
|
139
|
-
dep :odd, :num => 11, :compute => :canfail
|
140
|
-
dep :odd, :num => 12, :compute => :canfail
|
141
|
-
dep :odd, :num => 13, :compute => :canfail
|
142
|
-
task :sum_odds => :integer do
|
143
|
-
dependencies.inject(0) do |acc, dep|
|
144
|
-
acc += dep.load unless dep.error?
|
145
|
-
acc
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
dep :sum_odds
|
150
|
-
task :sum_odds_str => :string do
|
151
|
-
"Sum odds: " << step(:sum_odds).load.to_s
|
152
|
-
end
|
153
|
-
|
154
|
-
input :file, :file, "Save file"
|
155
|
-
task :reverse_file => :text do |file|
|
156
|
-
Open.read(file).reverse
|
157
|
-
end
|
158
|
-
|
159
|
-
task :create_file => :text do |file|
|
160
|
-
Open.write(file('a'), "A")
|
161
|
-
Open.write(file('b'), "B")
|
162
|
-
"DONE"
|
163
|
-
end
|
164
|
-
|
165
|
-
dep_task :reverse_step_file, TestWF, :reverse_file do |jobname, options, dependencies|
|
166
|
-
dep = dependencies.flatten.first
|
167
|
-
end
|
168
|
-
|
169
|
-
task :overr_orig => :string do
|
170
|
-
"over"
|
171
|
-
end
|
172
|
-
|
173
|
-
task :overr_alt => :string do
|
174
|
-
"alt"
|
175
|
-
end
|
176
|
-
|
177
|
-
task :overr_alt2 => :string do
|
178
|
-
"alt"
|
179
|
-
end
|
180
|
-
|
181
|
-
|
182
|
-
dep :overr_orig
|
183
|
-
task :overr_target => :string do
|
184
|
-
step(:overr_orig).load.reverse
|
185
|
-
end
|
186
|
-
|
187
|
-
dep :overr_alt, :not_overriden => true
|
188
|
-
dep :overr_target, "TestWF#overr_orig" => :overr_alt
|
189
|
-
task :overr_action => :string do
|
190
|
-
step(:overr_target).load.upcase
|
191
|
-
end
|
192
|
-
|
193
|
-
|
194
|
-
dep :overr_alt2, :not_overriden => true
|
195
|
-
dep :overr_target, "TestWF#overr_orig" => :overr_alt2, :not_overriden => true
|
196
|
-
task :not_overr_action => :string do
|
197
|
-
step(:overr_target).load.upcase
|
198
|
-
end
|
199
|
-
|
200
|
-
|
201
|
-
end
|
202
|
-
|
203
|
-
TestWF.workdir = Rbbt.tmp.test.jobs.TestWF
|
204
|
-
|
205
|
-
class TestWorkflow < Test::Unit::TestCase
|
206
|
-
|
207
|
-
|
208
|
-
def test_repo_marshal
|
209
|
-
TmpFile.with_file do |tmpdir|
|
210
|
-
tmpdir = Rbbt.tmp.repo_dir.find
|
211
|
-
repo = File.join(tmpdir, 'repo')
|
212
|
-
|
213
|
-
filename = 'file'
|
214
|
-
Open.repository_dirs.push(repo)
|
215
|
-
|
216
|
-
job = TestWF.job(:call_name, "Miguel")
|
217
|
-
job.run
|
218
|
-
|
219
|
-
obj = job.info
|
220
|
-
Open.write(File.join(repo, filename), Marshal.dump(obj))
|
221
|
-
new =Open.open(File.join(repo, filename)) do |f|
|
222
|
-
Marshal.load(f)
|
223
|
-
end
|
224
|
-
|
225
|
-
assert_equal new, obj
|
226
|
-
end
|
227
|
-
|
228
|
-
end
|
229
|
-
|
230
|
-
def test_in_repo
|
231
|
-
job = TestWF.job(:call_name, "Miguel")
|
232
|
-
assert_equal "Hi Miguel", job.run
|
233
|
-
assert_equal "Miguel", job.clean_name
|
234
|
-
end
|
235
|
-
|
236
|
-
def test_as_jobname
|
237
|
-
job = TestWF.job(:call_name, "Miguel")
|
238
|
-
assert_equal "Hi Miguel", job.run
|
239
|
-
assert_equal "Miguel", job.clean_name
|
240
|
-
|
241
|
-
job = TestWF.job(:call_name, nil, :name => "Miguel")
|
242
|
-
assert_equal "Hi Miguel", job.run
|
243
|
-
assert_equal "Miguel", job.clean_name
|
244
|
-
end
|
245
|
-
|
246
|
-
def test_update_on_input_dependency_update
|
247
|
-
Open.repository_dirs << File.join(ENV["HOME"],".rbbt/tmp/test/workflow")
|
248
|
-
Misc.with_env "RBBT_UPDATE", "true" do
|
249
|
-
send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
|
250
|
-
send_input_dep_to_reverse_job.clean
|
251
|
-
send_input_dep_to_reverse_job.run
|
252
|
-
|
253
|
-
input_dep_job = send_input_dep_to_reverse_job.step(:input_dep)
|
254
|
-
mtime_orig = Open.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
|
255
|
-
|
256
|
-
sleep 2
|
257
|
-
input_dep_job.clean
|
258
|
-
input_dep_job.run
|
259
|
-
send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
|
260
|
-
|
261
|
-
send_input_dep_to_reverse_job.run
|
262
|
-
mtime_new = Open.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
|
263
|
-
assert mtime_orig < mtime_new
|
264
|
-
end
|
265
|
-
end
|
266
|
-
|
267
|
-
def test_helper
|
268
|
-
assert_equal "User", TestWF.job(:user, "Default", :number => 3).run
|
269
|
-
end
|
270
|
-
|
271
|
-
def test_job
|
272
|
-
str = "TEST"
|
273
|
-
job = TestWF.job(:repeat2, "Default", :number => 3).clean.fork
|
274
|
-
while not job.done?
|
275
|
-
sleep 1
|
276
|
-
end
|
277
|
-
|
278
|
-
raise job.messages.last if job.error?
|
279
|
-
|
280
|
-
assert_equal ["TEST"] * 6 * "\n", job.load
|
281
|
-
end
|
282
|
-
|
283
|
-
def test_with_subdir
|
284
|
-
str = "TEST"
|
285
|
-
job = TestWF.job(:repeat2, "Default", :number => 3).fork
|
286
|
-
while not job.done?
|
287
|
-
sleep 1
|
288
|
-
end
|
289
|
-
|
290
|
-
raise job.messages.last if job.error?
|
291
|
-
|
292
|
-
assert_equal ["TEST"] * 6 * "\n", job.load
|
293
|
-
end
|
294
|
-
|
295
|
-
def test_search
|
296
|
-
str = "TEST"
|
297
|
-
TestWF.jobs(:repeat2).each do |name|
|
298
|
-
TestWF.load_name(:repeat2, name).clean
|
299
|
-
end
|
300
|
-
job1 = TestWF.job(:repeat2, "subdir/Default", :number => 3).clean.fork
|
301
|
-
job2 = TestWF.job(:repeat2, "subdir/Other", :number => 3).clean.fork
|
302
|
-
job3 = TestWF.job(:repeat2, "Default", :number => 3).clean.fork
|
303
|
-
|
304
|
-
while not job1.done? and not job2.done? and not job3.done?
|
305
|
-
sleep 1
|
306
|
-
end
|
307
|
-
|
308
|
-
assert_equal [job1.name, job2.name].sort, TestWF.jobs(:repeat2, "subdir/").sort
|
309
|
-
assert_equal [job1.name].sort, TestWF.jobs(:repeat2, "subdir/Default")
|
310
|
-
assert TestWF.jobs(:repeat2).include?(job1.name)
|
311
|
-
assert TestWF.jobs(:repeat2).include?(job2.name)
|
312
|
-
assert TestWF.jobs(:repeat2).include?(job3.name)
|
313
|
-
assert TestWF.load_name(:repeat2, job3.name).done?
|
314
|
-
assert_equal "TEST\nTEST\nTEST\nTEST\nTEST\nTEST", TestWF.load_name(:repeat2, TestWF.jobs(:repeat2).first).load
|
315
|
-
end
|
316
|
-
|
317
|
-
def test_double_dep
|
318
|
-
assert_equal ["TEST", "TEST\nTEST", "TEST\nTEST\nTEST\nTEST"], TestWF.job(:double_dep, "foo", :times => 2, :number => 2).clean.run
|
319
|
-
end
|
320
|
-
|
321
|
-
def test_object_workflow
|
322
|
-
a = ""
|
323
|
-
a.extend Workflow
|
324
|
-
a.task :foo => :string do
|
325
|
-
"bar"
|
326
|
-
end
|
327
|
-
|
328
|
-
job = a.job(:foo)
|
329
|
-
assert_equal 'bar', job.exec
|
330
|
-
end
|
331
|
-
|
332
|
-
def test_letter
|
333
|
-
assert_equal "D", TestWF.job(:letter).run
|
334
|
-
assert_equal "B", TestWF.job(:letter, nil, :letter => "B").run
|
335
|
-
assert_equal "BB", TestWF.job(:letter_repeat, nil, :letter => "B").run
|
336
|
-
job = TestWF.job(:two_letters, nil, :letter => "V")
|
337
|
-
assert_equal "V:AA", job.run
|
338
|
-
end
|
339
|
-
|
340
|
-
def test_override_dep
|
341
|
-
TmpFile.with_file("OTHER", false) do |file|
|
342
|
-
assert TestWF.job(:repeat2, nil, :number => 3, "TestWF#str" => file).clean.run.include? "OTHER"
|
343
|
-
end
|
344
|
-
end
|
345
|
-
|
346
|
-
def __test_stream
|
347
|
-
io = TestWF.job(:stream).run(:stream)
|
348
|
-
Misc.consume_stream(TSV.get_stream(io), false, STDOUT)
|
349
|
-
nil
|
350
|
-
end
|
351
|
-
|
352
|
-
def __test_fork_stream
|
353
|
-
job = TestWF.job(:stream)
|
354
|
-
job.clean
|
355
|
-
io = job.fork(:stream)
|
356
|
-
Misc.consume_stream(TSV.get_stream(io), false, STDOUT)
|
357
|
-
nil
|
358
|
-
end
|
359
|
-
|
360
|
-
def test_stream_order
|
361
|
-
|
362
|
-
job = TestWF.job(:stream2)
|
363
|
-
job.recursive_clean
|
364
|
-
job.produce
|
365
|
-
|
366
|
-
end
|
367
|
-
|
368
|
-
def test_rec_input_use
|
369
|
-
assert TestWF.rec_input_use(:double_dep).include?(:times)
|
370
|
-
assert TestWF.rec_input_use(:double_dep)[:times].include?(TestWF)
|
371
|
-
assert TestWF.rec_input_use(:double_dep)[:times][TestWF].include?(:repeat)
|
372
|
-
end
|
373
|
-
|
374
|
-
def test_shared_inputs
|
375
|
-
assert_equal "CB", TestWF.job(:t3).run
|
376
|
-
assert_equal "CB", TestWF.job(:t3).run
|
377
|
-
end
|
378
|
-
|
379
|
-
def test_transplant
|
380
|
-
listed = '/home/user/.rbbt/var/jobs/TestWF/task1/Default'
|
381
|
-
real = '/usr/local/var/rbbt/jobs/TestWF/task1/Default'
|
382
|
-
other = '/home/user/.rbbt/var/jobs/TestWF/task2/Default'
|
383
|
-
real_other = '/usr/local/var/rbbt/jobs/TestWF/task2/Default'
|
384
|
-
|
385
|
-
assert_equal real_other, Workflow.transplant(listed, real, other)
|
386
|
-
assert_equal real_other, Workflow.transplant(nil, real, other)
|
387
|
-
end
|
388
|
-
|
389
|
-
def test_relocate
|
390
|
-
TmpFile.with_file do |tmpdir|
|
391
|
-
listed = File.join(tmpdir, '/home/user/.rbbt/var/jobs/TestWF/task1/Default')
|
392
|
-
real = File.join(tmpdir, '/usr/local/var/rbbt/jobs/TestWF/task1/Default')
|
393
|
-
other = File.join(tmpdir, '/home/user/.rbbt/var/jobs/TestWF/task2/Default')
|
394
|
-
real_other = File.join(tmpdir, '/usr/local/var/rbbt/jobs/TestWF/task2/Default')
|
395
|
-
|
396
|
-
Open.write(real_other,'')
|
397
|
-
assert_equal real_other, Workflow.relocate(real, other)
|
398
|
-
assert_equal real_other, Workflow.relocate(real, other)
|
399
|
-
end
|
400
|
-
end
|
401
|
-
|
402
|
-
def test_relocate_alt
|
403
|
-
TmpFile.with_file do |tmpdir|
|
404
|
-
listed = File.join(tmpdir, '/scratch/tmp/rbbt/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC')
|
405
|
-
real = File.join(tmpdir, '/home/bsc26/bsc26892/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC')
|
406
|
-
other = File.join(tmpdir, '/scratch/tmp/rbbt/scratch/bsc26892/rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG')
|
407
|
-
real_other = File.join(tmpdir, '/home/bsc26/bsc26892/.rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG')
|
408
|
-
Open.write(real_other,'')
|
409
|
-
|
410
|
-
assert_equal real_other, Workflow.relocate(real, other)
|
411
|
-
end
|
412
|
-
end
|
413
|
-
|
414
|
-
def test_delete_dep
|
415
|
-
job = TestWF.job(:t3).recursive_clean
|
416
|
-
job.run
|
417
|
-
Misc.with_env "RBBT_UPDATE", 'true' do
|
418
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
419
|
-
#job = TestWF.job(:t3)
|
420
|
-
#job.step(:t1).clean
|
421
|
-
#assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
422
|
-
job = TestWF.job(:t3).recursive_clean
|
423
|
-
job.run
|
424
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
425
|
-
job = TestWF.job(:t3)
|
426
|
-
sleep 1
|
427
|
-
Open.touch job.step(:t1).path
|
428
|
-
Misc.with_env "RBBT_UPDATE", "false" do
|
429
|
-
assert job.updated?
|
430
|
-
end
|
431
|
-
Misc.with_env "RBBT_UPDATE", "true" do
|
432
|
-
assert ! job.updated?
|
433
|
-
end
|
434
|
-
end
|
435
|
-
end
|
436
|
-
|
437
|
-
def test_canfail
|
438
|
-
job = TestWF.job(:sum_odds)
|
439
|
-
assert_equal 24, job.run
|
440
|
-
|
441
|
-
job = TestWF.job(:sum_odds_str)
|
442
|
-
job.recursive_clean
|
443
|
-
assert_equal "Sum odds: 24", job.run
|
444
|
-
end
|
445
|
-
|
446
|
-
def test_save_inputs
|
447
|
-
TmpFile.with_file("Hi") do |file|
|
448
|
-
job = TestWF.job(:reverse_file, nil, :file => file)
|
449
|
-
TmpFile.with_file do |dir|
|
450
|
-
Path.setup(dir)
|
451
|
-
Step.save_job_inputs(job, dir)
|
452
|
-
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.as_path']
|
453
|
-
end
|
454
|
-
end
|
455
|
-
|
456
|
-
TmpFile.with_file("code") do |tmpfile|
|
457
|
-
job_orig = TestWF.job(:reverse_file, nil, :file => tmpfile)
|
458
|
-
TmpFile.with_file do |dir|
|
459
|
-
Path.setup(dir)
|
460
|
-
Step.save_job_inputs(job_orig, dir)
|
461
|
-
assert_equal [dir.file.find + '.as_path'], Dir.glob(dir + "/*")
|
462
|
-
inputs = Workflow.load_inputs(dir, [:file], :file => :file)
|
463
|
-
job = TestWF.job(:reverse_file, nil, inputs)
|
464
|
-
assert job_orig.path, job.path
|
465
|
-
end
|
466
|
-
end
|
467
|
-
|
468
|
-
end
|
469
|
-
|
470
|
-
def test_archive
|
471
|
-
job = TmpFile.with_file("Hi") do |file|
|
472
|
-
job = TestWF.job(:reverse_file, nil, :file => file)
|
473
|
-
job.run
|
474
|
-
job
|
475
|
-
end
|
476
|
-
TmpFile.with_file nil, true, :extension => 'tar.gz' do |targz|
|
477
|
-
Step.archive([job], targz)
|
478
|
-
TmpFile.with_file do |dir|
|
479
|
-
dir = Path.setup(dir)
|
480
|
-
Misc.untar targz, dir
|
481
|
-
assert dir.glob("**/*").collect{|f| File.basename(f)}.include? job.name
|
482
|
-
end
|
483
|
-
end
|
484
|
-
end
|
485
|
-
|
486
|
-
def test_input_step_file_check
|
487
|
-
job = TestWF.job(:t3).recursive_clean
|
488
|
-
job.run
|
489
|
-
Misc.with_env "RBBT_UPDATE", 'true' do
|
490
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
491
|
-
#job = TestWF.job(:t3)
|
492
|
-
#job.step(:t1).clean
|
493
|
-
#assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
494
|
-
job = TestWF.job(:t3).recursive_clean
|
495
|
-
job.run
|
496
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
497
|
-
job = TestWF.job(:t3)
|
498
|
-
sleep 1
|
499
|
-
Open.touch job.step(:t1).path
|
500
|
-
Misc.with_env "RBBT_UPDATE", "false" do
|
501
|
-
assert job.updated?
|
502
|
-
end
|
503
|
-
Misc.with_env "RBBT_UPDATE", "true" do
|
504
|
-
assert ! job.updated?
|
505
|
-
end
|
506
|
-
end
|
507
|
-
end
|
508
|
-
|
509
|
-
def test_overriden
|
510
|
-
job = TestWF.job(:overr_action)
|
511
|
-
job.recursive_clean
|
512
|
-
assert_equal "TLA", job.run
|
513
|
-
|
514
|
-
assert Symbol === job.step(:overr_orig).overriden
|
515
|
-
assert TrueClass === job.step(:overr_target).overriden
|
516
|
-
assert job.overriden
|
517
|
-
|
518
|
-
job = TestWF.job(:not_overr_action)
|
519
|
-
assert ! job.overriden
|
520
|
-
assert_equal "TLA", job.run
|
521
|
-
assert ! job.step(:overr_target).overriden
|
522
|
-
end
|
523
|
-
|
524
|
-
def test_anonymous_workflow
|
525
|
-
workflow = Module.new
|
526
|
-
workflow.extend Workflow
|
527
|
-
workflow.define_singleton_method(:to_s){"TestWorkflow"}
|
528
|
-
|
529
|
-
assert_equal "TestWorkflow", workflow.to_s
|
530
|
-
|
531
|
-
t1 = workflow.task :s1 => :string do
|
532
|
-
task_name.to_s
|
533
|
-
end
|
534
|
-
|
535
|
-
workflow.dep :s1
|
536
|
-
t2 = workflow.task :s2 => :string do
|
537
|
-
task_name.to_s
|
538
|
-
end
|
539
|
-
|
540
|
-
workflow.dep :s2
|
541
|
-
t3 = workflow.task :s3 => :string do
|
542
|
-
task_name.to_s
|
543
|
-
end
|
544
|
-
|
545
|
-
assert_equal "s3", workflow.job(:s3).run
|
546
|
-
end
|
547
|
-
|
548
|
-
def test_low_level_step
|
549
|
-
TmpFile.with_file do |tmpdir|
|
550
|
-
Path.setup tmpdir
|
551
|
-
|
552
|
-
s1 = Step.new tmpdir.s1 do "s1" end
|
553
|
-
s1.task_name = "s1"
|
554
|
-
|
555
|
-
# run clean run
|
556
|
-
assert_equal "s1", s1.run
|
557
|
-
s1.clean
|
558
|
-
assert_equal "s1", s1.run
|
559
|
-
assert_equal "s1", s1.run
|
560
|
-
|
561
|
-
# is persisted because it raises RbbtException
|
562
|
-
s1.task = proc do raise RbbtException end
|
563
|
-
assert_equal "s1", s1.run
|
564
|
-
s1.clean
|
565
|
-
assert_raises RbbtException do s1.run end
|
566
|
-
assert RbbtException === s1.get_exception
|
567
|
-
assert s1.error?
|
568
|
-
|
569
|
-
s1.recursive_clean
|
570
|
-
s1.task = proc do "s1" end
|
571
|
-
|
572
|
-
# add dependencies
|
573
|
-
s2 = Step.new tmpdir.s2 do
|
574
|
-
step(:s1).load + " -> s2"
|
575
|
-
end
|
576
|
-
s2.task_name = "s2"
|
577
|
-
s2.dependencies << s1
|
578
|
-
|
579
|
-
s3 = Step.new tmpdir.s3 do
|
580
|
-
step(:s2).load + " -> s3"
|
581
|
-
end
|
582
|
-
s3.task_name = "s3"
|
583
|
-
s3.dependencies << s2
|
584
|
-
|
585
|
-
assert_equal "s1 -> s2", s2.run
|
586
|
-
assert_equal "s1 -> s2 -> s3", s3.run
|
587
|
-
|
588
|
-
# Test recusive behaviour
|
589
|
-
s1.task = proc do raise RbbtException end
|
590
|
-
ComputeDependency.setup(s1, :produce)
|
591
|
-
s1.clean
|
592
|
-
s3.clean
|
593
|
-
s3.run
|
594
|
-
assert_equal "s1 -> s2 -> s3", s3.run
|
595
|
-
|
596
|
-
s1.task = proc do raise RbbtException end
|
597
|
-
ComputeDependency.setup(s1, :produce)
|
598
|
-
s3.recursive_clean
|
599
|
-
assert_raises RbbtException do s3.run end
|
600
|
-
|
601
|
-
s1.task = proc{|v| v }
|
602
|
-
s1.inputs = ["test"]
|
603
|
-
assert_equal "test", s1.run
|
604
|
-
s3.recursive_clean
|
605
|
-
assert_equal "test -> s2 -> s3", s3.run
|
606
|
-
end
|
607
|
-
end
|
608
|
-
|
609
|
-
end
|
@@ -1,40 +0,0 @@
|
|
1
|
-
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
-
require 'rbbt/tsv'
|
3
|
-
require 'rbbt/tsv/parallel'
|
4
|
-
|
5
|
-
class TestTSVParallelThrough < Test::Unit::TestCase
|
6
|
-
|
7
|
-
def test_pthrough
|
8
|
-
tsv = datafile_test('identifiers').tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
|
9
|
-
|
10
|
-
h = {}
|
11
|
-
tsv.monitor = true
|
12
|
-
tsv.unnamed = true
|
13
|
-
tsv.pthrough do |k,v|
|
14
|
-
h[k] = v.first
|
15
|
-
end
|
16
|
-
|
17
|
-
assert_equal tsv.size, h.size
|
18
|
-
assert_equal tsv.keys.sort, h.keys.sort
|
19
|
-
end
|
20
|
-
|
21
|
-
|
22
|
-
def test_ppthrough
|
23
|
-
tsv = datafile_test('identifiers').tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
|
24
|
-
|
25
|
-
h = {}
|
26
|
-
|
27
|
-
tsv.ppthrough_callback do |k,v|
|
28
|
-
h[k] = v
|
29
|
-
end
|
30
|
-
|
31
|
-
tsv.unnamed = true
|
32
|
-
tsv.monitor = true
|
33
|
-
tsv.ppthrough(3) do |k,v|
|
34
|
-
[k,v.first]
|
35
|
-
end
|
36
|
-
|
37
|
-
assert_equal tsv.size, h.size
|
38
|
-
assert_equal tsv.keys.sort, h.keys.sort
|
39
|
-
end
|
40
|
-
end
|