rbbt-util 5.43.0 → 5.44.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/association/util.rb +1 -1
  3. data/lib/rbbt/hpc/batch.rb +19 -17
  4. data/lib/rbbt/tsv/accessor.rb +6 -11
  5. data/lib/rbbt/tsv/dumper.rb +21 -10
  6. data/lib/rbbt/tsv/index.rb +2 -1
  7. data/lib/rbbt/util/misc/math.rb +0 -1
  8. data/lib/rbbt/util/open.rb +1 -1
  9. data/lib/rbbt/workflow/step/info.rb +1 -0
  10. data/lib/rbbt/workflow/step.rb +2 -1
  11. data/python/rbbt/__init__.py +2 -2
  12. data/share/rbbt_commands/workflow/task +1 -1
  13. metadata +4 -220
  14. data/test/rbbt/annotations/test_util.rb +0 -43
  15. data/test/rbbt/association/test_database.rb +0 -87
  16. data/test/rbbt/association/test_index.rb +0 -127
  17. data/test/rbbt/association/test_item.rb +0 -15
  18. data/test/rbbt/association/test_open.rb +0 -63
  19. data/test/rbbt/association/test_util.rb +0 -108
  20. data/test/rbbt/entity/test_identifiers.rb +0 -34
  21. data/test/rbbt/hpc/orchestrate/test_batches.rb +0 -70
  22. data/test/rbbt/hpc/orchestrate/test_chains.rb +0 -108
  23. data/test/rbbt/hpc/orchestrate/test_rules.rb +0 -59
  24. data/test/rbbt/hpc/test_batch.rb +0 -64
  25. data/test/rbbt/hpc/test_hpc_test_workflows.rb +0 -0
  26. data/test/rbbt/hpc/test_orchestrate.rb +0 -144
  27. data/test/rbbt/hpc/test_pbs.rb +0 -43
  28. data/test/rbbt/hpc/test_slurm.rb +0 -28
  29. data/test/rbbt/knowledge_base/test_enrichment.rb +0 -50
  30. data/test/rbbt/knowledge_base/test_entity.rb +0 -62
  31. data/test/rbbt/knowledge_base/test_query.rb +0 -46
  32. data/test/rbbt/knowledge_base/test_registry.rb +0 -74
  33. data/test/rbbt/knowledge_base/test_syndicate.rb +0 -48
  34. data/test/rbbt/knowledge_base/test_traverse.rb +0 -133
  35. data/test/rbbt/persist/test_tsv.rb +0 -88
  36. data/test/rbbt/persist/tsv/test_cdb.rb +0 -18
  37. data/test/rbbt/persist/tsv/test_kyotocabinet.rb +0 -27
  38. data/test/rbbt/persist/tsv/test_leveldb.rb +0 -18
  39. data/test/rbbt/persist/tsv/test_lmdb.rb +0 -20
  40. data/test/rbbt/persist/tsv/test_sharder.rb +0 -164
  41. data/test/rbbt/persist/tsv/test_tokyocabinet.rb +0 -262
  42. data/test/rbbt/resource/test_path.rb +0 -49
  43. data/test/rbbt/test_annotations.rb +0 -167
  44. data/test/rbbt/test_association.rb +0 -103
  45. data/test/rbbt/test_entity.rb +0 -252
  46. data/test/rbbt/test_fix_width_table.rb +0 -135
  47. data/test/rbbt/test_knowledge_base.rb +0 -226
  48. data/test/rbbt/test_monitor.rb +0 -11
  49. data/test/rbbt/test_packed_index.rb +0 -68
  50. data/test/rbbt/test_persist.rb +0 -85
  51. data/test/rbbt/test_resource.rb +0 -110
  52. data/test/rbbt/test_tsv.rb +0 -669
  53. data/test/rbbt/test_workflow.rb +0 -609
  54. data/test/rbbt/tsv/parallel/test_through.rb +0 -40
  55. data/test/rbbt/tsv/parallel/test_traverse.rb +0 -456
  56. data/test/rbbt/tsv/test_accessor.rb +0 -319
  57. data/test/rbbt/tsv/test_attach.rb +0 -715
  58. data/test/rbbt/tsv/test_change_id.rb +0 -61
  59. data/test/rbbt/tsv/test_csv.rb +0 -49
  60. data/test/rbbt/tsv/test_excel.rb +0 -171
  61. data/test/rbbt/tsv/test_field_index.rb +0 -19
  62. data/test/rbbt/tsv/test_filter.rb +0 -187
  63. data/test/rbbt/tsv/test_index.rb +0 -308
  64. data/test/rbbt/tsv/test_manipulate.rb +0 -334
  65. data/test/rbbt/tsv/test_marshal.rb +0 -24
  66. data/test/rbbt/tsv/test_matrix.rb +0 -20
  67. data/test/rbbt/tsv/test_parallel.rb +0 -7
  68. data/test/rbbt/tsv/test_parser.rb +0 -101
  69. data/test/rbbt/tsv/test_stream.rb +0 -253
  70. data/test/rbbt/tsv/test_util.rb +0 -52
  71. data/test/rbbt/util/R/test_eval.rb +0 -43
  72. data/test/rbbt/util/R/test_model.rb +0 -128
  73. data/test/rbbt/util/R/test_plot.rb +0 -38
  74. data/test/rbbt/util/concurrency/processes/test_socket.rb +0 -70
  75. data/test/rbbt/util/concurrency/test_processes.rb +0 -192
  76. data/test/rbbt/util/concurrency/test_threads.rb +0 -40
  77. data/test/rbbt/util/log/test_progress.rb +0 -111
  78. data/test/rbbt/util/misc/test_bgzf.rb +0 -48
  79. data/test/rbbt/util/misc/test_communication.rb +0 -13
  80. data/test/rbbt/util/misc/test_development.rb +0 -26
  81. data/test/rbbt/util/misc/test_format.rb +0 -10
  82. data/test/rbbt/util/misc/test_indiferent_hash.rb +0 -14
  83. data/test/rbbt/util/misc/test_lock.rb +0 -77
  84. data/test/rbbt/util/misc/test_multipart_payload.rb +0 -202
  85. data/test/rbbt/util/misc/test_omics.rb +0 -116
  86. data/test/rbbt/util/misc/test_pipes.rb +0 -343
  87. data/test/rbbt/util/misc/test_serialize.rb +0 -24
  88. data/test/rbbt/util/python/test_util.rb +0 -25
  89. data/test/rbbt/util/simpleopt/test_get.rb +0 -12
  90. data/test/rbbt/util/simpleopt/test_parse.rb +0 -10
  91. data/test/rbbt/util/simpleopt/test_setup.rb +0 -76
  92. data/test/rbbt/util/test_R.rb +0 -37
  93. data/test/rbbt/util/test_chain_methods.rb +0 -22
  94. data/test/rbbt/util/test_cmd.rb +0 -87
  95. data/test/rbbt/util/test_colorize.rb +0 -22
  96. data/test/rbbt/util/test_concurrency.rb +0 -6
  97. data/test/rbbt/util/test_config.rb +0 -69
  98. data/test/rbbt/util/test_excel2tsv.rb +0 -10
  99. data/test/rbbt/util/test_filecache.rb +0 -36
  100. data/test/rbbt/util/test_log.rb +0 -52
  101. data/test/rbbt/util/test_migrate.rb +0 -34
  102. data/test/rbbt/util/test_misc.rb +0 -728
  103. data/test/rbbt/util/test_open.rb +0 -200
  104. data/test/rbbt/util/test_procpath.rb +0 -23
  105. data/test/rbbt/util/test_python.rb +0 -144
  106. data/test/rbbt/util/test_semaphore.rb +0 -36
  107. data/test/rbbt/util/test_simpleDSL.rb +0 -55
  108. data/test/rbbt/util/test_simpleopt.rb +0 -11
  109. data/test/rbbt/util/test_ssh.rb +0 -10
  110. data/test/rbbt/util/test_tmpfile.rb +0 -32
  111. data/test/rbbt/workflow/step/test_dependencies.rb +0 -295
  112. data/test/rbbt/workflow/step/test_save_load_inputs.rb +0 -136
  113. data/test/rbbt/workflow/test_doc.rb +0 -30
  114. data/test/rbbt/workflow/test_remote_workflow.rb +0 -99
  115. data/test/rbbt/workflow/test_schedule.rb +0 -0
  116. data/test/rbbt/workflow/test_step.rb +0 -231
  117. data/test/rbbt/workflow/test_task.rb +0 -85
  118. data/test/rbbt/workflow/util/test_archive.rb +0 -27
  119. data/test/rbbt/workflow/util/test_data.rb +0 -67
  120. data/test/rbbt/workflow/util/test_orchestrator.rb +0 -263
  121. data/test/test_helper.rb +0 -114
@@ -1,456 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/tsv'
3
- require 'rbbt/tsv/parallel'
4
-
5
- class StopException < StandardError; end
6
-
7
- class TestTSVParallelThrough < Test::Unit::TestCase
8
-
9
- def test_traverse_tsv
10
- require 'rbbt/sources/organism'
11
-
12
- head = 100
13
-
14
- tsv = datafile_test('identifiers').tsv :head => head
15
- res = {}
16
- TSV.traverse tsv do |k,v|
17
- res[k] = v
18
- end
19
- assert_equal head, res.keys.compact.sort.length
20
-
21
- tsv = datafile_test('identifiers').tsv :head => head
22
- TSV.traverse tsv, :into => res do |k,v|
23
- [k,v]
24
- end
25
- assert_equal head, res.keys.compact.sort.length
26
- end
27
-
28
- def test_traverse_tsv_cpus
29
- require 'rbbt/sources/organism'
30
-
31
- head = 100
32
-
33
- tsv = datafile_test('identifiers').tsv :head => head
34
- res = {}
35
- TSV.traverse tsv do |k,v|
36
- res[k] = v
37
- end
38
- assert_equal head, res.keys.compact.sort.length
39
- assert res.values.compact.flatten.uniq.length > 0
40
-
41
- tsv = datafile_test('identifiers').tsv :head => head
42
- TSV.traverse tsv, :into => res, :cpus => 5 do |k,v|
43
- [k,v]
44
- end
45
-
46
- assert_equal head, res.keys.compact.sort.length
47
- assert res.values.compact.flatten.uniq.length > 0
48
- end
49
-
50
- def test_traverse_stream
51
- require 'rbbt/sources/organism'
52
-
53
- head = 3000
54
-
55
- tsv = datafile_test('identifiers').open
56
- res = {}
57
- TSV.traverse tsv, :head => head, :into => res, :bar => true do |k,v|
58
- [k,v]
59
- end
60
-
61
- assert_equal head, res.keys.compact.sort.length
62
- end
63
-
64
- def test_traverse_stream_cpus
65
- require 'rbbt/sources/organism'
66
-
67
- head = 1000
68
-
69
- tsv = datafile_test('identifiers')
70
- res = {}
71
- TSV.traverse tsv, :head => head, :cpus => 5, :into => res do |k,v|
72
- [k,v]
73
- end
74
-
75
- assert_equal head, res.keys.compact.sort.length
76
- end
77
-
78
- def test_traverse_stream_keys
79
- require 'rbbt/sources/organism'
80
-
81
- head = 1000
82
-
83
- tsv = datafile_test('identifiers').open
84
- res = []
85
-
86
- TSV.traverse tsv, :head => head, :type => :keys do |v|
87
- res << v
88
- end
89
-
90
- assert_equal res, datafile_test('identifiers').tsv(:head => head).keys
91
-
92
- tsv = datafile_test('identifiers').open
93
- res = []
94
-
95
- TSV.traverse tsv, :head => head, :type => :keys, :into => res do |v|
96
- v
97
- end
98
-
99
- assert_equal res.sort, datafile_test('identifiers').tsv(:head => head).keys.sort
100
- end
101
-
102
- def test_traverse_array
103
- require 'rbbt/sources/organism'
104
-
105
- array = []
106
- 100.times do array << rand(10).to_i end
107
-
108
- res = []
109
- TSV.traverse array do |v|
110
- res << v
111
- end
112
-
113
- assert_equal array, res
114
-
115
-
116
- res = []
117
- TSV.traverse array, :into => res do |v|
118
- v
119
- end
120
-
121
- assert_equal array, res
122
- end
123
-
124
- def test_traverse_priority
125
- require 'fc'
126
-
127
- queue = FastContainers::PriorityQueue.new(:min)
128
-
129
- array = []
130
- 100.times do e = rand(1000).to_i; array << e; queue.push(e,e) end
131
-
132
- res = TSV.traverse queue, :into => [] do |v|
133
- v
134
- end
135
-
136
- assert_equal array.sort, res
137
- end
138
-
139
- def test_traverse_array_threads
140
- require 'rbbt/sources/organism'
141
-
142
- array = []
143
- 100.times do array << rand(10).to_i end
144
-
145
- res = []
146
- TSV.traverse array, :threads => 5 do |v|
147
- res << v
148
- end
149
-
150
- assert_equal array.sort, res.sort
151
-
152
- res = []
153
- TSV.traverse array, :threads => 5, :into => res do |v|
154
- v
155
- end
156
-
157
- assert_equal array.sort, res.sort
158
- end
159
-
160
- def test_traverse_array_cpus
161
- require 'rbbt/sources/organism'
162
-
163
- array = []
164
- 100.times do array << rand(10).to_i end
165
-
166
- res = []
167
-
168
- TSV.traverse array, :cpus => 5, :into => res do |v|
169
- v
170
- end
171
-
172
- assert_equal array.sort, res.sort
173
- end
174
-
175
- def test_traverse_benchmark
176
- require 'rbbt/sources/organism'
177
-
178
- head = 5_000
179
-
180
- tsv = datafile_test('identifiers').open
181
- Misc.benchmark do
182
- res = {}
183
- TSV.traverse tsv, :head => head do |k,v|
184
- res[k] = v
185
- end
186
- end
187
-
188
- tsv = datafile_test('identifiers').open
189
- Misc.benchmark do
190
- res = {}
191
- TSV.traverse tsv, :head => head, :cpus => 5, :into => res do |k,v|
192
- [k,v]
193
- end
194
- end
195
- end
196
-
197
- def test_traverse_into_dumper
198
- require 'rbbt/sources/organism'
199
-
200
- head = 2_000
201
-
202
- stream = datafile_test('identifiers').open
203
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
204
- dumper.init
205
- TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
206
- k = k.first
207
- [k,v]
208
- end
209
-
210
- res = TSV.open(dumper.stream)
211
-
212
- assert_equal head, res.size
213
- end
214
-
215
- def test_traverse_into_dumper_error_bar
216
- require 'rbbt/sources/organism'
217
-
218
- head = 2_000
219
-
220
- stream = datafile_test('identifiers').open
221
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
222
- dumper.init
223
- TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
224
- k = k.first
225
- raise
226
- [k,v]
227
- end
228
-
229
- assert_raise do
230
- res = TSV.open(dumper.stream)
231
- end
232
-
233
- end
234
-
235
- def test_traverse_into_dumper_threads
236
- require 'rbbt/sources/organism'
237
-
238
- head = 2_000
239
- threads = 10
240
-
241
- stream = datafile_test('identifiers').open
242
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
243
- dumper.init
244
-
245
- TSV.traverse stream, :threads => threads, :head => head, :into => dumper do |k,v|
246
- k = k.first
247
- [k,v]
248
- end
249
-
250
- res = TSV.open(StringIO.new(dumper.stream.read))
251
-
252
- assert_equal head, res.size
253
- end
254
-
255
- def test_traverse_into_dumper_cpus
256
- require 'rbbt/sources/organism'
257
-
258
- head = 2_000
259
- cpus = 10
260
-
261
- stream = datafile_test('identifiers').open
262
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
263
- dumper.init
264
- TSV.traverse stream, :cpus => cpus, :head => head, :into => dumper do |k,v|
265
- k = k.first
266
- [k,v]
267
- end
268
-
269
- res = TSV.open(dumper.stream)
270
-
271
- assert_equal head, res.size
272
- end
273
-
274
- #{{{ TRAVERSE DUMPER
275
-
276
- def test_traverse_dumper
277
- require 'rbbt/sources/organism'
278
-
279
- head = 2_000
280
-
281
- tsv = TSV::Parser.new datafile_test('identifiers').open, :head => head
282
- dumper = TSV::Dumper.new tsv.options
283
-
284
- TSV.traverse tsv, :head => head, :into => dumper do |k,v|
285
- k = k.first
286
- [k,v]
287
- end
288
-
289
- res = {}
290
- TSV.traverse dumper.stream, :into => res do |k,v|
291
- [k, v.length]
292
- end
293
-
294
- assert_equal head, res.size
295
- end
296
-
297
- def test_traverse_dumper_threads
298
- require 'rbbt/sources/organism'
299
-
300
- head = 2_000
301
- threads = 3
302
-
303
- tsv = TSV::Parser.new datafile_test('identifiers').open, :head => head
304
-
305
- dumper = TSV::Dumper.new tsv.options
306
- dumper.init
307
-
308
- TSV.traverse tsv, :head => head, :threads => threads, :into => dumper do |k,v|
309
- k = k.first
310
- [k,v]
311
- end
312
-
313
- res = {}
314
- TSV.traverse dumper.stream, :threads => threads, :into => res do |k,v|
315
- [k, v.length]
316
- end
317
-
318
- assert_equal head, res.size
319
- end
320
-
321
- def test_traverse_dumper_cpus
322
- require 'rbbt/sources/organism'
323
-
324
- head = 5_000
325
- cpus = 4
326
-
327
- stream = datafile_test('identifiers').open
328
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
329
- dumper.init
330
-
331
- TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
332
- k = k.first
333
- [k,v]
334
- end
335
-
336
- res = {}
337
- TSV.traverse dumper.stream, :cpus => cpus, :into => res do |k,v|
338
- [k, v.length]
339
- end
340
-
341
-
342
- assert_equal head, res.size
343
- end
344
-
345
- def test_traverse_dumper_exception
346
- require 'rbbt/sources/organism'
347
-
348
- head = 2_000
349
-
350
- Log.info Log.color :red, "TRAVERSE EXCEPTION"
351
- stream = datafile_test('identifiers').open
352
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
353
- dumper.init
354
-
355
- assert_raise StopException do
356
- TSV.traverse stream, :head => head, :into => dumper do |k,v|
357
- k = k.first
358
- raise StopException if rand(100) < 20
359
- [k,v]
360
- end
361
- dumper.stream.join
362
- end
363
- end
364
-
365
- def test_traverse_dumper_cpus_exception
366
- require 'rbbt/sources/organism'
367
-
368
- head = 2_000
369
- cpus = 2
370
-
371
- Log.info Log.color :red, "TRAVERSE EXCEPTION"
372
- stream = datafile_test('identifiers').open
373
- dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
374
- dumper.init
375
-
376
- assert_raise ProcessFailed do
377
- TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
378
- k = k.first
379
- raise ProcessFailed if rand(100) < 20
380
- [k,v]
381
- end
382
- dumper.stream.join
383
- end
384
- end
385
-
386
- def test_traverse_into_stream
387
- size = 100
388
- array = (1..size).to_a.collect{|n| n.to_s}
389
- stream = TSV.traverse array, :into => :stream do |e|
390
- e
391
- end
392
- assert_equal size, stream.read.split("\n").length
393
- end
394
-
395
- def test_traverse_into_path
396
- size = 100
397
- array = (1..size).to_a.collect{|n| n.to_s}
398
- TmpFile.with_file do |tmpfile|
399
- Path.setup(tmpfile)
400
- io = TSV.traverse array, :into => tmpfile do |e|
401
- e
402
- end
403
- io.join
404
- assert_equal size, Open.read(tmpfile).split("\n").length
405
- end
406
- end
407
-
408
-
409
- def test_traverse_progress
410
- size = 1000
411
- array = (1..size).to_a.collect{|n| n.to_s}
412
- stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
413
- sleep 0.001
414
- e
415
- end
416
- assert_equal size, stream.read.split("\n").length
417
-
418
- size = 1000
419
- array = (1..size).to_a.collect{|n| n.to_s}
420
- stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
421
- sleep 0.001
422
- e
423
- end
424
- assert_equal size, stream.read.split("\n").length
425
-
426
- size = 1000
427
- array = (1..size).to_a.collect{|n| n.to_s}
428
- stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
429
- sleep 0.001
430
- e
431
- end
432
- assert_equal size, stream.read.split("\n").length
433
-
434
- size = 1000
435
- array = (1..size).to_a.collect{|n| n.to_s}
436
- stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
437
- sleep 0.01
438
- e
439
- end
440
- assert_equal size, stream.read.split("\n").length
441
- end
442
-
443
- def test_store_multiple
444
- size = 1000
445
- array = (1..size).to_a.collect{|n| n.to_s}
446
-
447
- stream = TSV.traverse array, :bar => {:max => size, :desc => "Multiple into stream"}, :cpus => 2, :into => :stream do |e|
448
- sleep 0.001
449
- res = [e,e+".alt"]
450
- res.extend MultipleResult
451
- res
452
- end
453
-
454
- assert_equal size*2, stream.read.split("\n").length
455
- end
456
- end