rbbt-util 5.43.0 → 5.44.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (121) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/association/util.rb +1 -1
  3. data/lib/rbbt/hpc/batch.rb +19 -17
  4. data/lib/rbbt/tsv/accessor.rb +6 -11
  5. data/lib/rbbt/tsv/dumper.rb +21 -10
  6. data/lib/rbbt/tsv/index.rb +2 -1
  7. data/lib/rbbt/util/misc/math.rb +0 -1
  8. data/lib/rbbt/util/open.rb +1 -1
  9. data/lib/rbbt/workflow/step/info.rb +1 -0
  10. data/lib/rbbt/workflow/step.rb +2 -1
  11. data/python/rbbt/__init__.py +2 -2
  12. data/share/rbbt_commands/workflow/task +1 -1
  13. metadata +4 -220
  14. data/test/rbbt/annotations/test_util.rb +0 -43
  15. data/test/rbbt/association/test_database.rb +0 -87
  16. data/test/rbbt/association/test_index.rb +0 -127
  17. data/test/rbbt/association/test_item.rb +0 -15
  18. data/test/rbbt/association/test_open.rb +0 -63
  19. data/test/rbbt/association/test_util.rb +0 -108
  20. data/test/rbbt/entity/test_identifiers.rb +0 -34
  21. data/test/rbbt/hpc/orchestrate/test_batches.rb +0 -70
  22. data/test/rbbt/hpc/orchestrate/test_chains.rb +0 -108
  23. data/test/rbbt/hpc/orchestrate/test_rules.rb +0 -59
  24. data/test/rbbt/hpc/test_batch.rb +0 -64
  25. data/test/rbbt/hpc/test_hpc_test_workflows.rb +0 -0
  26. data/test/rbbt/hpc/test_orchestrate.rb +0 -144
  27. data/test/rbbt/hpc/test_pbs.rb +0 -43
  28. data/test/rbbt/hpc/test_slurm.rb +0 -28
  29. data/test/rbbt/knowledge_base/test_enrichment.rb +0 -50
  30. data/test/rbbt/knowledge_base/test_entity.rb +0 -62
  31. data/test/rbbt/knowledge_base/test_query.rb +0 -46
  32. data/test/rbbt/knowledge_base/test_registry.rb +0 -74
  33. data/test/rbbt/knowledge_base/test_syndicate.rb +0 -48
  34. data/test/rbbt/knowledge_base/test_traverse.rb +0 -133
  35. data/test/rbbt/persist/test_tsv.rb +0 -88
  36. data/test/rbbt/persist/tsv/test_cdb.rb +0 -18
  37. data/test/rbbt/persist/tsv/test_kyotocabinet.rb +0 -27
  38. data/test/rbbt/persist/tsv/test_leveldb.rb +0 -18
  39. data/test/rbbt/persist/tsv/test_lmdb.rb +0 -20
  40. data/test/rbbt/persist/tsv/test_sharder.rb +0 -164
  41. data/test/rbbt/persist/tsv/test_tokyocabinet.rb +0 -262
  42. data/test/rbbt/resource/test_path.rb +0 -49
  43. data/test/rbbt/test_annotations.rb +0 -167
  44. data/test/rbbt/test_association.rb +0 -103
  45. data/test/rbbt/test_entity.rb +0 -252
  46. data/test/rbbt/test_fix_width_table.rb +0 -135
  47. data/test/rbbt/test_knowledge_base.rb +0 -226
  48. data/test/rbbt/test_monitor.rb +0 -11
  49. data/test/rbbt/test_packed_index.rb +0 -68
  50. data/test/rbbt/test_persist.rb +0 -85
  51. data/test/rbbt/test_resource.rb +0 -110
  52. data/test/rbbt/test_tsv.rb +0 -669
  53. data/test/rbbt/test_workflow.rb +0 -609
  54. data/test/rbbt/tsv/parallel/test_through.rb +0 -40
  55. data/test/rbbt/tsv/parallel/test_traverse.rb +0 -456
  56. data/test/rbbt/tsv/test_accessor.rb +0 -319
  57. data/test/rbbt/tsv/test_attach.rb +0 -715
  58. data/test/rbbt/tsv/test_change_id.rb +0 -61
  59. data/test/rbbt/tsv/test_csv.rb +0 -49
  60. data/test/rbbt/tsv/test_excel.rb +0 -171
  61. data/test/rbbt/tsv/test_field_index.rb +0 -19
  62. data/test/rbbt/tsv/test_filter.rb +0 -187
  63. data/test/rbbt/tsv/test_index.rb +0 -308
  64. data/test/rbbt/tsv/test_manipulate.rb +0 -334
  65. data/test/rbbt/tsv/test_marshal.rb +0 -24
  66. data/test/rbbt/tsv/test_matrix.rb +0 -20
  67. data/test/rbbt/tsv/test_parallel.rb +0 -7
  68. data/test/rbbt/tsv/test_parser.rb +0 -101
  69. data/test/rbbt/tsv/test_stream.rb +0 -253
  70. data/test/rbbt/tsv/test_util.rb +0 -52
  71. data/test/rbbt/util/R/test_eval.rb +0 -43
  72. data/test/rbbt/util/R/test_model.rb +0 -128
  73. data/test/rbbt/util/R/test_plot.rb +0 -38
  74. data/test/rbbt/util/concurrency/processes/test_socket.rb +0 -70
  75. data/test/rbbt/util/concurrency/test_processes.rb +0 -192
  76. data/test/rbbt/util/concurrency/test_threads.rb +0 -40
  77. data/test/rbbt/util/log/test_progress.rb +0 -111
  78. data/test/rbbt/util/misc/test_bgzf.rb +0 -48
  79. data/test/rbbt/util/misc/test_communication.rb +0 -13
  80. data/test/rbbt/util/misc/test_development.rb +0 -26
  81. data/test/rbbt/util/misc/test_format.rb +0 -10
  82. data/test/rbbt/util/misc/test_indiferent_hash.rb +0 -14
  83. data/test/rbbt/util/misc/test_lock.rb +0 -77
  84. data/test/rbbt/util/misc/test_multipart_payload.rb +0 -202
  85. data/test/rbbt/util/misc/test_omics.rb +0 -116
  86. data/test/rbbt/util/misc/test_pipes.rb +0 -343
  87. data/test/rbbt/util/misc/test_serialize.rb +0 -24
  88. data/test/rbbt/util/python/test_util.rb +0 -25
  89. data/test/rbbt/util/simpleopt/test_get.rb +0 -12
  90. data/test/rbbt/util/simpleopt/test_parse.rb +0 -10
  91. data/test/rbbt/util/simpleopt/test_setup.rb +0 -76
  92. data/test/rbbt/util/test_R.rb +0 -37
  93. data/test/rbbt/util/test_chain_methods.rb +0 -22
  94. data/test/rbbt/util/test_cmd.rb +0 -87
  95. data/test/rbbt/util/test_colorize.rb +0 -22
  96. data/test/rbbt/util/test_concurrency.rb +0 -6
  97. data/test/rbbt/util/test_config.rb +0 -69
  98. data/test/rbbt/util/test_excel2tsv.rb +0 -10
  99. data/test/rbbt/util/test_filecache.rb +0 -36
  100. data/test/rbbt/util/test_log.rb +0 -52
  101. data/test/rbbt/util/test_migrate.rb +0 -34
  102. data/test/rbbt/util/test_misc.rb +0 -728
  103. data/test/rbbt/util/test_open.rb +0 -200
  104. data/test/rbbt/util/test_procpath.rb +0 -23
  105. data/test/rbbt/util/test_python.rb +0 -144
  106. data/test/rbbt/util/test_semaphore.rb +0 -36
  107. data/test/rbbt/util/test_simpleDSL.rb +0 -55
  108. data/test/rbbt/util/test_simpleopt.rb +0 -11
  109. data/test/rbbt/util/test_ssh.rb +0 -10
  110. data/test/rbbt/util/test_tmpfile.rb +0 -32
  111. data/test/rbbt/workflow/step/test_dependencies.rb +0 -295
  112. data/test/rbbt/workflow/step/test_save_load_inputs.rb +0 -136
  113. data/test/rbbt/workflow/test_doc.rb +0 -30
  114. data/test/rbbt/workflow/test_remote_workflow.rb +0 -99
  115. data/test/rbbt/workflow/test_schedule.rb +0 -0
  116. data/test/rbbt/workflow/test_step.rb +0 -231
  117. data/test/rbbt/workflow/test_task.rb +0 -85
  118. data/test/rbbt/workflow/util/test_archive.rb +0 -27
  119. data/test/rbbt/workflow/util/test_data.rb +0 -67
  120. data/test/rbbt/workflow/util/test_orchestrator.rb +0 -263
  121. data/test/test_helper.rb +0 -114
@@ -1,192 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
-
3
- require 'rbbt-util'
4
- require 'rbbt/util/log'
5
- require 'rbbt/util/concurrency/processes'
6
-
7
- class TestConcurrencyProcess < Test::Unit::TestCase
8
-
9
- def setup
10
- #Log.severity = 0
11
- end
12
-
13
- def test_process_throttle
14
- q = RbbtProcessQueue.new 10
15
-
16
- times = 500
17
-
18
- res = []
19
- q.callback do |v|
20
- res << v
21
- end
22
-
23
- q.init do |i|
24
- sleep 0.001
25
- Process.pid
26
- end
27
-
28
- times.times do |i|
29
- q.process i
30
- end
31
-
32
- sleep 1 while res.length < times
33
- assert_equal 10, res.uniq.length
34
-
35
- q.add_process
36
- sleep 0.1
37
- q.add_process
38
- sleep 0.1
39
- q.add_process
40
- sleep 0.1
41
-
42
- times.times do |i|
43
- q.process i
44
- end
45
-
46
- sleep 1 while res.length < times * 2
47
- assert_equal 13, res[(times + 20)..-1].uniq.length
48
-
49
- q.remove_process
50
- sleep 0.1
51
- q.remove_process
52
- sleep 0.1
53
- q.remove_process
54
- sleep 0.1
55
- q.remove_process
56
- sleep 0.1
57
- q.remove_process
58
- sleep 0.1
59
-
60
- times.times do |i|
61
- q.process i
62
- end
63
-
64
- sleep 1 while res.length < times * 3
65
- assert_equal 8, res[(2*times + 30)..-1].uniq.length
66
-
67
- q.join
68
- assert_equal times * 3, res.length
69
- end
70
-
71
- def test_process
72
- q = RbbtProcessQueue.new 10
73
-
74
- res = []
75
-
76
- q.callback do |v|
77
- res << v
78
- end
79
-
80
- q.init do |i|
81
- i * 2
82
- end
83
-
84
- times = 500
85
-
86
- times.times do |i|
87
- q.process i
88
- end
89
-
90
- q.join
91
-
92
- assert_equal times, res.length
93
- assert_equal [0, 2, 4], res.sort[0..2]
94
- end
95
-
96
- def test_each
97
- times = 5000
98
- elems = (0..times-1).to_a
99
-
100
- TmpFile.with_file do |dir|
101
- RbbtProcessQueue.each(elems) do |elem|
102
- Open.write(File.join(dir, elem.to_s), "DONE")
103
- end
104
-
105
- assert_equal times, Dir.glob(File.join(dir, '*')).length
106
- end
107
- end
108
-
109
- def test_error
110
- assert_raise RbbtException do
111
- q = RbbtProcessQueue.new 3
112
-
113
- res = []
114
-
115
- q.callback do |v|
116
- res << v
117
- end
118
-
119
- q.init do |i|
120
- raise RbbtException.new "MY ERROR" if i == 300
121
- i * 2
122
- end
123
-
124
- times = 500
125
-
126
- times.times do |i|
127
- q.process i
128
- end
129
-
130
- q.join
131
-
132
- assert_equal times, res.length
133
- assert_equal [0, 2, 4], res.sort[0..2]
134
- end
135
- end
136
-
137
- def test_process_abort
138
- assert_raise Aborted do
139
- q = RbbtProcessQueue.new 10
140
-
141
- res = []
142
-
143
- q.callback do |v|
144
- res << v
145
- end
146
-
147
- q.init do |i|
148
- sleep 1 while true
149
- end
150
-
151
- times = 500
152
-
153
- times.times do |i|
154
- q.process i
155
- end
156
-
157
- sleep 1
158
- q.clean
159
- sleep 1
160
-
161
- q.join
162
- end
163
- end
164
-
165
-
166
- def test_process_respawn
167
- q = RbbtProcessQueue.new 2, nil, nil, true
168
-
169
- res = []
170
-
171
- q.callback do |v|
172
- res << v
173
- end
174
-
175
- q.init do |i|
176
- $str ||="-"
177
- $str = $str * (2 + rand(5).to_i)
178
- sleep 0.1
179
- "."
180
- end
181
-
182
- times = 50
183
-
184
- times.times do |i|
185
- q.process i
186
- end
187
-
188
- q.join
189
- end
190
- end
191
-
192
-
@@ -1,40 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/concurrency'
3
-
4
- class TestConcurrencyThreads < Test::Unit::TestCase
5
- def test_thread_queue
6
- q = RbbtThreadQueue.new 10
7
-
8
- res = []
9
-
10
- q.init do |v|
11
- res << v
12
- end
13
-
14
- times = 1_000_000
15
- times.times do |i|
16
- q.process [i*2]
17
- end
18
-
19
- q.join
20
- q.clean
21
-
22
- assert_equal times, res.length
23
- assert_equal [0, 2, 4], res.sort[0..2]
24
-
25
- end
26
-
27
- def test_each
28
- times = 10000
29
- elems = (0..times-1).to_a
30
-
31
- TmpFile.with_file do |dir|
32
- elems.each do |elem|
33
- Open.write(File.join(dir, elem.to_s), "DONE")
34
- end
35
-
36
- assert_equal times, Dir.glob(File.join(dir, '*')).length
37
- end
38
- end
39
- end
40
-
@@ -1,111 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/log/progress'
3
-
4
- class TestProgress < Test::Unit::TestCase
5
- def test_bar
6
- t1 = Thread.new do
7
- Log::ProgressBar.with_bar(20, :desc => "Bar 1") do |bar|
8
- 20.times do
9
- bar.tick
10
- sleep 0.3
11
- end
12
- Log.debug "Done progress"
13
- assert_equal 100, bar.percent
14
- end
15
- end
16
-
17
- t2 = Thread.new do
18
- Log::ProgressBar.with_bar(20, :desc => "Bar 2") do |bar|
19
- 20.times do
20
- bar.tick
21
- sleep 0.2
22
- end
23
- Log.debug "Done progress"
24
- assert_equal 100, bar.percent
25
- end
26
- end
27
- t1.join
28
- t2.join
29
- end
30
-
31
- def test_bar_no_size
32
- t1 = Thread.new do
33
- Log::ProgressBar.with_bar(nil, :desc => "Bar 1") do |bar|
34
- 20.times do
35
- bar.tick
36
- sleep 0.3
37
- end
38
- #Log.debug "Done progress"
39
- assert bar.history.length > 0
40
- end
41
- end
42
-
43
- t2 = Thread.new do
44
- Log::ProgressBar.with_bar(nil, :desc => "Bar 2") do |bar|
45
- 20.times do
46
- bar.tick
47
- sleep 0.2
48
- end
49
- #Log.debug "Done progress"
50
- assert bar.history.length > 0
51
- end
52
- end
53
- t1.join
54
- t2.join
55
- end
56
-
57
- def test_bar_nested
58
- Log::ProgressBar.with_bar(20, :desc => "Bar 1") do |bar|
59
- bar.init
60
- 20.times do
61
- Log::ProgressBar.with_bar(5, :desc => "Bar 2") do |bar|
62
- 5.times do
63
- bar.tick
64
- sleep 0.2
65
- end
66
- end
67
- bar.tick
68
- sleep 0.2
69
- end
70
- end
71
- end
72
-
73
- def test_pos
74
- size = 10000
75
-
76
- Log::ProgressBar.with_bar(size, :desc => "Bar 1") do |bar|
77
- bar.init
78
- nums = []
79
- 100.times do
80
- nums << rand(size)
81
- end
82
- nums.sort.each do |num|
83
- bar.pos num
84
- sleep 0.1
85
- end
86
- bar.tick
87
- end
88
- end
89
-
90
- def test_file
91
- size = 10000
92
-
93
- TmpFile.with_file do |file|
94
-
95
- Log::ProgressBar.with_bar(size, :desc => "Bar 1", :file => file) do |bar|
96
- bar.init
97
- nums = []
98
- 100.times do
99
- nums << rand(size)
100
- end
101
- nums.sort.each do |num|
102
- bar.pos num
103
- sleep 0.1
104
- end
105
- bar.tick
106
- end
107
- end
108
- end
109
-
110
- end
111
-
@@ -1,48 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/misc/bgzf'
3
-
4
- class TestBgzf < Test::Unit::TestCase
5
- def test_Bgzf
6
- content = "1234567890" * 1000000
7
- TmpFile.with_file(content) do |file|
8
- compressed = file + '.gz'
9
- `bgzip #{file} -c > #{compressed}`
10
- stream = Bgzf.setup Open.open(compressed, :noz => true)
11
- assert_equal "1234", stream.read(4)
12
- assert_equal "56", stream.read(2)
13
- stream.seek 500003
14
- assert_equal "4567", stream.read(4)
15
- assert_equal "89", stream.read(2)
16
- end
17
- end
18
-
19
- def test_tsv
20
- require 'rbbt/tsv'
21
- TmpFile.with_file(datafile_test(:identifiers).read) do |file|
22
- Misc.benchmark do
23
- tsv = TSV.open(Open.open(file))
24
- end
25
-
26
- `gzip #{file}`
27
- stream = Open.open(file + '.gz')
28
- Misc.benchmark do
29
- tsv = TSV.open(stream)
30
- end
31
-
32
- `gunzip #{file}.gz`
33
- compressed = file + '.bgz'
34
- `bgzip #{file} -c > #{compressed}`
35
- stream = Bgzf.setup File.open(compressed)
36
- Misc.benchmark do
37
- tsv = TSV.open(stream)
38
- end
39
-
40
- end
41
- end
42
-
43
- def test_bgzip
44
- assert File.exist?(Bgzf.bgzip_cmd)
45
- assert 'bgzip', File.basename(Bgzf.bgzip_cmd)
46
- end
47
- end
48
-
@@ -1,13 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/misc/communication'
3
-
4
- class TestCommunication < Test::Unit::TestCase
5
- def test_send_email
6
- keyword_test :mail do
7
- to = from = 'mvazque2@localhost'
8
- subject = message = "Test"
9
- Misc.send_email(to, from, subject, message)
10
- end
11
- end
12
- end
13
-
@@ -1,26 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/misc/development'
3
-
4
- class TestMiscDevelopment < Test::Unit::TestCase
5
- def test_mem
6
- assert Integer === Misc.memory_use
7
- end
8
-
9
- def ___test_ssh
10
- Misc.ssh_run 'mn1' do
11
- puts "hola"
12
- end
13
- end
14
-
15
- def __test_timeout
16
- Misc.timeout_insist(2) do
17
- puts "Start"
18
- 3.times do
19
- puts "ping"
20
- sleep rand(4)
21
- puts "pong"
22
- end
23
- end
24
- end
25
- end
26
-
@@ -1,10 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/misc/format'
3
-
4
- class TestClass < Test::Unit::TestCase
5
- def test_fixutf8
6
- a = "Camión"
7
- assert_equal a, Misc.fixutf8(a)
8
- end
9
- end
10
-
@@ -1,14 +0,0 @@
1
- require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
- require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
-
4
- class TestIndiferentHash < Test::Unit::TestCase
5
- def test_recursive
6
- a = {:a => {:b => 1 } }
7
- IndiferentHash.setup(a)
8
-
9
- assert IndiferentHash === a["a"]
10
- assert_equal 1, a[:a]["b"]
11
- assert_equal 1, a["a"][:b]
12
- end
13
- end
14
-
@@ -1,77 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
-
3
- require 'rbbt-util'
4
- require 'rbbt/util/misc/lock'
5
- require 'rbbt/workflow'
6
-
7
- class TestLock < Test::Unit::TestCase
8
- def __test_stress
9
- end
10
- end
11
-
12
- if __FILE__ == $0
13
- def deleted(pid = Process.pid)
14
- begin
15
- txt = `ls -la /proc/#{pid}/fd |grep deleted`
16
- puts Log.color(:magenta, [pid, txt.split("\n")*", "] * ": ")
17
- rescue Exception
18
- Log.exception $!
19
- end
20
- end
21
-
22
- cpus = 10
23
- file = "/tmp/test.lock"
24
-
25
- pids = []
26
- cpus.times do
27
- pids << Process.fork do
28
- while true do
29
- Lockfile.new file do
30
- Lockfile.new file + '.1' do
31
- end
32
- File.open(file){|f| puts f.read }
33
- end
34
- deleted
35
- end
36
- end
37
- end
38
-
39
-
40
-
41
- exit
42
- size = 1000000
43
- num = 1
44
- cpus = 5
45
-
46
- pdb = "http://interactome3d.irbbarcelona.org/pdb.php?dataset=human&type1=interactions&type2=pdb&pdb=Q99685-Q99685-EXP-3hju.pdb1-B-0-A-0.pdb"
47
- Workflow.require_workflow "Structure"
48
- TmpFile.with_file do |dir|
49
- Structure.workdir = dir
50
- Path.setup dir
51
- TSV.traverse (0..size).to_a, :cpus => cpus, :type => :array, :bar => true do |i|
52
- begin
53
- v = rand(num).to_s
54
- file = File.join(dir, "file-" << v.to_s)
55
-
56
- Misc.lock file + '.produce' do
57
- Misc.lock file do
58
- ##job = Structure.job(:neighbour_map, v, :pdb => pdb)
59
- #job = Translation.example_step(:translate, "Example")
60
- #job.path = file
61
- #if job.done?
62
- # job.clean if rand < 0.3
63
- #else
64
- # job.run(true)
65
- #end
66
- end
67
- end
68
- deleted Process.pid
69
-
70
- rescue Exception
71
- raise $!
72
- end
73
- end
74
- end
75
- end
76
-
77
-