rbbt-util 5.9.0 → 5.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/tsv/parallel/traverse.rb +3 -3
- data/lib/rbbt/util/concurrency/processes.rb +1 -1
- data/lib/rbbt/util/concurrency/processes/worker.rb +2 -1
- data/lib/rbbt/util/concurrency/threads.rb +0 -1
- data/lib/rbbt/util/misc.rb +12 -6
- data/test/rbbt/tsv/parallel/test_traverse.rb +29 -7
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 66364dac8f8a7eb98ad0b63b8d45405fd994a2ad
|
4
|
+
data.tar.gz: f1861d29eec6bc26661f829c80f1aa3367684a69
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ee58bf9d7f6abb6335f211e26ad2c3d84e6c6d3e56e0ed0279c00e8e6c5546f8065e5b14b6b881b7870e959f5c3f3f68f256f0685bf97de26aef09abf4bdef2e
|
7
|
+
data.tar.gz: 29cd31366e10b06ce60cd6df50d65067cce46450cddf188f6322fcd5a46c4c71b2a4669ef5521a1f6e21edea0967bf7b2b068e7acef6838bb08f833f377e8e13
|
@@ -91,7 +91,7 @@ module TSV
|
|
91
91
|
q.init &block
|
92
92
|
|
93
93
|
traverse_obj(obj, options) do |*p|
|
94
|
-
q.process p
|
94
|
+
q.process *p
|
95
95
|
end
|
96
96
|
|
97
97
|
q.join
|
@@ -119,8 +119,8 @@ module TSV
|
|
119
119
|
into = Misc.process_options options, :into
|
120
120
|
|
121
121
|
if into
|
122
|
-
callback = Proc.new do
|
123
|
-
store_into into,
|
122
|
+
callback = Proc.new do |e|
|
123
|
+
store_into into, e
|
124
124
|
end
|
125
125
|
options[:callback] = callback
|
126
126
|
end
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -981,7 +981,11 @@ end
|
|
981
981
|
else
|
982
982
|
Log.warn("Insisting after exception: #{$!.message}")
|
983
983
|
end
|
984
|
-
|
984
|
+
if sleep and try > 0
|
985
|
+
sleep sleep
|
986
|
+
else
|
987
|
+
Thread.pass
|
988
|
+
end
|
985
989
|
try += 1
|
986
990
|
retry if try < times
|
987
991
|
raise $!
|
@@ -1103,12 +1107,14 @@ end
|
|
1103
1107
|
lockfile = Lockfile.new(lock_path)
|
1104
1108
|
|
1105
1109
|
begin
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1110
|
+
Misc.insist 3 do
|
1111
|
+
if File.exists? lockfile and
|
1112
|
+
Misc.hostname == (info = Open.open(lockfile){|f| YAML.load(f) })["host"] and
|
1113
|
+
info["pid"] and not Misc.pid_exists?(info["pid"])
|
1109
1114
|
|
1110
|
-
|
1111
|
-
|
1115
|
+
Log.info("Removing lockfile: #{lockfile}. This pid #{Process.pid}. Content: #{info.inspect}")
|
1116
|
+
FileUtils.rm lockfile
|
1117
|
+
end
|
1112
1118
|
end
|
1113
1119
|
rescue
|
1114
1120
|
Log.warn("Error checking lockfile #{lockfile}: #{$!.message}. Removing. Content: #{begin Open.read(lockfile) rescue "Could not open file" end}")
|
@@ -4,7 +4,7 @@ require 'rbbt/tsv/parallel'
|
|
4
4
|
|
5
5
|
class TestTSVParallelThrough < Test::Unit::TestCase
|
6
6
|
|
7
|
-
def
|
7
|
+
def test_traverse_tsv
|
8
8
|
require 'rbbt/sources/organism'
|
9
9
|
|
10
10
|
head = 100
|
@@ -23,7 +23,29 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
23
23
|
assert_equal head, res.keys.compact.sort.length
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
26
|
+
def test_traverse_tsv_cpus
|
27
|
+
require 'rbbt/sources/organism'
|
28
|
+
|
29
|
+
head = 100
|
30
|
+
|
31
|
+
tsv = Organism.identifiers("Hsa").tsv :head => head
|
32
|
+
res = {}
|
33
|
+
TSV.traverse tsv do |k,v|
|
34
|
+
res[k] = v
|
35
|
+
end
|
36
|
+
assert_equal head, res.keys.compact.sort.length
|
37
|
+
assert res.values.compact.flatten.uniq.length > 0
|
38
|
+
|
39
|
+
tsv = Organism.identifiers("Hsa").tsv :head => head
|
40
|
+
TSV.traverse tsv, :into => res, :cpus => 5 do |k,v|
|
41
|
+
[k,v]
|
42
|
+
end
|
43
|
+
|
44
|
+
assert_equal head, res.keys.compact.sort.length
|
45
|
+
assert res.values.compact.flatten.uniq.length > 0
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_traverse_stream
|
27
49
|
require 'rbbt/sources/organism'
|
28
50
|
|
29
51
|
head = 100
|
@@ -44,7 +66,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
44
66
|
assert_equal head, res.keys.compact.sort.length
|
45
67
|
end
|
46
68
|
|
47
|
-
def
|
69
|
+
def test_traverse_stream_keys
|
48
70
|
require 'rbbt/sources/organism'
|
49
71
|
|
50
72
|
head = 100
|
@@ -68,7 +90,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
68
90
|
assert_equal res, Organism.identifiers("Hsa").tsv(:head => 100).keys
|
69
91
|
end
|
70
92
|
|
71
|
-
def
|
93
|
+
def test_traverse_array
|
72
94
|
require 'rbbt/sources/organism'
|
73
95
|
|
74
96
|
array = []
|
@@ -90,7 +112,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
90
112
|
assert_equal array, res
|
91
113
|
end
|
92
114
|
|
93
|
-
def
|
115
|
+
def test_traverse_array_threads
|
94
116
|
require 'rbbt/sources/organism'
|
95
117
|
|
96
118
|
array = []
|
@@ -111,7 +133,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
111
133
|
assert_equal array.sort, res.sort
|
112
134
|
end
|
113
135
|
|
114
|
-
def
|
136
|
+
def test_traverse_array_cpus
|
115
137
|
require 'rbbt/sources/organism'
|
116
138
|
|
117
139
|
array = []
|
@@ -129,7 +151,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
129
151
|
def test_traverse_benchmark
|
130
152
|
require 'rbbt/sources/organism'
|
131
153
|
|
132
|
-
head =
|
154
|
+
head = 8_000
|
133
155
|
|
134
156
|
tsv = Organism.identifiers("Hsa").open
|
135
157
|
Misc.benchmark do
|