rbbt-util 5.14.17 → 5.14.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/etc/app.d/base.rb +11 -9
- data/etc/app.d/finder.rb +2 -2
- data/lib/rbbt/association/index.rb +3 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +11 -16
- data/lib/rbbt/util/concurrency/processes.rb +4 -2
- data/lib/rbbt/util/concurrency/processes/socket.rb +5 -5
- data/lib/rbbt/util/misc/bgzf.rb +13 -1
- data/lib/rbbt/util/misc/development.rb +1 -2
- data/lib/rbbt/util/open.rb +1 -0
- data/share/rbbt_commands/stat/density +0 -4
- data/test/rbbt/tsv/parallel/test_traverse.rb +4 -4
- data/test/rbbt/util/test_misc.rb +10 -0
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 243728b311e7948c452e6de9e299e5938a3ebd24
|
4
|
+
data.tar.gz: 8eb9ee8a10b1653c79465eab2c68f340a39ecc9f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 46d41fe4394efa9c7ad329e8fa42d0fcd3ea6d2d72cd30a7d2b3f6d7fd8df61565f621ad339abc74d7662a5220df92d2de03beb46f0b5ded72d5a967e532f752
|
7
|
+
data.tar.gz: 20566c536d6b4f4946feedea1e5ad956caf883d8772db394fc8217bbc9eae1bacecb9d5b74fdfde0d45cd94784b82f8ceed01ac3ed286edfa8e35dd3782a12c7
|
data/etc/app.d/base.rb
CHANGED
@@ -23,17 +23,19 @@ use Rack::Session::Cookie, :key => 'rack.session',
|
|
23
23
|
|
24
24
|
|
25
25
|
#{{{ DIRECTORIES
|
26
|
-
global_var = Rbbt.var
|
27
|
-
local_var = Rbbt.var.app[$class_name]
|
26
|
+
global_var = Rbbt.var.sinatra
|
27
|
+
local_var = Rbbt.var.sinatra.app[$class_name]
|
28
28
|
|
29
|
-
set :cache_dir , local_var.
|
30
|
-
set :persist_dir , local_var.
|
29
|
+
set :cache_dir , local_var.cache
|
30
|
+
set :persist_dir , local_var.cache.persistence
|
31
31
|
set :persist_options , {:persist => true, :persist_dir => :persist_dir}
|
32
|
-
set :file_dir , local_var.
|
33
|
-
set :permalink_dir , local_var.
|
34
|
-
set :favourites_dir , local_var.
|
35
|
-
set :favourite_lists_dir , local_var.
|
36
|
-
set :favourite_maps_dir , local_var.
|
32
|
+
set :file_dir , local_var.files
|
33
|
+
set :permalink_dir , local_var.permalink
|
34
|
+
set :favourites_dir , local_var.favourites
|
35
|
+
set :favourite_lists_dir , local_var.favourite_lists
|
36
|
+
set :favourite_maps_dir , local_var.favourite_maps
|
37
|
+
|
38
|
+
#set :finder, nil
|
37
39
|
|
38
40
|
#{{{ WORKFLOWS
|
39
41
|
|
data/etc/app.d/finder.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
#{{{ FINDER
|
2
2
|
finder = Finder.new
|
3
3
|
if ENV['RBBT_FINDER']
|
4
|
-
finder.add_instance(KEGG.pathways, :grep => '^hsa\|^#', :fields => ["Pathway Name"], :namespace => "Hsa
|
5
|
-
finder.add_instance(Organism.lexicon("Hsa
|
4
|
+
finder.add_instance(KEGG.pathways, :grep => '^hsa\|^#', :fields => ["Pathway Name"], :namespace => Organism.default_code("Hsa"), :fix => Proc.new{|l| l.sub(/ - Homo sapiens.*/,'')})
|
5
|
+
finder.add_instance(Organism.lexicon(Organism.default_code("Hsa")), :persist => true, :namespace => Organism.default_code("Hsa"), :grep => '^LRG_', :invert_grep => true)
|
6
6
|
end
|
7
7
|
set :finder, finder
|
8
8
|
Log.debug("Finder started with: #{finder.instances.length} instances")
|
@@ -16,13 +16,15 @@ module Association
|
|
16
16
|
|
17
17
|
def reverse
|
18
18
|
@reverse ||= begin
|
19
|
+
persistence_path = self.persistence_path
|
20
|
+
persistence_path = persistence_path.find if Path === persistence_path
|
19
21
|
reverse_filename = persistence_path + '.reverse'
|
20
22
|
|
21
23
|
if File.exists?(reverse_filename)
|
22
24
|
new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
|
23
25
|
new
|
24
26
|
else
|
25
|
-
FileUtils.mkdir_p File.
|
27
|
+
FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exists?(File.basename(reverse_filename))
|
26
28
|
new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
|
27
29
|
new.write
|
28
30
|
through do |key, value|
|
@@ -336,38 +336,33 @@ module TSV
|
|
336
336
|
begin
|
337
337
|
callback, cleanup, join = Misc.process_options options, :callback, :cleanup, :join
|
338
338
|
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
q.init &block
|
339
|
+
q = RbbtProcessQueue.new num, cleanup, join
|
340
|
+
q.callback &callback
|
341
|
+
q.init &block
|
343
342
|
|
344
|
-
|
345
|
-
|
346
|
-
end
|
347
|
-
rescue Aborted, Errno::EPIPE
|
348
|
-
Log.medium "Aborted"
|
349
|
-
rescue Exception
|
350
|
-
Log.exception $!
|
351
|
-
raise $!
|
352
|
-
ensure
|
353
|
-
q.join
|
354
|
-
q.clean
|
343
|
+
traverse_obj(obj, options) do |*p|
|
344
|
+
q.process *p
|
355
345
|
end
|
346
|
+
|
347
|
+
q.join
|
356
348
|
rescue Interrupt, Aborted
|
357
|
-
Log.medium{"Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"}
|
358
349
|
q.abort
|
350
|
+
Log.medium{"Aborted traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.backtrace*","}"}
|
359
351
|
stream = obj_stream(obj)
|
360
352
|
stream.abort if stream.respond_to? :abort
|
361
353
|
stream = obj_stream(options[:into])
|
362
354
|
stream.abort if stream.respond_to? :abort
|
363
355
|
raise "Traversal aborted"
|
364
356
|
rescue Exception
|
357
|
+
q.abort
|
365
358
|
Log.medium "Exception during traversal in CPUs for #{stream_name(obj) || Misc.fingerprint(obj)}: #{$!.message}"
|
366
359
|
stream = obj_stream(obj)
|
367
360
|
stream.abort if stream.respond_to? :abort
|
368
361
|
stream = obj_stream(options[:into])
|
369
362
|
stream.abort if stream.respond_to? :abort
|
370
363
|
raise $!
|
364
|
+
ensure
|
365
|
+
q.clean
|
371
366
|
end
|
372
367
|
end
|
373
368
|
|
@@ -111,10 +111,12 @@ class RbbtProcessQueue
|
|
111
111
|
|
112
112
|
def clean
|
113
113
|
if (@process_monitor and @process_monitor.alive?) or (@callback_thread and @callback_thread.alive?)
|
114
|
-
self.abort
|
115
|
-
else
|
114
|
+
self.abort
|
116
115
|
self.join
|
117
116
|
end
|
117
|
+
|
118
|
+
@queue.clean if @queue
|
119
|
+
@callback_queue.clean if @callback_queue
|
118
120
|
end
|
119
121
|
|
120
122
|
def abort
|
@@ -10,10 +10,10 @@ class RbbtProcessQueue
|
|
10
10
|
@serializer = serializer || Marshal
|
11
11
|
|
12
12
|
|
13
|
-
key = "/" << rand(
|
14
|
-
@write_sem = key + '.in'
|
15
|
-
@read_sem = key + '.out'
|
16
|
-
Log.
|
13
|
+
@key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
|
14
|
+
@write_sem = @key + '.in'
|
15
|
+
@read_sem = @key + '.out'
|
16
|
+
Log.medium "Creating socket semaphores: #{@key}"
|
17
17
|
RbbtSemaphore.create_semaphore(@write_sem,1)
|
18
18
|
RbbtSemaphore.create_semaphore(@read_sem,1)
|
19
19
|
end
|
@@ -21,7 +21,7 @@ class RbbtProcessQueue
|
|
21
21
|
def clean
|
22
22
|
@sread.close unless @sread.closed?
|
23
23
|
@swrite.close unless @swrite.closed?
|
24
|
-
Log.
|
24
|
+
Log.medium "Destroying socket semaphores: #{[@key] * ", "}"
|
25
25
|
RbbtSemaphore.delete_semaphore(@write_sem)
|
26
26
|
RbbtSemaphore.delete_semaphore(@read_sem)
|
27
27
|
end
|
data/lib/rbbt/util/misc/bgzf.rb
CHANGED
@@ -50,6 +50,16 @@ module Bgzf
|
|
50
50
|
end
|
51
51
|
end
|
52
52
|
|
53
|
+
def read_all
|
54
|
+
str = ""
|
55
|
+
while true
|
56
|
+
block = read_block
|
57
|
+
break if block.nil?
|
58
|
+
str << block
|
59
|
+
end
|
60
|
+
str
|
61
|
+
end
|
62
|
+
|
53
63
|
def init
|
54
64
|
_index
|
55
65
|
end
|
@@ -119,7 +129,9 @@ module Bgzf
|
|
119
129
|
block[offset..-1]
|
120
130
|
end
|
121
131
|
|
122
|
-
def read(size)
|
132
|
+
def read(size=nil)
|
133
|
+
return read_all if size.nil?
|
134
|
+
|
123
135
|
block = get_block
|
124
136
|
return "" if block.nil? or block.empty?
|
125
137
|
len = block.length
|
@@ -286,9 +286,8 @@ module Misc
|
|
286
286
|
end
|
287
287
|
end
|
288
288
|
|
289
|
-
#file = caller.first + rand(1000000).to_s if file.nil?
|
290
289
|
index = (0..elems.length-1).to_a.collect{|v| v.to_s }
|
291
|
-
TSV.traverse index, :cpus => cpus, :bar => "Bootstrap in #{ cpus } cpus: #{ Misc.fingerprint elems }", :into => Set.new do |pos|
|
290
|
+
TSV.traverse index, :cpus => cpus, :bar => "Bootstrap in #{ cpus } cpus: #{ Misc.fingerprint Annotated.purge(elems) }", :into => Set.new do |pos|
|
292
291
|
elem = elems[pos.to_i]
|
293
292
|
elems.annotate elem if elems.respond_to? :annotate
|
294
293
|
begin
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -430,6 +430,7 @@ module Open
|
|
430
430
|
def self.write(file, content = nil, options = {})
|
431
431
|
options = Misc.add_defaults options, :mode => 'w'
|
432
432
|
|
433
|
+
file = file.find(options[:where]) if Path === file
|
433
434
|
mode = Misc.process_options options, :mode
|
434
435
|
|
435
436
|
FileUtils.mkdir_p File.dirname(file)
|
@@ -311,8 +311,8 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
311
311
|
head = 2_000
|
312
312
|
|
313
313
|
Log.info Log.color :red, "TRAVERSE EXCEPTION"
|
314
|
-
stream = Organism.identifiers("Hsa
|
315
|
-
dumper = TSV::Dumper.new Organism.identifiers("Hsa
|
314
|
+
stream = Organism.identifiers(Organism.default_code("Hsa")).open
|
315
|
+
dumper = TSV::Dumper.new Organism.identifiers(Organism.default_code("Hsa")).tsv_options
|
316
316
|
dumper.init
|
317
317
|
|
318
318
|
assert_raise StopException do
|
@@ -332,8 +332,8 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
332
332
|
cpus = 2
|
333
333
|
|
334
334
|
Log.info Log.color :red, "TRAVERSE EXCEPTION"
|
335
|
-
stream = Organism.identifiers("Hsa
|
336
|
-
dumper = TSV::Dumper.new Organism.identifiers("Hsa
|
335
|
+
stream = Organism.identifiers(Organism.default_code("Hsa")).open
|
336
|
+
dumper = TSV::Dumper.new Organism.identifiers(Organism.default_code("Hsa")).tsv_options
|
337
337
|
dumper.init
|
338
338
|
|
339
339
|
assert_raise ProcessFailed do
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -454,4 +454,14 @@ eum fugiat quo voluptas nulla pariatur?"
|
|
454
454
|
end
|
455
455
|
puts `ls -l /proc/#{ Process.pid }/fd`
|
456
456
|
end
|
457
|
+
|
458
|
+
def test_bootstrap
|
459
|
+
Misc.bootstrap (0..10).to_a do
|
460
|
+
Misc.bootstrap (0..10).to_a do
|
461
|
+
Misc.bootstrap (0..10).to_a do
|
462
|
+
sleep 0.1
|
463
|
+
end
|
464
|
+
end
|
465
|
+
end
|
466
|
+
end
|
457
467
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.14.
|
4
|
+
version: 5.14.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-08-
|
11
|
+
date: 2014-08-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bio-bgzf
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
- !ruby/object:Gem::Dependency
|
84
98
|
name: term-ansicolor
|
85
99
|
requirement: !ruby/object:Gem::Requirement
|