rbbt-util 5.27.11 → 5.28.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/util.rb +2 -2
- data/lib/rbbt/resource/path.rb +5 -5
- data/lib/rbbt/resource/util.rb +4 -3
- data/lib/rbbt/tsv/parallel/traverse.rb +6 -6
- data/lib/rbbt/util/misc.rb +1 -0
- data/lib/rbbt/util/misc/development.rb +12 -2
- data/lib/rbbt/util/misc/inspect.rb +2 -2
- data/lib/rbbt/util/open.rb +6 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +5 -0
- data/lib/rbbt/workflow/step/accessor.rb +1 -1
- data/lib/rbbt/workflow/step/dependencies.rb +2 -2
- data/share/rbbt_commands/app/start +4 -0
- data/share/rbbt_commands/tsv/uncollapse +0 -1
- data/share/workflow_config.ru +5 -1
- data/test/rbbt/test_resource.rb +3 -3
- data/test/rbbt/tsv/parallel/test_traverse.rb +21 -1
- data/test/rbbt/util/test_misc.rb +25 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6bc94b86c69f95806ae280d76df8da7696f5df0f008a3791385e4ccf11b70977
|
4
|
+
data.tar.gz: f7763e2dd88ba121b8a625b846bf0f4f1efaba090b873f2947e0a83ae9dabff1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 59fd76fc00eeeb892ad53bbbc0757d5e17dde600f505d31e8dabe43e0184e5c5bfcef2ccc3f7aa9bc295e36251b9f864706bc0dcf357cc3606b73b1f25f7d672
|
7
|
+
data.tar.gz: ce4d345fc9876378b57ba8753d2a7dfd13b4fd23330246bc2385186f81344026cf415e7cc6eb86253f86e709e2061139930734b2c77b7329165ed2892e3e5d0a
|
@@ -74,7 +74,7 @@ module Annotated
|
|
74
74
|
|
75
75
|
object = case
|
76
76
|
when literal_pos
|
77
|
-
values[literal_pos]
|
77
|
+
values[literal_pos].tap{|o| o.force_encoding(Encoding.default_external)}
|
78
78
|
else
|
79
79
|
id.dup
|
80
80
|
end
|
@@ -119,7 +119,7 @@ module Annotated
|
|
119
119
|
fields = AnnotatedArray === annotations ? annotations.annotations : annotations.compact.first.annotations
|
120
120
|
fields << :annotation_types
|
121
121
|
|
122
|
-
when (fields == [:literal] and
|
122
|
+
when (fields == [:literal] and ! annotations.compact.empty?)
|
123
123
|
fields << :literal
|
124
124
|
|
125
125
|
when (fields == [:all] && Annotated === annotations)
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -13,7 +13,7 @@ module Path
|
|
13
13
|
string.resource = resource
|
14
14
|
string.search_paths = search_paths
|
15
15
|
string.search_order = search_order
|
16
|
-
string.libdir = libdir || Path.caller_lib_dir
|
16
|
+
string.libdir = libdir || Path.caller_lib_dir
|
17
17
|
string
|
18
18
|
end
|
19
19
|
|
@@ -52,11 +52,11 @@ module Path
|
|
52
52
|
def join(name)
|
53
53
|
raise "Invalid path: #{ self }" if self.nil?
|
54
54
|
new = if self.empty?
|
55
|
-
self.annotate name.to_s.dup
|
55
|
+
self.annotate name.to_s.dup.chomp
|
56
56
|
else
|
57
|
-
self.annotate File.join(self, name.to_s)
|
57
|
+
self.annotate File.join(self, name.to_s.chomp)
|
58
58
|
end
|
59
|
-
new.original = File.join(self.original, name.to_s) if self.original
|
59
|
+
new.original = File.join(self.original, name.to_s.chomp) if self.original
|
60
60
|
new
|
61
61
|
end
|
62
62
|
|
@@ -76,7 +76,7 @@ module Path
|
|
76
76
|
return [] unless self.exists?
|
77
77
|
found = self.find
|
78
78
|
exp = File.join(found, pattern)
|
79
|
-
paths = Dir.glob(exp).collect{|f|
|
79
|
+
paths = Dir.glob(exp).collect{|f| self.annotate(f) }
|
80
80
|
|
81
81
|
paths.each do |p|
|
82
82
|
p.original = File.join(found.original, p.sub(/^#{found}/, ''))
|
data/lib/rbbt/resource/util.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module Path
|
2
2
|
|
3
|
-
def self.caller_lib_dir(file = nil, relative_to = 'lib')
|
3
|
+
def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
|
4
4
|
file = caller.reject{|l|
|
5
5
|
l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
|
6
6
|
l =~ /rbbt\/resource\/path\.rb/ or
|
@@ -9,13 +9,14 @@ module Path
|
|
9
9
|
l =~ /progress-monitor\.rb/
|
10
10
|
}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
|
11
11
|
|
12
|
+
relative_to = [relative_to] unless Array === relative_to
|
12
13
|
file = File.expand_path(file)
|
13
|
-
return Path.setup(file) if File.exist? File.join(file,
|
14
|
+
return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
|
14
15
|
|
15
16
|
while file != '/'
|
16
17
|
dir = File.dirname file
|
17
18
|
|
18
|
-
return dir if File.exist? File.join(dir,
|
19
|
+
return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
|
19
20
|
|
20
21
|
file = File.dirname file
|
21
22
|
end
|
@@ -101,8 +101,8 @@ module TSV
|
|
101
101
|
error = true
|
102
102
|
raise $!
|
103
103
|
ensure
|
104
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
105
104
|
join.call(error) if join
|
105
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
106
106
|
end
|
107
107
|
end
|
108
108
|
|
@@ -137,8 +137,8 @@ module TSV
|
|
137
137
|
error = true
|
138
138
|
raise $!
|
139
139
|
ensure
|
140
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
141
140
|
join.call(error) if join
|
141
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
142
142
|
end
|
143
143
|
end
|
144
144
|
|
@@ -177,8 +177,8 @@ module TSV
|
|
177
177
|
error = true
|
178
178
|
raise $!
|
179
179
|
ensure
|
180
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
181
180
|
join.call(error) if join
|
181
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
182
182
|
end
|
183
183
|
end
|
184
184
|
|
@@ -219,8 +219,8 @@ module TSV
|
|
219
219
|
error = true
|
220
220
|
raise $!
|
221
221
|
ensure
|
222
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
223
222
|
join.call(error) if join
|
223
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
224
224
|
end
|
225
225
|
end
|
226
226
|
|
@@ -273,8 +273,8 @@ module TSV
|
|
273
273
|
error = true
|
274
274
|
raise $!
|
275
275
|
ensure
|
276
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
277
276
|
join.call(error) if join
|
277
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
278
278
|
end
|
279
279
|
end
|
280
280
|
|
@@ -318,8 +318,8 @@ module TSV
|
|
318
318
|
error = true
|
319
319
|
raise $!
|
320
320
|
ensure
|
321
|
-
Log::ProgressBar.remove_bar(bar) if bar
|
322
321
|
join.call(error) if join
|
322
|
+
Log::ProgressBar.remove_bar(bar) if bar
|
323
323
|
end
|
324
324
|
end
|
325
325
|
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -295,13 +295,18 @@ def self.add_libdir(dir=nil)
|
|
295
295
|
def self.bootstrap(elems, num = nil, options = {}, &block)
|
296
296
|
IndiferentHash.setup options
|
297
297
|
|
298
|
-
num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil
|
298
|
+
num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil if defined?(Rbbt::Config)
|
299
299
|
num = :current if num.nil?
|
300
300
|
cpus = case num
|
301
301
|
when :current
|
302
302
|
n = Etc.nprocessors
|
303
303
|
n = elems.length / 2 if n > elems.length/2
|
304
|
-
|
304
|
+
|
305
|
+
if $BOOTSTRAPPED_CURRENT && $BOOTSTRAPPED_CURRENT + n > Etc.nprocessors
|
306
|
+
1
|
307
|
+
else
|
308
|
+
n
|
309
|
+
end
|
305
310
|
when String
|
306
311
|
num.to_i
|
307
312
|
when Integer
|
@@ -320,7 +325,12 @@ def self.add_libdir(dir=nil)
|
|
320
325
|
respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
|
321
326
|
|
322
327
|
index = (0..elems.length-1).to_a.collect{|v| v.to_s }
|
328
|
+
|
323
329
|
TSV.traverse index, options do |pos|
|
330
|
+
if num == :current
|
331
|
+
$BOOTSTRAPPED_CURRENT ||= n
|
332
|
+
$BOOTSTRAPPED_CURRENT += 0
|
333
|
+
end
|
324
334
|
elem = elems[pos.to_i]
|
325
335
|
elems.annotate elem if elems.respond_to? :annotate
|
326
336
|
res = begin
|
@@ -290,7 +290,7 @@ module Misc
|
|
290
290
|
if obj.exists?
|
291
291
|
if obj.directory?
|
292
292
|
files = obj.glob("**/*")
|
293
|
-
"directory: #{files}"
|
293
|
+
"directory: #{Misc.fingerprint(files)}"
|
294
294
|
else
|
295
295
|
"file: " << obj << "--" << mtime_str(obj)
|
296
296
|
end
|
@@ -401,7 +401,7 @@ module Misc
|
|
401
401
|
|
402
402
|
def self.scan_version_text(text, cmd = nil)
|
403
403
|
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
404
|
-
m = text.match(/(?:version.*?|#{cmd}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
404
|
+
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
405
405
|
return nil if m.nil?
|
406
406
|
m[1]
|
407
407
|
end
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -94,7 +94,8 @@ module Open
|
|
94
94
|
end
|
95
95
|
|
96
96
|
begin
|
97
|
-
wget_options = options.
|
97
|
+
wget_options = options.dup
|
98
|
+
wget_options = wget_options.merge( '-O' => '-') unless options.include?('--output-document')
|
98
99
|
wget_options[:pipe] = pipe unless pipe.nil?
|
99
100
|
wget_options[:stderr] = stderr unless stderr.nil?
|
100
101
|
|
@@ -824,4 +825,8 @@ module Open
|
|
824
825
|
def self.broken_link?(path)
|
825
826
|
File.symlink?(path) && ! File.exists?(File.readlink(path))
|
826
827
|
end
|
828
|
+
|
829
|
+
def self.download(url, path)
|
830
|
+
Open.wget(url, "--output-document" => path)
|
831
|
+
end
|
827
832
|
end
|
@@ -340,6 +340,11 @@ class RemoteStep < Step
|
|
340
340
|
@inputs = new_inputs
|
341
341
|
@info = nil
|
342
342
|
end
|
343
|
+
|
344
|
+
def init_info(*args)
|
345
|
+
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
346
|
+
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
347
|
+
end
|
343
348
|
end
|
344
349
|
|
345
350
|
require 'rbbt/workflow/remote_workflow/remote_step/rest'
|
@@ -208,7 +208,7 @@ class Step
|
|
208
208
|
end
|
209
209
|
|
210
210
|
def init_info(force = false)
|
211
|
-
return nil if @exec
|
211
|
+
return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
|
212
212
|
Open.lock(info_file, :lock => info_lock) do
|
213
213
|
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
214
214
|
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
@@ -24,7 +24,7 @@ class Step
|
|
24
24
|
Log.medium "Not duplicating stream #{stream_key}"
|
25
25
|
STREAM_CACHE[stream_key] = stream
|
26
26
|
when File
|
27
|
-
if Open.exists?
|
27
|
+
if Open.exists?(current.path)
|
28
28
|
Log.medium "Reopening file #{stream_key}"
|
29
29
|
Open.open(current.path)
|
30
30
|
else
|
@@ -97,7 +97,7 @@ class Step
|
|
97
97
|
end
|
98
98
|
|
99
99
|
job.dup_inputs unless status == 'done' or job.started?
|
100
|
-
job.init_info unless status == '
|
100
|
+
job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
|
101
101
|
|
102
102
|
canfail = ComputeDependency === job && job.canfail?
|
103
103
|
end
|
@@ -15,6 +15,7 @@ $ rbbt app start [options] <app_name>
|
|
15
15
|
-Ho--Host* Host name
|
16
16
|
-B--Bind* Bind IP
|
17
17
|
-p--port* TCP port
|
18
|
+
-w--workers* Number of workers for cluster mode (puma)
|
18
19
|
-s--server* Server type: thin, webrick, unicorn, etc
|
19
20
|
-f--finder Start server with finder functionality
|
20
21
|
-fs--file_server Activate file serving for resources
|
@@ -88,6 +89,8 @@ Misc.in_dir(app_dir) do
|
|
88
89
|
options.each do |k,v| fixed_options[k.to_sym] = v end
|
89
90
|
options = fixed_options
|
90
91
|
|
92
|
+
options[:workers] = options[:workers].to_i if options[:workers]
|
93
|
+
|
91
94
|
case server
|
92
95
|
when 'passenger'
|
93
96
|
system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"
|
@@ -95,6 +98,7 @@ Misc.in_dir(app_dir) do
|
|
95
98
|
system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
|
96
99
|
else
|
97
100
|
options[:config] = config_ru_file
|
101
|
+
options[:threads] = "8:8"
|
98
102
|
Rack::Server.start(options)
|
99
103
|
end
|
100
104
|
end
|
data/share/workflow_config.ru
CHANGED
@@ -43,7 +43,11 @@ etc_dir = Rbbt.etc
|
|
43
43
|
load_file etc_dir['app.d/pre.rb'].find
|
44
44
|
|
45
45
|
app.get '/' do
|
46
|
-
|
46
|
+
begin
|
47
|
+
template_render('main', params, 'main', :cache_type => :asynchronous)
|
48
|
+
rescue TemplateMissing
|
49
|
+
redirect to(File.join('/', wf.to_s))
|
50
|
+
end
|
47
51
|
end
|
48
52
|
|
49
53
|
#{{{ BASE
|
data/test/rbbt/test_resource.rb
CHANGED
@@ -69,11 +69,11 @@ class TestTSV < Test::Unit::TestCase
|
|
69
69
|
end
|
70
70
|
|
71
71
|
def test_libdir
|
72
|
-
assert File.exist?
|
73
|
-
assert File.exist?
|
72
|
+
assert File.exist?(TestResource[].share.Rlib["util.R"].find(:lib))
|
73
|
+
assert File.exist?(TestResource[].share.Rlib["util.R"].find)
|
74
74
|
end
|
75
75
|
|
76
|
-
def
|
76
|
+
def __test_server
|
77
77
|
require 'rbbt/sources/organism'
|
78
78
|
TmpFile.with_file :extension => 'gz' do |tmp|
|
79
79
|
Organism.get_from_server("Hsa/b37/known_sites/dbsnp_138.vcf.gz", tmp, 'http://rbbt.bsc.es')
|
@@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
202
202
|
stream = datafile_test('identifiers').open
|
203
203
|
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
204
204
|
dumper.init
|
205
|
-
TSV.traverse stream, :head => head, :into => dumper do |k,v|
|
205
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
206
206
|
k = k.first
|
207
207
|
[k,v]
|
208
208
|
end
|
@@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
212
212
|
assert_equal head, res.size
|
213
213
|
end
|
214
214
|
|
215
|
+
def test_traverse_into_dumper_error_bar
|
216
|
+
require 'rbbt/sources/organism'
|
217
|
+
|
218
|
+
head = 2_000
|
219
|
+
|
220
|
+
stream = datafile_test('identifiers').open
|
221
|
+
dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
|
222
|
+
dumper.init
|
223
|
+
TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
|
224
|
+
k = k.first
|
225
|
+
raise
|
226
|
+
[k,v]
|
227
|
+
end
|
228
|
+
|
229
|
+
assert_raise do
|
230
|
+
res = TSV.open(dumper.stream)
|
231
|
+
end
|
232
|
+
|
233
|
+
end
|
234
|
+
|
215
235
|
def test_traverse_into_dumper_threads
|
216
236
|
require 'rbbt/sources/organism'
|
217
237
|
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -690,5 +690,30 @@ Written by Mike Haertel and others, see <http://git.sv.gnu.org/cgit/grep.git/tre
|
|
690
690
|
EOF
|
691
691
|
assert_equal "3.1", Misc.scan_version_text(txt, "grep")
|
692
692
|
end
|
693
|
+
|
694
|
+
def test_bootstrap_current_only_once
|
695
|
+
|
696
|
+
|
697
|
+
max = Etc.nprocessors * 10
|
698
|
+
TmpFile.with_file do |f|
|
699
|
+
RbbtSemaphore.with_semaphore(1) do |sem|
|
700
|
+
Misc.bootstrap (1..max).to_a do
|
701
|
+
Open.open(f, :mode => 'a') do |sin|
|
702
|
+
sin.puts Process.pid.to_s
|
703
|
+
end
|
704
|
+
Misc.bootstrap (1..10).to_a do
|
705
|
+
RbbtSemaphore.synchronize(sem) do
|
706
|
+
Open.open(f, :mode => 'a') do |sin|
|
707
|
+
sin.puts [Process.ppid.to_s, Process.pid.to_s] * ":"
|
708
|
+
end
|
709
|
+
end
|
710
|
+
end
|
711
|
+
end
|
712
|
+
end
|
713
|
+
assert_equal max * 10 + max, Open.read(f).split("\n").length
|
714
|
+
assert_equal Etc.nprocessors * 2, Open.read(f).split("\n").uniq.length
|
715
|
+
end
|
716
|
+
|
717
|
+
end
|
693
718
|
end
|
694
719
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.28.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-08-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|