rbbt-util 5.44.1 → 6.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/rbbt +67 -90
- data/etc/app.d/base.rb +2 -2
- data/etc/app.d/semaphores.rb +3 -3
- data/lib/rbbt/annotations/annotated_array.rb +207 -207
- data/lib/rbbt/annotations/refactor.rb +27 -0
- data/lib/rbbt/annotations/util.rb +282 -282
- data/lib/rbbt/annotations.rb +343 -320
- data/lib/rbbt/association/database.rb +200 -225
- data/lib/rbbt/association/index.rb +294 -291
- data/lib/rbbt/association/item.rb +227 -227
- data/lib/rbbt/association/open.rb +35 -34
- data/lib/rbbt/association/util.rb +0 -169
- data/lib/rbbt/association.rb +2 -4
- data/lib/rbbt/entity/identifiers.rb +119 -118
- data/lib/rbbt/entity/refactor.rb +12 -0
- data/lib/rbbt/entity.rb +319 -315
- data/lib/rbbt/hpc/batch.rb +72 -53
- data/lib/rbbt/hpc/lsf.rb +2 -2
- data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
- data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
- data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
- data/lib/rbbt/hpc/orchestrate.rb +19 -13
- data/lib/rbbt/hpc/slurm.rb +18 -18
- data/lib/rbbt/knowledge_base/entity.rb +13 -5
- data/lib/rbbt/knowledge_base/query.rb +2 -2
- data/lib/rbbt/knowledge_base/registry.rb +32 -31
- data/lib/rbbt/knowledge_base/traverse.rb +1 -1
- data/lib/rbbt/knowledge_base.rb +1 -1
- data/lib/rbbt/monitor.rb +36 -25
- data/lib/rbbt/persist/refactor.rb +166 -0
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
- data/lib/rbbt/persist/tsv.rb +187 -185
- data/lib/rbbt/persist.rb +556 -551
- data/lib/rbbt/refactor.rb +20 -0
- data/lib/rbbt/resource/path/refactor.rb +178 -0
- data/lib/rbbt/resource/path.rb +317 -497
- data/lib/rbbt/resource/util.rb +0 -48
- data/lib/rbbt/resource.rb +3 -390
- data/lib/rbbt/tsv/accessor.rb +2 -838
- data/lib/rbbt/tsv/attach.rb +303 -299
- data/lib/rbbt/tsv/change_id.rb +244 -245
- data/lib/rbbt/tsv/csv.rb +87 -85
- data/lib/rbbt/tsv/dumper.rb +2 -100
- data/lib/rbbt/tsv/excel.rb +26 -24
- data/lib/rbbt/tsv/field_index.rb +4 -1
- data/lib/rbbt/tsv/filter.rb +3 -2
- data/lib/rbbt/tsv/index.rb +2 -284
- data/lib/rbbt/tsv/manipulate.rb +750 -747
- data/lib/rbbt/tsv/marshal.rb +3 -3
- data/lib/rbbt/tsv/matrix.rb +2 -2
- data/lib/rbbt/tsv/parallel/through.rb +2 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
- data/lib/rbbt/tsv/parser.rb +678 -678
- data/lib/rbbt/tsv/refactor.rb +195 -0
- data/lib/rbbt/tsv/stream.rb +253 -251
- data/lib/rbbt/tsv/util.rb +420 -420
- data/lib/rbbt/tsv.rb +210 -208
- data/lib/rbbt/util/R/eval.rb +4 -4
- data/lib/rbbt/util/R/plot.rb +62 -166
- data/lib/rbbt/util/R.rb +21 -18
- data/lib/rbbt/util/cmd.rb +2 -318
- data/lib/rbbt/util/color.rb +269 -269
- data/lib/rbbt/util/colorize.rb +89 -89
- data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
- data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
- data/lib/rbbt/util/concurrency/processes.rb +389 -386
- data/lib/rbbt/util/config.rb +169 -167
- data/lib/rbbt/util/iruby.rb +20 -0
- data/lib/rbbt/util/log/progress/report.rb +241 -241
- data/lib/rbbt/util/log/progress/util.rb +99 -99
- data/lib/rbbt/util/log/progress.rb +102 -102
- data/lib/rbbt/util/log/refactor.rb +49 -0
- data/lib/rbbt/util/log.rb +486 -532
- data/lib/rbbt/util/migrate.rb +1 -1
- data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
- data/lib/rbbt/util/misc/development.rb +12 -11
- data/lib/rbbt/util/misc/exceptions.rb +117 -112
- data/lib/rbbt/util/misc/format.rb +2 -230
- data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
- data/lib/rbbt/util/misc/inspect.rb +2 -476
- data/lib/rbbt/util/misc/lock.rb +109 -106
- data/lib/rbbt/util/misc/omics.rb +9 -1
- data/lib/rbbt/util/misc/pipes.rb +765 -793
- data/lib/rbbt/util/misc/refactor.rb +20 -0
- data/lib/rbbt/util/misc/ssw.rb +27 -17
- data/lib/rbbt/util/misc/system.rb +0 -15
- data/lib/rbbt/util/misc.rb +39 -20
- data/lib/rbbt/util/named_array/refactor.rb +4 -0
- data/lib/rbbt/util/named_array.rb +3 -220
- data/lib/rbbt/util/open/refactor.rb +7 -0
- data/lib/rbbt/util/open.rb +3 -857
- data/lib/rbbt/util/procpath.rb +6 -6
- data/lib/rbbt/util/python/paths.rb +27 -0
- data/lib/rbbt/util/python/run.rb +115 -0
- data/lib/rbbt/util/python/script.rb +110 -0
- data/lib/rbbt/util/python/util.rb +3 -3
- data/lib/rbbt/util/python.rb +22 -81
- data/lib/rbbt/util/semaphore.rb +152 -148
- data/lib/rbbt/util/simpleopt.rb +9 -8
- data/lib/rbbt/util/ssh/refactor.rb +19 -0
- data/lib/rbbt/util/ssh.rb +122 -118
- data/lib/rbbt/util/tar.rb +117 -115
- data/lib/rbbt/util/tmpfile.rb +69 -67
- data/lib/rbbt/util/version.rb +2 -0
- data/lib/rbbt/workflow/refactor/entity.rb +11 -0
- data/lib/rbbt/workflow/refactor/export.rb +66 -0
- data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
- data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
- data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
- data/lib/rbbt/workflow/refactor.rb +153 -0
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
- data/lib/rbbt/workflow/remote_workflow.rb +6 -1
- data/lib/rbbt/workflow/step/run.rb +766 -766
- data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
- data/lib/rbbt/workflow/step.rb +2 -362
- data/lib/rbbt/workflow/task.rb +118 -118
- data/lib/rbbt/workflow/usage.rb +289 -287
- data/lib/rbbt/workflow/util/archive.rb +6 -5
- data/lib/rbbt/workflow/util/data.rb +1 -1
- data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
- data/lib/rbbt/workflow/util/trace.rb +79 -44
- data/lib/rbbt/workflow.rb +4 -882
- data/lib/rbbt-util.rb +21 -13
- data/lib/rbbt.rb +16 -3
- data/python/rbbt/__init__.py +19 -1
- data/share/Rlib/plot.R +37 -37
- data/share/Rlib/svg.R +22 -5
- data/share/install/software/lib/install_helpers +1 -1
- data/share/rbbt_commands/hpc/list +2 -3
- data/share/rbbt_commands/hpc/orchestrate +4 -4
- data/share/rbbt_commands/hpc/tail +2 -0
- data/share/rbbt_commands/hpc/task +10 -7
- data/share/rbbt_commands/lsf/list +2 -3
- data/share/rbbt_commands/lsf/orchestrate +4 -4
- data/share/rbbt_commands/lsf/tail +2 -0
- data/share/rbbt_commands/lsf/task +10 -7
- data/share/rbbt_commands/migrate +1 -1
- data/share/rbbt_commands/pbs/list +2 -3
- data/share/rbbt_commands/pbs/orchestrate +4 -4
- data/share/rbbt_commands/pbs/tail +2 -0
- data/share/rbbt_commands/pbs/task +10 -7
- data/share/rbbt_commands/resource/produce +8 -1
- data/share/rbbt_commands/slurm/list +2 -3
- data/share/rbbt_commands/slurm/orchestrate +4 -4
- data/share/rbbt_commands/slurm/tail +2 -0
- data/share/rbbt_commands/slurm/task +10 -7
- data/share/rbbt_commands/system/clean +5 -5
- data/share/rbbt_commands/system/status +5 -5
- data/share/rbbt_commands/tsv/get +2 -3
- data/share/rbbt_commands/tsv/info +10 -13
- data/share/rbbt_commands/tsv/keys +18 -14
- data/share/rbbt_commands/tsv/slice +2 -2
- data/share/rbbt_commands/tsv/transpose +6 -2
- data/share/rbbt_commands/workflow/info +20 -24
- data/share/rbbt_commands/workflow/list +1 -1
- data/share/rbbt_commands/workflow/prov +20 -13
- data/share/rbbt_commands/workflow/server +11 -1
- data/share/rbbt_commands/workflow/task +76 -71
- data/share/rbbt_commands/workflow/write_info +26 -9
- data/share/software/opt/ssw/ssw.c +861 -0
- data/share/software/opt/ssw/ssw.h +130 -0
- data/share/workflow_config.ru +3 -3
- metadata +40 -2
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -1,318 +1,2 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
require 'open3'
|
4
|
-
require 'rbbt/util/misc/indiferent_hash'
|
5
|
-
|
6
|
-
module CMD
|
7
|
-
|
8
|
-
TOOLS = IndiferentHash.setup({})
|
9
|
-
def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
|
10
|
-
TOOLS[tool] = [claim, test, block, cmd]
|
11
|
-
end
|
12
|
-
|
13
|
-
def self.conda(tool, env = nil, channel = 'bioconda')
|
14
|
-
if env
|
15
|
-
CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
|
16
|
-
else
|
17
|
-
CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def self.get_tool(tool)
|
22
|
-
return tool.to_s unless TOOLS[tool]
|
23
|
-
|
24
|
-
@@init_cmd_tool ||= IndiferentHash.setup({})
|
25
|
-
|
26
|
-
claim, test, block, cmd = TOOLS[tool]
|
27
|
-
cmd = tool.to_s if cmd.nil?
|
28
|
-
|
29
|
-
if !@@init_cmd_tool[tool]
|
30
|
-
|
31
|
-
begin
|
32
|
-
if test
|
33
|
-
CMD.cmd(test + " ")
|
34
|
-
else
|
35
|
-
CMD.cmd("#{cmd} --help")
|
36
|
-
end
|
37
|
-
rescue
|
38
|
-
if claim
|
39
|
-
claim.produce
|
40
|
-
else
|
41
|
-
block.call
|
42
|
-
end
|
43
|
-
end
|
44
|
-
version_txt = ""
|
45
|
-
version = nil
|
46
|
-
["--version", "-version", "--help", ""].each do |f|
|
47
|
-
begin
|
48
|
-
version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
|
49
|
-
version = Misc.scan_version_text(version_txt, tool)
|
50
|
-
break if version
|
51
|
-
rescue
|
52
|
-
Log.exception $!
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
@@init_cmd_tool[tool] = version || true
|
57
|
-
|
58
|
-
return cmd if cmd
|
59
|
-
end
|
60
|
-
|
61
|
-
cmd
|
62
|
-
end
|
63
|
-
|
64
|
-
def self.versions
|
65
|
-
return {} unless defined? @@init_cmd_tool
|
66
|
-
@@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
|
67
|
-
end
|
68
|
-
|
69
|
-
def self.gzip_pipe(file)
|
70
|
-
Open.gzip?(file) ? "<(gunzip -c '#{file}')" : "'#{file}'"
|
71
|
-
end
|
72
|
-
|
73
|
-
def self.bash(cmd)
|
74
|
-
%Q(bash <<EOF\n#{cmd}\nEOF\n)
|
75
|
-
end
|
76
|
-
|
77
|
-
def self.process_cmd_options(options = {})
|
78
|
-
add_dashes = Misc.process_options options, :add_option_dashes
|
79
|
-
|
80
|
-
string = ""
|
81
|
-
options.each do |option, value|
|
82
|
-
raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
|
83
|
-
#raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
|
84
|
-
value = value.gsub("'","\\'") if value.to_s.include? "'"
|
85
|
-
|
86
|
-
option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
|
87
|
-
|
88
|
-
case
|
89
|
-
when value.nil? || FalseClass === value
|
90
|
-
next
|
91
|
-
when TrueClass === value
|
92
|
-
string << "#{option} "
|
93
|
-
else
|
94
|
-
if option.to_s.chars.to_a.last == "="
|
95
|
-
string << "#{option}'#{value}' "
|
96
|
-
else
|
97
|
-
string << "#{option} '#{value}' "
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
string.strip
|
103
|
-
end
|
104
|
-
|
105
|
-
def self.cmd(tool, cmd = nil, options = {}, &block)
|
106
|
-
options, cmd = cmd, nil if Hash === cmd
|
107
|
-
|
108
|
-
options = Misc.add_defaults options, :stderr => Log::DEBUG
|
109
|
-
in_content = options.delete(:in)
|
110
|
-
stderr = options.delete(:stderr)
|
111
|
-
pipe = options.delete(:pipe)
|
112
|
-
post = options.delete(:post)
|
113
|
-
log = options.delete(:log)
|
114
|
-
no_fail = options.delete(:no_fail)
|
115
|
-
no_fail = options.delete(:nofail) if no_fail.nil?
|
116
|
-
no_wait = options.delete(:no_wait)
|
117
|
-
xvfb = options.delete(:xvfb)
|
118
|
-
bar = options.delete(:progress_bar)
|
119
|
-
save_stderr = options.delete(:save_stderr)
|
120
|
-
|
121
|
-
dont_close_in = options.delete(:dont_close_in)
|
122
|
-
|
123
|
-
log = true if log.nil?
|
124
|
-
|
125
|
-
if cmd.nil? && ! Symbol === tool
|
126
|
-
cmd = tool
|
127
|
-
else
|
128
|
-
tool = get_tool(tool)
|
129
|
-
if cmd.nil?
|
130
|
-
cmd = tool
|
131
|
-
else
|
132
|
-
cmd = tool + ' ' + cmd
|
133
|
-
end
|
134
|
-
|
135
|
-
end
|
136
|
-
|
137
|
-
case xvfb
|
138
|
-
when TrueClass
|
139
|
-
cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
|
140
|
-
when String
|
141
|
-
cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
|
142
|
-
when String
|
143
|
-
end
|
144
|
-
|
145
|
-
if stderr == true
|
146
|
-
stderr = Log::HIGH
|
147
|
-
end
|
148
|
-
|
149
|
-
cmd_options = process_cmd_options options
|
150
|
-
if cmd =~ /'\{opt\}'/
|
151
|
-
cmd.sub!('\'{opt}\'', cmd_options)
|
152
|
-
else
|
153
|
-
cmd << " " << cmd_options
|
154
|
-
end
|
155
|
-
|
156
|
-
in_content = StringIO.new in_content if String === in_content
|
157
|
-
|
158
|
-
sin, sout, serr, wait_thr = begin
|
159
|
-
Open3.popen3(ENV, cmd)
|
160
|
-
rescue
|
161
|
-
Log.warn $!.message
|
162
|
-
raise ProcessFailed, nil, cmd unless no_fail
|
163
|
-
return
|
164
|
-
end
|
165
|
-
pid = wait_thr.pid
|
166
|
-
|
167
|
-
Log.debug{"CMD: [#{pid}] #{cmd}" if log}
|
168
|
-
|
169
|
-
if in_content.respond_to?(:read)
|
170
|
-
in_thread = Thread.new(Thread.current) do |parent|
|
171
|
-
Thread.current.report_on_exception = false if no_fail
|
172
|
-
begin
|
173
|
-
begin
|
174
|
-
while c = in_content.readpartial(Misc::BLOCK_SIZE)
|
175
|
-
sin << c
|
176
|
-
end
|
177
|
-
rescue EOFError
|
178
|
-
end
|
179
|
-
sin.close unless sin.closed?
|
180
|
-
|
181
|
-
unless dont_close_in
|
182
|
-
in_content.close unless in_content.closed?
|
183
|
-
in_content.join if in_content.respond_to? :join
|
184
|
-
end
|
185
|
-
rescue
|
186
|
-
Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
|
187
|
-
raise $!
|
188
|
-
end
|
189
|
-
end
|
190
|
-
else
|
191
|
-
in_thread = nil
|
192
|
-
sin.close
|
193
|
-
end
|
194
|
-
|
195
|
-
pids = [pid]
|
196
|
-
|
197
|
-
if pipe
|
198
|
-
|
199
|
-
ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
|
200
|
-
|
201
|
-
if (Integer === stderr and log) || bar
|
202
|
-
err_thread = Thread.new do
|
203
|
-
while line = serr.gets
|
204
|
-
bar.process(line) if bar
|
205
|
-
sout.log = line
|
206
|
-
sout.std_err << line if save_stderr
|
207
|
-
Log.log "STDERR [#{pid}]: " + line, stderr if log
|
208
|
-
end
|
209
|
-
serr.close
|
210
|
-
end
|
211
|
-
else
|
212
|
-
err_thread = Misc.consume_stream(serr, true)
|
213
|
-
end
|
214
|
-
|
215
|
-
sout.threads = [in_thread, err_thread, wait_thr].compact
|
216
|
-
|
217
|
-
sout
|
218
|
-
else
|
219
|
-
|
220
|
-
if bar
|
221
|
-
err = ""
|
222
|
-
err_thread = Thread.new do
|
223
|
-
while not serr.eof?
|
224
|
-
line = serr.gets
|
225
|
-
bar.process(line)
|
226
|
-
err << line if Integer === stderr and log
|
227
|
-
end
|
228
|
-
serr.close
|
229
|
-
end
|
230
|
-
elsif log and Integer === stderr
|
231
|
-
err = ""
|
232
|
-
err_thread = Thread.new do
|
233
|
-
while not serr.eof?
|
234
|
-
err << serr.gets
|
235
|
-
end
|
236
|
-
serr.close
|
237
|
-
end
|
238
|
-
else
|
239
|
-
Misc.consume_stream(serr, true)
|
240
|
-
#serr.close
|
241
|
-
err_thread = nil
|
242
|
-
err = ""
|
243
|
-
end
|
244
|
-
|
245
|
-
ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail
|
246
|
-
|
247
|
-
out = StringIO.new sout.read
|
248
|
-
sout.close unless sout.closed?
|
249
|
-
|
250
|
-
status = wait_thr.value
|
251
|
-
if not status.success? and not no_fail
|
252
|
-
if !err.empty?
|
253
|
-
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
|
254
|
-
else
|
255
|
-
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
|
256
|
-
end
|
257
|
-
else
|
258
|
-
Log.log err, stderr if Integer === stderr and log
|
259
|
-
end
|
260
|
-
|
261
|
-
out
|
262
|
-
end
|
263
|
-
end
|
264
|
-
|
265
|
-
def self.cmd_pid(*args)
|
266
|
-
all_args = *args
|
267
|
-
|
268
|
-
bar = all_args.last[:progress_bar] if Hash === all_args.last
|
269
|
-
|
270
|
-
all_args << {} unless Hash === all_args.last
|
271
|
-
|
272
|
-
level = all_args.last[:log] || 0
|
273
|
-
level = 0 if TrueClass === level
|
274
|
-
level = 10 if FalseClass === level
|
275
|
-
level = level.to_i
|
276
|
-
|
277
|
-
all_args.last[:log] = true
|
278
|
-
all_args.last[:pipe] = true
|
279
|
-
|
280
|
-
io = cmd(*all_args)
|
281
|
-
pid = io.pids.first
|
282
|
-
|
283
|
-
line = "" if bar
|
284
|
-
starting = true
|
285
|
-
while c = io.getc
|
286
|
-
if starting
|
287
|
-
if pid
|
288
|
-
Log.logn "STDOUT [#{pid}]: ", level
|
289
|
-
else
|
290
|
-
Log.logn "STDOUT: ", level
|
291
|
-
end
|
292
|
-
starting = false
|
293
|
-
end
|
294
|
-
STDERR << c if Log.severity <= level
|
295
|
-
line << c if bar
|
296
|
-
if c == "\n"
|
297
|
-
bar.process(line) if bar
|
298
|
-
starting = true
|
299
|
-
line = "" if bar
|
300
|
-
end
|
301
|
-
end
|
302
|
-
begin
|
303
|
-
io.join
|
304
|
-
bar.remove if bar
|
305
|
-
rescue
|
306
|
-
bar.remove(true) if bar
|
307
|
-
raise $!
|
308
|
-
end
|
309
|
-
|
310
|
-
nil
|
311
|
-
end
|
312
|
-
|
313
|
-
def self.cmd_log(*args)
|
314
|
-
cmd_pid(*args)
|
315
|
-
nil
|
316
|
-
end
|
317
|
-
|
318
|
-
end
|
1
|
+
require_relative '../refactor'
|
2
|
+
Rbbt.require_instead 'scout/cmd'
|