rbbt-util 5.44.1 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +67 -90
  3. data/etc/app.d/base.rb +2 -2
  4. data/etc/app.d/semaphores.rb +3 -3
  5. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  6. data/lib/rbbt/annotations/refactor.rb +27 -0
  7. data/lib/rbbt/annotations/util.rb +282 -282
  8. data/lib/rbbt/annotations.rb +343 -320
  9. data/lib/rbbt/association/database.rb +200 -225
  10. data/lib/rbbt/association/index.rb +294 -291
  11. data/lib/rbbt/association/item.rb +227 -227
  12. data/lib/rbbt/association/open.rb +35 -34
  13. data/lib/rbbt/association/util.rb +0 -169
  14. data/lib/rbbt/association.rb +2 -4
  15. data/lib/rbbt/entity/identifiers.rb +119 -118
  16. data/lib/rbbt/entity/refactor.rb +12 -0
  17. data/lib/rbbt/entity.rb +319 -315
  18. data/lib/rbbt/hpc/batch.rb +72 -53
  19. data/lib/rbbt/hpc/lsf.rb +2 -2
  20. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  21. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  22. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  24. data/lib/rbbt/hpc/slurm.rb +18 -18
  25. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  26. data/lib/rbbt/knowledge_base/query.rb +2 -2
  27. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  28. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  29. data/lib/rbbt/knowledge_base.rb +1 -1
  30. data/lib/rbbt/monitor.rb +36 -25
  31. data/lib/rbbt/persist/refactor.rb +166 -0
  32. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  33. data/lib/rbbt/persist/tsv.rb +187 -185
  34. data/lib/rbbt/persist.rb +556 -551
  35. data/lib/rbbt/refactor.rb +20 -0
  36. data/lib/rbbt/resource/path/refactor.rb +178 -0
  37. data/lib/rbbt/resource/path.rb +317 -497
  38. data/lib/rbbt/resource/util.rb +0 -48
  39. data/lib/rbbt/resource.rb +3 -390
  40. data/lib/rbbt/tsv/accessor.rb +2 -838
  41. data/lib/rbbt/tsv/attach.rb +303 -299
  42. data/lib/rbbt/tsv/change_id.rb +244 -245
  43. data/lib/rbbt/tsv/csv.rb +87 -85
  44. data/lib/rbbt/tsv/dumper.rb +2 -100
  45. data/lib/rbbt/tsv/excel.rb +26 -24
  46. data/lib/rbbt/tsv/field_index.rb +4 -1
  47. data/lib/rbbt/tsv/filter.rb +3 -2
  48. data/lib/rbbt/tsv/index.rb +2 -284
  49. data/lib/rbbt/tsv/manipulate.rb +750 -747
  50. data/lib/rbbt/tsv/marshal.rb +3 -3
  51. data/lib/rbbt/tsv/matrix.rb +2 -2
  52. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  53. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  54. data/lib/rbbt/tsv/parser.rb +678 -678
  55. data/lib/rbbt/tsv/refactor.rb +195 -0
  56. data/lib/rbbt/tsv/stream.rb +253 -251
  57. data/lib/rbbt/tsv/util.rb +420 -420
  58. data/lib/rbbt/tsv.rb +210 -208
  59. data/lib/rbbt/util/R/eval.rb +4 -4
  60. data/lib/rbbt/util/R/plot.rb +62 -166
  61. data/lib/rbbt/util/R.rb +21 -18
  62. data/lib/rbbt/util/cmd.rb +2 -318
  63. data/lib/rbbt/util/color.rb +269 -269
  64. data/lib/rbbt/util/colorize.rb +89 -89
  65. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  66. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  67. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  68. data/lib/rbbt/util/config.rb +169 -167
  69. data/lib/rbbt/util/iruby.rb +20 -0
  70. data/lib/rbbt/util/log/progress/report.rb +241 -241
  71. data/lib/rbbt/util/log/progress/util.rb +99 -99
  72. data/lib/rbbt/util/log/progress.rb +102 -102
  73. data/lib/rbbt/util/log/refactor.rb +49 -0
  74. data/lib/rbbt/util/log.rb +486 -532
  75. data/lib/rbbt/util/migrate.rb +1 -1
  76. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  77. data/lib/rbbt/util/misc/development.rb +12 -11
  78. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  79. data/lib/rbbt/util/misc/format.rb +2 -230
  80. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  81. data/lib/rbbt/util/misc/inspect.rb +2 -476
  82. data/lib/rbbt/util/misc/lock.rb +109 -106
  83. data/lib/rbbt/util/misc/omics.rb +9 -1
  84. data/lib/rbbt/util/misc/pipes.rb +765 -793
  85. data/lib/rbbt/util/misc/refactor.rb +20 -0
  86. data/lib/rbbt/util/misc/ssw.rb +27 -17
  87. data/lib/rbbt/util/misc/system.rb +0 -15
  88. data/lib/rbbt/util/misc.rb +39 -20
  89. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  90. data/lib/rbbt/util/named_array.rb +3 -220
  91. data/lib/rbbt/util/open/refactor.rb +7 -0
  92. data/lib/rbbt/util/open.rb +3 -857
  93. data/lib/rbbt/util/procpath.rb +6 -6
  94. data/lib/rbbt/util/python/paths.rb +27 -0
  95. data/lib/rbbt/util/python/run.rb +115 -0
  96. data/lib/rbbt/util/python/script.rb +110 -0
  97. data/lib/rbbt/util/python/util.rb +3 -3
  98. data/lib/rbbt/util/python.rb +22 -81
  99. data/lib/rbbt/util/semaphore.rb +152 -148
  100. data/lib/rbbt/util/simpleopt.rb +9 -8
  101. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  102. data/lib/rbbt/util/ssh.rb +122 -118
  103. data/lib/rbbt/util/tar.rb +117 -115
  104. data/lib/rbbt/util/tmpfile.rb +69 -67
  105. data/lib/rbbt/util/version.rb +2 -0
  106. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  107. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  108. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  109. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  110. data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
  111. data/lib/rbbt/workflow/refactor.rb +153 -0
  112. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  113. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  114. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  115. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  116. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  117. data/lib/rbbt/workflow/step/run.rb +766 -766
  118. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  119. data/lib/rbbt/workflow/step.rb +2 -362
  120. data/lib/rbbt/workflow/task.rb +118 -118
  121. data/lib/rbbt/workflow/usage.rb +289 -287
  122. data/lib/rbbt/workflow/util/archive.rb +6 -5
  123. data/lib/rbbt/workflow/util/data.rb +1 -1
  124. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  125. data/lib/rbbt/workflow/util/trace.rb +79 -44
  126. data/lib/rbbt/workflow.rb +4 -882
  127. data/lib/rbbt-util.rb +21 -13
  128. data/lib/rbbt.rb +16 -3
  129. data/python/rbbt/__init__.py +19 -1
  130. data/share/Rlib/plot.R +37 -37
  131. data/share/Rlib/svg.R +22 -5
  132. data/share/install/software/lib/install_helpers +1 -1
  133. data/share/rbbt_commands/hpc/list +2 -3
  134. data/share/rbbt_commands/hpc/orchestrate +4 -4
  135. data/share/rbbt_commands/hpc/tail +2 -0
  136. data/share/rbbt_commands/hpc/task +10 -7
  137. data/share/rbbt_commands/lsf/list +2 -3
  138. data/share/rbbt_commands/lsf/orchestrate +4 -4
  139. data/share/rbbt_commands/lsf/tail +2 -0
  140. data/share/rbbt_commands/lsf/task +10 -7
  141. data/share/rbbt_commands/migrate +1 -1
  142. data/share/rbbt_commands/pbs/list +2 -3
  143. data/share/rbbt_commands/pbs/orchestrate +4 -4
  144. data/share/rbbt_commands/pbs/tail +2 -0
  145. data/share/rbbt_commands/pbs/task +10 -7
  146. data/share/rbbt_commands/resource/produce +8 -1
  147. data/share/rbbt_commands/slurm/list +2 -3
  148. data/share/rbbt_commands/slurm/orchestrate +4 -4
  149. data/share/rbbt_commands/slurm/tail +2 -0
  150. data/share/rbbt_commands/slurm/task +10 -7
  151. data/share/rbbt_commands/system/clean +5 -5
  152. data/share/rbbt_commands/system/status +5 -5
  153. data/share/rbbt_commands/tsv/get +2 -3
  154. data/share/rbbt_commands/tsv/info +10 -13
  155. data/share/rbbt_commands/tsv/keys +18 -14
  156. data/share/rbbt_commands/tsv/slice +2 -2
  157. data/share/rbbt_commands/tsv/transpose +6 -2
  158. data/share/rbbt_commands/workflow/info +20 -24
  159. data/share/rbbt_commands/workflow/list +1 -1
  160. data/share/rbbt_commands/workflow/prov +20 -13
  161. data/share/rbbt_commands/workflow/server +11 -1
  162. data/share/rbbt_commands/workflow/task +76 -71
  163. data/share/rbbt_commands/workflow/write_info +26 -9
  164. data/share/software/opt/ssw/ssw.c +861 -0
  165. data/share/software/opt/ssw/ssw.h +130 -0
  166. data/share/workflow_config.ru +3 -3
  167. metadata +40 -2
data/lib/rbbt/util/cmd.rb CHANGED
@@ -1,318 +1,2 @@
1
- require 'rbbt/util/log'
2
- require 'stringio'
3
- require 'open3'
4
- require 'rbbt/util/misc/indiferent_hash'
5
-
6
- module CMD
7
-
8
- TOOLS = IndiferentHash.setup({})
9
- def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
10
- TOOLS[tool] = [claim, test, block, cmd]
11
- end
12
-
13
- def self.conda(tool, env = nil, channel = 'bioconda')
14
- if env
15
- CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
16
- else
17
- CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
18
- end
19
- end
20
-
21
- def self.get_tool(tool)
22
- return tool.to_s unless TOOLS[tool]
23
-
24
- @@init_cmd_tool ||= IndiferentHash.setup({})
25
-
26
- claim, test, block, cmd = TOOLS[tool]
27
- cmd = tool.to_s if cmd.nil?
28
-
29
- if !@@init_cmd_tool[tool]
30
-
31
- begin
32
- if test
33
- CMD.cmd(test + " ")
34
- else
35
- CMD.cmd("#{cmd} --help")
36
- end
37
- rescue
38
- if claim
39
- claim.produce
40
- else
41
- block.call
42
- end
43
- end
44
- version_txt = ""
45
- version = nil
46
- ["--version", "-version", "--help", ""].each do |f|
47
- begin
48
- version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
49
- version = Misc.scan_version_text(version_txt, tool)
50
- break if version
51
- rescue
52
- Log.exception $!
53
- end
54
- end
55
-
56
- @@init_cmd_tool[tool] = version || true
57
-
58
- return cmd if cmd
59
- end
60
-
61
- cmd
62
- end
63
-
64
- def self.versions
65
- return {} unless defined? @@init_cmd_tool
66
- @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
67
- end
68
-
69
- def self.gzip_pipe(file)
70
- Open.gzip?(file) ? "<(gunzip -c '#{file}')" : "'#{file}'"
71
- end
72
-
73
- def self.bash(cmd)
74
- %Q(bash <<EOF\n#{cmd}\nEOF\n)
75
- end
76
-
77
- def self.process_cmd_options(options = {})
78
- add_dashes = Misc.process_options options, :add_option_dashes
79
-
80
- string = ""
81
- options.each do |option, value|
82
- raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
83
- #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
84
- value = value.gsub("'","\\'") if value.to_s.include? "'"
85
-
86
- option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
87
-
88
- case
89
- when value.nil? || FalseClass === value
90
- next
91
- when TrueClass === value
92
- string << "#{option} "
93
- else
94
- if option.to_s.chars.to_a.last == "="
95
- string << "#{option}'#{value}' "
96
- else
97
- string << "#{option} '#{value}' "
98
- end
99
- end
100
- end
101
-
102
- string.strip
103
- end
104
-
105
- def self.cmd(tool, cmd = nil, options = {}, &block)
106
- options, cmd = cmd, nil if Hash === cmd
107
-
108
- options = Misc.add_defaults options, :stderr => Log::DEBUG
109
- in_content = options.delete(:in)
110
- stderr = options.delete(:stderr)
111
- pipe = options.delete(:pipe)
112
- post = options.delete(:post)
113
- log = options.delete(:log)
114
- no_fail = options.delete(:no_fail)
115
- no_fail = options.delete(:nofail) if no_fail.nil?
116
- no_wait = options.delete(:no_wait)
117
- xvfb = options.delete(:xvfb)
118
- bar = options.delete(:progress_bar)
119
- save_stderr = options.delete(:save_stderr)
120
-
121
- dont_close_in = options.delete(:dont_close_in)
122
-
123
- log = true if log.nil?
124
-
125
- if cmd.nil? && ! Symbol === tool
126
- cmd = tool
127
- else
128
- tool = get_tool(tool)
129
- if cmd.nil?
130
- cmd = tool
131
- else
132
- cmd = tool + ' ' + cmd
133
- end
134
-
135
- end
136
-
137
- case xvfb
138
- when TrueClass
139
- cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
140
- when String
141
- cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
142
- when String
143
- end
144
-
145
- if stderr == true
146
- stderr = Log::HIGH
147
- end
148
-
149
- cmd_options = process_cmd_options options
150
- if cmd =~ /'\{opt\}'/
151
- cmd.sub!('\'{opt}\'', cmd_options)
152
- else
153
- cmd << " " << cmd_options
154
- end
155
-
156
- in_content = StringIO.new in_content if String === in_content
157
-
158
- sin, sout, serr, wait_thr = begin
159
- Open3.popen3(ENV, cmd)
160
- rescue
161
- Log.warn $!.message
162
- raise ProcessFailed, nil, cmd unless no_fail
163
- return
164
- end
165
- pid = wait_thr.pid
166
-
167
- Log.debug{"CMD: [#{pid}] #{cmd}" if log}
168
-
169
- if in_content.respond_to?(:read)
170
- in_thread = Thread.new(Thread.current) do |parent|
171
- Thread.current.report_on_exception = false if no_fail
172
- begin
173
- begin
174
- while c = in_content.readpartial(Misc::BLOCK_SIZE)
175
- sin << c
176
- end
177
- rescue EOFError
178
- end
179
- sin.close unless sin.closed?
180
-
181
- unless dont_close_in
182
- in_content.close unless in_content.closed?
183
- in_content.join if in_content.respond_to? :join
184
- end
185
- rescue
186
- Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
187
- raise $!
188
- end
189
- end
190
- else
191
- in_thread = nil
192
- sin.close
193
- end
194
-
195
- pids = [pid]
196
-
197
- if pipe
198
-
199
- ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
200
-
201
- if (Integer === stderr and log) || bar
202
- err_thread = Thread.new do
203
- while line = serr.gets
204
- bar.process(line) if bar
205
- sout.log = line
206
- sout.std_err << line if save_stderr
207
- Log.log "STDERR [#{pid}]: " + line, stderr if log
208
- end
209
- serr.close
210
- end
211
- else
212
- err_thread = Misc.consume_stream(serr, true)
213
- end
214
-
215
- sout.threads = [in_thread, err_thread, wait_thr].compact
216
-
217
- sout
218
- else
219
-
220
- if bar
221
- err = ""
222
- err_thread = Thread.new do
223
- while not serr.eof?
224
- line = serr.gets
225
- bar.process(line)
226
- err << line if Integer === stderr and log
227
- end
228
- serr.close
229
- end
230
- elsif log and Integer === stderr
231
- err = ""
232
- err_thread = Thread.new do
233
- while not serr.eof?
234
- err << serr.gets
235
- end
236
- serr.close
237
- end
238
- else
239
- Misc.consume_stream(serr, true)
240
- #serr.close
241
- err_thread = nil
242
- err = ""
243
- end
244
-
245
- ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail
246
-
247
- out = StringIO.new sout.read
248
- sout.close unless sout.closed?
249
-
250
- status = wait_thr.value
251
- if not status.success? and not no_fail
252
- if !err.empty?
253
- raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
254
- else
255
- raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
256
- end
257
- else
258
- Log.log err, stderr if Integer === stderr and log
259
- end
260
-
261
- out
262
- end
263
- end
264
-
265
- def self.cmd_pid(*args)
266
- all_args = *args
267
-
268
- bar = all_args.last[:progress_bar] if Hash === all_args.last
269
-
270
- all_args << {} unless Hash === all_args.last
271
-
272
- level = all_args.last[:log] || 0
273
- level = 0 if TrueClass === level
274
- level = 10 if FalseClass === level
275
- level = level.to_i
276
-
277
- all_args.last[:log] = true
278
- all_args.last[:pipe] = true
279
-
280
- io = cmd(*all_args)
281
- pid = io.pids.first
282
-
283
- line = "" if bar
284
- starting = true
285
- while c = io.getc
286
- if starting
287
- if pid
288
- Log.logn "STDOUT [#{pid}]: ", level
289
- else
290
- Log.logn "STDOUT: ", level
291
- end
292
- starting = false
293
- end
294
- STDERR << c if Log.severity <= level
295
- line << c if bar
296
- if c == "\n"
297
- bar.process(line) if bar
298
- starting = true
299
- line = "" if bar
300
- end
301
- end
302
- begin
303
- io.join
304
- bar.remove if bar
305
- rescue
306
- bar.remove(true) if bar
307
- raise $!
308
- end
309
-
310
- nil
311
- end
312
-
313
- def self.cmd_log(*args)
314
- cmd_pid(*args)
315
- nil
316
- end
317
-
318
- end
1
+ require_relative '../refactor'
2
+ Rbbt.require_instead 'scout/cmd'