rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
data/lib/rbbt/util/cmd.rb CHANGED
@@ -1,318 +1,2 @@
1
- require 'rbbt/util/log'
2
- require 'stringio'
3
- require 'open3'
4
- require 'rbbt/util/misc/indiferent_hash'
5
-
6
- module CMD
7
-
8
- TOOLS = IndiferentHash.setup({})
9
- def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
10
- TOOLS[tool] = [claim, test, block, cmd]
11
- end
12
-
13
- def self.conda(tool, env = nil, channel = 'bioconda')
14
- if env
15
- CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
16
- else
17
- CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
18
- end
19
- end
20
-
21
- def self.get_tool(tool)
22
- return tool.to_s unless TOOLS[tool]
23
-
24
- @@init_cmd_tool ||= IndiferentHash.setup({})
25
-
26
- claim, test, block, cmd = TOOLS[tool]
27
- cmd = tool.to_s if cmd.nil?
28
-
29
- if !@@init_cmd_tool[tool]
30
-
31
- begin
32
- if test
33
- CMD.cmd(test + " ")
34
- else
35
- CMD.cmd("#{cmd} --help")
36
- end
37
- rescue
38
- if claim
39
- claim.produce
40
- else
41
- block.call
42
- end
43
- end
44
- version_txt = ""
45
- version = nil
46
- ["--version", "-version", "--help", ""].each do |f|
47
- begin
48
- version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
49
- version = Misc.scan_version_text(version_txt, tool)
50
- break if version
51
- rescue
52
- Log.exception $!
53
- end
54
- end
55
-
56
- @@init_cmd_tool[tool] = version || true
57
-
58
- return cmd if cmd
59
- end
60
-
61
- cmd
62
- end
63
-
64
- def self.versions
65
- return {} unless defined? @@init_cmd_tool
66
- @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
67
- end
68
-
69
- def self.gzip_pipe(file)
70
- Open.gzip?(file) ? "<(gunzip -c '#{file}')" : "'#{file}'"
71
- end
72
-
73
- def self.bash(cmd)
74
- %Q(bash <<EOF\n#{cmd}\nEOF\n)
75
- end
76
-
77
- def self.process_cmd_options(options = {})
78
- add_dashes = Misc.process_options options, :add_option_dashes
79
-
80
- string = ""
81
- options.each do |option, value|
82
- raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
83
- #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
84
- value = value.gsub("'","\\'") if value.to_s.include? "'"
85
-
86
- option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
87
-
88
- case
89
- when value.nil? || FalseClass === value
90
- next
91
- when TrueClass === value
92
- string << "#{option} "
93
- else
94
- if option.to_s.chars.to_a.last == "="
95
- string << "#{option}'#{value}' "
96
- else
97
- string << "#{option} '#{value}' "
98
- end
99
- end
100
- end
101
-
102
- string.strip
103
- end
104
-
105
- def self.cmd(tool, cmd = nil, options = {}, &block)
106
- options, cmd = cmd, nil if Hash === cmd
107
-
108
- options = Misc.add_defaults options, :stderr => Log::DEBUG
109
- in_content = options.delete(:in)
110
- stderr = options.delete(:stderr)
111
- pipe = options.delete(:pipe)
112
- post = options.delete(:post)
113
- log = options.delete(:log)
114
- no_fail = options.delete(:no_fail)
115
- no_fail = options.delete(:nofail) if no_fail.nil?
116
- no_wait = options.delete(:no_wait)
117
- xvfb = options.delete(:xvfb)
118
- bar = options.delete(:progress_bar)
119
- save_stderr = options.delete(:save_stderr)
120
-
121
- dont_close_in = options.delete(:dont_close_in)
122
-
123
- log = true if log.nil?
124
-
125
- if cmd.nil? && ! Symbol === tool
126
- cmd = tool
127
- else
128
- tool = get_tool(tool)
129
- if cmd.nil?
130
- cmd = tool
131
- else
132
- cmd = tool + ' ' + cmd
133
- end
134
-
135
- end
136
-
137
- case xvfb
138
- when TrueClass
139
- cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
140
- when String
141
- cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
142
- when String
143
- end
144
-
145
- if stderr == true
146
- stderr = Log::HIGH
147
- end
148
-
149
- cmd_options = process_cmd_options options
150
- if cmd =~ /'\{opt\}'/
151
- cmd.sub!('\'{opt}\'', cmd_options)
152
- else
153
- cmd << " " << cmd_options
154
- end
155
-
156
- in_content = StringIO.new in_content if String === in_content
157
-
158
- sin, sout, serr, wait_thr = begin
159
- Open3.popen3(ENV, cmd)
160
- rescue
161
- Log.warn $!.message
162
- raise ProcessFailed, nil, cmd unless no_fail
163
- return
164
- end
165
- pid = wait_thr.pid
166
-
167
- Log.debug{"CMD: [#{pid}] #{cmd}" if log}
168
-
169
- if in_content.respond_to?(:read)
170
- in_thread = Thread.new(Thread.current) do |parent|
171
- Thread.current.report_on_exception = false if no_fail
172
- begin
173
- begin
174
- while c = in_content.readpartial(Misc::BLOCK_SIZE)
175
- sin << c
176
- end
177
- rescue EOFError
178
- end
179
- sin.close unless sin.closed?
180
-
181
- unless dont_close_in
182
- in_content.close unless in_content.closed?
183
- in_content.join if in_content.respond_to? :join
184
- end
185
- rescue
186
- Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
187
- raise $!
188
- end
189
- end
190
- else
191
- in_thread = nil
192
- sin.close
193
- end
194
-
195
- pids = [pid]
196
-
197
- if pipe
198
-
199
- ConcurrentStream.setup sout, :pids => pids, :autojoin => no_wait, :no_fail => no_fail
200
-
201
- if (Integer === stderr and log) || bar
202
- err_thread = Thread.new do
203
- while line = serr.gets
204
- bar.process(line) if bar
205
- sout.log = line
206
- sout.std_err << line if save_stderr
207
- Log.log "STDERR [#{pid}]: " + line, stderr if log
208
- end
209
- serr.close
210
- end
211
- else
212
- err_thread = Misc.consume_stream(serr, true)
213
- end
214
-
215
- sout.threads = [in_thread, err_thread, wait_thr].compact
216
-
217
- sout
218
- else
219
-
220
- if bar
221
- err = ""
222
- err_thread = Thread.new do
223
- while not serr.eof?
224
- line = serr.gets
225
- bar.process(line)
226
- err << line if Integer === stderr and log
227
- end
228
- serr.close
229
- end
230
- elsif log and Integer === stderr
231
- err = ""
232
- err_thread = Thread.new do
233
- while not serr.eof?
234
- err << serr.gets
235
- end
236
- serr.close
237
- end
238
- else
239
- Misc.consume_stream(serr, true)
240
- #serr.close
241
- err_thread = nil
242
- err = ""
243
- end
244
-
245
- ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail
246
-
247
- out = StringIO.new sout.read
248
- sout.close unless sout.closed?
249
-
250
- status = wait_thr.value
251
- if not status.success? and not no_fail
252
- if !err.empty?
253
- raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
254
- else
255
- raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
256
- end
257
- else
258
- Log.log err, stderr if Integer === stderr and log
259
- end
260
-
261
- out
262
- end
263
- end
264
-
265
- def self.cmd_pid(*args)
266
- all_args = *args
267
-
268
- bar = all_args.last[:progress_bar] if Hash === all_args.last
269
-
270
- all_args << {} unless Hash === all_args.last
271
-
272
- level = all_args.last[:log] || 0
273
- level = 0 if TrueClass === level
274
- level = 10 if FalseClass === level
275
- level = level.to_i
276
-
277
- all_args.last[:log] = true
278
- all_args.last[:pipe] = true
279
-
280
- io = cmd(*all_args)
281
- pid = io.pids.first
282
-
283
- line = "" if bar
284
- starting = true
285
- while c = io.getc
286
- if starting
287
- if pid
288
- Log.logn "STDOUT [#{pid}]: ", level
289
- else
290
- Log.logn "STDOUT: ", level
291
- end
292
- starting = false
293
- end
294
- STDERR << c if Log.severity <= level
295
- line << c if bar
296
- if c == "\n"
297
- bar.process(line) if bar
298
- starting = true
299
- line = "" if bar
300
- end
301
- end
302
- begin
303
- io.join
304
- bar.remove if bar
305
- rescue
306
- bar.remove(true) if bar
307
- raise $!
308
- end
309
-
310
- nil
311
- end
312
-
313
- def self.cmd_log(*args)
314
- cmd_pid(*args)
315
- nil
316
- end
317
-
318
- end
1
+ require_relative '../refactor'
2
+ Rbbt.require_instead 'scout/cmd'