rbbt-util 5.44.1 → 6.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/bin/rbbt +67 -90
  4. data/bin/rbbt_exec.rb +2 -2
  5. data/etc/app.d/base.rb +2 -2
  6. data/etc/app.d/semaphores.rb +3 -3
  7. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  8. data/lib/rbbt/annotations/refactor.rb +27 -0
  9. data/lib/rbbt/annotations/util.rb +282 -282
  10. data/lib/rbbt/annotations.rb +343 -320
  11. data/lib/rbbt/association/database.rb +200 -225
  12. data/lib/rbbt/association/index.rb +294 -291
  13. data/lib/rbbt/association/item.rb +227 -227
  14. data/lib/rbbt/association/open.rb +35 -34
  15. data/lib/rbbt/association/util.rb +0 -169
  16. data/lib/rbbt/association.rb +2 -4
  17. data/lib/rbbt/entity/identifiers.rb +119 -118
  18. data/lib/rbbt/entity/refactor.rb +12 -0
  19. data/lib/rbbt/entity.rb +319 -315
  20. data/lib/rbbt/hpc/batch.rb +72 -53
  21. data/lib/rbbt/hpc/lsf.rb +2 -2
  22. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  24. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  25. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  26. data/lib/rbbt/hpc/slurm.rb +18 -18
  27. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  28. data/lib/rbbt/knowledge_base/query.rb +2 -2
  29. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  30. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  31. data/lib/rbbt/knowledge_base.rb +1 -1
  32. data/lib/rbbt/monitor.rb +36 -25
  33. data/lib/rbbt/persist/refactor.rb +166 -0
  34. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  35. data/lib/rbbt/persist/tsv.rb +187 -185
  36. data/lib/rbbt/persist.rb +556 -551
  37. data/lib/rbbt/refactor.rb +20 -0
  38. data/lib/rbbt/resource/path/refactor.rb +178 -0
  39. data/lib/rbbt/resource/path.rb +317 -497
  40. data/lib/rbbt/resource/util.rb +0 -48
  41. data/lib/rbbt/resource.rb +3 -390
  42. data/lib/rbbt/tsv/accessor.rb +2 -838
  43. data/lib/rbbt/tsv/attach.rb +303 -299
  44. data/lib/rbbt/tsv/change_id.rb +244 -245
  45. data/lib/rbbt/tsv/csv.rb +87 -85
  46. data/lib/rbbt/tsv/dumper.rb +2 -100
  47. data/lib/rbbt/tsv/excel.rb +26 -24
  48. data/lib/rbbt/tsv/field_index.rb +4 -1
  49. data/lib/rbbt/tsv/filter.rb +3 -2
  50. data/lib/rbbt/tsv/index.rb +2 -284
  51. data/lib/rbbt/tsv/manipulate.rb +750 -747
  52. data/lib/rbbt/tsv/marshal.rb +3 -3
  53. data/lib/rbbt/tsv/matrix.rb +2 -2
  54. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  55. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  56. data/lib/rbbt/tsv/parser.rb +678 -678
  57. data/lib/rbbt/tsv/refactor.rb +195 -0
  58. data/lib/rbbt/tsv/stream.rb +253 -251
  59. data/lib/rbbt/tsv/util.rb +420 -420
  60. data/lib/rbbt/tsv.rb +210 -208
  61. data/lib/rbbt/util/R/eval.rb +4 -4
  62. data/lib/rbbt/util/R/plot.rb +62 -166
  63. data/lib/rbbt/util/R.rb +21 -18
  64. data/lib/rbbt/util/cmd.rb +2 -318
  65. data/lib/rbbt/util/color.rb +269 -269
  66. data/lib/rbbt/util/colorize.rb +89 -89
  67. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  68. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  69. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  70. data/lib/rbbt/util/config.rb +169 -167
  71. data/lib/rbbt/util/filecache.rb +1 -1
  72. data/lib/rbbt/util/iruby.rb +20 -0
  73. data/lib/rbbt/util/log/progress/report.rb +241 -241
  74. data/lib/rbbt/util/log/progress/util.rb +99 -99
  75. data/lib/rbbt/util/log/progress.rb +102 -102
  76. data/lib/rbbt/util/log/refactor.rb +49 -0
  77. data/lib/rbbt/util/log.rb +486 -532
  78. data/lib/rbbt/util/migrate.rb +2 -2
  79. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  80. data/lib/rbbt/util/misc/development.rb +12 -11
  81. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  82. data/lib/rbbt/util/misc/format.rb +2 -230
  83. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  84. data/lib/rbbt/util/misc/inspect.rb +2 -476
  85. data/lib/rbbt/util/misc/lock.rb +109 -106
  86. data/lib/rbbt/util/misc/omics.rb +9 -1
  87. data/lib/rbbt/util/misc/pipes.rb +765 -793
  88. data/lib/rbbt/util/misc/refactor.rb +20 -0
  89. data/lib/rbbt/util/misc/ssw.rb +27 -17
  90. data/lib/rbbt/util/misc/system.rb +92 -105
  91. data/lib/rbbt/util/misc.rb +39 -20
  92. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  93. data/lib/rbbt/util/named_array.rb +3 -220
  94. data/lib/rbbt/util/open/refactor.rb +7 -0
  95. data/lib/rbbt/util/open.rb +3 -857
  96. data/lib/rbbt/util/procpath.rb +6 -6
  97. data/lib/rbbt/util/python/paths.rb +27 -0
  98. data/lib/rbbt/util/python/run.rb +115 -0
  99. data/lib/rbbt/util/python/script.rb +110 -0
  100. data/lib/rbbt/util/python/util.rb +3 -3
  101. data/lib/rbbt/util/python.rb +22 -81
  102. data/lib/rbbt/util/semaphore.rb +152 -148
  103. data/lib/rbbt/util/simpleopt.rb +9 -8
  104. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  105. data/lib/rbbt/util/ssh.rb +122 -118
  106. data/lib/rbbt/util/tar.rb +117 -115
  107. data/lib/rbbt/util/tmpfile.rb +69 -67
  108. data/lib/rbbt/util/version.rb +2 -0
  109. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  110. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  111. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  112. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  113. data/lib/rbbt/workflow/refactor/task_info.rb +66 -0
  114. data/lib/rbbt/workflow/refactor.rb +150 -0
  115. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +1 -2
  116. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  117. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  118. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  119. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  120. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  121. data/lib/rbbt/workflow/step/run.rb +766 -766
  122. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  123. data/lib/rbbt/workflow/step.rb +2 -362
  124. data/lib/rbbt/workflow/task.rb +118 -118
  125. data/lib/rbbt/workflow/usage.rb +289 -287
  126. data/lib/rbbt/workflow/util/archive.rb +6 -5
  127. data/lib/rbbt/workflow/util/data.rb +1 -1
  128. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  129. data/lib/rbbt/workflow/util/trace.rb +79 -44
  130. data/lib/rbbt/workflow.rb +4 -882
  131. data/lib/rbbt-util.rb +21 -13
  132. data/lib/rbbt.rb +16 -3
  133. data/python/rbbt/__init__.py +96 -4
  134. data/python/rbbt/workflow/remote.py +104 -0
  135. data/python/rbbt/workflow.py +64 -0
  136. data/python/test.py +10 -0
  137. data/share/Rlib/plot.R +37 -37
  138. data/share/Rlib/svg.R +22 -5
  139. data/share/install/software/lib/install_helpers +1 -1
  140. data/share/rbbt_commands/hpc/list +2 -3
  141. data/share/rbbt_commands/hpc/orchestrate +4 -4
  142. data/share/rbbt_commands/hpc/tail +2 -0
  143. data/share/rbbt_commands/hpc/task +10 -7
  144. data/share/rbbt_commands/lsf/list +2 -3
  145. data/share/rbbt_commands/lsf/orchestrate +4 -4
  146. data/share/rbbt_commands/lsf/tail +2 -0
  147. data/share/rbbt_commands/lsf/task +10 -7
  148. data/share/rbbt_commands/migrate +1 -1
  149. data/share/rbbt_commands/pbs/list +2 -3
  150. data/share/rbbt_commands/pbs/orchestrate +4 -4
  151. data/share/rbbt_commands/pbs/tail +2 -0
  152. data/share/rbbt_commands/pbs/task +10 -7
  153. data/share/rbbt_commands/resource/produce +8 -1
  154. data/share/rbbt_commands/slurm/list +2 -3
  155. data/share/rbbt_commands/slurm/orchestrate +4 -4
  156. data/share/rbbt_commands/slurm/tail +2 -0
  157. data/share/rbbt_commands/slurm/task +10 -7
  158. data/share/rbbt_commands/system/clean +5 -5
  159. data/share/rbbt_commands/system/status +5 -5
  160. data/share/rbbt_commands/tsv/get +2 -3
  161. data/share/rbbt_commands/tsv/info +10 -13
  162. data/share/rbbt_commands/tsv/keys +18 -14
  163. data/share/rbbt_commands/tsv/slice +2 -2
  164. data/share/rbbt_commands/tsv/transpose +6 -2
  165. data/share/rbbt_commands/workflow/info +20 -24
  166. data/share/rbbt_commands/workflow/list +1 -1
  167. data/share/rbbt_commands/workflow/prov +20 -13
  168. data/share/rbbt_commands/workflow/retry +43 -0
  169. data/share/rbbt_commands/workflow/server +12 -2
  170. data/share/rbbt_commands/workflow/task +80 -73
  171. data/share/rbbt_commands/workflow/write_info +26 -9
  172. data/share/software/opt/ssw/ssw.c +861 -0
  173. data/share/software/opt/ssw/ssw.h +130 -0
  174. data/share/workflow_config.ru +3 -3
  175. metadata +45 -6
data/lib/rbbt/util/log.rb CHANGED
@@ -1,532 +1,486 @@
1
- require 'term/ansicolor'
2
- require 'rbbt/util/color'
3
- require 'rbbt/util/log/progress'
4
-
5
- class MockMutex
6
- def synchronize
7
- yield
8
- end
9
- end
10
-
11
- module Log
12
- extend Term::ANSIColor
13
-
14
-
15
- #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
16
- LOG_MUTEX = MockMutex.new
17
- #LOG_MUTEX = Mutex.new
18
-
19
- SEVERITY_NAMES ||= begin
20
- names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
21
- names.each_with_index do |name,i|
22
- eval "#{ name } = #{ i }"
23
- end
24
- names
25
- end
26
-
27
- def self.terminal_width
28
- 80
29
- end
30
-
31
- def self.compact
32
- true
33
- end
34
-
35
-
36
- def self.last_caller(stack)
37
- line = nil
38
- pos ||= 0
39
- while line.nil? or line =~ /util\/log\.rb/ and stack.any?
40
- line = stack.shift
41
- end
42
- line ||= caller.first
43
- line.gsub('`', "'")
44
- end
45
-
46
-
47
- def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
48
- sout, sin = Misc.pipe
49
- soute, sine = Misc.pipe
50
- backup_stderr = STDERR.dup
51
- backup_stdout = STDOUT.dup
52
- old_logfile = Log.logfile
53
- Log.logfile(backup_stderr)
54
-
55
- severity_err ||= severity
56
- th_log = Thread.new do
57
- while line = sout.gets
58
- Log.logn "#{msg}: " + line, severity
59
- end
60
- end
61
-
62
- th_loge = Thread.new do
63
- while line = soute.gets
64
- Log.logn "#{msge}: " + line, severity_err
65
- end
66
- end
67
-
68
- begin
69
- STDOUT.reopen(sin)
70
- STDERR.reopen(sine)
71
- yield
72
- ensure
73
- STDERR.reopen backup_stderr
74
- STDOUT.reopen backup_stdout
75
- sin.close
76
- sine.close
77
- th_log.join
78
- th_loge.join
79
- backup_stdout.close
80
- backup_stderr.close
81
- Log.logfile = old_logfile
82
- end
83
- end
84
-
85
- def self.trap_stderr(msg = "STDERR", severity = 0)
86
- sout, sin = Misc.pipe
87
- backup_stderr = STDERR.dup
88
- old_logfile = Log.logfile
89
- Log.logfile(backup_stderr)
90
-
91
- th_log = Thread.new do
92
- while line = sout.gets
93
- Log.logn "#{msg}: " + line, severity
94
- end
95
- end
96
-
97
- begin
98
- STDERR.reopen(sin)
99
- yield
100
- sin.close
101
- ensure
102
- STDERR.reopen backup_stderr
103
- th_log.join
104
- backup_stderr.close
105
- Log.logfile = old_logfile
106
- end
107
- end
108
-
109
-
110
- def self._ignore_stderr
111
- begin
112
- File.open('/dev/null', 'w') do |f|
113
- backup_stderr = STDERR.dup
114
- STDERR.reopen(f)
115
- begin
116
- yield
117
- ensure
118
- STDERR.reopen backup_stderr
119
- backup_stderr.close
120
- end
121
- end
122
- rescue Errno::ENOENT
123
- yield
124
- end
125
- end
126
-
127
-
128
- def self.ignore_stderr(&block)
129
- LOG_MUTEX.synchronize do
130
- _ignore_stderr &block
131
- end
132
- end
133
-
134
- def self._ignore_stdout
135
- begin
136
- File.open('/dev/null', 'w') do |f|
137
- backup_stdout = STDOUT.dup
138
- STDOUT.reopen(f)
139
- begin
140
- yield
141
- ensure
142
- STDOUT.reopen backup_stdout
143
- backup_stdout.close
144
- end
145
- end
146
- rescue Errno::ENOENT
147
- yield
148
- end
149
- end
150
-
151
-
152
- def self.ignore_stdout(&block)
153
- LOG_MUTEX.synchronize do
154
- _ignore_stdout &block
155
- end
156
- end
157
-
158
- def self.get_level(level)
159
- case level
160
- when Numeric
161
- level.to_i
162
- when String
163
- begin
164
- Log.const_get(level.upcase)
165
- rescue
166
- Log.exception $!
167
- end
168
- when Symbol
169
- get_level(level.to_s)
170
- end || 0
171
- end
172
-
173
- class << self
174
- attr_accessor :logfile, :severity, :nocolor, :tty_size
175
- end
176
-
177
- self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
178
-
179
- self.ignore_stderr do
180
- self.tty_size = begin
181
- require "highline/system_extensions.rb"
182
- HighLine::SystemExtensions.terminal_size.first
183
- rescue Exception
184
- nil
185
- end
186
- end
187
-
188
- def self.with_severity(level)
189
- orig = Log.severity
190
- begin
191
- Log.severity = level
192
- yield
193
- ensure
194
- Log.severity = orig
195
- end
196
- end
197
-
198
- def self.logfile(file=nil)
199
- if file.nil?
200
- @logfile ||= nil
201
- else
202
- case file
203
- when String
204
- @logfile = File.open(file, :mode => 'a')
205
- when IO, File
206
- @logfile = file
207
- else
208
- raise "Unkown logfile format: #{file.inspect}"
209
- end
210
- end
211
- end
212
-
213
- WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
214
-
215
- SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
216
- HIGHLIGHT = "\033[1m"
217
-
218
- def self.uncolor(str)
219
- "" << Term::ANSIColor.uncolor(str)
220
- end
221
-
222
- def self.reset_color
223
- reset
224
- end
225
-
226
- def self.color(severity, str = nil, reset = false)
227
- return str.dup || "" if nocolor
228
- color = reset ? Term::ANSIColor.reset : ""
229
- color << SEVERITY_COLOR[severity] if Integer === severity
230
- color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity
231
- if str.nil?
232
- color
233
- else
234
- color + str.to_s + self.color(0)
235
- end
236
- end
237
-
238
- def self.up_lines(num = 1)
239
- nocolor ? "" : "\033[#{num+1}F\033[2K"
240
- end
241
-
242
- def self.down_lines(num = 1)
243
- nocolor ? "" : "\033[#{num+1}E"
244
- end
245
-
246
- def self.return_line
247
- nocolor ? "" : "\033[1A"
248
- end
249
-
250
- def self.clear_line(out = STDOUT)
251
- out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
252
- end
253
-
254
- def self.highlight(str = nil)
255
- if str.nil?
256
- return "" if nocolor
257
- HIGHLIGHT
258
- else
259
- return str if nocolor
260
- HIGHLIGHT + str + color(0)
261
- end
262
- end
263
-
264
- LAST = "log"
265
- def self.logn(message = nil, severity = MEDIUM, &block)
266
- return if severity < self.severity
267
- message ||= block.call if block_given?
268
- return if message.nil?
269
-
270
- time = Time.now.strftime("%m/%d/%y-%H:%M:%S.%L")
271
-
272
- sev_str = severity.to_s
273
-
274
- prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
275
- message = "" << highlight << message << color(0) if severity >= INFO
276
- str = prefix << " " << message.to_s
277
-
278
- LOG_MUTEX.synchronize do
279
- if logfile.nil?
280
- STDERR.write str
281
- else
282
- logfile.write str
283
- end
284
- Log::LAST.replace "log"
285
- nil
286
- end
287
- end
288
-
289
- def self.log(message = nil, severity = MEDIUM, &block)
290
- return if severity < self.severity
291
- message ||= block.call if block_given?
292
- return if message.nil?
293
- message = message + "\n" unless message[-1] == "\n"
294
- self.logn message, severity, &block
295
- end
296
-
297
- def self.log_obj_inspect(obj, level, file = $stdout)
298
- stack = caller
299
-
300
- line = Log.last_caller stack
301
-
302
- level = Log.get_level level
303
- name = Log::SEVERITY_NAMES[level] + ": "
304
- Log.log Log.color(level, name, true) << line, level
305
- Log.log "", level
306
- Log.log Log.color(level, "=> ", true) << obj.inspect, level
307
- Log.log "", level
308
- end
309
-
310
- def self.log_obj_fingerprint(obj, level, file = $stdout)
311
- stack = caller
312
-
313
- line = Log.last_caller stack
314
-
315
- level = Log.get_level level
316
- name = Log::SEVERITY_NAMES[level] + ": "
317
- Log.log Log.color(level, name, true) << line, level
318
- Log.log "", level
319
- Log.log Log.color(level, "=> ", true) << Misc.fingerprint(obj), level
320
- Log.log "", level
321
- end
322
-
323
- def self.debug(message = nil, &block)
324
- log(message, DEBUG, &block)
325
- end
326
-
327
- def self.low(message = nil, &block)
328
- log(message, LOW, &block)
329
- end
330
-
331
- def self.medium(message = nil, &block)
332
- log(message, MEDIUM, &block)
333
- end
334
-
335
- def self.high(message = nil, &block)
336
- log(message, HIGH, &block)
337
- end
338
-
339
- def self.info(message = nil, &block)
340
- log(message, INFO, &block)
341
- end
342
-
343
- def self.warn(message = nil, &block)
344
- log(message, WARN, &block)
345
- end
346
-
347
- def self.error(message = nil, &block)
348
- log(message, ERROR, &block)
349
- end
350
-
351
- def self.exception(e)
352
- stack = caller
353
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
354
- error([e.class.to_s, e.message].compact * ": " )
355
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
356
- else
357
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
358
- error([e.class.to_s, e.message].compact * ": " )
359
- end
360
- end
361
-
362
- def self.deprecated(m)
363
- stack = caller
364
- warn("DEPRECATED: " << Log.last_caller(stack))
365
- warn("* " << (m || "").to_s)
366
- end
367
-
368
- def self.color_stack(stack)
369
- stack.collect do |line|
370
- line = line.sub('`',"'")
371
- color = :green if line =~ /workflow/
372
- color = :blue if line =~ /rbbt-/
373
- Log.color color, line
374
- end unless stack.nil?
375
- end
376
-
377
- def self.tsv(tsv, example = false)
378
- STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
379
- STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true)
380
- STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
381
- if example && ! tsv.empty?
382
- key = case example
383
- when TrueClass, :first, "first"
384
- tsv.keys.first
385
- when :random, "random"
386
- tsv.keys.shuffle.first
387
- else
388
- example
389
- end
390
-
391
- values = tsv[key]
392
- values = [values] if tsv.type == :flat || tsv.type == :single
393
- if values.nil?
394
- STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
395
- else
396
- STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
397
- tsv.fields.zip(values).each do |field,value|
398
- STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
399
- end
400
- end
401
- end
402
- LOG_MUTEX.synchronize do
403
- Log::LAST.replace "log"
404
- end
405
- nil
406
- end
407
-
408
- def self.stack(stack)
409
- LOG_MUTEX.synchronize do
410
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
411
- STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
412
- color_stack(stack).each do |line|
413
- STDERR.puts line
414
- end
415
- else
416
- STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
417
- color_stack(stack.reverse).each do |line|
418
- STDERR.puts line
419
- end
420
- end
421
- end
422
- end
423
-
424
- def self.count_stack
425
- if ! $count_stacks
426
- Log.debug "Counting stacks at: " << caller.first
427
- return
428
- end
429
- $stack_counts ||= {}
430
- head = $count_stacks_head
431
- stack = caller[1..head+1]
432
- stack.reverse.each do |line,i|
433
- $stack_counts[line] ||= 0
434
- $stack_counts[line] += 1
435
- end
436
- end
437
-
438
- def self.with_stack_counts(head = 10, total = 100)
439
- $count_stacks_head = head
440
- $count_stacks = true
441
- $stack_counts = {}
442
- res = yield
443
- $count_stacks = false
444
- Log.debug "STACK_COUNTS:\n" + $stack_counts.sort_by{|line,c| c}.reverse.collect{|line,c| [c, line] * " - "}[0..total] * "\n"
445
- $stack_counts = {}
446
- res
447
- end
448
-
449
- case ENV['RBBT_LOG']
450
- when 'DEBUG'
451
- self.severity = DEBUG
452
- when 'LOW'
453
- self.severity = LOW
454
- when 'MEDIUM'
455
- self.severity = MEDIUM
456
- when 'HIGH'
457
- self.severity = HIGH
458
- when nil
459
- self.severity = INFO
460
- else
461
- self.severity = ENV['RBBT_LOG'].to_i
462
- end
463
- end
464
-
465
- def ppp(message)
466
- stack = caller
467
- puts "#{Log.color :cyan, "PRINT:"} " << stack.first
468
- puts ""
469
- if message.length > 200 or message.include? "\n"
470
- puts Log.color(:cyan, "=>|") << "\n" << message.to_s
471
- else
472
- puts Log.color(:cyan, "=> ") << message.to_s
473
- end
474
- puts ""
475
- end
476
-
477
- def fff(object)
478
- stack = caller
479
- Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
480
- Log.debug{""}
481
- Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
482
- Log.debug{""}
483
- end
484
-
485
- def ddd(obj, file = $stdout)
486
- Log.log_obj_inspect(obj, :debug, file)
487
- end
488
-
489
- def lll(obj, file = $stdout)
490
- Log.log_obj_inspect(obj, :low, file)
491
- end
492
-
493
- def mmm(obj, file = $stdout)
494
- Log.log_obj_inspect(obj, :medium, file)
495
- end
496
-
497
- def iii(obj=nil, file = $stdout)
498
- Log.log_obj_inspect(obj, :info, file)
499
- end
500
-
501
- def wwww(obj=nil, file = $stdout)
502
- Log.log_obj_inspect(obj, :warn, file)
503
- end
504
-
505
- def eee(obj=nil, file = $stdout)
506
- Log.log_obj_inspect(obj, :error, file)
507
- end
508
-
509
- def ddf(obj=nil, file = $stdout)
510
- Log.log_obj_fingerprint(obj, :debug, file)
511
- end
512
-
513
- def llf(obj=nil, file = $stdout)
514
- Log.log_obj_fingerprint(obj, :low, file)
515
- end
516
-
517
- def mmf(obj=nil, file = $stdout)
518
- Log.log_obj_fingerprint(obj, :medium, file)
519
- end
520
-
521
- def iif(obj=nil, file = $stdout)
522
- Log.log_obj_fingerprint(obj, :info, file)
523
- end
524
-
525
- def wwwf(obj=nil, file = $stdout)
526
- Log.log_obj_fingerprint(obj, :warn, file)
527
- end
528
-
529
- def eef(obj=nil, file = $stdout)
530
- Log.log_obj_fingerprint(obj, :error, file)
531
- end
532
-
1
+ require 'rbbt/refactor'
2
+ Rbbt.require_instead 'scout/log'
3
+ require_relative 'log/refactor'
4
+
5
+
6
+
7
+ #require 'term/ansicolor'
8
+ #require 'rbbt/util/color'
9
+ #require 'rbbt/util/log/progress'
10
+ #
11
+ #class MockMutex
12
+ # def synchronize
13
+ # yield
14
+ # end
15
+ #end
16
+ #
17
+ #module Log
18
+ # extend Term::ANSIColor
19
+ #
20
+ #
21
+ # #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
22
+ # LOG_MUTEX = MockMutex.new
23
+ # #LOG_MUTEX = Mutex.new
24
+ #
25
+ # SEVERITY_NAMES ||= begin
26
+ # names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
27
+ # names.each_with_index do |name,i|
28
+ # eval "#{ name } = #{ i }"
29
+ # end
30
+ # names
31
+ # end
32
+ #
33
+ # def self.terminal_width
34
+ # 80
35
+ # end
36
+ #
37
+ # def self.compact
38
+ # true
39
+ # end
40
+ #
41
+ #
42
+ # def self.last_caller(stack)
43
+ # line = nil
44
+ # pos ||= 0
45
+ # while line.nil? or line =~ /util\/log\.rb/ and stack.any?
46
+ # line = stack.shift
47
+ # end
48
+ # line ||= caller.first
49
+ # line.gsub('`', "'")
50
+ # end
51
+ #
52
+ #
53
+ # def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
54
+ # sout, sin = Misc.pipe
55
+ # soute, sine = Misc.pipe
56
+ # backup_stderr = STDERR.dup
57
+ # backup_stdout = STDOUT.dup
58
+ # old_logfile = Log.logfile
59
+ # Log.logfile(backup_stderr)
60
+ #
61
+ # severity_err ||= severity
62
+ # th_log = Thread.new do
63
+ # while line = sout.gets
64
+ # Log.logn "#{msg}: " + line, severity
65
+ # end
66
+ # end
67
+ #
68
+ # th_loge = Thread.new do
69
+ # while line = soute.gets
70
+ # Log.logn "#{msge}: " + line, severity_err
71
+ # end
72
+ # end
73
+ #
74
+ # begin
75
+ # STDOUT.reopen(sin)
76
+ # STDERR.reopen(sine)
77
+ # yield
78
+ # ensure
79
+ # STDERR.reopen backup_stderr
80
+ # STDOUT.reopen backup_stdout
81
+ # sin.close
82
+ # sine.close
83
+ # th_log.join
84
+ # th_loge.join
85
+ # backup_stdout.close
86
+ # backup_stderr.close
87
+ # Log.logfile = old_logfile
88
+ # end
89
+ # end
90
+ #
91
+ # def self.trap_stderr(msg = "STDERR", severity = 0)
92
+ # sout, sin = Misc.pipe
93
+ # backup_stderr = STDERR.dup
94
+ # old_logfile = Log.logfile
95
+ # Log.logfile(backup_stderr)
96
+ #
97
+ # th_log = Thread.new do
98
+ # while line = sout.gets
99
+ # Log.logn "#{msg}: " + line, severity
100
+ # end
101
+ # end
102
+ #
103
+ # begin
104
+ # STDERR.reopen(sin)
105
+ # yield
106
+ # sin.close
107
+ # ensure
108
+ # STDERR.reopen backup_stderr
109
+ # th_log.join
110
+ # backup_stderr.close
111
+ # Log.logfile = old_logfile
112
+ # end
113
+ # end
114
+ #
115
+ #
116
+ # def self.get_level(level)
117
+ # case level
118
+ # when Numeric
119
+ # level.to_i
120
+ # when String
121
+ # begin
122
+ # Log.const_get(level.upcase)
123
+ # rescue
124
+ # Log.exception $!
125
+ # end
126
+ # when Symbol
127
+ # get_level(level.to_s)
128
+ # end || 0
129
+ # end
130
+ #
131
+ # class << self
132
+ # attr_accessor :logfile, :severity, :nocolor, :tty_size
133
+ # end
134
+ #
135
+ # self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
136
+ #
137
+ # self.ignore_stderr do
138
+ # self.tty_size = begin
139
+ # require "highline/system_extensions.rb"
140
+ # HighLine::SystemExtensions.terminal_size.first
141
+ # rescue Exception
142
+ # nil
143
+ # end
144
+ # end
145
+ #
146
+ # def self.with_severity(level)
147
+ # orig = Log.severity
148
+ # begin
149
+ # Log.severity = level
150
+ # yield
151
+ # ensure
152
+ # Log.severity = orig
153
+ # end
154
+ # end
155
+ #
156
+ # def self.logfile(file=nil)
157
+ # if file.nil?
158
+ # @logfile ||= nil
159
+ # else
160
+ # case file
161
+ # when String
162
+ # @logfile = File.open(file, :mode => 'a')
163
+ # when IO, File
164
+ # @logfile = file
165
+ # else
166
+ # raise "Unkown logfile format: #{file.inspect}"
167
+ # end
168
+ # end
169
+ # end
170
+ #
171
+ # WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
172
+ #
173
+ # SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
174
+ # HIGHLIGHT = "\033[1m"
175
+ #
176
+ # def self.uncolor(str)
177
+ # "" << Term::ANSIColor.uncolor(str)
178
+ # end
179
+ #
180
+ # def self.reset_color
181
+ # reset
182
+ # end
183
+ #
184
+ # def self.color(severity, str = nil, reset = false)
185
+ # return str.dup || "" if nocolor
186
+ # color = reset ? Term::ANSIColor.reset : ""
187
+ # color << SEVERITY_COLOR[severity] if Integer === severity
188
+ # color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity
189
+ # if str.nil?
190
+ # color
191
+ # else
192
+ # color + str.to_s + self.color(0)
193
+ # end
194
+ # end
195
+ #
196
+ # def self.up_lines(num = 1)
197
+ # nocolor ? "" : "\033[#{num+1}F\033[2K"
198
+ # end
199
+ #
200
+ # def self.down_lines(num = 1)
201
+ # nocolor ? "" : "\033[#{num+1}E"
202
+ # end
203
+ #
204
+ # def self.return_line
205
+ # nocolor ? "" : "\033[1A"
206
+ # end
207
+ #
208
+ # def self.clear_line(out = STDOUT)
209
+ # out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
210
+ # end
211
+ #
212
+ # def self.highlight(str = nil)
213
+ # if str.nil?
214
+ # return "" if nocolor
215
+ # HIGHLIGHT
216
+ # else
217
+ # return str if nocolor
218
+ # HIGHLIGHT + str + color(0)
219
+ # end
220
+ # end
221
+ #
222
+ # LAST = "log"
223
+ # def self.logn(message = nil, severity = MEDIUM, &block)
224
+ # return if severity < self.severity
225
+ # message ||= block.call if block_given?
226
+ # return if message.nil?
227
+ #
228
+ # time = Time.now.strftime("%m/%d/%y-%H:%M:%S.%L")
229
+ #
230
+ # sev_str = severity.to_s
231
+ #
232
+ # prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
233
+ # message = "" << highlight << message << color(0) if severity >= INFO
234
+ # str = prefix << " " << message.to_s
235
+ #
236
+ # LOG_MUTEX.synchronize do
237
+ # if logfile.nil?
238
+ # STDERR.write str
239
+ # else
240
+ # logfile.write str
241
+ # end
242
+ # Log::LAST.replace "log"
243
+ # nil
244
+ # end
245
+ # end
246
+ #
247
+ # def self.log(message = nil, severity = MEDIUM, &block)
248
+ # return if severity < self.severity
249
+ # message ||= block.call if block_given?
250
+ # return if message.nil?
251
+ # message = message + "\n" unless message[-1] == "\n"
252
+ # self.logn message, severity, &block
253
+ # end
254
+ #
255
+ # def self.log_obj_inspect(obj, level, file = $stdout)
256
+ # stack = caller
257
+ #
258
+ # line = Log.last_caller stack
259
+ #
260
+ # level = Log.get_level level
261
+ # name = Log::SEVERITY_NAMES[level] + ": "
262
+ # Log.log Log.color(level, name, true) << line, level
263
+ # Log.log "", level
264
+ # Log.log Log.color(level, "=> ", true) << obj.inspect, level
265
+ # Log.log "", level
266
+ # end
267
+ #
268
+ # def self.log_obj_fingerprint(obj, level, file = $stdout)
269
+ # stack = caller
270
+ #
271
+ # line = Log.last_caller stack
272
+ #
273
+ # level = Log.get_level level
274
+ # name = Log::SEVERITY_NAMES[level] + ": "
275
+ # Log.log Log.color(level, name, true) << line, level
276
+ # Log.log "", level
277
+ # Log.log Log.color(level, "=> ", true) << Misc.fingerprint(obj), level
278
+ # Log.log "", level
279
+ # end
280
+ #
281
+ # def self.debug(message = nil, &block)
282
+ # log(message, DEBUG, &block)
283
+ # end
284
+ #
285
+ # def self.low(message = nil, &block)
286
+ # log(message, LOW, &block)
287
+ # end
288
+ #
289
+ # def self.medium(message = nil, &block)
290
+ # log(message, MEDIUM, &block)
291
+ # end
292
+ #
293
+ # def self.high(message = nil, &block)
294
+ # log(message, HIGH, &block)
295
+ # end
296
+ #
297
+ # def self.info(message = nil, &block)
298
+ # log(message, INFO, &block)
299
+ # end
300
+ #
301
+ # def self.warn(message = nil, &block)
302
+ # log(message, WARN, &block)
303
+ # end
304
+ #
305
+ # def self.error(message = nil, &block)
306
+ # log(message, ERROR, &block)
307
+ # end
308
+ #
309
+ # def self.exception(e)
310
+ # stack = caller
311
+ # if ENV["RBBT_ORIGINAL_STACK"] == 'true'
312
+ # error([e.class.to_s, e.message].compact * ": " )
313
+ # error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
314
+ # else
315
+ # error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
316
+ # error([e.class.to_s, e.message].compact * ": " )
317
+ # end
318
+ # end
319
+ #
320
+ # def self.deprecated(m)
321
+ # stack = caller
322
+ # warn("DEPRECATED: " << Log.last_caller(stack))
323
+ # warn("* " << (m || "").to_s)
324
+ # end
325
+ #
326
+ # def self.color_stack(stack)
327
+ # stack.collect do |line|
328
+ # line = line.sub('`',"'")
329
+ # color = :green if line =~ /workflow/
330
+ # color = :blue if line =~ /rbbt-/
331
+ # Log.color color, line
332
+ # end unless stack.nil?
333
+ # end
334
+ #
335
+ # def self.tsv(tsv, example = false)
336
+ # STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
337
+ # STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true)
338
+ # STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
339
+ # if example && ! tsv.empty?
340
+ # key = case example
341
+ # when TrueClass, :first, "first"
342
+ # tsv.keys.first
343
+ # when :random, "random"
344
+ # tsv.keys.shuffle.first
345
+ # else
346
+ # example
347
+ # end
348
+ #
349
+ # values = tsv[key]
350
+ # values = [values] if tsv.type == :flat || tsv.type == :single
351
+ # if values.nil?
352
+ # STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
353
+ # else
354
+ # STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
355
+ # tsv.fields.zip(values).each do |field,value|
356
+ # STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
357
+ # end
358
+ # end
359
+ # end
360
+ # end
361
+ #
362
+ # def self.stack(stack)
363
+ # LOG_MUTEX.synchronize do
364
+ # if ENV["RBBT_ORIGINAL_STACK"] == 'true'
365
+ # STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
366
+ # color_stack(stack).each do |line|
367
+ # STDERR.puts line
368
+ # end
369
+ # else
370
+ # STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
371
+ # color_stack(stack.reverse).each do |line|
372
+ # STDERR.puts line
373
+ # end
374
+ # end
375
+ # end
376
+ # end
377
+ #
378
+ # def self.count_stack
379
+ # if ! $count_stacks
380
+ # Log.debug "Counting stacks at: " << caller.first
381
+ # return
382
+ # end
383
+ # $stack_counts ||= {}
384
+ # head = $count_stacks_head
385
+ # stack = caller[1..head+1]
386
+ # stack.reverse.each do |line,i|
387
+ # $stack_counts[line] ||= 0
388
+ # $stack_counts[line] += 1
389
+ # end
390
+ # end
391
+ #
392
+ # def self.with_stack_counts(head = 10, total = 100)
393
+ # $count_stacks_head = head
394
+ # $count_stacks = true
395
+ # $stack_counts = {}
396
+ # res = yield
397
+ # $count_stacks = false
398
+ # Log.debug "STACK_COUNTS:\n" + $stack_counts.sort_by{|line,c| c}.reverse.collect{|line,c| [c, line] * " - "}[0..total] * "\n"
399
+ # $stack_counts = {}
400
+ # res
401
+ # end
402
+ #
403
+ # case ENV['RBBT_LOG']
404
+ # when 'DEBUG'
405
+ # self.severity = DEBUG
406
+ # when 'LOW'
407
+ # self.severity = LOW
408
+ # when 'MEDIUM'
409
+ # self.severity = MEDIUM
410
+ # when 'HIGH'
411
+ # self.severity = HIGH
412
+ # when nil
413
+ # self.severity = INFO
414
+ # else
415
+ # self.severity = ENV['RBBT_LOG'].to_i
416
+ # end
417
+ #end
418
+ #
419
+ #def ppp(message)
420
+ # stack = caller
421
+ # puts "#{Log.color :cyan, "PRINT:"} " << stack.first
422
+ # puts ""
423
+ # if message.length > 200 or message.include? "\n"
424
+ # puts Log.color(:cyan, "=>|") << "\n" << message.to_s
425
+ # else
426
+ # puts Log.color(:cyan, "=> ") << message.to_s
427
+ # end
428
+ # puts ""
429
+ #end
430
+ #
431
+ #def fff(object)
432
+ # stack = caller
433
+ # Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
434
+ # Log.debug{""}
435
+ # Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
436
+ # Log.debug{""}
437
+ #end
438
+ #
439
+ #def ddd(obj, file = $stdout)
440
+ # Log.log_obj_inspect(obj, :debug, file)
441
+ #end
442
+ #
443
+ #def lll(obj, file = $stdout)
444
+ # Log.log_obj_inspect(obj, :low, file)
445
+ #end
446
+ #
447
+ #def mmm(obj, file = $stdout)
448
+ # Log.log_obj_inspect(obj, :medium, file)
449
+ #end
450
+ #
451
+ #def iii(obj=nil, file = $stdout)
452
+ # Log.log_obj_inspect(obj, :info, file)
453
+ #end
454
+ #
455
+ #def wwww(obj=nil, file = $stdout)
456
+ # Log.log_obj_inspect(obj, :warn, file)
457
+ #end
458
+ #
459
+ #def eee(obj=nil, file = $stdout)
460
+ # Log.log_obj_inspect(obj, :error, file)
461
+ #end
462
+ #
463
+ #def ddf(obj=nil, file = $stdout)
464
+ # Log.log_obj_fingerprint(obj, :debug, file)
465
+ #end
466
+ #
467
+ #def llf(obj=nil, file = $stdout)
468
+ # Log.log_obj_fingerprint(obj, :low, file)
469
+ #end
470
+ #
471
+ #def mmf(obj=nil, file = $stdout)
472
+ # Log.log_obj_fingerprint(obj, :medium, file)
473
+ #end
474
+ #
475
+ #def iif(obj=nil, file = $stdout)
476
+ # Log.log_obj_fingerprint(obj, :info, file)
477
+ #end
478
+ #
479
+ #def wwwf(obj=nil, file = $stdout)
480
+ # Log.log_obj_fingerprint(obj, :warn, file)
481
+ #end
482
+ #
483
+ #def eef(obj=nil, file = $stdout)
484
+ # Log.log_obj_fingerprint(obj, :error, file)
485
+ #end
486
+ #