rbbt-util 5.44.1 → 6.0.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (167) hide show
  1. checksums.yaml +4 -4
  2. data/bin/rbbt +67 -90
  3. data/etc/app.d/base.rb +2 -2
  4. data/etc/app.d/semaphores.rb +3 -3
  5. data/lib/rbbt/annotations/annotated_array.rb +207 -207
  6. data/lib/rbbt/annotations/refactor.rb +27 -0
  7. data/lib/rbbt/annotations/util.rb +282 -282
  8. data/lib/rbbt/annotations.rb +343 -320
  9. data/lib/rbbt/association/database.rb +200 -225
  10. data/lib/rbbt/association/index.rb +294 -291
  11. data/lib/rbbt/association/item.rb +227 -227
  12. data/lib/rbbt/association/open.rb +35 -34
  13. data/lib/rbbt/association/util.rb +0 -169
  14. data/lib/rbbt/association.rb +2 -4
  15. data/lib/rbbt/entity/identifiers.rb +119 -118
  16. data/lib/rbbt/entity/refactor.rb +12 -0
  17. data/lib/rbbt/entity.rb +319 -315
  18. data/lib/rbbt/hpc/batch.rb +72 -53
  19. data/lib/rbbt/hpc/lsf.rb +2 -2
  20. data/lib/rbbt/hpc/orchestrate/batches.rb +2 -2
  21. data/lib/rbbt/hpc/orchestrate/chains.rb +25 -5
  22. data/lib/rbbt/hpc/orchestrate/rules.rb +2 -2
  23. data/lib/rbbt/hpc/orchestrate.rb +19 -13
  24. data/lib/rbbt/hpc/slurm.rb +18 -18
  25. data/lib/rbbt/knowledge_base/entity.rb +13 -5
  26. data/lib/rbbt/knowledge_base/query.rb +2 -2
  27. data/lib/rbbt/knowledge_base/registry.rb +32 -31
  28. data/lib/rbbt/knowledge_base/traverse.rb +1 -1
  29. data/lib/rbbt/knowledge_base.rb +1 -1
  30. data/lib/rbbt/monitor.rb +36 -25
  31. data/lib/rbbt/persist/refactor.rb +166 -0
  32. data/lib/rbbt/persist/tsv/tokyocabinet.rb +105 -105
  33. data/lib/rbbt/persist/tsv.rb +187 -185
  34. data/lib/rbbt/persist.rb +556 -551
  35. data/lib/rbbt/refactor.rb +20 -0
  36. data/lib/rbbt/resource/path/refactor.rb +178 -0
  37. data/lib/rbbt/resource/path.rb +317 -497
  38. data/lib/rbbt/resource/util.rb +0 -48
  39. data/lib/rbbt/resource.rb +3 -390
  40. data/lib/rbbt/tsv/accessor.rb +2 -838
  41. data/lib/rbbt/tsv/attach.rb +303 -299
  42. data/lib/rbbt/tsv/change_id.rb +244 -245
  43. data/lib/rbbt/tsv/csv.rb +87 -85
  44. data/lib/rbbt/tsv/dumper.rb +2 -100
  45. data/lib/rbbt/tsv/excel.rb +26 -24
  46. data/lib/rbbt/tsv/field_index.rb +4 -1
  47. data/lib/rbbt/tsv/filter.rb +3 -2
  48. data/lib/rbbt/tsv/index.rb +2 -284
  49. data/lib/rbbt/tsv/manipulate.rb +750 -747
  50. data/lib/rbbt/tsv/marshal.rb +3 -3
  51. data/lib/rbbt/tsv/matrix.rb +2 -2
  52. data/lib/rbbt/tsv/parallel/through.rb +2 -1
  53. data/lib/rbbt/tsv/parallel/traverse.rb +783 -781
  54. data/lib/rbbt/tsv/parser.rb +678 -678
  55. data/lib/rbbt/tsv/refactor.rb +195 -0
  56. data/lib/rbbt/tsv/stream.rb +253 -251
  57. data/lib/rbbt/tsv/util.rb +420 -420
  58. data/lib/rbbt/tsv.rb +210 -208
  59. data/lib/rbbt/util/R/eval.rb +4 -4
  60. data/lib/rbbt/util/R/plot.rb +62 -166
  61. data/lib/rbbt/util/R.rb +21 -18
  62. data/lib/rbbt/util/cmd.rb +2 -318
  63. data/lib/rbbt/util/color.rb +269 -269
  64. data/lib/rbbt/util/colorize.rb +89 -89
  65. data/lib/rbbt/util/concurrency/processes/refactor.rb +22 -0
  66. data/lib/rbbt/util/concurrency/processes/worker.rb +2 -2
  67. data/lib/rbbt/util/concurrency/processes.rb +389 -386
  68. data/lib/rbbt/util/config.rb +169 -167
  69. data/lib/rbbt/util/iruby.rb +20 -0
  70. data/lib/rbbt/util/log/progress/report.rb +241 -241
  71. data/lib/rbbt/util/log/progress/util.rb +99 -99
  72. data/lib/rbbt/util/log/progress.rb +102 -102
  73. data/lib/rbbt/util/log/refactor.rb +49 -0
  74. data/lib/rbbt/util/log.rb +486 -532
  75. data/lib/rbbt/util/migrate.rb +1 -1
  76. data/lib/rbbt/util/misc/concurrent_stream.rb +248 -246
  77. data/lib/rbbt/util/misc/development.rb +12 -11
  78. data/lib/rbbt/util/misc/exceptions.rb +117 -112
  79. data/lib/rbbt/util/misc/format.rb +2 -230
  80. data/lib/rbbt/util/misc/indiferent_hash.rb +2 -107
  81. data/lib/rbbt/util/misc/inspect.rb +2 -476
  82. data/lib/rbbt/util/misc/lock.rb +109 -106
  83. data/lib/rbbt/util/misc/omics.rb +9 -1
  84. data/lib/rbbt/util/misc/pipes.rb +765 -793
  85. data/lib/rbbt/util/misc/refactor.rb +20 -0
  86. data/lib/rbbt/util/misc/ssw.rb +27 -17
  87. data/lib/rbbt/util/misc/system.rb +0 -15
  88. data/lib/rbbt/util/misc.rb +39 -20
  89. data/lib/rbbt/util/named_array/refactor.rb +4 -0
  90. data/lib/rbbt/util/named_array.rb +3 -220
  91. data/lib/rbbt/util/open/refactor.rb +7 -0
  92. data/lib/rbbt/util/open.rb +3 -857
  93. data/lib/rbbt/util/procpath.rb +6 -6
  94. data/lib/rbbt/util/python/paths.rb +27 -0
  95. data/lib/rbbt/util/python/run.rb +115 -0
  96. data/lib/rbbt/util/python/script.rb +110 -0
  97. data/lib/rbbt/util/python/util.rb +3 -3
  98. data/lib/rbbt/util/python.rb +22 -81
  99. data/lib/rbbt/util/semaphore.rb +152 -148
  100. data/lib/rbbt/util/simpleopt.rb +9 -8
  101. data/lib/rbbt/util/ssh/refactor.rb +19 -0
  102. data/lib/rbbt/util/ssh.rb +122 -118
  103. data/lib/rbbt/util/tar.rb +117 -115
  104. data/lib/rbbt/util/tmpfile.rb +69 -67
  105. data/lib/rbbt/util/version.rb +2 -0
  106. data/lib/rbbt/workflow/refactor/entity.rb +11 -0
  107. data/lib/rbbt/workflow/refactor/export.rb +66 -0
  108. data/lib/rbbt/workflow/refactor/inputs.rb +24 -0
  109. data/lib/rbbt/workflow/refactor/recursive.rb +64 -0
  110. data/lib/rbbt/workflow/refactor/task_info.rb +65 -0
  111. data/lib/rbbt/workflow/refactor.rb +153 -0
  112. data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +55 -32
  113. data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +3 -1
  114. data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +14 -5
  115. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +19 -7
  116. data/lib/rbbt/workflow/remote_workflow.rb +6 -1
  117. data/lib/rbbt/workflow/step/run.rb +766 -766
  118. data/lib/rbbt/workflow/step/save_load_inputs.rb +254 -254
  119. data/lib/rbbt/workflow/step.rb +2 -362
  120. data/lib/rbbt/workflow/task.rb +118 -118
  121. data/lib/rbbt/workflow/usage.rb +289 -287
  122. data/lib/rbbt/workflow/util/archive.rb +6 -5
  123. data/lib/rbbt/workflow/util/data.rb +1 -1
  124. data/lib/rbbt/workflow/util/orchestrator.rb +249 -246
  125. data/lib/rbbt/workflow/util/trace.rb +79 -44
  126. data/lib/rbbt/workflow.rb +4 -882
  127. data/lib/rbbt-util.rb +21 -13
  128. data/lib/rbbt.rb +16 -3
  129. data/python/rbbt/__init__.py +19 -1
  130. data/share/Rlib/plot.R +37 -37
  131. data/share/Rlib/svg.R +22 -5
  132. data/share/install/software/lib/install_helpers +1 -1
  133. data/share/rbbt_commands/hpc/list +2 -3
  134. data/share/rbbt_commands/hpc/orchestrate +4 -4
  135. data/share/rbbt_commands/hpc/tail +2 -0
  136. data/share/rbbt_commands/hpc/task +10 -7
  137. data/share/rbbt_commands/lsf/list +2 -3
  138. data/share/rbbt_commands/lsf/orchestrate +4 -4
  139. data/share/rbbt_commands/lsf/tail +2 -0
  140. data/share/rbbt_commands/lsf/task +10 -7
  141. data/share/rbbt_commands/migrate +1 -1
  142. data/share/rbbt_commands/pbs/list +2 -3
  143. data/share/rbbt_commands/pbs/orchestrate +4 -4
  144. data/share/rbbt_commands/pbs/tail +2 -0
  145. data/share/rbbt_commands/pbs/task +10 -7
  146. data/share/rbbt_commands/resource/produce +8 -1
  147. data/share/rbbt_commands/slurm/list +2 -3
  148. data/share/rbbt_commands/slurm/orchestrate +4 -4
  149. data/share/rbbt_commands/slurm/tail +2 -0
  150. data/share/rbbt_commands/slurm/task +10 -7
  151. data/share/rbbt_commands/system/clean +5 -5
  152. data/share/rbbt_commands/system/status +5 -5
  153. data/share/rbbt_commands/tsv/get +2 -3
  154. data/share/rbbt_commands/tsv/info +10 -13
  155. data/share/rbbt_commands/tsv/keys +18 -14
  156. data/share/rbbt_commands/tsv/slice +2 -2
  157. data/share/rbbt_commands/tsv/transpose +6 -2
  158. data/share/rbbt_commands/workflow/info +20 -24
  159. data/share/rbbt_commands/workflow/list +1 -1
  160. data/share/rbbt_commands/workflow/prov +20 -13
  161. data/share/rbbt_commands/workflow/server +11 -1
  162. data/share/rbbt_commands/workflow/task +76 -71
  163. data/share/rbbt_commands/workflow/write_info +26 -9
  164. data/share/software/opt/ssw/ssw.c +861 -0
  165. data/share/software/opt/ssw/ssw.h +130 -0
  166. data/share/workflow_config.ru +3 -3
  167. metadata +40 -2
data/lib/rbbt/util/log.rb CHANGED
@@ -1,532 +1,486 @@
1
- require 'term/ansicolor'
2
- require 'rbbt/util/color'
3
- require 'rbbt/util/log/progress'
4
-
5
- class MockMutex
6
- def synchronize
7
- yield
8
- end
9
- end
10
-
11
- module Log
12
- extend Term::ANSIColor
13
-
14
-
15
- #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
16
- LOG_MUTEX = MockMutex.new
17
- #LOG_MUTEX = Mutex.new
18
-
19
- SEVERITY_NAMES ||= begin
20
- names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
21
- names.each_with_index do |name,i|
22
- eval "#{ name } = #{ i }"
23
- end
24
- names
25
- end
26
-
27
- def self.terminal_width
28
- 80
29
- end
30
-
31
- def self.compact
32
- true
33
- end
34
-
35
-
36
- def self.last_caller(stack)
37
- line = nil
38
- pos ||= 0
39
- while line.nil? or line =~ /util\/log\.rb/ and stack.any?
40
- line = stack.shift
41
- end
42
- line ||= caller.first
43
- line.gsub('`', "'")
44
- end
45
-
46
-
47
- def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
48
- sout, sin = Misc.pipe
49
- soute, sine = Misc.pipe
50
- backup_stderr = STDERR.dup
51
- backup_stdout = STDOUT.dup
52
- old_logfile = Log.logfile
53
- Log.logfile(backup_stderr)
54
-
55
- severity_err ||= severity
56
- th_log = Thread.new do
57
- while line = sout.gets
58
- Log.logn "#{msg}: " + line, severity
59
- end
60
- end
61
-
62
- th_loge = Thread.new do
63
- while line = soute.gets
64
- Log.logn "#{msge}: " + line, severity_err
65
- end
66
- end
67
-
68
- begin
69
- STDOUT.reopen(sin)
70
- STDERR.reopen(sine)
71
- yield
72
- ensure
73
- STDERR.reopen backup_stderr
74
- STDOUT.reopen backup_stdout
75
- sin.close
76
- sine.close
77
- th_log.join
78
- th_loge.join
79
- backup_stdout.close
80
- backup_stderr.close
81
- Log.logfile = old_logfile
82
- end
83
- end
84
-
85
- def self.trap_stderr(msg = "STDERR", severity = 0)
86
- sout, sin = Misc.pipe
87
- backup_stderr = STDERR.dup
88
- old_logfile = Log.logfile
89
- Log.logfile(backup_stderr)
90
-
91
- th_log = Thread.new do
92
- while line = sout.gets
93
- Log.logn "#{msg}: " + line, severity
94
- end
95
- end
96
-
97
- begin
98
- STDERR.reopen(sin)
99
- yield
100
- sin.close
101
- ensure
102
- STDERR.reopen backup_stderr
103
- th_log.join
104
- backup_stderr.close
105
- Log.logfile = old_logfile
106
- end
107
- end
108
-
109
-
110
- def self._ignore_stderr
111
- begin
112
- File.open('/dev/null', 'w') do |f|
113
- backup_stderr = STDERR.dup
114
- STDERR.reopen(f)
115
- begin
116
- yield
117
- ensure
118
- STDERR.reopen backup_stderr
119
- backup_stderr.close
120
- end
121
- end
122
- rescue Errno::ENOENT
123
- yield
124
- end
125
- end
126
-
127
-
128
- def self.ignore_stderr(&block)
129
- LOG_MUTEX.synchronize do
130
- _ignore_stderr &block
131
- end
132
- end
133
-
134
- def self._ignore_stdout
135
- begin
136
- File.open('/dev/null', 'w') do |f|
137
- backup_stdout = STDOUT.dup
138
- STDOUT.reopen(f)
139
- begin
140
- yield
141
- ensure
142
- STDOUT.reopen backup_stdout
143
- backup_stdout.close
144
- end
145
- end
146
- rescue Errno::ENOENT
147
- yield
148
- end
149
- end
150
-
151
-
152
- def self.ignore_stdout(&block)
153
- LOG_MUTEX.synchronize do
154
- _ignore_stdout &block
155
- end
156
- end
157
-
158
- def self.get_level(level)
159
- case level
160
- when Numeric
161
- level.to_i
162
- when String
163
- begin
164
- Log.const_get(level.upcase)
165
- rescue
166
- Log.exception $!
167
- end
168
- when Symbol
169
- get_level(level.to_s)
170
- end || 0
171
- end
172
-
173
- class << self
174
- attr_accessor :logfile, :severity, :nocolor, :tty_size
175
- end
176
-
177
- self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
178
-
179
- self.ignore_stderr do
180
- self.tty_size = begin
181
- require "highline/system_extensions.rb"
182
- HighLine::SystemExtensions.terminal_size.first
183
- rescue Exception
184
- nil
185
- end
186
- end
187
-
188
- def self.with_severity(level)
189
- orig = Log.severity
190
- begin
191
- Log.severity = level
192
- yield
193
- ensure
194
- Log.severity = orig
195
- end
196
- end
197
-
198
- def self.logfile(file=nil)
199
- if file.nil?
200
- @logfile ||= nil
201
- else
202
- case file
203
- when String
204
- @logfile = File.open(file, :mode => 'a')
205
- when IO, File
206
- @logfile = file
207
- else
208
- raise "Unkown logfile format: #{file.inspect}"
209
- end
210
- end
211
- end
212
-
213
- WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
214
-
215
- SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
216
- HIGHLIGHT = "\033[1m"
217
-
218
- def self.uncolor(str)
219
- "" << Term::ANSIColor.uncolor(str)
220
- end
221
-
222
- def self.reset_color
223
- reset
224
- end
225
-
226
- def self.color(severity, str = nil, reset = false)
227
- return str.dup || "" if nocolor
228
- color = reset ? Term::ANSIColor.reset : ""
229
- color << SEVERITY_COLOR[severity] if Integer === severity
230
- color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity
231
- if str.nil?
232
- color
233
- else
234
- color + str.to_s + self.color(0)
235
- end
236
- end
237
-
238
- def self.up_lines(num = 1)
239
- nocolor ? "" : "\033[#{num+1}F\033[2K"
240
- end
241
-
242
- def self.down_lines(num = 1)
243
- nocolor ? "" : "\033[#{num+1}E"
244
- end
245
-
246
- def self.return_line
247
- nocolor ? "" : "\033[1A"
248
- end
249
-
250
- def self.clear_line(out = STDOUT)
251
- out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
252
- end
253
-
254
- def self.highlight(str = nil)
255
- if str.nil?
256
- return "" if nocolor
257
- HIGHLIGHT
258
- else
259
- return str if nocolor
260
- HIGHLIGHT + str + color(0)
261
- end
262
- end
263
-
264
- LAST = "log"
265
- def self.logn(message = nil, severity = MEDIUM, &block)
266
- return if severity < self.severity
267
- message ||= block.call if block_given?
268
- return if message.nil?
269
-
270
- time = Time.now.strftime("%m/%d/%y-%H:%M:%S.%L")
271
-
272
- sev_str = severity.to_s
273
-
274
- prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
275
- message = "" << highlight << message << color(0) if severity >= INFO
276
- str = prefix << " " << message.to_s
277
-
278
- LOG_MUTEX.synchronize do
279
- if logfile.nil?
280
- STDERR.write str
281
- else
282
- logfile.write str
283
- end
284
- Log::LAST.replace "log"
285
- nil
286
- end
287
- end
288
-
289
- def self.log(message = nil, severity = MEDIUM, &block)
290
- return if severity < self.severity
291
- message ||= block.call if block_given?
292
- return if message.nil?
293
- message = message + "\n" unless message[-1] == "\n"
294
- self.logn message, severity, &block
295
- end
296
-
297
- def self.log_obj_inspect(obj, level, file = $stdout)
298
- stack = caller
299
-
300
- line = Log.last_caller stack
301
-
302
- level = Log.get_level level
303
- name = Log::SEVERITY_NAMES[level] + ": "
304
- Log.log Log.color(level, name, true) << line, level
305
- Log.log "", level
306
- Log.log Log.color(level, "=> ", true) << obj.inspect, level
307
- Log.log "", level
308
- end
309
-
310
- def self.log_obj_fingerprint(obj, level, file = $stdout)
311
- stack = caller
312
-
313
- line = Log.last_caller stack
314
-
315
- level = Log.get_level level
316
- name = Log::SEVERITY_NAMES[level] + ": "
317
- Log.log Log.color(level, name, true) << line, level
318
- Log.log "", level
319
- Log.log Log.color(level, "=> ", true) << Misc.fingerprint(obj), level
320
- Log.log "", level
321
- end
322
-
323
- def self.debug(message = nil, &block)
324
- log(message, DEBUG, &block)
325
- end
326
-
327
- def self.low(message = nil, &block)
328
- log(message, LOW, &block)
329
- end
330
-
331
- def self.medium(message = nil, &block)
332
- log(message, MEDIUM, &block)
333
- end
334
-
335
- def self.high(message = nil, &block)
336
- log(message, HIGH, &block)
337
- end
338
-
339
- def self.info(message = nil, &block)
340
- log(message, INFO, &block)
341
- end
342
-
343
- def self.warn(message = nil, &block)
344
- log(message, WARN, &block)
345
- end
346
-
347
- def self.error(message = nil, &block)
348
- log(message, ERROR, &block)
349
- end
350
-
351
- def self.exception(e)
352
- stack = caller
353
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
354
- error([e.class.to_s, e.message].compact * ": " )
355
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
356
- else
357
- error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
358
- error([e.class.to_s, e.message].compact * ": " )
359
- end
360
- end
361
-
362
- def self.deprecated(m)
363
- stack = caller
364
- warn("DEPRECATED: " << Log.last_caller(stack))
365
- warn("* " << (m || "").to_s)
366
- end
367
-
368
- def self.color_stack(stack)
369
- stack.collect do |line|
370
- line = line.sub('`',"'")
371
- color = :green if line =~ /workflow/
372
- color = :blue if line =~ /rbbt-/
373
- Log.color color, line
374
- end unless stack.nil?
375
- end
376
-
377
- def self.tsv(tsv, example = false)
378
- STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
379
- STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true)
380
- STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
381
- if example && ! tsv.empty?
382
- key = case example
383
- when TrueClass, :first, "first"
384
- tsv.keys.first
385
- when :random, "random"
386
- tsv.keys.shuffle.first
387
- else
388
- example
389
- end
390
-
391
- values = tsv[key]
392
- values = [values] if tsv.type == :flat || tsv.type == :single
393
- if values.nil?
394
- STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
395
- else
396
- STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
397
- tsv.fields.zip(values).each do |field,value|
398
- STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
399
- end
400
- end
401
- end
402
- LOG_MUTEX.synchronize do
403
- Log::LAST.replace "log"
404
- end
405
- nil
406
- end
407
-
408
- def self.stack(stack)
409
- LOG_MUTEX.synchronize do
410
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
411
- STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
412
- color_stack(stack).each do |line|
413
- STDERR.puts line
414
- end
415
- else
416
- STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
417
- color_stack(stack.reverse).each do |line|
418
- STDERR.puts line
419
- end
420
- end
421
- end
422
- end
423
-
424
- def self.count_stack
425
- if ! $count_stacks
426
- Log.debug "Counting stacks at: " << caller.first
427
- return
428
- end
429
- $stack_counts ||= {}
430
- head = $count_stacks_head
431
- stack = caller[1..head+1]
432
- stack.reverse.each do |line,i|
433
- $stack_counts[line] ||= 0
434
- $stack_counts[line] += 1
435
- end
436
- end
437
-
438
- def self.with_stack_counts(head = 10, total = 100)
439
- $count_stacks_head = head
440
- $count_stacks = true
441
- $stack_counts = {}
442
- res = yield
443
- $count_stacks = false
444
- Log.debug "STACK_COUNTS:\n" + $stack_counts.sort_by{|line,c| c}.reverse.collect{|line,c| [c, line] * " - "}[0..total] * "\n"
445
- $stack_counts = {}
446
- res
447
- end
448
-
449
- case ENV['RBBT_LOG']
450
- when 'DEBUG'
451
- self.severity = DEBUG
452
- when 'LOW'
453
- self.severity = LOW
454
- when 'MEDIUM'
455
- self.severity = MEDIUM
456
- when 'HIGH'
457
- self.severity = HIGH
458
- when nil
459
- self.severity = INFO
460
- else
461
- self.severity = ENV['RBBT_LOG'].to_i
462
- end
463
- end
464
-
465
- def ppp(message)
466
- stack = caller
467
- puts "#{Log.color :cyan, "PRINT:"} " << stack.first
468
- puts ""
469
- if message.length > 200 or message.include? "\n"
470
- puts Log.color(:cyan, "=>|") << "\n" << message.to_s
471
- else
472
- puts Log.color(:cyan, "=> ") << message.to_s
473
- end
474
- puts ""
475
- end
476
-
477
- def fff(object)
478
- stack = caller
479
- Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
480
- Log.debug{""}
481
- Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
482
- Log.debug{""}
483
- end
484
-
485
- def ddd(obj, file = $stdout)
486
- Log.log_obj_inspect(obj, :debug, file)
487
- end
488
-
489
- def lll(obj, file = $stdout)
490
- Log.log_obj_inspect(obj, :low, file)
491
- end
492
-
493
- def mmm(obj, file = $stdout)
494
- Log.log_obj_inspect(obj, :medium, file)
495
- end
496
-
497
- def iii(obj=nil, file = $stdout)
498
- Log.log_obj_inspect(obj, :info, file)
499
- end
500
-
501
- def wwww(obj=nil, file = $stdout)
502
- Log.log_obj_inspect(obj, :warn, file)
503
- end
504
-
505
- def eee(obj=nil, file = $stdout)
506
- Log.log_obj_inspect(obj, :error, file)
507
- end
508
-
509
- def ddf(obj=nil, file = $stdout)
510
- Log.log_obj_fingerprint(obj, :debug, file)
511
- end
512
-
513
- def llf(obj=nil, file = $stdout)
514
- Log.log_obj_fingerprint(obj, :low, file)
515
- end
516
-
517
- def mmf(obj=nil, file = $stdout)
518
- Log.log_obj_fingerprint(obj, :medium, file)
519
- end
520
-
521
- def iif(obj=nil, file = $stdout)
522
- Log.log_obj_fingerprint(obj, :info, file)
523
- end
524
-
525
- def wwwf(obj=nil, file = $stdout)
526
- Log.log_obj_fingerprint(obj, :warn, file)
527
- end
528
-
529
- def eef(obj=nil, file = $stdout)
530
- Log.log_obj_fingerprint(obj, :error, file)
531
- end
532
-
1
+ require 'rbbt/refactor'
2
+ Rbbt.require_instead 'scout/log'
3
+ require_relative 'log/refactor'
4
+
5
+
6
+
7
+ #require 'term/ansicolor'
8
+ #require 'rbbt/util/color'
9
+ #require 'rbbt/util/log/progress'
10
+ #
11
+ #class MockMutex
12
+ # def synchronize
13
+ # yield
14
+ # end
15
+ #end
16
+ #
17
+ #module Log
18
+ # extend Term::ANSIColor
19
+ #
20
+ #
21
+ # #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
22
+ # LOG_MUTEX = MockMutex.new
23
+ # #LOG_MUTEX = Mutex.new
24
+ #
25
+ # SEVERITY_NAMES ||= begin
26
+ # names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
27
+ # names.each_with_index do |name,i|
28
+ # eval "#{ name } = #{ i }"
29
+ # end
30
+ # names
31
+ # end
32
+ #
33
+ # def self.terminal_width
34
+ # 80
35
+ # end
36
+ #
37
+ # def self.compact
38
+ # true
39
+ # end
40
+ #
41
+ #
42
+ # def self.last_caller(stack)
43
+ # line = nil
44
+ # pos ||= 0
45
+ # while line.nil? or line =~ /util\/log\.rb/ and stack.any?
46
+ # line = stack.shift
47
+ # end
48
+ # line ||= caller.first
49
+ # line.gsub('`', "'")
50
+ # end
51
+ #
52
+ #
53
+ # def self.trap_std(msg = "STDOUT", msge = "STDERR", severity = 0, severity_err = nil)
54
+ # sout, sin = Misc.pipe
55
+ # soute, sine = Misc.pipe
56
+ # backup_stderr = STDERR.dup
57
+ # backup_stdout = STDOUT.dup
58
+ # old_logfile = Log.logfile
59
+ # Log.logfile(backup_stderr)
60
+ #
61
+ # severity_err ||= severity
62
+ # th_log = Thread.new do
63
+ # while line = sout.gets
64
+ # Log.logn "#{msg}: " + line, severity
65
+ # end
66
+ # end
67
+ #
68
+ # th_loge = Thread.new do
69
+ # while line = soute.gets
70
+ # Log.logn "#{msge}: " + line, severity_err
71
+ # end
72
+ # end
73
+ #
74
+ # begin
75
+ # STDOUT.reopen(sin)
76
+ # STDERR.reopen(sine)
77
+ # yield
78
+ # ensure
79
+ # STDERR.reopen backup_stderr
80
+ # STDOUT.reopen backup_stdout
81
+ # sin.close
82
+ # sine.close
83
+ # th_log.join
84
+ # th_loge.join
85
+ # backup_stdout.close
86
+ # backup_stderr.close
87
+ # Log.logfile = old_logfile
88
+ # end
89
+ # end
90
+ #
91
+ # def self.trap_stderr(msg = "STDERR", severity = 0)
92
+ # sout, sin = Misc.pipe
93
+ # backup_stderr = STDERR.dup
94
+ # old_logfile = Log.logfile
95
+ # Log.logfile(backup_stderr)
96
+ #
97
+ # th_log = Thread.new do
98
+ # while line = sout.gets
99
+ # Log.logn "#{msg}: " + line, severity
100
+ # end
101
+ # end
102
+ #
103
+ # begin
104
+ # STDERR.reopen(sin)
105
+ # yield
106
+ # sin.close
107
+ # ensure
108
+ # STDERR.reopen backup_stderr
109
+ # th_log.join
110
+ # backup_stderr.close
111
+ # Log.logfile = old_logfile
112
+ # end
113
+ # end
114
+ #
115
+ #
116
+ # def self.get_level(level)
117
+ # case level
118
+ # when Numeric
119
+ # level.to_i
120
+ # when String
121
+ # begin
122
+ # Log.const_get(level.upcase)
123
+ # rescue
124
+ # Log.exception $!
125
+ # end
126
+ # when Symbol
127
+ # get_level(level.to_s)
128
+ # end || 0
129
+ # end
130
+ #
131
+ # class << self
132
+ # attr_accessor :logfile, :severity, :nocolor, :tty_size
133
+ # end
134
+ #
135
+ # self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
136
+ #
137
+ # self.ignore_stderr do
138
+ # self.tty_size = begin
139
+ # require "highline/system_extensions.rb"
140
+ # HighLine::SystemExtensions.terminal_size.first
141
+ # rescue Exception
142
+ # nil
143
+ # end
144
+ # end
145
+ #
146
+ # def self.with_severity(level)
147
+ # orig = Log.severity
148
+ # begin
149
+ # Log.severity = level
150
+ # yield
151
+ # ensure
152
+ # Log.severity = orig
153
+ # end
154
+ # end
155
+ #
156
+ # def self.logfile(file=nil)
157
+ # if file.nil?
158
+ # @logfile ||= nil
159
+ # else
160
+ # case file
161
+ # when String
162
+ # @logfile = File.open(file, :mode => 'a')
163
+ # when IO, File
164
+ # @logfile = file
165
+ # else
166
+ # raise "Unkown logfile format: #{file.inspect}"
167
+ # end
168
+ # end
169
+ # end
170
+ #
171
+ # WHITE, DARK, GREEN, YELLOW, RED = Color::SOLARIZED.values_at :base0, :base00, :green, :yellow, :magenta
172
+ #
173
+ # SEVERITY_COLOR = [reset, cyan, green, magenta, blue, yellow, red] #.collect{|e| "\033[#{e}"}
174
+ # HIGHLIGHT = "\033[1m"
175
+ #
176
+ # def self.uncolor(str)
177
+ # "" << Term::ANSIColor.uncolor(str)
178
+ # end
179
+ #
180
+ # def self.reset_color
181
+ # reset
182
+ # end
183
+ #
184
+ # def self.color(severity, str = nil, reset = false)
185
+ # return str.dup || "" if nocolor
186
+ # color = reset ? Term::ANSIColor.reset : ""
187
+ # color << SEVERITY_COLOR[severity] if Integer === severity
188
+ # color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity
189
+ # if str.nil?
190
+ # color
191
+ # else
192
+ # color + str.to_s + self.color(0)
193
+ # end
194
+ # end
195
+ #
196
+ # def self.up_lines(num = 1)
197
+ # nocolor ? "" : "\033[#{num+1}F\033[2K"
198
+ # end
199
+ #
200
+ # def self.down_lines(num = 1)
201
+ # nocolor ? "" : "\033[#{num+1}E"
202
+ # end
203
+ #
204
+ # def self.return_line
205
+ # nocolor ? "" : "\033[1A"
206
+ # end
207
+ #
208
+ # def self.clear_line(out = STDOUT)
209
+ # out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
210
+ # end
211
+ #
212
+ # def self.highlight(str = nil)
213
+ # if str.nil?
214
+ # return "" if nocolor
215
+ # HIGHLIGHT
216
+ # else
217
+ # return str if nocolor
218
+ # HIGHLIGHT + str + color(0)
219
+ # end
220
+ # end
221
+ #
222
+ # LAST = "log"
223
+ # def self.logn(message = nil, severity = MEDIUM, &block)
224
+ # return if severity < self.severity
225
+ # message ||= block.call if block_given?
226
+ # return if message.nil?
227
+ #
228
+ # time = Time.now.strftime("%m/%d/%y-%H:%M:%S.%L")
229
+ #
230
+ # sev_str = severity.to_s
231
+ #
232
+ # prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
233
+ # message = "" << highlight << message << color(0) if severity >= INFO
234
+ # str = prefix << " " << message.to_s
235
+ #
236
+ # LOG_MUTEX.synchronize do
237
+ # if logfile.nil?
238
+ # STDERR.write str
239
+ # else
240
+ # logfile.write str
241
+ # end
242
+ # Log::LAST.replace "log"
243
+ # nil
244
+ # end
245
+ # end
246
+ #
247
+ # def self.log(message = nil, severity = MEDIUM, &block)
248
+ # return if severity < self.severity
249
+ # message ||= block.call if block_given?
250
+ # return if message.nil?
251
+ # message = message + "\n" unless message[-1] == "\n"
252
+ # self.logn message, severity, &block
253
+ # end
254
+ #
255
+ # def self.log_obj_inspect(obj, level, file = $stdout)
256
+ # stack = caller
257
+ #
258
+ # line = Log.last_caller stack
259
+ #
260
+ # level = Log.get_level level
261
+ # name = Log::SEVERITY_NAMES[level] + ": "
262
+ # Log.log Log.color(level, name, true) << line, level
263
+ # Log.log "", level
264
+ # Log.log Log.color(level, "=> ", true) << obj.inspect, level
265
+ # Log.log "", level
266
+ # end
267
+ #
268
+ # def self.log_obj_fingerprint(obj, level, file = $stdout)
269
+ # stack = caller
270
+ #
271
+ # line = Log.last_caller stack
272
+ #
273
+ # level = Log.get_level level
274
+ # name = Log::SEVERITY_NAMES[level] + ": "
275
+ # Log.log Log.color(level, name, true) << line, level
276
+ # Log.log "", level
277
+ # Log.log Log.color(level, "=> ", true) << Misc.fingerprint(obj), level
278
+ # Log.log "", level
279
+ # end
280
+ #
281
+ # def self.debug(message = nil, &block)
282
+ # log(message, DEBUG, &block)
283
+ # end
284
+ #
285
+ # def self.low(message = nil, &block)
286
+ # log(message, LOW, &block)
287
+ # end
288
+ #
289
+ # def self.medium(message = nil, &block)
290
+ # log(message, MEDIUM, &block)
291
+ # end
292
+ #
293
+ # def self.high(message = nil, &block)
294
+ # log(message, HIGH, &block)
295
+ # end
296
+ #
297
+ # def self.info(message = nil, &block)
298
+ # log(message, INFO, &block)
299
+ # end
300
+ #
301
+ # def self.warn(message = nil, &block)
302
+ # log(message, WARN, &block)
303
+ # end
304
+ #
305
+ # def self.error(message = nil, &block)
306
+ # log(message, ERROR, &block)
307
+ # end
308
+ #
309
+ # def self.exception(e)
310
+ # stack = caller
311
+ # if ENV["RBBT_ORIGINAL_STACK"] == 'true'
312
+ # error([e.class.to_s, e.message].compact * ": " )
313
+ # error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace)*"\n")
314
+ # else
315
+ # error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(e.backtrace.reverse)*"\n")
316
+ # error([e.class.to_s, e.message].compact * ": " )
317
+ # end
318
+ # end
319
+ #
320
+ # def self.deprecated(m)
321
+ # stack = caller
322
+ # warn("DEPRECATED: " << Log.last_caller(stack))
323
+ # warn("* " << (m || "").to_s)
324
+ # end
325
+ #
326
+ # def self.color_stack(stack)
327
+ # stack.collect do |line|
328
+ # line = line.sub('`',"'")
329
+ # color = :green if line =~ /workflow/
330
+ # color = :blue if line =~ /rbbt-/
331
+ # Log.color color, line
332
+ # end unless stack.nil?
333
+ # end
334
+ #
335
+ # def self.tsv(tsv, example = false)
336
+ # STDERR.puts Log.color :magenta, "TSV log: " << Log.last_caller(caller).gsub('`',"'")
337
+ # STDERR.puts Log.color(:blue, "=> "<< Misc.fingerprint(tsv), true)
338
+ # STDERR.puts Log.color(:cyan, "=> " << tsv.summary)
339
+ # if example && ! tsv.empty?
340
+ # key = case example
341
+ # when TrueClass, :first, "first"
342
+ # tsv.keys.first
343
+ # when :random, "random"
344
+ # tsv.keys.shuffle.first
345
+ # else
346
+ # example
347
+ # end
348
+ #
349
+ # values = tsv[key]
350
+ # values = [values] if tsv.type == :flat || tsv.type == :single
351
+ # if values.nil?
352
+ # STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}) not present: ") + key
353
+ # else
354
+ # STDERR.puts Log.color(:blue, "Key (#{tsv.key_field}): ") + key
355
+ # tsv.fields.zip(values).each do |field,value|
356
+ # STDERR.puts Log.color(:magenta, field + ": ") + (Array === value ? value * ", " : value.to_s)
357
+ # end
358
+ # end
359
+ # end
360
+ # end
361
+ #
362
+ # def self.stack(stack)
363
+ # LOG_MUTEX.synchronize do
364
+ # if ENV["RBBT_ORIGINAL_STACK"] == 'true'
365
+ # STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
366
+ # color_stack(stack).each do |line|
367
+ # STDERR.puts line
368
+ # end
369
+ # else
370
+ # STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
371
+ # color_stack(stack.reverse).each do |line|
372
+ # STDERR.puts line
373
+ # end
374
+ # end
375
+ # end
376
+ # end
377
+ #
378
+ # def self.count_stack
379
+ # if ! $count_stacks
380
+ # Log.debug "Counting stacks at: " << caller.first
381
+ # return
382
+ # end
383
+ # $stack_counts ||= {}
384
+ # head = $count_stacks_head
385
+ # stack = caller[1..head+1]
386
+ # stack.reverse.each do |line,i|
387
+ # $stack_counts[line] ||= 0
388
+ # $stack_counts[line] += 1
389
+ # end
390
+ # end
391
+ #
392
+ # def self.with_stack_counts(head = 10, total = 100)
393
+ # $count_stacks_head = head
394
+ # $count_stacks = true
395
+ # $stack_counts = {}
396
+ # res = yield
397
+ # $count_stacks = false
398
+ # Log.debug "STACK_COUNTS:\n" + $stack_counts.sort_by{|line,c| c}.reverse.collect{|line,c| [c, line] * " - "}[0..total] * "\n"
399
+ # $stack_counts = {}
400
+ # res
401
+ # end
402
+ #
403
+ # case ENV['RBBT_LOG']
404
+ # when 'DEBUG'
405
+ # self.severity = DEBUG
406
+ # when 'LOW'
407
+ # self.severity = LOW
408
+ # when 'MEDIUM'
409
+ # self.severity = MEDIUM
410
+ # when 'HIGH'
411
+ # self.severity = HIGH
412
+ # when nil
413
+ # self.severity = INFO
414
+ # else
415
+ # self.severity = ENV['RBBT_LOG'].to_i
416
+ # end
417
+ #end
418
+ #
419
+ #def ppp(message)
420
+ # stack = caller
421
+ # puts "#{Log.color :cyan, "PRINT:"} " << stack.first
422
+ # puts ""
423
+ # if message.length > 200 or message.include? "\n"
424
+ # puts Log.color(:cyan, "=>|") << "\n" << message.to_s
425
+ # else
426
+ # puts Log.color(:cyan, "=> ") << message.to_s
427
+ # end
428
+ # puts ""
429
+ #end
430
+ #
431
+ #def fff(object)
432
+ # stack = caller
433
+ # Log.debug{"#{Log.color :cyan, "FINGERPRINT:"} " << stack.first}
434
+ # Log.debug{""}
435
+ # Log.debug{require 'rbbt/util/misc'; "=> " << Misc.fingerprint(object) }
436
+ # Log.debug{""}
437
+ #end
438
+ #
439
+ #def ddd(obj, file = $stdout)
440
+ # Log.log_obj_inspect(obj, :debug, file)
441
+ #end
442
+ #
443
+ #def lll(obj, file = $stdout)
444
+ # Log.log_obj_inspect(obj, :low, file)
445
+ #end
446
+ #
447
+ #def mmm(obj, file = $stdout)
448
+ # Log.log_obj_inspect(obj, :medium, file)
449
+ #end
450
+ #
451
+ #def iii(obj=nil, file = $stdout)
452
+ # Log.log_obj_inspect(obj, :info, file)
453
+ #end
454
+ #
455
+ #def wwww(obj=nil, file = $stdout)
456
+ # Log.log_obj_inspect(obj, :warn, file)
457
+ #end
458
+ #
459
+ #def eee(obj=nil, file = $stdout)
460
+ # Log.log_obj_inspect(obj, :error, file)
461
+ #end
462
+ #
463
+ #def ddf(obj=nil, file = $stdout)
464
+ # Log.log_obj_fingerprint(obj, :debug, file)
465
+ #end
466
+ #
467
+ #def llf(obj=nil, file = $stdout)
468
+ # Log.log_obj_fingerprint(obj, :low, file)
469
+ #end
470
+ #
471
+ #def mmf(obj=nil, file = $stdout)
472
+ # Log.log_obj_fingerprint(obj, :medium, file)
473
+ #end
474
+ #
475
+ #def iif(obj=nil, file = $stdout)
476
+ # Log.log_obj_fingerprint(obj, :info, file)
477
+ #end
478
+ #
479
+ #def wwwf(obj=nil, file = $stdout)
480
+ # Log.log_obj_fingerprint(obj, :warn, file)
481
+ #end
482
+ #
483
+ #def eef(obj=nil, file = $stdout)
484
+ # Log.log_obj_fingerprint(obj, :error, file)
485
+ #end
486
+ #