scout-gear 7.2.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +51 -6
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +33 -29
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/exceptions.rb +1 -0
  10. data/lib/scout/log/color.rb +4 -2
  11. data/lib/scout/log/progress/report.rb +1 -1
  12. data/lib/scout/log/progress/util.rb +71 -2
  13. data/lib/scout/log/progress.rb +1 -1
  14. data/lib/scout/log/trap.rb +107 -0
  15. data/lib/scout/log.rb +56 -21
  16. data/lib/scout/meta_extension.rb +13 -6
  17. data/lib/scout/misc/digest.rb +1 -1
  18. data/lib/scout/misc/format.rb +12 -0
  19. data/lib/scout/misc/helper.rb +31 -0
  20. data/lib/scout/misc/insist.rb +1 -1
  21. data/lib/scout/misc/monitor.rb +12 -1
  22. data/lib/scout/misc/system.rb +10 -0
  23. data/lib/scout/misc.rb +1 -0
  24. data/lib/scout/named_array.rb +65 -3
  25. data/lib/scout/open/lock/lockfile.rb +587 -0
  26. data/lib/scout/open/lock.rb +28 -2
  27. data/lib/scout/open/remote.rb +4 -0
  28. data/lib/scout/open/stream.rb +111 -42
  29. data/lib/scout/open/util.rb +13 -3
  30. data/lib/scout/path/find.rb +9 -1
  31. data/lib/scout/path/util.rb +35 -0
  32. data/lib/scout/persist/serialize.rb +18 -5
  33. data/lib/scout/persist.rb +60 -30
  34. data/lib/scout/resource/path.rb +53 -0
  35. data/lib/scout/resource/produce.rb +0 -8
  36. data/lib/scout/resource/util.rb +2 -1
  37. data/lib/scout/semaphore.rb +8 -1
  38. data/lib/scout/tmpfile.rb +7 -8
  39. data/lib/scout/tsv/attach.rb +177 -0
  40. data/lib/scout/tsv/change_id.rb +40 -0
  41. data/lib/scout/tsv/dumper.rb +85 -54
  42. data/lib/scout/tsv/index.rb +188 -20
  43. data/lib/scout/tsv/open.rb +182 -0
  44. data/lib/scout/tsv/parser.rb +200 -118
  45. data/lib/scout/tsv/path.rb +5 -6
  46. data/lib/scout/tsv/persist/adapter.rb +26 -37
  47. data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
  48. data/lib/scout/tsv/persist/serialize.rb +117 -0
  49. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
  50. data/lib/scout/tsv/persist.rb +4 -2
  51. data/lib/scout/tsv/transformer.rb +141 -0
  52. data/lib/scout/tsv/traverse.rb +136 -37
  53. data/lib/scout/tsv/util/filter.rb +312 -0
  54. data/lib/scout/tsv/util/process.rb +73 -0
  55. data/lib/scout/tsv/util/reorder.rb +81 -0
  56. data/lib/scout/tsv/util/select.rb +265 -0
  57. data/lib/scout/tsv/util/unzip.rb +86 -0
  58. data/lib/scout/tsv/util.rb +126 -19
  59. data/lib/scout/tsv.rb +28 -5
  60. data/lib/scout/work_queue/socket.rb +6 -1
  61. data/lib/scout/work_queue/worker.rb +5 -2
  62. data/lib/scout/work_queue.rb +15 -8
  63. data/lib/scout/workflow/definition.rb +29 -2
  64. data/lib/scout/workflow/step/dependencies.rb +24 -4
  65. data/lib/scout/workflow/step/info.rb +40 -5
  66. data/lib/scout/workflow/step/progress.rb +14 -0
  67. data/lib/scout/workflow/step/provenance.rb +8 -7
  68. data/lib/scout/workflow/step/status.rb +45 -0
  69. data/lib/scout/workflow/step.rb +104 -33
  70. data/lib/scout/workflow/task/inputs.rb +14 -20
  71. data/lib/scout/workflow/task.rb +86 -47
  72. data/lib/scout/workflow/usage.rb +10 -6
  73. data/scout-gear.gemspec +30 -3
  74. data/scout_commands/workflow/task +37 -9
  75. data/scout_commands/workflow/task_old +2 -2
  76. data/test/scout/open/test_stream.rb +61 -59
  77. data/test/scout/path/test_find.rb +10 -1
  78. data/test/scout/resource/test_produce.rb +15 -0
  79. data/test/scout/test_meta_extension.rb +25 -0
  80. data/test/scout/test_named_array.rb +18 -0
  81. data/test/scout/test_persist.rb +67 -0
  82. data/test/scout/test_tmpfile.rb +1 -1
  83. data/test/scout/test_tsv.rb +222 -3
  84. data/test/scout/test_work_queue.rb +21 -18
  85. data/test/scout/tsv/persist/test_adapter.rb +11 -1
  86. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  87. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  88. data/test/scout/tsv/test_attach.rb +227 -0
  89. data/test/scout/tsv/test_change_id.rb +98 -0
  90. data/test/scout/tsv/test_dumper.rb +1 -1
  91. data/test/scout/tsv/test_index.rb +127 -3
  92. data/test/scout/tsv/test_open.rb +167 -0
  93. data/test/scout/tsv/test_parser.rb +45 -3
  94. data/test/scout/tsv/test_persist.rb +9 -0
  95. data/test/scout/tsv/test_transformer.rb +108 -0
  96. data/test/scout/tsv/test_traverse.rb +195 -3
  97. data/test/scout/tsv/test_util.rb +24 -0
  98. data/test/scout/tsv/util/test_filter.rb +188 -0
  99. data/test/scout/tsv/util/test_process.rb +47 -0
  100. data/test/scout/tsv/util/test_reorder.rb +94 -0
  101. data/test/scout/tsv/util/test_select.rb +58 -0
  102. data/test/scout/tsv/util/test_unzip.rb +112 -0
  103. data/test/scout/work_queue/test_socket.rb +0 -1
  104. data/test/scout/work_queue/test_worker.rb +63 -6
  105. data/test/scout/workflow/step/test_load.rb +3 -3
  106. data/test/scout/workflow/step/test_status.rb +31 -0
  107. data/test/scout/workflow/task/test_inputs.rb +14 -14
  108. data/test/scout/workflow/test_step.rb +13 -13
  109. data/test/scout/workflow/test_task.rb +168 -32
  110. data/test/scout/workflow/test_usage.rb +33 -6
  111. data/test/test_helper.rb +3 -1
  112. metadata +29 -2
data/lib/scout/log.rb CHANGED
@@ -1,10 +1,16 @@
1
1
  require_relative 'log/color'
2
2
  require_relative 'log/fingerprint'
3
3
  require_relative 'log/progress'
4
+ require_relative 'log/trap'
4
5
 
5
6
  require 'io/console'
6
7
 
7
8
  module Log
9
+ class << self
10
+ attr_accessor :severity
11
+ attr_writer :tty_size, :logfile
12
+ end
13
+
8
14
  SEVERITY_NAMES ||= begin
9
15
  names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
10
16
  names.each_with_index do |name,i|
@@ -12,13 +18,37 @@ module Log
12
18
  end
13
19
  names
14
20
  end
21
+ def self.default_severity
22
+ @@default_severity ||= begin
23
+ file = File.join(ENV["HOME"], '.scout/etc/log_severity')
24
+ if File.exist? file
25
+ File.open(file) do |f|
26
+ SEVERITY_NAMES.index f.read.strip
27
+ end
28
+ else
29
+ INFO
30
+ end
31
+ end
32
+ @@default_severity
33
+ end
15
34
 
16
- class << self
17
- attr_accessor :severity
18
- attr_writer :tty_size, :logfile
35
+ case ENV['SCOUT_LOG']
36
+ when 'DEBUG'
37
+ self.severity = DEBUG
38
+ when 'LOW'
39
+ self.severity = LOW
40
+ when 'MEDIUM'
41
+ self.severity = MEDIUM
42
+ when 'HIGH'
43
+ self.severity = HIGH
44
+ when nil
45
+ self.severity = default_severity
46
+ else
47
+ self.severity = default_severity
19
48
  end
20
49
 
21
50
 
51
+
22
52
  def self.tty_size
23
53
  @tty_size = begin
24
54
  IO.console.winsize.last
@@ -109,7 +139,11 @@ module Log
109
139
 
110
140
  sev_str = severity.to_s
111
141
 
112
- prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
142
+ if ENV["SCOUT_DEBUG_PID"] == "true"
143
+ prefix = time << "[" << Process.pid.to_s << "]" << color(severity) << "[" << sev_str << "]" << color(0)
144
+ else
145
+ prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
146
+ end
113
147
  message = "" << highlight << message << color(0) if severity >= INFO
114
148
  str = prefix << " " << message.to_s
115
149
 
@@ -187,7 +221,7 @@ module Log
187
221
  def self.exception(e)
188
222
  stack = caller
189
223
  backtrace = e.backtrace || []
190
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
224
+ if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
191
225
  error([e.class.to_s, e.message].compact * ": " )
192
226
  error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
193
227
  else
@@ -243,7 +277,7 @@ module Log
243
277
  end
244
278
 
245
279
  def self.stack(stack)
246
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
280
+ if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
247
281
  STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
248
282
  color_stack(stack).each do |line|
249
283
  STDERR.puts line
@@ -280,21 +314,6 @@ module Log
280
314
  $stack_counts = {}
281
315
  res
282
316
  end
283
-
284
- case ENV['RBBT_LOG']
285
- when 'DEBUG'
286
- self.severity = DEBUG
287
- when 'LOW'
288
- self.severity = LOW
289
- when 'MEDIUM'
290
- self.severity = MEDIUM
291
- when 'HIGH'
292
- self.severity = HIGH
293
- when nil
294
- self.severity = INFO
295
- else
296
- self.severity = ENV['RBBT_LOG'].to_i
297
- end
298
317
  end
299
318
 
300
319
  def ppp(message)
@@ -373,3 +392,19 @@ def sss(level, &block)
373
392
  end
374
393
  end
375
394
 
395
+ $scout_debug_log = false
396
+ def ccc(obj=nil, file = $stdout)
397
+ if block_given?
398
+ old_scout_debug_log = $scout_debug_log
399
+ $scout_debug_log = 'true'
400
+ begin
401
+ yield
402
+ ensure
403
+ $scout_debug_log = old_scout_debug_log
404
+ end
405
+ else
406
+ Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
407
+ end
408
+ end
409
+
410
+
@@ -2,7 +2,7 @@ module MetaExtension
2
2
  def self.extended(base)
3
3
  meta = class << base; self; end
4
4
 
5
- base.class_variable_set("@@extension_attrs", [])
5
+ base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
6
6
 
7
7
  meta.define_method(:extension_attr) do |*attrs|
8
8
  self.class_variable_get("@@extension_attrs").concat attrs
@@ -11,6 +11,14 @@ module MetaExtension
11
11
  end
12
12
  end
13
13
 
14
+ meta.define_method(:extended) do |obj|
15
+ attrs = self.class_variable_get("@@extension_attrs")
16
+
17
+ obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
18
+ extension_attrs = obj.instance_variable_get(:@extension_attrs)
19
+ extension_attrs.concat attrs
20
+ end
21
+
14
22
  meta.define_method(:setup) do |*args,&block|
15
23
  if block_given?
16
24
  obj, rest = block, args
@@ -18,7 +26,8 @@ module MetaExtension
18
26
  obj, *rest = args
19
27
  end
20
28
  obj = block if obj.nil?
21
- obj.extend base
29
+ obj.extend base unless base === obj
30
+
22
31
  attrs = self.class_variable_get("@@extension_attrs")
23
32
 
24
33
  return if attrs.nil? || attrs.empty?
@@ -27,8 +36,6 @@ module MetaExtension
27
36
  ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
28
37
  (! attrs.length != 1 ))
29
38
 
30
-
31
-
32
39
  pairs = rlast
33
40
  else
34
41
  pairs = attrs.zip(rest)
@@ -43,14 +50,14 @@ module MetaExtension
43
50
 
44
51
  base.define_method(:extension_attr_hash) do
45
52
  attr_hash = {}
46
- meta.class_variable_get("@@extension_attrs").each do |name|
53
+ @extension_attrs.each do |name|
47
54
  attr_hash[name] = self.instance_variable_get("@#{name}")
48
55
  end
49
56
  attr_hash
50
57
  end
51
58
 
52
59
  base.define_method(:annotate) do |other|
53
- attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
60
+ attr_values = @extension_attrs.collect do |a|
54
61
  self.instance_variable_get("@#{a}")
55
62
  end
56
63
  base.setup(other, *attr_values)
@@ -40,7 +40,7 @@ module Misc
40
40
  end
41
41
 
42
42
  def self.digest(obj)
43
- str = Misc.digest_str(obj)
43
+ str = String === obj ? obj : Misc.digest_str(obj)
44
44
  Digest::MD5.hexdigest(str)
45
45
  end
46
46
 
@@ -25,6 +25,18 @@ module Misc
25
25
  str
26
26
  end
27
27
 
28
+ CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
29
+ def self.format_seconds_short(time)
30
+ if time < 0.0001
31
+ "%.5g" % time + CHAR_SENCONDS
32
+ elsif time < 60
33
+ "%.2g" % time + CHAR_SENCONDS
34
+ else
35
+ format_seconds(time)
36
+ end
37
+ end
38
+
39
+
28
40
 
29
41
  MAX_WIDTH = 100
30
42
  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
@@ -0,0 +1,31 @@
1
+ module Misc
2
+ def self.intersect_sorted_arrays(a1, a2)
3
+ e1, e2 = a1.shift, a2.shift
4
+ intersect = []
5
+ while true
6
+ break if e1.nil? or e2.nil?
7
+ case e1 <=> e2
8
+ when 0
9
+ intersect << e1
10
+ e1, e2 = a1.shift, a2.shift
11
+ when -1
12
+ e1 = a1.shift while not e1.nil? and e1 < e2
13
+ when 1
14
+ e2 = a2.shift
15
+ e2 = a2.shift while not e2.nil? and e2 < e1
16
+ end
17
+ end
18
+ intersect
19
+ end
20
+
21
+ def self.counts(array)
22
+ counts = {}
23
+ array.each do |e|
24
+ counts[e] ||= 0
25
+ counts[e] += 1
26
+ end
27
+
28
+ counts
29
+ end
30
+
31
+ end
@@ -32,7 +32,7 @@ module Misc
32
32
  end
33
33
  raise $!
34
34
  rescue Exception
35
- Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
35
+ Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
36
36
  if msg
37
37
  Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
38
  elsif FalseClass === msg
@@ -14,7 +14,7 @@ module Misc
14
14
  if message
15
15
  puts "#{message }: #{ repeats } repeats"
16
16
  else
17
- puts "Benchmark for #{ repeats } repeats"
17
+ puts "Benchmark for #{ repeats } repeats (#{caller.first})"
18
18
  end
19
19
  puts measure
20
20
  rescue Exception
@@ -41,4 +41,15 @@ module Misc
41
41
 
42
42
  res
43
43
  end
44
+
45
+ def self.exec_time(&block)
46
+ start = Time.now
47
+ eend = nil
48
+ begin
49
+ yield
50
+ ensure
51
+ eend = Time.now
52
+ end
53
+ eend - start
54
+ end
44
55
  end
@@ -12,4 +12,14 @@ module Misc
12
12
  end
13
13
  end
14
14
  end
15
+
16
+ def self.with_env(var, value, &block)
17
+ old_value = ENV[var]
18
+ begin
19
+ ENV[var] = value
20
+ yield
21
+ ensure
22
+ ENV[var] = old_value
23
+ end
24
+ end
15
25
  end
data/lib/scout/misc.rb CHANGED
@@ -4,6 +4,7 @@ require_relative 'misc/digest'
4
4
  require_relative 'misc/filesystem'
5
5
  require_relative 'misc/monitor'
6
6
  require_relative 'misc/system'
7
+ require_relative 'misc/helper'
7
8
 
8
9
  module Misc
9
10
  end
@@ -1,13 +1,25 @@
1
1
  require_relative 'meta_extension'
2
2
  module NamedArray
3
3
  extend MetaExtension
4
- extension_attr :fields
4
+ extension_attr :fields, :key
5
5
 
6
- def self.identify_name(names, selected)
6
+ def self.field_match(field, name)
7
+ if (String === field) && (String === name)
8
+ field == name ||
9
+ field.start_with?(name) || field.include?("(" + name + ")") ||
10
+ name.start_with?(field) || name.include?("(" + field + ")")
11
+ else
12
+ field == name
13
+ end
14
+ end
15
+
16
+ def self.identify_name(names, selected, strict: false)
7
17
  res = (Array === selected ? selected : [selected]).collect do |field|
8
18
  case field
9
19
  when nil
10
20
  0
21
+ when Range
22
+ field
11
23
  when Integer
12
24
  field
13
25
  when Symbol
@@ -26,7 +38,8 @@ module NamedArray
26
38
  if field =~ /^\d+$/
27
39
  next identify_names(names, field.to_i)
28
40
  end
29
- pos = names.index{|name| name.start_with?(field) }
41
+ next pos if strict
42
+ pos = names.index{|name| field_match(field, name) }
30
43
  next pos if pos
31
44
  nil
32
45
  else
@@ -37,6 +50,10 @@ module NamedArray
37
50
  Array === selected ? res : res.first
38
51
  end
39
52
 
53
+ def identify_name(selected)
54
+ NamedArray.identify_name(fields, selected)
55
+ end
56
+
40
57
  def positions(fields)
41
58
  if Array == fields
42
59
  fields.collect{|field|
@@ -49,6 +66,7 @@ module NamedArray
49
66
 
50
67
  def [](key)
51
68
  pos = NamedArray.identify_name(@fields, key)
69
+ return nil if pos.nil?
52
70
  super(pos)
53
71
  end
54
72
 
@@ -65,4 +83,48 @@ module NamedArray
65
83
  end
66
84
  IndiferentHash.setup hash
67
85
  end
86
+
87
+ def values_at(*positions)
88
+ super(*identify_name(positions))
89
+ end
90
+
91
+ def self._zip_fields(array, max = nil)
92
+ return [] if array.nil? or array.empty? or (first = array.first).nil?
93
+
94
+ max = array.collect{|l| l.length }.max if max.nil?
95
+
96
+ rest = array[1..-1].collect{|v|
97
+ v.length == 1 & max > 1 ? v * max : v
98
+ }
99
+
100
+ first = first * max if first.length == 1 and max > 1
101
+
102
+ first.zip(*rest)
103
+ end
104
+
105
+ def self.zip_fields(array)
106
+ if array.length < 10000
107
+ _zip_fields(array)
108
+ else
109
+ zipped_slices = []
110
+ max = array.collect{|l| l.length}.max
111
+ array.each_slice(10000) do |slice|
112
+ zipped_slices << _zip_fields(slice, max)
113
+ end
114
+ new = zipped_slices.first
115
+ zipped_slices[1..-1].each do |rest|
116
+ rest.each_with_index do |list,i|
117
+ new[i].concat list
118
+ end
119
+ end
120
+ new
121
+ end
122
+ end
123
+
124
+ def self.add_zipped(source, new)
125
+ source.zip(new).each do |s,n|
126
+ s.concat(n)
127
+ end
128
+ source
129
+ end
68
130
  end