scout-gear 7.2.0 → 8.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +51 -6
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +33 -29
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/exceptions.rb +1 -0
  10. data/lib/scout/log/color.rb +4 -2
  11. data/lib/scout/log/progress/report.rb +1 -1
  12. data/lib/scout/log/progress/util.rb +71 -2
  13. data/lib/scout/log/progress.rb +1 -1
  14. data/lib/scout/log/trap.rb +107 -0
  15. data/lib/scout/log.rb +56 -21
  16. data/lib/scout/meta_extension.rb +13 -6
  17. data/lib/scout/misc/digest.rb +1 -1
  18. data/lib/scout/misc/format.rb +12 -0
  19. data/lib/scout/misc/helper.rb +31 -0
  20. data/lib/scout/misc/insist.rb +1 -1
  21. data/lib/scout/misc/monitor.rb +12 -1
  22. data/lib/scout/misc/system.rb +10 -0
  23. data/lib/scout/misc.rb +1 -0
  24. data/lib/scout/named_array.rb +65 -3
  25. data/lib/scout/open/lock/lockfile.rb +587 -0
  26. data/lib/scout/open/lock.rb +28 -2
  27. data/lib/scout/open/remote.rb +4 -0
  28. data/lib/scout/open/stream.rb +111 -42
  29. data/lib/scout/open/util.rb +13 -3
  30. data/lib/scout/path/find.rb +9 -1
  31. data/lib/scout/path/util.rb +35 -0
  32. data/lib/scout/persist/serialize.rb +18 -5
  33. data/lib/scout/persist.rb +60 -30
  34. data/lib/scout/resource/path.rb +53 -0
  35. data/lib/scout/resource/produce.rb +0 -8
  36. data/lib/scout/resource/util.rb +2 -1
  37. data/lib/scout/semaphore.rb +8 -1
  38. data/lib/scout/tmpfile.rb +7 -8
  39. data/lib/scout/tsv/attach.rb +177 -0
  40. data/lib/scout/tsv/change_id.rb +40 -0
  41. data/lib/scout/tsv/dumper.rb +85 -54
  42. data/lib/scout/tsv/index.rb +188 -20
  43. data/lib/scout/tsv/open.rb +182 -0
  44. data/lib/scout/tsv/parser.rb +200 -118
  45. data/lib/scout/tsv/path.rb +5 -6
  46. data/lib/scout/tsv/persist/adapter.rb +26 -37
  47. data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
  48. data/lib/scout/tsv/persist/serialize.rb +117 -0
  49. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
  50. data/lib/scout/tsv/persist.rb +4 -2
  51. data/lib/scout/tsv/transformer.rb +141 -0
  52. data/lib/scout/tsv/traverse.rb +136 -37
  53. data/lib/scout/tsv/util/filter.rb +312 -0
  54. data/lib/scout/tsv/util/process.rb +73 -0
  55. data/lib/scout/tsv/util/reorder.rb +81 -0
  56. data/lib/scout/tsv/util/select.rb +265 -0
  57. data/lib/scout/tsv/util/unzip.rb +86 -0
  58. data/lib/scout/tsv/util.rb +126 -19
  59. data/lib/scout/tsv.rb +28 -5
  60. data/lib/scout/work_queue/socket.rb +6 -1
  61. data/lib/scout/work_queue/worker.rb +5 -2
  62. data/lib/scout/work_queue.rb +15 -8
  63. data/lib/scout/workflow/definition.rb +29 -2
  64. data/lib/scout/workflow/step/dependencies.rb +24 -4
  65. data/lib/scout/workflow/step/info.rb +40 -5
  66. data/lib/scout/workflow/step/progress.rb +14 -0
  67. data/lib/scout/workflow/step/provenance.rb +8 -7
  68. data/lib/scout/workflow/step/status.rb +45 -0
  69. data/lib/scout/workflow/step.rb +104 -33
  70. data/lib/scout/workflow/task/inputs.rb +14 -20
  71. data/lib/scout/workflow/task.rb +86 -47
  72. data/lib/scout/workflow/usage.rb +10 -6
  73. data/scout-gear.gemspec +30 -3
  74. data/scout_commands/workflow/task +37 -9
  75. data/scout_commands/workflow/task_old +2 -2
  76. data/test/scout/open/test_stream.rb +61 -59
  77. data/test/scout/path/test_find.rb +10 -1
  78. data/test/scout/resource/test_produce.rb +15 -0
  79. data/test/scout/test_meta_extension.rb +25 -0
  80. data/test/scout/test_named_array.rb +18 -0
  81. data/test/scout/test_persist.rb +67 -0
  82. data/test/scout/test_tmpfile.rb +1 -1
  83. data/test/scout/test_tsv.rb +222 -3
  84. data/test/scout/test_work_queue.rb +21 -18
  85. data/test/scout/tsv/persist/test_adapter.rb +11 -1
  86. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  87. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  88. data/test/scout/tsv/test_attach.rb +227 -0
  89. data/test/scout/tsv/test_change_id.rb +98 -0
  90. data/test/scout/tsv/test_dumper.rb +1 -1
  91. data/test/scout/tsv/test_index.rb +127 -3
  92. data/test/scout/tsv/test_open.rb +167 -0
  93. data/test/scout/tsv/test_parser.rb +45 -3
  94. data/test/scout/tsv/test_persist.rb +9 -0
  95. data/test/scout/tsv/test_transformer.rb +108 -0
  96. data/test/scout/tsv/test_traverse.rb +195 -3
  97. data/test/scout/tsv/test_util.rb +24 -0
  98. data/test/scout/tsv/util/test_filter.rb +188 -0
  99. data/test/scout/tsv/util/test_process.rb +47 -0
  100. data/test/scout/tsv/util/test_reorder.rb +94 -0
  101. data/test/scout/tsv/util/test_select.rb +58 -0
  102. data/test/scout/tsv/util/test_unzip.rb +112 -0
  103. data/test/scout/work_queue/test_socket.rb +0 -1
  104. data/test/scout/work_queue/test_worker.rb +63 -6
  105. data/test/scout/workflow/step/test_load.rb +3 -3
  106. data/test/scout/workflow/step/test_status.rb +31 -0
  107. data/test/scout/workflow/task/test_inputs.rb +14 -14
  108. data/test/scout/workflow/test_step.rb +13 -13
  109. data/test/scout/workflow/test_task.rb +168 -32
  110. data/test/scout/workflow/test_usage.rb +33 -6
  111. data/test/test_helper.rb +3 -1
  112. metadata +29 -2
data/lib/scout/log.rb CHANGED
@@ -1,10 +1,16 @@
1
1
  require_relative 'log/color'
2
2
  require_relative 'log/fingerprint'
3
3
  require_relative 'log/progress'
4
+ require_relative 'log/trap'
4
5
 
5
6
  require 'io/console'
6
7
 
7
8
  module Log
9
+ class << self
10
+ attr_accessor :severity
11
+ attr_writer :tty_size, :logfile
12
+ end
13
+
8
14
  SEVERITY_NAMES ||= begin
9
15
  names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
10
16
  names.each_with_index do |name,i|
@@ -12,13 +18,37 @@ module Log
12
18
  end
13
19
  names
14
20
  end
21
+ def self.default_severity
22
+ @@default_severity ||= begin
23
+ file = File.join(ENV["HOME"], '.scout/etc/log_severity')
24
+ if File.exist? file
25
+ File.open(file) do |f|
26
+ SEVERITY_NAMES.index f.read.strip
27
+ end
28
+ else
29
+ INFO
30
+ end
31
+ end
32
+ @@default_severity
33
+ end
15
34
 
16
- class << self
17
- attr_accessor :severity
18
- attr_writer :tty_size, :logfile
35
+ case ENV['SCOUT_LOG']
36
+ when 'DEBUG'
37
+ self.severity = DEBUG
38
+ when 'LOW'
39
+ self.severity = LOW
40
+ when 'MEDIUM'
41
+ self.severity = MEDIUM
42
+ when 'HIGH'
43
+ self.severity = HIGH
44
+ when nil
45
+ self.severity = default_severity
46
+ else
47
+ self.severity = default_severity
19
48
  end
20
49
 
21
50
 
51
+
22
52
  def self.tty_size
23
53
  @tty_size = begin
24
54
  IO.console.winsize.last
@@ -109,7 +139,11 @@ module Log
109
139
 
110
140
  sev_str = severity.to_s
111
141
 
112
- prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
142
+ if ENV["SCOUT_DEBUG_PID"] == "true"
143
+ prefix = time << "[" << Process.pid.to_s << "]" << color(severity) << "[" << sev_str << "]" << color(0)
144
+ else
145
+ prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
146
+ end
113
147
  message = "" << highlight << message << color(0) if severity >= INFO
114
148
  str = prefix << " " << message.to_s
115
149
 
@@ -187,7 +221,7 @@ module Log
187
221
  def self.exception(e)
188
222
  stack = caller
189
223
  backtrace = e.backtrace || []
190
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
224
+ if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
191
225
  error([e.class.to_s, e.message].compact * ": " )
192
226
  error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
193
227
  else
@@ -243,7 +277,7 @@ module Log
243
277
  end
244
278
 
245
279
  def self.stack(stack)
246
- if ENV["RBBT_ORIGINAL_STACK"] == 'true'
280
+ if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
247
281
  STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
248
282
  color_stack(stack).each do |line|
249
283
  STDERR.puts line
@@ -280,21 +314,6 @@ module Log
280
314
  $stack_counts = {}
281
315
  res
282
316
  end
283
-
284
- case ENV['RBBT_LOG']
285
- when 'DEBUG'
286
- self.severity = DEBUG
287
- when 'LOW'
288
- self.severity = LOW
289
- when 'MEDIUM'
290
- self.severity = MEDIUM
291
- when 'HIGH'
292
- self.severity = HIGH
293
- when nil
294
- self.severity = INFO
295
- else
296
- self.severity = ENV['RBBT_LOG'].to_i
297
- end
298
317
  end
299
318
 
300
319
  def ppp(message)
@@ -373,3 +392,19 @@ def sss(level, &block)
373
392
  end
374
393
  end
375
394
 
395
+ $scout_debug_log = false
396
+ def ccc(obj=nil, file = $stdout)
397
+ if block_given?
398
+ old_scout_debug_log = $scout_debug_log
399
+ $scout_debug_log = 'true'
400
+ begin
401
+ yield
402
+ ensure
403
+ $scout_debug_log = old_scout_debug_log
404
+ end
405
+ else
406
+ Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
407
+ end
408
+ end
409
+
410
+
@@ -2,7 +2,7 @@ module MetaExtension
2
2
  def self.extended(base)
3
3
  meta = class << base; self; end
4
4
 
5
- base.class_variable_set("@@extension_attrs", [])
5
+ base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
6
6
 
7
7
  meta.define_method(:extension_attr) do |*attrs|
8
8
  self.class_variable_get("@@extension_attrs").concat attrs
@@ -11,6 +11,14 @@ module MetaExtension
11
11
  end
12
12
  end
13
13
 
14
+ meta.define_method(:extended) do |obj|
15
+ attrs = self.class_variable_get("@@extension_attrs")
16
+
17
+ obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
18
+ extension_attrs = obj.instance_variable_get(:@extension_attrs)
19
+ extension_attrs.concat attrs
20
+ end
21
+
14
22
  meta.define_method(:setup) do |*args,&block|
15
23
  if block_given?
16
24
  obj, rest = block, args
@@ -18,7 +26,8 @@ module MetaExtension
18
26
  obj, *rest = args
19
27
  end
20
28
  obj = block if obj.nil?
21
- obj.extend base
29
+ obj.extend base unless base === obj
30
+
22
31
  attrs = self.class_variable_get("@@extension_attrs")
23
32
 
24
33
  return if attrs.nil? || attrs.empty?
@@ -27,8 +36,6 @@ module MetaExtension
27
36
  ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
28
37
  (! attrs.length != 1 ))
29
38
 
30
-
31
-
32
39
  pairs = rlast
33
40
  else
34
41
  pairs = attrs.zip(rest)
@@ -43,14 +50,14 @@ module MetaExtension
43
50
 
44
51
  base.define_method(:extension_attr_hash) do
45
52
  attr_hash = {}
46
- meta.class_variable_get("@@extension_attrs").each do |name|
53
+ @extension_attrs.each do |name|
47
54
  attr_hash[name] = self.instance_variable_get("@#{name}")
48
55
  end
49
56
  attr_hash
50
57
  end
51
58
 
52
59
  base.define_method(:annotate) do |other|
53
- attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
60
+ attr_values = @extension_attrs.collect do |a|
54
61
  self.instance_variable_get("@#{a}")
55
62
  end
56
63
  base.setup(other, *attr_values)
@@ -40,7 +40,7 @@ module Misc
40
40
  end
41
41
 
42
42
  def self.digest(obj)
43
- str = Misc.digest_str(obj)
43
+ str = String === obj ? obj : Misc.digest_str(obj)
44
44
  Digest::MD5.hexdigest(str)
45
45
  end
46
46
 
@@ -25,6 +25,18 @@ module Misc
25
25
  str
26
26
  end
27
27
 
28
+ CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
29
+ def self.format_seconds_short(time)
30
+ if time < 0.0001
31
+ "%.5g" % time + CHAR_SENCONDS
32
+ elsif time < 60
33
+ "%.2g" % time + CHAR_SENCONDS
34
+ else
35
+ format_seconds(time)
36
+ end
37
+ end
38
+
39
+
28
40
 
29
41
  MAX_WIDTH = 100
30
42
  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
@@ -0,0 +1,31 @@
1
+ module Misc
2
+ def self.intersect_sorted_arrays(a1, a2)
3
+ e1, e2 = a1.shift, a2.shift
4
+ intersect = []
5
+ while true
6
+ break if e1.nil? or e2.nil?
7
+ case e1 <=> e2
8
+ when 0
9
+ intersect << e1
10
+ e1, e2 = a1.shift, a2.shift
11
+ when -1
12
+ e1 = a1.shift while not e1.nil? and e1 < e2
13
+ when 1
14
+ e2 = a2.shift
15
+ e2 = a2.shift while not e2.nil? and e2 < e1
16
+ end
17
+ end
18
+ intersect
19
+ end
20
+
21
+ def self.counts(array)
22
+ counts = {}
23
+ array.each do |e|
24
+ counts[e] ||= 0
25
+ counts[e] += 1
26
+ end
27
+
28
+ counts
29
+ end
30
+
31
+ end
@@ -32,7 +32,7 @@ module Misc
32
32
  end
33
33
  raise $!
34
34
  rescue Exception
35
- Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
35
+ Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
36
36
  if msg
37
37
  Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
38
  elsif FalseClass === msg
@@ -14,7 +14,7 @@ module Misc
14
14
  if message
15
15
  puts "#{message }: #{ repeats } repeats"
16
16
  else
17
- puts "Benchmark for #{ repeats } repeats"
17
+ puts "Benchmark for #{ repeats } repeats (#{caller.first})"
18
18
  end
19
19
  puts measure
20
20
  rescue Exception
@@ -41,4 +41,15 @@ module Misc
41
41
 
42
42
  res
43
43
  end
44
+
45
+ def self.exec_time(&block)
46
+ start = Time.now
47
+ eend = nil
48
+ begin
49
+ yield
50
+ ensure
51
+ eend = Time.now
52
+ end
53
+ eend - start
54
+ end
44
55
  end
@@ -12,4 +12,14 @@ module Misc
12
12
  end
13
13
  end
14
14
  end
15
+
16
+ def self.with_env(var, value, &block)
17
+ old_value = ENV[var]
18
+ begin
19
+ ENV[var] = value
20
+ yield
21
+ ensure
22
+ ENV[var] = old_value
23
+ end
24
+ end
15
25
  end
data/lib/scout/misc.rb CHANGED
@@ -4,6 +4,7 @@ require_relative 'misc/digest'
4
4
  require_relative 'misc/filesystem'
5
5
  require_relative 'misc/monitor'
6
6
  require_relative 'misc/system'
7
+ require_relative 'misc/helper'
7
8
 
8
9
  module Misc
9
10
  end
@@ -1,13 +1,25 @@
1
1
  require_relative 'meta_extension'
2
2
  module NamedArray
3
3
  extend MetaExtension
4
- extension_attr :fields
4
+ extension_attr :fields, :key
5
5
 
6
- def self.identify_name(names, selected)
6
+ def self.field_match(field, name)
7
+ if (String === field) && (String === name)
8
+ field == name ||
9
+ field.start_with?(name) || field.include?("(" + name + ")") ||
10
+ name.start_with?(field) || name.include?("(" + field + ")")
11
+ else
12
+ field == name
13
+ end
14
+ end
15
+
16
+ def self.identify_name(names, selected, strict: false)
7
17
  res = (Array === selected ? selected : [selected]).collect do |field|
8
18
  case field
9
19
  when nil
10
20
  0
21
+ when Range
22
+ field
11
23
  when Integer
12
24
  field
13
25
  when Symbol
@@ -26,7 +38,8 @@ module NamedArray
26
38
  if field =~ /^\d+$/
27
39
  next identify_names(names, field.to_i)
28
40
  end
29
- pos = names.index{|name| name.start_with?(field) }
41
+ next pos if strict
42
+ pos = names.index{|name| field_match(field, name) }
30
43
  next pos if pos
31
44
  nil
32
45
  else
@@ -37,6 +50,10 @@ module NamedArray
37
50
  Array === selected ? res : res.first
38
51
  end
39
52
 
53
+ def identify_name(selected)
54
+ NamedArray.identify_name(fields, selected)
55
+ end
56
+
40
57
  def positions(fields)
41
58
  if Array == fields
42
59
  fields.collect{|field|
@@ -49,6 +66,7 @@ module NamedArray
49
66
 
50
67
  def [](key)
51
68
  pos = NamedArray.identify_name(@fields, key)
69
+ return nil if pos.nil?
52
70
  super(pos)
53
71
  end
54
72
 
@@ -65,4 +83,48 @@ module NamedArray
65
83
  end
66
84
  IndiferentHash.setup hash
67
85
  end
86
+
87
+ def values_at(*positions)
88
+ super(*identify_name(positions))
89
+ end
90
+
91
+ def self._zip_fields(array, max = nil)
92
+ return [] if array.nil? or array.empty? or (first = array.first).nil?
93
+
94
+ max = array.collect{|l| l.length }.max if max.nil?
95
+
96
+ rest = array[1..-1].collect{|v|
97
+ v.length == 1 & max > 1 ? v * max : v
98
+ }
99
+
100
+ first = first * max if first.length == 1 and max > 1
101
+
102
+ first.zip(*rest)
103
+ end
104
+
105
+ def self.zip_fields(array)
106
+ if array.length < 10000
107
+ _zip_fields(array)
108
+ else
109
+ zipped_slices = []
110
+ max = array.collect{|l| l.length}.max
111
+ array.each_slice(10000) do |slice|
112
+ zipped_slices << _zip_fields(slice, max)
113
+ end
114
+ new = zipped_slices.first
115
+ zipped_slices[1..-1].each do |rest|
116
+ rest.each_with_index do |list,i|
117
+ new[i].concat list
118
+ end
119
+ end
120
+ new
121
+ end
122
+ end
123
+
124
+ def self.add_zipped(source, new)
125
+ source.zip(new).each do |s,n|
126
+ s.concat(n)
127
+ end
128
+ source
129
+ end
68
130
  end