scout-gear 7.3.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -2,7 +2,7 @@ module MetaExtension
2
2
  def self.extended(base)
3
3
  meta = class << base; self; end
4
4
 
5
- base.class_variable_set("@@extension_attrs", [])
5
+ base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
6
6
 
7
7
  meta.define_method(:extension_attr) do |*attrs|
8
8
  self.class_variable_get("@@extension_attrs").concat attrs
@@ -11,6 +11,14 @@ module MetaExtension
11
11
  end
12
12
  end
13
13
 
14
+ meta.define_method(:extended) do |obj|
15
+ attrs = self.class_variable_get("@@extension_attrs")
16
+
17
+ obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
18
+ extension_attrs = obj.instance_variable_get(:@extension_attrs)
19
+ extension_attrs.concat attrs
20
+ end
21
+
14
22
  meta.define_method(:setup) do |*args,&block|
15
23
  if block_given?
16
24
  obj, rest = block, args
@@ -18,7 +26,8 @@ module MetaExtension
18
26
  obj, *rest = args
19
27
  end
20
28
  obj = block if obj.nil?
21
- obj.extend base
29
+ obj.extend base unless base === obj
30
+
22
31
  attrs = self.class_variable_get("@@extension_attrs")
23
32
 
24
33
  return if attrs.nil? || attrs.empty?
@@ -27,8 +36,6 @@ module MetaExtension
27
36
  ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
28
37
  (! attrs.length != 1 ))
29
38
 
30
-
31
-
32
39
  pairs = rlast
33
40
  else
34
41
  pairs = attrs.zip(rest)
@@ -43,14 +50,14 @@ module MetaExtension
43
50
 
44
51
  base.define_method(:extension_attr_hash) do
45
52
  attr_hash = {}
46
- meta.class_variable_get("@@extension_attrs").each do |name|
53
+ @extension_attrs.each do |name|
47
54
  attr_hash[name] = self.instance_variable_get("@#{name}")
48
55
  end
49
56
  attr_hash
50
57
  end
51
58
 
52
59
  base.define_method(:annotate) do |other|
53
- attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
60
+ attr_values = @extension_attrs.collect do |a|
54
61
  self.instance_variable_get("@#{a}")
55
62
  end
56
63
  base.setup(other, *attr_values)
@@ -40,7 +40,7 @@ module Misc
40
40
  end
41
41
 
42
42
  def self.digest(obj)
43
- str = Misc.digest_str(obj)
43
+ str = String === obj ? obj : Misc.digest_str(obj)
44
44
  Digest::MD5.hexdigest(str)
45
45
  end
46
46
 
@@ -25,6 +25,18 @@ module Misc
25
25
  str
26
26
  end
27
27
 
28
+ CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
29
+ def self.format_seconds_short(time)
30
+ if time < 0.0001
31
+ "%.5g" % time + CHAR_SENCONDS
32
+ elsif time < 60
33
+ "%.2g" % time + CHAR_SENCONDS
34
+ else
35
+ format_seconds(time)
36
+ end
37
+ end
38
+
39
+
28
40
 
29
41
  MAX_WIDTH = 100
30
42
  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
@@ -32,7 +32,7 @@ module Misc
32
32
  end
33
33
  raise $!
34
34
  rescue Exception
35
- Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
35
+ Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
36
36
  if msg
37
37
  Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
38
  elsif FalseClass === msg
@@ -41,4 +41,15 @@ module Misc
41
41
 
42
42
  res
43
43
  end
44
+
45
+ def self.exec_time(&block)
46
+ start = Time.now
47
+ eend = nil
48
+ begin
49
+ yield
50
+ ensure
51
+ eend = Time.now
52
+ end
53
+ eend - start
54
+ end
44
55
  end
@@ -12,4 +12,14 @@ module Misc
12
12
  end
13
13
  end
14
14
  end
15
+
16
+ def self.with_env(var, value, &block)
17
+ old_value = ENV[var]
18
+ begin
19
+ ENV[var] = value
20
+ yield
21
+ ensure
22
+ ENV[var] = old_value
23
+ end
24
+ end
15
25
  end
@@ -1,13 +1,25 @@
1
1
  require_relative 'meta_extension'
2
2
  module NamedArray
3
3
  extend MetaExtension
4
- extension_attr :fields
4
+ extension_attr :fields, :key
5
5
 
6
- def self.identify_name(names, selected)
6
+ def self.field_match(field, name)
7
+ if (String === field) && (String === name)
8
+ field == name ||
9
+ field.start_with?(name) || field.include?("(" + name + ")") ||
10
+ name.start_with?(field) || name.include?("(" + field + ")")
11
+ else
12
+ field == name
13
+ end
14
+ end
15
+
16
+ def self.identify_name(names, selected, strict: false)
7
17
  res = (Array === selected ? selected : [selected]).collect do |field|
8
18
  case field
9
19
  when nil
10
20
  0
21
+ when Range
22
+ field
11
23
  when Integer
12
24
  field
13
25
  when Symbol
@@ -26,7 +38,8 @@ module NamedArray
26
38
  if field =~ /^\d+$/
27
39
  next identify_names(names, field.to_i)
28
40
  end
29
- pos = names.index{|name| name.start_with?(field) }
41
+ next pos if strict
42
+ pos = names.index{|name| field_match(field, name) }
30
43
  next pos if pos
31
44
  nil
32
45
  else
@@ -37,6 +50,10 @@ module NamedArray
37
50
  Array === selected ? res : res.first
38
51
  end
39
52
 
53
+ def identify_name(selected)
54
+ NamedArray.identify_name(fields, selected)
55
+ end
56
+
40
57
  def positions(fields)
41
58
  if Array == fields
42
59
  fields.collect{|field|
@@ -49,6 +66,7 @@ module NamedArray
49
66
 
50
67
  def [](key)
51
68
  pos = NamedArray.identify_name(@fields, key)
69
+ return nil if pos.nil?
52
70
  super(pos)
53
71
  end
54
72
 
@@ -65,4 +83,48 @@ module NamedArray
65
83
  end
66
84
  IndiferentHash.setup hash
67
85
  end
86
+
87
+ def values_at(*positions)
88
+ super(*identify_name(positions))
89
+ end
90
+
91
+ def self._zip_fields(array, max = nil)
92
+ return [] if array.nil? or array.empty? or (first = array.first).nil?
93
+
94
+ max = array.collect{|l| l.length }.max if max.nil?
95
+
96
+ rest = array[1..-1].collect{|v|
97
+ v.length == 1 & max > 1 ? v * max : v
98
+ }
99
+
100
+ first = first * max if first.length == 1 and max > 1
101
+
102
+ first.zip(*rest)
103
+ end
104
+
105
+ def self.zip_fields(array)
106
+ if array.length < 10000
107
+ _zip_fields(array)
108
+ else
109
+ zipped_slices = []
110
+ max = array.collect{|l| l.length}.max
111
+ array.each_slice(10000) do |slice|
112
+ zipped_slices << _zip_fields(slice, max)
113
+ end
114
+ new = zipped_slices.first
115
+ zipped_slices[1..-1].each do |rest|
116
+ rest.each_with_index do |list,i|
117
+ new[i].concat list
118
+ end
119
+ end
120
+ new
121
+ end
122
+ end
123
+
124
+ def self.add_zipped(source, new)
125
+ source.zip(new).each do |s,n|
126
+ s.concat(n)
127
+ end
128
+ source
129
+ end
68
130
  end