scout-gear 7.3.0 → 8.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -2,7 +2,7 @@ module MetaExtension
2
2
  def self.extended(base)
3
3
  meta = class << base; self; end
4
4
 
5
- base.class_variable_set("@@extension_attrs", [])
5
+ base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
6
6
 
7
7
  meta.define_method(:extension_attr) do |*attrs|
8
8
  self.class_variable_get("@@extension_attrs").concat attrs
@@ -11,6 +11,14 @@ module MetaExtension
11
11
  end
12
12
  end
13
13
 
14
+ meta.define_method(:extended) do |obj|
15
+ attrs = self.class_variable_get("@@extension_attrs")
16
+
17
+ obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
18
+ extension_attrs = obj.instance_variable_get(:@extension_attrs)
19
+ extension_attrs.concat attrs
20
+ end
21
+
14
22
  meta.define_method(:setup) do |*args,&block|
15
23
  if block_given?
16
24
  obj, rest = block, args
@@ -18,7 +26,8 @@ module MetaExtension
18
26
  obj, *rest = args
19
27
  end
20
28
  obj = block if obj.nil?
21
- obj.extend base
29
+ obj.extend base unless base === obj
30
+
22
31
  attrs = self.class_variable_get("@@extension_attrs")
23
32
 
24
33
  return if attrs.nil? || attrs.empty?
@@ -27,8 +36,6 @@ module MetaExtension
27
36
  ((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
28
37
  (! attrs.length != 1 ))
29
38
 
30
-
31
-
32
39
  pairs = rlast
33
40
  else
34
41
  pairs = attrs.zip(rest)
@@ -43,14 +50,14 @@ module MetaExtension
43
50
 
44
51
  base.define_method(:extension_attr_hash) do
45
52
  attr_hash = {}
46
- meta.class_variable_get("@@extension_attrs").each do |name|
53
+ @extension_attrs.each do |name|
47
54
  attr_hash[name] = self.instance_variable_get("@#{name}")
48
55
  end
49
56
  attr_hash
50
57
  end
51
58
 
52
59
  base.define_method(:annotate) do |other|
53
- attr_values = meta.class_variable_get("@@extension_attrs").collect do |a|
60
+ attr_values = @extension_attrs.collect do |a|
54
61
  self.instance_variable_get("@#{a}")
55
62
  end
56
63
  base.setup(other, *attr_values)
@@ -40,7 +40,7 @@ module Misc
40
40
  end
41
41
 
42
42
  def self.digest(obj)
43
- str = Misc.digest_str(obj)
43
+ str = String === obj ? obj : Misc.digest_str(obj)
44
44
  Digest::MD5.hexdigest(str)
45
45
  end
46
46
 
@@ -25,6 +25,18 @@ module Misc
25
25
  str
26
26
  end
27
27
 
28
+ CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
29
+ def self.format_seconds_short(time)
30
+ if time < 0.0001
31
+ "%.5g" % time + CHAR_SENCONDS
32
+ elsif time < 60
33
+ "%.2g" % time + CHAR_SENCONDS
34
+ else
35
+ format_seconds(time)
36
+ end
37
+ end
38
+
39
+
28
40
 
29
41
  MAX_WIDTH = 100
30
42
  def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
@@ -32,7 +32,7 @@ module Misc
32
32
  end
33
33
  raise $!
34
34
  rescue Exception
35
- Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
35
+ Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
36
36
  if msg
37
37
  Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
38
38
  elsif FalseClass === msg
@@ -41,4 +41,15 @@ module Misc
41
41
 
42
42
  res
43
43
  end
44
+
45
+ def self.exec_time(&block)
46
+ start = Time.now
47
+ eend = nil
48
+ begin
49
+ yield
50
+ ensure
51
+ eend = Time.now
52
+ end
53
+ eend - start
54
+ end
44
55
  end
@@ -12,4 +12,14 @@ module Misc
12
12
  end
13
13
  end
14
14
  end
15
+
16
+ def self.with_env(var, value, &block)
17
+ old_value = ENV[var]
18
+ begin
19
+ ENV[var] = value
20
+ yield
21
+ ensure
22
+ ENV[var] = old_value
23
+ end
24
+ end
15
25
  end
@@ -1,13 +1,25 @@
1
1
  require_relative 'meta_extension'
2
2
  module NamedArray
3
3
  extend MetaExtension
4
- extension_attr :fields
4
+ extension_attr :fields, :key
5
5
 
6
- def self.identify_name(names, selected)
6
+ def self.field_match(field, name)
7
+ if (String === field) && (String === name)
8
+ field == name ||
9
+ field.start_with?(name) || field.include?("(" + name + ")") ||
10
+ name.start_with?(field) || name.include?("(" + field + ")")
11
+ else
12
+ field == name
13
+ end
14
+ end
15
+
16
+ def self.identify_name(names, selected, strict: false)
7
17
  res = (Array === selected ? selected : [selected]).collect do |field|
8
18
  case field
9
19
  when nil
10
20
  0
21
+ when Range
22
+ field
11
23
  when Integer
12
24
  field
13
25
  when Symbol
@@ -26,7 +38,8 @@ module NamedArray
26
38
  if field =~ /^\d+$/
27
39
  next identify_names(names, field.to_i)
28
40
  end
29
- pos = names.index{|name| name.start_with?(field) }
41
+ next pos if strict
42
+ pos = names.index{|name| field_match(field, name) }
30
43
  next pos if pos
31
44
  nil
32
45
  else
@@ -37,6 +50,10 @@ module NamedArray
37
50
  Array === selected ? res : res.first
38
51
  end
39
52
 
53
+ def identify_name(selected)
54
+ NamedArray.identify_name(fields, selected)
55
+ end
56
+
40
57
  def positions(fields)
41
58
  if Array == fields
42
59
  fields.collect{|field|
@@ -49,6 +66,7 @@ module NamedArray
49
66
 
50
67
  def [](key)
51
68
  pos = NamedArray.identify_name(@fields, key)
69
+ return nil if pos.nil?
52
70
  super(pos)
53
71
  end
54
72
 
@@ -65,4 +83,48 @@ module NamedArray
65
83
  end
66
84
  IndiferentHash.setup hash
67
85
  end
86
+
87
+ def values_at(*positions)
88
+ super(*identify_name(positions))
89
+ end
90
+
91
+ def self._zip_fields(array, max = nil)
92
+ return [] if array.nil? or array.empty? or (first = array.first).nil?
93
+
94
+ max = array.collect{|l| l.length }.max if max.nil?
95
+
96
+ rest = array[1..-1].collect{|v|
97
+ v.length == 1 & max > 1 ? v * max : v
98
+ }
99
+
100
+ first = first * max if first.length == 1 and max > 1
101
+
102
+ first.zip(*rest)
103
+ end
104
+
105
+ def self.zip_fields(array)
106
+ if array.length < 10000
107
+ _zip_fields(array)
108
+ else
109
+ zipped_slices = []
110
+ max = array.collect{|l| l.length}.max
111
+ array.each_slice(10000) do |slice|
112
+ zipped_slices << _zip_fields(slice, max)
113
+ end
114
+ new = zipped_slices.first
115
+ zipped_slices[1..-1].each do |rest|
116
+ rest.each_with_index do |list,i|
117
+ new[i].concat list
118
+ end
119
+ end
120
+ new
121
+ end
122
+ end
123
+
124
+ def self.add_zipped(source, new)
125
+ source.zip(new).each do |s,n|
126
+ s.concat(n)
127
+ end
128
+ source
129
+ end
68
130
  end