scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
data/lib/scout/meta_extension.rb
CHANGED
@@ -2,7 +2,7 @@ module MetaExtension
|
|
2
2
|
def self.extended(base)
|
3
3
|
meta = class << base; self; end
|
4
4
|
|
5
|
-
base.class_variable_set("@@extension_attrs", [])
|
5
|
+
base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
|
6
6
|
|
7
7
|
meta.define_method(:extension_attr) do |*attrs|
|
8
8
|
self.class_variable_get("@@extension_attrs").concat attrs
|
@@ -11,6 +11,14 @@ module MetaExtension
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
meta.define_method(:extended) do |obj|
|
15
|
+
attrs = self.class_variable_get("@@extension_attrs")
|
16
|
+
|
17
|
+
obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
|
18
|
+
extension_attrs = obj.instance_variable_get(:@extension_attrs)
|
19
|
+
extension_attrs.concat attrs
|
20
|
+
end
|
21
|
+
|
14
22
|
meta.define_method(:setup) do |*args,&block|
|
15
23
|
if block_given?
|
16
24
|
obj, rest = block, args
|
@@ -18,7 +26,8 @@ module MetaExtension
|
|
18
26
|
obj, *rest = args
|
19
27
|
end
|
20
28
|
obj = block if obj.nil?
|
21
|
-
obj.extend base
|
29
|
+
obj.extend base unless base === obj
|
30
|
+
|
22
31
|
attrs = self.class_variable_get("@@extension_attrs")
|
23
32
|
|
24
33
|
return if attrs.nil? || attrs.empty?
|
@@ -27,8 +36,6 @@ module MetaExtension
|
|
27
36
|
((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
|
28
37
|
(! attrs.length != 1 ))
|
29
38
|
|
30
|
-
|
31
|
-
|
32
39
|
pairs = rlast
|
33
40
|
else
|
34
41
|
pairs = attrs.zip(rest)
|
@@ -43,14 +50,14 @@ module MetaExtension
|
|
43
50
|
|
44
51
|
base.define_method(:extension_attr_hash) do
|
45
52
|
attr_hash = {}
|
46
|
-
|
53
|
+
@extension_attrs.each do |name|
|
47
54
|
attr_hash[name] = self.instance_variable_get("@#{name}")
|
48
55
|
end
|
49
56
|
attr_hash
|
50
57
|
end
|
51
58
|
|
52
59
|
base.define_method(:annotate) do |other|
|
53
|
-
attr_values =
|
60
|
+
attr_values = @extension_attrs.collect do |a|
|
54
61
|
self.instance_variable_get("@#{a}")
|
55
62
|
end
|
56
63
|
base.setup(other, *attr_values)
|
data/lib/scout/misc/digest.rb
CHANGED
data/lib/scout/misc/format.rb
CHANGED
@@ -25,6 +25,18 @@ module Misc
|
|
25
25
|
str
|
26
26
|
end
|
27
27
|
|
28
|
+
CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
|
29
|
+
def self.format_seconds_short(time)
|
30
|
+
if time < 0.0001
|
31
|
+
"%.5g" % time + CHAR_SENCONDS
|
32
|
+
elsif time < 60
|
33
|
+
"%.2g" % time + CHAR_SENCONDS
|
34
|
+
else
|
35
|
+
format_seconds(time)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
28
40
|
|
29
41
|
MAX_WIDTH = 100
|
30
42
|
def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
|
data/lib/scout/misc/insist.rb
CHANGED
@@ -32,7 +32,7 @@ module Misc
|
|
32
32
|
end
|
33
33
|
raise $!
|
34
34
|
rescue Exception
|
35
|
-
Log.exception $! if ENV["
|
35
|
+
Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
|
36
36
|
if msg
|
37
37
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
|
38
38
|
elsif FalseClass === msg
|
data/lib/scout/misc/monitor.rb
CHANGED
data/lib/scout/misc/system.rb
CHANGED
data/lib/scout/named_array.rb
CHANGED
@@ -1,13 +1,25 @@
|
|
1
1
|
require_relative 'meta_extension'
|
2
2
|
module NamedArray
|
3
3
|
extend MetaExtension
|
4
|
-
extension_attr :fields
|
4
|
+
extension_attr :fields, :key
|
5
5
|
|
6
|
-
def self.
|
6
|
+
def self.field_match(field, name)
|
7
|
+
if (String === field) && (String === name)
|
8
|
+
field == name ||
|
9
|
+
field.start_with?(name) || field.include?("(" + name + ")") ||
|
10
|
+
name.start_with?(field) || name.include?("(" + field + ")")
|
11
|
+
else
|
12
|
+
field == name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.identify_name(names, selected, strict: false)
|
7
17
|
res = (Array === selected ? selected : [selected]).collect do |field|
|
8
18
|
case field
|
9
19
|
when nil
|
10
20
|
0
|
21
|
+
when Range
|
22
|
+
field
|
11
23
|
when Integer
|
12
24
|
field
|
13
25
|
when Symbol
|
@@ -26,7 +38,8 @@ module NamedArray
|
|
26
38
|
if field =~ /^\d+$/
|
27
39
|
next identify_names(names, field.to_i)
|
28
40
|
end
|
29
|
-
pos
|
41
|
+
next pos if strict
|
42
|
+
pos = names.index{|name| field_match(field, name) }
|
30
43
|
next pos if pos
|
31
44
|
nil
|
32
45
|
else
|
@@ -37,6 +50,10 @@ module NamedArray
|
|
37
50
|
Array === selected ? res : res.first
|
38
51
|
end
|
39
52
|
|
53
|
+
def identify_name(selected)
|
54
|
+
NamedArray.identify_name(fields, selected)
|
55
|
+
end
|
56
|
+
|
40
57
|
def positions(fields)
|
41
58
|
if Array == fields
|
42
59
|
fields.collect{|field|
|
@@ -49,6 +66,7 @@ module NamedArray
|
|
49
66
|
|
50
67
|
def [](key)
|
51
68
|
pos = NamedArray.identify_name(@fields, key)
|
69
|
+
return nil if pos.nil?
|
52
70
|
super(pos)
|
53
71
|
end
|
54
72
|
|
@@ -65,4 +83,48 @@ module NamedArray
|
|
65
83
|
end
|
66
84
|
IndiferentHash.setup hash
|
67
85
|
end
|
86
|
+
|
87
|
+
def values_at(*positions)
|
88
|
+
super(*identify_name(positions))
|
89
|
+
end
|
90
|
+
|
91
|
+
def self._zip_fields(array, max = nil)
|
92
|
+
return [] if array.nil? or array.empty? or (first = array.first).nil?
|
93
|
+
|
94
|
+
max = array.collect{|l| l.length }.max if max.nil?
|
95
|
+
|
96
|
+
rest = array[1..-1].collect{|v|
|
97
|
+
v.length == 1 & max > 1 ? v * max : v
|
98
|
+
}
|
99
|
+
|
100
|
+
first = first * max if first.length == 1 and max > 1
|
101
|
+
|
102
|
+
first.zip(*rest)
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.zip_fields(array)
|
106
|
+
if array.length < 10000
|
107
|
+
_zip_fields(array)
|
108
|
+
else
|
109
|
+
zipped_slices = []
|
110
|
+
max = array.collect{|l| l.length}.max
|
111
|
+
array.each_slice(10000) do |slice|
|
112
|
+
zipped_slices << _zip_fields(slice, max)
|
113
|
+
end
|
114
|
+
new = zipped_slices.first
|
115
|
+
zipped_slices[1..-1].each do |rest|
|
116
|
+
rest.each_with_index do |list,i|
|
117
|
+
new[i].concat list
|
118
|
+
end
|
119
|
+
end
|
120
|
+
new
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.add_zipped(source, new)
|
125
|
+
source.zip(new).each do |s,n|
|
126
|
+
s.concat(n)
|
127
|
+
end
|
128
|
+
source
|
129
|
+
end
|
68
130
|
end
|