scout-gear 7.3.0 → 8.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
data/lib/scout/meta_extension.rb
CHANGED
@@ -2,7 +2,7 @@ module MetaExtension
|
|
2
2
|
def self.extended(base)
|
3
3
|
meta = class << base; self; end
|
4
4
|
|
5
|
-
base.class_variable_set("@@extension_attrs", [])
|
5
|
+
base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
|
6
6
|
|
7
7
|
meta.define_method(:extension_attr) do |*attrs|
|
8
8
|
self.class_variable_get("@@extension_attrs").concat attrs
|
@@ -11,6 +11,14 @@ module MetaExtension
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
meta.define_method(:extended) do |obj|
|
15
|
+
attrs = self.class_variable_get("@@extension_attrs")
|
16
|
+
|
17
|
+
obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
|
18
|
+
extension_attrs = obj.instance_variable_get(:@extension_attrs)
|
19
|
+
extension_attrs.concat attrs
|
20
|
+
end
|
21
|
+
|
14
22
|
meta.define_method(:setup) do |*args,&block|
|
15
23
|
if block_given?
|
16
24
|
obj, rest = block, args
|
@@ -18,7 +26,8 @@ module MetaExtension
|
|
18
26
|
obj, *rest = args
|
19
27
|
end
|
20
28
|
obj = block if obj.nil?
|
21
|
-
obj.extend base
|
29
|
+
obj.extend base unless base === obj
|
30
|
+
|
22
31
|
attrs = self.class_variable_get("@@extension_attrs")
|
23
32
|
|
24
33
|
return if attrs.nil? || attrs.empty?
|
@@ -27,8 +36,6 @@ module MetaExtension
|
|
27
36
|
((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
|
28
37
|
(! attrs.length != 1 ))
|
29
38
|
|
30
|
-
|
31
|
-
|
32
39
|
pairs = rlast
|
33
40
|
else
|
34
41
|
pairs = attrs.zip(rest)
|
@@ -43,14 +50,14 @@ module MetaExtension
|
|
43
50
|
|
44
51
|
base.define_method(:extension_attr_hash) do
|
45
52
|
attr_hash = {}
|
46
|
-
|
53
|
+
@extension_attrs.each do |name|
|
47
54
|
attr_hash[name] = self.instance_variable_get("@#{name}")
|
48
55
|
end
|
49
56
|
attr_hash
|
50
57
|
end
|
51
58
|
|
52
59
|
base.define_method(:annotate) do |other|
|
53
|
-
attr_values =
|
60
|
+
attr_values = @extension_attrs.collect do |a|
|
54
61
|
self.instance_variable_get("@#{a}")
|
55
62
|
end
|
56
63
|
base.setup(other, *attr_values)
|
data/lib/scout/misc/digest.rb
CHANGED
data/lib/scout/misc/format.rb
CHANGED
@@ -25,6 +25,18 @@ module Misc
|
|
25
25
|
str
|
26
26
|
end
|
27
27
|
|
28
|
+
CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
|
29
|
+
def self.format_seconds_short(time)
|
30
|
+
if time < 0.0001
|
31
|
+
"%.5g" % time + CHAR_SENCONDS
|
32
|
+
elsif time < 60
|
33
|
+
"%.2g" % time + CHAR_SENCONDS
|
34
|
+
else
|
35
|
+
format_seconds(time)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
28
40
|
|
29
41
|
MAX_WIDTH = 100
|
30
42
|
def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
|
data/lib/scout/misc/insist.rb
CHANGED
@@ -32,7 +32,7 @@ module Misc
|
|
32
32
|
end
|
33
33
|
raise $!
|
34
34
|
rescue Exception
|
35
|
-
Log.exception $! if ENV["
|
35
|
+
Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
|
36
36
|
if msg
|
37
37
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
|
38
38
|
elsif FalseClass === msg
|
data/lib/scout/misc/monitor.rb
CHANGED
data/lib/scout/misc/system.rb
CHANGED
data/lib/scout/named_array.rb
CHANGED
@@ -1,13 +1,25 @@
|
|
1
1
|
require_relative 'meta_extension'
|
2
2
|
module NamedArray
|
3
3
|
extend MetaExtension
|
4
|
-
extension_attr :fields
|
4
|
+
extension_attr :fields, :key
|
5
5
|
|
6
|
-
def self.
|
6
|
+
def self.field_match(field, name)
|
7
|
+
if (String === field) && (String === name)
|
8
|
+
field == name ||
|
9
|
+
field.start_with?(name) || field.include?("(" + name + ")") ||
|
10
|
+
name.start_with?(field) || name.include?("(" + field + ")")
|
11
|
+
else
|
12
|
+
field == name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.identify_name(names, selected, strict: false)
|
7
17
|
res = (Array === selected ? selected : [selected]).collect do |field|
|
8
18
|
case field
|
9
19
|
when nil
|
10
20
|
0
|
21
|
+
when Range
|
22
|
+
field
|
11
23
|
when Integer
|
12
24
|
field
|
13
25
|
when Symbol
|
@@ -26,7 +38,8 @@ module NamedArray
|
|
26
38
|
if field =~ /^\d+$/
|
27
39
|
next identify_names(names, field.to_i)
|
28
40
|
end
|
29
|
-
pos
|
41
|
+
next pos if strict
|
42
|
+
pos = names.index{|name| field_match(field, name) }
|
30
43
|
next pos if pos
|
31
44
|
nil
|
32
45
|
else
|
@@ -37,6 +50,10 @@ module NamedArray
|
|
37
50
|
Array === selected ? res : res.first
|
38
51
|
end
|
39
52
|
|
53
|
+
def identify_name(selected)
|
54
|
+
NamedArray.identify_name(fields, selected)
|
55
|
+
end
|
56
|
+
|
40
57
|
def positions(fields)
|
41
58
|
if Array == fields
|
42
59
|
fields.collect{|field|
|
@@ -49,6 +66,7 @@ module NamedArray
|
|
49
66
|
|
50
67
|
def [](key)
|
51
68
|
pos = NamedArray.identify_name(@fields, key)
|
69
|
+
return nil if pos.nil?
|
52
70
|
super(pos)
|
53
71
|
end
|
54
72
|
|
@@ -65,4 +83,48 @@ module NamedArray
|
|
65
83
|
end
|
66
84
|
IndiferentHash.setup hash
|
67
85
|
end
|
86
|
+
|
87
|
+
def values_at(*positions)
|
88
|
+
super(*identify_name(positions))
|
89
|
+
end
|
90
|
+
|
91
|
+
def self._zip_fields(array, max = nil)
|
92
|
+
return [] if array.nil? or array.empty? or (first = array.first).nil?
|
93
|
+
|
94
|
+
max = array.collect{|l| l.length }.max if max.nil?
|
95
|
+
|
96
|
+
rest = array[1..-1].collect{|v|
|
97
|
+
v.length == 1 & max > 1 ? v * max : v
|
98
|
+
}
|
99
|
+
|
100
|
+
first = first * max if first.length == 1 and max > 1
|
101
|
+
|
102
|
+
first.zip(*rest)
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.zip_fields(array)
|
106
|
+
if array.length < 10000
|
107
|
+
_zip_fields(array)
|
108
|
+
else
|
109
|
+
zipped_slices = []
|
110
|
+
max = array.collect{|l| l.length}.max
|
111
|
+
array.each_slice(10000) do |slice|
|
112
|
+
zipped_slices << _zip_fields(slice, max)
|
113
|
+
end
|
114
|
+
new = zipped_slices.first
|
115
|
+
zipped_slices[1..-1].each do |rest|
|
116
|
+
rest.each_with_index do |list,i|
|
117
|
+
new[i].concat list
|
118
|
+
end
|
119
|
+
end
|
120
|
+
new
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.add_zipped(source, new)
|
125
|
+
source.zip(new).each do |s,n|
|
126
|
+
s.concat(n)
|
127
|
+
end
|
128
|
+
source
|
129
|
+
end
|
68
130
|
end
|