scout-gear 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +51 -6
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +33 -29
- data/lib/scout/config.rb +1 -1
- data/lib/scout/exceptions.rb +1 -0
- data/lib/scout/log/color.rb +4 -2
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +71 -2
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/helper.rb +31 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +12 -1
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +111 -42
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +60 -30
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/semaphore.rb +8 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +85 -54
- data/lib/scout/tsv/index.rb +188 -20
- data/lib/scout/tsv/open.rb +182 -0
- data/lib/scout/tsv/parser.rb +200 -118
- data/lib/scout/tsv/path.rb +5 -6
- data/lib/scout/tsv/persist/adapter.rb +26 -37
- data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
- data/lib/scout/tsv/persist/serialize.rb +117 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
- data/lib/scout/tsv/persist.rb +4 -2
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +136 -37
- data/lib/scout/tsv/util/filter.rb +312 -0
- data/lib/scout/tsv/util/process.rb +73 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +265 -0
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +126 -19
- data/lib/scout/tsv.rb +28 -5
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +29 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +40 -5
- data/lib/scout/workflow/step/progress.rb +14 -0
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +104 -33
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +86 -47
- data/lib/scout/workflow/usage.rb +10 -6
- data/scout-gear.gemspec +30 -3
- data/scout_commands/workflow/task +37 -9
- data/scout_commands/workflow/task_old +2 -2
- data/test/scout/open/test_stream.rb +61 -59
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +67 -0
- data/test/scout/test_tmpfile.rb +1 -1
- data/test/scout/test_tsv.rb +222 -3
- data/test/scout/test_work_queue.rb +21 -18
- data/test/scout/tsv/persist/test_adapter.rb +11 -1
- data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +127 -3
- data/test/scout/tsv/test_open.rb +167 -0
- data/test/scout/tsv/test_parser.rb +45 -3
- data/test/scout/tsv/test_persist.rb +9 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +195 -3
- data/test/scout/tsv/test_util.rb +24 -0
- data/test/scout/tsv/util/test_filter.rb +188 -0
- data/test/scout/tsv/util/test_process.rb +47 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +58 -0
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/work_queue/test_worker.rb +63 -6
- data/test/scout/workflow/step/test_load.rb +3 -3
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- data/test/test_helper.rb +3 -1
- metadata +29 -2
data/lib/scout/log.rb
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
require_relative 'log/color'
|
2
2
|
require_relative 'log/fingerprint'
|
3
3
|
require_relative 'log/progress'
|
4
|
+
require_relative 'log/trap'
|
4
5
|
|
5
6
|
require 'io/console'
|
6
7
|
|
7
8
|
module Log
|
9
|
+
class << self
|
10
|
+
attr_accessor :severity
|
11
|
+
attr_writer :tty_size, :logfile
|
12
|
+
end
|
13
|
+
|
8
14
|
SEVERITY_NAMES ||= begin
|
9
15
|
names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
|
10
16
|
names.each_with_index do |name,i|
|
@@ -12,13 +18,37 @@ module Log
|
|
12
18
|
end
|
13
19
|
names
|
14
20
|
end
|
21
|
+
def self.default_severity
|
22
|
+
@@default_severity ||= begin
|
23
|
+
file = File.join(ENV["HOME"], '.scout/etc/log_severity')
|
24
|
+
if File.exist? file
|
25
|
+
File.open(file) do |f|
|
26
|
+
SEVERITY_NAMES.index f.read.strip
|
27
|
+
end
|
28
|
+
else
|
29
|
+
INFO
|
30
|
+
end
|
31
|
+
end
|
32
|
+
@@default_severity
|
33
|
+
end
|
15
34
|
|
16
|
-
|
17
|
-
|
18
|
-
|
35
|
+
case ENV['SCOUT_LOG']
|
36
|
+
when 'DEBUG'
|
37
|
+
self.severity = DEBUG
|
38
|
+
when 'LOW'
|
39
|
+
self.severity = LOW
|
40
|
+
when 'MEDIUM'
|
41
|
+
self.severity = MEDIUM
|
42
|
+
when 'HIGH'
|
43
|
+
self.severity = HIGH
|
44
|
+
when nil
|
45
|
+
self.severity = default_severity
|
46
|
+
else
|
47
|
+
self.severity = default_severity
|
19
48
|
end
|
20
49
|
|
21
50
|
|
51
|
+
|
22
52
|
def self.tty_size
|
23
53
|
@tty_size = begin
|
24
54
|
IO.console.winsize.last
|
@@ -109,7 +139,11 @@ module Log
|
|
109
139
|
|
110
140
|
sev_str = severity.to_s
|
111
141
|
|
112
|
-
|
142
|
+
if ENV["SCOUT_DEBUG_PID"] == "true"
|
143
|
+
prefix = time << "[" << Process.pid.to_s << "]" << color(severity) << "[" << sev_str << "]" << color(0)
|
144
|
+
else
|
145
|
+
prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
|
146
|
+
end
|
113
147
|
message = "" << highlight << message << color(0) if severity >= INFO
|
114
148
|
str = prefix << " " << message.to_s
|
115
149
|
|
@@ -187,7 +221,7 @@ module Log
|
|
187
221
|
def self.exception(e)
|
188
222
|
stack = caller
|
189
223
|
backtrace = e.backtrace || []
|
190
|
-
if ENV["
|
224
|
+
if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
191
225
|
error([e.class.to_s, e.message].compact * ": " )
|
192
226
|
error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
|
193
227
|
else
|
@@ -243,7 +277,7 @@ module Log
|
|
243
277
|
end
|
244
278
|
|
245
279
|
def self.stack(stack)
|
246
|
-
if ENV["
|
280
|
+
if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
247
281
|
STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
|
248
282
|
color_stack(stack).each do |line|
|
249
283
|
STDERR.puts line
|
@@ -280,21 +314,6 @@ module Log
|
|
280
314
|
$stack_counts = {}
|
281
315
|
res
|
282
316
|
end
|
283
|
-
|
284
|
-
case ENV['RBBT_LOG']
|
285
|
-
when 'DEBUG'
|
286
|
-
self.severity = DEBUG
|
287
|
-
when 'LOW'
|
288
|
-
self.severity = LOW
|
289
|
-
when 'MEDIUM'
|
290
|
-
self.severity = MEDIUM
|
291
|
-
when 'HIGH'
|
292
|
-
self.severity = HIGH
|
293
|
-
when nil
|
294
|
-
self.severity = INFO
|
295
|
-
else
|
296
|
-
self.severity = ENV['RBBT_LOG'].to_i
|
297
|
-
end
|
298
317
|
end
|
299
318
|
|
300
319
|
def ppp(message)
|
@@ -373,3 +392,19 @@ def sss(level, &block)
|
|
373
392
|
end
|
374
393
|
end
|
375
394
|
|
395
|
+
$scout_debug_log = false
|
396
|
+
def ccc(obj=nil, file = $stdout)
|
397
|
+
if block_given?
|
398
|
+
old_scout_debug_log = $scout_debug_log
|
399
|
+
$scout_debug_log = 'true'
|
400
|
+
begin
|
401
|
+
yield
|
402
|
+
ensure
|
403
|
+
$scout_debug_log = old_scout_debug_log
|
404
|
+
end
|
405
|
+
else
|
406
|
+
Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
|
data/lib/scout/meta_extension.rb
CHANGED
@@ -2,7 +2,7 @@ module MetaExtension
|
|
2
2
|
def self.extended(base)
|
3
3
|
meta = class << base; self; end
|
4
4
|
|
5
|
-
base.class_variable_set("@@extension_attrs", [])
|
5
|
+
base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
|
6
6
|
|
7
7
|
meta.define_method(:extension_attr) do |*attrs|
|
8
8
|
self.class_variable_get("@@extension_attrs").concat attrs
|
@@ -11,6 +11,14 @@ module MetaExtension
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
meta.define_method(:extended) do |obj|
|
15
|
+
attrs = self.class_variable_get("@@extension_attrs")
|
16
|
+
|
17
|
+
obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
|
18
|
+
extension_attrs = obj.instance_variable_get(:@extension_attrs)
|
19
|
+
extension_attrs.concat attrs
|
20
|
+
end
|
21
|
+
|
14
22
|
meta.define_method(:setup) do |*args,&block|
|
15
23
|
if block_given?
|
16
24
|
obj, rest = block, args
|
@@ -18,7 +26,8 @@ module MetaExtension
|
|
18
26
|
obj, *rest = args
|
19
27
|
end
|
20
28
|
obj = block if obj.nil?
|
21
|
-
obj.extend base
|
29
|
+
obj.extend base unless base === obj
|
30
|
+
|
22
31
|
attrs = self.class_variable_get("@@extension_attrs")
|
23
32
|
|
24
33
|
return if attrs.nil? || attrs.empty?
|
@@ -27,8 +36,6 @@ module MetaExtension
|
|
27
36
|
((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
|
28
37
|
(! attrs.length != 1 ))
|
29
38
|
|
30
|
-
|
31
|
-
|
32
39
|
pairs = rlast
|
33
40
|
else
|
34
41
|
pairs = attrs.zip(rest)
|
@@ -43,14 +50,14 @@ module MetaExtension
|
|
43
50
|
|
44
51
|
base.define_method(:extension_attr_hash) do
|
45
52
|
attr_hash = {}
|
46
|
-
|
53
|
+
@extension_attrs.each do |name|
|
47
54
|
attr_hash[name] = self.instance_variable_get("@#{name}")
|
48
55
|
end
|
49
56
|
attr_hash
|
50
57
|
end
|
51
58
|
|
52
59
|
base.define_method(:annotate) do |other|
|
53
|
-
attr_values =
|
60
|
+
attr_values = @extension_attrs.collect do |a|
|
54
61
|
self.instance_variable_get("@#{a}")
|
55
62
|
end
|
56
63
|
base.setup(other, *attr_values)
|
data/lib/scout/misc/digest.rb
CHANGED
data/lib/scout/misc/format.rb
CHANGED
@@ -25,6 +25,18 @@ module Misc
|
|
25
25
|
str
|
26
26
|
end
|
27
27
|
|
28
|
+
CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
|
29
|
+
def self.format_seconds_short(time)
|
30
|
+
if time < 0.0001
|
31
|
+
"%.5g" % time + CHAR_SENCONDS
|
32
|
+
elsif time < 60
|
33
|
+
"%.2g" % time + CHAR_SENCONDS
|
34
|
+
else
|
35
|
+
format_seconds(time)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
28
40
|
|
29
41
|
MAX_WIDTH = 100
|
30
42
|
def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Misc
|
2
|
+
def self.intersect_sorted_arrays(a1, a2)
|
3
|
+
e1, e2 = a1.shift, a2.shift
|
4
|
+
intersect = []
|
5
|
+
while true
|
6
|
+
break if e1.nil? or e2.nil?
|
7
|
+
case e1 <=> e2
|
8
|
+
when 0
|
9
|
+
intersect << e1
|
10
|
+
e1, e2 = a1.shift, a2.shift
|
11
|
+
when -1
|
12
|
+
e1 = a1.shift while not e1.nil? and e1 < e2
|
13
|
+
when 1
|
14
|
+
e2 = a2.shift
|
15
|
+
e2 = a2.shift while not e2.nil? and e2 < e1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
intersect
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.counts(array)
|
22
|
+
counts = {}
|
23
|
+
array.each do |e|
|
24
|
+
counts[e] ||= 0
|
25
|
+
counts[e] += 1
|
26
|
+
end
|
27
|
+
|
28
|
+
counts
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
data/lib/scout/misc/insist.rb
CHANGED
@@ -32,7 +32,7 @@ module Misc
|
|
32
32
|
end
|
33
33
|
raise $!
|
34
34
|
rescue Exception
|
35
|
-
Log.exception $! if ENV["
|
35
|
+
Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
|
36
36
|
if msg
|
37
37
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
|
38
38
|
elsif FalseClass === msg
|
data/lib/scout/misc/monitor.rb
CHANGED
@@ -14,7 +14,7 @@ module Misc
|
|
14
14
|
if message
|
15
15
|
puts "#{message }: #{ repeats } repeats"
|
16
16
|
else
|
17
|
-
puts "Benchmark for #{ repeats } repeats"
|
17
|
+
puts "Benchmark for #{ repeats } repeats (#{caller.first})"
|
18
18
|
end
|
19
19
|
puts measure
|
20
20
|
rescue Exception
|
@@ -41,4 +41,15 @@ module Misc
|
|
41
41
|
|
42
42
|
res
|
43
43
|
end
|
44
|
+
|
45
|
+
def self.exec_time(&block)
|
46
|
+
start = Time.now
|
47
|
+
eend = nil
|
48
|
+
begin
|
49
|
+
yield
|
50
|
+
ensure
|
51
|
+
eend = Time.now
|
52
|
+
end
|
53
|
+
eend - start
|
54
|
+
end
|
44
55
|
end
|
data/lib/scout/misc/system.rb
CHANGED
data/lib/scout/misc.rb
CHANGED
data/lib/scout/named_array.rb
CHANGED
@@ -1,13 +1,25 @@
|
|
1
1
|
require_relative 'meta_extension'
|
2
2
|
module NamedArray
|
3
3
|
extend MetaExtension
|
4
|
-
extension_attr :fields
|
4
|
+
extension_attr :fields, :key
|
5
5
|
|
6
|
-
def self.
|
6
|
+
def self.field_match(field, name)
|
7
|
+
if (String === field) && (String === name)
|
8
|
+
field == name ||
|
9
|
+
field.start_with?(name) || field.include?("(" + name + ")") ||
|
10
|
+
name.start_with?(field) || name.include?("(" + field + ")")
|
11
|
+
else
|
12
|
+
field == name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.identify_name(names, selected, strict: false)
|
7
17
|
res = (Array === selected ? selected : [selected]).collect do |field|
|
8
18
|
case field
|
9
19
|
when nil
|
10
20
|
0
|
21
|
+
when Range
|
22
|
+
field
|
11
23
|
when Integer
|
12
24
|
field
|
13
25
|
when Symbol
|
@@ -26,7 +38,8 @@ module NamedArray
|
|
26
38
|
if field =~ /^\d+$/
|
27
39
|
next identify_names(names, field.to_i)
|
28
40
|
end
|
29
|
-
pos
|
41
|
+
next pos if strict
|
42
|
+
pos = names.index{|name| field_match(field, name) }
|
30
43
|
next pos if pos
|
31
44
|
nil
|
32
45
|
else
|
@@ -37,6 +50,10 @@ module NamedArray
|
|
37
50
|
Array === selected ? res : res.first
|
38
51
|
end
|
39
52
|
|
53
|
+
def identify_name(selected)
|
54
|
+
NamedArray.identify_name(fields, selected)
|
55
|
+
end
|
56
|
+
|
40
57
|
def positions(fields)
|
41
58
|
if Array == fields
|
42
59
|
fields.collect{|field|
|
@@ -49,6 +66,7 @@ module NamedArray
|
|
49
66
|
|
50
67
|
def [](key)
|
51
68
|
pos = NamedArray.identify_name(@fields, key)
|
69
|
+
return nil if pos.nil?
|
52
70
|
super(pos)
|
53
71
|
end
|
54
72
|
|
@@ -65,4 +83,48 @@ module NamedArray
|
|
65
83
|
end
|
66
84
|
IndiferentHash.setup hash
|
67
85
|
end
|
86
|
+
|
87
|
+
def values_at(*positions)
|
88
|
+
super(*identify_name(positions))
|
89
|
+
end
|
90
|
+
|
91
|
+
def self._zip_fields(array, max = nil)
|
92
|
+
return [] if array.nil? or array.empty? or (first = array.first).nil?
|
93
|
+
|
94
|
+
max = array.collect{|l| l.length }.max if max.nil?
|
95
|
+
|
96
|
+
rest = array[1..-1].collect{|v|
|
97
|
+
v.length == 1 & max > 1 ? v * max : v
|
98
|
+
}
|
99
|
+
|
100
|
+
first = first * max if first.length == 1 and max > 1
|
101
|
+
|
102
|
+
first.zip(*rest)
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.zip_fields(array)
|
106
|
+
if array.length < 10000
|
107
|
+
_zip_fields(array)
|
108
|
+
else
|
109
|
+
zipped_slices = []
|
110
|
+
max = array.collect{|l| l.length}.max
|
111
|
+
array.each_slice(10000) do |slice|
|
112
|
+
zipped_slices << _zip_fields(slice, max)
|
113
|
+
end
|
114
|
+
new = zipped_slices.first
|
115
|
+
zipped_slices[1..-1].each do |rest|
|
116
|
+
rest.each_with_index do |list,i|
|
117
|
+
new[i].concat list
|
118
|
+
end
|
119
|
+
end
|
120
|
+
new
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.add_zipped(source, new)
|
125
|
+
source.zip(new).each do |s,n|
|
126
|
+
s.concat(n)
|
127
|
+
end
|
128
|
+
source
|
129
|
+
end
|
68
130
|
end
|