scout-gear 7.2.0 → 8.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +51 -6
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +33 -29
- data/lib/scout/config.rb +1 -1
- data/lib/scout/exceptions.rb +1 -0
- data/lib/scout/log/color.rb +4 -2
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +71 -2
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/helper.rb +31 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +12 -1
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +111 -42
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +60 -30
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/semaphore.rb +8 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +85 -54
- data/lib/scout/tsv/index.rb +188 -20
- data/lib/scout/tsv/open.rb +182 -0
- data/lib/scout/tsv/parser.rb +200 -118
- data/lib/scout/tsv/path.rb +5 -6
- data/lib/scout/tsv/persist/adapter.rb +26 -37
- data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
- data/lib/scout/tsv/persist/serialize.rb +117 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
- data/lib/scout/tsv/persist.rb +4 -2
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +136 -37
- data/lib/scout/tsv/util/filter.rb +312 -0
- data/lib/scout/tsv/util/process.rb +73 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +265 -0
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +126 -19
- data/lib/scout/tsv.rb +28 -5
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +29 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +40 -5
- data/lib/scout/workflow/step/progress.rb +14 -0
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +104 -33
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +86 -47
- data/lib/scout/workflow/usage.rb +10 -6
- data/scout-gear.gemspec +30 -3
- data/scout_commands/workflow/task +37 -9
- data/scout_commands/workflow/task_old +2 -2
- data/test/scout/open/test_stream.rb +61 -59
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +67 -0
- data/test/scout/test_tmpfile.rb +1 -1
- data/test/scout/test_tsv.rb +222 -3
- data/test/scout/test_work_queue.rb +21 -18
- data/test/scout/tsv/persist/test_adapter.rb +11 -1
- data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +127 -3
- data/test/scout/tsv/test_open.rb +167 -0
- data/test/scout/tsv/test_parser.rb +45 -3
- data/test/scout/tsv/test_persist.rb +9 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +195 -3
- data/test/scout/tsv/test_util.rb +24 -0
- data/test/scout/tsv/util/test_filter.rb +188 -0
- data/test/scout/tsv/util/test_process.rb +47 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +58 -0
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/work_queue/test_worker.rb +63 -6
- data/test/scout/workflow/step/test_load.rb +3 -3
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- data/test/test_helper.rb +3 -1
- metadata +29 -2
data/lib/scout/log.rb
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
require_relative 'log/color'
|
2
2
|
require_relative 'log/fingerprint'
|
3
3
|
require_relative 'log/progress'
|
4
|
+
require_relative 'log/trap'
|
4
5
|
|
5
6
|
require 'io/console'
|
6
7
|
|
7
8
|
module Log
|
9
|
+
class << self
|
10
|
+
attr_accessor :severity
|
11
|
+
attr_writer :tty_size, :logfile
|
12
|
+
end
|
13
|
+
|
8
14
|
SEVERITY_NAMES ||= begin
|
9
15
|
names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE )
|
10
16
|
names.each_with_index do |name,i|
|
@@ -12,13 +18,37 @@ module Log
|
|
12
18
|
end
|
13
19
|
names
|
14
20
|
end
|
21
|
+
def self.default_severity
|
22
|
+
@@default_severity ||= begin
|
23
|
+
file = File.join(ENV["HOME"], '.scout/etc/log_severity')
|
24
|
+
if File.exist? file
|
25
|
+
File.open(file) do |f|
|
26
|
+
SEVERITY_NAMES.index f.read.strip
|
27
|
+
end
|
28
|
+
else
|
29
|
+
INFO
|
30
|
+
end
|
31
|
+
end
|
32
|
+
@@default_severity
|
33
|
+
end
|
15
34
|
|
16
|
-
|
17
|
-
|
18
|
-
|
35
|
+
case ENV['SCOUT_LOG']
|
36
|
+
when 'DEBUG'
|
37
|
+
self.severity = DEBUG
|
38
|
+
when 'LOW'
|
39
|
+
self.severity = LOW
|
40
|
+
when 'MEDIUM'
|
41
|
+
self.severity = MEDIUM
|
42
|
+
when 'HIGH'
|
43
|
+
self.severity = HIGH
|
44
|
+
when nil
|
45
|
+
self.severity = default_severity
|
46
|
+
else
|
47
|
+
self.severity = default_severity
|
19
48
|
end
|
20
49
|
|
21
50
|
|
51
|
+
|
22
52
|
def self.tty_size
|
23
53
|
@tty_size = begin
|
24
54
|
IO.console.winsize.last
|
@@ -109,7 +139,11 @@ module Log
|
|
109
139
|
|
110
140
|
sev_str = severity.to_s
|
111
141
|
|
112
|
-
|
142
|
+
if ENV["SCOUT_DEBUG_PID"] == "true"
|
143
|
+
prefix = time << "[" << Process.pid.to_s << "]" << color(severity) << "[" << sev_str << "]" << color(0)
|
144
|
+
else
|
145
|
+
prefix = time << color(severity) << "[" << sev_str << "]" << color(0)
|
146
|
+
end
|
113
147
|
message = "" << highlight << message << color(0) if severity >= INFO
|
114
148
|
str = prefix << " " << message.to_s
|
115
149
|
|
@@ -187,7 +221,7 @@ module Log
|
|
187
221
|
def self.exception(e)
|
188
222
|
stack = caller
|
189
223
|
backtrace = e.backtrace || []
|
190
|
-
if ENV["
|
224
|
+
if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
191
225
|
error([e.class.to_s, e.message].compact * ": " )
|
192
226
|
error("BACKTRACE [#{Process.pid}]: " << Log.last_caller(stack) << "\n" + color_stack(backtrace)*"\n")
|
193
227
|
else
|
@@ -243,7 +277,7 @@ module Log
|
|
243
277
|
end
|
244
278
|
|
245
279
|
def self.stack(stack)
|
246
|
-
if ENV["
|
280
|
+
if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
247
281
|
STDERR.puts Log.color :magenta, "Stack trace [#{Process.pid}]: " << Log.last_caller(caller)
|
248
282
|
color_stack(stack).each do |line|
|
249
283
|
STDERR.puts line
|
@@ -280,21 +314,6 @@ module Log
|
|
280
314
|
$stack_counts = {}
|
281
315
|
res
|
282
316
|
end
|
283
|
-
|
284
|
-
case ENV['RBBT_LOG']
|
285
|
-
when 'DEBUG'
|
286
|
-
self.severity = DEBUG
|
287
|
-
when 'LOW'
|
288
|
-
self.severity = LOW
|
289
|
-
when 'MEDIUM'
|
290
|
-
self.severity = MEDIUM
|
291
|
-
when 'HIGH'
|
292
|
-
self.severity = HIGH
|
293
|
-
when nil
|
294
|
-
self.severity = INFO
|
295
|
-
else
|
296
|
-
self.severity = ENV['RBBT_LOG'].to_i
|
297
|
-
end
|
298
317
|
end
|
299
318
|
|
300
319
|
def ppp(message)
|
@@ -373,3 +392,19 @@ def sss(level, &block)
|
|
373
392
|
end
|
374
393
|
end
|
375
394
|
|
395
|
+
$scout_debug_log = false
|
396
|
+
def ccc(obj=nil, file = $stdout)
|
397
|
+
if block_given?
|
398
|
+
old_scout_debug_log = $scout_debug_log
|
399
|
+
$scout_debug_log = 'true'
|
400
|
+
begin
|
401
|
+
yield
|
402
|
+
ensure
|
403
|
+
$scout_debug_log = old_scout_debug_log
|
404
|
+
end
|
405
|
+
else
|
406
|
+
Log.log_obj_inspect(obj, :info, file) if $scout_debug_log
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
|
data/lib/scout/meta_extension.rb
CHANGED
@@ -2,7 +2,7 @@ module MetaExtension
|
|
2
2
|
def self.extended(base)
|
3
3
|
meta = class << base; self; end
|
4
4
|
|
5
|
-
base.class_variable_set("@@extension_attrs", [])
|
5
|
+
base.class_variable_set("@@extension_attrs", []) unless base.class_variables.include?("@@extension_attrs")
|
6
6
|
|
7
7
|
meta.define_method(:extension_attr) do |*attrs|
|
8
8
|
self.class_variable_get("@@extension_attrs").concat attrs
|
@@ -11,6 +11,14 @@ module MetaExtension
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
meta.define_method(:extended) do |obj|
|
15
|
+
attrs = self.class_variable_get("@@extension_attrs")
|
16
|
+
|
17
|
+
obj.instance_variable_set(:@extension_attrs, []) unless obj.instance_variables.include?(:@extension_attrs)
|
18
|
+
extension_attrs = obj.instance_variable_get(:@extension_attrs)
|
19
|
+
extension_attrs.concat attrs
|
20
|
+
end
|
21
|
+
|
14
22
|
meta.define_method(:setup) do |*args,&block|
|
15
23
|
if block_given?
|
16
24
|
obj, rest = block, args
|
@@ -18,7 +26,8 @@ module MetaExtension
|
|
18
26
|
obj, *rest = args
|
19
27
|
end
|
20
28
|
obj = block if obj.nil?
|
21
|
-
obj.extend base
|
29
|
+
obj.extend base unless base === obj
|
30
|
+
|
22
31
|
attrs = self.class_variable_get("@@extension_attrs")
|
23
32
|
|
24
33
|
return if attrs.nil? || attrs.empty?
|
@@ -27,8 +36,6 @@ module MetaExtension
|
|
27
36
|
((! (rlkey = rlast.keys.first).nil? && attrs.include?(rlkey.to_sym)) ||
|
28
37
|
(! attrs.length != 1 ))
|
29
38
|
|
30
|
-
|
31
|
-
|
32
39
|
pairs = rlast
|
33
40
|
else
|
34
41
|
pairs = attrs.zip(rest)
|
@@ -43,14 +50,14 @@ module MetaExtension
|
|
43
50
|
|
44
51
|
base.define_method(:extension_attr_hash) do
|
45
52
|
attr_hash = {}
|
46
|
-
|
53
|
+
@extension_attrs.each do |name|
|
47
54
|
attr_hash[name] = self.instance_variable_get("@#{name}")
|
48
55
|
end
|
49
56
|
attr_hash
|
50
57
|
end
|
51
58
|
|
52
59
|
base.define_method(:annotate) do |other|
|
53
|
-
attr_values =
|
60
|
+
attr_values = @extension_attrs.collect do |a|
|
54
61
|
self.instance_variable_get("@#{a}")
|
55
62
|
end
|
56
63
|
base.setup(other, *attr_values)
|
data/lib/scout/misc/digest.rb
CHANGED
data/lib/scout/misc/format.rb
CHANGED
@@ -25,6 +25,18 @@ module Misc
|
|
25
25
|
str
|
26
26
|
end
|
27
27
|
|
28
|
+
CHAR_SENCONDS = ENV["SCOUT_NOCOLOR"] == "true" ? "sec" : "″"
|
29
|
+
def self.format_seconds_short(time)
|
30
|
+
if time < 0.0001
|
31
|
+
"%.5g" % time + CHAR_SENCONDS
|
32
|
+
elsif time < 60
|
33
|
+
"%.2g" % time + CHAR_SENCONDS
|
34
|
+
else
|
35
|
+
format_seconds(time)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
28
40
|
|
29
41
|
MAX_WIDTH = 100
|
30
42
|
def self.format_paragraph(text, size = nil, indent = nil, offset = nil)
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Misc
|
2
|
+
def self.intersect_sorted_arrays(a1, a2)
|
3
|
+
e1, e2 = a1.shift, a2.shift
|
4
|
+
intersect = []
|
5
|
+
while true
|
6
|
+
break if e1.nil? or e2.nil?
|
7
|
+
case e1 <=> e2
|
8
|
+
when 0
|
9
|
+
intersect << e1
|
10
|
+
e1, e2 = a1.shift, a2.shift
|
11
|
+
when -1
|
12
|
+
e1 = a1.shift while not e1.nil? and e1 < e2
|
13
|
+
when 1
|
14
|
+
e2 = a2.shift
|
15
|
+
e2 = a2.shift while not e2.nil? and e2 < e1
|
16
|
+
end
|
17
|
+
end
|
18
|
+
intersect
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.counts(array)
|
22
|
+
counts = {}
|
23
|
+
array.each do |e|
|
24
|
+
counts[e] ||= 0
|
25
|
+
counts[e] += 1
|
26
|
+
end
|
27
|
+
|
28
|
+
counts
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
data/lib/scout/misc/insist.rb
CHANGED
@@ -32,7 +32,7 @@ module Misc
|
|
32
32
|
end
|
33
33
|
raise $!
|
34
34
|
rescue Exception
|
35
|
-
Log.exception $! if ENV["
|
35
|
+
Log.exception $! if ENV["SCOUT_LOG_INSIST"] == 'true'
|
36
36
|
if msg
|
37
37
|
Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
|
38
38
|
elsif FalseClass === msg
|
data/lib/scout/misc/monitor.rb
CHANGED
@@ -14,7 +14,7 @@ module Misc
|
|
14
14
|
if message
|
15
15
|
puts "#{message }: #{ repeats } repeats"
|
16
16
|
else
|
17
|
-
puts "Benchmark for #{ repeats } repeats"
|
17
|
+
puts "Benchmark for #{ repeats } repeats (#{caller.first})"
|
18
18
|
end
|
19
19
|
puts measure
|
20
20
|
rescue Exception
|
@@ -41,4 +41,15 @@ module Misc
|
|
41
41
|
|
42
42
|
res
|
43
43
|
end
|
44
|
+
|
45
|
+
def self.exec_time(&block)
|
46
|
+
start = Time.now
|
47
|
+
eend = nil
|
48
|
+
begin
|
49
|
+
yield
|
50
|
+
ensure
|
51
|
+
eend = Time.now
|
52
|
+
end
|
53
|
+
eend - start
|
54
|
+
end
|
44
55
|
end
|
data/lib/scout/misc/system.rb
CHANGED
data/lib/scout/misc.rb
CHANGED
data/lib/scout/named_array.rb
CHANGED
@@ -1,13 +1,25 @@
|
|
1
1
|
require_relative 'meta_extension'
|
2
2
|
module NamedArray
|
3
3
|
extend MetaExtension
|
4
|
-
extension_attr :fields
|
4
|
+
extension_attr :fields, :key
|
5
5
|
|
6
|
-
def self.
|
6
|
+
def self.field_match(field, name)
|
7
|
+
if (String === field) && (String === name)
|
8
|
+
field == name ||
|
9
|
+
field.start_with?(name) || field.include?("(" + name + ")") ||
|
10
|
+
name.start_with?(field) || name.include?("(" + field + ")")
|
11
|
+
else
|
12
|
+
field == name
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.identify_name(names, selected, strict: false)
|
7
17
|
res = (Array === selected ? selected : [selected]).collect do |field|
|
8
18
|
case field
|
9
19
|
when nil
|
10
20
|
0
|
21
|
+
when Range
|
22
|
+
field
|
11
23
|
when Integer
|
12
24
|
field
|
13
25
|
when Symbol
|
@@ -26,7 +38,8 @@ module NamedArray
|
|
26
38
|
if field =~ /^\d+$/
|
27
39
|
next identify_names(names, field.to_i)
|
28
40
|
end
|
29
|
-
pos
|
41
|
+
next pos if strict
|
42
|
+
pos = names.index{|name| field_match(field, name) }
|
30
43
|
next pos if pos
|
31
44
|
nil
|
32
45
|
else
|
@@ -37,6 +50,10 @@ module NamedArray
|
|
37
50
|
Array === selected ? res : res.first
|
38
51
|
end
|
39
52
|
|
53
|
+
def identify_name(selected)
|
54
|
+
NamedArray.identify_name(fields, selected)
|
55
|
+
end
|
56
|
+
|
40
57
|
def positions(fields)
|
41
58
|
if Array == fields
|
42
59
|
fields.collect{|field|
|
@@ -49,6 +66,7 @@ module NamedArray
|
|
49
66
|
|
50
67
|
def [](key)
|
51
68
|
pos = NamedArray.identify_name(@fields, key)
|
69
|
+
return nil if pos.nil?
|
52
70
|
super(pos)
|
53
71
|
end
|
54
72
|
|
@@ -65,4 +83,48 @@ module NamedArray
|
|
65
83
|
end
|
66
84
|
IndiferentHash.setup hash
|
67
85
|
end
|
86
|
+
|
87
|
+
def values_at(*positions)
|
88
|
+
super(*identify_name(positions))
|
89
|
+
end
|
90
|
+
|
91
|
+
def self._zip_fields(array, max = nil)
|
92
|
+
return [] if array.nil? or array.empty? or (first = array.first).nil?
|
93
|
+
|
94
|
+
max = array.collect{|l| l.length }.max if max.nil?
|
95
|
+
|
96
|
+
rest = array[1..-1].collect{|v|
|
97
|
+
v.length == 1 & max > 1 ? v * max : v
|
98
|
+
}
|
99
|
+
|
100
|
+
first = first * max if first.length == 1 and max > 1
|
101
|
+
|
102
|
+
first.zip(*rest)
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.zip_fields(array)
|
106
|
+
if array.length < 10000
|
107
|
+
_zip_fields(array)
|
108
|
+
else
|
109
|
+
zipped_slices = []
|
110
|
+
max = array.collect{|l| l.length}.max
|
111
|
+
array.each_slice(10000) do |slice|
|
112
|
+
zipped_slices << _zip_fields(slice, max)
|
113
|
+
end
|
114
|
+
new = zipped_slices.first
|
115
|
+
zipped_slices[1..-1].each do |rest|
|
116
|
+
rest.each_with_index do |list,i|
|
117
|
+
new[i].concat list
|
118
|
+
end
|
119
|
+
end
|
120
|
+
new
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.add_zipped(source, new)
|
125
|
+
source.zip(new).each do |s,n|
|
126
|
+
s.concat(n)
|
127
|
+
end
|
128
|
+
source
|
129
|
+
end
|
68
130
|
end
|