scout-gear 8.1.0 → 9.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vimproject +22 -0
- data/VERSION +1 -1
- data/bin/scout +1 -1
- data/lib/rbbt-scout.rb +4 -2
- data/lib/scout/cmd.rb +9 -5
- data/lib/scout/config.rb +1 -1
- data/lib/scout/indiferent_hash/options.rb +1 -1
- data/lib/scout/indiferent_hash.rb +17 -0
- data/lib/scout/log/progress/report.rb +1 -0
- data/lib/scout/log.rb +14 -11
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/filesystem.rb +2 -3
- data/lib/scout/offsite/ssh.rb +3 -0
- data/lib/scout/offsite/step.rb +20 -3
- data/lib/scout/open/stream.rb +0 -1
- data/lib/scout/open.rb +8 -8
- data/lib/scout/path/util.rb +20 -0
- data/lib/scout/path.rb +1 -1
- data/lib/scout/persist/serialize.rb +1 -1
- data/lib/scout/resource/open.rb +8 -0
- data/lib/scout/resource/path.rb +12 -9
- data/lib/scout/resource/software.rb +4 -2
- data/lib/scout/resource/util.rb +5 -1
- data/lib/scout/resource.rb +2 -0
- data/lib/scout/tsv/dumper.rb +4 -1
- data/lib/scout/tsv/open.rb +7 -1
- data/lib/scout/tsv/parser.rb +13 -3
- data/lib/scout/tsv/util.rb +1 -1
- data/lib/scout/work_queue.rb +1 -0
- data/lib/scout/workflow/deployment/orchestrator.rb +17 -8
- data/lib/scout/workflow/step/dependencies.rb +11 -4
- data/lib/scout/workflow/step/file.rb +5 -1
- data/lib/scout/workflow/step/info.rb +16 -2
- data/lib/scout/workflow/step/inputs.rb +5 -0
- data/lib/scout/workflow/step/provenance.rb +28 -10
- data/lib/scout/workflow/step/status.rb +26 -2
- data/lib/scout/workflow/step.rb +26 -7
- data/lib/scout/workflow/task/dependencies.rb +3 -0
- data/lib/scout/workflow/task/inputs.rb +19 -9
- data/lib/scout/workflow/task.rb +24 -3
- data/lib/scout/workflow/usage.rb +22 -33
- data/lib/scout/workflow.rb +25 -12
- data/scout-gear.gemspec +9 -3
- data/scout_commands/resource/produce +66 -0
- data/scout_commands/template +52 -0
- data/scout_commands/workflow/install +3 -0
- data/scout_commands/workflow/task +25 -7
- data/share/software/install_helpers +2 -2
- data/share/templates/command +25 -0
- data/share/templates/workflow.rb +14 -0
- data/test/scout/offsite/test_step.rb +0 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/resource/test_util.rb +9 -0
- data/test/scout/test_persist.rb +2 -2
- data/test/scout/test_work_queue.rb +1 -1
- data/test/scout/tsv/test_parser.rb +21 -0
- data/test/scout/workflow/step/test_info.rb +0 -1
- data/test/scout/workflow/step/test_provenance.rb +24 -1
- data/test/scout/workflow/task/test_dependencies.rb +2 -0
- data/test/scout/workflow/task/test_inputs.rb +0 -1
- data/test/scout/workflow/test_definition.rb +1 -1
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bbe916f2d82030e26b1049319216f45341b7e7f7410df65e3953c0a733af2c21
|
4
|
+
data.tar.gz: 152b31b552f90b2bb06aed9bc3cdd12a3bff48746468e52b39e5754977a40f90
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51f41961b2a95d847e4a60a70a2774c34098c4fa502818be8b9d5ec30ced53fce81d02862a1104498060c04a88138ca329ae82ba09fa264b6d9ec0daca80a3a4
|
7
|
+
data.tar.gz: 113d51ff9798416073a86debb07ce47708254a4594e5785cfc4543794c3ff4e09da8809c29d67f64d57aeacd30c1c2ad49998d5eca4f544f3381991c296db1c5
|
data/.vimproject
CHANGED
@@ -10,6 +10,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
10
10
|
glob
|
11
11
|
doc
|
12
12
|
update
|
13
|
+
template
|
13
14
|
offsite
|
14
15
|
workflow=workflow{
|
15
16
|
task
|
@@ -17,6 +18,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
17
18
|
info
|
18
19
|
install
|
19
20
|
}
|
21
|
+
resource=resource{
|
22
|
+
produce
|
23
|
+
}
|
20
24
|
}
|
21
25
|
lib=lib {
|
22
26
|
scout-gear.rb
|
@@ -115,6 +119,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
115
119
|
progress.rb
|
116
120
|
provenance.rb
|
117
121
|
status.rb
|
122
|
+
inputs.rb
|
118
123
|
}
|
119
124
|
task.rb
|
120
125
|
task=task{
|
@@ -168,6 +173,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
168
173
|
sync.rb
|
169
174
|
step.rb
|
170
175
|
}
|
176
|
+
hpc=hpc{
|
177
|
+
slurm.rb
|
178
|
+
}
|
171
179
|
}
|
172
180
|
}
|
173
181
|
test=test {
|
@@ -198,7 +206,18 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
198
206
|
test_task.rb
|
199
207
|
test_usage.rb
|
200
208
|
test_util.rb
|
209
|
+
step=step{
|
210
|
+
test_dependencies.rb
|
211
|
+
test_info.rb
|
212
|
+
test_load.rb
|
213
|
+
test_provenance.rb
|
214
|
+
test_status.rb
|
201
215
|
}
|
216
|
+
task=task{
|
217
|
+
test_dependencies.rb
|
218
|
+
test_inputs.rb
|
219
|
+
}
|
220
|
+
}
|
202
221
|
indiferent_hash=indiferent_hash{
|
203
222
|
test_case_insensitive.rb
|
204
223
|
test_options.rb
|
@@ -209,6 +228,9 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
|
|
209
228
|
software=software{
|
210
229
|
install_helpers
|
211
230
|
}
|
231
|
+
templates=templates{
|
232
|
+
workflow.rb
|
233
|
+
}
|
212
234
|
}
|
213
235
|
|
214
236
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
9.1.0
|
data/bin/scout
CHANGED
@@ -54,6 +54,7 @@ if dev_dir
|
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
57
|
+
require 'scout'
|
57
58
|
|
58
59
|
require 'scout/simple_opt'
|
59
60
|
|
@@ -98,7 +99,6 @@ if config_keys = options.delete(:config_keys)
|
|
98
99
|
end
|
99
100
|
end
|
100
101
|
|
101
|
-
|
102
102
|
$scout_command_dir = Scout.bin.scout
|
103
103
|
$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
|
104
104
|
|
data/lib/rbbt-scout.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
$LOAD_PATH.unshift File.join(__dir__, '../modules/rbbt-util/lib')
|
2
|
-
module
|
2
|
+
module Scout
|
3
3
|
extend Resource
|
4
|
-
self.pkgdir = 'rbbt'
|
5
4
|
self.path_maps = Path.path_maps.merge(:rbbt_lib => File.expand_path(File.join(__dir__, '../modules/rbbt-util/', '{TOPLEVEL}','{SUBPATH}')))
|
6
5
|
end
|
6
|
+
Rbbt = Scout
|
7
|
+
|
8
|
+
Resource.set_software_env Rbbt.software
|
data/lib/scout/cmd.rb
CHANGED
@@ -20,17 +20,22 @@ module CMD
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
+
|
23
24
|
def self.get_tool(tool)
|
24
25
|
return tool.to_s unless TOOLS[tool]
|
25
26
|
|
26
27
|
@@init_cmd_tool ||= IndiferentHash.setup({})
|
28
|
+
|
29
|
+
claim, test, block, cmd = TOOLS[tool]
|
30
|
+
cmd = tool.to_s if cmd.nil?
|
31
|
+
|
27
32
|
if !@@init_cmd_tool[tool]
|
28
|
-
|
33
|
+
|
29
34
|
begin
|
30
35
|
if test
|
31
36
|
CMD.cmd(test + " ")
|
32
37
|
else
|
33
|
-
CMD.cmd("#{
|
38
|
+
CMD.cmd("#{cmd} --help")
|
34
39
|
end
|
35
40
|
rescue
|
36
41
|
if claim
|
@@ -43,7 +48,7 @@ module CMD
|
|
43
48
|
version = nil
|
44
49
|
["--version", "-version", "--help", ""].each do |f|
|
45
50
|
begin
|
46
|
-
version_txt += CMD.cmd("#{
|
51
|
+
version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
|
47
52
|
version = CMD.scan_version_text(version_txt, tool)
|
48
53
|
break if version
|
49
54
|
rescue
|
@@ -56,9 +61,8 @@ module CMD
|
|
56
61
|
return cmd if cmd
|
57
62
|
end
|
58
63
|
|
59
|
-
|
64
|
+
cmd
|
60
65
|
end
|
61
|
-
|
62
66
|
def self.scan_version_text(text, cmd = nil)
|
63
67
|
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
64
68
|
text = Misc.fixutf8 text
|
data/lib/scout/config.rb
CHANGED
@@ -75,5 +75,22 @@ module IndiferentHash
|
|
75
75
|
end
|
76
76
|
clean
|
77
77
|
end
|
78
|
+
|
79
|
+
def slice(*list)
|
80
|
+
ext_list = []
|
81
|
+
list.each do |e|
|
82
|
+
case e
|
83
|
+
when Symbol
|
84
|
+
ext_list << e
|
85
|
+
ext_list << e.to_s
|
86
|
+
when String
|
87
|
+
ext_list << e
|
88
|
+
ext_list << e.to_sym
|
89
|
+
else
|
90
|
+
ext_list << e
|
91
|
+
end
|
92
|
+
end
|
93
|
+
IndiferentHash.setup(super(*ext_list))
|
94
|
+
end
|
78
95
|
end
|
79
96
|
|
@@ -161,6 +161,7 @@ module Log
|
|
161
161
|
def save
|
162
162
|
info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
|
163
163
|
info.delete_if{|k,v| v.nil?}
|
164
|
+
FileUtils.mkdir_p File.dirname(file) unless File.exist?(File.dirname(file))
|
164
165
|
File.write(file, info.to_yaml)
|
165
166
|
end
|
166
167
|
|
data/lib/scout/log.rb
CHANGED
@@ -51,17 +51,19 @@ module Log
|
|
51
51
|
|
52
52
|
def self.tty_size
|
53
53
|
@@tty_size ||= Log.ignore_stderr do
|
54
|
-
begin
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
54
|
+
size = begin
|
55
|
+
IO.console.winsize.last
|
56
|
+
rescue Exception
|
57
|
+
begin
|
58
|
+
res = `tput li`
|
59
|
+
res = nil if res == ""
|
60
|
+
res || ENV["TTY_SIZE"] || 80
|
61
|
+
rescue Exception
|
62
|
+
ENV["TTY_SIZE"] || 80
|
63
|
+
end
|
64
|
+
end
|
65
|
+
size = size.to_i if String === size
|
66
|
+
size
|
65
67
|
end
|
66
68
|
end
|
67
69
|
|
@@ -269,6 +271,7 @@ module Log
|
|
269
271
|
line = line.sub('`',"'")
|
270
272
|
color = :green if line =~ /workflow/
|
271
273
|
color = :blue if line =~ /scout-/
|
274
|
+
color = :cyan if line =~ /rbbt-/
|
272
275
|
if color
|
273
276
|
Log.color color, line
|
274
277
|
else
|
data/lib/scout/misc/digest.rb
CHANGED
@@ -16,11 +16,10 @@ module Misc
|
|
16
16
|
|
17
17
|
basedir += "/" unless basedir.slice(-2,-1) == "/"
|
18
18
|
|
19
|
-
if path.
|
20
|
-
return path
|
19
|
+
if path.start_with?(basedir)
|
20
|
+
return path.slice(basedir.length, basedir.length)
|
21
21
|
else
|
22
22
|
return nil
|
23
23
|
end
|
24
24
|
end
|
25
|
-
|
26
25
|
end
|
data/lib/scout/offsite/ssh.rb
CHANGED
@@ -21,6 +21,9 @@ class SSHLine
|
|
21
21
|
ch.exec 'bash -l'
|
22
22
|
end
|
23
23
|
|
24
|
+
@ch.send_data("[[ -f ~/.scout/environment ]] && source ~/.scout/environment\n")
|
25
|
+
@ch.send_data("[[ -f ~/.rbbt/environment ]] && source ~/.rbbt/environment\n")
|
26
|
+
|
24
27
|
@ch.on_data do |_,data|
|
25
28
|
if m = data.match(/DONECMD: (\d+)\n/)
|
26
29
|
@exit_status = m[1].to_i
|
data/lib/scout/offsite/step.rb
CHANGED
@@ -5,14 +5,14 @@ require_relative 'sync'
|
|
5
5
|
module OffsiteStep
|
6
6
|
|
7
7
|
extend MetaExtension
|
8
|
-
extension_attr :server, :workflow_name, :clean_id, :
|
8
|
+
extension_attr :server, :workflow_name, :clean_id, :slurm
|
9
9
|
|
10
10
|
def inputs_directory
|
11
11
|
@inputs_directory ||= begin
|
12
12
|
if provided_inputs && provided_inputs.any?
|
13
13
|
file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
|
14
14
|
TmpFile.with_path do |inputs_dir|
|
15
|
-
|
15
|
+
save_inputs(inputs_dir)
|
16
16
|
SSHLine.rsync(inputs_dir, file, target: server, directory: true)
|
17
17
|
end
|
18
18
|
file
|
@@ -68,6 +68,19 @@ job = wf.job(:#{task_name}, "#{clean_name}");
|
|
68
68
|
status == :done
|
69
69
|
end
|
70
70
|
|
71
|
+
def orchestrate_slurm
|
72
|
+
bundle_files = offsite_job_ssh <<~EOF
|
73
|
+
require 'rbbt/hpc'
|
74
|
+
HPC::BATCH_MODULE = HPC.batch_system "SLURM"
|
75
|
+
HPC::BATCH_MODULE.orchestrate_job(job, {})
|
76
|
+
job.join
|
77
|
+
job.bundle_files
|
78
|
+
EOF
|
79
|
+
SSHLine.sync(bundle_files, source: server)
|
80
|
+
self.load
|
81
|
+
end
|
82
|
+
|
83
|
+
|
71
84
|
def exec
|
72
85
|
bundle_files = offsite_job_ssh <<~EOF
|
73
86
|
job.run
|
@@ -78,6 +91,10 @@ job = wf.job(:#{task_name}, "#{clean_name}");
|
|
78
91
|
end
|
79
92
|
|
80
93
|
def run
|
81
|
-
|
94
|
+
if slurm
|
95
|
+
orchestrate_slurm
|
96
|
+
else
|
97
|
+
exec
|
98
|
+
end
|
82
99
|
end
|
83
100
|
end
|
data/lib/scout/open/stream.rb
CHANGED
@@ -66,7 +66,6 @@ module Open
|
|
66
66
|
into.close if into and into_close and not into.closed?
|
67
67
|
block.call if block_given?
|
68
68
|
|
69
|
-
Log.debug "Consume stream done #{Log.fingerprint io} -> #{Log.fingerprint(into_path ||into)}"
|
70
69
|
last_c
|
71
70
|
rescue Aborted
|
72
71
|
Thread.current["exception"] = true
|
data/lib/scout/open.rb
CHANGED
@@ -20,21 +20,21 @@ module Open
|
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
23
|
-
def self.get_stream(file, mode = 'r')
|
23
|
+
def self.get_stream(file, mode = 'r', options = {})
|
24
24
|
return file if Open.is_stream?(file)
|
25
25
|
return file.stream if Open.has_stream?(file)
|
26
26
|
file = file.find if Path === file
|
27
27
|
|
28
|
-
return Open.ssh(file) if Open.ssh?(file)
|
29
|
-
return Open.wget(file) if Open.remote?(file)
|
28
|
+
return Open.ssh(file, options) if Open.ssh?(file)
|
29
|
+
return Open.wget(file, options) if Open.remote?(file)
|
30
30
|
|
31
31
|
File.open(file, mode)
|
32
32
|
end
|
33
33
|
|
34
|
-
def self.file_open(file, grep = false, mode = 'r', invert_grep = false)
|
34
|
+
def self.file_open(file, grep = false, mode = 'r', invert_grep = false, options = {})
|
35
35
|
Open.mkdir File.dirname(file) if mode.include? 'w'
|
36
36
|
|
37
|
-
stream = get_stream(file, mode)
|
37
|
+
stream = get_stream(file, mode, options)
|
38
38
|
|
39
39
|
if grep
|
40
40
|
grep(stream, grep, invert_grep)
|
@@ -58,7 +58,7 @@ module Open
|
|
58
58
|
def self.open(file, options = {})
|
59
59
|
if IO === file || StringIO === file
|
60
60
|
if block_given?
|
61
|
-
res = yield file
|
61
|
+
res = yield file, options
|
62
62
|
file.close
|
63
63
|
return res
|
64
64
|
else
|
@@ -72,7 +72,7 @@ module Open
|
|
72
72
|
|
73
73
|
options[:noz] = true if mode.include? "w"
|
74
74
|
|
75
|
-
io = file_open(file, options[:grep], mode, options[:invert_grep])
|
75
|
+
io = file_open(file, options[:grep], mode, options[:invert_grep], options)
|
76
76
|
|
77
77
|
io = unzip(io) if ((String === file and zip?(file)) and not options[:noz]) or options[:zip]
|
78
78
|
io = gunzip(io) if ((String === file and gzip?(file)) and not options[:noz]) or options[:gzip]
|
@@ -88,7 +88,7 @@ module Open
|
|
88
88
|
rescue DontClose
|
89
89
|
res = $!.payload
|
90
90
|
rescue Exception
|
91
|
-
io.abort if io.respond_to? :abort
|
91
|
+
io.abort $! if io.respond_to? :abort
|
92
92
|
io.join if io.respond_to? :join
|
93
93
|
raise $!
|
94
94
|
ensure
|
data/lib/scout/path/util.rb
CHANGED
@@ -86,6 +86,26 @@ module Path
|
|
86
86
|
self.annotate(self.split(".")[0..-2] * ".")
|
87
87
|
end
|
88
88
|
|
89
|
+
def remove_extension(extension = nil)
|
90
|
+
if extension.nil?
|
91
|
+
unset_extension
|
92
|
+
else
|
93
|
+
self.annotate(self.sub(/\.#{extension}$/,''))
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def replace_extension(new_extension = nil, multiple = false)
|
98
|
+
if String === multiple
|
99
|
+
new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
|
100
|
+
elsif multiple
|
101
|
+
new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
|
102
|
+
else
|
103
|
+
new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
|
104
|
+
end
|
105
|
+
new_path = new_path + "." + new_extension.to_s
|
106
|
+
self.annotate(new_path)
|
107
|
+
end
|
108
|
+
|
89
109
|
|
90
110
|
# Is 'file' newer than 'path'? return non-true if path is newer than file
|
91
111
|
def self.newer?(path, file, by_link = false)
|
data/lib/scout/path.rb
CHANGED
@@ -32,7 +32,7 @@ module Path
|
|
32
32
|
prevpath = prevpath.to_s if Symbol === prevpath
|
33
33
|
|
34
34
|
subpath = File.join(prevpath.to_s, subpath) if prevpath
|
35
|
-
new = self.empty? ? subpath : File.join(self, subpath)
|
35
|
+
new = self.empty? ? subpath.dup : File.join(self, subpath)
|
36
36
|
self.annotate(new)
|
37
37
|
new
|
38
38
|
end
|
data/lib/scout/resource/path.rb
CHANGED
@@ -3,7 +3,7 @@ module Path
|
|
3
3
|
return self if ! force && (Open.exist?(self) || @produced)
|
4
4
|
begin
|
5
5
|
if Resource === self.pkgdir
|
6
|
-
self.pkgdir.produce self
|
6
|
+
self.pkgdir.produce self, force
|
7
7
|
else
|
8
8
|
false
|
9
9
|
end
|
@@ -13,7 +13,7 @@ module Path
|
|
13
13
|
message = $!.message
|
14
14
|
message = "No exception message" if message.nil? || message.empty?
|
15
15
|
Log.warn "Error producing #{self}: #{message}"
|
16
|
-
|
16
|
+
raise $!
|
17
17
|
ensure
|
18
18
|
@produced = true
|
19
19
|
end
|
@@ -33,13 +33,16 @@ module Path
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def produce_and_find(extension = nil, *args)
|
36
|
-
if extension
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
36
|
+
found = if extension
|
37
|
+
found = find_with_extension(extension, *args)
|
38
|
+
found.exists? ? found : produce_with_extension(extension, *args)
|
39
|
+
else
|
40
|
+
found = find
|
41
|
+
found.exists? ? found : produce(*args)
|
42
|
+
end
|
43
|
+
raise "Not found: #{self}" unless found
|
44
|
+
|
45
|
+
found
|
43
46
|
end
|
44
47
|
|
45
48
|
def relocate
|
@@ -6,7 +6,7 @@ module Resource
|
|
6
6
|
|
7
7
|
def self.install(content, name, software_dir = Path.setup('software'), &block)
|
8
8
|
software_dir ||= Path.setup('software')
|
9
|
-
software_dir = software_dir.find if Path === software_dir
|
9
|
+
software_dir = software_dir.find(:user) if Path === software_dir
|
10
10
|
|
11
11
|
content = block if block_given?
|
12
12
|
|
@@ -166,11 +166,13 @@ url='#{url}'
|
|
166
166
|
value.sub!(/^['"]/,'')
|
167
167
|
value.sub!(/['"]$/,'')
|
168
168
|
value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
|
169
|
-
Log.debug "Set variable export from .post_install: #{
|
169
|
+
Log.debug "Set variable export from .post_install: #{Log.fingerprint [var,value]*"="}"
|
170
170
|
ENV[var] = value
|
171
171
|
end
|
172
172
|
end
|
173
173
|
end
|
174
174
|
end
|
175
175
|
end
|
176
|
+
|
177
|
+
self.set_software_env
|
176
178
|
end
|
data/lib/scout/resource/util.rb
CHANGED
@@ -28,7 +28,11 @@ module Resource
|
|
28
28
|
m.named_captures.include?(c) ? m[c] : nil
|
29
29
|
}.compact * "/"
|
30
30
|
unlocated.gsub!(/\/+/,'/')
|
31
|
-
|
31
|
+
if self.subdir && ! self.subdir.empty?
|
32
|
+
subdir = self.subdir
|
33
|
+
subdir += "/" unless subdir.end_with?("/")
|
34
|
+
unlocated[subdir] = ""
|
35
|
+
end
|
32
36
|
choices << self.annotate(unlocated)
|
33
37
|
end
|
34
38
|
end
|
data/lib/scout/resource.rb
CHANGED
@@ -2,7 +2,9 @@ require_relative 'log'
|
|
2
2
|
require_relative 'path'
|
3
3
|
require_relative 'resource/produce'
|
4
4
|
require_relative 'resource/path'
|
5
|
+
require_relative 'resource/open'
|
5
6
|
require_relative 'resource/util'
|
7
|
+
require_relative 'resource/software'
|
6
8
|
|
7
9
|
module Resource
|
8
10
|
extend MetaExtension
|
data/lib/scout/tsv/dumper.rb
CHANGED
@@ -11,12 +11,15 @@ module TSV
|
|
11
11
|
fields_str = "#{header_hash}#{key_field || "Id"}#{sep}#{fields*sep}"
|
12
12
|
end
|
13
13
|
|
14
|
-
if
|
14
|
+
if String === preamble
|
15
|
+
preamble_str = preamble
|
16
|
+
elsif preamble && options.values.compact.any?
|
15
17
|
preamble_str = "#: " << IndiferentHash.hash2string(options)
|
16
18
|
else
|
17
19
|
preamble_str = nil
|
18
20
|
end
|
19
21
|
|
22
|
+
preamble_str = preamble_str.strip if preamble_str
|
20
23
|
[preamble_str, fields_str].compact * "\n"
|
21
24
|
end
|
22
25
|
|
data/lib/scout/tsv/open.rb
CHANGED
@@ -20,7 +20,7 @@ module Open
|
|
20
20
|
into.add *res
|
21
21
|
when TSV, Hash
|
22
22
|
key, value = res
|
23
|
-
if into.type == :double
|
23
|
+
if TSV === into && into.type == :double
|
24
24
|
into.zip_new key, value, insitu: false
|
25
25
|
else
|
26
26
|
into[key] = value
|
@@ -118,6 +118,12 @@ module Open
|
|
118
118
|
callback.call res if callback
|
119
119
|
nil
|
120
120
|
end
|
121
|
+
when Hash
|
122
|
+
obj.each do |key,value|
|
123
|
+
res = block.call(key,value)
|
124
|
+
callback.call res if callback
|
125
|
+
nil
|
126
|
+
end
|
121
127
|
when Array
|
122
128
|
obj.each do |line|
|
123
129
|
res = block.call(line)
|
data/lib/scout/tsv/parser.rb
CHANGED
@@ -51,7 +51,7 @@ module TSV
|
|
51
51
|
[key, items]
|
52
52
|
end
|
53
53
|
|
54
|
-
def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, **kargs, &block)
|
54
|
+
def self.parse_stream(stream, data: nil, source_type: nil, type: :list, merge: true, one2one: false, fix: true, bar: false, first_line: nil, field_names: nil, head: nil, **kargs, &block)
|
55
55
|
begin
|
56
56
|
bar = "Parsing #{Log.fingerprint stream}" if TrueClass === bar
|
57
57
|
bar = Log::ProgressBar.get_obj_bar(stream, bar) if bar
|
@@ -62,7 +62,8 @@ module TSV
|
|
62
62
|
data = {} if data.nil?
|
63
63
|
merge = false if type != :double && type != :flat
|
64
64
|
line = first_line || stream.gets
|
65
|
-
while line
|
65
|
+
while line
|
66
|
+
break if head && head <= 0
|
66
67
|
begin
|
67
68
|
line.chomp!
|
68
69
|
if Proc === fix
|
@@ -171,6 +172,7 @@ module TSV
|
|
171
172
|
stream.abort($!) if stream.respond_to?(:abort)
|
172
173
|
raise $!
|
173
174
|
ensure
|
175
|
+
head = head - 1 if head
|
174
176
|
if stream.closed?
|
175
177
|
line = nil
|
176
178
|
else
|
@@ -185,7 +187,15 @@ module TSV
|
|
185
187
|
else
|
186
188
|
bar.remove
|
187
189
|
end if bar
|
188
|
-
|
190
|
+
|
191
|
+
if stream.respond_to?(:join)
|
192
|
+
eof = begin
|
193
|
+
stream.eof?
|
194
|
+
rescue IOError
|
195
|
+
true
|
196
|
+
end
|
197
|
+
stream.join if eof
|
198
|
+
end
|
189
199
|
end
|
190
200
|
end
|
191
201
|
|
data/lib/scout/tsv/util.rb
CHANGED
data/lib/scout/work_queue.rb
CHANGED
@@ -6,6 +6,7 @@ class WorkQueue
|
|
6
6
|
attr_accessor :workers, :worker_proc, :callback
|
7
7
|
|
8
8
|
def initialize(workers = 0, &block)
|
9
|
+
workers = workers.to_i if String === workers
|
9
10
|
@input = WorkQueue::Socket.new
|
10
11
|
@output = WorkQueue::Socket.new
|
11
12
|
@workers = workers.times.collect{ Worker.new }
|