scout-gear 1.2.0 → 5.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitmodules +4 -0
- data/.vimproject +663 -4
- data/Rakefile +1 -0
- data/VERSION +1 -1
- data/bin/scout +235 -0
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/indiferent_hash.rb +0 -30
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +15 -1
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +226 -0
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc.rb +5 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +372 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +78 -26
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +17 -5
- data/lib/scout/path.rb +13 -31
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +140 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +15 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource.rb +37 -0
- data/lib/scout/simple_opt/accessor.rb +54 -0
- data/lib/scout/simple_opt/doc.rb +102 -0
- data/lib/scout/simple_opt/get.rb +57 -0
- data/lib/scout/simple_opt/parse.rb +67 -0
- data/lib/scout/simple_opt/setup.rb +26 -0
- data/lib/scout/simple_opt.rb +5 -0
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +72 -0
- data/lib/scout/workflow/documentation.rb +77 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step.rb +96 -0
- data/lib/scout/workflow/task/inputs.rb +112 -0
- data/lib/scout/workflow/task.rb +141 -0
- data/lib/scout/workflow/usage.rb +294 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +39 -0
- data/lib/scout-gear.rb +5 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +78 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/task +707 -0
- data/test/scout/indiferent_hash/test_case_insensitive.rb +16 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +37 -1
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/simple_opt/test_get.rb +11 -0
- data/test/scout/simple_opt/test_parse.rb +10 -0
- data/test/scout/simple_opt/test_setup.rb +77 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +28 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +178 -0
- data/test/scout/workflow/test_usage.rb +26 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +76 -3
data/bin/scout
CHANGED
@@ -0,0 +1,235 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift File.join(__dir__, '../lib')
|
4
|
+
|
5
|
+
require 'scout-gear'
|
6
|
+
|
7
|
+
class CmdStop < Exception
|
8
|
+
attr_accessor :exit_status
|
9
|
+
def initialize(exit_status = 0)
|
10
|
+
@exit_status = exit_status
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Add paths to scout repos under --dev to LOAD_PATHS
|
15
|
+
dev_dir = nil
|
16
|
+
if _i = ARGV.index("--dev")
|
17
|
+
dev_dir = ARGV[_i+1]
|
18
|
+
ARGV.delete "--dev"
|
19
|
+
ARGV.delete dev_dir
|
20
|
+
end
|
21
|
+
|
22
|
+
if dev_dir.nil?
|
23
|
+
_s = nil
|
24
|
+
ARGV.each_with_index do |s,i|
|
25
|
+
if s.match(/^--dev(?:=(.*))?/)
|
26
|
+
dev_dir = $1
|
27
|
+
_s = s
|
28
|
+
next
|
29
|
+
end
|
30
|
+
end
|
31
|
+
ARGV.delete _s if _s
|
32
|
+
end
|
33
|
+
|
34
|
+
if dev_dir
|
35
|
+
['scout-*/lib'].each do |pattern|
|
36
|
+
Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
|
37
|
+
$LOAD_PATH.unshift f
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
require 'scout/simple_opt'
|
43
|
+
|
44
|
+
options = SOPT.setup <<EOF
|
45
|
+
Ruby bioinformatics toolkit
|
46
|
+
|
47
|
+
$ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value'
|
48
|
+
|
49
|
+
|
50
|
+
--log* #{Log.color :yellow, "Log level from 0 (debug) 6 (errors)"}
|
51
|
+
--dev* #{Log.color :yellow, "Find development libraries in the directory specified"}
|
52
|
+
--nocolor #{Log.color :yellow, "Disable colored output"}
|
53
|
+
--nobar #{Log.color :yellow, "Disable progress report"}
|
54
|
+
--locate_file #{Log.color :yellow, "Report the location of the script instead of executing it"}
|
55
|
+
EOF
|
56
|
+
|
57
|
+
Log.nocolor = true if options[:nocolor]
|
58
|
+
|
59
|
+
locate = options.delete :locate_file
|
60
|
+
|
61
|
+
if options[:log_file]
|
62
|
+
Log.logfile(options[:log_file])
|
63
|
+
end
|
64
|
+
|
65
|
+
if options[:log]
|
66
|
+
Log.severity = options[:log].to_i
|
67
|
+
else
|
68
|
+
global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
|
69
|
+
if ENV["SCOUT_LOG"]
|
70
|
+
Log.severity = ENV["SCOUT_LOG"].to_i
|
71
|
+
else
|
72
|
+
global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
|
73
|
+
Log.severity = global_severity.to_i if global_severity
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
$scout_command_dir = Scout.bin.scout
|
78
|
+
$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
|
79
|
+
|
80
|
+
SOPT.description =<<EOF
|
81
|
+
This command controls many aspects of the Scout framework, from configuration tasks to running applications.
|
82
|
+
|
83
|
+
Commands are implemented in separate files under the Scout path '#{$scout_command_dir}'.
|
84
|
+
Known locations are: #{([$scout_command_dir] + $scout_command_dir.find_all) * ", " }.
|
85
|
+
You can place your own commads at #{$scout_command_dir.find(:user)}.
|
86
|
+
EOF
|
87
|
+
|
88
|
+
if options[:profile]
|
89
|
+
require 'ruby-prof'
|
90
|
+
RubyProf.start
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
def prev_dir(prev)
|
95
|
+
scout_command_dir = $scout_command_dir
|
96
|
+
|
97
|
+
prev.each do |previous_command|
|
98
|
+
scout_command_dir = scout_command_dir[previous_command]
|
99
|
+
end
|
100
|
+
|
101
|
+
scout_command_dir
|
102
|
+
end
|
103
|
+
|
104
|
+
def commands(prev)
|
105
|
+
scout_command_dir = prev_dir(prev)
|
106
|
+
|
107
|
+
command_file_dirs = scout_command_dir.find_all
|
108
|
+
command_files = command_file_dirs.collect{|d| d.glob('*') }.flatten
|
109
|
+
command_files.collect{|p| File.basename(p) }.uniq.reject{|p| p =~ /\.desc$/}.sort
|
110
|
+
end
|
111
|
+
|
112
|
+
def scout_usage(prev = nil)
|
113
|
+
puts
|
114
|
+
puts SOPT.doc
|
115
|
+
|
116
|
+
if prev
|
117
|
+
puts
|
118
|
+
puts Log.color :magenta, "## COMMANDS"
|
119
|
+
puts
|
120
|
+
puts Log.color :magenta, "Command:"
|
121
|
+
puts
|
122
|
+
puts " #{File.basename($0)} #{prev * " "}"
|
123
|
+
puts
|
124
|
+
puts Log.color :magenta, "Subcommands:"
|
125
|
+
puts
|
126
|
+
prev_dir = prev_dir(prev)
|
127
|
+
commands(prev).each do |command|
|
128
|
+
directory = File.directory? prev_dir[command].find
|
129
|
+
if directory
|
130
|
+
puts " " << Log.color(:blue, command)
|
131
|
+
else
|
132
|
+
puts " " << command
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
puts
|
137
|
+
true
|
138
|
+
end
|
139
|
+
|
140
|
+
alias usage scout_usage
|
141
|
+
|
142
|
+
def print_error(error, backtrace = nil)
|
143
|
+
puts Log.color :magenta, "## ERROR"
|
144
|
+
puts
|
145
|
+
if backtrace
|
146
|
+
puts Log.color :red, "Backtrace: "
|
147
|
+
puts
|
148
|
+
puts Log.color_stack(backtrace.reverse) * "\n"
|
149
|
+
puts
|
150
|
+
end
|
151
|
+
puts Log.color :red, error
|
152
|
+
puts
|
153
|
+
end
|
154
|
+
|
155
|
+
def aliases
|
156
|
+
@aliases ||= Scout.etc.cmd_alias.exists? ? Scout.etc.cmd_alias.yaml : {}
|
157
|
+
end
|
158
|
+
|
159
|
+
def tokenize_cmd_params(str)
|
160
|
+
return str if Array === str
|
161
|
+
str.scan(/
|
162
|
+
(?:["']([^"']*?)["']) |
|
163
|
+
([^"'\s]+)
|
164
|
+
/x).flatten.compact
|
165
|
+
end
|
166
|
+
|
167
|
+
def cmd_alias
|
168
|
+
while ARGV[0] && aliases.include?(ARGV[0])
|
169
|
+
ARGV.replace tokenize_cmd_params(aliases[ARGV[0]]) + ARGV[1..-1]
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
dir = $scout_command_dir
|
174
|
+
$previous_commands = []
|
175
|
+
|
176
|
+
cmd_alias
|
177
|
+
|
178
|
+
exit_status = 0
|
179
|
+
begin
|
180
|
+
while ARGV.any?
|
181
|
+
$command = ARGV.shift
|
182
|
+
case
|
183
|
+
when File.directory?(dir[$command].find)
|
184
|
+
$previous_commands << $command
|
185
|
+
dir = dir[$command]
|
186
|
+
when dir[$command].exists?
|
187
|
+
if locate
|
188
|
+
puts dir[$command].find
|
189
|
+
exit_status = 0
|
190
|
+
exit exit_status
|
191
|
+
else
|
192
|
+
load dir[$command].find
|
193
|
+
exit_status = 0
|
194
|
+
exit exit_status
|
195
|
+
end
|
196
|
+
when File.exist?($command)
|
197
|
+
load $command
|
198
|
+
exit_status = 0
|
199
|
+
exit exit_status
|
200
|
+
else
|
201
|
+
error = "Command '#{$command }' not understood"
|
202
|
+
scout_usage($previous_commands)
|
203
|
+
print_error(error)
|
204
|
+
exit_status = -1
|
205
|
+
exit exit_status
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
scout_usage($previous_commands)
|
210
|
+
exit_status = 0
|
211
|
+
exit exit_status
|
212
|
+
|
213
|
+
rescue ParameterException
|
214
|
+
puts
|
215
|
+
scout_usage
|
216
|
+
print_error($!.message, $!.backtrace)
|
217
|
+
puts
|
218
|
+
exit_status = -1
|
219
|
+
exit exit_status
|
220
|
+
rescue SystemExit,CmdStop
|
221
|
+
exit_status = $!.status
|
222
|
+
exit exit_status
|
223
|
+
rescue Exception
|
224
|
+
Log.exception $!
|
225
|
+
exit_status = -1
|
226
|
+
exit exit_status
|
227
|
+
ensure
|
228
|
+
if options[:profile]
|
229
|
+
result = RubyProf.stop
|
230
|
+
printer = RubyProf::FlatPrinter.new(result)
|
231
|
+
printer.print(STDOUT, :min_percent => 10)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
exit exit_status
|
data/lib/scout/cmd.rb
ADDED
@@ -0,0 +1,344 @@
|
|
1
|
+
require_relative 'indiferent_hash'
|
2
|
+
require_relative 'concurrent_stream'
|
3
|
+
require_relative 'log'
|
4
|
+
require_relative 'open/stream'
|
5
|
+
require 'stringio'
|
6
|
+
require 'open3'
|
7
|
+
|
8
|
+
module CMD
|
9
|
+
|
10
|
+
TOOLS = IndiferentHash.setup({})
|
11
|
+
def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
|
12
|
+
TOOLS[tool] = [claim, test, block, cmd]
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.conda(tool, env = nil, channel = 'bioconda')
|
16
|
+
if env
|
17
|
+
CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
|
18
|
+
else
|
19
|
+
CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.get_tool(tool)
|
24
|
+
return tool.to_s unless TOOLS[tool]
|
25
|
+
|
26
|
+
@@init_cmd_tool ||= IndiferentHash.setup({})
|
27
|
+
if !@@init_cmd_tool[tool]
|
28
|
+
claim, test, block, cmd = TOOLS[tool]
|
29
|
+
begin
|
30
|
+
if test
|
31
|
+
CMD.cmd(test + " ")
|
32
|
+
else
|
33
|
+
CMD.cmd("#{tool} --help")
|
34
|
+
end
|
35
|
+
rescue
|
36
|
+
if claim
|
37
|
+
claim.produce
|
38
|
+
else
|
39
|
+
block.call
|
40
|
+
end
|
41
|
+
end
|
42
|
+
version_txt = ""
|
43
|
+
version = nil
|
44
|
+
["--version", "-version", "--help", ""].each do |f|
|
45
|
+
begin
|
46
|
+
version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
|
47
|
+
version = CMD.scan_version_text(version_txt, tool)
|
48
|
+
break if version
|
49
|
+
rescue
|
50
|
+
Log.exception $!
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
@@init_cmd_tool[tool] = version || true
|
55
|
+
|
56
|
+
return cmd if cmd
|
57
|
+
end
|
58
|
+
|
59
|
+
tool.to_s
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.scan_version_text(text, cmd = nil)
|
63
|
+
cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
|
64
|
+
text.split("\n").each do |line|
|
65
|
+
next unless line =~ /\W#{cmd}\W/i
|
66
|
+
m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
67
|
+
return m[1] if m
|
68
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
|
69
|
+
return m[1] if m
|
70
|
+
next unless line =~ /\Wversion\W/i
|
71
|
+
m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
72
|
+
return m[1] if m
|
73
|
+
end
|
74
|
+
m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
|
75
|
+
return m[1] if m
|
76
|
+
m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
|
77
|
+
return m[1] if m
|
78
|
+
nil
|
79
|
+
end
|
80
|
+
def self.versions
|
81
|
+
return {} unless defined? @@init_cmd_tool
|
82
|
+
@@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.bash(cmd)
|
86
|
+
cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
|
87
|
+
CMD.cmd(cmd, :autojoin => true)
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.process_cmd_options(options = {})
|
91
|
+
add_dashes = IndiferentHash.process_options options, :add_option_dashes
|
92
|
+
|
93
|
+
string = ""
|
94
|
+
options.each do |option, value|
|
95
|
+
raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
|
96
|
+
#raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
|
97
|
+
value = value.gsub("'","\\'") if value.to_s.include? "'"
|
98
|
+
|
99
|
+
option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
|
100
|
+
|
101
|
+
case
|
102
|
+
when value.nil? || FalseClass === value
|
103
|
+
next
|
104
|
+
when TrueClass === value
|
105
|
+
string << "#{option} "
|
106
|
+
else
|
107
|
+
if option.to_s.chars.to_a.last == "="
|
108
|
+
string << "#{option}'#{value}' "
|
109
|
+
else
|
110
|
+
string << "#{option} '#{value}' "
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
string.strip
|
116
|
+
end
|
117
|
+
|
118
|
+
def self.cmd(tool, cmd = nil, options = {}, &block)
|
119
|
+
options, cmd = cmd, nil if Hash === cmd
|
120
|
+
|
121
|
+
options = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
|
122
|
+
in_content = options.delete(:in)
|
123
|
+
stderr = options.delete(:stderr)
|
124
|
+
post = options.delete(:post)
|
125
|
+
pipe = options.delete(:pipe)
|
126
|
+
log = options.delete(:log)
|
127
|
+
no_fail = options.delete(:no_fail)
|
128
|
+
no_fail = options.delete(:nofail) if no_fail.nil?
|
129
|
+
no_wait = options.delete(:no_wait)
|
130
|
+
xvfb = options.delete(:xvfb)
|
131
|
+
bar = options.delete(:progress_bar)
|
132
|
+
save_stderr = options.delete(:save_stderr)
|
133
|
+
autojoin = options.delete(:autojoin)
|
134
|
+
autojoin = no_wait if autojoin.nil?
|
135
|
+
|
136
|
+
dont_close_in = options.delete(:dont_close_in)
|
137
|
+
|
138
|
+
log = true if log.nil?
|
139
|
+
|
140
|
+
if cmd.nil? && ! Symbol === tool
|
141
|
+
cmd = tool
|
142
|
+
else
|
143
|
+
tool = get_tool(tool)
|
144
|
+
if cmd.nil?
|
145
|
+
cmd = tool
|
146
|
+
else
|
147
|
+
cmd = tool + ' ' + cmd
|
148
|
+
end
|
149
|
+
|
150
|
+
end
|
151
|
+
|
152
|
+
case xvfb
|
153
|
+
when TrueClass
|
154
|
+
cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
|
155
|
+
when String
|
156
|
+
cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
|
157
|
+
when String
|
158
|
+
end
|
159
|
+
|
160
|
+
if stderr == true
|
161
|
+
stderr = Log::HIGH
|
162
|
+
end
|
163
|
+
|
164
|
+
cmd_options = process_cmd_options options
|
165
|
+
if cmd =~ /'\{opt\}'/
|
166
|
+
cmd.sub!('\'{opt}\'', cmd_options)
|
167
|
+
else
|
168
|
+
cmd << " " << cmd_options
|
169
|
+
end
|
170
|
+
|
171
|
+
in_content = StringIO.new in_content if String === in_content
|
172
|
+
|
173
|
+
sin, sout, serr, wait_thr = begin
|
174
|
+
Open3.popen3(ENV, cmd)
|
175
|
+
rescue
|
176
|
+
Log.warn $!.message
|
177
|
+
raise ProcessFailed, nil, cmd unless no_fail
|
178
|
+
return
|
179
|
+
end
|
180
|
+
pid = wait_thr.pid
|
181
|
+
|
182
|
+
Log.debug{"CMD: [#{pid}] #{cmd}" if log}
|
183
|
+
|
184
|
+
if in_content.respond_to?(:read)
|
185
|
+
in_thread = Thread.new(Thread.current) do |parent|
|
186
|
+
Thread.current.report_on_exception = false if no_fail
|
187
|
+
begin
|
188
|
+
begin
|
189
|
+
while c = in_content.readpartial(Open::BLOCK_SIZE)
|
190
|
+
sin << c
|
191
|
+
end
|
192
|
+
rescue EOFError
|
193
|
+
end
|
194
|
+
sin.close unless sin.closed?
|
195
|
+
|
196
|
+
unless dont_close_in
|
197
|
+
in_content.close unless in_content.closed?
|
198
|
+
in_content.join if in_content.respond_to? :join
|
199
|
+
end
|
200
|
+
rescue
|
201
|
+
Log.error "Error in CMD [#{pid}] #{cmd}: #{$!.message}" unless no_fail
|
202
|
+
raise $!
|
203
|
+
end
|
204
|
+
end
|
205
|
+
else
|
206
|
+
in_thread = nil
|
207
|
+
sin.close
|
208
|
+
end
|
209
|
+
|
210
|
+
pids = [pid]
|
211
|
+
|
212
|
+
if pipe
|
213
|
+
|
214
|
+
ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
|
215
|
+
|
216
|
+
sout.callback = post if post
|
217
|
+
|
218
|
+
if (Integer === stderr and log) || bar
|
219
|
+
err_thread = Thread.new do
|
220
|
+
Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
|
221
|
+
begin
|
222
|
+
while line = serr.gets
|
223
|
+
bar.process(line) if bar
|
224
|
+
sout.log = line
|
225
|
+
sout.std_err << line if save_stderr
|
226
|
+
Log.log "STDERR [#{pid}]: " + line, stderr if log
|
227
|
+
end
|
228
|
+
serr.close
|
229
|
+
rescue
|
230
|
+
Log.exception $!
|
231
|
+
raise $!
|
232
|
+
end
|
233
|
+
end
|
234
|
+
else
|
235
|
+
err_thread = Open.consume_stream(serr, true)
|
236
|
+
end
|
237
|
+
|
238
|
+
sout.threads = [in_thread, err_thread, wait_thr].compact
|
239
|
+
|
240
|
+
sout
|
241
|
+
else
|
242
|
+
|
243
|
+
if bar
|
244
|
+
err = ""
|
245
|
+
err_thread = Thread.new do
|
246
|
+
while not serr.eof?
|
247
|
+
line = serr.gets
|
248
|
+
bar.process(line)
|
249
|
+
err << line if Integer === stderr and log
|
250
|
+
end
|
251
|
+
serr.close
|
252
|
+
end
|
253
|
+
elsif log and Integer === stderr
|
254
|
+
err = ""
|
255
|
+
err_thread = Thread.new do
|
256
|
+
while not serr.eof?
|
257
|
+
err << serr.gets
|
258
|
+
end
|
259
|
+
serr.close
|
260
|
+
end
|
261
|
+
else
|
262
|
+
Open.consume_stream(serr, true)
|
263
|
+
#serr.close
|
264
|
+
err_thread = nil
|
265
|
+
err = ""
|
266
|
+
end
|
267
|
+
|
268
|
+
ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
|
269
|
+
|
270
|
+
begin
|
271
|
+
out = StringIO.new sout.read
|
272
|
+
sout.close unless sout.closed?
|
273
|
+
|
274
|
+
status = wait_thr.value
|
275
|
+
if not status.success? and not no_fail
|
276
|
+
if !err.empty?
|
277
|
+
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
|
278
|
+
else
|
279
|
+
raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
|
280
|
+
end
|
281
|
+
else
|
282
|
+
Log.log err, stderr if Integer === stderr and log
|
283
|
+
end
|
284
|
+
out
|
285
|
+
ensure
|
286
|
+
post.call if post
|
287
|
+
end
|
288
|
+
end
|
289
|
+
end
|
290
|
+
|
291
|
+
def self.cmd_pid(*args)
|
292
|
+
all_args = *args
|
293
|
+
|
294
|
+
bar = all_args.last[:progress_bar] if Hash === all_args.last
|
295
|
+
|
296
|
+
all_args << {} unless Hash === all_args.last
|
297
|
+
|
298
|
+
level = all_args.last[:log] || 0
|
299
|
+
level = 0 if TrueClass === level
|
300
|
+
level = 10 if FalseClass === level
|
301
|
+
level = level.to_i
|
302
|
+
|
303
|
+
all_args.last[:log] = true
|
304
|
+
all_args.last[:pipe] = true
|
305
|
+
|
306
|
+
io = cmd(*all_args)
|
307
|
+
pid = io.pids.first
|
308
|
+
|
309
|
+
line = "" if bar
|
310
|
+
starting = true
|
311
|
+
while c = io.getc
|
312
|
+
if starting
|
313
|
+
if pid
|
314
|
+
Log.logn "STDOUT [#{pid}]: ", level
|
315
|
+
else
|
316
|
+
Log.logn "STDOUT: ", level
|
317
|
+
end
|
318
|
+
starting = false
|
319
|
+
end
|
320
|
+
STDERR << c if Log.severity <= level
|
321
|
+
line << c if bar
|
322
|
+
if c == "\n"
|
323
|
+
bar.process(line) if bar
|
324
|
+
starting = true
|
325
|
+
line = "" if bar
|
326
|
+
end
|
327
|
+
end
|
328
|
+
begin
|
329
|
+
io.join
|
330
|
+
bar.remove if bar
|
331
|
+
rescue
|
332
|
+
bar.remove(true) if bar
|
333
|
+
raise $!
|
334
|
+
end
|
335
|
+
|
336
|
+
nil
|
337
|
+
end
|
338
|
+
|
339
|
+
def self.cmd_log(*args)
|
340
|
+
cmd_pid(*args)
|
341
|
+
nil
|
342
|
+
end
|
343
|
+
|
344
|
+
end
|