bosh_deployer 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +139 -0
- data/Rakefile +51 -0
- data/config/aws_defaults.yml +45 -0
- data/config/vsphere_defaults.yml +38 -0
- data/lib/bosh/cli/commands/micro.rb +393 -0
- data/lib/deployer/config.rb +137 -0
- data/lib/deployer/errors.rb +11 -0
- data/lib/deployer/instance_manager.rb +452 -0
- data/lib/deployer/models/instance.rb +6 -0
- data/lib/deployer/version.rb +7 -0
- data/lib/deployer.rb +21 -0
- metadata +128 -0
@@ -0,0 +1,137 @@
|
|
1
|
+
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
+
|
3
|
+
module Bosh; end
|
4
|
+
|
5
|
+
module Bosh::Deployer
|
6
|
+
class Config
|
7
|
+
|
8
|
+
class << self
|
9
|
+
|
10
|
+
attr_accessor :logger, :db, :uuid, :resources, :cloud_options, :spec_properties, :bosh_ip
|
11
|
+
|
12
|
+
def configure(config)
|
13
|
+
if config["cloud"].nil?
|
14
|
+
raise ConfigError, "No cloud properties defined"
|
15
|
+
end
|
16
|
+
if config["cloud"]["plugin"].nil?
|
17
|
+
raise ConfigError, "No cloud plugin defined"
|
18
|
+
end
|
19
|
+
|
20
|
+
config = deep_merge(load_defaults(config["cloud"]["plugin"]), config)
|
21
|
+
|
22
|
+
@base_dir = config["dir"]
|
23
|
+
FileUtils.mkdir_p(@base_dir)
|
24
|
+
|
25
|
+
@cloud_options = config["cloud"]
|
26
|
+
@net_conf = config["network"]
|
27
|
+
@bosh_ip = @net_conf["ip"]
|
28
|
+
@resources = config["resources"]
|
29
|
+
|
30
|
+
@logger = Logger.new(config["logging"]["file"] || STDOUT)
|
31
|
+
@logger.level = Logger.const_get(config["logging"]["level"].upcase)
|
32
|
+
@logger.formatter = ThreadFormatter.new
|
33
|
+
|
34
|
+
@spec_properties = config["apply_spec"]["properties"]
|
35
|
+
|
36
|
+
@db = Sequel.sqlite
|
37
|
+
|
38
|
+
@db.create_table :vsphere_disk do
|
39
|
+
primary_key :id
|
40
|
+
column :path, :text
|
41
|
+
column :datacenter, :text
|
42
|
+
column :datastore, :text
|
43
|
+
column :size, :integer
|
44
|
+
end
|
45
|
+
|
46
|
+
@db.create_table :instances do
|
47
|
+
primary_key :id
|
48
|
+
column :name, :text, :unique => true, :null => false
|
49
|
+
column :uuid, :text
|
50
|
+
column :stemcell_cid, :text
|
51
|
+
column :stemcell_name, :text
|
52
|
+
column :vm_cid, :text
|
53
|
+
column :disk_cid, :text
|
54
|
+
end
|
55
|
+
|
56
|
+
Sequel::Model.plugin :validation_helpers
|
57
|
+
|
58
|
+
Bosh::Clouds::Config.configure(self)
|
59
|
+
|
60
|
+
require "deployer/models/instance"
|
61
|
+
|
62
|
+
@cloud_options["properties"]["agent"]["mbus"] ||=
|
63
|
+
"http://vcap:b00tstrap@0.0.0.0:6868"
|
64
|
+
|
65
|
+
@disk_model = nil
|
66
|
+
@cloud = nil
|
67
|
+
@networks = nil
|
68
|
+
end
|
69
|
+
|
70
|
+
def disk_model
|
71
|
+
if @disk_model.nil?
|
72
|
+
case @cloud_options["plugin"]
|
73
|
+
when "vsphere"
|
74
|
+
require "cloud/vsphere"
|
75
|
+
@disk_model = VSphereCloud::Models::Disk
|
76
|
+
else
|
77
|
+
end
|
78
|
+
end
|
79
|
+
@disk_model
|
80
|
+
end
|
81
|
+
|
82
|
+
def cloud
|
83
|
+
if @cloud.nil?
|
84
|
+
@cloud = Bosh::Clouds::Provider.create(@cloud_options["plugin"],
|
85
|
+
@cloud_options["properties"])
|
86
|
+
end
|
87
|
+
@cloud
|
88
|
+
end
|
89
|
+
|
90
|
+
def agent
|
91
|
+
uri = URI.parse(@cloud_options["properties"]["agent"]["mbus"])
|
92
|
+
uri.host = bosh_ip
|
93
|
+
user, password = uri.userinfo.split(":", 2)
|
94
|
+
uri.userinfo = nil
|
95
|
+
Bosh::Agent::HTTPClient.new(uri.to_s,
|
96
|
+
{ "user" => user,
|
97
|
+
"password" => password,
|
98
|
+
"reply_to" => uuid })
|
99
|
+
end
|
100
|
+
|
101
|
+
def networks
|
102
|
+
@networks ||= {
|
103
|
+
"bosh" => {
|
104
|
+
"cloud_properties" => @net_conf["cloud_properties"],
|
105
|
+
"netmask" => @net_conf["netmask"],
|
106
|
+
"gateway" => @net_conf["gateway"],
|
107
|
+
"ip" => @net_conf["ip"],
|
108
|
+
"dns" => @net_conf["dns"],
|
109
|
+
"type" => @net_conf["type"],
|
110
|
+
"default" => ["dns", "gateway"]
|
111
|
+
}
|
112
|
+
}
|
113
|
+
end
|
114
|
+
|
115
|
+
private
|
116
|
+
|
117
|
+
def deep_merge(src, dst)
|
118
|
+
src.merge(dst) do |key, old, new|
|
119
|
+
if new.respond_to?(:blank) && new.blank?
|
120
|
+
old
|
121
|
+
elsif old.kind_of?(Hash) and new.kind_of?(Hash)
|
122
|
+
deep_merge(old, new)
|
123
|
+
elsif old.kind_of?(Array) and new.kind_of?(Array)
|
124
|
+
old.concat(new).uniq
|
125
|
+
else
|
126
|
+
new
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def load_defaults(provider)
|
132
|
+
file = File.join(File.dirname(File.expand_path(__FILE__)), "../../config/#{provider}_defaults.yml")
|
133
|
+
YAML.load_file(file)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,452 @@
|
|
1
|
+
# Copyright (c) 2009-2012 VMware, Inc.
|
2
|
+
|
3
|
+
module Bosh::Deployer
|
4
|
+
class InstanceManager
|
5
|
+
|
6
|
+
DEPLOYMENTS_FILE = "bosh-deployments.yml"
|
7
|
+
|
8
|
+
attr_reader :state
|
9
|
+
attr_accessor :renderer
|
10
|
+
|
11
|
+
class LoggerRenderer
|
12
|
+
attr_accessor :stage, :total, :index
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
enter_stage("Deployer", 0)
|
16
|
+
end
|
17
|
+
|
18
|
+
def enter_stage(stage, total)
|
19
|
+
@stage = stage
|
20
|
+
@total = total
|
21
|
+
@index = 0
|
22
|
+
end
|
23
|
+
|
24
|
+
def update(state, task)
|
25
|
+
Config.logger.info("#{@stage} - #{state} #{task}")
|
26
|
+
@index += 1 if state == :finished
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(config)
|
31
|
+
Config.configure(config)
|
32
|
+
|
33
|
+
@state_yml = File.join(config["dir"], DEPLOYMENTS_FILE)
|
34
|
+
load_state(config["name"])
|
35
|
+
|
36
|
+
Config.uuid = state.uuid
|
37
|
+
|
38
|
+
@renderer = LoggerRenderer.new
|
39
|
+
end
|
40
|
+
|
41
|
+
def cloud
|
42
|
+
Config.cloud
|
43
|
+
end
|
44
|
+
|
45
|
+
def agent
|
46
|
+
Config.agent
|
47
|
+
end
|
48
|
+
|
49
|
+
def logger
|
50
|
+
Config.logger
|
51
|
+
end
|
52
|
+
|
53
|
+
def disk_model
|
54
|
+
Config.disk_model
|
55
|
+
end
|
56
|
+
|
57
|
+
def instance_model
|
58
|
+
Models::Instance
|
59
|
+
end
|
60
|
+
|
61
|
+
def exists?
|
62
|
+
state.vm_cid != nil
|
63
|
+
end
|
64
|
+
|
65
|
+
def step(task)
|
66
|
+
renderer.update(:started, task)
|
67
|
+
result = yield
|
68
|
+
renderer.update(:finished, task)
|
69
|
+
result
|
70
|
+
end
|
71
|
+
|
72
|
+
def create(stemcell_tgz)
|
73
|
+
if state.vm_cid
|
74
|
+
raise ConfigError, "VM #{state.vm_cid} already exists"
|
75
|
+
end
|
76
|
+
if state.stemcell_cid && state.stemcell_cid != state.stemcell_name
|
77
|
+
raise ConfigError, "stemcell #{state.stemcell_cid} already exists"
|
78
|
+
end
|
79
|
+
|
80
|
+
renderer.enter_stage("Deploy Micro BOSH", 11)
|
81
|
+
|
82
|
+
state.stemcell_cid = create_stemcell(stemcell_tgz)
|
83
|
+
state.stemcell_name = File.basename(stemcell_tgz, ".tgz")
|
84
|
+
save_state
|
85
|
+
|
86
|
+
begin
|
87
|
+
step "Creating VM from #{state.stemcell_cid}" do
|
88
|
+
state.vm_cid = create_vm(state.stemcell_cid)
|
89
|
+
discover_bosh_ip
|
90
|
+
end
|
91
|
+
save_state
|
92
|
+
rescue => e
|
93
|
+
delete_stemcell
|
94
|
+
raise e
|
95
|
+
end
|
96
|
+
|
97
|
+
step "Waiting for the agent" do
|
98
|
+
wait_until_agent_ready
|
99
|
+
end
|
100
|
+
|
101
|
+
step "Updating persistent disk" do
|
102
|
+
update_persistent_disk
|
103
|
+
end
|
104
|
+
|
105
|
+
apply(@apply_spec)
|
106
|
+
|
107
|
+
step "Waiting for the director" do
|
108
|
+
wait_until_director_ready
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def destroy
|
113
|
+
renderer.enter_stage("Delete micro BOSH", 6)
|
114
|
+
agent_stop
|
115
|
+
if state.disk_cid
|
116
|
+
delete_disk(state.disk_cid, state.vm_cid)
|
117
|
+
end
|
118
|
+
delete_vm
|
119
|
+
delete_stemcell
|
120
|
+
end
|
121
|
+
|
122
|
+
def update(stemcell_tgz)
|
123
|
+
renderer.enter_stage("Prepare for update", 5)
|
124
|
+
agent_stop
|
125
|
+
detach_disk
|
126
|
+
delete_vm
|
127
|
+
delete_stemcell
|
128
|
+
create(stemcell_tgz)
|
129
|
+
end
|
130
|
+
|
131
|
+
def create_stemcell(stemcell_tgz)
|
132
|
+
if File.directory?(stemcell_tgz)
|
133
|
+
step "Using existing stemcell" do
|
134
|
+
end
|
135
|
+
|
136
|
+
step "Loading apply spec" do
|
137
|
+
@apply_spec = load_apply_spec("#{stemcell_tgz}/apply_spec.yml")
|
138
|
+
end
|
139
|
+
|
140
|
+
return stemcell_tgz
|
141
|
+
end
|
142
|
+
|
143
|
+
Dir.mktmpdir("sc-") do |stemcell|
|
144
|
+
step "Unpacking stemcell" do
|
145
|
+
run_command("tar -zxf #{stemcell_tgz} -C #{stemcell}")
|
146
|
+
end
|
147
|
+
|
148
|
+
@apply_spec = load_apply_spec("#{stemcell}/apply_spec.yml")
|
149
|
+
properties = Config.cloud_options["properties"]["stemcell"]
|
150
|
+
|
151
|
+
step "Uploading stemcell" do
|
152
|
+
cloud.create_stemcell("#{stemcell}/image", properties)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def create_vm(stemcell_cid)
|
158
|
+
resources = Config.resources['cloud_properties']
|
159
|
+
networks = Config.networks
|
160
|
+
cloud.create_vm(state.uuid, stemcell_cid, resources, networks)
|
161
|
+
end
|
162
|
+
|
163
|
+
def mount_disk(disk_cid)
|
164
|
+
step "Mount disk" do
|
165
|
+
agent.run_task(:mount_disk, disk_cid.to_s)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def unmount_disk(disk_cid)
|
170
|
+
step "Unmount disk" do
|
171
|
+
if disk_info.include?(disk_cid)
|
172
|
+
agent.run_task(:unmount_disk, disk_cid.to_s)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def migrate_disk(src_disk_cid, dst_disk_cid)
|
178
|
+
step "Migrate disk" do
|
179
|
+
agent.run_task(:migrate_disk, src_disk_cid.to_s, dst_disk_cid.to_s)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def disk_info
|
184
|
+
return @disk_list if @disk_list
|
185
|
+
@disk_list = agent.list_disk
|
186
|
+
end
|
187
|
+
|
188
|
+
def create_disk
|
189
|
+
step "Create disk" do
|
190
|
+
state.disk_cid = cloud.create_disk(Config.resources['persistent_disk'], state.vm_cid)
|
191
|
+
save_state
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
def delete_disk(disk_cid, vm_cid)
|
196
|
+
unmount_disk(disk_cid)
|
197
|
+
|
198
|
+
begin
|
199
|
+
step "Detach disk" do
|
200
|
+
cloud.detach_disk(vm_cid, disk_cid) if vm_cid
|
201
|
+
end
|
202
|
+
rescue Bosh::Clouds::DiskNotAttached
|
203
|
+
end
|
204
|
+
|
205
|
+
begin
|
206
|
+
step "Delete disk" do
|
207
|
+
cloud.delete_disk(disk_cid)
|
208
|
+
end
|
209
|
+
state.disk_cid = nil
|
210
|
+
save_state
|
211
|
+
rescue Bosh::Clouds::DiskNotFound
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
def attach_disk(is_create=false)
|
216
|
+
return if state.disk_cid.nil?
|
217
|
+
|
218
|
+
cloud.attach_disk(state.vm_cid, state.disk_cid)
|
219
|
+
save_state
|
220
|
+
|
221
|
+
begin
|
222
|
+
mount_disk(state.disk_cid)
|
223
|
+
rescue
|
224
|
+
if is_create
|
225
|
+
logger.warn("!!! mount_disk(#{state.disk_cid}) failed !!! retrying...")
|
226
|
+
mount_disk(state.disk_cid)
|
227
|
+
else
|
228
|
+
raise
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
def detach_disk
|
234
|
+
if state.disk_cid.nil?
|
235
|
+
raise "Error while detaching disk: unknown disk attached to instance"
|
236
|
+
end
|
237
|
+
|
238
|
+
unmount_disk(state.disk_cid)
|
239
|
+
step "Detach disk" do
|
240
|
+
cloud.detach_disk(state.vm_cid, state.disk_cid)
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def attach_missing_disk
|
245
|
+
if state.disk_cid
|
246
|
+
attach_disk(true)
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
def check_persistent_disk
|
251
|
+
return if state.disk_cid.nil?
|
252
|
+
agent_disk_cid = disk_info.first
|
253
|
+
if agent_disk_cid != state.disk_cid
|
254
|
+
raise "instance #{state.vm_cid} has invalid disk: Agent reports #{agent_disk_cid} while deployer's record shows #{state.disk_cid}"
|
255
|
+
end
|
256
|
+
end
|
257
|
+
|
258
|
+
def update_persistent_disk
|
259
|
+
attach_missing_disk
|
260
|
+
check_persistent_disk
|
261
|
+
|
262
|
+
#XXX handle disk size change
|
263
|
+
if state.disk_cid.nil?
|
264
|
+
create_disk
|
265
|
+
attach_disk(true)
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def update_spec(spec)
|
270
|
+
properties = spec["properties"]
|
271
|
+
|
272
|
+
%w{blobstore postgres director redis nats aws_registry}.each do |service|
|
273
|
+
next unless properties[service]
|
274
|
+
properties[service]["address"] = bosh_ip
|
275
|
+
|
276
|
+
if override = Config.spec_properties[service]
|
277
|
+
properties[service].merge!(override)
|
278
|
+
end
|
279
|
+
end
|
280
|
+
|
281
|
+
case Config.cloud_options["plugin"]
|
282
|
+
when "vsphere"
|
283
|
+
properties["vcenter"] =
|
284
|
+
Config.spec_properties["vcenter"] ||
|
285
|
+
Config.cloud_options["properties"]["vcenters"].first.dup
|
286
|
+
|
287
|
+
properties["vcenter"]["address"] ||= properties["vcenter"]["host"]
|
288
|
+
when "aws"
|
289
|
+
properties["aws"] =
|
290
|
+
Config.spec_properties["aws"] ||
|
291
|
+
Config.cloud_options["properties"]["aws"].dup
|
292
|
+
|
293
|
+
properties["aws"]["registry"] = Config.cloud_options["properties"]["registry"]
|
294
|
+
properties["aws"]["stemcell"] = Config.cloud_options["properties"]["stemcell"]
|
295
|
+
else
|
296
|
+
end
|
297
|
+
|
298
|
+
spec
|
299
|
+
end
|
300
|
+
|
301
|
+
def apply(spec)
|
302
|
+
agent_stop
|
303
|
+
|
304
|
+
step "Applying micro BOSH spec" do
|
305
|
+
agent.run_task(:apply, update_spec(spec.dup))
|
306
|
+
end
|
307
|
+
|
308
|
+
agent_start
|
309
|
+
end
|
310
|
+
|
311
|
+
def discover_bosh_ip
|
312
|
+
if exists? and cloud.respond_to?(:ec2)
|
313
|
+
Config.bosh_ip = cloud.ec2.instances[state.vm_cid].private_ip_address
|
314
|
+
logger.info("discovered bosh ip=#{Config.bosh_ip}")
|
315
|
+
end
|
316
|
+
Config.bosh_ip
|
317
|
+
end
|
318
|
+
|
319
|
+
private
|
320
|
+
|
321
|
+
def bosh_ip
|
322
|
+
Config.bosh_ip
|
323
|
+
end
|
324
|
+
|
325
|
+
def agent_stop
|
326
|
+
step "Stopping agent services" do
|
327
|
+
begin
|
328
|
+
agent.run_task(:stop)
|
329
|
+
rescue
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
def agent_start
|
335
|
+
step "Starting agent services" do
|
336
|
+
agent.run_task(:start)
|
337
|
+
end
|
338
|
+
end
|
339
|
+
|
340
|
+
def wait_until_ready
|
341
|
+
timeout_time = Time.now.to_f + (60 * 5)
|
342
|
+
begin
|
343
|
+
yield
|
344
|
+
sleep 0.5
|
345
|
+
rescue Bosh::Agent::Error, Errno::ECONNREFUSED => e
|
346
|
+
if timeout_time - Time.now.to_f > 0
|
347
|
+
retry
|
348
|
+
else
|
349
|
+
raise e
|
350
|
+
end
|
351
|
+
end
|
352
|
+
end
|
353
|
+
|
354
|
+
def wait_until_agent_ready #XXX >> agent_client
|
355
|
+
wait_until_ready { agent.ping }
|
356
|
+
end
|
357
|
+
|
358
|
+
def wait_until_director_ready
|
359
|
+
port = @apply_spec["properties"]["director"]["port"]
|
360
|
+
url = "http://#{bosh_ip}:#{port}/info"
|
361
|
+
wait_until_ready do
|
362
|
+
info = Yajl::Parser.parse(HTTPClient.new.get(url).body)
|
363
|
+
logger.info("Director is ready: #{info.inspect}")
|
364
|
+
end
|
365
|
+
end
|
366
|
+
|
367
|
+
def delete_stemcell
|
368
|
+
unless state.stemcell_cid
|
369
|
+
raise ConfigError, "Cannot find existing stemcell"
|
370
|
+
end
|
371
|
+
|
372
|
+
if state.stemcell_cid == state.stemcell_name
|
373
|
+
step "Preserving stemcell" do
|
374
|
+
end
|
375
|
+
else
|
376
|
+
step "Delete stemcell" do
|
377
|
+
cloud.delete_stemcell(state.stemcell_cid)
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
state.stemcell_cid = nil
|
382
|
+
state.stemcell_name = nil
|
383
|
+
save_state
|
384
|
+
end
|
385
|
+
|
386
|
+
def delete_vm
|
387
|
+
unless state.vm_cid
|
388
|
+
raise ConfigError, "Cannot find existing VM"
|
389
|
+
end
|
390
|
+
step "Delete VM" do
|
391
|
+
cloud.delete_vm(state.vm_cid)
|
392
|
+
end
|
393
|
+
state.vm_cid = nil
|
394
|
+
save_state
|
395
|
+
end
|
396
|
+
|
397
|
+
def load_deployments
|
398
|
+
if File.exists?(@state_yml)
|
399
|
+
logger.info("Loading existing deployment data from: #{@state_yml}")
|
400
|
+
YAML.load_file(@state_yml)
|
401
|
+
else
|
402
|
+
logger.info("No existing deployments found (will save to #{@state_yml})")
|
403
|
+
{ "instances" => [], "disks" => [] }
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
def load_apply_spec(file)
|
408
|
+
logger.info("Loading apply spec from #{file}")
|
409
|
+
YAML.load_file(file)
|
410
|
+
end
|
411
|
+
|
412
|
+
def generate_unique_name
|
413
|
+
UUIDTools::UUID.random_create.to_s
|
414
|
+
end
|
415
|
+
|
416
|
+
def load_state(name)
|
417
|
+
@deployments = load_deployments
|
418
|
+
|
419
|
+
disk_model.insert_multiple(@deployments["disks"]) if disk_model
|
420
|
+
instance_model.insert_multiple(@deployments["instances"])
|
421
|
+
|
422
|
+
@state = instance_model.find(:name => name)
|
423
|
+
if @state.nil?
|
424
|
+
@state = instance_model.new
|
425
|
+
@state.uuid = "bm-#{generate_unique_name}"
|
426
|
+
@state.name = name
|
427
|
+
@state.save
|
428
|
+
else
|
429
|
+
discover_bosh_ip
|
430
|
+
end
|
431
|
+
end
|
432
|
+
|
433
|
+
def save_state
|
434
|
+
state.save
|
435
|
+
@deployments["instances"] = instance_model.map { |instance| instance.values }
|
436
|
+
@deployments["disks"] = disk_model.map { |disk| disk.values } if disk_model
|
437
|
+
|
438
|
+
File.open(@state_yml, "w") do |file|
|
439
|
+
file.write(YAML.dump(@deployments))
|
440
|
+
end
|
441
|
+
end
|
442
|
+
|
443
|
+
def run_command(command)
|
444
|
+
output = `#{command} 2>&1`
|
445
|
+
if $?.exitstatus != 0
|
446
|
+
$stderr.puts output
|
447
|
+
raise "'#{command}' failed with exit status=#{$?.exitstatus} [#{output}]"
|
448
|
+
end
|
449
|
+
end
|
450
|
+
|
451
|
+
end
|
452
|
+
end
|