skytap-yf 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.DS_Store +0 -0
- data/Gemfile +4 -0
- data/README.md +0 -0
- data/README.rdoc +6 -0
- data/api_schema.yaml +1016 -0
- data/bin/skytap +4 -0
- data/ca-bundle.crt +3721 -0
- data/data/.DS_Store +0 -0
- data/lib/skytap/api_schema.rb +11 -0
- data/lib/skytap/command_line.rb +145 -0
- data/lib/skytap/commands/base.rb +294 -0
- data/lib/skytap/commands/help.rb +82 -0
- data/lib/skytap/commands/http.rb +196 -0
- data/lib/skytap/commands/root.rb +79 -0
- data/lib/skytap/commands.rb +9 -0
- data/lib/skytap/core_ext.rb +8 -0
- data/lib/skytap/error.rb +4 -0
- data/lib/skytap/help_templates/help.erb +52 -0
- data/lib/skytap/ip_address.rb +63 -0
- data/lib/skytap/logger.rb +52 -0
- data/lib/skytap/plugins/vm_copy_to_region.rb +200 -0
- data/lib/skytap/plugins/vm_download.rb +431 -0
- data/lib/skytap/plugins/vm_upload.rb +401 -0
- data/lib/skytap/requester.rb +134 -0
- data/lib/skytap/response.rb +61 -0
- data/lib/skytap/skytaprc.rb +28 -0
- data/lib/skytap/subnet.rb +92 -0
- data/lib/skytap/templates.rb +216 -0
- data/lib/skytap/version.rb +5 -0
- data/lib/skytap.rb +149 -0
- data/skytap.gemspec +25 -0
- data/skytap.rdoc +5 -0
- metadata +143 -0
@@ -0,0 +1,431 @@
|
|
1
|
+
module Skytap
|
2
|
+
module Commands
|
3
|
+
class Download < Skytap::Commands::Base
|
4
|
+
MAX_CONCURRENCY = 5
|
5
|
+
CHECK_PERIOD = 15
|
6
|
+
AT_CAPACITY_RETRY_PERIOD = 15.minutes.to_i
|
7
|
+
|
8
|
+
attr_reader :vm_ids
|
9
|
+
attr_reader :downloaders
|
10
|
+
|
11
|
+
self.parent = Vm
|
12
|
+
self.plugin = true
|
13
|
+
|
14
|
+
def self.description
|
15
|
+
<<-"EOF"
|
16
|
+
Download the specified Skytap template VM to the local filesystem
|
17
|
+
|
18
|
+
The VM must belong to a template, not a configuration. The template cannot be
|
19
|
+
public.
|
20
|
+
EOF
|
21
|
+
end
|
22
|
+
|
23
|
+
def expected_args
|
24
|
+
ActiveSupport::OrderedHash[
|
25
|
+
'vm_id*', 'One or more IDs of template VMs to donwload'
|
26
|
+
]
|
27
|
+
end
|
28
|
+
|
29
|
+
def expected_options
|
30
|
+
ActiveSupport::OrderedHash[
|
31
|
+
:dir, {:flag_arg => 'DIR', :desc => 'Directory into which to download VM and metadata'},
|
32
|
+
]
|
33
|
+
end
|
34
|
+
|
35
|
+
attr_reader :downloaders
|
36
|
+
|
37
|
+
def run!
|
38
|
+
@vm_ids = args.collect {|a| find_id(a)}
|
39
|
+
@downloaders = []
|
40
|
+
|
41
|
+
until finished?
|
42
|
+
if vm_ids.present? && slots_available?
|
43
|
+
kick_off_export(vm_ids.shift)
|
44
|
+
else
|
45
|
+
sleep CHECK_PERIOD
|
46
|
+
print_status
|
47
|
+
invoker.try(:call, self)
|
48
|
+
|
49
|
+
signal_available if signal_stale? || (concurrency_at_overage && concurrency < concurrency_at_overage)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
signal_available
|
54
|
+
print_summary
|
55
|
+
return response
|
56
|
+
end
|
57
|
+
|
58
|
+
def concurrency_at_overage
|
59
|
+
@concurrency_at_overage
|
60
|
+
end
|
61
|
+
|
62
|
+
def kick_off_export(vm_id)
|
63
|
+
begin
|
64
|
+
# Create export job on server in main thread.
|
65
|
+
job = VmExportJob.new(logger, username, api_token, vm_id, command_options[:dir])
|
66
|
+
|
67
|
+
# If successful, start FTP download and subsequent steps in new thread.
|
68
|
+
dl = Downloader.new(job)
|
69
|
+
downloaders << dl
|
70
|
+
log_line(dl.status_line)
|
71
|
+
rescue NoSlotsAvailable => ex
|
72
|
+
vm_ids << vm_id
|
73
|
+
signal_full
|
74
|
+
log_line(("VM #{vm_id}: " << no_capacity_message).color(:yellow))
|
75
|
+
rescue Exception => ex
|
76
|
+
dl = DeadDownloader.new(vm_id, ex)
|
77
|
+
downloaders << dl
|
78
|
+
log_line(dl.status_line)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def no_capacity_message
|
83
|
+
m = AT_CAPACITY_RETRY_PERIOD / 60
|
84
|
+
"No export capacity is currently available on Skytap. Will retry in #{m} minutes".tap do |msg|
|
85
|
+
if active_downloaders.present?
|
86
|
+
msg << ' or when another export completes.'
|
87
|
+
else
|
88
|
+
msg << '.'
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def log_line(msg, include_separator=true)
|
94
|
+
line = msg
|
95
|
+
line += "\n---" if include_separator
|
96
|
+
logger.info line
|
97
|
+
end
|
98
|
+
|
99
|
+
def print_status
|
100
|
+
logger.info "#{status_lines}\n---"
|
101
|
+
end
|
102
|
+
|
103
|
+
def print_summary
|
104
|
+
unless response.error?
|
105
|
+
logger.info "#{'Summary'.bright}\n#{response.payload}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def finished?
|
110
|
+
vm_ids.empty? && concurrency == 0
|
111
|
+
end
|
112
|
+
|
113
|
+
def slots_available?
|
114
|
+
estimated_available_slots > 0
|
115
|
+
end
|
116
|
+
|
117
|
+
def signal_stale?
|
118
|
+
full? && Time.now - @full_at > AT_CAPACITY_RETRY_PERIOD
|
119
|
+
end
|
120
|
+
|
121
|
+
def seconds_until_retry
|
122
|
+
return unless full?
|
123
|
+
[0, AT_CAPACITY_RETRY_PERIOD - (Time.now - @full_at)].max
|
124
|
+
end
|
125
|
+
|
126
|
+
def estimated_available_slots
|
127
|
+
if full?
|
128
|
+
0
|
129
|
+
else
|
130
|
+
MAX_CONCURRENCY - concurrency
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def signal_full
|
135
|
+
@concurrency_at_overage = concurrency
|
136
|
+
@full_at = Time.now
|
137
|
+
end
|
138
|
+
|
139
|
+
def signal_available
|
140
|
+
@concurrency_at_overage = @full_at = nil
|
141
|
+
end
|
142
|
+
|
143
|
+
def full?
|
144
|
+
@full_at
|
145
|
+
end
|
146
|
+
|
147
|
+
def status_lines
|
148
|
+
active_downloaders.collect(&:status_line).join("\n")
|
149
|
+
end
|
150
|
+
|
151
|
+
def active_downloaders
|
152
|
+
downloaders.reject(&:finished?)
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
private
|
157
|
+
|
158
|
+
def concurrency
|
159
|
+
downloaders.select(&:alive?).length
|
160
|
+
end
|
161
|
+
|
162
|
+
def response
|
163
|
+
@_response ||= begin
|
164
|
+
error = !downloaders.any?(&:success?)
|
165
|
+
Response.build(downloaders.collect(&:status_line).join("\n"), error)
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
class NoSlotsAvailable < RuntimeError
|
171
|
+
end
|
172
|
+
|
173
|
+
class VmExportJob
|
174
|
+
attr_reader :logger, :vm, :vm_id, :export_dir, :username, :api_token
|
175
|
+
|
176
|
+
def initialize(logger, username, api_token, vm_id, dir=nil)
|
177
|
+
@logger = logger
|
178
|
+
@username = username
|
179
|
+
@api_token = api_token
|
180
|
+
@vm_id = vm_id
|
181
|
+
|
182
|
+
@vm = Skytap.invoke!(username, api_token, "vm show #{vm_id}")
|
183
|
+
@export_dir = File.join(File.expand_path(dir || '.'), "vm_#{vm_id}")
|
184
|
+
FileUtils.mkdir_p(export_dir)
|
185
|
+
|
186
|
+
create_on_server
|
187
|
+
end
|
188
|
+
|
189
|
+
def create_on_server
|
190
|
+
begin
|
191
|
+
export
|
192
|
+
rescue Exception => ex
|
193
|
+
if at_capacity?(ex)
|
194
|
+
raise NoSlotsAvailable.new
|
195
|
+
else
|
196
|
+
raise
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
def at_capacity?(exception)
|
202
|
+
exception.message.include?('You cannot export a VM because you may not have more than')
|
203
|
+
end
|
204
|
+
|
205
|
+
def export(force_reload=false)
|
206
|
+
return @export unless @export.nil? || force_reload
|
207
|
+
|
208
|
+
if @export
|
209
|
+
id = @export['id']
|
210
|
+
@export = Skytap.invoke!(username, api_token, "export show #{id}")
|
211
|
+
else
|
212
|
+
@export = Skytap.invoke!(username, api_token, "export create", {}, :param => {'vm_id' => vm_id})
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
class DeadDownloader
|
218
|
+
def initialize(vm_id, exception)
|
219
|
+
@vm_id = vm_id
|
220
|
+
@exception = exception
|
221
|
+
end
|
222
|
+
|
223
|
+
def finished?
|
224
|
+
true
|
225
|
+
end
|
226
|
+
|
227
|
+
def alive?
|
228
|
+
false
|
229
|
+
end
|
230
|
+
|
231
|
+
def success?
|
232
|
+
false
|
233
|
+
end
|
234
|
+
|
235
|
+
def status_line
|
236
|
+
"VM #{@vm_id}: Error: #{@exception}"
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
class Downloader < Thread
|
241
|
+
MAX_WAIT = 2.days
|
242
|
+
EXPORT_CHECK_PERIOD = 5
|
243
|
+
|
244
|
+
attr_reader :job, :bytes_transferred, :bytes_total, :result
|
245
|
+
delegate :logger, :vm, :vm_id, :export, :export_dir, :username, :api_token, :to => :job
|
246
|
+
|
247
|
+
def initialize(job)
|
248
|
+
@job = job
|
249
|
+
@bytes_transferred = @bytes_total = 0
|
250
|
+
|
251
|
+
super do
|
252
|
+
begin
|
253
|
+
run
|
254
|
+
rescue Exception => ex
|
255
|
+
@result = Response.build(ex)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
def run
|
261
|
+
wait_until_ready
|
262
|
+
ftp_download
|
263
|
+
download_data
|
264
|
+
Skytap.invoke!(username, api_token, "export destroy #{id}")
|
265
|
+
@result = Response.build(export_dir)
|
266
|
+
end
|
267
|
+
|
268
|
+
def finished?
|
269
|
+
!!@finished
|
270
|
+
end
|
271
|
+
|
272
|
+
def success?
|
273
|
+
result && !result.error?
|
274
|
+
end
|
275
|
+
|
276
|
+
def status_line
|
277
|
+
prefix = "VM #{vm_id}".tap do |str|
|
278
|
+
if vm
|
279
|
+
str << " (#{vm['name']})"
|
280
|
+
end
|
281
|
+
end
|
282
|
+
prefix << ': ' << status
|
283
|
+
end
|
284
|
+
|
285
|
+
def status
|
286
|
+
if result.try(:error?)
|
287
|
+
@finished = true
|
288
|
+
"Error: #{result.error_message}".color(:red).bright
|
289
|
+
elsif result
|
290
|
+
@finished = true
|
291
|
+
"Downloaded: #{result.payload}".color(:green).bright
|
292
|
+
elsif bytes_transferred == 0
|
293
|
+
'Exporting'.color(:yellow)
|
294
|
+
else
|
295
|
+
gb_transferred = bytes_transferred / 1.gigabyte.to_f
|
296
|
+
gb_total = bytes_total / 1.gigabyte.to_f
|
297
|
+
percent_done = 100.0 * bytes_transferred / bytes_total
|
298
|
+
"Downloading #{'%0.1f' % percent_done}% (#{'%0.1f' % gb_transferred} / #{'%0.1f' % gb_total} GB)".color(:yellow)
|
299
|
+
end
|
300
|
+
end
|
301
|
+
|
302
|
+
def ftp_download
|
303
|
+
remote_path = export['filename']
|
304
|
+
local_path = File.join(export_dir, File.basename(export['filename']))
|
305
|
+
FileUtils.mkdir_p(export_dir)
|
306
|
+
|
307
|
+
ftp = Net::FTP.new(export['ftp_host'])
|
308
|
+
ftp.login(export['ftp_user_name'], export['ftp_password'])
|
309
|
+
ftp.chdir(File.dirname(remote_path))
|
310
|
+
@bytes_total = ftp.size(File.basename(remote_path))
|
311
|
+
ftp.getbinaryfile(File.basename(remote_path), local_path) do |data|
|
312
|
+
@bytes_transferred += data.size
|
313
|
+
end
|
314
|
+
ftp.close
|
315
|
+
end
|
316
|
+
|
317
|
+
def download_data
|
318
|
+
vm = Skytap.invoke!(username, api_token, "vm show #{vm_id}")
|
319
|
+
template_id = export['template_url'] =~ /templates\/(\d+)/ && $1
|
320
|
+
template = Skytap.invoke!(username, api_token, "template show #{template_id}")
|
321
|
+
|
322
|
+
exportable_vm = ExportableVm.new(vm, template)
|
323
|
+
|
324
|
+
File.open(File.join(export_dir, 'vm.yaml'), 'w') do |f|
|
325
|
+
f << YAML.dump(exportable_vm.data)
|
326
|
+
end
|
327
|
+
end
|
328
|
+
|
329
|
+
def id
|
330
|
+
export['id']
|
331
|
+
end
|
332
|
+
|
333
|
+
def wait_until_ready
|
334
|
+
cutoff = MAX_WAIT.from_now
|
335
|
+
finished = nil
|
336
|
+
|
337
|
+
while Time.now < cutoff
|
338
|
+
case export(true)['status']
|
339
|
+
when 'processing'
|
340
|
+
when 'complete'
|
341
|
+
finished = true
|
342
|
+
break
|
343
|
+
else
|
344
|
+
raise Skytap::Error.new "Export job had unexpected state of #{export['status'].inspect}"
|
345
|
+
end
|
346
|
+
|
347
|
+
sleep EXPORT_CHECK_PERIOD
|
348
|
+
end
|
349
|
+
|
350
|
+
unless finished
|
351
|
+
raise Skytap::Error.new 'Timed out waiting for export job to complete'
|
352
|
+
end
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
#TODO:NLA Probably should pull this into a method that also sets e.g., Download.parent = Vm.
|
357
|
+
Vm.subcommands << Download
|
358
|
+
|
359
|
+
class ExportableVm
|
360
|
+
DEFAULT_IP = '10.0.0.1'
|
361
|
+
DEFAULT_HOSTNAME = 'host-1'
|
362
|
+
DEFAULT_SUBNET = '10.0.0.0/24'
|
363
|
+
DEFAULT_DOMAIN = 'test.net'
|
364
|
+
|
365
|
+
attr_reader :vm, :template
|
366
|
+
attr_reader :name, :description, :credentials, :ip, :hostname, :subnet, :domain
|
367
|
+
|
368
|
+
def initialize(vm, template)
|
369
|
+
@vm = vm
|
370
|
+
@template = template
|
371
|
+
|
372
|
+
@name = vm['name']
|
373
|
+
@description = template['description'].present? ? template['description'] : @name
|
374
|
+
|
375
|
+
if iface = vm['interfaces'][0]
|
376
|
+
case iface['network_type']
|
377
|
+
when 'automatic'
|
378
|
+
@ip = iface['ip']
|
379
|
+
@hostname = iface['hostname']
|
380
|
+
|
381
|
+
network = template['networks'].detect {|net| net['id'] == iface['network_id']}
|
382
|
+
raise Skytap::Error.new('Network for VM interface not found') unless network
|
383
|
+
@subnet = network['subnet']
|
384
|
+
@domain = network['domain']
|
385
|
+
when 'manual'
|
386
|
+
@manual_network = true
|
387
|
+
|
388
|
+
network = template['networks'].detect {|net| net['id'] == iface['network_id']}
|
389
|
+
raise Skytap::Error.new('Network for VM interface not found') unless network
|
390
|
+
|
391
|
+
@subnet = network['subnet']
|
392
|
+
@domain = DEFAULT_DOMAIN
|
393
|
+
|
394
|
+
@ip = Subnet.new(@subnet).min_machine_ip.to_s
|
395
|
+
@hostname = DEFAULT_HOSTNAME
|
396
|
+
else # not connected
|
397
|
+
@ip = DEFAULT_IP
|
398
|
+
@hostname = iface['hostname'] || DEFAULT_HOSTNAME
|
399
|
+
@subnet = DEFAULT_SUBNET
|
400
|
+
@domain = DEFAULT_DOMAIN
|
401
|
+
end
|
402
|
+
else
|
403
|
+
# Choose default everything for VM hostname, address and network subnet and domain
|
404
|
+
@ip = DEFAULT_IP
|
405
|
+
@hostname = DEFAULT_HOSTNAME
|
406
|
+
@domain = DEFAULT_DOMAIN
|
407
|
+
@subnet = DEFAULT_SUBNET
|
408
|
+
end
|
409
|
+
end
|
410
|
+
|
411
|
+
def data
|
412
|
+
@data ||= {
|
413
|
+
'template_name' => @name,
|
414
|
+
'template_description' => @description,
|
415
|
+
'network_domain' => @domain,
|
416
|
+
'network_subnet' => @subnet,
|
417
|
+
'interface_ip' => @ip,
|
418
|
+
'interface_hostname' => @hostname,
|
419
|
+
}.tap do |d|
|
420
|
+
if creds = vm['credentials'].try(:collect){|c| c['text']}
|
421
|
+
d['credentials'] = creds
|
422
|
+
end
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
def manual_network?
|
427
|
+
@manual_network
|
428
|
+
end
|
429
|
+
end
|
430
|
+
end
|
431
|
+
end
|