skytap-yf 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,431 @@
1
+ module Skytap
2
+ module Commands
3
+ class Download < Skytap::Commands::Base
4
+ MAX_CONCURRENCY = 5
5
+ CHECK_PERIOD = 15
6
+ AT_CAPACITY_RETRY_PERIOD = 15.minutes.to_i
7
+
8
+ attr_reader :vm_ids
9
+ attr_reader :downloaders
10
+
11
+ self.parent = Vm
12
+ self.plugin = true
13
+
14
+ def self.description
15
+ <<-"EOF"
16
+ Download the specified Skytap template VM to the local filesystem
17
+
18
+ The VM must belong to a template, not a configuration. The template cannot be
19
+ public.
20
+ EOF
21
+ end
22
+
23
+ def expected_args
24
+ ActiveSupport::OrderedHash[
25
+ 'vm_id*', 'One or more IDs of template VMs to donwload'
26
+ ]
27
+ end
28
+
29
+ def expected_options
30
+ ActiveSupport::OrderedHash[
31
+ :dir, {:flag_arg => 'DIR', :desc => 'Directory into which to download VM and metadata'},
32
+ ]
33
+ end
34
+
35
+ attr_reader :downloaders
36
+
37
+ def run!
38
+ @vm_ids = args.collect {|a| find_id(a)}
39
+ @downloaders = []
40
+
41
+ until finished?
42
+ if vm_ids.present? && slots_available?
43
+ kick_off_export(vm_ids.shift)
44
+ else
45
+ sleep CHECK_PERIOD
46
+ print_status
47
+ invoker.try(:call, self)
48
+
49
+ signal_available if signal_stale? || (concurrency_at_overage && concurrency < concurrency_at_overage)
50
+ end
51
+ end
52
+
53
+ signal_available
54
+ print_summary
55
+ return response
56
+ end
57
+
58
+ def concurrency_at_overage
59
+ @concurrency_at_overage
60
+ end
61
+
62
+ def kick_off_export(vm_id)
63
+ begin
64
+ # Create export job on server in main thread.
65
+ job = VmExportJob.new(logger, username, api_token, vm_id, command_options[:dir])
66
+
67
+ # If successful, start FTP download and subsequent steps in new thread.
68
+ dl = Downloader.new(job)
69
+ downloaders << dl
70
+ log_line(dl.status_line)
71
+ rescue NoSlotsAvailable => ex
72
+ vm_ids << vm_id
73
+ signal_full
74
+ log_line(("VM #{vm_id}: " << no_capacity_message).color(:yellow))
75
+ rescue Exception => ex
76
+ dl = DeadDownloader.new(vm_id, ex)
77
+ downloaders << dl
78
+ log_line(dl.status_line)
79
+ end
80
+ end
81
+
82
+ def no_capacity_message
83
+ m = AT_CAPACITY_RETRY_PERIOD / 60
84
+ "No export capacity is currently available on Skytap. Will retry in #{m} minutes".tap do |msg|
85
+ if active_downloaders.present?
86
+ msg << ' or when another export completes.'
87
+ else
88
+ msg << '.'
89
+ end
90
+ end
91
+ end
92
+
93
+ def log_line(msg, include_separator=true)
94
+ line = msg
95
+ line += "\n---" if include_separator
96
+ logger.info line
97
+ end
98
+
99
+ def print_status
100
+ logger.info "#{status_lines}\n---"
101
+ end
102
+
103
+ def print_summary
104
+ unless response.error?
105
+ logger.info "#{'Summary'.bright}\n#{response.payload}"
106
+ end
107
+ end
108
+
109
+ def finished?
110
+ vm_ids.empty? && concurrency == 0
111
+ end
112
+
113
+ def slots_available?
114
+ estimated_available_slots > 0
115
+ end
116
+
117
+ def signal_stale?
118
+ full? && Time.now - @full_at > AT_CAPACITY_RETRY_PERIOD
119
+ end
120
+
121
+ def seconds_until_retry
122
+ return unless full?
123
+ [0, AT_CAPACITY_RETRY_PERIOD - (Time.now - @full_at)].max
124
+ end
125
+
126
+ def estimated_available_slots
127
+ if full?
128
+ 0
129
+ else
130
+ MAX_CONCURRENCY - concurrency
131
+ end
132
+ end
133
+
134
+ def signal_full
135
+ @concurrency_at_overage = concurrency
136
+ @full_at = Time.now
137
+ end
138
+
139
+ def signal_available
140
+ @concurrency_at_overage = @full_at = nil
141
+ end
142
+
143
+ def full?
144
+ @full_at
145
+ end
146
+
147
+ def status_lines
148
+ active_downloaders.collect(&:status_line).join("\n")
149
+ end
150
+
151
+ def active_downloaders
152
+ downloaders.reject(&:finished?)
153
+ end
154
+
155
+
156
+ private
157
+
158
+ def concurrency
159
+ downloaders.select(&:alive?).length
160
+ end
161
+
162
+ def response
163
+ @_response ||= begin
164
+ error = !downloaders.any?(&:success?)
165
+ Response.build(downloaders.collect(&:status_line).join("\n"), error)
166
+ end
167
+ end
168
+ end
169
+
170
+ class NoSlotsAvailable < RuntimeError
171
+ end
172
+
173
+ class VmExportJob
174
+ attr_reader :logger, :vm, :vm_id, :export_dir, :username, :api_token
175
+
176
+ def initialize(logger, username, api_token, vm_id, dir=nil)
177
+ @logger = logger
178
+ @username = username
179
+ @api_token = api_token
180
+ @vm_id = vm_id
181
+
182
+ @vm = Skytap.invoke!(username, api_token, "vm show #{vm_id}")
183
+ @export_dir = File.join(File.expand_path(dir || '.'), "vm_#{vm_id}")
184
+ FileUtils.mkdir_p(export_dir)
185
+
186
+ create_on_server
187
+ end
188
+
189
+ def create_on_server
190
+ begin
191
+ export
192
+ rescue Exception => ex
193
+ if at_capacity?(ex)
194
+ raise NoSlotsAvailable.new
195
+ else
196
+ raise
197
+ end
198
+ end
199
+ end
200
+
201
+ def at_capacity?(exception)
202
+ exception.message.include?('You cannot export a VM because you may not have more than')
203
+ end
204
+
205
+ def export(force_reload=false)
206
+ return @export unless @export.nil? || force_reload
207
+
208
+ if @export
209
+ id = @export['id']
210
+ @export = Skytap.invoke!(username, api_token, "export show #{id}")
211
+ else
212
+ @export = Skytap.invoke!(username, api_token, "export create", {}, :param => {'vm_id' => vm_id})
213
+ end
214
+ end
215
+ end
216
+
217
+ class DeadDownloader
218
+ def initialize(vm_id, exception)
219
+ @vm_id = vm_id
220
+ @exception = exception
221
+ end
222
+
223
+ def finished?
224
+ true
225
+ end
226
+
227
+ def alive?
228
+ false
229
+ end
230
+
231
+ def success?
232
+ false
233
+ end
234
+
235
+ def status_line
236
+ "VM #{@vm_id}: Error: #{@exception}"
237
+ end
238
+ end
239
+
240
+ class Downloader < Thread
241
+ MAX_WAIT = 2.days
242
+ EXPORT_CHECK_PERIOD = 5
243
+
244
+ attr_reader :job, :bytes_transferred, :bytes_total, :result
245
+ delegate :logger, :vm, :vm_id, :export, :export_dir, :username, :api_token, :to => :job
246
+
247
+ def initialize(job)
248
+ @job = job
249
+ @bytes_transferred = @bytes_total = 0
250
+
251
+ super do
252
+ begin
253
+ run
254
+ rescue Exception => ex
255
+ @result = Response.build(ex)
256
+ end
257
+ end
258
+ end
259
+
260
+ def run
261
+ wait_until_ready
262
+ ftp_download
263
+ download_data
264
+ Skytap.invoke!(username, api_token, "export destroy #{id}")
265
+ @result = Response.build(export_dir)
266
+ end
267
+
268
+ def finished?
269
+ !!@finished
270
+ end
271
+
272
+ def success?
273
+ result && !result.error?
274
+ end
275
+
276
+ def status_line
277
+ prefix = "VM #{vm_id}".tap do |str|
278
+ if vm
279
+ str << " (#{vm['name']})"
280
+ end
281
+ end
282
+ prefix << ': ' << status
283
+ end
284
+
285
+ def status
286
+ if result.try(:error?)
287
+ @finished = true
288
+ "Error: #{result.error_message}".color(:red).bright
289
+ elsif result
290
+ @finished = true
291
+ "Downloaded: #{result.payload}".color(:green).bright
292
+ elsif bytes_transferred == 0
293
+ 'Exporting'.color(:yellow)
294
+ else
295
+ gb_transferred = bytes_transferred / 1.gigabyte.to_f
296
+ gb_total = bytes_total / 1.gigabyte.to_f
297
+ percent_done = 100.0 * bytes_transferred / bytes_total
298
+ "Downloading #{'%0.1f' % percent_done}% (#{'%0.1f' % gb_transferred} / #{'%0.1f' % gb_total} GB)".color(:yellow)
299
+ end
300
+ end
301
+
302
+ def ftp_download
303
+ remote_path = export['filename']
304
+ local_path = File.join(export_dir, File.basename(export['filename']))
305
+ FileUtils.mkdir_p(export_dir)
306
+
307
+ ftp = Net::FTP.new(export['ftp_host'])
308
+ ftp.login(export['ftp_user_name'], export['ftp_password'])
309
+ ftp.chdir(File.dirname(remote_path))
310
+ @bytes_total = ftp.size(File.basename(remote_path))
311
+ ftp.getbinaryfile(File.basename(remote_path), local_path) do |data|
312
+ @bytes_transferred += data.size
313
+ end
314
+ ftp.close
315
+ end
316
+
317
+ def download_data
318
+ vm = Skytap.invoke!(username, api_token, "vm show #{vm_id}")
319
+ template_id = export['template_url'] =~ /templates\/(\d+)/ && $1
320
+ template = Skytap.invoke!(username, api_token, "template show #{template_id}")
321
+
322
+ exportable_vm = ExportableVm.new(vm, template)
323
+
324
+ File.open(File.join(export_dir, 'vm.yaml'), 'w') do |f|
325
+ f << YAML.dump(exportable_vm.data)
326
+ end
327
+ end
328
+
329
+ def id
330
+ export['id']
331
+ end
332
+
333
+ def wait_until_ready
334
+ cutoff = MAX_WAIT.from_now
335
+ finished = nil
336
+
337
+ while Time.now < cutoff
338
+ case export(true)['status']
339
+ when 'processing'
340
+ when 'complete'
341
+ finished = true
342
+ break
343
+ else
344
+ raise Skytap::Error.new "Export job had unexpected state of #{export['status'].inspect}"
345
+ end
346
+
347
+ sleep EXPORT_CHECK_PERIOD
348
+ end
349
+
350
+ unless finished
351
+ raise Skytap::Error.new 'Timed out waiting for export job to complete'
352
+ end
353
+ end
354
+ end
355
+
356
+ #TODO:NLA Probably should pull this into a method that also sets e.g., Download.parent = Vm.
357
+ Vm.subcommands << Download
358
+
359
+ class ExportableVm
360
+ DEFAULT_IP = '10.0.0.1'
361
+ DEFAULT_HOSTNAME = 'host-1'
362
+ DEFAULT_SUBNET = '10.0.0.0/24'
363
+ DEFAULT_DOMAIN = 'test.net'
364
+
365
+ attr_reader :vm, :template
366
+ attr_reader :name, :description, :credentials, :ip, :hostname, :subnet, :domain
367
+
368
+ def initialize(vm, template)
369
+ @vm = vm
370
+ @template = template
371
+
372
+ @name = vm['name']
373
+ @description = template['description'].present? ? template['description'] : @name
374
+
375
+ if iface = vm['interfaces'][0]
376
+ case iface['network_type']
377
+ when 'automatic'
378
+ @ip = iface['ip']
379
+ @hostname = iface['hostname']
380
+
381
+ network = template['networks'].detect {|net| net['id'] == iface['network_id']}
382
+ raise Skytap::Error.new('Network for VM interface not found') unless network
383
+ @subnet = network['subnet']
384
+ @domain = network['domain']
385
+ when 'manual'
386
+ @manual_network = true
387
+
388
+ network = template['networks'].detect {|net| net['id'] == iface['network_id']}
389
+ raise Skytap::Error.new('Network for VM interface not found') unless network
390
+
391
+ @subnet = network['subnet']
392
+ @domain = DEFAULT_DOMAIN
393
+
394
+ @ip = Subnet.new(@subnet).min_machine_ip.to_s
395
+ @hostname = DEFAULT_HOSTNAME
396
+ else # not connected
397
+ @ip = DEFAULT_IP
398
+ @hostname = iface['hostname'] || DEFAULT_HOSTNAME
399
+ @subnet = DEFAULT_SUBNET
400
+ @domain = DEFAULT_DOMAIN
401
+ end
402
+ else
403
+ # Choose default everything for VM hostname, address and network subnet and domain
404
+ @ip = DEFAULT_IP
405
+ @hostname = DEFAULT_HOSTNAME
406
+ @domain = DEFAULT_DOMAIN
407
+ @subnet = DEFAULT_SUBNET
408
+ end
409
+ end
410
+
411
+ def data
412
+ @data ||= {
413
+ 'template_name' => @name,
414
+ 'template_description' => @description,
415
+ 'network_domain' => @domain,
416
+ 'network_subnet' => @subnet,
417
+ 'interface_ip' => @ip,
418
+ 'interface_hostname' => @hostname,
419
+ }.tap do |d|
420
+ if creds = vm['credentials'].try(:collect){|c| c['text']}
421
+ d['credentials'] = creds
422
+ end
423
+ end
424
+ end
425
+
426
+ def manual_network?
427
+ @manual_network
428
+ end
429
+ end
430
+ end
431
+ end