skytap-yf 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,401 @@
1
+ module Skytap
2
+ module Commands
3
+ class Upload < Skytap::Commands::Base
4
+ MAX_CONCURRENCY = 5
5
+ CHECK_PERIOD = 15
6
+ AT_CAPACITY_RETRY_PERIOD = 15.minutes.to_i
7
+
8
+ # What was passed in, either a reference to a VM directory or a VM archive file.
9
+ attr_reader :vms
10
+ attr_reader :uploaders
11
+
12
+ self.parent = Vm
13
+ self.plugin = true
14
+ self.ask_interactively = true
15
+
16
+ def self.description
17
+ <<-"EOF"
18
+ Upload one or more VM directories or VM archive files to Skytap
19
+
20
+ The file must be one of the following archive types:
21
+ .tgz .tar.gz .tbz .tbz2 .tar.bz2 .tar .ova .ovf .7z .7z.001 .zip
22
+
23
+ The contents of the archive file must contain a VMware image in either
24
+ VMX/VMDK or OVF/VMDK format.
25
+
26
+ See the following page for more information:
27
+ https://cloud.skytap.com/docs/index.php/Importing_and_Exporting_Virtual_Machines#How_to_Import_VMs_in_to_Skytap
28
+ EOF
29
+ end
30
+
31
+ def expected_args
32
+ ActiveSupport::OrderedHash[
33
+ 'file*', 'One or more directories or VM archive files to upload'
34
+ ]
35
+ end
36
+
37
+ def parameters
38
+ # Only display import parameters if a single VM will be uploaded.
39
+ if args.length == 1
40
+ Import.subcommands.detect{|s| s.command_name == 'create'}.spec['params']
41
+ else
42
+ {}
43
+ end
44
+ end
45
+
46
+ def run!
47
+ @vms = args
48
+ @uploaders = []
49
+ params = composed_params
50
+
51
+ until finished?
52
+ if vms.present? && slots_available?
53
+ kick_off_import(vms.shift, params.dup)
54
+ else
55
+ sleep CHECK_PERIOD
56
+ print_status
57
+ invoker.try(:call, self)
58
+
59
+ signal_available if signal_stale? || (concurrency_at_overage && concurrency < concurrency_at_overage)
60
+ end
61
+ end
62
+
63
+ signal_available
64
+ print_summary
65
+ return response
66
+ end
67
+
68
+ def concurrency_at_overage
69
+ @concurrency_at_overage
70
+ end
71
+
72
+ def kick_off_import(vm, params)
73
+ begin
74
+ # Create import job on server in main thread.
75
+ job = VmImportJob.new(logger, username, api_token, vm, params)
76
+
77
+ # If successful, start FTP upload and subsequent steps in new thread.
78
+ up = Uploader.new(job)
79
+ uploaders << up
80
+ log_line(up.status_line)
81
+ rescue NoSlotsAvailable => ex
82
+ vms << vm
83
+ signal_full
84
+ log_line(("#{vm}: " << no_capacity_message).color(:yellow))
85
+ rescue Exception => ex
86
+ up = DeadUploader.new(vm, ex)
87
+ uploaders << up
88
+ log_line(up.status_line)
89
+ end
90
+ end
91
+
92
+ def no_capacity_message
93
+ m = AT_CAPACITY_RETRY_PERIOD / 60
94
+ "No import capacity is currently available on Skytap. Will retry in #{m} minutes".tap do |msg|
95
+ if active_uploaders.present?
96
+ msg << ' or when another import completes.'
97
+ else
98
+ msg << '.'
99
+ end
100
+ end
101
+ end
102
+
103
+ def log_line(msg, include_separator=true)
104
+ line = msg
105
+ line += "\n---" if include_separator
106
+ logger.info line
107
+ end
108
+
109
+ def print_status
110
+ if (stat = status_lines).present?
111
+ log_line(stat)
112
+ end
113
+ end
114
+
115
+ def print_summary
116
+ unless response.error?
117
+ logger.info "#{'Summary'.bright}\n#{response.payload}"
118
+ end
119
+ end
120
+
121
+ def finished?
122
+ vms.empty? && concurrency == 0
123
+ end
124
+
125
+ def slots_available?
126
+ estimated_available_slots > 0
127
+ end
128
+
129
+ def signal_stale?
130
+ full? && Time.now - @full_at > AT_CAPACITY_RETRY_PERIOD
131
+ end
132
+
133
+ def seconds_until_retry
134
+ return unless full?
135
+ [0, AT_CAPACITY_RETRY_PERIOD - (Time.now - @full_at)].max
136
+ end
137
+
138
+ def estimated_available_slots
139
+ if full?
140
+ 0
141
+ else
142
+ MAX_CONCURRENCY - concurrency
143
+ end
144
+ end
145
+
146
+ def signal_full
147
+ @concurrency_at_overage = concurrency
148
+ @full_at = Time.now
149
+ end
150
+
151
+ def signal_available
152
+ @concurrency_at_overage = @full_at = nil
153
+ end
154
+
155
+ def full?
156
+ @full_at
157
+ end
158
+
159
+ def status_lines
160
+ active_uploaders.collect(&:status_line).join("\n")
161
+ end
162
+
163
+ def active_uploaders
164
+ uploaders.reject(&:finished?)
165
+ end
166
+
167
+
168
+ private
169
+
170
+ def concurrency
171
+ uploaders.select(&:alive?).length
172
+ end
173
+
174
+ def response
175
+ @_response ||= begin
176
+ error = !uploaders.any?(&:success?)
177
+ Response.build(uploaders.collect(&:status_line).join("\n"), error)
178
+ end
179
+ end
180
+ end
181
+
182
+ class NoSlotsAvailable < RuntimeError
183
+ end
184
+
185
+ class VmImportJob
186
+ attr_reader :logger, :import_path, :params, :vm, :username, :api_token, :other_credentials
187
+
188
+ def initialize(logger, username, api_token, vm, params = {})
189
+ @logger = logger
190
+ @username = username
191
+ @api_token = api_token
192
+ @vm = File.expand_path(vm)
193
+ @params = params
194
+ @vm_filename = File.basename(@vm)
195
+
196
+ create_on_server
197
+ end
198
+
199
+ def create_on_server
200
+ setup
201
+ begin
202
+ import
203
+ rescue Exception => ex
204
+ if at_capacity?(ex)
205
+ raise NoSlotsAvailable.new
206
+ else
207
+ raise
208
+ end
209
+ end
210
+ end
211
+
212
+ def at_capacity?(exception)
213
+ exception.message.include?('You cannot import a VM because you may not have more than')
214
+ end
215
+
216
+ def setup
217
+ if File.directory?(vm)
218
+ @import_path = File.join(vm, 'vm.7z')
219
+ unless File.exist?(@import_path)
220
+ raise Skytap::Error.new("Directory provided (#{vm}) but no vm.7z file found inside")
221
+ end
222
+
223
+ metadata_file = File.join(vm, 'vm.yaml')
224
+ if File.exist?(metadata_file)
225
+ #TODO:NLA If hostname is present, truncate it to 15 chars. This is the max import hostname length.
226
+ extra_params = YAML.load_file(metadata_file)
227
+ end
228
+
229
+ elsif File.exist?(vm)
230
+ @import_path = vm
231
+ else
232
+ raise Skytap::Error.new("File does not exist: #{vm}")
233
+ end
234
+
235
+ @params = (extra_params || {}).merge(params)
236
+ if @params['credentials'].is_a?(Array)
237
+ cred = @params['credentials'].shift
238
+ @other_credentials = @params['credentials']
239
+ @params['credentials'] = cred
240
+ end
241
+ end
242
+
243
+ def import(force_reload=false)
244
+ return @import unless @import.nil? || force_reload
245
+
246
+ if @import
247
+ id = @import['id']
248
+ @import = Skytap.invoke!(username, api_token, "import show #{id}")
249
+ else
250
+ @import = Skytap.invoke!(username, api_token, 'import create', {}, :param => params)
251
+ end
252
+ end
253
+ end
254
+
255
+ class DeadUploader
256
+ def initialize(vm, exception)
257
+ @vm = vm
258
+ @exception = exception
259
+ end
260
+
261
+ def finished?
262
+ true
263
+ end
264
+
265
+ def alive?
266
+ false
267
+ end
268
+
269
+ def success?
270
+ false
271
+ end
272
+
273
+ def status_line
274
+ "VM #{@vm}: Error: #{@exception}"
275
+ end
276
+ end
277
+
278
+ class Uploader < Thread
279
+ MAX_WAIT = 2.days
280
+ IMPORT_CHECK_PERIOD = 5
281
+
282
+ attr_reader :job, :bytes_transferred, :bytes_total, :result
283
+ delegate :logger, :import, :import_path, :params, :vm, :username, :api_token, :other_credentials, :to => :job
284
+
285
+ def initialize(job)
286
+ @job = job
287
+ @bytes_transferred = @bytes_total = 0
288
+ path, basename = File.split(File.expand_path(import_path))
289
+ @vm_filename = File.join(File.basename(path), basename) # E.g., "myfolder/vm.7z"
290
+ @vm_filename << " (#{params['template_name']})" if params['template_name']
291
+
292
+ super do
293
+ begin
294
+ run
295
+ rescue Exception => ex
296
+ @result = Response.build(ex)
297
+ end
298
+ end
299
+ end
300
+
301
+ def run
302
+ ftp_upload
303
+ Skytap.invoke!(username, api_token, "import update #{id}", {}, :param => {'status' => 'processing'})
304
+ wait_until_ready
305
+ add_other_credentials
306
+ Skytap.invoke!(username, api_token, "import destroy #{id}")
307
+ @result = Response.build(import['template_url'])
308
+ end
309
+
310
+
311
+ def finished?
312
+ !!@finished
313
+ end
314
+
315
+ def success?
316
+ result && !result.error?
317
+ end
318
+
319
+ def status_line
320
+ "#{@vm_filename}: #{status}"
321
+ end
322
+
323
+ def status
324
+ if result.try(:error?)
325
+ @finished = true
326
+ "Error: #{result.error_message}".color(:red).bright
327
+ elsif result
328
+ @finished = true
329
+ "Uploaded to: #{result.payload}".color(:green).bright
330
+ elsif bytes_transferred == 0
331
+ 'Starting'.color(:yellow)
332
+ elsif bytes_transferred >= bytes_total
333
+ 'Importing'.color(:yellow)
334
+ else
335
+ gb_transferred = bytes_transferred / 1.gigabyte.to_f
336
+ gb_total = bytes_total / 1.gigabyte.to_f
337
+ percent_done = 100.0 * bytes_transferred / bytes_total
338
+ "Uploading #{'%0.1f' % percent_done}% (#{'%0.1f' % gb_transferred} / #{'%0.1f' % gb_total} GB)".color(:yellow)
339
+ end
340
+ end
341
+
342
+
343
+ private
344
+
345
+
346
+ def ftp_upload
347
+ local_path = import_path
348
+ # FTP URL looks like: ftp://67x8meqCr:a3hnalxZLg@ftp.cloud.skytap.com/upload/
349
+ remote_dir = import['ftp_url'] =~ %r{@.*?(/.+)$} && $1
350
+
351
+ ftp = Net::FTP.new(import['ftp_host'])
352
+ ftp.login(import['ftp_user_name'], import['ftp_password'])
353
+ ftp.chdir(remote_dir)
354
+ @bytes_total = File.size(local_path)
355
+ ftp.putbinaryfile(local_path) do |data|
356
+ @bytes_transferred += data.size
357
+ end
358
+ ftp.close
359
+ end
360
+
361
+ def add_other_credentials
362
+ template_id = import['template_url'] =~ /\/templates\/(\d+)/ && $1
363
+ template = Skytap.invoke!(username, api_token, "template show #{template_id}")
364
+ vm_id = template['vms'].first['id']
365
+ (other_credentials || []).each do |cred|
366
+ Skytap.invoke!(username, api_token, "credential create /vms/#{vm_id}", {}, :param => {'vm_id' => vm_id, 'text' => cred})
367
+ end
368
+ end
369
+
370
+ def id
371
+ import['id']
372
+ end
373
+
374
+ def wait_until_ready
375
+ cutoff = MAX_WAIT.from_now
376
+ finished = nil
377
+
378
+ while Time.now < cutoff
379
+ case import(true)['status']
380
+ when 'processing'
381
+ when 'complete'
382
+ finished = true
383
+ break
384
+ else
385
+ #TODO:NLA Check that this actually is displayed to the user, for both normal `vm upload` and for `vm copytoregion` too.
386
+ raise Skytap::Error.new "Import job had unexpected state of #{import['status'].inspect}"
387
+ end
388
+
389
+ sleep IMPORT_CHECK_PERIOD
390
+ end
391
+
392
+ unless finished
393
+ raise Skytap::Error.new 'Timed out waiting for import job to complete'
394
+ end
395
+ end
396
+ end
397
+
398
+ #TODO:NLA Probably should pull this into a method that also sets e.g., Upload.parent = Vm.
399
+ Vm.subcommands << Upload
400
+ end
401
+ end
@@ -0,0 +1,134 @@
1
+ module Skytap
2
+ class Requester
3
+ #TODO:NLA Move more of the implementation in request() below into this class.
4
+ class Response
5
+ attr_reader :logger, :http_response
6
+ def initialize(logger, http_response, options = {})
7
+ @logger = logger
8
+ @http_response = http_response
9
+
10
+ logger.debug 'Response:'.color(:cyan).bright
11
+ each_header do |k, v|
12
+ logger.debug "#{k}: #{v}".color(:cyan)
13
+ end
14
+ logger.debug "\n#{body}".color(:cyan)
15
+ logger.debug '---'.color(:cyan).bright
16
+
17
+ # Raise unless response is 2XX.
18
+ @http_response.value if options[:raise]
19
+ end
20
+
21
+ def method_missing(*args, &block)
22
+ http_response.send(*args, &block)
23
+ end
24
+
25
+ def pretty_body
26
+ return body if body.blank?
27
+ if content_type.include?('json')
28
+ begin
29
+ JSON.pretty_generate(JSON.load(body))
30
+ rescue
31
+ body
32
+ end
33
+ else
34
+ body
35
+ end
36
+ end
37
+ end
38
+
39
+ attr_reader :logger, :format
40
+
41
+ def initialize(logger, username, api_token, base_url, http_format, verify_certs=true)
42
+ @logger = logger
43
+ @username = username or raise Skytap::Error.new 'No username provided'
44
+ @api_token = api_token or raise Skytap::Error.new 'No API token provided'
45
+ @base_uri = URI.parse(base_url) or raise Skytap::Error.new 'No base URL provided'
46
+ @format = "application/#{http_format}" or raise Skytap::Error.new 'No HTTP format provided'
47
+ @verify_certs = verify_certs
48
+ end
49
+
50
+ # Raises on error code unless :raise => false is passed.
51
+ def request(method, url, options = {})
52
+ options = options.symbolize_keys
53
+ options[:raise] = true unless options.has_key?(:raise)
54
+
55
+ with_session do |http|
56
+ begin
57
+ #TODO:NLA Move this into method
58
+ if p = options[:params]
59
+ if p.is_a?(Hash)
60
+ p = p.collect {|k,v| CGI.escape(k) + '=' + CGI.escape(v)}.join('&')
61
+ end
62
+
63
+ path, params = url.split('?', 2)
64
+ if params
65
+ params << '&'
66
+ else
67
+ params = ''
68
+ end
69
+ params << p
70
+ url = [path, params].join('?')
71
+ end
72
+
73
+ body = options[:body] || ''
74
+ headers = base_headers.merge(options[:headers] || {})
75
+ logger.debug 'Request:'.color(:cyan).bright
76
+ logger.debug "#{method} #{url}\n#{headers.collect {|k,v| "#{k}:#{v}"}.join("\n")}\n\n#{body}".color(:cyan)
77
+ logger.debug '---'.color(:cyan).bright
78
+
79
+ response = Response.new(logger, http.send_request(method, url, body, headers),
80
+ :raise => options[:raise])
81
+ rescue OpenSSL::SSL::SSLError => ex
82
+ $stderr.puts <<-"EOF"
83
+ An SSL error occurred (probably certificate verification failed).
84
+
85
+ If you are pointed against an internal test environment, set
86
+ "verify-certs: false" in your ~/.skytaprc file or use --no-verify-certs.
87
+
88
+ If not, then you may be subject to a man-in-the-middle attack, or the web
89
+ site's SSL certificate may have expired.
90
+
91
+ The error was: #{ex}
92
+ EOF
93
+ exit(-1)
94
+ rescue Net::HTTPServerException => ex
95
+ logger.debug 'Response:'.color(:cyan).bright
96
+ logger.info "Code: #{ex.response.code} #{ex.response.message}".color(:red).bright
97
+ logger.info ex.response.body
98
+ raise
99
+ end
100
+ end
101
+ end
102
+
103
+ def base_headers
104
+ headers = {
105
+ 'Authorization' => auth_header,
106
+ 'Content-Type' => format,
107
+ 'Accept' => format,
108
+ 'User-Agent' => "SkytapCLI/#{Skytap::VERSION}",
109
+ }
110
+ end
111
+
112
+ def auth_header
113
+ "Basic #{Base64.encode64(@username + ":" + @api_token)}".gsub("\n", '')
114
+ end
115
+
116
+ def with_session
117
+ http = Net::HTTP.new(@base_uri.host, @base_uri.port)
118
+ http.use_ssl = @base_uri.port == 443 || @base_uri.scheme == 'https'
119
+
120
+ if http.use_ssl?
121
+ # Allow cert-checking to be disabled, since internal test environments
122
+ # have bad certs.
123
+ if @verify_certs
124
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
125
+ http.ca_file = File.join(File.dirname(__FILE__), '..', '..', 'ca-bundle.crt')
126
+ else
127
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
128
+ end
129
+ end
130
+
131
+ yield http
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,61 @@
1
+ module Skytap
2
+ class Response
3
+ def self.build(result, is_error=nil)
4
+ case result
5
+ when self
6
+ result
7
+ when Requester::Response
8
+ is_error ||= result.code !~ /^[123]/
9
+ if is_error && result.payload.is_a?(Hash)
10
+ err_msg = "Server error (code #{result.code}): " +
11
+ (result.payload['error'] ||
12
+ result.payload['errors'].try(:join, ' ') ||
13
+ result.body)
14
+ end
15
+ new(result.payload, is_error, err_msg)
16
+ when Skytap::Error
17
+ new(result, true, result.to_s)
18
+ when Exception
19
+ log_exception(result)
20
+ new(result, true, "Internal error: #{result}")
21
+ else
22
+ new(result, is_error)
23
+ end
24
+ end
25
+
26
+ attr_reader :payload
27
+
28
+ def initialize(payload, error = nil, error_message = nil)
29
+ @payload = payload
30
+ @error = error
31
+ if @error
32
+ @error_message = error_message || @payload.to_s
33
+ end
34
+ end
35
+
36
+ def error?
37
+ @error
38
+ end
39
+
40
+ def error_message
41
+ @error_message.color(:red).bright
42
+ end
43
+
44
+
45
+ private
46
+
47
+
48
+ def self.log_exception(ex)
49
+ begin
50
+ message = "#{Time.now}: -- #{Skytap::VERSION} -- " +
51
+ "#{ex} (#{ex.class.name})\n#{ex.backtrace.join("\n")}\n"
52
+
53
+ File.open(File.expand_path(File.join(ENV['HOME'], '.skytap.log')), 'a') do |f|
54
+ f << message
55
+ end
56
+ rescue
57
+ # No-op
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,28 @@
1
+ module Skytap
2
+ module SkytapRC
3
+ extend self
4
+
5
+ RC_FILE = File.expand_path(File.join(ENV['HOME'], '.skytaprc'))
6
+ RECOGNIZED_OPTIONS = [:'verify-certs', :'base-url', :'api-token', :'http-format', :ask, :'log-level', :username, :colorize]
7
+
8
+ def exists?
9
+ File.exist?(RC_FILE)
10
+ end
11
+
12
+ def load
13
+ @rc_contents ||= (if File.exist?(RC_FILE)
14
+ YAML.load_file(RC_FILE).symbolize_keys
15
+ else
16
+ {}
17
+ end).subset(RECOGNIZED_OPTIONS)
18
+ end
19
+
20
+ def write(hash)
21
+ @rc_contents = nil
22
+ hash = (hash || {}).subset(RECOGNIZED_OPTIONS)
23
+ File.open(RC_FILE, 'w') do |f|
24
+ f << YAML.dump(hash)
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,92 @@
1
+ require 'skytap/ip_address'
2
+
3
+ class Subnet
4
+ class InvalidSubnet < RuntimeError
5
+ end
6
+
7
+ attr_reader :size, :mask, :address
8
+
9
+ def initialize(cidr_block)
10
+ unless cidr_block =~ /^(.*)\/(.*)/
11
+ raise InvalidSubnet.new 'Not in CIDR block form (XX.XX.XX.XX/YY)'
12
+ end
13
+
14
+ network = $1
15
+ begin
16
+ @size = Integer($2)
17
+ rescue
18
+ raise InvalidSubnet.new 'Invalid size'
19
+ end
20
+
21
+ @address = IpAddress.new(network)
22
+ @mask = size_to_mask(@size)
23
+ end
24
+
25
+ def network_portion
26
+ @network_portion ||= (mask & address)
27
+ end
28
+ alias_method :min, :network_portion
29
+
30
+ def ==(other)
31
+ other.is_a?(Subnet) && \
32
+ (size == other.size) && \
33
+ network_portion == other.network_portion
34
+ end
35
+
36
+ def overlaps?(other)
37
+ min <= other.max && max >= other.min
38
+ end
39
+
40
+ def subsumes?(other)
41
+ min <= other.min && max >= other.max
42
+ end
43
+
44
+ def strictly_subsumes?(other)
45
+ subsumes?(other) && self != other
46
+ end
47
+
48
+ def max
49
+ @max ||= (mask.inverse | network_portion)
50
+ end
51
+
52
+ def each_address
53
+ (min.to_i..max.to_i).each{|i| yield IpAddress.new(i)}
54
+ end
55
+
56
+ def contains?(ip)
57
+ ip = IpAddress.new(ip) unless ip.is_a?(IpAddress)
58
+ (ip & mask) == network_portion
59
+ end
60
+
61
+ def num_addresses
62
+ 2 ** (32-size)
63
+ end
64
+
65
+ def min_machine_ip
66
+ min + 1
67
+ end
68
+
69
+ def normalized?
70
+ address == network_portion
71
+ end
72
+
73
+ def normalize
74
+ Subnet.new("#{network_portion}/#{size}")
75
+ end
76
+
77
+ def to_s
78
+ "#{address}/#{size}"
79
+ end
80
+
81
+ def <=>(other)
82
+ to_s <=> other.to_s
83
+ end
84
+
85
+ private
86
+
87
+ def size_to_mask(numbits)
88
+ raise InvalidSubnet.new("Subnet size in bits must be between 0 and 32") unless numbits >= 0 and numbits <= 32
89
+ IpAddress.new((2**numbits - 1) << (32 - numbits))
90
+ end
91
+ end
92
+