ec2_amitools 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +54 -0
- data/bin/console +14 -0
- data/bin/ec2-ami-tools-version +6 -0
- data/bin/ec2-bundle-image +6 -0
- data/bin/ec2-bundle-vol +6 -0
- data/bin/ec2-delete-bundle +6 -0
- data/bin/ec2-download-bundle +6 -0
- data/bin/ec2-migrate-bundle +6 -0
- data/bin/ec2-migrate-manifest +6 -0
- data/bin/ec2-unbundle +6 -0
- data/bin/ec2-upload-bundle +6 -0
- data/bin/setup +8 -0
- data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
- data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
- data/etc/ec2/amitools/cert-ec2.pem +23 -0
- data/etc/ec2/amitools/mappings.csv +9 -0
- data/lib/ec2/amitools/bundle.rb +251 -0
- data/lib/ec2/amitools/bundle_base.rb +58 -0
- data/lib/ec2/amitools/bundleimage.rb +94 -0
- data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
- data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
- data/lib/ec2/amitools/bundleparameters.rb +120 -0
- data/lib/ec2/amitools/bundlevol.rb +240 -0
- data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
- data/lib/ec2/amitools/crypto.rb +379 -0
- data/lib/ec2/amitools/decryptmanifest.rb +20 -0
- data/lib/ec2/amitools/defaults.rb +12 -0
- data/lib/ec2/amitools/deletebundle.rb +212 -0
- data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
- data/lib/ec2/amitools/downloadbundle.rb +161 -0
- data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
- data/lib/ec2/amitools/exception.rb +86 -0
- data/lib/ec2/amitools/fileutil.rb +219 -0
- data/lib/ec2/amitools/format.rb +127 -0
- data/lib/ec2/amitools/instance-data.rb +97 -0
- data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
- data/lib/ec2/amitools/manifestv20070829.rb +361 -0
- data/lib/ec2/amitools/manifestv20071010.rb +403 -0
- data/lib/ec2/amitools/manifestv3.rb +331 -0
- data/lib/ec2/amitools/mapids.rb +148 -0
- data/lib/ec2/amitools/migratebundle.rb +222 -0
- data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
- data/lib/ec2/amitools/migratemanifest.rb +225 -0
- data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
- data/lib/ec2/amitools/minimalec2.rb +116 -0
- data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
- data/lib/ec2/amitools/parameters_base.rb +168 -0
- data/lib/ec2/amitools/region.rb +93 -0
- data/lib/ec2/amitools/s3toolparameters.rb +183 -0
- data/lib/ec2/amitools/showversion.rb +12 -0
- data/lib/ec2/amitools/syschecks.rb +27 -0
- data/lib/ec2/amitools/tool_base.rb +224 -0
- data/lib/ec2/amitools/unbundle.rb +107 -0
- data/lib/ec2/amitools/unbundleparameters.rb +65 -0
- data/lib/ec2/amitools/uploadbundle.rb +361 -0
- data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
- data/lib/ec2/amitools/util.rb +532 -0
- data/lib/ec2/amitools/version.rb +33 -0
- data/lib/ec2/amitools/xmlbuilder.rb +237 -0
- data/lib/ec2/amitools/xmlutil.rb +55 -0
- data/lib/ec2/common/constants.rb +16 -0
- data/lib/ec2/common/curl.rb +110 -0
- data/lib/ec2/common/headers.rb +95 -0
- data/lib/ec2/common/headersv4.rb +173 -0
- data/lib/ec2/common/http.rb +333 -0
- data/lib/ec2/common/s3support.rb +231 -0
- data/lib/ec2/common/signature.rb +68 -0
- data/lib/ec2/oem/LICENSE.txt +58 -0
- data/lib/ec2/oem/open4.rb +399 -0
- data/lib/ec2/platform/base/architecture.rb +26 -0
- data/lib/ec2/platform/base/constants.rb +54 -0
- data/lib/ec2/platform/base/pipeline.rb +181 -0
- data/lib/ec2/platform/base.rb +57 -0
- data/lib/ec2/platform/current.rb +55 -0
- data/lib/ec2/platform/linux/architecture.rb +35 -0
- data/lib/ec2/platform/linux/constants.rb +23 -0
- data/lib/ec2/platform/linux/fstab.rb +99 -0
- data/lib/ec2/platform/linux/identity.rb +16 -0
- data/lib/ec2/platform/linux/image.rb +811 -0
- data/lib/ec2/platform/linux/mtab.rb +74 -0
- data/lib/ec2/platform/linux/pipeline.rb +40 -0
- data/lib/ec2/platform/linux/rsync.rb +114 -0
- data/lib/ec2/platform/linux/tar.rb +124 -0
- data/lib/ec2/platform/linux/uname.rb +50 -0
- data/lib/ec2/platform/linux.rb +83 -0
- data/lib/ec2/platform/solaris/architecture.rb +28 -0
- data/lib/ec2/platform/solaris/constants.rb +30 -0
- data/lib/ec2/platform/solaris/fstab.rb +43 -0
- data/lib/ec2/platform/solaris/identity.rb +16 -0
- data/lib/ec2/platform/solaris/image.rb +327 -0
- data/lib/ec2/platform/solaris/mtab.rb +29 -0
- data/lib/ec2/platform/solaris/pipeline.rb +40 -0
- data/lib/ec2/platform/solaris/rsync.rb +24 -0
- data/lib/ec2/platform/solaris/tar.rb +36 -0
- data/lib/ec2/platform/solaris/uname.rb +21 -0
- data/lib/ec2/platform/solaris.rb +38 -0
- data/lib/ec2/platform.rb +69 -0
- data/lib/ec2/version.rb +8 -0
- data/lib/ec2_amitools +1 -0
- data/lib/ec2_amitools.rb +7 -0
- metadata +184 -0
@@ -0,0 +1,12 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/version'
|
12
|
+
puts EC2Version::version_copyright_string()
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/fileutil'
|
12
|
+
require 'ec2/platform/current'
|
13
|
+
|
14
|
+
module SysChecks
|
15
|
+
def self.rsync_usable?()
|
16
|
+
EC2::Platform::Current::Rsync.usable?
|
17
|
+
end
|
18
|
+
def self.good_tar_version?()
|
19
|
+
EC2::Platform::Current::Tar::Version.current.usable?
|
20
|
+
end
|
21
|
+
def self.get_system_arch()
|
22
|
+
EC2::Platform::Current::System::BUNDLING_ARCHITECTURE
|
23
|
+
end
|
24
|
+
def self.root_user?()
|
25
|
+
EC2::Platform::Current::System.superuser?
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,224 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
module AMIToolExceptions
|
12
|
+
# All fatal errors should inherit from this.
|
13
|
+
class EC2FatalError < RuntimeError
|
14
|
+
attr_accessor :code
|
15
|
+
def initialize(code, msg)
|
16
|
+
super(msg)
|
17
|
+
@code = code
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class FileNotFound < EC2FatalError
|
22
|
+
def initialize(path)
|
23
|
+
super(2, "File not found: #{path}")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class S3Error < EC2FatalError
|
28
|
+
def initialize(msg)
|
29
|
+
super(3, "Error talking to S3: #{msg}")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class PromptTimeout < EC2FatalError
|
34
|
+
def initialize(msg=nil)
|
35
|
+
message = "Timed out waiting for user input"
|
36
|
+
message += ": #{msg}" unless msg.nil?
|
37
|
+
super(5, message)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# This is more for flow control than anything else.
|
42
|
+
# Raising it should terminate execution, but not print an error.
|
43
|
+
class EC2StopExecution < RuntimeError
|
44
|
+
attr_accessor :code
|
45
|
+
def initialize(code=0)
|
46
|
+
super()
|
47
|
+
@code = code
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class TryFailed < RuntimeError
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
class AMITool
|
57
|
+
|
58
|
+
include AMIToolExceptions
|
59
|
+
|
60
|
+
PROMPT_TIMEOUT = 30
|
61
|
+
MAX_TRIES = 5
|
62
|
+
BACKOFF_PERIOD = 5
|
63
|
+
|
64
|
+
#------------------------------------------------------------------------------#
|
65
|
+
# Methods to override in subclasses
|
66
|
+
#------------------------------------------------------------------------------#
|
67
|
+
|
68
|
+
def get_manual()
|
69
|
+
# We have to get the manual text into here.
|
70
|
+
raise "NotImplemented: get_manual()"
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_name()
|
74
|
+
# We have to get the tool name into here.
|
75
|
+
raise "NotImplemented: get_name()"
|
76
|
+
end
|
77
|
+
|
78
|
+
def main(params)
|
79
|
+
# Main entry point.
|
80
|
+
raise "NotImplemented: main()"
|
81
|
+
end
|
82
|
+
|
83
|
+
#------------------------------------------------------------------------------#
|
84
|
+
# Utility methods
|
85
|
+
#------------------------------------------------------------------------------#
|
86
|
+
|
87
|
+
# Display a message (without appending a newline) and ask for a response.
|
88
|
+
# Returns user response or nil if interactivity is not desired.
|
89
|
+
# Raises exception on timeout.
|
90
|
+
def interactive_prompt(message, name=nil)
|
91
|
+
return nil unless interactive?
|
92
|
+
begin
|
93
|
+
$stdout.print(message)
|
94
|
+
$stdout.flush
|
95
|
+
Timeout::timeout(PROMPT_TIMEOUT) do
|
96
|
+
return gets
|
97
|
+
end
|
98
|
+
rescue Timeout::Error
|
99
|
+
raise PromptTimeout.new(name)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
#------------------------------------------------------------------------------#
|
104
|
+
|
105
|
+
# Display a message on stderr.
|
106
|
+
# If interactive, asks for confirmation (yes/no).
|
107
|
+
# Returns true if in batch mode or user agrees, false if user disagrees.
|
108
|
+
# Raises exception on timeout.
|
109
|
+
def warn_confirm(message)
|
110
|
+
$stderr.puts(message)
|
111
|
+
$stderr.flush
|
112
|
+
return true unless interactive?
|
113
|
+
response = interactive_prompt("Are you sure you want to continue? [y/N]")
|
114
|
+
if response =~ /^[Yy]/
|
115
|
+
return true
|
116
|
+
end
|
117
|
+
return false
|
118
|
+
end
|
119
|
+
|
120
|
+
#----------------------------------------------------------------------------#
|
121
|
+
|
122
|
+
def retry_s3(retrying=true)
|
123
|
+
tries = 0
|
124
|
+
while true
|
125
|
+
tries += 1
|
126
|
+
begin
|
127
|
+
result = yield
|
128
|
+
return result
|
129
|
+
rescue TryFailed => e
|
130
|
+
$stderr.puts e.message
|
131
|
+
if retrying and tries < MAX_TRIES
|
132
|
+
$stdout.puts "Retrying in #{BACKOFF_PERIOD}s ..."
|
133
|
+
else
|
134
|
+
raise EC2FatalError.new(3, e.message)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
#------------------------------------------------------------------------------#
|
141
|
+
# Standard behaviour
|
142
|
+
#------------------------------------------------------------------------------#
|
143
|
+
|
144
|
+
def handle_early_exit_parameters(params)
|
145
|
+
if params.version
|
146
|
+
puts get_name() + " " + params.version_copyright_string()
|
147
|
+
return
|
148
|
+
end
|
149
|
+
|
150
|
+
if params.show_help
|
151
|
+
puts params.help
|
152
|
+
return
|
153
|
+
end
|
154
|
+
|
155
|
+
if params.manual
|
156
|
+
puts get_manual()
|
157
|
+
return
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
#------------------------------------------------------------------------------#
|
162
|
+
|
163
|
+
def interactive?
|
164
|
+
@interactive
|
165
|
+
end
|
166
|
+
|
167
|
+
#------------------------------------------------------------------------------#
|
168
|
+
|
169
|
+
def get_parameters(params_class)
|
170
|
+
# Parse the parameters and die on errors.
|
171
|
+
# Assume that if we're parsing parameters, it's safe to exit.
|
172
|
+
begin
|
173
|
+
params = params_class.new(ARGV)
|
174
|
+
rescue StandardError => e
|
175
|
+
$stderr.puts e.message
|
176
|
+
$stderr.puts "Try '#{get_name} --help'"
|
177
|
+
exit 1
|
178
|
+
end
|
179
|
+
|
180
|
+
# Deal with help, verion, etc.
|
181
|
+
if params.early_exit?
|
182
|
+
handle_early_exit_parameters(params)
|
183
|
+
exit 0
|
184
|
+
end
|
185
|
+
|
186
|
+
# Some general flags that we want to set
|
187
|
+
@debug = params.debug
|
188
|
+
@interactive = params.interactive?
|
189
|
+
|
190
|
+
# Finally, return the leftovers.
|
191
|
+
params
|
192
|
+
end
|
193
|
+
|
194
|
+
#------------------------------------------------------------------------------#
|
195
|
+
|
196
|
+
def run(params_class)
|
197
|
+
# We want to be able to reuse bits without having to parse
|
198
|
+
# parameters, so run() is not called from the constructor.
|
199
|
+
begin
|
200
|
+
params = get_parameters(params_class)
|
201
|
+
main(params)
|
202
|
+
rescue AMIToolExceptions::EC2StopExecution => e
|
203
|
+
# We've been asked to stop.
|
204
|
+
exit e.code
|
205
|
+
rescue AMIToolExceptions::PromptTimeout => e
|
206
|
+
$stderr.puts e.message
|
207
|
+
exit e.code
|
208
|
+
rescue AMIToolExceptions::EC2FatalError => e
|
209
|
+
$stderr.puts "ERROR: #{e.message}"
|
210
|
+
puts e.backtrace if @debug
|
211
|
+
exit e.code
|
212
|
+
rescue Interrupt => e
|
213
|
+
$stderr.puts "\n#{get_name} interrupted."
|
214
|
+
puts e.backtrace if @debug
|
215
|
+
exit 255
|
216
|
+
rescue => e
|
217
|
+
$stderr.puts "ERROR: #{e.message}"
|
218
|
+
puts e.inspect if @debug
|
219
|
+
puts e.backtrace if @debug
|
220
|
+
exit 254
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/format'
|
12
|
+
require 'ec2/amitools/manifestv3'
|
13
|
+
require 'ec2/amitools/unbundleparameters'
|
14
|
+
require 'ec2/platform/current'
|
15
|
+
require 'ec2/amitools/tool_base'
|
16
|
+
|
17
|
+
UNBUNDLE_NAME = 'ec2-unbundle'
|
18
|
+
|
19
|
+
UNBUNDLE_MANUAL =<<TEXT
|
20
|
+
#{UNBUNDLE_NAME} extracts a filesystem image from a bundle AMI.
|
21
|
+
|
22
|
+
#{UNBUNDLE_NAME} will:
|
23
|
+
- read relevant information from the manifest file
|
24
|
+
- concatenate all the parts
|
25
|
+
- decrypt and uncompress the image
|
26
|
+
TEXT
|
27
|
+
|
28
|
+
class Unbundler < AMITool
|
29
|
+
|
30
|
+
def unbundle(p)
|
31
|
+
begin
|
32
|
+
manifest_path = p.manifest_path
|
33
|
+
src_dir = p.source
|
34
|
+
dst_dir = p.destination
|
35
|
+
|
36
|
+
digest_pipe = File::join( '/tmp', "ec2-unbundle-image-digest-pipe" )
|
37
|
+
File::delete( digest_pipe ) if File::exist?( digest_pipe )
|
38
|
+
unless system( "mkfifo #{digest_pipe}" )
|
39
|
+
raise "error creating named pipe #{digest_pipe}"
|
40
|
+
end
|
41
|
+
|
42
|
+
# Load manifest and the user's private key.
|
43
|
+
manifest = ManifestV3.new(File.open( manifest_path ) { |f| f.read() })
|
44
|
+
pk = Crypto::loadprivkey( p.user_pk_path )
|
45
|
+
|
46
|
+
# Extract key and IV from XML manifest.
|
47
|
+
key = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_key))
|
48
|
+
iv = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_iv))
|
49
|
+
|
50
|
+
# Create a string of space separated part paths.
|
51
|
+
part_files = manifest.parts.collect do |part|
|
52
|
+
File::join( src_dir, part.filename )
|
53
|
+
end.join( ' ' )
|
54
|
+
|
55
|
+
# Join, decrypt, decompress and untar.
|
56
|
+
untar = EC2::Platform::Current::Tar::Command.new.extract.chdir(dst_dir)
|
57
|
+
pipeline = EC2::Platform::Current::Pipeline.new('image-unbundle-pipeline', @debug)
|
58
|
+
pipeline.concat([
|
59
|
+
['cat', "openssl sha1 < #{digest_pipe} & cat #{part_files}"],
|
60
|
+
['decrypt', "openssl enc -d -aes-128-cbc -K #{key} -iv #{iv}"],
|
61
|
+
['gunzip', "gunzip"],
|
62
|
+
['tee', "tee #{digest_pipe}"],
|
63
|
+
['untar', untar.expand]
|
64
|
+
])
|
65
|
+
digest = nil
|
66
|
+
begin
|
67
|
+
digest = pipeline.execute.split(/\s+/).last.strip
|
68
|
+
rescue EC2::Platform::Current::Pipeline::ExecutionError => e
|
69
|
+
$stderr.puts e.message
|
70
|
+
end
|
71
|
+
|
72
|
+
# Verify digest.
|
73
|
+
unless manifest.digest == digest
|
74
|
+
raise "invalid digest, expected #{manifest.digest} received #{digest}"
|
75
|
+
end
|
76
|
+
|
77
|
+
puts "Unbundle complete."
|
78
|
+
return 0
|
79
|
+
ensure
|
80
|
+
File::delete( digest_pipe ) if (digest_pipe && File::exist?( digest_pipe ))
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
#------------------------------------------------------------------------------#
|
85
|
+
# Overrides
|
86
|
+
#------------------------------------------------------------------------------#
|
87
|
+
|
88
|
+
def get_manual()
|
89
|
+
UNBUNDLE_MANUAL
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_name()
|
93
|
+
UNBUNDLE_NAME
|
94
|
+
end
|
95
|
+
|
96
|
+
def main(p)
|
97
|
+
unbundle(p)
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
#------------------------------------------------------------------------------#
|
103
|
+
# Script entry point. Execute only if this file is being executed.
|
104
|
+
|
105
|
+
if __FILE__ == $0 || $0.match(/bin\/ec2-unbundle/)
|
106
|
+
Unbundler.new().run(UnbundleParameters)
|
107
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/parameters_base'
|
12
|
+
|
13
|
+
class UnbundleParameters < ParametersBase
|
14
|
+
|
15
|
+
MANIFEST_DESCRIPTION = "The path to the AMI manifest file."
|
16
|
+
SOURCE_DESCRIPTION = 'The directory containing bundled AMI parts to unbundle. Defaults to ".".'
|
17
|
+
DESTINATION_DESCRIPTION = 'The directory to unbundle the AMI into. Defaults to the ".".'
|
18
|
+
|
19
|
+
attr_accessor :manifest_path,
|
20
|
+
:user_pk_path,
|
21
|
+
:source,
|
22
|
+
:destination
|
23
|
+
|
24
|
+
#----------------------------------------------------------------------------#
|
25
|
+
|
26
|
+
def mandatory_params()
|
27
|
+
on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
|
28
|
+
assert_file_exists(path, '--privatekey')
|
29
|
+
@user_pk_path = path
|
30
|
+
end
|
31
|
+
|
32
|
+
on('-m', '--manifest PATH', String, MANIFEST_DESCRIPTION) do |manifest|
|
33
|
+
assert_file_exists(manifest, '--manifest')
|
34
|
+
@manifest_path = manifest
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
#----------------------------------------------------------------------------#
|
39
|
+
|
40
|
+
def optional_params()
|
41
|
+
on('-s', '--source DIRECTORY', String, SOURCE_DESCRIPTION) do |directory|
|
42
|
+
assert_directory_exists(directory, '--source')
|
43
|
+
@source = directory
|
44
|
+
end
|
45
|
+
|
46
|
+
on('-d', '--destination DIRECTORY', String, DESTINATION_DESCRIPTION) do |directory|
|
47
|
+
assert_directory_exists(directory, '--destination')
|
48
|
+
@destination = directory
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
#----------------------------------------------------------------------------#
|
53
|
+
|
54
|
+
def validate_params()
|
55
|
+
raise MissingMandatory.new('--manifest') unless @manifest_path
|
56
|
+
raise MissingMandatory.new('--privatekey') unless @user_pk_path
|
57
|
+
end
|
58
|
+
|
59
|
+
#----------------------------------------------------------------------------#
|
60
|
+
|
61
|
+
def set_defaults()
|
62
|
+
@source ||= Dir::pwd()
|
63
|
+
@destination ||= Dir::pwd()
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,361 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/common/s3support'
|
12
|
+
require 'ec2/amitools/uploadbundleparameters'
|
13
|
+
require 'uri'
|
14
|
+
require 'ec2/amitools/instance-data'
|
15
|
+
require 'ec2/amitools/manifestv20071010'
|
16
|
+
require 'rexml/document'
|
17
|
+
require 'digest/md5'
|
18
|
+
require 'base64'
|
19
|
+
require 'ec2/amitools/tool_base'
|
20
|
+
require 'ec2/amitools/region'
|
21
|
+
|
22
|
+
#------------------------------------------------------------------------------#
|
23
|
+
|
24
|
+
UPLOAD_BUNDLE_NAME = 'ec2-upload-bundle'
|
25
|
+
|
26
|
+
UPLOAD_BUNDLE_MANUAL =<<TEXT
|
27
|
+
#{UPLOAD_BUNDLE_NAME} is a command line tool to upload a bundled Amazon Image to S3 storage
|
28
|
+
for use by EC2. An Amazon Image may be one of the following:
|
29
|
+
- Amazon Machine Image (AMI)
|
30
|
+
- Amazon Kernel Image (AKI)
|
31
|
+
- Amazon Ramdisk Image (ARI)
|
32
|
+
|
33
|
+
#{UPLOAD_BUNDLE_NAME} will:
|
34
|
+
- create an S3 bucket to store the bundled AMI in if it does not already exist
|
35
|
+
- upload the AMI manifest and parts files to S3, granting specified privileges
|
36
|
+
- on them (defaults to EC2 read privileges)
|
37
|
+
|
38
|
+
To manually retry an upload that failed, #{UPLOAD_BUNDLE_NAME} can optionally:
|
39
|
+
- skip uploading the manifest
|
40
|
+
- only upload bundled AMI parts from a specified part onwards
|
41
|
+
TEXT
|
42
|
+
|
43
|
+
#------------------------------------------------------------------------------#
|
44
|
+
|
45
|
+
class BucketLocationError < AMIToolExceptions::EC2FatalError
|
46
|
+
def initialize(bucket, location, bucket_location)
|
47
|
+
location = "US" if location == :unconstrained
|
48
|
+
bucket_location = "US" if bucket_location == :unconstrained
|
49
|
+
super(10, "Bucket \"#{bucket}\" already exists in \"#{bucket_location}\" and \"#{location}\" was specified.")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
#----------------------------------------------------------------------------#
|
54
|
+
|
55
|
+
# Upload the specified file.
|
56
|
+
|
57
|
+
class BundleUploader < AMITool
|
58
|
+
|
59
|
+
def upload(s3_conn, bucket, key, file, acl, retry_upload)
|
60
|
+
retry_s3(retry_upload) do
|
61
|
+
begin
|
62
|
+
md5 = get_md5(file)
|
63
|
+
s3_conn.put(bucket, key, file, {"x-amz-acl"=>acl, "content-md5"=>md5})
|
64
|
+
return
|
65
|
+
rescue EC2::Common::HTTP::Error::PathInvalid => e
|
66
|
+
raise FileNotFound(file)
|
67
|
+
rescue => e
|
68
|
+
raise TryFailed.new("Failed to upload \"#{file}\": #{e.message}")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
#----------------------------------------------------------------------------#
|
74
|
+
|
75
|
+
def get_md5(file)
|
76
|
+
Base64::encode64(Digest::MD5::digest(File.open(file) { |f| f.read })).strip
|
77
|
+
end
|
78
|
+
|
79
|
+
#----------------------------------------------------------------------------#
|
80
|
+
|
81
|
+
#
|
82
|
+
# Availability zone names are generally in the format => ${REGION}${ZONENUMBER}.
|
83
|
+
# Examples being us-east-1b, us-east-1c, etc.
|
84
|
+
#
|
85
|
+
def get_availability_zone()
|
86
|
+
instance_data = EC2::InstanceData.new
|
87
|
+
instance_data.availability_zone
|
88
|
+
end
|
89
|
+
|
90
|
+
#----------------------------------------------------------------------------#
|
91
|
+
|
92
|
+
# Return a list of bundle part filename and part number tuples from the manifest.
|
93
|
+
def get_part_info(manifest)
|
94
|
+
parts = manifest.ami_part_info_list.map do |part|
|
95
|
+
[part['filename'], part['index']]
|
96
|
+
end
|
97
|
+
parts.sort
|
98
|
+
end
|
99
|
+
|
100
|
+
#------------------------------------------------------------------------------#
|
101
|
+
|
102
|
+
def uri2string(uri)
|
103
|
+
s = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
|
104
|
+
# Remove the trailing '/'.
|
105
|
+
return (s[-1..-1] == "/" ? s[0..-2] : s)
|
106
|
+
end
|
107
|
+
|
108
|
+
#------------------------------------------------------------------------------#
|
109
|
+
|
110
|
+
# Get the bucket's location.
|
111
|
+
def get_bucket_location(s3_conn, bucket)
|
112
|
+
begin
|
113
|
+
response = s3_conn.get_bucket_location(bucket)
|
114
|
+
rescue EC2::Common::HTTP::Error::Retrieve => e
|
115
|
+
if e.code == 404
|
116
|
+
# We have a "Not found" S3 response, which probably means the bucket doesn't exist.
|
117
|
+
return nil
|
118
|
+
end
|
119
|
+
raise e
|
120
|
+
end
|
121
|
+
$stdout.puts "check_bucket_location response: #{response.body}" if @debug and response.text?
|
122
|
+
docroot = REXML::Document.new(response.body).root
|
123
|
+
bucket_location = REXML::XPath.first(docroot, '/LocationConstraint').text
|
124
|
+
bucket_location ||= :unconstrained
|
125
|
+
end
|
126
|
+
|
127
|
+
#------------------------------------------------------------------------------#
|
128
|
+
|
129
|
+
# Check if the bucket exists and is in an appropriate location.
|
130
|
+
def check_bucket_location(bucket, bucket_location, location)
|
131
|
+
if bucket_location.nil?
|
132
|
+
# The bucket does not exist. Safe, but we need to create it.
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
if location.nil?
|
136
|
+
# The bucket exists and we don't care where it is.
|
137
|
+
return true
|
138
|
+
end
|
139
|
+
unless [bucket_location, AwsRegion.guess_region_from_s3_bucket(bucket_location)].include?(location)
|
140
|
+
# The bucket isn't where we want it. This is a problem.
|
141
|
+
raise BucketLocationError.new(bucket, location, bucket_location)
|
142
|
+
end
|
143
|
+
# The bucket exists and is in the right place.
|
144
|
+
return true
|
145
|
+
end
|
146
|
+
|
147
|
+
#------------------------------------------------------------------------------#
|
148
|
+
|
149
|
+
# Create the specified bucket if it does not exist.
|
150
|
+
def create_bucket(s3_conn, bucket, bucket_location, location, retry_create)
|
151
|
+
begin
|
152
|
+
if check_bucket_location(bucket, bucket_location, location)
|
153
|
+
return true
|
154
|
+
end
|
155
|
+
$stdout.puts "Creating bucket..."
|
156
|
+
|
157
|
+
retry_s3(retry_create) do
|
158
|
+
error = "Could not create or access bucket #{bucket}"
|
159
|
+
begin
|
160
|
+
rsp = s3_conn.create_bucket(bucket, location == :unconstrained ? nil : location)
|
161
|
+
rescue EC2::Common::HTTP::Error::Retrieve => e
|
162
|
+
error += ": server response #{e.message} #{e.code}"
|
163
|
+
raise TryFailed.new(e.message)
|
164
|
+
rescue RuntimeError => e
|
165
|
+
error += ": error message #{e.message}"
|
166
|
+
raise e
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
#------------------------------------------------------------------------------#
|
173
|
+
|
174
|
+
# If we return true, we have a v2-compliant name.
|
175
|
+
# If we return false, we wish to use a bad name.
|
176
|
+
# Otherwise we quietly wander off to die in peace.
|
177
|
+
def check_bucket_name(bucket)
|
178
|
+
if EC2::Common::S3Support::bucket_name_s3_v2_safe?(bucket)
|
179
|
+
return true
|
180
|
+
end
|
181
|
+
message = "The specified bucket is not S3 v2 safe (see S3 documentation for details):\n#{bucket}"
|
182
|
+
if warn_confirm(message)
|
183
|
+
# Assume the customer knows what he's doing.
|
184
|
+
return false
|
185
|
+
else
|
186
|
+
# We've been asked to stop, so quietly wander off to die in peace.
|
187
|
+
raise EC2StopExecution.new()
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
# force v1 S3 addressing when using govcloud endpoint
|
192
|
+
def check_govcloud_override(s3_url)
|
193
|
+
if s3_url =~ /s3-us-gov-west-1/
|
194
|
+
false
|
195
|
+
else
|
196
|
+
true
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
#------------------------------------------------------------------------------#
|
201
|
+
|
202
|
+
def get_region()
|
203
|
+
zone = get_availability_zone()
|
204
|
+
if zone.nil?
|
205
|
+
return nil
|
206
|
+
end
|
207
|
+
# assume region names do not have a common naming scheme. Therefore we manually go through all known region names
|
208
|
+
AwsRegion.regions.each do |region|
|
209
|
+
match = zone.match(region)
|
210
|
+
if not match.nil?
|
211
|
+
return region
|
212
|
+
end
|
213
|
+
end
|
214
|
+
nil
|
215
|
+
end
|
216
|
+
|
217
|
+
# This is very much a best effort attempt. If in doubt, we don't warn.
|
218
|
+
def cross_region?(location, bucket_location)
|
219
|
+
|
220
|
+
# If the bucket exists, its S3 location is canonical.
|
221
|
+
s3_region = bucket_location
|
222
|
+
s3_region ||= location
|
223
|
+
s3_region ||= :unconstrained
|
224
|
+
|
225
|
+
region = get_region()
|
226
|
+
|
227
|
+
if region.nil?
|
228
|
+
# If we can't get the region, assume we're fine since there's
|
229
|
+
# nothing more we can do.
|
230
|
+
return false
|
231
|
+
end
|
232
|
+
|
233
|
+
return s3_region != AwsRegion.get_s3_location(region)
|
234
|
+
end
|
235
|
+
|
236
|
+
#------------------------------------------------------------------------------#
|
237
|
+
|
238
|
+
def warn_about_migrating()
|
239
|
+
message = ["You are bundling in one region, but uploading to another. If the kernel",
|
240
|
+
"or ramdisk associated with this AMI are not in the target region, AMI",
|
241
|
+
"registration will fail.",
|
242
|
+
"You can use the ec2-migrate-manifest tool to update your manifest file",
|
243
|
+
"with a kernel and ramdisk that exist in the target region.",
|
244
|
+
].join("\n")
|
245
|
+
unless warn_confirm(message)
|
246
|
+
raise EC2StopExecution.new()
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
#------------------------------------------------------------------------------#
|
251
|
+
|
252
|
+
def get_s3_conn(s3_url, user, pass, method, sigv, region=nil)
|
253
|
+
EC2::Common::S3Support.new(s3_url, user, pass, method, @debug, sigv, region)
|
254
|
+
end
|
255
|
+
|
256
|
+
#------------------------------------------------------------------------------#
|
257
|
+
|
258
|
+
#
|
259
|
+
# Get parameters and display help or manual if necessary.
|
260
|
+
#
|
261
|
+
def upload_bundle(url,
|
262
|
+
bucket,
|
263
|
+
keyprefix,
|
264
|
+
user,
|
265
|
+
pass,
|
266
|
+
location,
|
267
|
+
manifest_file,
|
268
|
+
retry_stuff,
|
269
|
+
part,
|
270
|
+
directory,
|
271
|
+
acl,
|
272
|
+
skipmanifest,
|
273
|
+
sigv,
|
274
|
+
region)
|
275
|
+
begin
|
276
|
+
# Get the S3 URL.
|
277
|
+
s3_uri = URI.parse(url)
|
278
|
+
s3_url = uri2string(s3_uri)
|
279
|
+
v2_bucket = check_bucket_name(bucket) and check_govcloud_override(s3_url)
|
280
|
+
s3_conn = get_s3_conn(s3_url, user, pass, (v2_bucket ? nil : :path), sigv, region)
|
281
|
+
|
282
|
+
# Get current location and bucket location.
|
283
|
+
bucket_location = get_bucket_location(s3_conn, bucket)
|
284
|
+
|
285
|
+
# Load manifest.
|
286
|
+
xml = File.open(manifest_file) { |f| f.read }
|
287
|
+
manifest = ManifestV20071010.new(xml)
|
288
|
+
|
289
|
+
# If in interactive mode, warn when bundling a kernel into our AMI and we are uploading cross-region
|
290
|
+
if interactive? and manifest.kernel_id and cross_region?(location, bucket_location)
|
291
|
+
warn_about_migrating()
|
292
|
+
end
|
293
|
+
|
294
|
+
# Create storage bucket if required.
|
295
|
+
create_bucket(s3_conn, bucket, bucket_location, location, retry_stuff)
|
296
|
+
|
297
|
+
# Upload AMI bundle parts.
|
298
|
+
$stdout.puts "Uploading bundled image parts to the S3 bucket #{bucket} ..."
|
299
|
+
get_part_info(manifest).each do |part_info|
|
300
|
+
if part.nil? or (part_info[1] >= part)
|
301
|
+
path = File.join(directory, part_info[0])
|
302
|
+
upload(s3_conn, bucket, keyprefix + part_info[0], path, acl, retry_stuff)
|
303
|
+
$stdout.puts "Uploaded #{part_info[0]}"
|
304
|
+
else
|
305
|
+
$stdout.puts "Skipping #{part_info[0]}"
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
# Encrypt and upload manifest.
|
310
|
+
unless skipmanifest
|
311
|
+
$stdout.puts "Uploading manifest ..."
|
312
|
+
upload(s3_conn, bucket, keyprefix + File::basename(manifest_file), manifest_file, acl, retry_stuff)
|
313
|
+
$stdout.puts "Uploaded manifest."
|
314
|
+
$stdout.puts 'Manifest uploaded to: %s/%s' % [bucket, keyprefix + File::basename(manifest_file)]
|
315
|
+
else
|
316
|
+
$stdout.puts "Skipping manifest."
|
317
|
+
end
|
318
|
+
|
319
|
+
$stdout.puts 'Bundle upload completed.'
|
320
|
+
rescue EC2::Common::HTTP::Error => e
|
321
|
+
$stderr.puts e.backtrace if @debug
|
322
|
+
raise S3Error.new(e.message)
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
#------------------------------------------------------------------------------#
|
327
|
+
# Overrides
|
328
|
+
#------------------------------------------------------------------------------#
|
329
|
+
|
330
|
+
def get_manual()
|
331
|
+
UPLOAD_BUNDLE_MANUAL
|
332
|
+
end
|
333
|
+
|
334
|
+
def get_name()
|
335
|
+
UPLOAD_BUNDLE_NAME
|
336
|
+
end
|
337
|
+
|
338
|
+
def main(p)
|
339
|
+
upload_bundle(p.url,
|
340
|
+
p.bucket,
|
341
|
+
p.keyprefix,
|
342
|
+
p.user,
|
343
|
+
p.pass,
|
344
|
+
p.location,
|
345
|
+
p.manifest,
|
346
|
+
p.retry,
|
347
|
+
p.part,
|
348
|
+
p.directory,
|
349
|
+
p.acl,
|
350
|
+
p.skipmanifest,
|
351
|
+
p.sigv,
|
352
|
+
p.region)
|
353
|
+
end
|
354
|
+
|
355
|
+
end
|
356
|
+
|
357
|
+
#------------------------------------------------------------------------------#
|
358
|
+
# Script entry point. Execute only if this file is being executed.
|
359
|
+
if __FILE__ == $0 || $0.match(/bin\/ec2-upload-bundle/)
|
360
|
+
BundleUploader.new().run(UploadBundleParameters)
|
361
|
+
end
|