ec2_amitools 1.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +54 -0
- data/bin/console +14 -0
- data/bin/ec2-ami-tools-version +6 -0
- data/bin/ec2-bundle-image +6 -0
- data/bin/ec2-bundle-vol +6 -0
- data/bin/ec2-delete-bundle +6 -0
- data/bin/ec2-download-bundle +6 -0
- data/bin/ec2-migrate-bundle +6 -0
- data/bin/ec2-migrate-manifest +6 -0
- data/bin/ec2-unbundle +6 -0
- data/bin/ec2-upload-bundle +6 -0
- data/bin/setup +8 -0
- data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
- data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
- data/etc/ec2/amitools/cert-ec2.pem +23 -0
- data/etc/ec2/amitools/mappings.csv +9 -0
- data/lib/ec2/amitools/bundle.rb +251 -0
- data/lib/ec2/amitools/bundle_base.rb +58 -0
- data/lib/ec2/amitools/bundleimage.rb +94 -0
- data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
- data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
- data/lib/ec2/amitools/bundleparameters.rb +120 -0
- data/lib/ec2/amitools/bundlevol.rb +240 -0
- data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
- data/lib/ec2/amitools/crypto.rb +379 -0
- data/lib/ec2/amitools/decryptmanifest.rb +20 -0
- data/lib/ec2/amitools/defaults.rb +12 -0
- data/lib/ec2/amitools/deletebundle.rb +212 -0
- data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
- data/lib/ec2/amitools/downloadbundle.rb +161 -0
- data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
- data/lib/ec2/amitools/exception.rb +86 -0
- data/lib/ec2/amitools/fileutil.rb +219 -0
- data/lib/ec2/amitools/format.rb +127 -0
- data/lib/ec2/amitools/instance-data.rb +97 -0
- data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
- data/lib/ec2/amitools/manifestv20070829.rb +361 -0
- data/lib/ec2/amitools/manifestv20071010.rb +403 -0
- data/lib/ec2/amitools/manifestv3.rb +331 -0
- data/lib/ec2/amitools/mapids.rb +148 -0
- data/lib/ec2/amitools/migratebundle.rb +222 -0
- data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
- data/lib/ec2/amitools/migratemanifest.rb +225 -0
- data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
- data/lib/ec2/amitools/minimalec2.rb +116 -0
- data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
- data/lib/ec2/amitools/parameters_base.rb +168 -0
- data/lib/ec2/amitools/region.rb +93 -0
- data/lib/ec2/amitools/s3toolparameters.rb +183 -0
- data/lib/ec2/amitools/showversion.rb +12 -0
- data/lib/ec2/amitools/syschecks.rb +27 -0
- data/lib/ec2/amitools/tool_base.rb +224 -0
- data/lib/ec2/amitools/unbundle.rb +107 -0
- data/lib/ec2/amitools/unbundleparameters.rb +65 -0
- data/lib/ec2/amitools/uploadbundle.rb +361 -0
- data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
- data/lib/ec2/amitools/util.rb +532 -0
- data/lib/ec2/amitools/version.rb +33 -0
- data/lib/ec2/amitools/xmlbuilder.rb +237 -0
- data/lib/ec2/amitools/xmlutil.rb +55 -0
- data/lib/ec2/common/constants.rb +16 -0
- data/lib/ec2/common/curl.rb +110 -0
- data/lib/ec2/common/headers.rb +95 -0
- data/lib/ec2/common/headersv4.rb +173 -0
- data/lib/ec2/common/http.rb +333 -0
- data/lib/ec2/common/s3support.rb +231 -0
- data/lib/ec2/common/signature.rb +68 -0
- data/lib/ec2/oem/LICENSE.txt +58 -0
- data/lib/ec2/oem/open4.rb +399 -0
- data/lib/ec2/platform/base/architecture.rb +26 -0
- data/lib/ec2/platform/base/constants.rb +54 -0
- data/lib/ec2/platform/base/pipeline.rb +181 -0
- data/lib/ec2/platform/base.rb +57 -0
- data/lib/ec2/platform/current.rb +55 -0
- data/lib/ec2/platform/linux/architecture.rb +35 -0
- data/lib/ec2/platform/linux/constants.rb +23 -0
- data/lib/ec2/platform/linux/fstab.rb +99 -0
- data/lib/ec2/platform/linux/identity.rb +16 -0
- data/lib/ec2/platform/linux/image.rb +811 -0
- data/lib/ec2/platform/linux/mtab.rb +74 -0
- data/lib/ec2/platform/linux/pipeline.rb +40 -0
- data/lib/ec2/platform/linux/rsync.rb +114 -0
- data/lib/ec2/platform/linux/tar.rb +124 -0
- data/lib/ec2/platform/linux/uname.rb +50 -0
- data/lib/ec2/platform/linux.rb +83 -0
- data/lib/ec2/platform/solaris/architecture.rb +28 -0
- data/lib/ec2/platform/solaris/constants.rb +30 -0
- data/lib/ec2/platform/solaris/fstab.rb +43 -0
- data/lib/ec2/platform/solaris/identity.rb +16 -0
- data/lib/ec2/platform/solaris/image.rb +327 -0
- data/lib/ec2/platform/solaris/mtab.rb +29 -0
- data/lib/ec2/platform/solaris/pipeline.rb +40 -0
- data/lib/ec2/platform/solaris/rsync.rb +24 -0
- data/lib/ec2/platform/solaris/tar.rb +36 -0
- data/lib/ec2/platform/solaris/uname.rb +21 -0
- data/lib/ec2/platform/solaris.rb +38 -0
- data/lib/ec2/platform.rb +69 -0
- data/lib/ec2/version.rb +8 -0
- data/lib/ec2_amitools +1 -0
- data/lib/ec2_amitools.rb +7 -0
- metadata +184 -0
@@ -0,0 +1,12 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/version'
|
12
|
+
puts EC2Version::version_copyright_string()
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/fileutil'
|
12
|
+
require 'ec2/platform/current'
|
13
|
+
|
14
|
+
module SysChecks
|
15
|
+
def self.rsync_usable?()
|
16
|
+
EC2::Platform::Current::Rsync.usable?
|
17
|
+
end
|
18
|
+
def self.good_tar_version?()
|
19
|
+
EC2::Platform::Current::Tar::Version.current.usable?
|
20
|
+
end
|
21
|
+
def self.get_system_arch()
|
22
|
+
EC2::Platform::Current::System::BUNDLING_ARCHITECTURE
|
23
|
+
end
|
24
|
+
def self.root_user?()
|
25
|
+
EC2::Platform::Current::System.superuser?
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,224 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
module AMIToolExceptions
|
12
|
+
# All fatal errors should inherit from this.
|
13
|
+
class EC2FatalError < RuntimeError
|
14
|
+
attr_accessor :code
|
15
|
+
def initialize(code, msg)
|
16
|
+
super(msg)
|
17
|
+
@code = code
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
class FileNotFound < EC2FatalError
|
22
|
+
def initialize(path)
|
23
|
+
super(2, "File not found: #{path}")
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
class S3Error < EC2FatalError
|
28
|
+
def initialize(msg)
|
29
|
+
super(3, "Error talking to S3: #{msg}")
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class PromptTimeout < EC2FatalError
|
34
|
+
def initialize(msg=nil)
|
35
|
+
message = "Timed out waiting for user input"
|
36
|
+
message += ": #{msg}" unless msg.nil?
|
37
|
+
super(5, message)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# This is more for flow control than anything else.
|
42
|
+
# Raising it should terminate execution, but not print an error.
|
43
|
+
class EC2StopExecution < RuntimeError
|
44
|
+
attr_accessor :code
|
45
|
+
def initialize(code=0)
|
46
|
+
super()
|
47
|
+
@code = code
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class TryFailed < RuntimeError
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
class AMITool
|
57
|
+
|
58
|
+
include AMIToolExceptions
|
59
|
+
|
60
|
+
PROMPT_TIMEOUT = 30
|
61
|
+
MAX_TRIES = 5
|
62
|
+
BACKOFF_PERIOD = 5
|
63
|
+
|
64
|
+
#------------------------------------------------------------------------------#
|
65
|
+
# Methods to override in subclasses
|
66
|
+
#------------------------------------------------------------------------------#
|
67
|
+
|
68
|
+
def get_manual()
|
69
|
+
# We have to get the manual text into here.
|
70
|
+
raise "NotImplemented: get_manual()"
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_name()
|
74
|
+
# We have to get the tool name into here.
|
75
|
+
raise "NotImplemented: get_name()"
|
76
|
+
end
|
77
|
+
|
78
|
+
def main(params)
|
79
|
+
# Main entry point.
|
80
|
+
raise "NotImplemented: main()"
|
81
|
+
end
|
82
|
+
|
83
|
+
#------------------------------------------------------------------------------#
|
84
|
+
# Utility methods
|
85
|
+
#------------------------------------------------------------------------------#
|
86
|
+
|
87
|
+
# Display a message (without appending a newline) and ask for a response.
|
88
|
+
# Returns user response or nil if interactivity is not desired.
|
89
|
+
# Raises exception on timeout.
|
90
|
+
def interactive_prompt(message, name=nil)
|
91
|
+
return nil unless interactive?
|
92
|
+
begin
|
93
|
+
$stdout.print(message)
|
94
|
+
$stdout.flush
|
95
|
+
Timeout::timeout(PROMPT_TIMEOUT) do
|
96
|
+
return gets
|
97
|
+
end
|
98
|
+
rescue Timeout::Error
|
99
|
+
raise PromptTimeout.new(name)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
#------------------------------------------------------------------------------#
|
104
|
+
|
105
|
+
# Display a message on stderr.
|
106
|
+
# If interactive, asks for confirmation (yes/no).
|
107
|
+
# Returns true if in batch mode or user agrees, false if user disagrees.
|
108
|
+
# Raises exception on timeout.
|
109
|
+
def warn_confirm(message)
|
110
|
+
$stderr.puts(message)
|
111
|
+
$stderr.flush
|
112
|
+
return true unless interactive?
|
113
|
+
response = interactive_prompt("Are you sure you want to continue? [y/N]")
|
114
|
+
if response =~ /^[Yy]/
|
115
|
+
return true
|
116
|
+
end
|
117
|
+
return false
|
118
|
+
end
|
119
|
+
|
120
|
+
#----------------------------------------------------------------------------#
|
121
|
+
|
122
|
+
def retry_s3(retrying=true)
|
123
|
+
tries = 0
|
124
|
+
while true
|
125
|
+
tries += 1
|
126
|
+
begin
|
127
|
+
result = yield
|
128
|
+
return result
|
129
|
+
rescue TryFailed => e
|
130
|
+
$stderr.puts e.message
|
131
|
+
if retrying and tries < MAX_TRIES
|
132
|
+
$stdout.puts "Retrying in #{BACKOFF_PERIOD}s ..."
|
133
|
+
else
|
134
|
+
raise EC2FatalError.new(3, e.message)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
#------------------------------------------------------------------------------#
|
141
|
+
# Standard behaviour
|
142
|
+
#------------------------------------------------------------------------------#
|
143
|
+
|
144
|
+
def handle_early_exit_parameters(params)
|
145
|
+
if params.version
|
146
|
+
puts get_name() + " " + params.version_copyright_string()
|
147
|
+
return
|
148
|
+
end
|
149
|
+
|
150
|
+
if params.show_help
|
151
|
+
puts params.help
|
152
|
+
return
|
153
|
+
end
|
154
|
+
|
155
|
+
if params.manual
|
156
|
+
puts get_manual()
|
157
|
+
return
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
#------------------------------------------------------------------------------#
|
162
|
+
|
163
|
+
def interactive?
|
164
|
+
@interactive
|
165
|
+
end
|
166
|
+
|
167
|
+
#------------------------------------------------------------------------------#
|
168
|
+
|
169
|
+
def get_parameters(params_class)
|
170
|
+
# Parse the parameters and die on errors.
|
171
|
+
# Assume that if we're parsing parameters, it's safe to exit.
|
172
|
+
begin
|
173
|
+
params = params_class.new(ARGV)
|
174
|
+
rescue StandardError => e
|
175
|
+
$stderr.puts e.message
|
176
|
+
$stderr.puts "Try '#{get_name} --help'"
|
177
|
+
exit 1
|
178
|
+
end
|
179
|
+
|
180
|
+
# Deal with help, verion, etc.
|
181
|
+
if params.early_exit?
|
182
|
+
handle_early_exit_parameters(params)
|
183
|
+
exit 0
|
184
|
+
end
|
185
|
+
|
186
|
+
# Some general flags that we want to set
|
187
|
+
@debug = params.debug
|
188
|
+
@interactive = params.interactive?
|
189
|
+
|
190
|
+
# Finally, return the leftovers.
|
191
|
+
params
|
192
|
+
end
|
193
|
+
|
194
|
+
#------------------------------------------------------------------------------#
|
195
|
+
|
196
|
+
def run(params_class)
|
197
|
+
# We want to be able to reuse bits without having to parse
|
198
|
+
# parameters, so run() is not called from the constructor.
|
199
|
+
begin
|
200
|
+
params = get_parameters(params_class)
|
201
|
+
main(params)
|
202
|
+
rescue AMIToolExceptions::EC2StopExecution => e
|
203
|
+
# We've been asked to stop.
|
204
|
+
exit e.code
|
205
|
+
rescue AMIToolExceptions::PromptTimeout => e
|
206
|
+
$stderr.puts e.message
|
207
|
+
exit e.code
|
208
|
+
rescue AMIToolExceptions::EC2FatalError => e
|
209
|
+
$stderr.puts "ERROR: #{e.message}"
|
210
|
+
puts e.backtrace if @debug
|
211
|
+
exit e.code
|
212
|
+
rescue Interrupt => e
|
213
|
+
$stderr.puts "\n#{get_name} interrupted."
|
214
|
+
puts e.backtrace if @debug
|
215
|
+
exit 255
|
216
|
+
rescue => e
|
217
|
+
$stderr.puts "ERROR: #{e.message}"
|
218
|
+
puts e.inspect if @debug
|
219
|
+
puts e.backtrace if @debug
|
220
|
+
exit 254
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/format'
|
12
|
+
require 'ec2/amitools/manifestv3'
|
13
|
+
require 'ec2/amitools/unbundleparameters'
|
14
|
+
require 'ec2/platform/current'
|
15
|
+
require 'ec2/amitools/tool_base'
|
16
|
+
|
17
|
+
UNBUNDLE_NAME = 'ec2-unbundle'
|
18
|
+
|
19
|
+
UNBUNDLE_MANUAL =<<TEXT
|
20
|
+
#{UNBUNDLE_NAME} extracts a filesystem image from a bundle AMI.
|
21
|
+
|
22
|
+
#{UNBUNDLE_NAME} will:
|
23
|
+
- read relevant information from the manifest file
|
24
|
+
- concatenate all the parts
|
25
|
+
- decrypt and uncompress the image
|
26
|
+
TEXT
|
27
|
+
|
28
|
+
class Unbundler < AMITool
|
29
|
+
|
30
|
+
def unbundle(p)
|
31
|
+
begin
|
32
|
+
manifest_path = p.manifest_path
|
33
|
+
src_dir = p.source
|
34
|
+
dst_dir = p.destination
|
35
|
+
|
36
|
+
digest_pipe = File::join( '/tmp', "ec2-unbundle-image-digest-pipe" )
|
37
|
+
File::delete( digest_pipe ) if File::exist?( digest_pipe )
|
38
|
+
unless system( "mkfifo #{digest_pipe}" )
|
39
|
+
raise "error creating named pipe #{digest_pipe}"
|
40
|
+
end
|
41
|
+
|
42
|
+
# Load manifest and the user's private key.
|
43
|
+
manifest = ManifestV3.new(File.open( manifest_path ) { |f| f.read() })
|
44
|
+
pk = Crypto::loadprivkey( p.user_pk_path )
|
45
|
+
|
46
|
+
# Extract key and IV from XML manifest.
|
47
|
+
key = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_key))
|
48
|
+
iv = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_iv))
|
49
|
+
|
50
|
+
# Create a string of space separated part paths.
|
51
|
+
part_files = manifest.parts.collect do |part|
|
52
|
+
File::join( src_dir, part.filename )
|
53
|
+
end.join( ' ' )
|
54
|
+
|
55
|
+
# Join, decrypt, decompress and untar.
|
56
|
+
untar = EC2::Platform::Current::Tar::Command.new.extract.chdir(dst_dir)
|
57
|
+
pipeline = EC2::Platform::Current::Pipeline.new('image-unbundle-pipeline', @debug)
|
58
|
+
pipeline.concat([
|
59
|
+
['cat', "openssl sha1 < #{digest_pipe} & cat #{part_files}"],
|
60
|
+
['decrypt', "openssl enc -d -aes-128-cbc -K #{key} -iv #{iv}"],
|
61
|
+
['gunzip', "gunzip"],
|
62
|
+
['tee', "tee #{digest_pipe}"],
|
63
|
+
['untar', untar.expand]
|
64
|
+
])
|
65
|
+
digest = nil
|
66
|
+
begin
|
67
|
+
digest = pipeline.execute.split(/\s+/).last.strip
|
68
|
+
rescue EC2::Platform::Current::Pipeline::ExecutionError => e
|
69
|
+
$stderr.puts e.message
|
70
|
+
end
|
71
|
+
|
72
|
+
# Verify digest.
|
73
|
+
unless manifest.digest == digest
|
74
|
+
raise "invalid digest, expected #{manifest.digest} received #{digest}"
|
75
|
+
end
|
76
|
+
|
77
|
+
puts "Unbundle complete."
|
78
|
+
return 0
|
79
|
+
ensure
|
80
|
+
File::delete( digest_pipe ) if (digest_pipe && File::exist?( digest_pipe ))
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
#------------------------------------------------------------------------------#
|
85
|
+
# Overrides
|
86
|
+
#------------------------------------------------------------------------------#
|
87
|
+
|
88
|
+
def get_manual()
|
89
|
+
UNBUNDLE_MANUAL
|
90
|
+
end
|
91
|
+
|
92
|
+
def get_name()
|
93
|
+
UNBUNDLE_NAME
|
94
|
+
end
|
95
|
+
|
96
|
+
def main(p)
|
97
|
+
unbundle(p)
|
98
|
+
end
|
99
|
+
|
100
|
+
end
|
101
|
+
|
102
|
+
#------------------------------------------------------------------------------#
|
103
|
+
# Script entry point. Execute only if this file is being executed.
|
104
|
+
|
105
|
+
if __FILE__ == $0 || $0.match(/bin\/ec2-unbundle/)
|
106
|
+
Unbundler.new().run(UnbundleParameters)
|
107
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/amitools/parameters_base'
|
12
|
+
|
13
|
+
class UnbundleParameters < ParametersBase
|
14
|
+
|
15
|
+
MANIFEST_DESCRIPTION = "The path to the AMI manifest file."
|
16
|
+
SOURCE_DESCRIPTION = 'The directory containing bundled AMI parts to unbundle. Defaults to ".".'
|
17
|
+
DESTINATION_DESCRIPTION = 'The directory to unbundle the AMI into. Defaults to the ".".'
|
18
|
+
|
19
|
+
attr_accessor :manifest_path,
|
20
|
+
:user_pk_path,
|
21
|
+
:source,
|
22
|
+
:destination
|
23
|
+
|
24
|
+
#----------------------------------------------------------------------------#
|
25
|
+
|
26
|
+
def mandatory_params()
|
27
|
+
on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
|
28
|
+
assert_file_exists(path, '--privatekey')
|
29
|
+
@user_pk_path = path
|
30
|
+
end
|
31
|
+
|
32
|
+
on('-m', '--manifest PATH', String, MANIFEST_DESCRIPTION) do |manifest|
|
33
|
+
assert_file_exists(manifest, '--manifest')
|
34
|
+
@manifest_path = manifest
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
#----------------------------------------------------------------------------#
|
39
|
+
|
40
|
+
def optional_params()
|
41
|
+
on('-s', '--source DIRECTORY', String, SOURCE_DESCRIPTION) do |directory|
|
42
|
+
assert_directory_exists(directory, '--source')
|
43
|
+
@source = directory
|
44
|
+
end
|
45
|
+
|
46
|
+
on('-d', '--destination DIRECTORY', String, DESTINATION_DESCRIPTION) do |directory|
|
47
|
+
assert_directory_exists(directory, '--destination')
|
48
|
+
@destination = directory
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
#----------------------------------------------------------------------------#
|
53
|
+
|
54
|
+
def validate_params()
|
55
|
+
raise MissingMandatory.new('--manifest') unless @manifest_path
|
56
|
+
raise MissingMandatory.new('--privatekey') unless @user_pk_path
|
57
|
+
end
|
58
|
+
|
59
|
+
#----------------------------------------------------------------------------#
|
60
|
+
|
61
|
+
def set_defaults()
|
62
|
+
@source ||= Dir::pwd()
|
63
|
+
@destination ||= Dir::pwd()
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,361 @@
|
|
1
|
+
# Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
|
2
|
+
# Reserved. Licensed under the Amazon Software License (the
|
3
|
+
# "License"). You may not use this file except in compliance with the
|
4
|
+
# License. A copy of the License is located at
|
5
|
+
# http://aws.amazon.com/asl or in the "license" file accompanying this
|
6
|
+
# file. This file is distributed on an "AS IS" BASIS, WITHOUT
|
7
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
|
8
|
+
# the License for the specific language governing permissions and
|
9
|
+
# limitations under the License.
|
10
|
+
|
11
|
+
require 'ec2/common/s3support'
|
12
|
+
require 'ec2/amitools/uploadbundleparameters'
|
13
|
+
require 'uri'
|
14
|
+
require 'ec2/amitools/instance-data'
|
15
|
+
require 'ec2/amitools/manifestv20071010'
|
16
|
+
require 'rexml/document'
|
17
|
+
require 'digest/md5'
|
18
|
+
require 'base64'
|
19
|
+
require 'ec2/amitools/tool_base'
|
20
|
+
require 'ec2/amitools/region'
|
21
|
+
|
22
|
+
#------------------------------------------------------------------------------#
|
23
|
+
|
24
|
+
UPLOAD_BUNDLE_NAME = 'ec2-upload-bundle'
|
25
|
+
|
26
|
+
UPLOAD_BUNDLE_MANUAL =<<TEXT
|
27
|
+
#{UPLOAD_BUNDLE_NAME} is a command line tool to upload a bundled Amazon Image to S3 storage
|
28
|
+
for use by EC2. An Amazon Image may be one of the following:
|
29
|
+
- Amazon Machine Image (AMI)
|
30
|
+
- Amazon Kernel Image (AKI)
|
31
|
+
- Amazon Ramdisk Image (ARI)
|
32
|
+
|
33
|
+
#{UPLOAD_BUNDLE_NAME} will:
|
34
|
+
- create an S3 bucket to store the bundled AMI in if it does not already exist
|
35
|
+
- upload the AMI manifest and parts files to S3, granting specified privileges
|
36
|
+
- on them (defaults to EC2 read privileges)
|
37
|
+
|
38
|
+
To manually retry an upload that failed, #{UPLOAD_BUNDLE_NAME} can optionally:
|
39
|
+
- skip uploading the manifest
|
40
|
+
- only upload bundled AMI parts from a specified part onwards
|
41
|
+
TEXT
|
42
|
+
|
43
|
+
#------------------------------------------------------------------------------#
|
44
|
+
|
45
|
+
class BucketLocationError < AMIToolExceptions::EC2FatalError
|
46
|
+
def initialize(bucket, location, bucket_location)
|
47
|
+
location = "US" if location == :unconstrained
|
48
|
+
bucket_location = "US" if bucket_location == :unconstrained
|
49
|
+
super(10, "Bucket \"#{bucket}\" already exists in \"#{bucket_location}\" and \"#{location}\" was specified.")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
#----------------------------------------------------------------------------#
|
54
|
+
|
55
|
+
# Upload the specified file.
|
56
|
+
|
57
|
+
class BundleUploader < AMITool
|
58
|
+
|
59
|
+
def upload(s3_conn, bucket, key, file, acl, retry_upload)
|
60
|
+
retry_s3(retry_upload) do
|
61
|
+
begin
|
62
|
+
md5 = get_md5(file)
|
63
|
+
s3_conn.put(bucket, key, file, {"x-amz-acl"=>acl, "content-md5"=>md5})
|
64
|
+
return
|
65
|
+
rescue EC2::Common::HTTP::Error::PathInvalid => e
|
66
|
+
raise FileNotFound(file)
|
67
|
+
rescue => e
|
68
|
+
raise TryFailed.new("Failed to upload \"#{file}\": #{e.message}")
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
#----------------------------------------------------------------------------#
|
74
|
+
|
75
|
+
def get_md5(file)
|
76
|
+
Base64::encode64(Digest::MD5::digest(File.open(file) { |f| f.read })).strip
|
77
|
+
end
|
78
|
+
|
79
|
+
#----------------------------------------------------------------------------#
|
80
|
+
|
81
|
+
#
|
82
|
+
# Availability zone names are generally in the format => ${REGION}${ZONENUMBER}.
|
83
|
+
# Examples being us-east-1b, us-east-1c, etc.
|
84
|
+
#
|
85
|
+
def get_availability_zone()
|
86
|
+
instance_data = EC2::InstanceData.new
|
87
|
+
instance_data.availability_zone
|
88
|
+
end
|
89
|
+
|
90
|
+
#----------------------------------------------------------------------------#
|
91
|
+
|
92
|
+
# Return a list of bundle part filename and part number tuples from the manifest.
|
93
|
+
def get_part_info(manifest)
|
94
|
+
parts = manifest.ami_part_info_list.map do |part|
|
95
|
+
[part['filename'], part['index']]
|
96
|
+
end
|
97
|
+
parts.sort
|
98
|
+
end
|
99
|
+
|
100
|
+
#------------------------------------------------------------------------------#
|
101
|
+
|
102
|
+
def uri2string(uri)
|
103
|
+
s = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
|
104
|
+
# Remove the trailing '/'.
|
105
|
+
return (s[-1..-1] == "/" ? s[0..-2] : s)
|
106
|
+
end
|
107
|
+
|
108
|
+
#------------------------------------------------------------------------------#
|
109
|
+
|
110
|
+
# Get the bucket's location.
|
111
|
+
def get_bucket_location(s3_conn, bucket)
|
112
|
+
begin
|
113
|
+
response = s3_conn.get_bucket_location(bucket)
|
114
|
+
rescue EC2::Common::HTTP::Error::Retrieve => e
|
115
|
+
if e.code == 404
|
116
|
+
# We have a "Not found" S3 response, which probably means the bucket doesn't exist.
|
117
|
+
return nil
|
118
|
+
end
|
119
|
+
raise e
|
120
|
+
end
|
121
|
+
$stdout.puts "check_bucket_location response: #{response.body}" if @debug and response.text?
|
122
|
+
docroot = REXML::Document.new(response.body).root
|
123
|
+
bucket_location = REXML::XPath.first(docroot, '/LocationConstraint').text
|
124
|
+
bucket_location ||= :unconstrained
|
125
|
+
end
|
126
|
+
|
127
|
+
#------------------------------------------------------------------------------#
|
128
|
+
|
129
|
+
# Check if the bucket exists and is in an appropriate location.
|
130
|
+
def check_bucket_location(bucket, bucket_location, location)
|
131
|
+
if bucket_location.nil?
|
132
|
+
# The bucket does not exist. Safe, but we need to create it.
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
if location.nil?
|
136
|
+
# The bucket exists and we don't care where it is.
|
137
|
+
return true
|
138
|
+
end
|
139
|
+
unless [bucket_location, AwsRegion.guess_region_from_s3_bucket(bucket_location)].include?(location)
|
140
|
+
# The bucket isn't where we want it. This is a problem.
|
141
|
+
raise BucketLocationError.new(bucket, location, bucket_location)
|
142
|
+
end
|
143
|
+
# The bucket exists and is in the right place.
|
144
|
+
return true
|
145
|
+
end
|
146
|
+
|
147
|
+
#------------------------------------------------------------------------------#
|
148
|
+
|
149
|
+
# Create the specified bucket if it does not exist.
|
150
|
+
def create_bucket(s3_conn, bucket, bucket_location, location, retry_create)
|
151
|
+
begin
|
152
|
+
if check_bucket_location(bucket, bucket_location, location)
|
153
|
+
return true
|
154
|
+
end
|
155
|
+
$stdout.puts "Creating bucket..."
|
156
|
+
|
157
|
+
retry_s3(retry_create) do
|
158
|
+
error = "Could not create or access bucket #{bucket}"
|
159
|
+
begin
|
160
|
+
rsp = s3_conn.create_bucket(bucket, location == :unconstrained ? nil : location)
|
161
|
+
rescue EC2::Common::HTTP::Error::Retrieve => e
|
162
|
+
error += ": server response #{e.message} #{e.code}"
|
163
|
+
raise TryFailed.new(e.message)
|
164
|
+
rescue RuntimeError => e
|
165
|
+
error += ": error message #{e.message}"
|
166
|
+
raise e
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
#------------------------------------------------------------------------------#
|
173
|
+
|
174
|
+
# If we return true, we have a v2-compliant name.
|
175
|
+
# If we return false, we wish to use a bad name.
|
176
|
+
# Otherwise we quietly wander off to die in peace.
|
177
|
+
def check_bucket_name(bucket)
|
178
|
+
if EC2::Common::S3Support::bucket_name_s3_v2_safe?(bucket)
|
179
|
+
return true
|
180
|
+
end
|
181
|
+
message = "The specified bucket is not S3 v2 safe (see S3 documentation for details):\n#{bucket}"
|
182
|
+
if warn_confirm(message)
|
183
|
+
# Assume the customer knows what he's doing.
|
184
|
+
return false
|
185
|
+
else
|
186
|
+
# We've been asked to stop, so quietly wander off to die in peace.
|
187
|
+
raise EC2StopExecution.new()
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
# force v1 S3 addressing when using govcloud endpoint
|
192
|
+
def check_govcloud_override(s3_url)
|
193
|
+
if s3_url =~ /s3-us-gov-west-1/
|
194
|
+
false
|
195
|
+
else
|
196
|
+
true
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
#------------------------------------------------------------------------------#
|
201
|
+
|
202
|
+
def get_region()
|
203
|
+
zone = get_availability_zone()
|
204
|
+
if zone.nil?
|
205
|
+
return nil
|
206
|
+
end
|
207
|
+
# assume region names do not have a common naming scheme. Therefore we manually go through all known region names
|
208
|
+
AwsRegion.regions.each do |region|
|
209
|
+
match = zone.match(region)
|
210
|
+
if not match.nil?
|
211
|
+
return region
|
212
|
+
end
|
213
|
+
end
|
214
|
+
nil
|
215
|
+
end
|
216
|
+
|
217
|
+
# This is very much a best effort attempt. If in doubt, we don't warn.
|
218
|
+
def cross_region?(location, bucket_location)
|
219
|
+
|
220
|
+
# If the bucket exists, its S3 location is canonical.
|
221
|
+
s3_region = bucket_location
|
222
|
+
s3_region ||= location
|
223
|
+
s3_region ||= :unconstrained
|
224
|
+
|
225
|
+
region = get_region()
|
226
|
+
|
227
|
+
if region.nil?
|
228
|
+
# If we can't get the region, assume we're fine since there's
|
229
|
+
# nothing more we can do.
|
230
|
+
return false
|
231
|
+
end
|
232
|
+
|
233
|
+
return s3_region != AwsRegion.get_s3_location(region)
|
234
|
+
end
|
235
|
+
|
236
|
+
#------------------------------------------------------------------------------#
|
237
|
+
|
238
|
+
def warn_about_migrating()
|
239
|
+
message = ["You are bundling in one region, but uploading to another. If the kernel",
|
240
|
+
"or ramdisk associated with this AMI are not in the target region, AMI",
|
241
|
+
"registration will fail.",
|
242
|
+
"You can use the ec2-migrate-manifest tool to update your manifest file",
|
243
|
+
"with a kernel and ramdisk that exist in the target region.",
|
244
|
+
].join("\n")
|
245
|
+
unless warn_confirm(message)
|
246
|
+
raise EC2StopExecution.new()
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
#------------------------------------------------------------------------------#
|
251
|
+
|
252
|
+
def get_s3_conn(s3_url, user, pass, method, sigv, region=nil)
|
253
|
+
EC2::Common::S3Support.new(s3_url, user, pass, method, @debug, sigv, region)
|
254
|
+
end
|
255
|
+
|
256
|
+
#------------------------------------------------------------------------------#
|
257
|
+
|
258
|
+
#
|
259
|
+
# Get parameters and display help or manual if necessary.
|
260
|
+
#
|
261
|
+
def upload_bundle(url,
|
262
|
+
bucket,
|
263
|
+
keyprefix,
|
264
|
+
user,
|
265
|
+
pass,
|
266
|
+
location,
|
267
|
+
manifest_file,
|
268
|
+
retry_stuff,
|
269
|
+
part,
|
270
|
+
directory,
|
271
|
+
acl,
|
272
|
+
skipmanifest,
|
273
|
+
sigv,
|
274
|
+
region)
|
275
|
+
begin
|
276
|
+
# Get the S3 URL.
|
277
|
+
s3_uri = URI.parse(url)
|
278
|
+
s3_url = uri2string(s3_uri)
|
279
|
+
v2_bucket = check_bucket_name(bucket) and check_govcloud_override(s3_url)
|
280
|
+
s3_conn = get_s3_conn(s3_url, user, pass, (v2_bucket ? nil : :path), sigv, region)
|
281
|
+
|
282
|
+
# Get current location and bucket location.
|
283
|
+
bucket_location = get_bucket_location(s3_conn, bucket)
|
284
|
+
|
285
|
+
# Load manifest.
|
286
|
+
xml = File.open(manifest_file) { |f| f.read }
|
287
|
+
manifest = ManifestV20071010.new(xml)
|
288
|
+
|
289
|
+
# If in interactive mode, warn when bundling a kernel into our AMI and we are uploading cross-region
|
290
|
+
if interactive? and manifest.kernel_id and cross_region?(location, bucket_location)
|
291
|
+
warn_about_migrating()
|
292
|
+
end
|
293
|
+
|
294
|
+
# Create storage bucket if required.
|
295
|
+
create_bucket(s3_conn, bucket, bucket_location, location, retry_stuff)
|
296
|
+
|
297
|
+
# Upload AMI bundle parts.
|
298
|
+
$stdout.puts "Uploading bundled image parts to the S3 bucket #{bucket} ..."
|
299
|
+
get_part_info(manifest).each do |part_info|
|
300
|
+
if part.nil? or (part_info[1] >= part)
|
301
|
+
path = File.join(directory, part_info[0])
|
302
|
+
upload(s3_conn, bucket, keyprefix + part_info[0], path, acl, retry_stuff)
|
303
|
+
$stdout.puts "Uploaded #{part_info[0]}"
|
304
|
+
else
|
305
|
+
$stdout.puts "Skipping #{part_info[0]}"
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
# Encrypt and upload manifest.
|
310
|
+
unless skipmanifest
|
311
|
+
$stdout.puts "Uploading manifest ..."
|
312
|
+
upload(s3_conn, bucket, keyprefix + File::basename(manifest_file), manifest_file, acl, retry_stuff)
|
313
|
+
$stdout.puts "Uploaded manifest."
|
314
|
+
$stdout.puts 'Manifest uploaded to: %s/%s' % [bucket, keyprefix + File::basename(manifest_file)]
|
315
|
+
else
|
316
|
+
$stdout.puts "Skipping manifest."
|
317
|
+
end
|
318
|
+
|
319
|
+
$stdout.puts 'Bundle upload completed.'
|
320
|
+
rescue EC2::Common::HTTP::Error => e
|
321
|
+
$stderr.puts e.backtrace if @debug
|
322
|
+
raise S3Error.new(e.message)
|
323
|
+
end
|
324
|
+
end
|
325
|
+
|
326
|
+
#------------------------------------------------------------------------------#
|
327
|
+
# Overrides
|
328
|
+
#------------------------------------------------------------------------------#
|
329
|
+
|
330
|
+
def get_manual()
|
331
|
+
UPLOAD_BUNDLE_MANUAL
|
332
|
+
end
|
333
|
+
|
334
|
+
def get_name()
|
335
|
+
UPLOAD_BUNDLE_NAME
|
336
|
+
end
|
337
|
+
|
338
|
+
def main(p)
|
339
|
+
upload_bundle(p.url,
|
340
|
+
p.bucket,
|
341
|
+
p.keyprefix,
|
342
|
+
p.user,
|
343
|
+
p.pass,
|
344
|
+
p.location,
|
345
|
+
p.manifest,
|
346
|
+
p.retry,
|
347
|
+
p.part,
|
348
|
+
p.directory,
|
349
|
+
p.acl,
|
350
|
+
p.skipmanifest,
|
351
|
+
p.sigv,
|
352
|
+
p.region)
|
353
|
+
end
|
354
|
+
|
355
|
+
end
|
356
|
+
|
357
|
+
#------------------------------------------------------------------------------#
|
358
|
+
# Script entry point. Execute only if this file is being executed.
|
359
|
+
if __FILE__ == $0 || $0.match(/bin\/ec2-upload-bundle/)
|
360
|
+
BundleUploader.new().run(UploadBundleParameters)
|
361
|
+
end
|