ec2_amitools 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +54 -0
  3. data/bin/console +14 -0
  4. data/bin/ec2-ami-tools-version +6 -0
  5. data/bin/ec2-bundle-image +6 -0
  6. data/bin/ec2-bundle-vol +6 -0
  7. data/bin/ec2-delete-bundle +6 -0
  8. data/bin/ec2-download-bundle +6 -0
  9. data/bin/ec2-migrate-bundle +6 -0
  10. data/bin/ec2-migrate-manifest +6 -0
  11. data/bin/ec2-unbundle +6 -0
  12. data/bin/ec2-upload-bundle +6 -0
  13. data/bin/setup +8 -0
  14. data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
  15. data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
  16. data/etc/ec2/amitools/cert-ec2.pem +23 -0
  17. data/etc/ec2/amitools/mappings.csv +9 -0
  18. data/lib/ec2/amitools/bundle.rb +251 -0
  19. data/lib/ec2/amitools/bundle_base.rb +58 -0
  20. data/lib/ec2/amitools/bundleimage.rb +94 -0
  21. data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
  22. data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
  23. data/lib/ec2/amitools/bundleparameters.rb +120 -0
  24. data/lib/ec2/amitools/bundlevol.rb +240 -0
  25. data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
  26. data/lib/ec2/amitools/crypto.rb +379 -0
  27. data/lib/ec2/amitools/decryptmanifest.rb +20 -0
  28. data/lib/ec2/amitools/defaults.rb +12 -0
  29. data/lib/ec2/amitools/deletebundle.rb +212 -0
  30. data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
  31. data/lib/ec2/amitools/downloadbundle.rb +161 -0
  32. data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
  33. data/lib/ec2/amitools/exception.rb +86 -0
  34. data/lib/ec2/amitools/fileutil.rb +219 -0
  35. data/lib/ec2/amitools/format.rb +127 -0
  36. data/lib/ec2/amitools/instance-data.rb +97 -0
  37. data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
  38. data/lib/ec2/amitools/manifestv20070829.rb +361 -0
  39. data/lib/ec2/amitools/manifestv20071010.rb +403 -0
  40. data/lib/ec2/amitools/manifestv3.rb +331 -0
  41. data/lib/ec2/amitools/mapids.rb +148 -0
  42. data/lib/ec2/amitools/migratebundle.rb +222 -0
  43. data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
  44. data/lib/ec2/amitools/migratemanifest.rb +225 -0
  45. data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
  46. data/lib/ec2/amitools/minimalec2.rb +116 -0
  47. data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
  48. data/lib/ec2/amitools/parameters_base.rb +168 -0
  49. data/lib/ec2/amitools/region.rb +93 -0
  50. data/lib/ec2/amitools/s3toolparameters.rb +183 -0
  51. data/lib/ec2/amitools/showversion.rb +12 -0
  52. data/lib/ec2/amitools/syschecks.rb +27 -0
  53. data/lib/ec2/amitools/tool_base.rb +224 -0
  54. data/lib/ec2/amitools/unbundle.rb +107 -0
  55. data/lib/ec2/amitools/unbundleparameters.rb +65 -0
  56. data/lib/ec2/amitools/uploadbundle.rb +361 -0
  57. data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
  58. data/lib/ec2/amitools/util.rb +532 -0
  59. data/lib/ec2/amitools/version.rb +33 -0
  60. data/lib/ec2/amitools/xmlbuilder.rb +237 -0
  61. data/lib/ec2/amitools/xmlutil.rb +55 -0
  62. data/lib/ec2/common/constants.rb +16 -0
  63. data/lib/ec2/common/curl.rb +110 -0
  64. data/lib/ec2/common/headers.rb +95 -0
  65. data/lib/ec2/common/headersv4.rb +173 -0
  66. data/lib/ec2/common/http.rb +333 -0
  67. data/lib/ec2/common/s3support.rb +231 -0
  68. data/lib/ec2/common/signature.rb +68 -0
  69. data/lib/ec2/oem/LICENSE.txt +58 -0
  70. data/lib/ec2/oem/open4.rb +399 -0
  71. data/lib/ec2/platform/base/architecture.rb +26 -0
  72. data/lib/ec2/platform/base/constants.rb +54 -0
  73. data/lib/ec2/platform/base/pipeline.rb +181 -0
  74. data/lib/ec2/platform/base.rb +57 -0
  75. data/lib/ec2/platform/current.rb +55 -0
  76. data/lib/ec2/platform/linux/architecture.rb +35 -0
  77. data/lib/ec2/platform/linux/constants.rb +23 -0
  78. data/lib/ec2/platform/linux/fstab.rb +99 -0
  79. data/lib/ec2/platform/linux/identity.rb +16 -0
  80. data/lib/ec2/platform/linux/image.rb +811 -0
  81. data/lib/ec2/platform/linux/mtab.rb +74 -0
  82. data/lib/ec2/platform/linux/pipeline.rb +40 -0
  83. data/lib/ec2/platform/linux/rsync.rb +114 -0
  84. data/lib/ec2/platform/linux/tar.rb +124 -0
  85. data/lib/ec2/platform/linux/uname.rb +50 -0
  86. data/lib/ec2/platform/linux.rb +83 -0
  87. data/lib/ec2/platform/solaris/architecture.rb +28 -0
  88. data/lib/ec2/platform/solaris/constants.rb +30 -0
  89. data/lib/ec2/platform/solaris/fstab.rb +43 -0
  90. data/lib/ec2/platform/solaris/identity.rb +16 -0
  91. data/lib/ec2/platform/solaris/image.rb +327 -0
  92. data/lib/ec2/platform/solaris/mtab.rb +29 -0
  93. data/lib/ec2/platform/solaris/pipeline.rb +40 -0
  94. data/lib/ec2/platform/solaris/rsync.rb +24 -0
  95. data/lib/ec2/platform/solaris/tar.rb +36 -0
  96. data/lib/ec2/platform/solaris/uname.rb +21 -0
  97. data/lib/ec2/platform/solaris.rb +38 -0
  98. data/lib/ec2/platform.rb +69 -0
  99. data/lib/ec2/version.rb +8 -0
  100. data/lib/ec2_amitools +1 -0
  101. data/lib/ec2_amitools.rb +7 -0
  102. metadata +184 -0
@@ -0,0 +1,12 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/version'
12
+ puts EC2Version::version_copyright_string()
@@ -0,0 +1,27 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/fileutil'
12
+ require 'ec2/platform/current'
13
+
14
+ module SysChecks
15
+ def self.rsync_usable?()
16
+ EC2::Platform::Current::Rsync.usable?
17
+ end
18
+ def self.good_tar_version?()
19
+ EC2::Platform::Current::Tar::Version.current.usable?
20
+ end
21
+ def self.get_system_arch()
22
+ EC2::Platform::Current::System::BUNDLING_ARCHITECTURE
23
+ end
24
+ def self.root_user?()
25
+ EC2::Platform::Current::System.superuser?
26
+ end
27
+ end
@@ -0,0 +1,224 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ module AMIToolExceptions
12
+ # All fatal errors should inherit from this.
13
+ class EC2FatalError < RuntimeError
14
+ attr_accessor :code
15
+ def initialize(code, msg)
16
+ super(msg)
17
+ @code = code
18
+ end
19
+ end
20
+
21
+ class FileNotFound < EC2FatalError
22
+ def initialize(path)
23
+ super(2, "File not found: #{path}")
24
+ end
25
+ end
26
+
27
+ class S3Error < EC2FatalError
28
+ def initialize(msg)
29
+ super(3, "Error talking to S3: #{msg}")
30
+ end
31
+ end
32
+
33
+ class PromptTimeout < EC2FatalError
34
+ def initialize(msg=nil)
35
+ message = "Timed out waiting for user input"
36
+ message += ": #{msg}" unless msg.nil?
37
+ super(5, message)
38
+ end
39
+ end
40
+
41
+ # This is more for flow control than anything else.
42
+ # Raising it should terminate execution, but not print an error.
43
+ class EC2StopExecution < RuntimeError
44
+ attr_accessor :code
45
+ def initialize(code=0)
46
+ super()
47
+ @code = code
48
+ end
49
+ end
50
+
51
+ class TryFailed < RuntimeError
52
+ end
53
+
54
+ end
55
+
56
+ class AMITool
57
+
58
+ include AMIToolExceptions
59
+
60
+ PROMPT_TIMEOUT = 30
61
+ MAX_TRIES = 5
62
+ BACKOFF_PERIOD = 5
63
+
64
+ #------------------------------------------------------------------------------#
65
+ # Methods to override in subclasses
66
+ #------------------------------------------------------------------------------#
67
+
68
+ def get_manual()
69
+ # We have to get the manual text into here.
70
+ raise "NotImplemented: get_manual()"
71
+ end
72
+
73
+ def get_name()
74
+ # We have to get the tool name into here.
75
+ raise "NotImplemented: get_name()"
76
+ end
77
+
78
+ def main(params)
79
+ # Main entry point.
80
+ raise "NotImplemented: main()"
81
+ end
82
+
83
+ #------------------------------------------------------------------------------#
84
+ # Utility methods
85
+ #------------------------------------------------------------------------------#
86
+
87
+ # Display a message (without appending a newline) and ask for a response.
88
+ # Returns user response or nil if interactivity is not desired.
89
+ # Raises exception on timeout.
90
+ def interactive_prompt(message, name=nil)
91
+ return nil unless interactive?
92
+ begin
93
+ $stdout.print(message)
94
+ $stdout.flush
95
+ Timeout::timeout(PROMPT_TIMEOUT) do
96
+ return gets
97
+ end
98
+ rescue Timeout::Error
99
+ raise PromptTimeout.new(name)
100
+ end
101
+ end
102
+
103
+ #------------------------------------------------------------------------------#
104
+
105
+ # Display a message on stderr.
106
+ # If interactive, asks for confirmation (yes/no).
107
+ # Returns true if in batch mode or user agrees, false if user disagrees.
108
+ # Raises exception on timeout.
109
+ def warn_confirm(message)
110
+ $stderr.puts(message)
111
+ $stderr.flush
112
+ return true unless interactive?
113
+ response = interactive_prompt("Are you sure you want to continue? [y/N]")
114
+ if response =~ /^[Yy]/
115
+ return true
116
+ end
117
+ return false
118
+ end
119
+
120
+ #----------------------------------------------------------------------------#
121
+
122
+ def retry_s3(retrying=true)
123
+ tries = 0
124
+ while true
125
+ tries += 1
126
+ begin
127
+ result = yield
128
+ return result
129
+ rescue TryFailed => e
130
+ $stderr.puts e.message
131
+ if retrying and tries < MAX_TRIES
132
+ $stdout.puts "Retrying in #{BACKOFF_PERIOD}s ..."
133
+ else
134
+ raise EC2FatalError.new(3, e.message)
135
+ end
136
+ end
137
+ end
138
+ end
139
+
140
+ #------------------------------------------------------------------------------#
141
+ # Standard behaviour
142
+ #------------------------------------------------------------------------------#
143
+
144
+ def handle_early_exit_parameters(params)
145
+ if params.version
146
+ puts get_name() + " " + params.version_copyright_string()
147
+ return
148
+ end
149
+
150
+ if params.show_help
151
+ puts params.help
152
+ return
153
+ end
154
+
155
+ if params.manual
156
+ puts get_manual()
157
+ return
158
+ end
159
+ end
160
+
161
+ #------------------------------------------------------------------------------#
162
+
163
+ def interactive?
164
+ @interactive
165
+ end
166
+
167
+ #------------------------------------------------------------------------------#
168
+
169
+ def get_parameters(params_class)
170
+ # Parse the parameters and die on errors.
171
+ # Assume that if we're parsing parameters, it's safe to exit.
172
+ begin
173
+ params = params_class.new(ARGV)
174
+ rescue StandardError => e
175
+ $stderr.puts e.message
176
+ $stderr.puts "Try '#{get_name} --help'"
177
+ exit 1
178
+ end
179
+
180
+ # Deal with help, verion, etc.
181
+ if params.early_exit?
182
+ handle_early_exit_parameters(params)
183
+ exit 0
184
+ end
185
+
186
+ # Some general flags that we want to set
187
+ @debug = params.debug
188
+ @interactive = params.interactive?
189
+
190
+ # Finally, return the leftovers.
191
+ params
192
+ end
193
+
194
+ #------------------------------------------------------------------------------#
195
+
196
+ def run(params_class)
197
+ # We want to be able to reuse bits without having to parse
198
+ # parameters, so run() is not called from the constructor.
199
+ begin
200
+ params = get_parameters(params_class)
201
+ main(params)
202
+ rescue AMIToolExceptions::EC2StopExecution => e
203
+ # We've been asked to stop.
204
+ exit e.code
205
+ rescue AMIToolExceptions::PromptTimeout => e
206
+ $stderr.puts e.message
207
+ exit e.code
208
+ rescue AMIToolExceptions::EC2FatalError => e
209
+ $stderr.puts "ERROR: #{e.message}"
210
+ puts e.backtrace if @debug
211
+ exit e.code
212
+ rescue Interrupt => e
213
+ $stderr.puts "\n#{get_name} interrupted."
214
+ puts e.backtrace if @debug
215
+ exit 255
216
+ rescue => e
217
+ $stderr.puts "ERROR: #{e.message}"
218
+ puts e.inspect if @debug
219
+ puts e.backtrace if @debug
220
+ exit 254
221
+ end
222
+ end
223
+
224
+ end
@@ -0,0 +1,107 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/format'
12
+ require 'ec2/amitools/manifestv3'
13
+ require 'ec2/amitools/unbundleparameters'
14
+ require 'ec2/platform/current'
15
+ require 'ec2/amitools/tool_base'
16
+
17
+ UNBUNDLE_NAME = 'ec2-unbundle'
18
+
19
+ UNBUNDLE_MANUAL =<<TEXT
20
+ #{UNBUNDLE_NAME} extracts a filesystem image from a bundle AMI.
21
+
22
+ #{UNBUNDLE_NAME} will:
23
+ - read relevant information from the manifest file
24
+ - concatenate all the parts
25
+ - decrypt and uncompress the image
26
+ TEXT
27
+
28
+ class Unbundler < AMITool
29
+
30
+ def unbundle(p)
31
+ begin
32
+ manifest_path = p.manifest_path
33
+ src_dir = p.source
34
+ dst_dir = p.destination
35
+
36
+ digest_pipe = File::join( '/tmp', "ec2-unbundle-image-digest-pipe" )
37
+ File::delete( digest_pipe ) if File::exist?( digest_pipe )
38
+ unless system( "mkfifo #{digest_pipe}" )
39
+ raise "error creating named pipe #{digest_pipe}"
40
+ end
41
+
42
+ # Load manifest and the user's private key.
43
+ manifest = ManifestV3.new(File.open( manifest_path ) { |f| f.read() })
44
+ pk = Crypto::loadprivkey( p.user_pk_path )
45
+
46
+ # Extract key and IV from XML manifest.
47
+ key = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_key))
48
+ iv = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_iv))
49
+
50
+ # Create a string of space separated part paths.
51
+ part_files = manifest.parts.collect do |part|
52
+ File::join( src_dir, part.filename )
53
+ end.join( ' ' )
54
+
55
+ # Join, decrypt, decompress and untar.
56
+ untar = EC2::Platform::Current::Tar::Command.new.extract.chdir(dst_dir)
57
+ pipeline = EC2::Platform::Current::Pipeline.new('image-unbundle-pipeline', @debug)
58
+ pipeline.concat([
59
+ ['cat', "openssl sha1 < #{digest_pipe} & cat #{part_files}"],
60
+ ['decrypt', "openssl enc -d -aes-128-cbc -K #{key} -iv #{iv}"],
61
+ ['gunzip', "gunzip"],
62
+ ['tee', "tee #{digest_pipe}"],
63
+ ['untar', untar.expand]
64
+ ])
65
+ digest = nil
66
+ begin
67
+ digest = pipeline.execute.split(/\s+/).last.strip
68
+ rescue EC2::Platform::Current::Pipeline::ExecutionError => e
69
+ $stderr.puts e.message
70
+ end
71
+
72
+ # Verify digest.
73
+ unless manifest.digest == digest
74
+ raise "invalid digest, expected #{manifest.digest} received #{digest}"
75
+ end
76
+
77
+ puts "Unbundle complete."
78
+ return 0
79
+ ensure
80
+ File::delete( digest_pipe ) if (digest_pipe && File::exist?( digest_pipe ))
81
+ end
82
+ end
83
+
84
+ #------------------------------------------------------------------------------#
85
+ # Overrides
86
+ #------------------------------------------------------------------------------#
87
+
88
+ def get_manual()
89
+ UNBUNDLE_MANUAL
90
+ end
91
+
92
+ def get_name()
93
+ UNBUNDLE_NAME
94
+ end
95
+
96
+ def main(p)
97
+ unbundle(p)
98
+ end
99
+
100
+ end
101
+
102
+ #------------------------------------------------------------------------------#
103
+ # Script entry point. Execute only if this file is being executed.
104
+
105
+ if __FILE__ == $0 || $0.match(/bin\/ec2-unbundle/)
106
+ Unbundler.new().run(UnbundleParameters)
107
+ end
@@ -0,0 +1,65 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/parameters_base'
12
+
13
+ class UnbundleParameters < ParametersBase
14
+
15
+ MANIFEST_DESCRIPTION = "The path to the AMI manifest file."
16
+ SOURCE_DESCRIPTION = 'The directory containing bundled AMI parts to unbundle. Defaults to ".".'
17
+ DESTINATION_DESCRIPTION = 'The directory to unbundle the AMI into. Defaults to the ".".'
18
+
19
+ attr_accessor :manifest_path,
20
+ :user_pk_path,
21
+ :source,
22
+ :destination
23
+
24
+ #----------------------------------------------------------------------------#
25
+
26
+ def mandatory_params()
27
+ on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
28
+ assert_file_exists(path, '--privatekey')
29
+ @user_pk_path = path
30
+ end
31
+
32
+ on('-m', '--manifest PATH', String, MANIFEST_DESCRIPTION) do |manifest|
33
+ assert_file_exists(manifest, '--manifest')
34
+ @manifest_path = manifest
35
+ end
36
+ end
37
+
38
+ #----------------------------------------------------------------------------#
39
+
40
+ def optional_params()
41
+ on('-s', '--source DIRECTORY', String, SOURCE_DESCRIPTION) do |directory|
42
+ assert_directory_exists(directory, '--source')
43
+ @source = directory
44
+ end
45
+
46
+ on('-d', '--destination DIRECTORY', String, DESTINATION_DESCRIPTION) do |directory|
47
+ assert_directory_exists(directory, '--destination')
48
+ @destination = directory
49
+ end
50
+ end
51
+
52
+ #----------------------------------------------------------------------------#
53
+
54
+ def validate_params()
55
+ raise MissingMandatory.new('--manifest') unless @manifest_path
56
+ raise MissingMandatory.new('--privatekey') unless @user_pk_path
57
+ end
58
+
59
+ #----------------------------------------------------------------------------#
60
+
61
+ def set_defaults()
62
+ @source ||= Dir::pwd()
63
+ @destination ||= Dir::pwd()
64
+ end
65
+ end
@@ -0,0 +1,361 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/common/s3support'
12
+ require 'ec2/amitools/uploadbundleparameters'
13
+ require 'uri'
14
+ require 'ec2/amitools/instance-data'
15
+ require 'ec2/amitools/manifestv20071010'
16
+ require 'rexml/document'
17
+ require 'digest/md5'
18
+ require 'base64'
19
+ require 'ec2/amitools/tool_base'
20
+ require 'ec2/amitools/region'
21
+
22
+ #------------------------------------------------------------------------------#
23
+
24
+ UPLOAD_BUNDLE_NAME = 'ec2-upload-bundle'
25
+
26
+ UPLOAD_BUNDLE_MANUAL =<<TEXT
27
+ #{UPLOAD_BUNDLE_NAME} is a command line tool to upload a bundled Amazon Image to S3 storage
28
+ for use by EC2. An Amazon Image may be one of the following:
29
+ - Amazon Machine Image (AMI)
30
+ - Amazon Kernel Image (AKI)
31
+ - Amazon Ramdisk Image (ARI)
32
+
33
+ #{UPLOAD_BUNDLE_NAME} will:
34
+ - create an S3 bucket to store the bundled AMI in if it does not already exist
35
+ - upload the AMI manifest and parts files to S3, granting specified privileges
36
+ - on them (defaults to EC2 read privileges)
37
+
38
+ To manually retry an upload that failed, #{UPLOAD_BUNDLE_NAME} can optionally:
39
+ - skip uploading the manifest
40
+ - only upload bundled AMI parts from a specified part onwards
41
+ TEXT
42
+
43
+ #------------------------------------------------------------------------------#
44
+
45
+ class BucketLocationError < AMIToolExceptions::EC2FatalError
46
+ def initialize(bucket, location, bucket_location)
47
+ location = "US" if location == :unconstrained
48
+ bucket_location = "US" if bucket_location == :unconstrained
49
+ super(10, "Bucket \"#{bucket}\" already exists in \"#{bucket_location}\" and \"#{location}\" was specified.")
50
+ end
51
+ end
52
+
53
+ #----------------------------------------------------------------------------#
54
+
55
+ # Upload the specified file.
56
+
57
+ class BundleUploader < AMITool
58
+
59
+ def upload(s3_conn, bucket, key, file, acl, retry_upload)
60
+ retry_s3(retry_upload) do
61
+ begin
62
+ md5 = get_md5(file)
63
+ s3_conn.put(bucket, key, file, {"x-amz-acl"=>acl, "content-md5"=>md5})
64
+ return
65
+ rescue EC2::Common::HTTP::Error::PathInvalid => e
66
+ raise FileNotFound(file)
67
+ rescue => e
68
+ raise TryFailed.new("Failed to upload \"#{file}\": #{e.message}")
69
+ end
70
+ end
71
+ end
72
+
73
+ #----------------------------------------------------------------------------#
74
+
75
+ def get_md5(file)
76
+ Base64::encode64(Digest::MD5::digest(File.open(file) { |f| f.read })).strip
77
+ end
78
+
79
+ #----------------------------------------------------------------------------#
80
+
81
+ #
82
+ # Availability zone names are generally in the format => ${REGION}${ZONENUMBER}.
83
+ # Examples being us-east-1b, us-east-1c, etc.
84
+ #
85
+ def get_availability_zone()
86
+ instance_data = EC2::InstanceData.new
87
+ instance_data.availability_zone
88
+ end
89
+
90
+ #----------------------------------------------------------------------------#
91
+
92
+ # Return a list of bundle part filename and part number tuples from the manifest.
93
+ def get_part_info(manifest)
94
+ parts = manifest.ami_part_info_list.map do |part|
95
+ [part['filename'], part['index']]
96
+ end
97
+ parts.sort
98
+ end
99
+
100
+ #------------------------------------------------------------------------------#
101
+
102
+ def uri2string(uri)
103
+ s = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
104
+ # Remove the trailing '/'.
105
+ return (s[-1..-1] == "/" ? s[0..-2] : s)
106
+ end
107
+
108
+ #------------------------------------------------------------------------------#
109
+
110
+ # Get the bucket's location.
111
+ def get_bucket_location(s3_conn, bucket)
112
+ begin
113
+ response = s3_conn.get_bucket_location(bucket)
114
+ rescue EC2::Common::HTTP::Error::Retrieve => e
115
+ if e.code == 404
116
+ # We have a "Not found" S3 response, which probably means the bucket doesn't exist.
117
+ return nil
118
+ end
119
+ raise e
120
+ end
121
+ $stdout.puts "check_bucket_location response: #{response.body}" if @debug and response.text?
122
+ docroot = REXML::Document.new(response.body).root
123
+ bucket_location = REXML::XPath.first(docroot, '/LocationConstraint').text
124
+ bucket_location ||= :unconstrained
125
+ end
126
+
127
+ #------------------------------------------------------------------------------#
128
+
129
+ # Check if the bucket exists and is in an appropriate location.
130
+ def check_bucket_location(bucket, bucket_location, location)
131
+ if bucket_location.nil?
132
+ # The bucket does not exist. Safe, but we need to create it.
133
+ return false
134
+ end
135
+ if location.nil?
136
+ # The bucket exists and we don't care where it is.
137
+ return true
138
+ end
139
+ unless [bucket_location, AwsRegion.guess_region_from_s3_bucket(bucket_location)].include?(location)
140
+ # The bucket isn't where we want it. This is a problem.
141
+ raise BucketLocationError.new(bucket, location, bucket_location)
142
+ end
143
+ # The bucket exists and is in the right place.
144
+ return true
145
+ end
146
+
147
+ #------------------------------------------------------------------------------#
148
+
149
+ # Create the specified bucket if it does not exist.
150
+ def create_bucket(s3_conn, bucket, bucket_location, location, retry_create)
151
+ begin
152
+ if check_bucket_location(bucket, bucket_location, location)
153
+ return true
154
+ end
155
+ $stdout.puts "Creating bucket..."
156
+
157
+ retry_s3(retry_create) do
158
+ error = "Could not create or access bucket #{bucket}"
159
+ begin
160
+ rsp = s3_conn.create_bucket(bucket, location == :unconstrained ? nil : location)
161
+ rescue EC2::Common::HTTP::Error::Retrieve => e
162
+ error += ": server response #{e.message} #{e.code}"
163
+ raise TryFailed.new(e.message)
164
+ rescue RuntimeError => e
165
+ error += ": error message #{e.message}"
166
+ raise e
167
+ end
168
+ end
169
+ end
170
+ end
171
+
172
+ #------------------------------------------------------------------------------#
173
+
174
+ # If we return true, we have a v2-compliant name.
175
+ # If we return false, we wish to use a bad name.
176
+ # Otherwise we quietly wander off to die in peace.
177
+ def check_bucket_name(bucket)
178
+ if EC2::Common::S3Support::bucket_name_s3_v2_safe?(bucket)
179
+ return true
180
+ end
181
+ message = "The specified bucket is not S3 v2 safe (see S3 documentation for details):\n#{bucket}"
182
+ if warn_confirm(message)
183
+ # Assume the customer knows what he's doing.
184
+ return false
185
+ else
186
+ # We've been asked to stop, so quietly wander off to die in peace.
187
+ raise EC2StopExecution.new()
188
+ end
189
+ end
190
+
191
+ # force v1 S3 addressing when using govcloud endpoint
192
+ def check_govcloud_override(s3_url)
193
+ if s3_url =~ /s3-us-gov-west-1/
194
+ false
195
+ else
196
+ true
197
+ end
198
+ end
199
+
200
+ #------------------------------------------------------------------------------#
201
+
202
+ def get_region()
203
+ zone = get_availability_zone()
204
+ if zone.nil?
205
+ return nil
206
+ end
207
+ # assume region names do not have a common naming scheme. Therefore we manually go through all known region names
208
+ AwsRegion.regions.each do |region|
209
+ match = zone.match(region)
210
+ if not match.nil?
211
+ return region
212
+ end
213
+ end
214
+ nil
215
+ end
216
+
217
+ # This is very much a best effort attempt. If in doubt, we don't warn.
218
+ def cross_region?(location, bucket_location)
219
+
220
+ # If the bucket exists, its S3 location is canonical.
221
+ s3_region = bucket_location
222
+ s3_region ||= location
223
+ s3_region ||= :unconstrained
224
+
225
+ region = get_region()
226
+
227
+ if region.nil?
228
+ # If we can't get the region, assume we're fine since there's
229
+ # nothing more we can do.
230
+ return false
231
+ end
232
+
233
+ return s3_region != AwsRegion.get_s3_location(region)
234
+ end
235
+
236
+ #------------------------------------------------------------------------------#
237
+
238
+ def warn_about_migrating()
239
+ message = ["You are bundling in one region, but uploading to another. If the kernel",
240
+ "or ramdisk associated with this AMI are not in the target region, AMI",
241
+ "registration will fail.",
242
+ "You can use the ec2-migrate-manifest tool to update your manifest file",
243
+ "with a kernel and ramdisk that exist in the target region.",
244
+ ].join("\n")
245
+ unless warn_confirm(message)
246
+ raise EC2StopExecution.new()
247
+ end
248
+ end
249
+
250
+ #------------------------------------------------------------------------------#
251
+
252
+ def get_s3_conn(s3_url, user, pass, method, sigv, region=nil)
253
+ EC2::Common::S3Support.new(s3_url, user, pass, method, @debug, sigv, region)
254
+ end
255
+
256
+ #------------------------------------------------------------------------------#
257
+
258
+ #
259
+ # Get parameters and display help or manual if necessary.
260
+ #
261
+ def upload_bundle(url,
262
+ bucket,
263
+ keyprefix,
264
+ user,
265
+ pass,
266
+ location,
267
+ manifest_file,
268
+ retry_stuff,
269
+ part,
270
+ directory,
271
+ acl,
272
+ skipmanifest,
273
+ sigv,
274
+ region)
275
+ begin
276
+ # Get the S3 URL.
277
+ s3_uri = URI.parse(url)
278
+ s3_url = uri2string(s3_uri)
279
+ v2_bucket = check_bucket_name(bucket) and check_govcloud_override(s3_url)
280
+ s3_conn = get_s3_conn(s3_url, user, pass, (v2_bucket ? nil : :path), sigv, region)
281
+
282
+ # Get current location and bucket location.
283
+ bucket_location = get_bucket_location(s3_conn, bucket)
284
+
285
+ # Load manifest.
286
+ xml = File.open(manifest_file) { |f| f.read }
287
+ manifest = ManifestV20071010.new(xml)
288
+
289
+ # If in interactive mode, warn when bundling a kernel into our AMI and we are uploading cross-region
290
+ if interactive? and manifest.kernel_id and cross_region?(location, bucket_location)
291
+ warn_about_migrating()
292
+ end
293
+
294
+ # Create storage bucket if required.
295
+ create_bucket(s3_conn, bucket, bucket_location, location, retry_stuff)
296
+
297
+ # Upload AMI bundle parts.
298
+ $stdout.puts "Uploading bundled image parts to the S3 bucket #{bucket} ..."
299
+ get_part_info(manifest).each do |part_info|
300
+ if part.nil? or (part_info[1] >= part)
301
+ path = File.join(directory, part_info[0])
302
+ upload(s3_conn, bucket, keyprefix + part_info[0], path, acl, retry_stuff)
303
+ $stdout.puts "Uploaded #{part_info[0]}"
304
+ else
305
+ $stdout.puts "Skipping #{part_info[0]}"
306
+ end
307
+ end
308
+
309
+ # Encrypt and upload manifest.
310
+ unless skipmanifest
311
+ $stdout.puts "Uploading manifest ..."
312
+ upload(s3_conn, bucket, keyprefix + File::basename(manifest_file), manifest_file, acl, retry_stuff)
313
+ $stdout.puts "Uploaded manifest."
314
+ $stdout.puts 'Manifest uploaded to: %s/%s' % [bucket, keyprefix + File::basename(manifest_file)]
315
+ else
316
+ $stdout.puts "Skipping manifest."
317
+ end
318
+
319
+ $stdout.puts 'Bundle upload completed.'
320
+ rescue EC2::Common::HTTP::Error => e
321
+ $stderr.puts e.backtrace if @debug
322
+ raise S3Error.new(e.message)
323
+ end
324
+ end
325
+
326
+ #------------------------------------------------------------------------------#
327
+ # Overrides
328
+ #------------------------------------------------------------------------------#
329
+
330
+ def get_manual()
331
+ UPLOAD_BUNDLE_MANUAL
332
+ end
333
+
334
+ def get_name()
335
+ UPLOAD_BUNDLE_NAME
336
+ end
337
+
338
+ def main(p)
339
+ upload_bundle(p.url,
340
+ p.bucket,
341
+ p.keyprefix,
342
+ p.user,
343
+ p.pass,
344
+ p.location,
345
+ p.manifest,
346
+ p.retry,
347
+ p.part,
348
+ p.directory,
349
+ p.acl,
350
+ p.skipmanifest,
351
+ p.sigv,
352
+ p.region)
353
+ end
354
+
355
+ end
356
+
357
+ #------------------------------------------------------------------------------#
358
+ # Script entry point. Execute only if this file is being executed.
359
+ if __FILE__ == $0 || $0.match(/bin\/ec2-upload-bundle/)
360
+ BundleUploader.new().run(UploadBundleParameters)
361
+ end