ec2_amitools 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +54 -0
  3. data/bin/console +14 -0
  4. data/bin/ec2-ami-tools-version +6 -0
  5. data/bin/ec2-bundle-image +6 -0
  6. data/bin/ec2-bundle-vol +6 -0
  7. data/bin/ec2-delete-bundle +6 -0
  8. data/bin/ec2-download-bundle +6 -0
  9. data/bin/ec2-migrate-bundle +6 -0
  10. data/bin/ec2-migrate-manifest +6 -0
  11. data/bin/ec2-unbundle +6 -0
  12. data/bin/ec2-upload-bundle +6 -0
  13. data/bin/setup +8 -0
  14. data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
  15. data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
  16. data/etc/ec2/amitools/cert-ec2.pem +23 -0
  17. data/etc/ec2/amitools/mappings.csv +9 -0
  18. data/lib/ec2/amitools/bundle.rb +251 -0
  19. data/lib/ec2/amitools/bundle_base.rb +58 -0
  20. data/lib/ec2/amitools/bundleimage.rb +94 -0
  21. data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
  22. data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
  23. data/lib/ec2/amitools/bundleparameters.rb +120 -0
  24. data/lib/ec2/amitools/bundlevol.rb +240 -0
  25. data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
  26. data/lib/ec2/amitools/crypto.rb +379 -0
  27. data/lib/ec2/amitools/decryptmanifest.rb +20 -0
  28. data/lib/ec2/amitools/defaults.rb +12 -0
  29. data/lib/ec2/amitools/deletebundle.rb +212 -0
  30. data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
  31. data/lib/ec2/amitools/downloadbundle.rb +161 -0
  32. data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
  33. data/lib/ec2/amitools/exception.rb +86 -0
  34. data/lib/ec2/amitools/fileutil.rb +219 -0
  35. data/lib/ec2/amitools/format.rb +127 -0
  36. data/lib/ec2/amitools/instance-data.rb +97 -0
  37. data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
  38. data/lib/ec2/amitools/manifestv20070829.rb +361 -0
  39. data/lib/ec2/amitools/manifestv20071010.rb +403 -0
  40. data/lib/ec2/amitools/manifestv3.rb +331 -0
  41. data/lib/ec2/amitools/mapids.rb +148 -0
  42. data/lib/ec2/amitools/migratebundle.rb +222 -0
  43. data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
  44. data/lib/ec2/amitools/migratemanifest.rb +225 -0
  45. data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
  46. data/lib/ec2/amitools/minimalec2.rb +116 -0
  47. data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
  48. data/lib/ec2/amitools/parameters_base.rb +168 -0
  49. data/lib/ec2/amitools/region.rb +93 -0
  50. data/lib/ec2/amitools/s3toolparameters.rb +183 -0
  51. data/lib/ec2/amitools/showversion.rb +12 -0
  52. data/lib/ec2/amitools/syschecks.rb +27 -0
  53. data/lib/ec2/amitools/tool_base.rb +224 -0
  54. data/lib/ec2/amitools/unbundle.rb +107 -0
  55. data/lib/ec2/amitools/unbundleparameters.rb +65 -0
  56. data/lib/ec2/amitools/uploadbundle.rb +361 -0
  57. data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
  58. data/lib/ec2/amitools/util.rb +532 -0
  59. data/lib/ec2/amitools/version.rb +33 -0
  60. data/lib/ec2/amitools/xmlbuilder.rb +237 -0
  61. data/lib/ec2/amitools/xmlutil.rb +55 -0
  62. data/lib/ec2/common/constants.rb +16 -0
  63. data/lib/ec2/common/curl.rb +110 -0
  64. data/lib/ec2/common/headers.rb +95 -0
  65. data/lib/ec2/common/headersv4.rb +173 -0
  66. data/lib/ec2/common/http.rb +333 -0
  67. data/lib/ec2/common/s3support.rb +231 -0
  68. data/lib/ec2/common/signature.rb +68 -0
  69. data/lib/ec2/oem/LICENSE.txt +58 -0
  70. data/lib/ec2/oem/open4.rb +399 -0
  71. data/lib/ec2/platform/base/architecture.rb +26 -0
  72. data/lib/ec2/platform/base/constants.rb +54 -0
  73. data/lib/ec2/platform/base/pipeline.rb +181 -0
  74. data/lib/ec2/platform/base.rb +57 -0
  75. data/lib/ec2/platform/current.rb +55 -0
  76. data/lib/ec2/platform/linux/architecture.rb +35 -0
  77. data/lib/ec2/platform/linux/constants.rb +23 -0
  78. data/lib/ec2/platform/linux/fstab.rb +99 -0
  79. data/lib/ec2/platform/linux/identity.rb +16 -0
  80. data/lib/ec2/platform/linux/image.rb +811 -0
  81. data/lib/ec2/platform/linux/mtab.rb +74 -0
  82. data/lib/ec2/platform/linux/pipeline.rb +40 -0
  83. data/lib/ec2/platform/linux/rsync.rb +114 -0
  84. data/lib/ec2/platform/linux/tar.rb +124 -0
  85. data/lib/ec2/platform/linux/uname.rb +50 -0
  86. data/lib/ec2/platform/linux.rb +83 -0
  87. data/lib/ec2/platform/solaris/architecture.rb +28 -0
  88. data/lib/ec2/platform/solaris/constants.rb +30 -0
  89. data/lib/ec2/platform/solaris/fstab.rb +43 -0
  90. data/lib/ec2/platform/solaris/identity.rb +16 -0
  91. data/lib/ec2/platform/solaris/image.rb +327 -0
  92. data/lib/ec2/platform/solaris/mtab.rb +29 -0
  93. data/lib/ec2/platform/solaris/pipeline.rb +40 -0
  94. data/lib/ec2/platform/solaris/rsync.rb +24 -0
  95. data/lib/ec2/platform/solaris/tar.rb +36 -0
  96. data/lib/ec2/platform/solaris/uname.rb +21 -0
  97. data/lib/ec2/platform/solaris.rb +38 -0
  98. data/lib/ec2/platform.rb +69 -0
  99. data/lib/ec2/version.rb +8 -0
  100. data/lib/ec2_amitools +1 -0
  101. data/lib/ec2_amitools.rb +7 -0
  102. metadata +184 -0
@@ -0,0 +1,12 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/version'
12
+ puts EC2Version::version_copyright_string()
@@ -0,0 +1,27 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/fileutil'
12
+ require 'ec2/platform/current'
13
+
14
+ module SysChecks
15
+ def self.rsync_usable?()
16
+ EC2::Platform::Current::Rsync.usable?
17
+ end
18
+ def self.good_tar_version?()
19
+ EC2::Platform::Current::Tar::Version.current.usable?
20
+ end
21
+ def self.get_system_arch()
22
+ EC2::Platform::Current::System::BUNDLING_ARCHITECTURE
23
+ end
24
+ def self.root_user?()
25
+ EC2::Platform::Current::System.superuser?
26
+ end
27
+ end
@@ -0,0 +1,224 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ module AMIToolExceptions
12
+ # All fatal errors should inherit from this.
13
+ class EC2FatalError < RuntimeError
14
+ attr_accessor :code
15
+ def initialize(code, msg)
16
+ super(msg)
17
+ @code = code
18
+ end
19
+ end
20
+
21
+ class FileNotFound < EC2FatalError
22
+ def initialize(path)
23
+ super(2, "File not found: #{path}")
24
+ end
25
+ end
26
+
27
+ class S3Error < EC2FatalError
28
+ def initialize(msg)
29
+ super(3, "Error talking to S3: #{msg}")
30
+ end
31
+ end
32
+
33
+ class PromptTimeout < EC2FatalError
34
+ def initialize(msg=nil)
35
+ message = "Timed out waiting for user input"
36
+ message += ": #{msg}" unless msg.nil?
37
+ super(5, message)
38
+ end
39
+ end
40
+
41
+ # This is more for flow control than anything else.
42
+ # Raising it should terminate execution, but not print an error.
43
+ class EC2StopExecution < RuntimeError
44
+ attr_accessor :code
45
+ def initialize(code=0)
46
+ super()
47
+ @code = code
48
+ end
49
+ end
50
+
51
+ class TryFailed < RuntimeError
52
+ end
53
+
54
+ end
55
+
56
+ class AMITool
57
+
58
+ include AMIToolExceptions
59
+
60
+ PROMPT_TIMEOUT = 30
61
+ MAX_TRIES = 5
62
+ BACKOFF_PERIOD = 5
63
+
64
+ #------------------------------------------------------------------------------#
65
+ # Methods to override in subclasses
66
+ #------------------------------------------------------------------------------#
67
+
68
+ def get_manual()
69
+ # We have to get the manual text into here.
70
+ raise "NotImplemented: get_manual()"
71
+ end
72
+
73
+ def get_name()
74
+ # We have to get the tool name into here.
75
+ raise "NotImplemented: get_name()"
76
+ end
77
+
78
+ def main(params)
79
+ # Main entry point.
80
+ raise "NotImplemented: main()"
81
+ end
82
+
83
+ #------------------------------------------------------------------------------#
84
+ # Utility methods
85
+ #------------------------------------------------------------------------------#
86
+
87
+ # Display a message (without appending a newline) and ask for a response.
88
+ # Returns user response or nil if interactivity is not desired.
89
+ # Raises exception on timeout.
90
+ def interactive_prompt(message, name=nil)
91
+ return nil unless interactive?
92
+ begin
93
+ $stdout.print(message)
94
+ $stdout.flush
95
+ Timeout::timeout(PROMPT_TIMEOUT) do
96
+ return gets
97
+ end
98
+ rescue Timeout::Error
99
+ raise PromptTimeout.new(name)
100
+ end
101
+ end
102
+
103
+ #------------------------------------------------------------------------------#
104
+
105
+ # Display a message on stderr.
106
+ # If interactive, asks for confirmation (yes/no).
107
+ # Returns true if in batch mode or user agrees, false if user disagrees.
108
+ # Raises exception on timeout.
109
+ def warn_confirm(message)
110
+ $stderr.puts(message)
111
+ $stderr.flush
112
+ return true unless interactive?
113
+ response = interactive_prompt("Are you sure you want to continue? [y/N]")
114
+ if response =~ /^[Yy]/
115
+ return true
116
+ end
117
+ return false
118
+ end
119
+
120
+ #----------------------------------------------------------------------------#
121
+
122
+ def retry_s3(retrying=true)
123
+ tries = 0
124
+ while true
125
+ tries += 1
126
+ begin
127
+ result = yield
128
+ return result
129
+ rescue TryFailed => e
130
+ $stderr.puts e.message
131
+ if retrying and tries < MAX_TRIES
132
+ $stdout.puts "Retrying in #{BACKOFF_PERIOD}s ..."
133
+ else
134
+ raise EC2FatalError.new(3, e.message)
135
+ end
136
+ end
137
+ end
138
+ end
139
+
140
+ #------------------------------------------------------------------------------#
141
+ # Standard behaviour
142
+ #------------------------------------------------------------------------------#
143
+
144
+ def handle_early_exit_parameters(params)
145
+ if params.version
146
+ puts get_name() + " " + params.version_copyright_string()
147
+ return
148
+ end
149
+
150
+ if params.show_help
151
+ puts params.help
152
+ return
153
+ end
154
+
155
+ if params.manual
156
+ puts get_manual()
157
+ return
158
+ end
159
+ end
160
+
161
+ #------------------------------------------------------------------------------#
162
+
163
+ def interactive?
164
+ @interactive
165
+ end
166
+
167
+ #------------------------------------------------------------------------------#
168
+
169
+ def get_parameters(params_class)
170
+ # Parse the parameters and die on errors.
171
+ # Assume that if we're parsing parameters, it's safe to exit.
172
+ begin
173
+ params = params_class.new(ARGV)
174
+ rescue StandardError => e
175
+ $stderr.puts e.message
176
+ $stderr.puts "Try '#{get_name} --help'"
177
+ exit 1
178
+ end
179
+
180
+ # Deal with help, verion, etc.
181
+ if params.early_exit?
182
+ handle_early_exit_parameters(params)
183
+ exit 0
184
+ end
185
+
186
+ # Some general flags that we want to set
187
+ @debug = params.debug
188
+ @interactive = params.interactive?
189
+
190
+ # Finally, return the leftovers.
191
+ params
192
+ end
193
+
194
+ #------------------------------------------------------------------------------#
195
+
196
+ def run(params_class)
197
+ # We want to be able to reuse bits without having to parse
198
+ # parameters, so run() is not called from the constructor.
199
+ begin
200
+ params = get_parameters(params_class)
201
+ main(params)
202
+ rescue AMIToolExceptions::EC2StopExecution => e
203
+ # We've been asked to stop.
204
+ exit e.code
205
+ rescue AMIToolExceptions::PromptTimeout => e
206
+ $stderr.puts e.message
207
+ exit e.code
208
+ rescue AMIToolExceptions::EC2FatalError => e
209
+ $stderr.puts "ERROR: #{e.message}"
210
+ puts e.backtrace if @debug
211
+ exit e.code
212
+ rescue Interrupt => e
213
+ $stderr.puts "\n#{get_name} interrupted."
214
+ puts e.backtrace if @debug
215
+ exit 255
216
+ rescue => e
217
+ $stderr.puts "ERROR: #{e.message}"
218
+ puts e.inspect if @debug
219
+ puts e.backtrace if @debug
220
+ exit 254
221
+ end
222
+ end
223
+
224
+ end
@@ -0,0 +1,107 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/format'
12
+ require 'ec2/amitools/manifestv3'
13
+ require 'ec2/amitools/unbundleparameters'
14
+ require 'ec2/platform/current'
15
+ require 'ec2/amitools/tool_base'
16
+
17
+ UNBUNDLE_NAME = 'ec2-unbundle'
18
+
19
+ UNBUNDLE_MANUAL =<<TEXT
20
+ #{UNBUNDLE_NAME} extracts a filesystem image from a bundle AMI.
21
+
22
+ #{UNBUNDLE_NAME} will:
23
+ - read relevant information from the manifest file
24
+ - concatenate all the parts
25
+ - decrypt and uncompress the image
26
+ TEXT
27
+
28
+ class Unbundler < AMITool
29
+
30
+ def unbundle(p)
31
+ begin
32
+ manifest_path = p.manifest_path
33
+ src_dir = p.source
34
+ dst_dir = p.destination
35
+
36
+ digest_pipe = File::join( '/tmp', "ec2-unbundle-image-digest-pipe" )
37
+ File::delete( digest_pipe ) if File::exist?( digest_pipe )
38
+ unless system( "mkfifo #{digest_pipe}" )
39
+ raise "error creating named pipe #{digest_pipe}"
40
+ end
41
+
42
+ # Load manifest and the user's private key.
43
+ manifest = ManifestV3.new(File.open( manifest_path ) { |f| f.read() })
44
+ pk = Crypto::loadprivkey( p.user_pk_path )
45
+
46
+ # Extract key and IV from XML manifest.
47
+ key = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_key))
48
+ iv = pk.private_decrypt(Format::hex2bin( manifest.user_encrypted_iv))
49
+
50
+ # Create a string of space separated part paths.
51
+ part_files = manifest.parts.collect do |part|
52
+ File::join( src_dir, part.filename )
53
+ end.join( ' ' )
54
+
55
+ # Join, decrypt, decompress and untar.
56
+ untar = EC2::Platform::Current::Tar::Command.new.extract.chdir(dst_dir)
57
+ pipeline = EC2::Platform::Current::Pipeline.new('image-unbundle-pipeline', @debug)
58
+ pipeline.concat([
59
+ ['cat', "openssl sha1 < #{digest_pipe} & cat #{part_files}"],
60
+ ['decrypt', "openssl enc -d -aes-128-cbc -K #{key} -iv #{iv}"],
61
+ ['gunzip', "gunzip"],
62
+ ['tee', "tee #{digest_pipe}"],
63
+ ['untar', untar.expand]
64
+ ])
65
+ digest = nil
66
+ begin
67
+ digest = pipeline.execute.split(/\s+/).last.strip
68
+ rescue EC2::Platform::Current::Pipeline::ExecutionError => e
69
+ $stderr.puts e.message
70
+ end
71
+
72
+ # Verify digest.
73
+ unless manifest.digest == digest
74
+ raise "invalid digest, expected #{manifest.digest} received #{digest}"
75
+ end
76
+
77
+ puts "Unbundle complete."
78
+ return 0
79
+ ensure
80
+ File::delete( digest_pipe ) if (digest_pipe && File::exist?( digest_pipe ))
81
+ end
82
+ end
83
+
84
+ #------------------------------------------------------------------------------#
85
+ # Overrides
86
+ #------------------------------------------------------------------------------#
87
+
88
+ def get_manual()
89
+ UNBUNDLE_MANUAL
90
+ end
91
+
92
+ def get_name()
93
+ UNBUNDLE_NAME
94
+ end
95
+
96
+ def main(p)
97
+ unbundle(p)
98
+ end
99
+
100
+ end
101
+
102
+ #------------------------------------------------------------------------------#
103
+ # Script entry point. Execute only if this file is being executed.
104
+
105
+ if __FILE__ == $0 || $0.match(/bin\/ec2-unbundle/)
106
+ Unbundler.new().run(UnbundleParameters)
107
+ end
@@ -0,0 +1,65 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/amitools/parameters_base'
12
+
13
+ class UnbundleParameters < ParametersBase
14
+
15
+ MANIFEST_DESCRIPTION = "The path to the AMI manifest file."
16
+ SOURCE_DESCRIPTION = 'The directory containing bundled AMI parts to unbundle. Defaults to ".".'
17
+ DESTINATION_DESCRIPTION = 'The directory to unbundle the AMI into. Defaults to the ".".'
18
+
19
+ attr_accessor :manifest_path,
20
+ :user_pk_path,
21
+ :source,
22
+ :destination
23
+
24
+ #----------------------------------------------------------------------------#
25
+
26
+ def mandatory_params()
27
+ on('-k', '--privatekey PATH', String, USER_PK_PATH_DESCRIPTION) do |path|
28
+ assert_file_exists(path, '--privatekey')
29
+ @user_pk_path = path
30
+ end
31
+
32
+ on('-m', '--manifest PATH', String, MANIFEST_DESCRIPTION) do |manifest|
33
+ assert_file_exists(manifest, '--manifest')
34
+ @manifest_path = manifest
35
+ end
36
+ end
37
+
38
+ #----------------------------------------------------------------------------#
39
+
40
+ def optional_params()
41
+ on('-s', '--source DIRECTORY', String, SOURCE_DESCRIPTION) do |directory|
42
+ assert_directory_exists(directory, '--source')
43
+ @source = directory
44
+ end
45
+
46
+ on('-d', '--destination DIRECTORY', String, DESTINATION_DESCRIPTION) do |directory|
47
+ assert_directory_exists(directory, '--destination')
48
+ @destination = directory
49
+ end
50
+ end
51
+
52
+ #----------------------------------------------------------------------------#
53
+
54
+ def validate_params()
55
+ raise MissingMandatory.new('--manifest') unless @manifest_path
56
+ raise MissingMandatory.new('--privatekey') unless @user_pk_path
57
+ end
58
+
59
+ #----------------------------------------------------------------------------#
60
+
61
+ def set_defaults()
62
+ @source ||= Dir::pwd()
63
+ @destination ||= Dir::pwd()
64
+ end
65
+ end
@@ -0,0 +1,361 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'ec2/common/s3support'
12
+ require 'ec2/amitools/uploadbundleparameters'
13
+ require 'uri'
14
+ require 'ec2/amitools/instance-data'
15
+ require 'ec2/amitools/manifestv20071010'
16
+ require 'rexml/document'
17
+ require 'digest/md5'
18
+ require 'base64'
19
+ require 'ec2/amitools/tool_base'
20
+ require 'ec2/amitools/region'
21
+
22
+ #------------------------------------------------------------------------------#
23
+
24
+ UPLOAD_BUNDLE_NAME = 'ec2-upload-bundle'
25
+
26
+ UPLOAD_BUNDLE_MANUAL =<<TEXT
27
+ #{UPLOAD_BUNDLE_NAME} is a command line tool to upload a bundled Amazon Image to S3 storage
28
+ for use by EC2. An Amazon Image may be one of the following:
29
+ - Amazon Machine Image (AMI)
30
+ - Amazon Kernel Image (AKI)
31
+ - Amazon Ramdisk Image (ARI)
32
+
33
+ #{UPLOAD_BUNDLE_NAME} will:
34
+ - create an S3 bucket to store the bundled AMI in if it does not already exist
35
+ - upload the AMI manifest and parts files to S3, granting specified privileges
36
+ - on them (defaults to EC2 read privileges)
37
+
38
+ To manually retry an upload that failed, #{UPLOAD_BUNDLE_NAME} can optionally:
39
+ - skip uploading the manifest
40
+ - only upload bundled AMI parts from a specified part onwards
41
+ TEXT
42
+
43
+ #------------------------------------------------------------------------------#
44
+
45
+ class BucketLocationError < AMIToolExceptions::EC2FatalError
46
+ def initialize(bucket, location, bucket_location)
47
+ location = "US" if location == :unconstrained
48
+ bucket_location = "US" if bucket_location == :unconstrained
49
+ super(10, "Bucket \"#{bucket}\" already exists in \"#{bucket_location}\" and \"#{location}\" was specified.")
50
+ end
51
+ end
52
+
53
+ #----------------------------------------------------------------------------#
54
+
55
+ # Upload the specified file.
56
+
57
+ class BundleUploader < AMITool
58
+
59
+ def upload(s3_conn, bucket, key, file, acl, retry_upload)
60
+ retry_s3(retry_upload) do
61
+ begin
62
+ md5 = get_md5(file)
63
+ s3_conn.put(bucket, key, file, {"x-amz-acl"=>acl, "content-md5"=>md5})
64
+ return
65
+ rescue EC2::Common::HTTP::Error::PathInvalid => e
66
+ raise FileNotFound(file)
67
+ rescue => e
68
+ raise TryFailed.new("Failed to upload \"#{file}\": #{e.message}")
69
+ end
70
+ end
71
+ end
72
+
73
+ #----------------------------------------------------------------------------#
74
+
75
+ def get_md5(file)
76
+ Base64::encode64(Digest::MD5::digest(File.open(file) { |f| f.read })).strip
77
+ end
78
+
79
+ #----------------------------------------------------------------------------#
80
+
81
+ #
82
+ # Availability zone names are generally in the format => ${REGION}${ZONENUMBER}.
83
+ # Examples being us-east-1b, us-east-1c, etc.
84
+ #
85
+ def get_availability_zone()
86
+ instance_data = EC2::InstanceData.new
87
+ instance_data.availability_zone
88
+ end
89
+
90
+ #----------------------------------------------------------------------------#
91
+
92
+ # Return a list of bundle part filename and part number tuples from the manifest.
93
+ def get_part_info(manifest)
94
+ parts = manifest.ami_part_info_list.map do |part|
95
+ [part['filename'], part['index']]
96
+ end
97
+ parts.sort
98
+ end
99
+
100
+ #------------------------------------------------------------------------------#
101
+
102
+ def uri2string(uri)
103
+ s = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}"
104
+ # Remove the trailing '/'.
105
+ return (s[-1..-1] == "/" ? s[0..-2] : s)
106
+ end
107
+
108
+ #------------------------------------------------------------------------------#
109
+
110
+ # Get the bucket's location.
111
+ def get_bucket_location(s3_conn, bucket)
112
+ begin
113
+ response = s3_conn.get_bucket_location(bucket)
114
+ rescue EC2::Common::HTTP::Error::Retrieve => e
115
+ if e.code == 404
116
+ # We have a "Not found" S3 response, which probably means the bucket doesn't exist.
117
+ return nil
118
+ end
119
+ raise e
120
+ end
121
+ $stdout.puts "check_bucket_location response: #{response.body}" if @debug and response.text?
122
+ docroot = REXML::Document.new(response.body).root
123
+ bucket_location = REXML::XPath.first(docroot, '/LocationConstraint').text
124
+ bucket_location ||= :unconstrained
125
+ end
126
+
127
+ #------------------------------------------------------------------------------#
128
+
129
+ # Check if the bucket exists and is in an appropriate location.
130
+ def check_bucket_location(bucket, bucket_location, location)
131
+ if bucket_location.nil?
132
+ # The bucket does not exist. Safe, but we need to create it.
133
+ return false
134
+ end
135
+ if location.nil?
136
+ # The bucket exists and we don't care where it is.
137
+ return true
138
+ end
139
+ unless [bucket_location, AwsRegion.guess_region_from_s3_bucket(bucket_location)].include?(location)
140
+ # The bucket isn't where we want it. This is a problem.
141
+ raise BucketLocationError.new(bucket, location, bucket_location)
142
+ end
143
+ # The bucket exists and is in the right place.
144
+ return true
145
+ end
146
+
147
+ #------------------------------------------------------------------------------#
148
+
149
+ # Create the specified bucket if it does not exist.
150
+ def create_bucket(s3_conn, bucket, bucket_location, location, retry_create)
151
+ begin
152
+ if check_bucket_location(bucket, bucket_location, location)
153
+ return true
154
+ end
155
+ $stdout.puts "Creating bucket..."
156
+
157
+ retry_s3(retry_create) do
158
+ error = "Could not create or access bucket #{bucket}"
159
+ begin
160
+ rsp = s3_conn.create_bucket(bucket, location == :unconstrained ? nil : location)
161
+ rescue EC2::Common::HTTP::Error::Retrieve => e
162
+ error += ": server response #{e.message} #{e.code}"
163
+ raise TryFailed.new(e.message)
164
+ rescue RuntimeError => e
165
+ error += ": error message #{e.message}"
166
+ raise e
167
+ end
168
+ end
169
+ end
170
+ end
171
+
172
+ #------------------------------------------------------------------------------#
173
+
174
+ # If we return true, we have a v2-compliant name.
175
+ # If we return false, we wish to use a bad name.
176
+ # Otherwise we quietly wander off to die in peace.
177
+ def check_bucket_name(bucket)
178
+ if EC2::Common::S3Support::bucket_name_s3_v2_safe?(bucket)
179
+ return true
180
+ end
181
+ message = "The specified bucket is not S3 v2 safe (see S3 documentation for details):\n#{bucket}"
182
+ if warn_confirm(message)
183
+ # Assume the customer knows what he's doing.
184
+ return false
185
+ else
186
+ # We've been asked to stop, so quietly wander off to die in peace.
187
+ raise EC2StopExecution.new()
188
+ end
189
+ end
190
+
191
+ # force v1 S3 addressing when using govcloud endpoint
192
+ def check_govcloud_override(s3_url)
193
+ if s3_url =~ /s3-us-gov-west-1/
194
+ false
195
+ else
196
+ true
197
+ end
198
+ end
199
+
200
+ #------------------------------------------------------------------------------#
201
+
202
+ def get_region()
203
+ zone = get_availability_zone()
204
+ if zone.nil?
205
+ return nil
206
+ end
207
+ # assume region names do not have a common naming scheme. Therefore we manually go through all known region names
208
+ AwsRegion.regions.each do |region|
209
+ match = zone.match(region)
210
+ if not match.nil?
211
+ return region
212
+ end
213
+ end
214
+ nil
215
+ end
216
+
217
+ # This is very much a best effort attempt. If in doubt, we don't warn.
218
+ def cross_region?(location, bucket_location)
219
+
220
+ # If the bucket exists, its S3 location is canonical.
221
+ s3_region = bucket_location
222
+ s3_region ||= location
223
+ s3_region ||= :unconstrained
224
+
225
+ region = get_region()
226
+
227
+ if region.nil?
228
+ # If we can't get the region, assume we're fine since there's
229
+ # nothing more we can do.
230
+ return false
231
+ end
232
+
233
+ return s3_region != AwsRegion.get_s3_location(region)
234
+ end
235
+
236
+ #------------------------------------------------------------------------------#
237
+
238
+ def warn_about_migrating()
239
+ message = ["You are bundling in one region, but uploading to another. If the kernel",
240
+ "or ramdisk associated with this AMI are not in the target region, AMI",
241
+ "registration will fail.",
242
+ "You can use the ec2-migrate-manifest tool to update your manifest file",
243
+ "with a kernel and ramdisk that exist in the target region.",
244
+ ].join("\n")
245
+ unless warn_confirm(message)
246
+ raise EC2StopExecution.new()
247
+ end
248
+ end
249
+
250
+ #------------------------------------------------------------------------------#
251
+
252
+ def get_s3_conn(s3_url, user, pass, method, sigv, region=nil)
253
+ EC2::Common::S3Support.new(s3_url, user, pass, method, @debug, sigv, region)
254
+ end
255
+
256
+ #------------------------------------------------------------------------------#
257
+
258
+ #
259
+ # Get parameters and display help or manual if necessary.
260
+ #
261
+ def upload_bundle(url,
262
+ bucket,
263
+ keyprefix,
264
+ user,
265
+ pass,
266
+ location,
267
+ manifest_file,
268
+ retry_stuff,
269
+ part,
270
+ directory,
271
+ acl,
272
+ skipmanifest,
273
+ sigv,
274
+ region)
275
+ begin
276
+ # Get the S3 URL.
277
+ s3_uri = URI.parse(url)
278
+ s3_url = uri2string(s3_uri)
279
+ v2_bucket = check_bucket_name(bucket) and check_govcloud_override(s3_url)
280
+ s3_conn = get_s3_conn(s3_url, user, pass, (v2_bucket ? nil : :path), sigv, region)
281
+
282
+ # Get current location and bucket location.
283
+ bucket_location = get_bucket_location(s3_conn, bucket)
284
+
285
+ # Load manifest.
286
+ xml = File.open(manifest_file) { |f| f.read }
287
+ manifest = ManifestV20071010.new(xml)
288
+
289
+ # If in interactive mode, warn when bundling a kernel into our AMI and we are uploading cross-region
290
+ if interactive? and manifest.kernel_id and cross_region?(location, bucket_location)
291
+ warn_about_migrating()
292
+ end
293
+
294
+ # Create storage bucket if required.
295
+ create_bucket(s3_conn, bucket, bucket_location, location, retry_stuff)
296
+
297
+ # Upload AMI bundle parts.
298
+ $stdout.puts "Uploading bundled image parts to the S3 bucket #{bucket} ..."
299
+ get_part_info(manifest).each do |part_info|
300
+ if part.nil? or (part_info[1] >= part)
301
+ path = File.join(directory, part_info[0])
302
+ upload(s3_conn, bucket, keyprefix + part_info[0], path, acl, retry_stuff)
303
+ $stdout.puts "Uploaded #{part_info[0]}"
304
+ else
305
+ $stdout.puts "Skipping #{part_info[0]}"
306
+ end
307
+ end
308
+
309
+ # Encrypt and upload manifest.
310
+ unless skipmanifest
311
+ $stdout.puts "Uploading manifest ..."
312
+ upload(s3_conn, bucket, keyprefix + File::basename(manifest_file), manifest_file, acl, retry_stuff)
313
+ $stdout.puts "Uploaded manifest."
314
+ $stdout.puts 'Manifest uploaded to: %s/%s' % [bucket, keyprefix + File::basename(manifest_file)]
315
+ else
316
+ $stdout.puts "Skipping manifest."
317
+ end
318
+
319
+ $stdout.puts 'Bundle upload completed.'
320
+ rescue EC2::Common::HTTP::Error => e
321
+ $stderr.puts e.backtrace if @debug
322
+ raise S3Error.new(e.message)
323
+ end
324
+ end
325
+
326
+ #------------------------------------------------------------------------------#
327
+ # Overrides
328
+ #------------------------------------------------------------------------------#
329
+
330
+ def get_manual()
331
+ UPLOAD_BUNDLE_MANUAL
332
+ end
333
+
334
+ def get_name()
335
+ UPLOAD_BUNDLE_NAME
336
+ end
337
+
338
+ def main(p)
339
+ upload_bundle(p.url,
340
+ p.bucket,
341
+ p.keyprefix,
342
+ p.user,
343
+ p.pass,
344
+ p.location,
345
+ p.manifest,
346
+ p.retry,
347
+ p.part,
348
+ p.directory,
349
+ p.acl,
350
+ p.skipmanifest,
351
+ p.sigv,
352
+ p.region)
353
+ end
354
+
355
+ end
356
+
357
+ #------------------------------------------------------------------------------#
358
+ # Script entry point. Execute only if this file is being executed.
359
+ if __FILE__ == $0 || $0.match(/bin\/ec2-upload-bundle/)
360
+ BundleUploader.new().run(UploadBundleParameters)
361
+ end