ec2_amitools 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +54 -0
  3. data/bin/console +14 -0
  4. data/bin/ec2-ami-tools-version +6 -0
  5. data/bin/ec2-bundle-image +6 -0
  6. data/bin/ec2-bundle-vol +6 -0
  7. data/bin/ec2-delete-bundle +6 -0
  8. data/bin/ec2-download-bundle +6 -0
  9. data/bin/ec2-migrate-bundle +6 -0
  10. data/bin/ec2-migrate-manifest +6 -0
  11. data/bin/ec2-unbundle +6 -0
  12. data/bin/ec2-upload-bundle +6 -0
  13. data/bin/setup +8 -0
  14. data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
  15. data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
  16. data/etc/ec2/amitools/cert-ec2.pem +23 -0
  17. data/etc/ec2/amitools/mappings.csv +9 -0
  18. data/lib/ec2/amitools/bundle.rb +251 -0
  19. data/lib/ec2/amitools/bundle_base.rb +58 -0
  20. data/lib/ec2/amitools/bundleimage.rb +94 -0
  21. data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
  22. data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
  23. data/lib/ec2/amitools/bundleparameters.rb +120 -0
  24. data/lib/ec2/amitools/bundlevol.rb +240 -0
  25. data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
  26. data/lib/ec2/amitools/crypto.rb +379 -0
  27. data/lib/ec2/amitools/decryptmanifest.rb +20 -0
  28. data/lib/ec2/amitools/defaults.rb +12 -0
  29. data/lib/ec2/amitools/deletebundle.rb +212 -0
  30. data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
  31. data/lib/ec2/amitools/downloadbundle.rb +161 -0
  32. data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
  33. data/lib/ec2/amitools/exception.rb +86 -0
  34. data/lib/ec2/amitools/fileutil.rb +219 -0
  35. data/lib/ec2/amitools/format.rb +127 -0
  36. data/lib/ec2/amitools/instance-data.rb +97 -0
  37. data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
  38. data/lib/ec2/amitools/manifestv20070829.rb +361 -0
  39. data/lib/ec2/amitools/manifestv20071010.rb +403 -0
  40. data/lib/ec2/amitools/manifestv3.rb +331 -0
  41. data/lib/ec2/amitools/mapids.rb +148 -0
  42. data/lib/ec2/amitools/migratebundle.rb +222 -0
  43. data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
  44. data/lib/ec2/amitools/migratemanifest.rb +225 -0
  45. data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
  46. data/lib/ec2/amitools/minimalec2.rb +116 -0
  47. data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
  48. data/lib/ec2/amitools/parameters_base.rb +168 -0
  49. data/lib/ec2/amitools/region.rb +93 -0
  50. data/lib/ec2/amitools/s3toolparameters.rb +183 -0
  51. data/lib/ec2/amitools/showversion.rb +12 -0
  52. data/lib/ec2/amitools/syschecks.rb +27 -0
  53. data/lib/ec2/amitools/tool_base.rb +224 -0
  54. data/lib/ec2/amitools/unbundle.rb +107 -0
  55. data/lib/ec2/amitools/unbundleparameters.rb +65 -0
  56. data/lib/ec2/amitools/uploadbundle.rb +361 -0
  57. data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
  58. data/lib/ec2/amitools/util.rb +532 -0
  59. data/lib/ec2/amitools/version.rb +33 -0
  60. data/lib/ec2/amitools/xmlbuilder.rb +237 -0
  61. data/lib/ec2/amitools/xmlutil.rb +55 -0
  62. data/lib/ec2/common/constants.rb +16 -0
  63. data/lib/ec2/common/curl.rb +110 -0
  64. data/lib/ec2/common/headers.rb +95 -0
  65. data/lib/ec2/common/headersv4.rb +173 -0
  66. data/lib/ec2/common/http.rb +333 -0
  67. data/lib/ec2/common/s3support.rb +231 -0
  68. data/lib/ec2/common/signature.rb +68 -0
  69. data/lib/ec2/oem/LICENSE.txt +58 -0
  70. data/lib/ec2/oem/open4.rb +399 -0
  71. data/lib/ec2/platform/base/architecture.rb +26 -0
  72. data/lib/ec2/platform/base/constants.rb +54 -0
  73. data/lib/ec2/platform/base/pipeline.rb +181 -0
  74. data/lib/ec2/platform/base.rb +57 -0
  75. data/lib/ec2/platform/current.rb +55 -0
  76. data/lib/ec2/platform/linux/architecture.rb +35 -0
  77. data/lib/ec2/platform/linux/constants.rb +23 -0
  78. data/lib/ec2/platform/linux/fstab.rb +99 -0
  79. data/lib/ec2/platform/linux/identity.rb +16 -0
  80. data/lib/ec2/platform/linux/image.rb +811 -0
  81. data/lib/ec2/platform/linux/mtab.rb +74 -0
  82. data/lib/ec2/platform/linux/pipeline.rb +40 -0
  83. data/lib/ec2/platform/linux/rsync.rb +114 -0
  84. data/lib/ec2/platform/linux/tar.rb +124 -0
  85. data/lib/ec2/platform/linux/uname.rb +50 -0
  86. data/lib/ec2/platform/linux.rb +83 -0
  87. data/lib/ec2/platform/solaris/architecture.rb +28 -0
  88. data/lib/ec2/platform/solaris/constants.rb +30 -0
  89. data/lib/ec2/platform/solaris/fstab.rb +43 -0
  90. data/lib/ec2/platform/solaris/identity.rb +16 -0
  91. data/lib/ec2/platform/solaris/image.rb +327 -0
  92. data/lib/ec2/platform/solaris/mtab.rb +29 -0
  93. data/lib/ec2/platform/solaris/pipeline.rb +40 -0
  94. data/lib/ec2/platform/solaris/rsync.rb +24 -0
  95. data/lib/ec2/platform/solaris/tar.rb +36 -0
  96. data/lib/ec2/platform/solaris/uname.rb +21 -0
  97. data/lib/ec2/platform/solaris.rb +38 -0
  98. data/lib/ec2/platform.rb +69 -0
  99. data/lib/ec2/version.rb +8 -0
  100. data/lib/ec2_amitools +1 -0
  101. data/lib/ec2_amitools.rb +7 -0
  102. metadata +184 -0
@@ -0,0 +1,16 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ module EC2
12
+ module Platform
13
+ module Linux; end
14
+ EC2::Platform::PEER = EC2::Platform::Linux unless defined? EC2::Platform::PEER
15
+ end
16
+ end
@@ -0,0 +1,811 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'fileutils'
12
+ require 'pathname'
13
+ require 'ec2/oem/open4'
14
+ require 'ec2/amitools/fileutil'
15
+ require 'ec2/amitools/syschecks'
16
+ require 'ec2/amitools/exception'
17
+ require 'ec2/platform/linux/mtab'
18
+ require 'ec2/platform/linux/fstab'
19
+ require 'ec2/platform/linux/constants'
20
+
21
+ module EC2
22
+ module Platform
23
+ module Linux
24
+
25
+ # This class encapsulate functionality to create an file loopback image
26
+ # from a volume. The image is created using dd. Sub-directories of the
27
+ # volume, including mounts of local filesystems, are copied to the image.
28
+ # Symbolic links are preserved.
29
+ class Image
30
+ IMG_MNT = '/mnt/img-mnt'
31
+ EXCLUDES= ['/dev', '/media', '/mnt', '/proc', '/sys']
32
+ DEFAULT_FSTAB = EC2::Platform::Linux::Fstab::DEFAULT
33
+ LEGACY_FSTAB = EC2::Platform::Linux::Fstab::LEGACY
34
+ BASE_UTILS = [ 'modprobe', 'mount', 'umount', 'dd' ]
35
+ PART_UTILS = [ 'dmsetup', 'kpartx', 'losetup' ]
36
+ CHROOT_UTILS = [ 'grub' ]
37
+
38
+ #---------------------------------------------------------------------#
39
+
40
+ # Initialize the instance with the required parameters.
41
+ # * _volume_ The path to the volume to create the image file from.
42
+ # * _image_filename_ The name of the image file to create.
43
+ # * _mb_image_size_ The image file size in MB.
44
+ # * _exclude_ List of directories to exclude.
45
+ # * _debug_ Make extra noise.
46
+ def initialize( volume,
47
+ image_filename,
48
+ mb_image_size,
49
+ exclude,
50
+ includes,
51
+ filter = true,
52
+ fstab = nil,
53
+ part_type = nil,
54
+ arch = nil,
55
+ script = nil,
56
+ debug = false,
57
+ grub_config = nil )
58
+ @volume = volume
59
+ @image_filename = image_filename
60
+ @mb_image_size = mb_image_size
61
+ @exclude = exclude
62
+ @includes = includes
63
+ @filter = filter
64
+ @arch = arch || EC2::Platform::Linux::Uname.platform
65
+ @script = script
66
+ @fstab = nil
67
+ @conf = grub_config
68
+ @warnings = Array.new
69
+
70
+ self.verify_runtime(BASE_UTILS)
71
+ self.set_partition_type(part_type)
72
+
73
+ # Cunning plan or horrible hack?
74
+ # If :legacy is passed in as the fstab, we use the old v3 manifest's
75
+ # device naming and fstab.
76
+ if [:legacy, :default].include? fstab
77
+ @fstab = fstab
78
+ elsif not fstab.nil?
79
+ @fstab = File.open(fstab).read()
80
+ end
81
+ @debug = debug
82
+
83
+ # Exclude the temporary image mount point if it is under the volume
84
+ # being bundled.
85
+ if IMG_MNT.index( volume ) == 0
86
+ @exclude << IMG_MNT
87
+ end
88
+ end
89
+
90
+ #--------------------------------------------------------------------#
91
+
92
+ def make_hash(array)
93
+ hash = Hash.new
94
+ array.each do |entry|
95
+ # Split on the first '='
96
+ key, value = entry.split('=', 2)
97
+ hash[key] = value || ''
98
+ end
99
+ hash
100
+ end
101
+
102
+ def compare_hashes(a, b)
103
+ a.each do |key,value|
104
+ if not b.has_key?(key)
105
+ @warnings.push "\t* Missing key '#{key}' with value '#{value}'"
106
+ elsif b[key] != value
107
+ @warnings.push "\t* Key '#{key}' value '#{b[key]}' differs from /proc/cmdline value '#{value}'"
108
+ end
109
+ end
110
+ end
111
+
112
+ def remove_kernel(tokens)
113
+ if tokens.include?('kernel')
114
+ tokens.delete('kernel')
115
+ # The kernel can receive optional arguments, drop those along with the kernel
116
+ kernel_index = tokens.index{ |token| !token.include?('--') }
117
+ tokens = tokens.drop(kernel_index + 1)
118
+ end
119
+ tokens
120
+ end
121
+
122
+ def tokenize(string)
123
+ string.strip.split(/\s+/)
124
+ end
125
+
126
+ def check_kernel_parameters(conf)
127
+ cmdline = File.read('/proc/cmdline')
128
+ cmdline_hash = make_hash(tokenize(cmdline))
129
+
130
+ default = nil
131
+ File.readlines(conf).each do |line|
132
+ match = line.match(/^default.*([\d+])/)
133
+ if match
134
+ default = match.captures[0]
135
+ end
136
+ end
137
+ if not default
138
+ STDERR.puts "Couldn't find default kernel designation in grub config. The resulting image may not boot."
139
+ return
140
+ end
141
+ default = default.to_i
142
+
143
+ kernels = File.readlines(conf).grep(/^\s*kernel/)
144
+ kernel_line = kernels[default]
145
+
146
+ kernel_line = remove_kernel(tokenize(kernel_line))
147
+ kernel_hash = make_hash(kernel_line)
148
+
149
+ compare_hashes(cmdline_hash, kernel_hash)
150
+
151
+ if not @warnings.empty?
152
+ $stdout.puts "Found the following differences between your kernel " +
153
+ "commandline and the grub configuration on the volume:"
154
+
155
+ @warnings.each do |warning|
156
+ $stdout.puts warning
157
+ end
158
+
159
+ $stdout.puts "Please verify that the kernel command line in " +
160
+ "#{File.expand_path(conf)} is correct for your new AMI."
161
+ end
162
+ end
163
+
164
+ # Create the loopback image file and copy volume to it.
165
+ def make
166
+ begin
167
+ puts( "Copying #{@volume} into the image file #{@image_filename}...")
168
+ puts( 'Excluding: ' )
169
+ @exclude.each { |x| puts( "\t #{x}" ) }
170
+
171
+ create_image_file
172
+ format_image
173
+ execute( 'sync' ) # Flush so newly formatted filesystem is ready to mount.
174
+ mount_image
175
+ copy_rec( @volume, IMG_MNT)
176
+ update_fstab
177
+ customize_image
178
+ finalize_image
179
+ ensure
180
+ cleanup
181
+ end
182
+ end
183
+
184
+ #---------------------------------------------------------------------#
185
+ # Ensure we have the specified commonly-needed utils in the PATH.
186
+ def verify_runtime(utils, chroot = nil)
187
+ unless ENV['PATH']
188
+ raise FatalError.new('PATH not set, cannot find needed utilities')
189
+ end
190
+
191
+ paths = ENV['PATH'].split(File::PATH_SEPARATOR)
192
+ paths.map! { |path| File.join(chroot, path) } if chroot
193
+
194
+ utils.each do |util|
195
+ unless paths.any? { |dir| File.executable?(File.join(dir, util)) }
196
+ raise FatalError.new("Required utility '%s' not found in PATH - is it installed?" % util)
197
+ end
198
+ end
199
+ end
200
+
201
+ def check_deps(part_type)
202
+ if part_type == EC2::Platform::PartitionType::MBR
203
+ self.verify_runtime([ 'parted' ])
204
+ self.verify_runtime(CHROOT_UTILS, @volume)
205
+ elsif part_type == EC2::Platform::PartitionType::GPT
206
+ self.verify_runtime([ 'sgdisk' ])
207
+ self.verify_runtime(CHROOT_UTILS, @volume)
208
+ end
209
+ end
210
+
211
+ #---------------------------------------------------------------------#
212
+ # Assign an appropriate partition type. The current implementation will
213
+ # fail to bundle volumes that reside on devices whose partition schemes
214
+ # deviate from what is commonly available in EC2, namely a partitioned
215
+ # disk with the root file system residing on the first partition.
216
+ ROOT_DEVICE_REGEX = /^(\/dev\/(?:xvd|sd)(?:[a-z]|[a-c][a-z]|d[a-x]))[1]?$/
217
+
218
+ def set_partition_type(input)
219
+ input ||= EC2::Platform::PartitionType::NONE
220
+ if input == EC2::Platform::PartitionType::NONE
221
+ # We are not doing anything interesting. Return early.
222
+ puts('Not partitioning boot device.')
223
+ @part_type = EC2::Platform::PartitionType::NONE
224
+ return
225
+ end
226
+
227
+ # Verify that general partitioning utilities are present
228
+ self.verify_runtime(PART_UTILS)
229
+
230
+ value = input
231
+ puts('Setting partition type to bundle "%s" with...' % [@volume])
232
+ if File.directory?(@volume)
233
+ mtab = EC2::Platform::Linux::Mtab.load
234
+ entry = mtab.entries[@volume]
235
+ if entry
236
+ # Volume is a mounted file system:
237
+ # * Determine the mounted device
238
+ # * Ensure device partition scheme is one that we understand
239
+ # * Ensure device and it's container(if applicable) are block devices
240
+ # * Determine the current partition type using parted if appropriate.
241
+ device = entry.device
242
+ root = nil
243
+ if (match = ROOT_DEVICE_REGEX.match(device))
244
+ root = match[1]
245
+ root = device unless File.exists?(root) # classic AMI with no partition table
246
+ # Be paranoid. Bail unless the device and parent are block devices
247
+ [device, root].each do |dev|
248
+ unless File.blockdev?(dev)
249
+ raise FatalError.new('Not a block device: %s' % [dev])
250
+ end
251
+ end
252
+ else
253
+ raise FatalError.new('Non-standard volume device "%s"' % [device])
254
+ end
255
+ if input == :auto
256
+ self.verify_runtime([ 'parted' ])
257
+ puts('Auto-detecting partition type for "%s"' % [@volume])
258
+ cmd = "parted -s %s print|awk -F: '/^Partition Table/{print $2}'" % [root]
259
+ value = evaluate(cmd).strip
260
+ raise FatalError.new('Cannot determine partition type') if value.empty?
261
+ puts('Partition label detected using parted: "%s"' % value)
262
+ end
263
+ else
264
+ # Volume specified is possibly a file system root:
265
+ # * Proceed cautiously using sane defaults if no partition type
266
+ # has been provided.
267
+ puts('Volume "%s" is not a mount point.' % [@volume])
268
+ value = EC2::Platform::PartitionType::GPT if input == :auto
269
+ puts('Treating it as a file system root and using "%s"...' % [value])
270
+ end
271
+ elsif File.blockdev?(@volume)
272
+ # Volume specified is a block device:
273
+ # * Not sure how we got here.
274
+ # * We only support bundling of file system roots and not block
275
+ # devices, so throw an exception.
276
+ raise FatalError('Volume cannot be a block device "%s".' % [@volume])
277
+ else
278
+ # Volume specified is not a file system (mounted or otherwise):
279
+ # * Bail!
280
+ raise FatalError.new('Cannot determine partition type of "%s"' % [@volume])
281
+ end
282
+
283
+ if EC2::Platform::PartitionType.valid?(value)
284
+ @part_type = value
285
+ elsif value == 'msdos'
286
+ # This is the parted label value reported for MBR partition tables.
287
+ @part_type = EC2::Platform::PartitionType::MBR
288
+ elsif value == 'loop'
289
+ # This typically indicates that we have a bare partition that is not
290
+ # part of a partition table. This is typically the case for pv amis.
291
+ @part_type = EC2::Platform::PartitionType::NONE
292
+ elsif value == 'gpt'
293
+ @part_type = EC2::Platform::PartitionType::GPT
294
+ elsif value
295
+ if input == :auto
296
+ # Somehow we failed to determine a partition type that we support
297
+ raise FatalError.new('Could not determine a suitable partition type')
298
+ else
299
+ # User specified a format we currently do not support. Bail.
300
+ raise FatalError.new('Unsupported partition table type %s' % input)
301
+ end
302
+ else
303
+ raise FatalError.ne('Cannot determine partition type for %s' % [@volume])
304
+ end
305
+ puts('Using partition type "%s"' % @part_type)
306
+
307
+ self.check_deps(@part_type)
308
+ end
309
+
310
+ #---------------------------------------------------------------------#
311
+
312
+ def settle
313
+ # Run sync and udevadm settle to quiet device.
314
+ execute('sync||:')
315
+ if File.executable?('/usr/sbin/udevsettle')
316
+ execute('/usr/sbin/udevsettle||:')
317
+ elsif File.executable?('/sbin/udevadm')
318
+ execute('/sbin/udevadm settle||:')
319
+ end
320
+ end
321
+
322
+ #---------------------------------------------------------------------#
323
+ # Returns true if we are trying to build a valid disk image
324
+ def is_disk_image?
325
+ EC2::Platform::PartitionType.valid?(@part_type)
326
+ end
327
+
328
+ private
329
+
330
+ #---------------------------------------------------------------------#
331
+
332
+ def unmount(mpoint)
333
+ if mounted?(mpoint) then
334
+ execute('umount -d ' + mpoint)
335
+ end
336
+ end
337
+
338
+ #---------------------------------------------------------------------#
339
+
340
+ def mounted?(mpoint)
341
+ EC2::Platform::Linux::Mtab.load.entries.keys.include? mpoint
342
+ end
343
+
344
+ #---------------------------------------------------------------------#
345
+
346
+ # Unmount devices. Delete temporary files.
347
+ def cleanup
348
+ # Unmount image file.
349
+ if self.is_disk_image?
350
+ unmount('%s/sys' % IMG_MNT)
351
+ unmount('%s/proc' % IMG_MNT)
352
+ unmount('%s/dev' % IMG_MNT)
353
+ end
354
+
355
+ unmount(IMG_MNT)
356
+ if self.is_disk_image? and @diskdev
357
+ diskname = File.basename(@diskdev)
358
+ execute('kpartx -d %s' % @diskdev)
359
+ execute('dmsetup remove %s' % diskname)
360
+ execute('losetup -d %s' % @diskloop)
361
+ end
362
+ end
363
+
364
+ #---------------------------------------------------------------------#
365
+
366
+ # Call dd to create the image file.
367
+ def create_image_file
368
+ cmd = "dd if=/dev/zero status=noxfer of=" + @image_filename +
369
+ " bs=1M count=1 seek=" + (@mb_image_size-1).to_s
370
+ execute( cmd )
371
+ end
372
+
373
+ #---------------------------------------------------------------------#
374
+
375
+ # Format the image file, tune filesystem not to fsck based on interval.
376
+ # Where available and possible, retain the original root volume label
377
+ # uuid and file-system type falling back to using ext3 if not sure of
378
+ # what to do.
379
+ def format_image
380
+ mtab = EC2::Platform::Linux::Mtab.load
381
+ root = mtab.entries[Pathname(@volume).realpath.to_s].device rescue nil
382
+ info = fsinfo( root )
383
+ label= info[:label]
384
+ uuid = info[:uuid]
385
+ type = info[:type] || 'ext3'
386
+ execute('modprobe loop') unless File.blockdev?('/dev/loop0')
387
+
388
+ target = nil
389
+ if self.is_disk_image?
390
+ cmd = []
391
+ img = @image_filename
392
+ size = (@mb_image_size * 1024 * 1024 / 512)
393
+ case @part_type
394
+ when EC2::Platform::PartitionType::MBR
395
+ # Add a partition table and leave space to install a boot-loader.
396
+ # The boot partition fills up the disk. Note that '-1s' indicates
397
+ # the end of disk (and not 1 sector in from the end.
398
+ head = 63
399
+ cmd << ['unit s']
400
+ cmd << ['mklabel msdos']
401
+ cmd << ['mkpart primary %s -1s' % head]
402
+ cmd << ['set 1 boot on print quit']
403
+ cmd = "parted --script %s -- '%s'" % [img, cmd.join(' ')]
404
+ execute(cmd)
405
+ self.settle
406
+ when EC2::Platform::PartitionType::GPT
407
+ # Add a 1M (2048 sector) BIOS Boot Partition (BPP) to hold the
408
+ # GRUB bootloader and then fill the rest of the disk with the
409
+ # boot partition.
410
+ #
411
+ # * GRUB2 is BBP-aware and will automatically use this partition for
412
+ # stage2.
413
+ #
414
+ # * Legacy GRUB is not smart enough to use the BBP for stage 1.5.
415
+ # We deal with that during GRUB setup in #finalize_image.
416
+ #
417
+ # * Legacy GRUB knows enough about GPT partitions to reference
418
+ # them by number (avoiding the need for the hybrid MBR hack), so
419
+ # we set this partition to the maximum partition available to
420
+ # avoid incrementing the root partition number.
421
+ last = evaluate('sgdisk --print %s |grep "^Partition table holds up to"|cut -d" " -f6 ||:' % img).strip
422
+ last = 4 if last.empty? # fallback to 4.
423
+ execute('sgdisk --new %s:1M:+1M --change-name %s:"BIOS Boot Partition" --typecode %s:ef02 %s' % [last,last,last, img])
424
+ self.settle
425
+ execute('sgdisk --largest-new=1 --change-name 1:"Linux" --typecode 1:8300 %s' % img)
426
+ self.settle
427
+ execute('sgdisk --print %s' % img)
428
+ self.settle
429
+ else
430
+ raise NotImplementedError, "Partition table type %s not supported" % @part_type
431
+ end
432
+ self.settle
433
+
434
+ # The series of activities below are to ensure we can workaround
435
+ # the vagaries of various versions of GRUB. After we have created
436
+ # partition table above, we fake an hda-named device by leveraging
437
+ # device mapper to create a linear device named "/dev/mapper/hda".
438
+ # We then set @target to its first partition. This makes @target
439
+ # easy to manipulate pretty much in the same way that we handle
440
+ # non-partitioned images. All cleanup happens during #cleanup.
441
+ @diskloop = evaluate('losetup -f').strip
442
+ execute('losetup %s %s' % [@diskloop, @image_filename])
443
+ @diskdev = '/dev/mapper/hda'
444
+ @partdev = '%s1' % [ @diskdev]
445
+ diskname = File.basename(@diskdev)
446
+ loopname = File.basename(@diskloop)
447
+ majmin = IO.read('/sys/block/%s/dev' % loopname).strip
448
+ execute( 'echo 0 %s linear %s 0|dmsetup create %s' % [size, majmin, diskname] )
449
+ execute( 'kpartx -a %s' % [ @diskdev ] )
450
+ self.settle
451
+ @target = @partdev
452
+ @fstype = type
453
+ else
454
+ # Not creating a disk image.
455
+ @target = @image_filename
456
+ end
457
+
458
+ tune = nil
459
+ mkfs = [ '/sbin/mkfs.' + type ]
460
+ case type
461
+ when 'btrfs'
462
+ mkfs << [ '-L', label] unless label.to_s.empty?
463
+ mkfs << [ @target ]
464
+ when 'xfs'
465
+ mkfs << [ '-L', label] unless label.to_s.empty?
466
+ mkfs << [ @target ]
467
+ tune = [ '/usr/sbin/xfs_admin' ]
468
+ tune << [ '-U', uuid ] unless uuid.to_s.empty?
469
+ tune << [ @target ]
470
+ else
471
+ # Type unknown or ext2 or ext3 or ext4
472
+ # New e2fsprogs changed the default inode size to 256 which is
473
+ # incompatible with some older kernels, and older versions of
474
+ # grub. The options below change the behavior back to the
475
+ # expected RHEL5 behavior if we are bundling disk images. This
476
+ # is not ideal, but oh well.
477
+ if ['ext2', 'ext3', 'ext4'].include?(type)
478
+ # Clear all the defaults specified in /etc/mke2fs.conf
479
+ features = ['none']
480
+
481
+ # Get Filesytem Features as reported by dumpe2fs
482
+ output = evaluate("dumpe2fs -h %s | grep 'Filesystem features'" % root)
483
+ parts = output.split(':')[1].lstrip.split(' ')
484
+ features.concat(parts)
485
+ features.delete('needs_recovery')
486
+ if features.include?('64bit')
487
+ puts "WARNING: 64bit filesystem flag detected on root device (#{root}), resulting image may not boot"
488
+ end
489
+
490
+ if self.is_disk_image?
491
+ mkfs = [ '/sbin/mke2fs -t %s -v -m 1' % type ]
492
+ mkfs << ['-O', features.join(',')]
493
+ if ['ext2', 'ext3',].include?(type)
494
+ mkfs << [ '-I 128 -i 8192' ]
495
+ end
496
+ else
497
+ mkfs << ['-F']
498
+ mkfs << ['-O', features.join(',')]
499
+ end
500
+ else
501
+ # Unknown case
502
+ mkfs << ['-F']
503
+ end
504
+ mkfs << [ '-L', label] unless label.to_s.empty?
505
+ mkfs << [ @target ]
506
+ tune = [ '/sbin/tune2fs -i 0' ]
507
+ tune << [ '-U', uuid ] unless uuid.to_s.empty?
508
+ tune << [ @target ]
509
+ end
510
+ execute( mkfs.join( ' ' ) )
511
+ execute( tune.join( ' ' ) ) if tune
512
+ end
513
+
514
+ def customize_image
515
+ return unless @script and File.executable?(@script)
516
+ puts('Customizing cloned volume mounted at %s with script %s' % [IMG_MNT, @script])
517
+ output = evaluate('%s "%s"' % [@script, IMG_MNT])
518
+ STDERR.puts output if @debug
519
+ end
520
+
521
+ def finalize_image
522
+ return unless self.is_disk_image?
523
+ begin
524
+ # GRUB needs to reference the device.map file to know about our
525
+ # disk layout. So let's write out a simple one.
526
+ devmapfile = '%s/boot/grub/device.map' % IMG_MNT
527
+ FileUtils.mkdir_p(File.dirname(devmapfile))
528
+ File.open(devmapfile, 'w') do|io|
529
+ io << "(hd0) %s\n" % @diskdev
530
+ end
531
+
532
+ # Provide a suitable /etc/mtab if it doesn't already exist
533
+ fixmtab = false
534
+ mtabfile = '%s/etc/mtab' % IMG_MNT
535
+ execute('ln -s /proc/mounts %s' % mtabfile) unless File.exists?(mtabfile)
536
+
537
+ puts('Installing GRUB on root device with %s boot scheme' % @part_type)
538
+ # Newish versions of old GRUB expect the first partition
539
+ # of /dev/mapper/hda to be /dev/mapper/hda1. Lie to GRUB
540
+ # by adding a symlink to /dev/mapper/hda1.
541
+ hdap1 = '%s/dev/mapper/hdap1' % IMG_MNT
542
+ execute('ln -s ./hda1 %s' % hdap1) unless File.exists?(hdap1)
543
+
544
+ # Try to find the grub stages. There isn't a good way to know where
545
+ # exactly these will be on a given system, so this glob is a little
546
+ # excessive, but it should usually result in finding the path to
547
+ # the grub stages. It's also possible that it will exist outside
548
+ # /usr, but unlikely enough to not merit another glob there.
549
+ stage1 = Dir.glob("#{IMG_MNT}/usr/**/grub*/**/stage1")
550
+ if stage1.empty?
551
+ raise RuntimeError, "Couldn't find grub stages under #{IMG_MNT}"
552
+ end
553
+ stagesdir = File.dirname(stage1[0])
554
+
555
+ # Copy the stages into the grub dir on /boot
556
+ # Normally you'd let grub-install do this, but it can be very picky
557
+ # and doing the setup manually seems to be more reliable.
558
+ Dir.glob(["#{stagesdir}/stage{1,2}", "#{stagesdir}/*_stage1_5"]).each do |stage|
559
+ dest = "#{IMG_MNT}/boot/grub/%s" % File.basename(stage)
560
+ FileUtils.rm_f(dest)
561
+ FileUtils.cp(stage, dest)
562
+ end
563
+
564
+ # We're now ready to install GRUB.
565
+ case @part_type
566
+ when EC2::Platform::PartitionType::MBR
567
+ cmd = 'device (hd0) %s\nroot (hd0,0)\nsetup (hd0)' % @diskdev
568
+ execute('echo -e "%s" | grub --device-map=/dev/null --batch' % cmd, IMG_MNT)
569
+ when EC2::Platform::PartitionType::GPT
570
+ case @fstype
571
+ when /ext[234]/
572
+ file = '/boot/grub/e2fs_stage1_5'
573
+ when 'xfs'
574
+ file = '/boot/grub/xfs_stage1_5'
575
+ else
576
+ raise RuntimeError, 'File system type %s unsupported' % [@fstype]
577
+ end
578
+
579
+ file = File.join(IMG_MNT, file)
580
+ size = (File.size(file) + 511)/512
581
+ head = 2048 # start of the BIOS Boot Partition
582
+ cmd = 'dd if=%s of=%s seek=%s conv=fsync status=noxfer' % [file, @diskdev, head]
583
+ execute(cmd)
584
+ cmd = 'device (hd0) %s\n' % @diskdev
585
+ cmd += 'root (hd0,0)\n'
586
+ cmd += 'install /boot/grub/stage1 (hd0) (hd0)%s+%s ' % [head, size]
587
+ cmd += 'p (hd0,0)/boot/grub/stage2 /boot/grub/grub.conf'
588
+ execute('echo -e "%s" | grub --device-map=/dev/null --batch' % cmd, IMG_MNT)
589
+ else
590
+ raise RuntimeError, 'Unknown partition table type %s' % @part_type
591
+ end
592
+
593
+ # Check for reasonable kernel parameters
594
+ if @conf # user-supplied
595
+ src = Pathname(@conf).realpath.to_s
596
+ puts "Using user supplied grub config #{src}"
597
+ check_kernel_parameters(@conf)
598
+
599
+ dst_dir = File.join(IMG_MNT, '/boot/grub')
600
+ FileUtils.mkdir_p(dst_dir) unless File.exists?(dst_dir)
601
+
602
+ menulst = File.join(dst_dir, '/menu.lst')
603
+ # Copy the user supplied grub-config over any default ones
604
+ FileUtils.copy_entry(src, menulst, remove_destination=true)
605
+
606
+ grubconf = File.join(dst_dir, '/grub.conf')
607
+ File.delete(grubconf) unless not File.exists?(grubconf)
608
+ File.symlink('menu.lst', grubconf)
609
+ @conf = menulst
610
+ else
611
+ default_confs = [File.join(IMG_MNT, '/boot/grub/grub.conf'),
612
+ File.join(IMG_MNT, '/boot/grub/menu.lst')]
613
+ @conf = default_confs.find { |file| File.file?(file) }
614
+ if @conf
615
+ puts "Using default grub config"
616
+ check_kernel_parameters(@conf)
617
+ else
618
+ STDERR.puts('WARNING: No GRUB config found. The resulting image may not boot')
619
+ end
620
+ end
621
+
622
+ # Finally, tweak grub.conf to ensure we can boot.
623
+ adjustconf(@conf)
624
+ ensure
625
+ FileUtils.rm_f(hdap1)
626
+ FileUtils.rm_f(mtabfile) if fixmtab
627
+ end
628
+ end
629
+
630
+ def adjustconf(conf)
631
+ if conf
632
+ puts("Adjusting #{File.expand_path(conf)}")
633
+ conf = File.expand_path(conf)
634
+ data = IO.read(conf).split(/\n/).map do |line|
635
+ line.gsub(/root\s+\(hd0\)/, 'root (hd0,0)')
636
+ end
637
+ File.open(conf, 'w'){|io| io << data.join("\n")}
638
+ puts(evaluate('cat %s' % conf))
639
+ end
640
+ end
641
+
642
+ def fsinfo( fs )
643
+ result = {}
644
+ if fs and File.exists?( fs )
645
+ ['LABEL', 'UUID', 'TYPE' ].each do |tag|
646
+ begin
647
+ property = tag.downcase.to_sym
648
+ value = evaluate( '/sbin/blkid -o value -s %s %s' % [tag, fs] ).strip
649
+ result[property] = value if value and not value.empty?
650
+ rescue FatalError => e
651
+ if @debug
652
+ STDERR.puts e.message
653
+ STDERR.puts "Could not replicate file system #{property}. Proceeding..."
654
+ end
655
+ end
656
+ end
657
+ end
658
+ result
659
+ end
660
+
661
+ #---------------------------------------------------------------------#
662
+
663
+ # Mount the image file as a loopback device. The mount point is created
664
+ # if necessary.
665
+ def mount_image
666
+ Dir.mkdir(IMG_MNT) if not FileUtil::exists?(IMG_MNT)
667
+ raise FatalError.new("image already mounted") if mounted?(IMG_MNT)
668
+ dirs = ['mnt', 'proc', 'sys', 'dev']
669
+ if self.is_disk_image?
670
+ execute( 'mount -t %s %s %s' % [@fstype, @target, IMG_MNT] )
671
+ dirs.each{|dir| FileUtils.mkdir_p( '%s/%s' % [IMG_MNT, dir])}
672
+ make_special_devices
673
+ execute( 'mount -o bind /proc %s/proc' % IMG_MNT )
674
+ execute( 'mount -o bind /sys %s/sys' % IMG_MNT )
675
+ execute( 'mount -o bind /dev %s/dev' % IMG_MNT )
676
+ else
677
+ execute( 'mount -o loop ' + @target + ' ' + IMG_MNT )
678
+ dirs.each{ |dir| FileUtils.mkdir_p( '%s/%s' % [IMG_MNT, dir]) }
679
+ make_special_devices
680
+ end
681
+ end
682
+
683
+ #---------------------------------------------------------------------#
684
+ # Copy the contents of the specified source directory to the specified
685
+ # target directory, recursing sub-directories. Directories within the
686
+ # exclusion list are not copied. Symlinks are retained but not traversed.
687
+ #
688
+ # src: The source directory name.
689
+ # dst: The destination directory name.
690
+ # options: A set of options to try.
691
+ def copy_rec( src, dst, options={:xattributes => true} )
692
+ begin
693
+ rsync = EC2::Platform::Linux::Rsync::Command.new
694
+ rsync.archive.times.recursive.sparse.links.quietly.include(@includes).exclude(@exclude)
695
+ if @filter
696
+ rsync.exclude(EC2::Platform::Linux::Constants::Security::FILE_FILTER)
697
+ end
698
+ rsync.xattributes if options[ :xattributes ]
699
+ rsync.src(File::join( src, '*' )).dst(dst)
700
+ execute(rsync.expand)
701
+ return true
702
+ rescue Exception => e
703
+ rc = $?.exitstatus
704
+ return true if rc == 0
705
+ if rc == 23 and SysChecks::rsync_usable?
706
+ STDERR.puts [
707
+ 'NOTE: rsync seemed successful but exited with error code 23. This probably means',
708
+ 'that your version of rsync was built against a kernel with HAVE_LUTIMES defined,',
709
+ 'although the current kernel was not built with this option enabled. The bundling',
710
+ 'process will thus ignore the error and continue bundling. If bundling completes',
711
+ 'successfully, your image should be perfectly usable. We, however, recommend that',
712
+ 'you install a version of rsync that handles this situation more elegantly.'
713
+ ].join("\n")
714
+ return true
715
+ elsif rc == 1 and options[ :xattributes ]
716
+ STDERR.puts [
717
+ 'NOTE: rsync with preservation of extended file attributes failed. Retrying rsync',
718
+ 'without attempting to preserve extended file attributes...'
719
+ ].join("\n")
720
+ o = options.clone
721
+ o[ :xattributes ] = false
722
+ return copy_rec( src, dst, o)
723
+ end
724
+ raise e
725
+ end
726
+ end
727
+
728
+ #----------------------------------------------------------------------------#
729
+
730
+ def make_special_devices
731
+ execute("mknod %s/dev/null c 1 3" % IMG_MNT)
732
+ execute("mknod %s/dev/zero c 1 5" % IMG_MNT)
733
+ execute("mknod %s/dev/tty c 5 0" % IMG_MNT)
734
+ execute("mknod %s/dev/console c 5 1" % IMG_MNT)
735
+ execute("ln -s null %s/dev/X0R" % IMG_MNT)
736
+ end
737
+
738
+ #----------------------------------------------------------------------------#
739
+
740
+ def make_fstab
741
+ case @fstab
742
+ when :legacy
743
+ return LEGACY_FSTAB
744
+ when :default
745
+ return DEFAULT_FSTAB
746
+ else
747
+ return @fstab
748
+ end
749
+ end
750
+
751
+ #----------------------------------------------------------------------------#
752
+
753
+ def update_fstab
754
+ if @fstab
755
+ etc = File::join( IMG_MNT, 'etc')
756
+ fstab = File::join( etc, 'fstab' )
757
+
758
+ FileUtils::mkdir_p( etc ) unless File::exist?( etc)
759
+ execute( "cp #{fstab} #{fstab}.old" ) if File.exist?( fstab )
760
+ fstab_content = make_fstab
761
+ File.open( fstab, 'w' ) { |f| f.write( fstab_content ) }
762
+ puts "/etc/fstab:"
763
+ fstab_content.each_line do |s|
764
+ puts "\t #{s}"
765
+ end
766
+ end
767
+ end
768
+
769
+ #----------------------------------------------------------------------------#
770
+
771
+ # Execute the command line _cmd_.
772
+ def execute( cmd, chroot = nil, nullenv = true )
773
+ command = cmd
774
+ if chroot and not File.directory?(chroot)
775
+ raise FatalError.new('Cannot chroot into %s. Not a directory' % [chroot])
776
+ end
777
+ if chroot
778
+ env = nullenv ? 'env -i' : ''
779
+ command = 'setarch %s chroot %s %s %s' % [@arch, chroot, env, cmd]
780
+ else
781
+ command = cmd
782
+ end
783
+
784
+ if @debug
785
+ if chroot
786
+ STDERR.puts( 'Executing(chroot=%s): %s' % [chroot, command ] )
787
+ else
788
+ STDERR.puts( 'Executing: %s' % command )
789
+ end
790
+ else
791
+ command += ' >/dev/null 2>&1'
792
+ end
793
+ raise FatalError.new("Failed to execute: '#{cmd}'") unless system( command )
794
+ end
795
+
796
+ #---------------------------------------------------------------------------#
797
+ # Execute command line passed in and return STDOUT output if successful.
798
+ def evaluate( cmd, success = 0, verbattim = nil )
799
+ verbattim = @debug if verbattim.nil?
800
+ STDERR.puts( "Evaluating: %s" % cmd ) if verbattim
801
+ pid, stdin, stdout, stderr = Open4::popen4( cmd )
802
+ pid, status = Process::waitpid2 pid
803
+ unless status.exitstatus == success
804
+ raise FatalError.new( "Failed to evaluate '#{cmd }'. Reason: #{stderr.read}." )
805
+ end
806
+ stdout.read
807
+ end
808
+ end
809
+ end
810
+ end
811
+ end