ec2_amitools 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +54 -0
  3. data/bin/console +14 -0
  4. data/bin/ec2-ami-tools-version +6 -0
  5. data/bin/ec2-bundle-image +6 -0
  6. data/bin/ec2-bundle-vol +6 -0
  7. data/bin/ec2-delete-bundle +6 -0
  8. data/bin/ec2-download-bundle +6 -0
  9. data/bin/ec2-migrate-bundle +6 -0
  10. data/bin/ec2-migrate-manifest +6 -0
  11. data/bin/ec2-unbundle +6 -0
  12. data/bin/ec2-upload-bundle +6 -0
  13. data/bin/setup +8 -0
  14. data/etc/ec2/amitools/cert-ec2-cn-north-1.pem +28 -0
  15. data/etc/ec2/amitools/cert-ec2-gov.pem +17 -0
  16. data/etc/ec2/amitools/cert-ec2.pem +23 -0
  17. data/etc/ec2/amitools/mappings.csv +9 -0
  18. data/lib/ec2/amitools/bundle.rb +251 -0
  19. data/lib/ec2/amitools/bundle_base.rb +58 -0
  20. data/lib/ec2/amitools/bundleimage.rb +94 -0
  21. data/lib/ec2/amitools/bundleimageparameters.rb +42 -0
  22. data/lib/ec2/amitools/bundlemachineparameters.rb +60 -0
  23. data/lib/ec2/amitools/bundleparameters.rb +120 -0
  24. data/lib/ec2/amitools/bundlevol.rb +240 -0
  25. data/lib/ec2/amitools/bundlevolparameters.rb +164 -0
  26. data/lib/ec2/amitools/crypto.rb +379 -0
  27. data/lib/ec2/amitools/decryptmanifest.rb +20 -0
  28. data/lib/ec2/amitools/defaults.rb +12 -0
  29. data/lib/ec2/amitools/deletebundle.rb +212 -0
  30. data/lib/ec2/amitools/deletebundleparameters.rb +78 -0
  31. data/lib/ec2/amitools/downloadbundle.rb +161 -0
  32. data/lib/ec2/amitools/downloadbundleparameters.rb +84 -0
  33. data/lib/ec2/amitools/exception.rb +86 -0
  34. data/lib/ec2/amitools/fileutil.rb +219 -0
  35. data/lib/ec2/amitools/format.rb +127 -0
  36. data/lib/ec2/amitools/instance-data.rb +97 -0
  37. data/lib/ec2/amitools/manifest_wrapper.rb +132 -0
  38. data/lib/ec2/amitools/manifestv20070829.rb +361 -0
  39. data/lib/ec2/amitools/manifestv20071010.rb +403 -0
  40. data/lib/ec2/amitools/manifestv3.rb +331 -0
  41. data/lib/ec2/amitools/mapids.rb +148 -0
  42. data/lib/ec2/amitools/migratebundle.rb +222 -0
  43. data/lib/ec2/amitools/migratebundleparameters.rb +173 -0
  44. data/lib/ec2/amitools/migratemanifest.rb +225 -0
  45. data/lib/ec2/amitools/migratemanifestparameters.rb +118 -0
  46. data/lib/ec2/amitools/minimalec2.rb +116 -0
  47. data/lib/ec2/amitools/parameter_exceptions.rb +34 -0
  48. data/lib/ec2/amitools/parameters_base.rb +168 -0
  49. data/lib/ec2/amitools/region.rb +93 -0
  50. data/lib/ec2/amitools/s3toolparameters.rb +183 -0
  51. data/lib/ec2/amitools/showversion.rb +12 -0
  52. data/lib/ec2/amitools/syschecks.rb +27 -0
  53. data/lib/ec2/amitools/tool_base.rb +224 -0
  54. data/lib/ec2/amitools/unbundle.rb +107 -0
  55. data/lib/ec2/amitools/unbundleparameters.rb +65 -0
  56. data/lib/ec2/amitools/uploadbundle.rb +361 -0
  57. data/lib/ec2/amitools/uploadbundleparameters.rb +108 -0
  58. data/lib/ec2/amitools/util.rb +532 -0
  59. data/lib/ec2/amitools/version.rb +33 -0
  60. data/lib/ec2/amitools/xmlbuilder.rb +237 -0
  61. data/lib/ec2/amitools/xmlutil.rb +55 -0
  62. data/lib/ec2/common/constants.rb +16 -0
  63. data/lib/ec2/common/curl.rb +110 -0
  64. data/lib/ec2/common/headers.rb +95 -0
  65. data/lib/ec2/common/headersv4.rb +173 -0
  66. data/lib/ec2/common/http.rb +333 -0
  67. data/lib/ec2/common/s3support.rb +231 -0
  68. data/lib/ec2/common/signature.rb +68 -0
  69. data/lib/ec2/oem/LICENSE.txt +58 -0
  70. data/lib/ec2/oem/open4.rb +399 -0
  71. data/lib/ec2/platform/base/architecture.rb +26 -0
  72. data/lib/ec2/platform/base/constants.rb +54 -0
  73. data/lib/ec2/platform/base/pipeline.rb +181 -0
  74. data/lib/ec2/platform/base.rb +57 -0
  75. data/lib/ec2/platform/current.rb +55 -0
  76. data/lib/ec2/platform/linux/architecture.rb +35 -0
  77. data/lib/ec2/platform/linux/constants.rb +23 -0
  78. data/lib/ec2/platform/linux/fstab.rb +99 -0
  79. data/lib/ec2/platform/linux/identity.rb +16 -0
  80. data/lib/ec2/platform/linux/image.rb +811 -0
  81. data/lib/ec2/platform/linux/mtab.rb +74 -0
  82. data/lib/ec2/platform/linux/pipeline.rb +40 -0
  83. data/lib/ec2/platform/linux/rsync.rb +114 -0
  84. data/lib/ec2/platform/linux/tar.rb +124 -0
  85. data/lib/ec2/platform/linux/uname.rb +50 -0
  86. data/lib/ec2/platform/linux.rb +83 -0
  87. data/lib/ec2/platform/solaris/architecture.rb +28 -0
  88. data/lib/ec2/platform/solaris/constants.rb +30 -0
  89. data/lib/ec2/platform/solaris/fstab.rb +43 -0
  90. data/lib/ec2/platform/solaris/identity.rb +16 -0
  91. data/lib/ec2/platform/solaris/image.rb +327 -0
  92. data/lib/ec2/platform/solaris/mtab.rb +29 -0
  93. data/lib/ec2/platform/solaris/pipeline.rb +40 -0
  94. data/lib/ec2/platform/solaris/rsync.rb +24 -0
  95. data/lib/ec2/platform/solaris/tar.rb +36 -0
  96. data/lib/ec2/platform/solaris/uname.rb +21 -0
  97. data/lib/ec2/platform/solaris.rb +38 -0
  98. data/lib/ec2/platform.rb +69 -0
  99. data/lib/ec2/version.rb +8 -0
  100. data/lib/ec2_amitools +1 -0
  101. data/lib/ec2_amitools.rb +7 -0
  102. metadata +184 -0
@@ -0,0 +1,16 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ module EC2
12
+ module Platform
13
+ module Linux; end
14
+ EC2::Platform::PEER = EC2::Platform::Linux unless defined? EC2::Platform::PEER
15
+ end
16
+ end
@@ -0,0 +1,811 @@
1
+ # Copyright 2008-2014 Amazon.com, Inc. or its affiliates. All Rights
2
+ # Reserved. Licensed under the Amazon Software License (the
3
+ # "License"). You may not use this file except in compliance with the
4
+ # License. A copy of the License is located at
5
+ # http://aws.amazon.com/asl or in the "license" file accompanying this
6
+ # file. This file is distributed on an "AS IS" BASIS, WITHOUT
7
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See
8
+ # the License for the specific language governing permissions and
9
+ # limitations under the License.
10
+
11
+ require 'fileutils'
12
+ require 'pathname'
13
+ require 'ec2/oem/open4'
14
+ require 'ec2/amitools/fileutil'
15
+ require 'ec2/amitools/syschecks'
16
+ require 'ec2/amitools/exception'
17
+ require 'ec2/platform/linux/mtab'
18
+ require 'ec2/platform/linux/fstab'
19
+ require 'ec2/platform/linux/constants'
20
+
21
+ module EC2
22
+ module Platform
23
+ module Linux
24
+
25
+ # This class encapsulate functionality to create an file loopback image
26
+ # from a volume. The image is created using dd. Sub-directories of the
27
+ # volume, including mounts of local filesystems, are copied to the image.
28
+ # Symbolic links are preserved.
29
+ class Image
30
+ IMG_MNT = '/mnt/img-mnt'
31
+ EXCLUDES= ['/dev', '/media', '/mnt', '/proc', '/sys']
32
+ DEFAULT_FSTAB = EC2::Platform::Linux::Fstab::DEFAULT
33
+ LEGACY_FSTAB = EC2::Platform::Linux::Fstab::LEGACY
34
+ BASE_UTILS = [ 'modprobe', 'mount', 'umount', 'dd' ]
35
+ PART_UTILS = [ 'dmsetup', 'kpartx', 'losetup' ]
36
+ CHROOT_UTILS = [ 'grub' ]
37
+
38
+ #---------------------------------------------------------------------#
39
+
40
+ # Initialize the instance with the required parameters.
41
+ # * _volume_ The path to the volume to create the image file from.
42
+ # * _image_filename_ The name of the image file to create.
43
+ # * _mb_image_size_ The image file size in MB.
44
+ # * _exclude_ List of directories to exclude.
45
+ # * _debug_ Make extra noise.
46
+ def initialize( volume,
47
+ image_filename,
48
+ mb_image_size,
49
+ exclude,
50
+ includes,
51
+ filter = true,
52
+ fstab = nil,
53
+ part_type = nil,
54
+ arch = nil,
55
+ script = nil,
56
+ debug = false,
57
+ grub_config = nil )
58
+ @volume = volume
59
+ @image_filename = image_filename
60
+ @mb_image_size = mb_image_size
61
+ @exclude = exclude
62
+ @includes = includes
63
+ @filter = filter
64
+ @arch = arch || EC2::Platform::Linux::Uname.platform
65
+ @script = script
66
+ @fstab = nil
67
+ @conf = grub_config
68
+ @warnings = Array.new
69
+
70
+ self.verify_runtime(BASE_UTILS)
71
+ self.set_partition_type(part_type)
72
+
73
+ # Cunning plan or horrible hack?
74
+ # If :legacy is passed in as the fstab, we use the old v3 manifest's
75
+ # device naming and fstab.
76
+ if [:legacy, :default].include? fstab
77
+ @fstab = fstab
78
+ elsif not fstab.nil?
79
+ @fstab = File.open(fstab).read()
80
+ end
81
+ @debug = debug
82
+
83
+ # Exclude the temporary image mount point if it is under the volume
84
+ # being bundled.
85
+ if IMG_MNT.index( volume ) == 0
86
+ @exclude << IMG_MNT
87
+ end
88
+ end
89
+
90
+ #--------------------------------------------------------------------#
91
+
92
+ def make_hash(array)
93
+ hash = Hash.new
94
+ array.each do |entry|
95
+ # Split on the first '='
96
+ key, value = entry.split('=', 2)
97
+ hash[key] = value || ''
98
+ end
99
+ hash
100
+ end
101
+
102
+ def compare_hashes(a, b)
103
+ a.each do |key,value|
104
+ if not b.has_key?(key)
105
+ @warnings.push "\t* Missing key '#{key}' with value '#{value}'"
106
+ elsif b[key] != value
107
+ @warnings.push "\t* Key '#{key}' value '#{b[key]}' differs from /proc/cmdline value '#{value}'"
108
+ end
109
+ end
110
+ end
111
+
112
+ def remove_kernel(tokens)
113
+ if tokens.include?('kernel')
114
+ tokens.delete('kernel')
115
+ # The kernel can receive optional arguments, drop those along with the kernel
116
+ kernel_index = tokens.index{ |token| !token.include?('--') }
117
+ tokens = tokens.drop(kernel_index + 1)
118
+ end
119
+ tokens
120
+ end
121
+
122
+ def tokenize(string)
123
+ string.strip.split(/\s+/)
124
+ end
125
+
126
+ def check_kernel_parameters(conf)
127
+ cmdline = File.read('/proc/cmdline')
128
+ cmdline_hash = make_hash(tokenize(cmdline))
129
+
130
+ default = nil
131
+ File.readlines(conf).each do |line|
132
+ match = line.match(/^default.*([\d+])/)
133
+ if match
134
+ default = match.captures[0]
135
+ end
136
+ end
137
+ if not default
138
+ STDERR.puts "Couldn't find default kernel designation in grub config. The resulting image may not boot."
139
+ return
140
+ end
141
+ default = default.to_i
142
+
143
+ kernels = File.readlines(conf).grep(/^\s*kernel/)
144
+ kernel_line = kernels[default]
145
+
146
+ kernel_line = remove_kernel(tokenize(kernel_line))
147
+ kernel_hash = make_hash(kernel_line)
148
+
149
+ compare_hashes(cmdline_hash, kernel_hash)
150
+
151
+ if not @warnings.empty?
152
+ $stdout.puts "Found the following differences between your kernel " +
153
+ "commandline and the grub configuration on the volume:"
154
+
155
+ @warnings.each do |warning|
156
+ $stdout.puts warning
157
+ end
158
+
159
+ $stdout.puts "Please verify that the kernel command line in " +
160
+ "#{File.expand_path(conf)} is correct for your new AMI."
161
+ end
162
+ end
163
+
164
+ # Create the loopback image file and copy volume to it.
165
+ def make
166
+ begin
167
+ puts( "Copying #{@volume} into the image file #{@image_filename}...")
168
+ puts( 'Excluding: ' )
169
+ @exclude.each { |x| puts( "\t #{x}" ) }
170
+
171
+ create_image_file
172
+ format_image
173
+ execute( 'sync' ) # Flush so newly formatted filesystem is ready to mount.
174
+ mount_image
175
+ copy_rec( @volume, IMG_MNT)
176
+ update_fstab
177
+ customize_image
178
+ finalize_image
179
+ ensure
180
+ cleanup
181
+ end
182
+ end
183
+
184
+ #---------------------------------------------------------------------#
185
+ # Ensure we have the specified commonly-needed utils in the PATH.
186
+ def verify_runtime(utils, chroot = nil)
187
+ unless ENV['PATH']
188
+ raise FatalError.new('PATH not set, cannot find needed utilities')
189
+ end
190
+
191
+ paths = ENV['PATH'].split(File::PATH_SEPARATOR)
192
+ paths.map! { |path| File.join(chroot, path) } if chroot
193
+
194
+ utils.each do |util|
195
+ unless paths.any? { |dir| File.executable?(File.join(dir, util)) }
196
+ raise FatalError.new("Required utility '%s' not found in PATH - is it installed?" % util)
197
+ end
198
+ end
199
+ end
200
+
201
+ def check_deps(part_type)
202
+ if part_type == EC2::Platform::PartitionType::MBR
203
+ self.verify_runtime([ 'parted' ])
204
+ self.verify_runtime(CHROOT_UTILS, @volume)
205
+ elsif part_type == EC2::Platform::PartitionType::GPT
206
+ self.verify_runtime([ 'sgdisk' ])
207
+ self.verify_runtime(CHROOT_UTILS, @volume)
208
+ end
209
+ end
210
+
211
+ #---------------------------------------------------------------------#
212
+ # Assign an appropriate partition type. The current implementation will
213
+ # fail to bundle volumes that reside on devices whose partition schemes
214
+ # deviate from what is commonly available in EC2, namely a partitioned
215
+ # disk with the root file system residing on the first partition.
216
+ ROOT_DEVICE_REGEX = /^(\/dev\/(?:xvd|sd)(?:[a-z]|[a-c][a-z]|d[a-x]))[1]?$/
217
+
218
+ def set_partition_type(input)
219
+ input ||= EC2::Platform::PartitionType::NONE
220
+ if input == EC2::Platform::PartitionType::NONE
221
+ # We are not doing anything interesting. Return early.
222
+ puts('Not partitioning boot device.')
223
+ @part_type = EC2::Platform::PartitionType::NONE
224
+ return
225
+ end
226
+
227
+ # Verify that general partitioning utilities are present
228
+ self.verify_runtime(PART_UTILS)
229
+
230
+ value = input
231
+ puts('Setting partition type to bundle "%s" with...' % [@volume])
232
+ if File.directory?(@volume)
233
+ mtab = EC2::Platform::Linux::Mtab.load
234
+ entry = mtab.entries[@volume]
235
+ if entry
236
+ # Volume is a mounted file system:
237
+ # * Determine the mounted device
238
+ # * Ensure device partition scheme is one that we understand
239
+ # * Ensure device and it's container(if applicable) are block devices
240
+ # * Determine the current partition type using parted if appropriate.
241
+ device = entry.device
242
+ root = nil
243
+ if (match = ROOT_DEVICE_REGEX.match(device))
244
+ root = match[1]
245
+ root = device unless File.exists?(root) # classic AMI with no partition table
246
+ # Be paranoid. Bail unless the device and parent are block devices
247
+ [device, root].each do |dev|
248
+ unless File.blockdev?(dev)
249
+ raise FatalError.new('Not a block device: %s' % [dev])
250
+ end
251
+ end
252
+ else
253
+ raise FatalError.new('Non-standard volume device "%s"' % [device])
254
+ end
255
+ if input == :auto
256
+ self.verify_runtime([ 'parted' ])
257
+ puts('Auto-detecting partition type for "%s"' % [@volume])
258
+ cmd = "parted -s %s print|awk -F: '/^Partition Table/{print $2}'" % [root]
259
+ value = evaluate(cmd).strip
260
+ raise FatalError.new('Cannot determine partition type') if value.empty?
261
+ puts('Partition label detected using parted: "%s"' % value)
262
+ end
263
+ else
264
+ # Volume specified is possibly a file system root:
265
+ # * Proceed cautiously using sane defaults if no partition type
266
+ # has been provided.
267
+ puts('Volume "%s" is not a mount point.' % [@volume])
268
+ value = EC2::Platform::PartitionType::GPT if input == :auto
269
+ puts('Treating it as a file system root and using "%s"...' % [value])
270
+ end
271
+ elsif File.blockdev?(@volume)
272
+ # Volume specified is a block device:
273
+ # * Not sure how we got here.
274
+ # * We only support bundling of file system roots and not block
275
+ # devices, so throw an exception.
276
+ raise FatalError('Volume cannot be a block device "%s".' % [@volume])
277
+ else
278
+ # Volume specified is not a file system (mounted or otherwise):
279
+ # * Bail!
280
+ raise FatalError.new('Cannot determine partition type of "%s"' % [@volume])
281
+ end
282
+
283
+ if EC2::Platform::PartitionType.valid?(value)
284
+ @part_type = value
285
+ elsif value == 'msdos'
286
+ # This is the parted label value reported for MBR partition tables.
287
+ @part_type = EC2::Platform::PartitionType::MBR
288
+ elsif value == 'loop'
289
+ # This typically indicates that we have a bare partition that is not
290
+ # part of a partition table. This is typically the case for pv amis.
291
+ @part_type = EC2::Platform::PartitionType::NONE
292
+ elsif value == 'gpt'
293
+ @part_type = EC2::Platform::PartitionType::GPT
294
+ elsif value
295
+ if input == :auto
296
+ # Somehow we failed to determine a partition type that we support
297
+ raise FatalError.new('Could not determine a suitable partition type')
298
+ else
299
+ # User specified a format we currently do not support. Bail.
300
+ raise FatalError.new('Unsupported partition table type %s' % input)
301
+ end
302
+ else
303
+ raise FatalError.ne('Cannot determine partition type for %s' % [@volume])
304
+ end
305
+ puts('Using partition type "%s"' % @part_type)
306
+
307
+ self.check_deps(@part_type)
308
+ end
309
+
310
+ #---------------------------------------------------------------------#
311
+
312
+ def settle
313
+ # Run sync and udevadm settle to quiet device.
314
+ execute('sync||:')
315
+ if File.executable?('/usr/sbin/udevsettle')
316
+ execute('/usr/sbin/udevsettle||:')
317
+ elsif File.executable?('/sbin/udevadm')
318
+ execute('/sbin/udevadm settle||:')
319
+ end
320
+ end
321
+
322
+ #---------------------------------------------------------------------#
323
+ # Returns true if we are trying to build a valid disk image
324
+ def is_disk_image?
325
+ EC2::Platform::PartitionType.valid?(@part_type)
326
+ end
327
+
328
+ private
329
+
330
+ #---------------------------------------------------------------------#
331
+
332
+ def unmount(mpoint)
333
+ if mounted?(mpoint) then
334
+ execute('umount -d ' + mpoint)
335
+ end
336
+ end
337
+
338
+ #---------------------------------------------------------------------#
339
+
340
+ def mounted?(mpoint)
341
+ EC2::Platform::Linux::Mtab.load.entries.keys.include? mpoint
342
+ end
343
+
344
+ #---------------------------------------------------------------------#
345
+
346
+ # Unmount devices. Delete temporary files.
347
+ def cleanup
348
+ # Unmount image file.
349
+ if self.is_disk_image?
350
+ unmount('%s/sys' % IMG_MNT)
351
+ unmount('%s/proc' % IMG_MNT)
352
+ unmount('%s/dev' % IMG_MNT)
353
+ end
354
+
355
+ unmount(IMG_MNT)
356
+ if self.is_disk_image? and @diskdev
357
+ diskname = File.basename(@diskdev)
358
+ execute('kpartx -d %s' % @diskdev)
359
+ execute('dmsetup remove %s' % diskname)
360
+ execute('losetup -d %s' % @diskloop)
361
+ end
362
+ end
363
+
364
+ #---------------------------------------------------------------------#
365
+
366
+ # Call dd to create the image file.
367
+ def create_image_file
368
+ cmd = "dd if=/dev/zero status=noxfer of=" + @image_filename +
369
+ " bs=1M count=1 seek=" + (@mb_image_size-1).to_s
370
+ execute( cmd )
371
+ end
372
+
373
+ #---------------------------------------------------------------------#
374
+
375
+ # Format the image file, tune filesystem not to fsck based on interval.
376
+ # Where available and possible, retain the original root volume label
377
+ # uuid and file-system type falling back to using ext3 if not sure of
378
+ # what to do.
379
+ def format_image
380
+ mtab = EC2::Platform::Linux::Mtab.load
381
+ root = mtab.entries[Pathname(@volume).realpath.to_s].device rescue nil
382
+ info = fsinfo( root )
383
+ label= info[:label]
384
+ uuid = info[:uuid]
385
+ type = info[:type] || 'ext3'
386
+ execute('modprobe loop') unless File.blockdev?('/dev/loop0')
387
+
388
+ target = nil
389
+ if self.is_disk_image?
390
+ cmd = []
391
+ img = @image_filename
392
+ size = (@mb_image_size * 1024 * 1024 / 512)
393
+ case @part_type
394
+ when EC2::Platform::PartitionType::MBR
395
+ # Add a partition table and leave space to install a boot-loader.
396
+ # The boot partition fills up the disk. Note that '-1s' indicates
397
+ # the end of disk (and not 1 sector in from the end.
398
+ head = 63
399
+ cmd << ['unit s']
400
+ cmd << ['mklabel msdos']
401
+ cmd << ['mkpart primary %s -1s' % head]
402
+ cmd << ['set 1 boot on print quit']
403
+ cmd = "parted --script %s -- '%s'" % [img, cmd.join(' ')]
404
+ execute(cmd)
405
+ self.settle
406
+ when EC2::Platform::PartitionType::GPT
407
+ # Add a 1M (2048 sector) BIOS Boot Partition (BPP) to hold the
408
+ # GRUB bootloader and then fill the rest of the disk with the
409
+ # boot partition.
410
+ #
411
+ # * GRUB2 is BBP-aware and will automatically use this partition for
412
+ # stage2.
413
+ #
414
+ # * Legacy GRUB is not smart enough to use the BBP for stage 1.5.
415
+ # We deal with that during GRUB setup in #finalize_image.
416
+ #
417
+ # * Legacy GRUB knows enough about GPT partitions to reference
418
+ # them by number (avoiding the need for the hybrid MBR hack), so
419
+ # we set this partition to the maximum partition available to
420
+ # avoid incrementing the root partition number.
421
+ last = evaluate('sgdisk --print %s |grep "^Partition table holds up to"|cut -d" " -f6 ||:' % img).strip
422
+ last = 4 if last.empty? # fallback to 4.
423
+ execute('sgdisk --new %s:1M:+1M --change-name %s:"BIOS Boot Partition" --typecode %s:ef02 %s' % [last,last,last, img])
424
+ self.settle
425
+ execute('sgdisk --largest-new=1 --change-name 1:"Linux" --typecode 1:8300 %s' % img)
426
+ self.settle
427
+ execute('sgdisk --print %s' % img)
428
+ self.settle
429
+ else
430
+ raise NotImplementedError, "Partition table type %s not supported" % @part_type
431
+ end
432
+ self.settle
433
+
434
+ # The series of activities below are to ensure we can workaround
435
+ # the vagaries of various versions of GRUB. After we have created
436
+ # partition table above, we fake an hda-named device by leveraging
437
+ # device mapper to create a linear device named "/dev/mapper/hda".
438
+ # We then set @target to its first partition. This makes @target
439
+ # easy to manipulate pretty much in the same way that we handle
440
+ # non-partitioned images. All cleanup happens during #cleanup.
441
+ @diskloop = evaluate('losetup -f').strip
442
+ execute('losetup %s %s' % [@diskloop, @image_filename])
443
+ @diskdev = '/dev/mapper/hda'
444
+ @partdev = '%s1' % [ @diskdev]
445
+ diskname = File.basename(@diskdev)
446
+ loopname = File.basename(@diskloop)
447
+ majmin = IO.read('/sys/block/%s/dev' % loopname).strip
448
+ execute( 'echo 0 %s linear %s 0|dmsetup create %s' % [size, majmin, diskname] )
449
+ execute( 'kpartx -a %s' % [ @diskdev ] )
450
+ self.settle
451
+ @target = @partdev
452
+ @fstype = type
453
+ else
454
+ # Not creating a disk image.
455
+ @target = @image_filename
456
+ end
457
+
458
+ tune = nil
459
+ mkfs = [ '/sbin/mkfs.' + type ]
460
+ case type
461
+ when 'btrfs'
462
+ mkfs << [ '-L', label] unless label.to_s.empty?
463
+ mkfs << [ @target ]
464
+ when 'xfs'
465
+ mkfs << [ '-L', label] unless label.to_s.empty?
466
+ mkfs << [ @target ]
467
+ tune = [ '/usr/sbin/xfs_admin' ]
468
+ tune << [ '-U', uuid ] unless uuid.to_s.empty?
469
+ tune << [ @target ]
470
+ else
471
+ # Type unknown or ext2 or ext3 or ext4
472
+ # New e2fsprogs changed the default inode size to 256 which is
473
+ # incompatible with some older kernels, and older versions of
474
+ # grub. The options below change the behavior back to the
475
+ # expected RHEL5 behavior if we are bundling disk images. This
476
+ # is not ideal, but oh well.
477
+ if ['ext2', 'ext3', 'ext4'].include?(type)
478
+ # Clear all the defaults specified in /etc/mke2fs.conf
479
+ features = ['none']
480
+
481
+ # Get Filesytem Features as reported by dumpe2fs
482
+ output = evaluate("dumpe2fs -h %s | grep 'Filesystem features'" % root)
483
+ parts = output.split(':')[1].lstrip.split(' ')
484
+ features.concat(parts)
485
+ features.delete('needs_recovery')
486
+ if features.include?('64bit')
487
+ puts "WARNING: 64bit filesystem flag detected on root device (#{root}), resulting image may not boot"
488
+ end
489
+
490
+ if self.is_disk_image?
491
+ mkfs = [ '/sbin/mke2fs -t %s -v -m 1' % type ]
492
+ mkfs << ['-O', features.join(',')]
493
+ if ['ext2', 'ext3',].include?(type)
494
+ mkfs << [ '-I 128 -i 8192' ]
495
+ end
496
+ else
497
+ mkfs << ['-F']
498
+ mkfs << ['-O', features.join(',')]
499
+ end
500
+ else
501
+ # Unknown case
502
+ mkfs << ['-F']
503
+ end
504
+ mkfs << [ '-L', label] unless label.to_s.empty?
505
+ mkfs << [ @target ]
506
+ tune = [ '/sbin/tune2fs -i 0' ]
507
+ tune << [ '-U', uuid ] unless uuid.to_s.empty?
508
+ tune << [ @target ]
509
+ end
510
+ execute( mkfs.join( ' ' ) )
511
+ execute( tune.join( ' ' ) ) if tune
512
+ end
513
+
514
+ def customize_image
515
+ return unless @script and File.executable?(@script)
516
+ puts('Customizing cloned volume mounted at %s with script %s' % [IMG_MNT, @script])
517
+ output = evaluate('%s "%s"' % [@script, IMG_MNT])
518
+ STDERR.puts output if @debug
519
+ end
520
+
521
+ def finalize_image
522
+ return unless self.is_disk_image?
523
+ begin
524
+ # GRUB needs to reference the device.map file to know about our
525
+ # disk layout. So let's write out a simple one.
526
+ devmapfile = '%s/boot/grub/device.map' % IMG_MNT
527
+ FileUtils.mkdir_p(File.dirname(devmapfile))
528
+ File.open(devmapfile, 'w') do|io|
529
+ io << "(hd0) %s\n" % @diskdev
530
+ end
531
+
532
+ # Provide a suitable /etc/mtab if it doesn't already exist
533
+ fixmtab = false
534
+ mtabfile = '%s/etc/mtab' % IMG_MNT
535
+ execute('ln -s /proc/mounts %s' % mtabfile) unless File.exists?(mtabfile)
536
+
537
+ puts('Installing GRUB on root device with %s boot scheme' % @part_type)
538
+ # Newish versions of old GRUB expect the first partition
539
+ # of /dev/mapper/hda to be /dev/mapper/hda1. Lie to GRUB
540
+ # by adding a symlink to /dev/mapper/hda1.
541
+ hdap1 = '%s/dev/mapper/hdap1' % IMG_MNT
542
+ execute('ln -s ./hda1 %s' % hdap1) unless File.exists?(hdap1)
543
+
544
+ # Try to find the grub stages. There isn't a good way to know where
545
+ # exactly these will be on a given system, so this glob is a little
546
+ # excessive, but it should usually result in finding the path to
547
+ # the grub stages. It's also possible that it will exist outside
548
+ # /usr, but unlikely enough to not merit another glob there.
549
+ stage1 = Dir.glob("#{IMG_MNT}/usr/**/grub*/**/stage1")
550
+ if stage1.empty?
551
+ raise RuntimeError, "Couldn't find grub stages under #{IMG_MNT}"
552
+ end
553
+ stagesdir = File.dirname(stage1[0])
554
+
555
+ # Copy the stages into the grub dir on /boot
556
+ # Normally you'd let grub-install do this, but it can be very picky
557
+ # and doing the setup manually seems to be more reliable.
558
+ Dir.glob(["#{stagesdir}/stage{1,2}", "#{stagesdir}/*_stage1_5"]).each do |stage|
559
+ dest = "#{IMG_MNT}/boot/grub/%s" % File.basename(stage)
560
+ FileUtils.rm_f(dest)
561
+ FileUtils.cp(stage, dest)
562
+ end
563
+
564
+ # We're now ready to install GRUB.
565
+ case @part_type
566
+ when EC2::Platform::PartitionType::MBR
567
+ cmd = 'device (hd0) %s\nroot (hd0,0)\nsetup (hd0)' % @diskdev
568
+ execute('echo -e "%s" | grub --device-map=/dev/null --batch' % cmd, IMG_MNT)
569
+ when EC2::Platform::PartitionType::GPT
570
+ case @fstype
571
+ when /ext[234]/
572
+ file = '/boot/grub/e2fs_stage1_5'
573
+ when 'xfs'
574
+ file = '/boot/grub/xfs_stage1_5'
575
+ else
576
+ raise RuntimeError, 'File system type %s unsupported' % [@fstype]
577
+ end
578
+
579
+ file = File.join(IMG_MNT, file)
580
+ size = (File.size(file) + 511)/512
581
+ head = 2048 # start of the BIOS Boot Partition
582
+ cmd = 'dd if=%s of=%s seek=%s conv=fsync status=noxfer' % [file, @diskdev, head]
583
+ execute(cmd)
584
+ cmd = 'device (hd0) %s\n' % @diskdev
585
+ cmd += 'root (hd0,0)\n'
586
+ cmd += 'install /boot/grub/stage1 (hd0) (hd0)%s+%s ' % [head, size]
587
+ cmd += 'p (hd0,0)/boot/grub/stage2 /boot/grub/grub.conf'
588
+ execute('echo -e "%s" | grub --device-map=/dev/null --batch' % cmd, IMG_MNT)
589
+ else
590
+ raise RuntimeError, 'Unknown partition table type %s' % @part_type
591
+ end
592
+
593
+ # Check for reasonable kernel parameters
594
+ if @conf # user-supplied
595
+ src = Pathname(@conf).realpath.to_s
596
+ puts "Using user supplied grub config #{src}"
597
+ check_kernel_parameters(@conf)
598
+
599
+ dst_dir = File.join(IMG_MNT, '/boot/grub')
600
+ FileUtils.mkdir_p(dst_dir) unless File.exists?(dst_dir)
601
+
602
+ menulst = File.join(dst_dir, '/menu.lst')
603
+ # Copy the user supplied grub-config over any default ones
604
+ FileUtils.copy_entry(src, menulst, remove_destination=true)
605
+
606
+ grubconf = File.join(dst_dir, '/grub.conf')
607
+ File.delete(grubconf) unless not File.exists?(grubconf)
608
+ File.symlink('menu.lst', grubconf)
609
+ @conf = menulst
610
+ else
611
+ default_confs = [File.join(IMG_MNT, '/boot/grub/grub.conf'),
612
+ File.join(IMG_MNT, '/boot/grub/menu.lst')]
613
+ @conf = default_confs.find { |file| File.file?(file) }
614
+ if @conf
615
+ puts "Using default grub config"
616
+ check_kernel_parameters(@conf)
617
+ else
618
+ STDERR.puts('WARNING: No GRUB config found. The resulting image may not boot')
619
+ end
620
+ end
621
+
622
+ # Finally, tweak grub.conf to ensure we can boot.
623
+ adjustconf(@conf)
624
+ ensure
625
+ FileUtils.rm_f(hdap1)
626
+ FileUtils.rm_f(mtabfile) if fixmtab
627
+ end
628
+ end
629
+
630
+ def adjustconf(conf)
631
+ if conf
632
+ puts("Adjusting #{File.expand_path(conf)}")
633
+ conf = File.expand_path(conf)
634
+ data = IO.read(conf).split(/\n/).map do |line|
635
+ line.gsub(/root\s+\(hd0\)/, 'root (hd0,0)')
636
+ end
637
+ File.open(conf, 'w'){|io| io << data.join("\n")}
638
+ puts(evaluate('cat %s' % conf))
639
+ end
640
+ end
641
+
642
+ def fsinfo( fs )
643
+ result = {}
644
+ if fs and File.exists?( fs )
645
+ ['LABEL', 'UUID', 'TYPE' ].each do |tag|
646
+ begin
647
+ property = tag.downcase.to_sym
648
+ value = evaluate( '/sbin/blkid -o value -s %s %s' % [tag, fs] ).strip
649
+ result[property] = value if value and not value.empty?
650
+ rescue FatalError => e
651
+ if @debug
652
+ STDERR.puts e.message
653
+ STDERR.puts "Could not replicate file system #{property}. Proceeding..."
654
+ end
655
+ end
656
+ end
657
+ end
658
+ result
659
+ end
660
+
661
+ #---------------------------------------------------------------------#
662
+
663
+ # Mount the image file as a loopback device. The mount point is created
664
+ # if necessary.
665
+ def mount_image
666
+ Dir.mkdir(IMG_MNT) if not FileUtil::exists?(IMG_MNT)
667
+ raise FatalError.new("image already mounted") if mounted?(IMG_MNT)
668
+ dirs = ['mnt', 'proc', 'sys', 'dev']
669
+ if self.is_disk_image?
670
+ execute( 'mount -t %s %s %s' % [@fstype, @target, IMG_MNT] )
671
+ dirs.each{|dir| FileUtils.mkdir_p( '%s/%s' % [IMG_MNT, dir])}
672
+ make_special_devices
673
+ execute( 'mount -o bind /proc %s/proc' % IMG_MNT )
674
+ execute( 'mount -o bind /sys %s/sys' % IMG_MNT )
675
+ execute( 'mount -o bind /dev %s/dev' % IMG_MNT )
676
+ else
677
+ execute( 'mount -o loop ' + @target + ' ' + IMG_MNT )
678
+ dirs.each{ |dir| FileUtils.mkdir_p( '%s/%s' % [IMG_MNT, dir]) }
679
+ make_special_devices
680
+ end
681
+ end
682
+
683
+ #---------------------------------------------------------------------#
684
+ # Copy the contents of the specified source directory to the specified
685
+ # target directory, recursing sub-directories. Directories within the
686
+ # exclusion list are not copied. Symlinks are retained but not traversed.
687
+ #
688
+ # src: The source directory name.
689
+ # dst: The destination directory name.
690
+ # options: A set of options to try.
691
+ def copy_rec( src, dst, options={:xattributes => true} )
692
+ begin
693
+ rsync = EC2::Platform::Linux::Rsync::Command.new
694
+ rsync.archive.times.recursive.sparse.links.quietly.include(@includes).exclude(@exclude)
695
+ if @filter
696
+ rsync.exclude(EC2::Platform::Linux::Constants::Security::FILE_FILTER)
697
+ end
698
+ rsync.xattributes if options[ :xattributes ]
699
+ rsync.src(File::join( src, '*' )).dst(dst)
700
+ execute(rsync.expand)
701
+ return true
702
+ rescue Exception => e
703
+ rc = $?.exitstatus
704
+ return true if rc == 0
705
+ if rc == 23 and SysChecks::rsync_usable?
706
+ STDERR.puts [
707
+ 'NOTE: rsync seemed successful but exited with error code 23. This probably means',
708
+ 'that your version of rsync was built against a kernel with HAVE_LUTIMES defined,',
709
+ 'although the current kernel was not built with this option enabled. The bundling',
710
+ 'process will thus ignore the error and continue bundling. If bundling completes',
711
+ 'successfully, your image should be perfectly usable. We, however, recommend that',
712
+ 'you install a version of rsync that handles this situation more elegantly.'
713
+ ].join("\n")
714
+ return true
715
+ elsif rc == 1 and options[ :xattributes ]
716
+ STDERR.puts [
717
+ 'NOTE: rsync with preservation of extended file attributes failed. Retrying rsync',
718
+ 'without attempting to preserve extended file attributes...'
719
+ ].join("\n")
720
+ o = options.clone
721
+ o[ :xattributes ] = false
722
+ return copy_rec( src, dst, o)
723
+ end
724
+ raise e
725
+ end
726
+ end
727
+
728
+ #----------------------------------------------------------------------------#
729
+
730
+ def make_special_devices
731
+ execute("mknod %s/dev/null c 1 3" % IMG_MNT)
732
+ execute("mknod %s/dev/zero c 1 5" % IMG_MNT)
733
+ execute("mknod %s/dev/tty c 5 0" % IMG_MNT)
734
+ execute("mknod %s/dev/console c 5 1" % IMG_MNT)
735
+ execute("ln -s null %s/dev/X0R" % IMG_MNT)
736
+ end
737
+
738
+ #----------------------------------------------------------------------------#
739
+
740
+ def make_fstab
741
+ case @fstab
742
+ when :legacy
743
+ return LEGACY_FSTAB
744
+ when :default
745
+ return DEFAULT_FSTAB
746
+ else
747
+ return @fstab
748
+ end
749
+ end
750
+
751
+ #----------------------------------------------------------------------------#
752
+
753
+ def update_fstab
754
+ if @fstab
755
+ etc = File::join( IMG_MNT, 'etc')
756
+ fstab = File::join( etc, 'fstab' )
757
+
758
+ FileUtils::mkdir_p( etc ) unless File::exist?( etc)
759
+ execute( "cp #{fstab} #{fstab}.old" ) if File.exist?( fstab )
760
+ fstab_content = make_fstab
761
+ File.open( fstab, 'w' ) { |f| f.write( fstab_content ) }
762
+ puts "/etc/fstab:"
763
+ fstab_content.each_line do |s|
764
+ puts "\t #{s}"
765
+ end
766
+ end
767
+ end
768
+
769
+ #----------------------------------------------------------------------------#
770
+
771
+ # Execute the command line _cmd_.
772
+ def execute( cmd, chroot = nil, nullenv = true )
773
+ command = cmd
774
+ if chroot and not File.directory?(chroot)
775
+ raise FatalError.new('Cannot chroot into %s. Not a directory' % [chroot])
776
+ end
777
+ if chroot
778
+ env = nullenv ? 'env -i' : ''
779
+ command = 'setarch %s chroot %s %s %s' % [@arch, chroot, env, cmd]
780
+ else
781
+ command = cmd
782
+ end
783
+
784
+ if @debug
785
+ if chroot
786
+ STDERR.puts( 'Executing(chroot=%s): %s' % [chroot, command ] )
787
+ else
788
+ STDERR.puts( 'Executing: %s' % command )
789
+ end
790
+ else
791
+ command += ' >/dev/null 2>&1'
792
+ end
793
+ raise FatalError.new("Failed to execute: '#{cmd}'") unless system( command )
794
+ end
795
+
796
+ #---------------------------------------------------------------------------#
797
+ # Execute command line passed in and return STDOUT output if successful.
798
+ def evaluate( cmd, success = 0, verbattim = nil )
799
+ verbattim = @debug if verbattim.nil?
800
+ STDERR.puts( "Evaluating: %s" % cmd ) if verbattim
801
+ pid, stdin, stdout, stderr = Open4::popen4( cmd )
802
+ pid, status = Process::waitpid2 pid
803
+ unless status.exitstatus == success
804
+ raise FatalError.new( "Failed to evaluate '#{cmd }'. Reason: #{stderr.read}." )
805
+ end
806
+ stdout.read
807
+ end
808
+ end
809
+ end
810
+ end
811
+ end