boxgrinder-build 0.9.0 → 0.9.1

data/lib/boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin.rb

@@ -75,12 +75,12 @@ module BoxGrinder
 
       @log.info "Building #{@appliance_config.name} appliance..."
 
-      @exec_helper.execute "appliance-creator -d -v -t '#{@dir.tmp}' --cache=#{@config.dir.cache}/rpms-cache/#{@appliance_config.path.main} --config '#{kickstart_file}' -o '#{@dir.tmp}' --name '#{@appliance_config.name}' --vmem #{@appliance_config.hardware.memory} --vcpu #{@appliance_config.hardware.cpus} --format #{@plugin_config['format']}"
+      execute_appliance_creator(kickstart_file)
 
       FileUtils.mv(Dir.glob("#{@dir.tmp}/#{@appliance_config.name}/*"), @dir.tmp)
       FileUtils.rm_rf("#{@dir.tmp}/#{@appliance_config.name}/")
 
-      @image_helper.customize(@deliverables.disk) do |guestfs, guestfs_helper|
+      @image_helper.customize([@deliverables.disk]) do |guestfs, guestfs_helper|
         # TODO is this really needed?
         @log.debug "Uploading '/etc/resolv.conf'..."
         guestfs.upload("/etc/resolv.conf", "/etc/resolv.conf")
@@ -89,7 +89,6 @@ module BoxGrinder
         change_configuration(guestfs_helper)
         # TODO check if this is still required
         apply_root_password(guestfs)
-        fix_partition_labels(guestfs)
         use_labels_for_partitions(guestfs)
         disable_firewall(guestfs)
         set_motd(guestfs)
@@ -116,6 +115,56 @@ module BoxGrinder
       @log.info "Base image for #{@appliance_config.name} appliance was built successfully."
     end
 
+    def execute_appliance_creator(kickstart_file)
+      begin
+        @exec_helper.execute "appliance-creator -d -v -t '#{@dir.tmp}' --cache=#{@config.dir.cache}/rpms-cache/#{@appliance_config.path.main} --config '#{kickstart_file}' -o '#{@dir.tmp}' --name '#{@appliance_config.name}' --vmem #{@appliance_config.hardware.memory} --vcpu #{@appliance_config.hardware.cpus} --format #{@plugin_config['format']}"
+      rescue InterruptionError => e
+        cleanup_after_appliance_creator(e.pid)
+        abort
+      end
+    end
+
+    def cleanup_after_appliance_creator(pid)
+      @log.debug "Sending TERM signal to process '#{pid}'..."
+      Process.kill("TERM", pid)
+
+      @log.debug "Waiting for process to be terminated..."
+      Process.wait(pid)
+
+      @log.debug "Cleaning appliance-creator mount points..."
+
+      Dir["#{@dir.tmp}/imgcreate-*"].each do |dir|
+        dev_mapper = @exec_helper.execute "mount | grep #{dir} | awk '{print $1}'"
+
+        mappings = {}
+
+        dev_mapper.each do |mapping|
+          if mapping =~ /(loop\d+)p(\d+)/
+            mappings[$1] = [] if mappings[$1].nil?
+            mappings[$1] << $2 unless mappings[$1].include?($2)
+          end
+        end
+
+        (['/var/cache/yum', '/dev/shm', '/dev/pts', '/proc', '/sys'] + @appliance_config.hardware.partitions.keys.reverse).each do |mount_point|
+          @log.trace "Unmounting '#{mount_point}'..."
+          @exec_helper.execute "umount -d #{dir}/install_root#{mount_point}"
+        end
+
+        mappings.each do |loop, partitions|
+          @log.trace "Removing mappings from loop device #{loop}..."
+          @exec_helper.execute "/sbin/kpartx -d /dev/#{loop}"
+          @exec_helper.execute "losetup -d /dev/#{loop}"
+
+          partitions.each do |part|
+            @log.trace "Removing mapping for partition #{part} from loop device #{loop}..."
+            @exec_helper.execute "rm /dev/#{loop}#{part}"
+          end
+        end
+      end
+
+      @log.debug "Cleaned up after appliance-creator."
+    end
+
     # https://issues.jboss.org/browse/BGBUILD-177
     def disable_firewall(guestfs)
       @log.debug "Disabling firewall..."
@@ -123,12 +172,6 @@ module BoxGrinder
       @log.debug "Firewall disabled."
     end
 
-    def fix_partition_labels(guestfs)
-      guestfs.list_partitions.each do |partition|
-        guestfs.sh("/sbin/e2label #{partition} #{read_label(guestfs, partition)}")
-      end
-    end
-
     def use_labels_for_partitions(guestfs)
       device = guestfs.list_devices.first
 

data/lib/boxgrinder-build/plugins/os/rpm-based/src/appliance.ks.erb

@@ -17,8 +17,12 @@ services --enabled=network
 
 rootpw --iscrypted <%= appliance_config.os.password.crypt((0...8).map { 65.+(rand(25)).chr }.join) %>
 
-<% appliance_config.hardware.partitions.each do |root, partition| %>
-part <%= root %> --size <%= (partition['size'].to_f * 1024).to_i %> --fstype <%= partition['type'] %> <% unless partition['options'].nil? %> --fsoptions '<%= partition['options'] %>' <% end %> <% if partition['passphrase'] %> --encrypted --passphrase='<%= partition['passphrase'] %>' <% end %> --ondisk sda --label <%= root %><% end %>
+# appliance-creator doesn't support labels...
+# --label Zlib.crc32(root).to_s(16)
+
+<% mount_points.each do |root| %>
+<% partition = appliance_config.hardware.partitions[root]%>
+part <%= root %> --size <%= (partition['size'].to_f * 1024).to_i %> --fstype <%= partition['type'] %> <% unless partition['options'].nil? %> --fsoptions '<%= partition['options'] %>' <% end %> <% if partition['passphrase'] %> --encrypted --passphrase='<%= partition['passphrase'] %>' <% end %> --ondisk sda<% end %>
 
 <% for repo in repos %>
 <%= repo %><% end %>

data/lib/boxgrinder-build/plugins/platform/ec2/ec2-plugin.rb

@@ -35,10 +35,11 @@ module BoxGrinder
 
       @log.info "Converting #{@appliance_config.name} appliance image to EC2 format..."
 
-      create_ec2_disk
-      sync_files
+      @image_helper.create_disk(@deliverables.disk, 10) # 10 GB destination disk
+
+      @image_helper.customize([@previous_deliverables.disk, @deliverables.disk], :automount => false) do |guestfs, guestfs_helper|
+        @image_helper.sync_filesystem(guestfs, guestfs_helper)
 
-      @image_helper.customize(@deliverables.disk) do |guestfs, guestfs_helper|
         if (@appliance_config.os.name == 'rhel' or @appliance_config.os.name == 'centos') and @appliance_config.os.version == '5'
           # Not sure why it's messed but this prevents booting on AWS
           recreate_journal(guestfs)
@@ -75,47 +76,6 @@ module BoxGrinder
       @log.info "Image converted to EC2 format."
     end
 
-    def sync_files
-      ec2_disk_mount_dir = "#{@dir.tmp}/ec2-#{rand(9999999999).to_s.center(10, rand(9).to_s)}"
-      raw_disk_mount_dir = "#{@dir.tmp}/raw-#{rand(9999999999).to_s.center(10, rand(9).to_s)}"
-
-      tmp_disk = "#{@dir.tmp}/#{@appliance_config.name}.raw"
-
-      @log.debug "Conveting disk to RAW format..."
-      @image_helper.convert_disk(@previous_deliverables.disk, 'raw', tmp_disk)
-
-      begin
-        ec2_mounts = @image_helper.mount_image(@deliverables.disk, ec2_disk_mount_dir)
-        raw_mounts = @image_helper.mount_image(tmp_disk, raw_disk_mount_dir)
-      rescue => e
-        @log.error e
-        @log.error "Mouting failed, trying to clean up."
-
-        @image_helper.umount_image(tmp_disk, raw_disk_mount_dir, raw_mounts) unless raw_mounts.nil?
-        @image_helper.umount_image(@deliverables.disk, ec2_disk_mount_dir, ec2_mounts) unless ec2_mounts.nil?
-
-        raise "Error while mounting image. See logs for more info."
-      end
-
-      @image_helper.sync_files(raw_disk_mount_dir, ec2_disk_mount_dir)
-
-      @image_helper.umount_image(tmp_disk, raw_disk_mount_dir, raw_mounts)
-      @image_helper.umount_image(@deliverables.disk, ec2_disk_mount_dir, ec2_mounts)
-
-      FileUtils.rm_rf tmp_disk
-    end
-
-    def create_ec2_disk
-      begin
-        # TODO using whole 10GB is fine?
-        @image_helper.create_disk(@deliverables.disk, 10)
-        @image_helper.create_filesystem(@deliverables.disk)
-      rescue => e
-        @log.error e
-        raise "Error while preparing EC2 disk image. See logs for more info."
-      end
-    end
-
     def execute_post(guestfs_helper)
       unless @appliance_config.post['ec2'].nil?
         @appliance_config.post['ec2'].each do |cmd|

data/lib/boxgrinder-build/plugins/platform/ec2/src/fstab_32bit

@@ -1,4 +1,4 @@
-LABEL=/    /         #FILESYSTEM_TYPE#    defaults         1 1
+LABEL=79d3d2d4    /         #FILESYSTEM_TYPE#    defaults         1 1
 /dev/#DISK_DEVICE_PREFIX#da2  /mnt      ext3    defaults         1 2
 /dev/#DISK_DEVICE_PREFIX#da3  swap      swap    defaults         0 0
 none       /dev/pts  devpts  gid=5,mode=620   0 0

data/lib/boxgrinder-build/plugins/platform/ec2/src/fstab_64bit

@@ -1,4 +1,4 @@
-LABEL=/    /         #FILESYSTEM_TYPE#    defaults         1 1
+LABEL=79d3d2d4    /         #FILESYSTEM_TYPE#    defaults         1 1
 /dev/#DISK_DEVICE_PREFIX#db   /mnt      ext3    defaults         0 0
 /dev/#DISK_DEVICE_PREFIX#dc   /data     ext3    defaults         0 0
 none       /dev/pts  devpts  gid=5,mode=620   0 0

data/lib/boxgrinder-build/plugins/platform/ec2/src/menu.lst

@@ -2,5 +2,5 @@ default=0
 timeout=0
 title #TITLE#
         root (hd0)
-        kernel /boot/vmlinuz-#KERNEL_VERSION# ro root=LABEL=/ rd_NO_PLYMOUTH
-        initrd /boot/#KERNEL_IMAGE_NAME#-#KERNEL_VERSION#.img
+        kernel /boot/vmlinuz-#KERNEL_VERSION# ro root=LABEL=79d3d2d4 rd_NO_PLYMOUTH
+        initrd /boot/#KERNEL_IMAGE_NAME#-#KERNEL_VERSION#.img

data/lib/boxgrinder-build/plugins/platform/virtualbox/virtualbox-plugin.rb

@@ -17,7 +17,7 @@
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
 require 'boxgrinder-build/plugins/base-plugin'
-require 'boxgrinder-build/helpers/appliance-customize-helper'
+require 'boxgrinder-build/helpers/linux-helper'
 
 module BoxGrinder
   class VirtualBoxPlugin < BasePlugin
@@ -29,11 +29,21 @@ module BoxGrinder
       @log.info "Converting image to VirtualBox format..."
 
       build_virtualbox
-      customize
+
+      if is_os_old? or !@appliance_config.post['virtualbox'].nil?
+        @image_helper.customize([@deliverables.disk]) do |guestfs, guestfs_helper|
+          recreate_kernel(guestfs) if is_os_old?
+          customize(guestfs_helper) unless @appliance_config.post['virtualbox'].nil?
+        end
+      end
 
       @log.info "Image converted to VirtualBox format."
     end
 
+    def is_os_old?
+      (@appliance_config.os.name == 'centos' or @appliance_config.os.name == 'rhel') and @appliance_config.os.version == '5'
+    end
+
     def build_virtualbox
       @log.debug "Building VirtualBox image."
 
@@ -44,19 +54,20 @@ module BoxGrinder
       @log.debug "VirtualBox image was built."
     end
 
-    def customize
-      unless @appliance_config.post['virtualbox'].nil?
-        @image_helper.customize(@deliverables.disk) do |guestfs, guestfs_helper|
-          @appliance_config.post['virtualbox'].each do |cmd|
-            guestfs_helper.sh(cmd, :arch => @appliance_config.hardware.arch)
-          end
-          @log.debug "Post commands from appliance definition file executed."
-        end
-      else
-        @log.debug "No commands specified, skipping."
+    def recreate_kernel(guestfs)
+      @log.info "Recreating kernel to include require modules..."
+      LinuxHelper.new(:log => @log).recreate_kernel_image(guestfs, ['ahci'])
+      @log.info "Kernel recreated."
+    end
+
+    def customize(guestfs_helper)
+      @log.debug "Executing post commands from appliance definition..."
+      @appliance_config.post['virtualbox'].each do |cmd|
+        guestfs_helper.sh(cmd, :arch => @appliance_config.hardware.arch)
       end
+      @log.debug "Post commands from appliance definition file executed."
     end
   end
 end
 
-plugin :class => BoxGrinder::VirtualBoxPlugin, :type => :platform, :name => :virtualbox, :full_name  => "VirtualBox"
+plugin :class => BoxGrinder::VirtualBoxPlugin, :type => :platform, :name => :virtualbox, :full_name => "VirtualBox"

data/lib/boxgrinder-build/plugins/platform/vmware/src/base.vmx

@@ -5,9 +5,9 @@ config.version = "8"
 virtualHW.version = "4"
 guestOS = "#GUESTOS#"
 displayName = "#NAME#"
-annotation = "#SUMMARY# | Version: #VERSION# | Built using BoxGrinder"
+annotation = "#NAME# | Version: #VERSION# | Built using BoxGrinder"
 guestinfo.vmware.product.long = "#NAME#"
-guestinfo.vmware.product.url = "http://www.jboss.org/boxgrinder"
+guestinfo.vmware.product.url = "http://boxgrinder.org"
 guestinfo.vmware.product.class = "virtual machine"
 numvcpus = "#VCPU#"
 memsize = "#MEM_SIZE#"

data/lib/boxgrinder-build/plugins/platform/vmware/vmware-plugin.rb

@@ -17,7 +17,6 @@
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
 require 'boxgrinder-build/plugins/base-plugin'
-require 'boxgrinder-build/helpers/appliance-customize-helper'
 
 module BoxGrinder
   class VMwarePlugin < BasePlugin

data/rubygem-boxgrinder-build.spec

@@ -5,7 +5,7 @@
 
 Summary: A tool for creating appliances from simple plain text files
 Name: rubygem-%{gemname}
-Version: 0.9.0
+Version: 0.9.1
 Release: 1%{?dist}
 Group: Development/Languages
 License: LGPLv3+
@@ -19,6 +19,9 @@ Requires: ruby-libguestfs
 Requires: parted
 Requires: e2fsprogs
 
+# Fix for rubygem-aws package
+Requires: rubygem(activesupport)
+
 # For EL5
 BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
 BuildArch: noarch
@@ -30,6 +33,9 @@ BuildRequires: rubygem(boxgrinder-core) < 0.4.0
 BuildRequires: rubygem(echoe)
 BuildRequires: ruby-libguestfs
 
+# Fix for rubygem-aws package
+BuildRequires: rubygem(activesupport)
+
 # EBS and S3
 Requires: rubygem(amazon-ec2)
 # Fixes blankslate error
@@ -60,6 +66,11 @@ Requires: rsync
 Requires: wget
 Requires: util-linux
 
+# ElasticHosts
+Requires: rubygem(rest-client)
+
+BuildRequires: rubygem(rest-client)
+
 Provides: rubygem(%{gemname}) = %{version}
 
 Obsoletes: rubygem(boxgrinder-build-ebs-delivery-plugin) < 0.0.4-2
@@ -137,6 +148,17 @@ popd
 %{gemdir}/doc/%{gemname}-%{version}
 
 %changelog
+* Thu Mar 17 2011 <mgoldman@redhat.com> - 0.9.1-1
+- Upstream release: 0.9.1
+- [BGBUILD-188] Use libuestfs instead mounting partitions manually for EC2 appliances
+- [BGBUILD-97] some filesystems dont get unmounted on BG interruption
+- [BGBUILD-155] Images built on Centos5.x (el5) for VirtualBox kernel panic (/dev/root missing)
+- [BGBUILD-190] Allow to specify kernel variant (PAE or not) for Fedora OS
+- [BGBUILD-196] GuestFS fails mounting partitions where more then 3 partitions are present
+- [BGBUILD-200] /sbin/e2label: Filesystem has unsupported feature(s) while trying to open /dev/sda1
+- [BGBUILD-194] Add support for ElasticHosts cloud
+- [BGBUILD-202] Unable to get valid context for ec2-user after login on AMI
+
 * Tue Mar 01 2011 <msavy@redhat.com> - 0.9.0-1
 - Upstream release: 0.9.0
 - [BGBUILD-103] README to indicate supported operating systems / requirements

data/spec/appliance-spec.rb

@@ -145,7 +145,8 @@ module BoxGrinder
         appliance_config = ApplianceConfig.new
 
         appliance_helper = mock(ApplianceDefinitionHelper)
-        appliance_helper.should_receive(:read_definitions).with("#{File.dirname(__FILE__)}/rspec/src/appliances/jeos-f13.ks").and_raise("Unknown format")
+        appliance_helper.should_receive(:read_definitions).with("#{File.dirname(__FILE__)}/rspec/src/appliances/jeos-f13.ks")
+        appliance_helper.should_receive(:appliance_configs).and_return([])
 
         clazz = mock('PluginClass')
 
@@ -179,7 +180,8 @@ module BoxGrinder
         prepare_appliance({}, "#{File.dirname(__FILE__)}/rspec/src/appliances/jeos-f13.ks")
 
         appliance_helper = mock(ApplianceDefinitionHelper)
-        appliance_helper.should_receive(:read_definitions).with("#{File.dirname(__FILE__)}/rspec/src/appliances/jeos-f13.ks").and_raise("Unknown format")
+        appliance_helper.should_receive(:read_definitions).with("#{File.dirname(__FILE__)}/rspec/src/appliances/jeos-f13.ks")
+        appliance_helper.should_receive(:appliance_configs).and_return([])
 
         plugin_manager = mock(PluginManager)
         plugin_manager.should_receive(:plugins).and_return({:os => {}})
@@ -191,83 +193,7 @@ module BoxGrinder
 
         lambda {
           @appliance.read_definition
-        }.should raise_error("Couldn't read appliance definition file: jeos-f13.ks")
-      end
-    end
-
-    describe ".validate_definition" do
-      it "should validate definition and pass" do
-        prepare_appliance
-        @appliance.instance_variable_set(:@appliance_config, prepare_appliance_config)
-
-        appliance_config_validator = mock(ApplianceConfigValidator)
-        appliance_config_validator.should_receive(:validate)
-
-        ApplianceConfigValidator.should_receive(:new).with(@appliance_config).and_return(appliance_config_validator)
-
-        plugin_manager = mock(PluginManager)
-        plugin_manager.stub!(:plugins).and_return({:os => {:fedora => {:type => :os, :name => :fedora, :full_name => "Fedora", :versions => ["11", "12", "13", "14", "rawhide"]}}})
-
-        PluginManager.stub!(:instance).and_return(plugin_manager)
-
-        @appliance.validate_definition
-      end
-
-      it "should validate definition and fail because no operating system plugins are installed" do
-        prepare_appliance
-        @appliance.instance_variable_set(:@appliance_config, prepare_appliance_config)
-
-        appliance_config_validator = mock(ApplianceConfigValidator)
-        appliance_config_validator.should_receive(:validate)
-
-        ApplianceConfigValidator.should_receive(:new).with(@appliance_config).and_return(appliance_config_validator)
-
-        plugin_manager = mock(PluginManager)
-        plugin_manager.stub!(:plugins).and_return({:os => {}})
-
-        PluginManager.stub!(:instance).and_return(plugin_manager)
-
-        lambda {
-          @appliance.validate_definition
-        }.should raise_error(RuntimeError, "No operating system plugins installed. Install one or more operating system plugin. See http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#Operating_system_plugins for more info.")
-      end
-
-      it "should validate definition and fail because no supported operating system plugins is installed" do
-        prepare_appliance
-        @appliance.instance_variable_set(:@appliance_config, prepare_appliance_config)
-
-        appliance_config_validator = mock(ApplianceConfigValidator)
-        appliance_config_validator.should_receive(:validate)
-
-        ApplianceConfigValidator.should_receive(:new).with(@appliance_config).and_return(appliance_config_validator)
-
-        plugin_manager = mock(PluginManager)
-        plugin_manager.stub!(:plugins).and_return({:os => {:rhel => {}}})
-
-        PluginManager.stub!(:instance).and_return(plugin_manager)
-
-        lambda {
-          @appliance.validate_definition
-        }.should raise_error(RuntimeError, "Not supported operating system selected: fedora. Make sure you have installed right operating system plugin, see http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#Operating_system_plugins. Supported OSes are: rhel")
-      end
-
-      it "should validate definition and fail because no supported operating system version plugins is installed" do
-        prepare_appliance
-        @appliance.instance_variable_set(:@appliance_config, prepare_appliance_config)
-
-        appliance_config_validator = mock(ApplianceConfigValidator)
-        appliance_config_validator.should_receive(:validate)
-
-        ApplianceConfigValidator.should_receive(:new).with(@appliance_config).and_return(appliance_config_validator)
-
-        plugin_manager = mock(PluginManager)
-        plugin_manager.stub!(:plugins).and_return({:os => {:fedora => {:type => :os, :name => :fedora, :full_name => "Fedora", :versions => ["xyz"]}}})
-
-        PluginManager.stub!(:instance).and_return(plugin_manager)
-
-        lambda {
-          @appliance.validate_definition
-        }.should raise_error(RuntimeError, "Not supported operating system version selected: 11. Supported versions are: xyz")
+        }.should raise_error(ValidationError, "Couldn't read appliance definition file: jeos-f13.ks.")
       end
     end
 

data/spec/helpers/augeas-helper-spec.rb

@@ -16,6 +16,7 @@
 # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
+require 'rubygems'
 require 'boxgrinder-build/helpers/augeas-helper'
 
 module BoxGrinder