boxgrinder-build 0.0.1

data/lib/boxgrinder-build/helpers/ssh-helper.rb

@@ -0,0 +1,140 @@
+# JBoss, Home of Professional Open Source
+# Copyright 2009, Red Hat Middleware LLC, and individual contributors
+# by the @authors tag. See the copyright.txt in the distribution for a
+# full listing of individual contributors.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 2.1 of
+# the License, or (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this software; if not, write to the Free
+# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+
+require 'net/ssh'
+require 'net/sftp'
+require 'progressbar/progressbar'
+require 'boxgrinder-core/helpers/exec-helper'
+
+module BoxGrinder
+  class SSHHelper
+    def initialize( ssh_options, options = {} )
+      @ssh_options    = ssh_options
+
+      @log            = options[:log]         || Logger.new(STDOUT)
+      @exec_helper    = options[:exec_helper] || ExecHelper.new( :log => @log )
+    end
+
+    def connect
+      @log.info "Connecting to #{@ssh_options['host']}..."
+      @ssh = Net::SSH.start( @ssh_options['host'], @ssh_options['username'], { :password => @ssh_options['password'] } )
+    end
+
+    def connected?
+      return true if !@ssh.nil? and !@ssh.closed?
+      false
+    end
+
+    def disconnect
+      @log.info "Disconnecting from #{@ssh_options['host']}..."
+      @ssh.close if connected?
+      @ssh = nil
+    end
+
+    def upload_files( path, files = {} )
+
+      return if files.size == 0
+
+      raise "You're not connected to server" unless connected?
+
+      global_size = 0
+
+      files.each_value do |file|
+        global_size += File.size( file )
+      end
+
+      global_size_kb = global_size / 1024
+      global_size_mb = global_size_kb / 1024
+
+      @log.info "#{files.size} files to upload (#{global_size_mb > 0 ? global_size_mb.to_s + "MB" : global_size_kb > 0 ? global_size_kb.to_s + "kB" : global_size.to_s})"
+
+      @ssh.sftp.connect do |sftp|
+
+        begin
+          sftp.stat!( path )
+        rescue Net::SFTP::StatusException => e
+          raise unless e.code == 2
+          if @ssh_options['sftp_create_path']
+            @ssh.exec!( "mkdir -p #{path}" )
+          else
+            raise
+          end
+        end
+
+        nb = 0
+
+        files.each do |key, local|
+          name       = File.basename( local )
+          remote     = "#{path}/#{key}"
+          size_b     = File.size( local )
+          size_kb    = size_b / 1024
+          nb_of      = "#{nb += 1}/#{files.size}"
+
+          begin
+            sftp.stat!( remote )
+
+            unless @ssh_options['sftp_overwrite']
+
+              local_md5_sum   = `md5sum #{local} | awk '{ print $1 }'`.strip
+              remote_md5_sum  = @ssh.exec!( "md5sum #{remote} | awk '{ print $1 }'" ).strip
+
+              if (local_md5_sum.eql?( remote_md5_sum ))
+                @log.info "#{nb_of} #{name}: files are identical (md5sum: #{local_md5_sum}), skipping..."
+                next
+              end
+            end
+
+          rescue Net::SFTP::StatusException => e
+            raise unless e.code == 2
+          end
+
+          @ssh.exec!( "mkdir -p #{File.dirname( remote ) }" )
+
+          pbar = ProgressBar.new( "#{nb_of} #{name}", size_b )
+          pbar.file_transfer_mode
+
+          sftp.upload!( local, remote ) do |event, uploader, *args|
+            case event
+              when :open then
+              when :put then
+                pbar.set( args[1] )
+              when :close then
+              when :mkdir then
+              when :finish then
+                pbar.finish
+            end
+          end
+
+          sftp.setstat(remote, :permissions => @ssh_options['sftp_default_permissions'])
+
+        end
+      end
+    end
+
+    def createrepo( path, directories = [] )
+      raise "You're not connected to server" unless connected?
+
+      directories.each do |directory|
+        @log.info "Refreshing repository information in #{directory}..."
+        @ssh.exec!( "createrepo #{path}/#{directory}" )
+      end
+    end
+  end
+end

data/lib/boxgrinder-build/images/ec2-image.rb

@@ -0,0 +1,301 @@
+# JBoss, Home of Professional Open Source
+# Copyright 2009, Red Hat Middleware LLC, and individual contributors
+# by the @authors tag. See the copyright.txt in the distribution for a
+# full listing of individual contributors.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 2.1 of
+# the License, or (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this software; if not, write to the Free
+# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+
+require 'rake/tasklib'
+require 'fileutils'
+require 'boxgrinder-core/validators/errors'
+require 'yaml'
+require 'AWS'
+require 'aws/s3'
+require 'boxgrinder-build/helpers/aws-helper'
+include AWS::S3
+
+module BoxGrinder
+  class EC2Image < Rake::TaskLib
+
+    def initialize( config, appliance_config, options = {} )
+      @config            = config
+      @appliance_config  = appliance_config
+
+      @log          = options[:log]         || Logger.new(STDOUT)
+      @exec_helper  = options[:exec_helper] || ExecHelper.new( :log => @log )
+
+      define_tasks
+    end
+
+    def define_tasks
+      directory @appliance_config.path.dir.ec2.build
+      directory @appliance_config.path.dir.ec2.bundle
+
+      # TODO we should depend on actual disk file, not xml I think
+      file @appliance_config.path.file.ec2.disk  => [ @appliance_config.path.file.raw.xml, @appliance_config.path.dir.ec2.build ] do
+        convert_image_to_ec2_format
+      end
+
+      file @appliance_config.path.file.ec2.manifest => [ @appliance_config.path.file.ec2.disk, @appliance_config.path.dir.ec2.bundle ] do
+        @aws_helper = AWSHelper.new( @config, @appliance_config )
+        bundle_image
+      end
+
+      task "appliance:#{@appliance_config.name}:ec2:bundle" => [ @appliance_config.path.file.ec2.manifest ]
+
+      task "appliance:#{@appliance_config.name}:ec2:upload" => [ "appliance:#{@appliance_config.name}:ec2:bundle" ] do
+        @aws_helper = AWSHelper.new( @config, @appliance_config )
+        upload_image
+      end
+
+      task "appliance:#{@appliance_config.name}:ec2:register" => [ "appliance:#{@appliance_config.name}:ec2:upload" ] do
+        @aws_helper = AWSHelper.new( @config, @appliance_config )
+        register_image
+      end
+
+      desc "Build #{@appliance_config.simple_name} appliance for Amazon EC2"
+      task "appliance:#{@appliance_config.name}:ec2" => [ @appliance_config.path.file.ec2.disk ]
+    end
+
+    def bundle_image
+      @log.info "Bundling AMI..."
+
+      @exec_helper.execute( "ec2-bundle-image -i #{@appliance_config.path.file.ec2.disk} --kernel #{AWS_DEFAULTS[:kernel_id][@appliance_config.hardware.arch]} --ramdisk #{AWS_DEFAULTS[:ramdisk_id][@appliance_config.hardware.arch]} -c #{@aws_helper.aws_data['cert_file']} -k #{@aws_helper.aws_data['key_file']} -u #{@aws_helper.aws_data['account_number']} -r #{@appliance_config.hardware.arch} -d #{@appliance_config.path.dir.ec2.bundle}" )
+
+      @log.info "Bundling AMI finished."
+    end
+
+    def appliance_already_uploaded?
+      begin
+        bucket = Bucket.find( @aws_helper.aws_data['bucket_name'] )
+      rescue
+        return false
+      end
+
+      manifest_location = @aws_helper.bucket_manifest_key( @appliance_config.name )
+      manifest_location = manifest_location[ manifest_location.index( "/" ) + 1, manifest_location.length ]
+
+      for object in bucket.objects do
+        return true if object.key.eql?( manifest_location )
+      end
+
+      false
+    end
+
+    def upload_image
+      if appliance_already_uploaded?
+        @log.debug "Image for #{@appliance_config.simple_name} appliance is already uploaded, skipping..."
+        return
+      end
+
+      @log.info "Uploading #{@appliance_config.simple_name} AMI to bucket '#{@aws_helper.aws_data['bucket_name']}'..."
+
+      @exec_helper.execute( "ec2-upload-bundle -b #{@aws_helper.bucket_key( @appliance_config.name )} -m #{@appliance_config.path.file.ec2.manifest} -a #{@aws_helper.aws_data['access_key']} -s #{@aws_helper.aws_data['secret_access_key']} --retry" )
+    end
+
+    def register_image
+      ami_info    = @aws_helper.ami_info( @appliance_config.name )
+
+      if ami_info
+        @log.info "Image is registered under id: #{ami_info.imageId}"
+        return
+      else
+        ami_info = @aws_helper.ec2.register_image( :image_location => @aws_helper.bucket_manifest_key( @appliance_config.name ) )
+        @log.info "Image successfully registered under id: #{ami_info.imageId}."
+      end
+    end
+
+    def convert_image_to_ec2_format
+      @log.info "Converting #{@appliance_config.simple_name} appliance image to EC2 format..."
+
+      ec2_disk_mount_dir = "#{@config.dir.build}/#{@appliance_config.appliance_path}/tmp/ec2-#{rand(9999999999).to_s.center(10, rand(9).to_s)}"
+      raw_disk_mount_dir = "#{@config.dir.build}/#{@appliance_config.appliance_path}/tmp/raw-#{rand(9999999999).to_s.center(10, rand(9).to_s)}"
+
+      ec2_prepare_disk
+      ec2_create_filesystem
+
+      raw_disk_offset = calculate_disk_offset( @appliance_config.path.file.raw.disk )
+
+      ec2_loop_device = mount_image(@appliance_config.path.file.ec2.disk, ec2_disk_mount_dir )
+      raw_loop_device = mount_image(@appliance_config.path.file.raw.disk, raw_disk_mount_dir, raw_disk_offset )
+
+      sync_files( raw_disk_mount_dir, ec2_disk_mount_dir )
+
+      umount_image( @appliance_config.path.file.raw.disk, raw_disk_mount_dir, raw_loop_device )
+      umount_image( @appliance_config.path.file.ec2.disk, ec2_disk_mount_dir, ec2_loop_device )
+
+      guestfs_helper = GuestFSHelper.new( @appliance_config.path.file.ec2.disk, :log => @log )
+      guestfs = guestfs_helper.guestfs
+
+      create_devices( guestfs )
+      upload_fstab( guestfs )
+
+      guestfs.mkdir( "/data" ) if @appliance_config.is64bit?
+
+      enable_networking( guestfs )
+      upload_rc_local( guestfs )
+
+      guestfs_helper.rebuild_rpm_database
+
+      install_additional_packages( guestfs )
+      change_configuration( guestfs )
+
+      if @appliance_config.os.name.eql?("fedora") and @appliance_config.os.version.to_s.eql?("12")
+        @log.debug "Downgrading udev package to use in EC2 environment..."
+        guestfs.sh( "yum -y downgrade udev-142" )
+        guestfs.upload( "#{@config.dir.base}/src/f12/yum.conf", "/etc/yum.conf" )
+        @log.debug "Package udev downgraded."
+      end
+
+      guestfs.close
+
+      @log.info "Image converted to EC2 format."
+    end
+
+    def ec2_prepare_disk
+      # TODO add progress bar?
+      @log.debug "Preparing disk for EC2 image..."
+      @exec_helper.execute "dd if=/dev/zero of=#{@appliance_config.path.file.ec2.disk} bs=1 count=0 seek=#{10 * 1024}M"
+      @log.debug "Disk for EC2 image prepared"
+    end
+
+    def ec2_create_filesystem
+      @log.debug "Creating filesystem..."
+      @exec_helper.execute "mkfs.ext3 -F #{@appliance_config.path.file.ec2.disk}"
+      @log.debug "Filesystem created"
+    end
+
+    def calculate_disk_offset( disk )
+      loop_device = get_loop_device
+
+      @exec_helper.execute( "sudo losetup #{loop_device} #{disk}" )
+      offset = @exec_helper.execute("sudo parted -m #{loop_device} 'unit B print' | grep '^1' | awk -F: '{ print $2 }'").strip.chop
+      @exec_helper.execute( "sudo losetup -d #{loop_device}" )
+
+      offset
+    end
+
+    def mount_image( disk, mount_dir, offset = 0 )
+      loop_device = get_loop_device
+
+      @log.debug "Mounting image #{File.basename( disk )} in #{mount_dir} using #{loop_device} with offset #{offset}"
+      FileUtils.mkdir_p( mount_dir )
+      @exec_helper.execute( "sudo losetup -o #{offset.to_s} #{loop_device} #{disk}" )
+      @exec_helper.execute( "sudo mount #{loop_device} -t ext3 #{ mount_dir}")
+
+      loop_device
+    end
+
+    def umount_image( disk, mount_dir, loop_device )
+      @log.debug "Unmounting image #{File.basename( disk )}"
+      @exec_helper.execute( "sudo umount -d #{loop_device}" )
+      FileUtils.rm_rf( mount_dir )
+    end
+
+
+    def sync_files( from_dir, to_dir )
+      @log.debug "Syncing files between #{from_dir} and #{to_dir}..."
+      @exec_helper.execute "sudo rsync -u -r -a  #{from_dir}/* #{to_dir}"
+      @log.debug "Sync finished."
+    end
+
+    def cache_rpms( rpms )
+      for name in rpms.keys
+        cache_file = "#{@config.dir.src_cache}/#{name}"
+
+        if ( ! File.exist?( cache_file ) )
+          FileUtils.mkdir_p( @config.dir.src_cache )
+          @exec_helper.execute( "wget #{rpms[name]} -O #{cache_file}" )
+        end
+      end
+    end
+
+    def create_devices( guestfs )
+      @log.debug "Creating required devices..."
+      guestfs.sh( "/sbin/MAKEDEV -d /dev -x console" )
+      guestfs.sh( "/sbin/MAKEDEV -d /dev -x null" )
+      guestfs.sh( "/sbin/MAKEDEV -d /dev -x zero" )
+      @log.debug "Devices created."
+    end
+
+    def upload_fstab( guestfs )
+      @log.debug "Uploading '/etc/fstab' file..."
+      fstab_file = @appliance_config.is64bit? ? "#{@config.dir.base}/src/ec2/fstab_64bit" : "#{@config.dir.base}/src/ec2/fstab_32bit"
+      guestfs.upload( fstab_file, "/etc/fstab" )
+      @log.debug "'/etc/fstab' file uploaded."
+    end
+
+    # enable networking on default runlevels
+    def enable_networking( guestfs )
+      @log.debug "Enabling networking..."
+      guestfs.sh( "/sbin/chkconfig network on" )
+      guestfs.upload( "#{@config.dir.base}/src/ec2/ifcfg-eth0", "/etc/sysconfig/network-scripts/ifcfg-eth0" )
+      @log.debug "Networking enabled."
+    end
+
+    def upload_rc_local( guestfs )
+      @log.debug "Uploading '/etc/rc.local' file..."
+      rc_local = Tempfile.new('rc_local')
+      rc_local << guestfs.read_file( "/etc/rc.local" ) + File.read( "#{@config.dir.base}/src/ec2/rc_local" )
+      rc_local.flush
+
+      guestfs.upload( rc_local.path, "/etc/rc.local" )
+
+      rc_local.close
+      @log.debug "'/etc/rc.local' file uploaded."
+    end
+
+    def install_additional_packages( guestfs )
+      rpms = {
+              File.basename(AWS_DEFAULTS[:kernel_rpm][@appliance_config.hardware.arch]) => AWS_DEFAULTS[:kernel_rpm][@appliance_config.hardware.arch],
+              "ec2-ami-tools.noarch.rpm" => "http://s3.amazonaws.com/ec2-downloads/ec2-ami-tools.noarch.rpm"
+      }
+
+      cache_rpms( rpms )
+
+      @log.debug "Installing additional packages (#{rpms.keys.join( ", " )})..."
+      guestfs.mkdir_p("/tmp/rpms")
+
+      for name in rpms.keys
+        cache_file = "#{@config.dir.src_cache}/#{name}"
+        guestfs.upload( cache_file, "/tmp/rpms/#{name}" )
+      end
+
+      guestfs.sh( "rpm -Uvh --nodeps /tmp/rpms/*.rpm" )
+      guestfs.rm_rf("/tmp/rpms")
+      @log.debug "Additional packages installed."
+    end
+
+    def change_configuration( guestfs )
+      @log.debug "Changing configuration files using augeas..."
+      guestfs.aug_init( "/", 0 )
+      # disable password authentication
+      guestfs.aug_set( "/files/etc/ssh/sshd_config/PasswordAuthentication", "no" )
+      guestfs.aug_save
+      @log.debug "Augeas changes saved."
+    end
+
+    def get_loop_device
+      begin
+        loop_device = @exec_helper.execute("sudo losetup -f 2>&1").strip
+      rescue
+        raise "No free loop devices available, please free at least one. See 'losetup -d' command."
+      end
+
+      loop_device
+    end
+  end
+end

data/lib/boxgrinder-build/images/raw-image.rb

@@ -0,0 +1,137 @@
+# JBoss, Home of Professional Open Source
+# Copyright 2009, Red Hat Middleware LLC, and individual contributors
+# by the @authors tag. See the copyright.txt in the distribution for a
+# full listing of individual contributors.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 2.1 of
+# the License, or (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this software; if not, write to the Free
+# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+
+require 'rake/tasklib'
+require 'yaml'
+require 'boxgrinder-build/helpers/guestfs-helper'
+require 'boxgrinder-core/helpers/exec-helper'
+
+module BoxGrinder
+  class RAWImage < Rake::TaskLib
+
+    def initialize( config, appliance_config, options = {} )
+      @config = config
+      @appliance_config = appliance_config
+
+      @log          = options[:log]         || Logger.new(STDOUT)
+      @exec_helper  = options[:exec_helper] || ExecHelper.new( { :log => @log } )
+
+      @tmp_dir = "#{@config.dir.root}/#{@config.dir.build}/tmp"
+
+      define_tasks
+    end
+
+    def define_tasks
+      desc "Build #{@appliance_config.simple_name} appliance."
+      task "appliance:#{@appliance_config.name}" => [ @appliance_config.path.file.raw.xml, "appliance:#{@appliance_config.name}:validate:dependencies" ]
+
+      directory @tmp_dir
+
+      file @appliance_config.path.file.raw.xml => [ @appliance_config.path.file.raw.kickstart, "appliance:#{@appliance_config.name}:validate:dependencies", @tmp_dir ] do
+        build_raw_image
+        do_post_build_operations
+      end
+    end
+
+    def build_raw_image
+      @log.info "Building #{@appliance_config.simple_name} appliance..."
+
+      @exec_helper.execute "sudo PYTHONUNBUFFERED=1 appliance-creator -d -v -t #{@tmp_dir} --cache=#{@config.dir.rpms_cache}/#{@appliance_config.main_path} --config #{@appliance_config.path.file.raw.kickstart} -o #{@appliance_config.path.dir.raw.build} --name #{@appliance_config.name} --vmem #{@appliance_config.hardware.memory} --vcpu #{@appliance_config.hardware.cpus}"
+
+      # fix permissions
+      @exec_helper.execute "sudo chmod 777 #{@appliance_config.path.dir.raw.build_full}"
+      @exec_helper.execute "sudo chmod 666 #{@appliance_config.path.file.raw.disk}"
+      @exec_helper.execute "sudo chmod 666 #{@appliance_config.path.file.raw.xml}"
+
+      @log.info "Appliance #{@appliance_config.simple_name} was built successfully."
+    end
+
+    def do_post_build_operations
+      @log.info "Executing post operations after build..."
+
+      guestfs_helper = GuestFSHelper.new( @appliance_config.path.file.raw.disk, :log => @log )
+      guestfs = guestfs_helper.guestfs
+
+      change_configuration( guestfs )
+      set_motd( guestfs )
+      install_version_files( guestfs )
+      install_repos( guestfs )
+
+      @log.debug "Executing post commands from appliance definition file..."
+      if @appliance_config.post.base.size > 0
+        for cmd in @appliance_config.post.base
+          @log.debug "Executing #{cmd}"
+          guestfs.sh( cmd )
+        end
+        @log.debug "Post commands from appliance definition file executed."
+      else
+        @log.debug "No commands specified, skipping."
+      end
+
+      guestfs.close
+
+      @log.info "Post operations executed."
+    end
+
+    def change_configuration( guestfs )
+      @log.debug "Changing configuration files using augeas..."
+      guestfs.aug_init( "/", 0 )
+      # don't use DNS for SSH
+      guestfs.aug_set( "/files/etc/ssh/sshd_config/UseDNS", "no" )
+      guestfs.aug_save
+      @log.debug "Augeas changes saved."
+    end
+
+    def set_motd( guestfs )
+      @log.debug "Setting up '/etc/motd'..."
+      # set nice banner for SSH
+      motd_file = "/etc/init.d/motd"
+      guestfs.upload( "#{@config.dir.base}/src/motd.init", motd_file )
+      guestfs.sh( "sed -i s/#VERSION#/'#{@appliance_config.version}.#{@appliance_config.release}'/ #{motd_file}" )
+      guestfs.sh( "sed -i s/#APPLIANCE#/'#{@appliance_config.name} appliance'/ #{motd_file}" )
+
+      guestfs.sh( "/bin/chmod +x #{motd_file}" )
+      guestfs.sh( "/sbin/chkconfig --add motd" )
+      @log.debug "'/etc/motd' is nice now."
+    end
+
+    def install_version_files( guestfs )
+      @log.debug "Installing BoxGrinder version files..."
+      guestfs.sh( "echo 'BOXGRINDER_VERSION=#{@config.version_with_release}' > /etc/sysconfig/boxgrinder" )
+      guestfs.sh( "echo 'APPLIANCE_NAME=#{@appliance_config.name}' >> /etc/sysconfig/boxgrinder" )
+      @log.debug "Version files installed."
+    end
+
+    def install_repos( guestfs )
+      @log.debug "Installing repositories from appliance definition file..."
+      @appliance_config.repos.each do |repo|
+        @log.debug "Installing #{repo['name']} repo..."
+        repo_file = File.read( "#{@config.dir.base}/src/base.repo").gsub( /#NAME#/, repo['name'] )
+
+        ['baseurl', 'mirrorlist'].each  do |type|
+          repo_file << ("#{type}=#{repo[type]}") unless repo[type].nil?
+        end
+
+        guestfs.sh("echo '#{repo_file}' > /etc/yum.repos.d/#{repo['name']}.repo")
+      end
+      @log.debug "Repositories installed."
+    end
+  end
+end