boxgrinder-build 0.9.2 → 0.9.3

data/lib/boxgrinder-build/plugins/delivery/elastichosts/elastichosts-plugin.rb

@@ -23,20 +23,19 @@ require 'cgi'
 
 module BoxGrinder
   class ElasticHostsPlugin < BasePlugin
-    def after_init
+    def validate
       set_default_config_value('chunk', 64) # chunk size in MB
       set_default_config_value('start_part', 0) # part number to start uploading
       set_default_config_value('wait', 5) # wait time before retrying upload
       set_default_config_value('retry', 3) # number of retries
       set_default_config_value('ssl', false) # use SSL?
       set_default_config_value('drive_name', @appliance_config.name)
-    end
 
-    def execute(type = :elastichosts)
       validate_plugin_config(['endpoint', 'username', 'password'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#ElasticHosts_Delivery_Plugin')
-
       raise PluginValidationError, "You can use ElasticHosts plugin with base appliances (appliances created with operating system plugins) only, see http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#ElasticHosts_Delivery_Plugin." unless @previous_plugin_info[:type] == :os
+    end
 
+    def execute
       upload
       create_server
     end

data/lib/boxgrinder-build/plugins/delivery/local/local-plugin.rb

@@ -23,36 +23,40 @@ require 'boxgrinder-build/plugins/base-plugin'
 module BoxGrinder
   class LocalPlugin < BasePlugin
     def after_init
+      @package_name = "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
+    end
+
+    def validate
       set_default_config_value('overwrite', false)
       set_default_config_value('package', true)
 
-      if @plugin_config['package']
-        register_deliverable(:package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz")
-      end
-    end
-
-    def execute(type = :local)
       validate_plugin_config(['path'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#Local_delivery_plugin')
+    end
 
+    def execute
       if @plugin_config['overwrite'] or !deliverables_exists?
-        PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(@previous_deliverables[:disk]), @deliverables[:package]) if @plugin_config['package']
-
         FileUtils.mkdir_p @plugin_config['path']
 
-        @log.debug "Copying files to '#{@plugin_config['path']}'..."
+        if @plugin_config['package']
+          PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(@previous_deliverables[:disk]), "#{@plugin_config['path']}/#{@package_name}") if @plugin_config['package']
+        else
+          @log.debug "Copying files to '#{@plugin_config['path']}'..."
 
-        (@plugin_config['package'] ? @deliverables : @previous_deliverables).each_value do |file|
-          @log.debug "Copying '#{file}'..."
-          @exec_helper.execute("cp '#{file}' '#{@plugin_config['path']}'")
+          @previous_deliverables.each_value do |file|
+            @log.debug "Copying '#{file}'..."
+            @exec_helper.execute("cp '#{file}' '#{@plugin_config['path']}'")
+          end
+          @log.info "Appliance delivered to '#{@plugin_config['path']}'."
         end
-        @log.info "Appliance delivered to '#{@plugin_config['path']}'."
       else
         @log.info "Appliance already delivered to '#{@plugin_config['path']}'."
       end
     end
 
-    def deliverables_exists?
-      (@plugin_config['package'] ? @deliverables : @previous_deliverables).each_value do |file|
+    def deliverables_exists?      
+      return File.exists?("#{@plugin_config['path']}/#{@package_name}") if @plugin_config['package']
+
+      @previous_deliverables.each_value do |file|
         return false unless File.exists?("#{@plugin_config['path']}/#{File.basename(file)}")
       end
 
@@ -63,4 +67,4 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::LocalPlugin, :type => :delivery, :name => :local, :full_name  => "Local file system"
+plugin :class => BoxGrinder::LocalPlugin, :type => :delivery, :name => :local, :full_name  => "Local file system"

data/lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb

@@ -22,16 +22,11 @@ require 'boxgrinder-build/helpers/package-helper'
 require 'AWS'
 require 'aws'
 
-# TODO remove this when it'll become not necessary
-# quick fix for old active_support require issue in EPEL 5
-require 'active_support/basic_object'
-require 'active_support/duration'
-
 module BoxGrinder
   class S3Plugin < BasePlugin
     REGION_OPTIONS = {
         'eu-west-1' => {
-            :endpoint => 's3.amazonaws.com',
+            :endpoint => 's3-eu-west-1.amazonaws.com',
             :location => 'EU',
             :kernel => {
                 'i386' => {:aki => 'aki-4deec439'},
@@ -48,6 +43,16 @@ module BoxGrinder
             }
         },
 
+        'ap-northeast-1' => {
+            :endpoint => 's3-ap-northeast-1.amazonaws.com',
+            :location => 'ap-northeast-1',
+            :kernel => {
+                'i386' => {:aki => 'aki-d209a2d3'},
+                'x86_64' => {:aki => 'aki-d409a2d5'}
+            }
+
+        },
+
         'us-west-1' => {
             :endpoint => 's3-us-west-1.amazonaws.com',
             :location => 'us-west-1',
@@ -68,38 +73,55 @@ module BoxGrinder
     }
 
     def after_init
-      set_default_config_value('overwrite', false)
-      set_default_config_value('path', '/')
-      set_default_config_value('region', 'us-east-1')
-
       register_supported_os("fedora", ['13', '14', '15'])
       register_supported_os("centos", ['5'])
       register_supported_os("rhel", ['5', '6'])
+      register_supported_os("sl", ['5', '6'])
 
       @ami_build_dir = "#{@dir.base}/ami"
       @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml"
     end
 
-    def execute(type = :ami)
+    def validate
+      set_default_config_value('overwrite', false)
+      set_default_config_value('path', '/')
+      set_default_config_value('region', 'us-east-1')
       validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
 
-      case type
+      subtype(:ami) do
+        set_default_config_value('snapshot', false)
+        validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
+      end
+
+      raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin only aware of the following regions: #{REGION_OPTIONS.keys.join(", ")}" unless REGION_OPTIONS.has_key?(@plugin_config['region'])
+
+    end
+
+    def execute
+      case @type
         when :s3
           upload_to_bucket(@previous_deliverables)
         when :cloudfront
           upload_to_bucket(@previous_deliverables, 'public-read')
         when :ami
-          set_default_config_value('snapshot', false)
-          validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
-
           @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')
 
           @ec2 = AWS::EC2::Base.new(:access_key_id => @plugin_config['access_key'], :secret_access_key => @plugin_config['secret_access_key'], :server => "ec2.#{@plugin_config['region']}.amazonaws.com")
 
           ami_dir = ami_key(@appliance_config.name, @plugin_config['path'])
           ami_manifest_key = "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml"
+          s3_object_exists = s3_object_exists?(ami_manifest_key)
 
-          if !s3_object_exists?(ami_manifest_key) or @plugin_config['snapshot']
+           @log.debug "Going to check whether s3 object exists"
+
+          if s3_object_exists and @plugin_config['overwrite']
+            @log.info "Object exists, attempting to deregister an existing image"
+            deregister_image(ami_manifest_key) # Remove existing image
+            bucket().delete_folder(ami_dir) # Avoid triggering dupe detection
+          end
+
+          if !s3_object_exists or @plugin_config['snapshot']
+            @log.info "Doing bundle/snapshot"
             bundle_image(@previous_deliverables)
             fix_sha1_sum
             upload_image(ami_dir)
@@ -131,10 +153,10 @@ module BoxGrinder
 
       remote_path = "#{s3_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
       size_b = File.size(@deliverables[:package])
-
       key = bucket(true, permissions).key(remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
 
-      unless key.exists? or @plugin_config['overwrite']
+      if !key.exists? or @plugin_config['overwrite']
+        @log.info "Will overwrite existing file #{remote_path}" if key.exists? and @plugin_config['overwrite']
         @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_b/1024/1024}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
         key.put(open(@deliverables[:package]), permissions, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint])
         @log.info "Appliance #{@appliance_config.name} uploaded to S3."
@@ -185,6 +207,16 @@ module BoxGrinder
       end
     end
 
+    def deregister_image(ami_manifest_key)
+      info = ami_info(ami_manifest_key)
+      if info
+        @ec2.deregister_image(:image_id => info.imageId)
+        @log.info "Preexisting image '#{info.imageLocation}' for #{@appliance_config.name} was successfully de-registered, it had id: #{info.imageId} (region: #{@plugin_config['region']})."
+      else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new".
+        @log.info "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
+      end
+    end
+
     def ami_info(ami_manifest_key)
       ami_info = nil
 
@@ -210,12 +242,17 @@ module BoxGrinder
 
       return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot']
 
+      @log.info "Determining snapshot name"
+
       snapshot = 1
 
       while s3_object_exists?("#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/")
         snapshot += 1
       end
 
+      # Reuse the last key (if there was one)
+      snapshot -=1 if snapshot > 1 and @plugin_config['overwrite']
+
       "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}"
     end
 
@@ -224,7 +261,7 @@ module BoxGrinder
 
       begin
         b = bucket(false)
-        # Retrieve only one or no keys (if bucket is empty), throw an exception if bucket doesn't exists
+        # Retrieve only one or no keys (if bucket is empty), throw an exception if bucket doesn't exist
         b.keys('max-keys' => 1)
 
         if b.key(path).exists?
@@ -239,4 +276,4 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::S3Plugin, :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]
+plugin :class => BoxGrinder::S3Plugin, :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]

data/lib/boxgrinder-build/plugins/delivery/sftp/sftp-plugin.rb

@@ -25,16 +25,18 @@ require 'boxgrinder-build/helpers/package-helper'
 
 module BoxGrinder
   class SFTPPlugin < BasePlugin
-    def after_init
+    def validate
       set_default_config_value('overwrite', false)
       set_default_config_value('default_permissions', 0644)
 
-      register_deliverable(:package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz")
+      validate_plugin_config(['path', 'username', 'host'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#SFTP_Delivery_Plugin')
     end
 
-    def execute( type = :sftp )
-      validate_plugin_config(['path', 'username', 'host'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#SFTP_Delivery_Plugin')
+    def after_init
+      register_deliverable(:package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz")
+    end
 
+    def execute
       PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package( File.dirname(@previous_deliverables[:disk]), @deliverables[:package] )
 
       @log.info "Uploading #{@appliance_config.name} appliance via SSH..."

data/lib/boxgrinder-build/plugins/os/fedora/fedora-plugin.rb

@@ -23,6 +23,7 @@ module BoxGrinder
     def after_init
       super
       register_supported_os('fedora', ["13", "14", "15", "rawhide"])
+      set_default_config_value('PAE',true)
     end
 
     def execute(appliance_definition_file)
@@ -56,7 +57,7 @@ module BoxGrinder
       if @appliance_config.is64bit?
         packages << "kernel"
       else
-        @appliance_config.os.pae ? packages << "kernel-PAE" : packages << "kernel"
+        @plugin_config['PAE'] ? packages << "kernel-PAE" : packages << "kernel"
       end
     end
   end

data/lib/boxgrinder-build/plugins/os/rpm-based/kickstart.rb

@@ -24,9 +24,8 @@ require 'zlib'
 module BoxGrinder
 
   class Kickstart
-    def initialize(config, appliance_config, repos, dir, options = {})
+    def initialize(config, appliance_config, dir, options = {})
       @config           = config
-      @repos            = repos
       @appliance_config = appliance_config
       @dir              = dir
       @log              = options[:log] || Logger.new(STDOUT)
@@ -59,11 +58,7 @@ module BoxGrinder
 
       definition['mount_points'] = @linux_helper.partition_mount_points(@appliance_config.hardware.partitions)
 
-      repos = []
-      repos += default_repos if @appliance_config.default_repos
-      repos += @appliance_config.repos
-
-      for repo in repos
+      for repo in @appliance_config.repos
         if repo.keys.include?('mirrorlist')
           urltype = 'mirrorlist'
         else
@@ -92,30 +87,5 @@ module BoxGrinder
 
       definition
     end
-
-    def default_repos
-      os_repos = @repos[@appliance_config.os.version]
-
-      repos = Array.new
-
-      for type in ["base", "updates"]
-        unless os_repos.nil? or os_repos[type].nil?
-
-          mirrorlist = os_repos[type]['mirrorlist']
-          baseurl = os_repos[type]['baseurl']
-
-          name = "#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{type}"
-
-          if mirrorlist.nil?
-            repos.push({"name" => name, "baseurl" => baseurl})
-          else
-            repos.push({"name" => name, "mirrorlist" => mirrorlist})
-          end
-        end
-      end
-
-      repos
-    end
   end
-
 end

data/lib/boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin.rb

@@ -25,8 +25,6 @@ require 'boxgrinder-build/helpers/linux-helper'
 module BoxGrinder
   class RPMBasedOSPlugin < BasePlugin
     def after_init
-      set_default_config_value('format', 'raw')
-
       register_deliverable(
           :disk => "#{@appliance_config.name}-sda.#{@plugin_config['format']}",
           :descriptor => "#{@appliance_config.name}.xml"
@@ -35,6 +33,10 @@ module BoxGrinder
       @linux_helper = LinuxHelper.new(:log => @log)
     end
 
+    def validate
+      set_default_config_value('format', 'raw')
+    end
+
     def read_file(file)
       read_kickstart(file) if File.extname(file).eql?('.ks')
     end
@@ -88,12 +90,35 @@ module BoxGrinder
       appliance_config
     end
 
+    # Add default repos (if present) to the list of additional repositories specified in appliance definition.
+    def add_repos(repos)
+      return if repos.empty?
+
+      repos[@appliance_config.os.version].each do |name, repo|
+        r = { 'name' => name, 'ephemeral' => true }
+
+        ['baseurl', 'mirrorlist'].each { |type| r[type] = substitute_vars(repo[type]) unless repo[type].nil? }
+
+        @appliance_config.repos << r
+      end
+    end
+
+    # Substitute variables in selected string.
+    def substitute_vars(str)
+      return if str.nil?
+      @appliance_config.variables.keys.each do |var|
+        str = str.gsub("##{var}#", @appliance_config.variables[var])
+      end
+      str
+    end
+
     def build_with_appliance_creator(appliance_definition_file, repos = {})
       if File.extname(appliance_definition_file).eql?('.ks')
         kickstart_file = appliance_definition_file
       else
-        kickstart_file = Kickstart.new(@config, @appliance_config, repos, @dir, :log => @log).create
-        RPMDependencyValidator.new(@config, @appliance_config, @dir, kickstart_file, @options).resolve_packages
+        add_repos(repos) if @appliance_config.default_repos
+        kickstart_file = Kickstart.new(@config, @appliance_config, @dir, :log => @log).create
+        RPMDependencyValidator.new(@config, @appliance_config, @dir, :log => @log).resolve_packages
       end
 
       @log.info "Building #{@appliance_config.name} appliance..."

data/lib/boxgrinder-build/plugins/os/rpm-based/rpm-dependency-validator.rb

@@ -16,6 +16,7 @@
 # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
+require 'boxgrinder-core/helpers/log-helper'
 require 'yaml'
 
 module BoxGrinder
@@ -32,14 +33,13 @@ module BoxGrinder
   end
 
   class RPMDependencyValidator
-    def initialize( config, appliance_config, dir, kickstart_file, options = {} )
+    def initialize( config, appliance_config, dir, options = {} )
       @config           = config
       @appliance_config = appliance_config
-      @kickstart_file   = kickstart_file
       @dir              = dir
 
-      @log          = options[:log]         || Logger.new(STDOUT)
-      @exec_helper  = options[:exec_helper] || ExecHelper.new( { :log => @log } )
+      @log          = options[:log]         || LogHelper.new
+      @exec_helper  = options[:exec_helper] || ExecHelper.new( :log => @log )
 
       @yum_config_file = "#{@dir.tmp}/yum.conf"
 
@@ -51,18 +51,15 @@ module BoxGrinder
     def resolve_packages
       @log.info "Resolving packages added to #{@appliance_config.name} appliance definition file..."
 
-      repos = read_repos_from_kickstart_file
       package_list = generate_package_list
-      repo_list = generate_repo_list( repos )
+      generate_yum_config
 
-      generate_yum_config( repos )
+      invalid = invalid_names( @appliance_config.repos, package_list )
 
-      invalid_names = invalid_names( repo_list, package_list )
-
-      if invalid_names.size == 0
+      if invalid.empty?
         @log.info "All additional packages for #{@appliance_config.name} appliance successfully resolved."
       else
-        raise "Package#{invalid_names.size > 1 ? "s" : ""} #{invalid_names.join(', ')} for #{@appliance_config.name} appliance not found in repositories. Please check package names in appliance definition file."
+        raise "Package#{invalid.size > 1 ? "s" : ""} #{invalid.join(', ')} for #{@appliance_config.name} appliance not found in repositories. Please check package names in appliance definition file."
       end
     end
 
@@ -75,7 +72,7 @@ module BoxGrinder
         arches = "x86_64"
       end
 
-      repoquery_output = @exec_helper.execute( "repoquery --quiet --disablerepo=* --enablerepo=#{repo_list} -c '#{@yum_config_file}' list available #{package_list.join( ' ' )} --nevra --archlist=#{arches},noarch" )
+      repoquery_output = @exec_helper.execute( "repoquery --quiet --disablerepo=* --enablerepo=#{@appliance_config.repos.collect {|r| "#{@magic_hash}#{r['name']}"}.join(",")} -c '#{@yum_config_file}' list available #{package_list.join( ' ' )} --nevra --archlist=#{arches},noarch" )
 
       invalid_names = []
 
@@ -106,44 +103,16 @@ module BoxGrinder
       packages
     end
 
-    def generate_repo_list(repos)
-      repo_list = ""
-
-      repos.each do |repo|
-        repo_list += "#{@magic_hash}#{repo.name},"
-      end
-
-      repo_list = repo_list[0, repo_list.length - 1]
-    end
-
-    def read_repos_from_kickstart_file
-      repos = `grep -e "^repo" '#{@kickstart_file}'`
-      repo_list = []
-
-      repos.each do |repo_line|
-        name = repo_line.match( /--name=([\w\-]+)/ )[1]
-        baseurl = repo_line.match( /--baseurl=([\w\-\:\/\.&\?=]+)/ )
-        mirrorlist = repo_line.match( /--mirrorlist=([\w\-\:\/\.&\?=]+)/ )
-
-        baseurl = baseurl[1] unless baseurl.nil?
-        mirrorlist = mirrorlist[1] unless mirrorlist.nil?
-
-        repo_list.push( Repo.new( name, baseurl, mirrorlist ) )
-      end
-
-      repo_list
-    end
-
-    def generate_yum_config( repo_list )
+    def generate_yum_config
       File.open( @yum_config_file, "w") do |f|
 
         f.puts( "[main]\r\ncachedir=#{Dir.pwd}/#{@dir.tmp}/#{@magic_hash}#{@appliance_config.hardware.arch}-yum-cache/\r\n\r\n" )
 
-        for repo in repo_list
-          f.puts( "[#{@magic_hash}#{repo.name}]" )
-          f.puts( "name=#{repo.name}" )
-          f.puts( "baseurl=#{repo.baseurl}" ) unless repo.baseurl.nil?
-          f.puts( "mirrorlist=#{repo.mirrorlist}" ) unless repo.mirrorlist.nil?
+        for repo in @appliance_config.repos
+          f.puts( "[#{@magic_hash}#{repo['name']}]" )
+          f.puts( "name=#{repo['name']}" )
+          f.puts( "baseurl=#{repo['baseurl']}" ) unless repo['baseurl'].nil?
+          f.puts( "mirrorlist=#{repo['mirrorlist']}" ) unless repo['mirrorlist'].nil?
           f.puts( "enabled=1" )
           f.puts
         end