boxgrinder-build 0.9.3 → 0.9.4

data/lib/boxgrinder-build/plugins/delivery/elastichosts/elastichosts-plugin.rb

@@ -17,7 +17,8 @@
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
 require 'boxgrinder-build/plugins/base-plugin'
-require 'restclient'
+require 'rubygems'
+require 'rest_client'
 require 'zlib'
 require 'cgi'
 

data/lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb

@@ -17,60 +17,14 @@
 # 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 
 require 'rubygems'
+require 'aws-sdk'
 require 'boxgrinder-build/plugins/base-plugin'
 require 'boxgrinder-build/helpers/package-helper'
-require 'AWS'
-require 'aws'
+require 'boxgrinder-build/helpers/s3-helper'
+require 'boxgrinder-build/helpers/ec2-helper'
 
 module BoxGrinder
   class S3Plugin < BasePlugin
-    REGION_OPTIONS = {
-        'eu-west-1' => {
-            :endpoint => 's3-eu-west-1.amazonaws.com',
-            :location => 'EU',
-            :kernel => {
-                'i386' => {:aki => 'aki-4deec439'},
-                'x86_64' => {:aki => 'aki-4feec43b'}
-            }
-        },
-
-        'ap-southeast-1' => {
-            :endpoint => 's3-ap-southeast-1.amazonaws.com',
-            :location => 'ap-southeast-1',
-            :kernel => {
-                'i386' => {:aki => 'aki-13d5aa41'},
-                'x86_64' => {:aki => 'aki-11d5aa43'}
-            }
-        },
-
-        'ap-northeast-1' => {
-            :endpoint => 's3-ap-northeast-1.amazonaws.com',
-            :location => 'ap-northeast-1',
-            :kernel => {
-                'i386' => {:aki => 'aki-d209a2d3'},
-                'x86_64' => {:aki => 'aki-d409a2d5'}
-            }
-
-        },
-
-        'us-west-1' => {
-            :endpoint => 's3-us-west-1.amazonaws.com',
-            :location => 'us-west-1',
-            :kernel => {
-                'i386' => {:aki => 'aki-99a0f1dc'},
-                'x86_64' => {:aki => 'aki-9ba0f1de'}
-            }
-        },
-
-        'us-east-1' => {
-            :endpoint => 's3.amazonaws.com',
-            :location => '',
-            :kernel => {
-                'i386' => {:aki => 'aki-407d9529'},
-                'x86_64' => {:aki => 'aki-427d952b'}
-            }
-        }
-    }
 
     def after_init
       register_supported_os("fedora", ['13', '14', '15'])
@@ -93,8 +47,24 @@ module BoxGrinder
         validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
       end
 
-      raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin only aware of the following regions: #{REGION_OPTIONS.keys.join(", ")}" unless REGION_OPTIONS.has_key?(@plugin_config['region'])
+      @s3_endpoints = S3Helper::endpoints
+      raise PluginValidationError, "Invalid region specified: #{@plugin_config['region']}. This plugin is only aware of the following regions: #{@s3_endpoints.keys.join(", ")}" unless @s3_endpoints.has_key?(@plugin_config['region'])
 
+      @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')
+
+      # Set global AWS configuration
+      AWS.config(:access_key_id => @plugin_config['access_key'],
+        :secret_access_key => @plugin_config['secret_access_key'],
+        :ec2_endpoint => EC2Helper::endpoints[@plugin_config['region']][:endpoint],
+        :s3_endpoint => @s3_endpoints[@plugin_config['region']][:endpoint],
+        :max_retries => 5,
+        :use_ssl => @plugin_config['use_ssl'])
+        #:logger => @log)   need to  modify our logger to accept blah.log(:level, 'message')
+
+      @ec2 = AWS::EC2.new
+      @s3 = AWS::S3.new
+      @s3helper = S3Helper.new(@ec2, @s3, :log => @log)
+      @ec2helper = EC2Helper.new(@ec2, :log => @log)
     end
 
     def execute
@@ -102,33 +72,25 @@ module BoxGrinder
         when :s3
           upload_to_bucket(@previous_deliverables)
         when :cloudfront
-          upload_to_bucket(@previous_deliverables, 'public-read')
+          upload_to_bucket(@previous_deliverables, :public_read)
         when :ami
-          @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '')
-
-          @ec2 = AWS::EC2::Base.new(:access_key_id => @plugin_config['access_key'], :secret_access_key => @plugin_config['secret_access_key'], :server => "ec2.#{@plugin_config['region']}.amazonaws.com")
-
           ami_dir = ami_key(@appliance_config.name, @plugin_config['path'])
-          ami_manifest_key = "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml"
-          s3_object_exists = s3_object_exists?(ami_manifest_key)
+          ami_manifest_key = @s3helper.stub_s3obj(asset_bucket, "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml")
 
-           @log.debug "Going to check whether s3 object exists"
+          @log.debug "Going to check whether s3 object exists"
 
-          if s3_object_exists and @plugin_config['overwrite']
+          if @s3helper.object_exists?(ami_manifest_key) and @plugin_config['overwrite']
             @log.info "Object exists, attempting to deregister an existing image"
             deregister_image(ami_manifest_key) # Remove existing image
-            bucket().delete_folder(ami_dir) # Avoid triggering dupe detection
+            @s3helper.delete_folder(asset_bucket, ami_dir) # Avoid triggering dupe detection
           end
 
-          if !s3_object_exists or @plugin_config['snapshot']
+          if !@s3helper.object_exists?(ami_manifest_key) or @plugin_config['snapshot']
             @log.info "Doing bundle/snapshot"
             bundle_image(@previous_deliverables)
             fix_sha1_sum
             upload_image(ami_dir)
-          else
-            @log.debug "AMI for #{@appliance_config.name} appliance already uploaded, skipping..."
           end
-
           register_image(ami_manifest_key)
       end
     end
@@ -141,7 +103,7 @@ module BoxGrinder
       File.open(@ami_manifest, "w") { |f| f.write(ami_manifest) }
     end
 
-    def upload_to_bucket(previous_deliverables, permissions = 'private')
+    def upload_to_bucket(previous_deliverables, permissions = :private)
       register_deliverable(
           :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
       )
@@ -151,25 +113,29 @@ module BoxGrinder
 
       PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package])
 
-      remote_path = "#{s3_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
-      size_b = File.size(@deliverables[:package])
-      key = bucket(true, permissions).key(remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
-
-      if !key.exists? or @plugin_config['overwrite']
-        @log.info "Will overwrite existing file #{remote_path}" if key.exists? and @plugin_config['overwrite']
-        @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_b/1024/1024}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
-        key.put(open(@deliverables[:package]), permissions, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint])
+      remote_path = "#{@s3helper.parse_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}"
+      size_m = File.size(@deliverables[:package])/1024**2
+      s3_obj = @s3helper.stub_s3obj(asset_bucket,remote_path.gsub(/^\//, '').gsub(/\/\//, ''))
+      # Does it really exist?
+      obj_exists = @s3helper.object_exists?(s3_obj)
+
+      if !obj_exists or @plugin_config['overwrite']
+        @log.info "Will overwrite existing file #{remote_path}" if obj_exists and @plugin_config['overwrite']
+        @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_m}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..."
+        s3_obj.write(:file => @deliverables[:package],
+                        :acl => permissions)
         @log.info "Appliance #{@appliance_config.name} uploaded to S3."
       else
         @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping."
       end
-
-      @s3.close_connection
     end
 
-    def bucket(create_if_missing = true, permissions = 'private')
-      @s3 ||= Aws::S3.new(@plugin_config['access_key'], @plugin_config['secret_access_key'], :connection_mode => :single, :logger => @log, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint])
-      @s3.bucket(@plugin_config['bucket'], create_if_missing, permissions, :location => REGION_OPTIONS[@plugin_config['region']][:location])
+    def asset_bucket(create_if_missing = true, permissions = :private)
+      @s3helper.bucket(:bucket => @plugin_config['bucket'],
+        :acl => permissions,
+        :create_of_missing => create_if_missing,
+        :location_constraint => @s3_endpoints[@plugin_config['region']][:location]
+      )
     end
 
     def bundle_image(deliverables)
@@ -184,96 +150,64 @@ module BoxGrinder
 
       FileUtils.mkdir_p(@ami_build_dir)
 
-      @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{REGION_OPTIONS[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
+      @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{@s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
 
       @log.info "Bundling AMI finished."
     end
 
     def upload_image(ami_dir)
-      bucket # this will create the bucket if needed
+      asset_bucket(true,:private) # this will create the bucket if needed
       @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..."
 
-      @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{REGION_OPTIONS[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']])
+      @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{@s3_endpoints[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']])
     end
 
     def register_image(ami_manifest_key)
-      info = ami_info(ami_manifest_key)
-
-      if info
-        @log.info "Image for #{@appliance_config.name} is registered under id: #{info.imageId} (region: #{@plugin_config['region']})."
+      if ami = ami_by_manifest_key(ami_manifest_key)
+        @log.info "Image for #{@appliance_config.name} is already registered under id: #{ami.id} (region: #{@plugin_config['region']})."
       else
-        info = @ec2.register_image(:image_location => "#{@plugin_config['bucket']}/#{ami_manifest_key}")
-        @log.info "Image for #{@appliance_config.name} successfully registered under id: #{info.imageId} (region: #{@plugin_config['region']})."
+        ami = @ec2.images.create(:image_location =>  "#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
+        @ec2helper.wait_for_image_state(:available, ami)
+        @log.info "Image for #{@appliance_config.name} successfully registered under id: #{ami.id} (region: #{@plugin_config['region']})."
       end
     end
 
     def deregister_image(ami_manifest_key)
-      info = ami_info(ami_manifest_key)
-      if info
-        @ec2.deregister_image(:image_id => info.imageId)
-        @log.info "Preexisting image '#{info.imageLocation}' for #{@appliance_config.name} was successfully de-registered, it had id: #{info.imageId} (region: #{@plugin_config['region']})."
+      if ami = ami_by_manifest_key(ami_manifest_key)
+        @log.info "Preexisting image '#{ami.location}' for #{@appliance_config.name} will be de-registered, it had id: #{ami.id} (region: #{@plugin_config['region']})."
+        ami.deregister
+        @ec2helper.wait_for_image_death(ami)
       else # This occurs when the AMI is de-registered externally but the file structure is left intact in S3. In this instance, we simply overwrite and register the image as if it were "new".
-        @log.info "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
+        @log.debug "Possible dangling/unregistered AMI skeleton structure in S3, there is nothing to deregister"
       end
     end
 
-    def ami_info(ami_manifest_key)
-      ami_info = nil
-
-      images = @ec2.describe_images(:owner_id => @plugin_config['account_number']).imagesSet
-
-      return nil if images.nil?
-
-      for image in images.item do
-        ami_info = image if (image.imageLocation.eql?("#{@plugin_config['bucket']}/#{ami_manifest_key}"))
-      end
-
-      ami_info
-    end
-
-    def s3_path(path)
-      return '' if path == '/'
-
-      "#{path.gsub(/^(\/)*/, '').gsub(/(\/)*$/, '')}/"
+    def ami_by_manifest_key(ami_manifest_key)
+      ami = @ec2.images.with_owner(@plugin_config['account_number']).
+          filter("manifest-location","#{@plugin_config['bucket']}/#{ami_manifest_key.key}")
+      return nil unless ami.any?
+      ami.first
     end
 
     def ami_key(appliance_name, path)
-      base_path = "#{s3_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}"
+      base_path = "#{@s3helper.parse_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}"
 
       return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot']
 
       @log.info "Determining snapshot name"
-
       snapshot = 1
-
-      while s3_object_exists?("#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/")
+      while @s3helper.object_exists?(
+          @s3helper.stub_s3obj(asset_bucket, "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/")
+      )
         snapshot += 1
       end
-
       # Reuse the last key (if there was one)
       snapshot -=1 if snapshot > 1 and @plugin_config['overwrite']
 
       "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}"
     end
 
-    def s3_object_exists?(path)
-      @log.trace "Checking if '#{path}' path exists in #{@plugin_config['bucket']}..."
-
-      begin
-        b = bucket(false)
-        # Retrieve only one or no keys (if bucket is empty), throw an exception if bucket doesn't exist
-        b.keys('max-keys' => 1)
-
-        if b.key(path).exists?
-          @log.trace "Path exists!"
-          return true
-        end
-      rescue
-      end
-      @log.trace "Path doesn't exist!"
-      false
-    end
   end
 end
 
-plugin :class => BoxGrinder::S3Plugin, :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]
+plugin :class => BoxGrinder::S3Plugin, :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]

data/lib/boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin.rb

@@ -151,23 +151,12 @@ module BoxGrinder
 
         guestfs.sh("chkconfig firstboot off") if guestfs.exists('/etc/init.d/firstboot') != 0
 
-        @log.info "Executing post operations after build..."
+        # https://issues.jboss.org/browse/BGBUILD-148
+        recreate_rpm_database(guestfs, guestfs_helper) if @config.os.name != @appliance_config.os.name or @config.os.version != @appliance_config.os.version
 
-        unless @appliance_config.post['base'].nil?
-          @appliance_config.post['base'].each do |cmd|
-            guestfs_helper.sh(cmd, :arch => @appliance_config.hardware.arch)
-          end
-          @log.debug "Post commands from appliance definition file executed."
-        else
-          @log.debug "No commands specified, skipping."
-        end
+        execute_post(guestfs_helper)
 
         yield guestfs, guestfs_helper if block_given?
-
-        @log.info "Post operations executed."
-
-        # https://issues.jboss.org/browse/BGBUILD-148
-        recreate_rpm_database(guestfs, guestfs_helper) if @config.os.name != @appliance_config.os.name or @config.os.version != @appliance_config.os.version
       end
 
       @log.info "Base image for #{@appliance_config.name} appliance was built successfully."
@@ -182,6 +171,18 @@ module BoxGrinder
       end
     end
 
+    def execute_post(guestfs_helper)
+      @log.info "Executing post operations after build..."
+      unless @appliance_config.post['base'].nil?
+        @appliance_config.post['base'].each do |cmd|
+          guestfs_helper.sh(cmd, :arch => @appliance_config.hardware.arch)
+        end
+        @log.debug "Post commands from appliance definition file executed."
+      else
+        @log.debug "No commands specified, skipping."
+      end
+    end
+
     # https://issues.jboss.org/browse/BGBUILD-204
     def disable_biosdevname(guestfs)
       @log.debug "Disabling biosdevname for Fedora 15..."

data/rubygem-boxgrinder-build.spec

@@ -5,7 +5,7 @@
 
 Summary: A tool for creating appliances from simple plain text files
 Name: rubygem-%{gemname}
-Version: 0.9.3
+Version: 0.9.4
 Release: 1%{?dist}
 Group: Development/Languages
 License: LGPLv3+
@@ -16,35 +16,22 @@ Requires: ruby(abi) = %{rubyabi}
 Requires: rubygem(boxgrinder-core) >= 0.3.0
 Requires: rubygem(boxgrinder-core) < 0.4.0
 Requires: ruby-libguestfs
-Requires: parted
-Requires: e2fsprogs
-
-# Fix for rubygem-aws package
-Requires: rubygem(activesupport)
 
 BuildArch: noarch
 
 BuildRequires: rubygem(rake)
-BuildRequires: rubygem(rspec)
 BuildRequires: rubygem(boxgrinder-core) >= 0.3.0
 BuildRequires: rubygem(boxgrinder-core) < 0.4.0
 BuildRequires: rubygem(echoe)
 BuildRequires: ruby-libguestfs
+# Use rspec-core until rspec are migrated to RSpec 2.x
+BuildRequires: rubygem(rspec-core)
 
-# Fix for rubygem-aws package
-BuildRequires: rubygem(activesupport)
-
-# EBS and S3
-Requires: rubygem(amazon-ec2)
-# Fixes blankslate error
-Requires: rubygem(builder)
-Requires: rubygem(aws)
+# AWS
 Requires: euca2ools >= 1.3.1-4
+Requires: rubygem(aws-sdk) >= 1.0.4
 
-BuildRequires: rubygem(amazon-ec2)
-BuildRequires: rubygem(aws)
-# Fixes blankslate error
-BuildRequires: rubygem(builder)
+BuildRequires: rubygem(aws-sdk) >= 1.0.4
 
 # SFTP
 Requires: rubygem(net-sftp)
@@ -59,15 +46,9 @@ BuildRequires: rubygem(progressbar)
 Requires: appliance-tools
 Requires: yum-utils
 
-# EC2
-Requires: rsync
-Requires: wget
-Requires: util-linux
-
-# ElasticHosts
-Requires: rubygem(rest-client)
-
+#Elastichosts
 BuildRequires: rubygem(rest-client)
+Requires: rubygem(rest-client)
 
 Provides: rubygem(%{gemname}) = %{version}
 
@@ -117,6 +98,7 @@ mkdir -p %{buildroot}/%{gemdir}
             --force --rdoc %{SOURCE0}
 mv %{_builddir}%{gemdir}/bin/* %{buildroot}/%{_bindir}
 find %{_builddir}%{geminstdir}/bin -type f | xargs chmod a+x
+rm -rf %{_builddir}/%{geminstdir}/integ/packages/*.rpm
 cp -r %{_builddir}%{gemdir}/* %{buildroot}/%{gemdir}
 
 %check
@@ -148,6 +130,19 @@ popd
 %{gemdir}/doc/%{gemname}-%{version}
 
 %changelog
+
+* Fri Aug 12 2011 Marc Savy <msavy@redhat.com> - 0.9.4-1
+- Upstream release: 0.9.4
+- [BGBUILD-263] NoMethodError: undefined method `item' for nil:NilClass while creating EBS appliance
+- [BGBUILD-246] Detect when insufficient system memory is available for standard libguestfs, and reduce allocation.
+- [BGBUILD-269] RPM database is recreated after post section execution preventing installing RPM in post section
+- [BGBUILD-273] Move to RSpec2
+- [BGBUILD-272] Move from aws and amazon-ec2 to official aws-sdk gem
+- [BGBUILD-238] Stop AWS gem warnings
+- [BGBUILD-265] Resolve concurrency issues in S3 plugin for overwriting
+- [BGBUILD-249] Warning from S3 AMI plugin that BG is attempting to create a bucket that already exists
+- [BGBUILD-242]	Additional EBS overwrite edge cases
+
 * Fri Jun 17 2011 Marc Savy <msavy@redhat.com> - 0.9.3-1
 - Upstream release: 0.9.3
 - [BGBUILD-232] boxgrinder doesn't validate config early enough
@@ -158,7 +153,7 @@ popd
 - [BGBUILD-222] Allow overwrite of uploaded ec2 image
 - [BGBUILD-225] Move PAE configuration parameter to operating system configuration
 - [BGBUILD-224] EBS Plugin Support for CentOS v5.5 and fix for non-integer EBS disk sizes
-- [BGBUILD-231] Cannot register Fedora 15 EC2 AMI with S3 delivery plugin in eu-west-1 availability zone
+- [BGBUILD-231] Cannot register Fedora 15 EC2 AMI with S3 delivery plugin in eu-west-1 availability zone
 - [BGBUILD-193] EBS delivery plugin timing/concurrency issues
 - [BGBUILD-247] ap-northeast-1 end-point is missing in S3 plugin (added Tokyo region)
 - [BGBUILD-251] Add ap-northeast-1 (tokyo) region for EBS plugin