boxgrinder-build 0.9.8 → 0.10.0

data/lib/boxgrinder-build/plugins/delivery/local/local-plugin.rb

@@ -22,6 +22,8 @@ require 'boxgrinder-build/plugins/base-plugin'
 
 module BoxGrinder
   class LocalPlugin < BasePlugin
+    plugin :type => :delivery, :name => :local, :full_name  => "Local file system"
+
     def after_init
       @package_name = "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz"
     end
@@ -67,4 +69,3 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::LocalPlugin, :type => :delivery, :name => :local, :full_name  => "Local file system"

data/lib/boxgrinder-build/plugins/delivery/openstack/openstack-plugin.rb

@@ -0,0 +1,133 @@
+#
+# Copyright 2010 Red Hat, Inc.
+#
+# This is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation; either version 3 of
+# the License, or (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this software; if not, write to the Free
+# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+# 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+
+require 'rubygems'
+require 'boxgrinder-build/plugins/base-plugin'
+require 'rest_client'
+require 'json'
+
+module BoxGrinder
+  class OpenStackPlugin < BasePlugin
+    plugin :type => :delivery, :name => :openstack, :full_name  => "OpenStack"
+
+    def after_init
+      set_default_config_value('host', 'localhost')
+      set_default_config_value('port', '9292')
+      set_default_config_value('schema', 'http')
+      set_default_config_value('overwrite', false)
+      set_default_config_value('public', false)
+
+      register_supported_platform(:ec2)
+      register_supported_platform(:vmware)
+      register_supported_platform(:virtualbox)
+
+      @disk_format, @container_format = disk_and_container_format
+      @appliance_name = "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@disk_format}"
+    end
+
+    def execute
+      @log.debug "Checking if '#{@appliance_name}' appliance is already registered..."
+      images = get_images(:name => @appliance_name)
+
+      unless images.empty?
+        @log.debug "We found one or more appliances with the name '#{@appliance_name}'."
+
+        unless @plugin_config['overwrite']
+          @log.error "One or more appliances are already registered with the name '#{@appliance_name}'. You can specify 'overwrite' parameter to remove them."
+          return
+        end
+
+        @log.info "Removing all images with name '#{@appliance_name}' because 'overwrite' parameter is set to true..."
+        images.each {|i| delete_image(i['id']) }
+        @log.info "Images removed."
+      end
+
+      disk_format, container_format = disk_and_container_format
+
+      post_image(:disk_format => disk_format, :container_format => container_format, :public => @plugin_config['public'])
+    end
+
+    def disk_and_container_format
+      disk_format = :raw
+      container_format = :bare
+
+      if @previous_plugin_info[:type] == :platform
+        case @previous_plugin_info[:name]
+          when :ec2:
+            disk_format = :ami
+            container_format = :ami
+          when :vmware:
+            disk_format = :vmdk
+          when :virtualbox:
+            disk_format = :vmdk
+        end
+      end
+
+      [disk_format, container_format]
+    end
+
+    def post_image(options = {})
+      options = {
+          :disk_format => :raw, # raw, vhd, vmdk, vdi, qcow2, aki, ari, ami
+          :container_format => :bare, # ovf, bare, aki, ari, ami
+          :public => true
+      }.merge(options)
+
+      @log.info "Uploading and registering '#{@appliance_name}' appliance in OpenStack..."
+
+      file_size = File.size(@previous_deliverables.disk)
+
+      @log.trace "Disk format: #{options[:disk_format]}, container format: #{options[:container_format]}, public: #{options[:public]}, size: #{file_size}."
+
+      image = JSON.parse(RestClient.post("#{url}/v1/images",
+        File.new(@previous_deliverables.disk, 'rb'),
+        :content_type => 'application/octet-stream',
+        'x-image-meta-size' => file_size,
+        'x-image-meta-name' => @appliance_name,
+        'x-image-meta-disk-format' => options[:disk_format],
+        'x-image-meta-container-format' => options[:container_format],
+        'x-image-meta-is-public' => options[:public] ? "true" : false,
+        'x-image-meta-property-distro' => "#{@appliance_config.os.name.capitalize} #{@appliance_config.os.version}"
+      ))['image']
+
+      @log.info "Appliance registered under id = #{image['id']}."
+    end
+
+    # Removes image from the server for specified id.
+    #
+    def delete_image(id)
+      @log.trace "Removing image with id = #{id}..."
+      RestClient.delete("#{url}/v1/images/#{id}")
+      @log.trace "Image removed."
+    end
+
+    # Retrieves a list of public images with specified filter. If no filter is specified - all images are returned.
+    #
+    def get_images(params = {})
+      @log.trace "Listing images with params = #{params.to_json}..."
+      data = JSON.parse(RestClient.get("#{url}/v1/images", :params => params))['images']
+      @log.trace "Listing done."
+      data
+    end
+
+    def url
+      "#{@plugin_config['schema']}://#{@plugin_config['host']}:#{@plugin_config['port']}"
+    end
+  end
+end
+

data/lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb

@@ -25,6 +25,7 @@ require 'boxgrinder-build/helpers/ec2-helper'
 
 module BoxGrinder
   class S3Plugin < BasePlugin
+    plugin :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]
 
     def after_init
       register_supported_os("fedora", ['13', '14', '15', '16'])
@@ -38,6 +39,8 @@ module BoxGrinder
 
     def validate
       set_default_config_value('overwrite', false)
+      set_default_config_value('kernel', false)
+      set_default_config_value('ramdisk', false)
       set_default_config_value('path', '/')
       set_default_config_value('region', 'us-east-1')
       validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin')
@@ -153,7 +156,18 @@ module BoxGrinder
 
       FileUtils.mkdir_p(@ami_build_dir)
 
-      @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{@s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
+      cmd_str = "euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem " <<
+        "-i #{deliverables[:disk]} " <<
+        "--kernel #{@plugin_config['kernel'] || @s3_endpoints[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch.intern][:aki]} " <<
+        "-c #{@plugin_config['cert_file']} "<<
+        "-k #{@plugin_config['key_file']} " <<
+        "-u #{@plugin_config['account_number']} " <<
+        "-r #{@appliance_config.hardware.base_arch} " <<
+        "-d #{@ami_build_dir} "
+
+      cmd_str << "--ramdisk #{@plugin_config['ramdisk']}" if @plugin_config['ramdisk']
+
+      @exec_helper.execute(cmd_str, :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']])
 
       @log.info "Bundling AMI finished."
     end
@@ -216,4 +230,3 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::S3Plugin, :type => :delivery, :name => :s3, :full_name => "Amazon Simple Storage Service (Amazon S3)", :types => [:s3, :cloudfront, :ami]

data/lib/boxgrinder-build/plugins/delivery/sftp/sftp-plugin.rb

@@ -22,14 +22,21 @@ require 'net/sftp'
 require 'progressbar'
 require 'boxgrinder-build/plugins/base-plugin'
 require 'boxgrinder-build/helpers/package-helper'
+require 'boxgrinder-build/helpers/sftp-helper'
 
 module BoxGrinder
   class SFTPPlugin < BasePlugin
+    plugin :type => :delivery, :name => :sftp, :full_name => "SSH File Transfer Protocol"
+
     def validate
       set_default_config_value('overwrite', false)
       set_default_config_value('default_permissions', 0644)
+      set_default_config_value('identity', false)
 
       validate_plugin_config(['path', 'username', 'host'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#SFTP_Delivery_Plugin')
+
+      @identity = (@plugin_config['identity'] || @plugin_config['i'])
+      @sftp_helper = SFTPHelper.new(:log => @log)
     end
 
     def after_init
@@ -41,114 +48,21 @@ module BoxGrinder
 
       @log.info "Uploading #{@appliance_config.name} appliance via SSH..."
 
-      begin
-        #TODO move to a block
-        connect
-        upload_files(@plugin_config['path'], File.basename(@deliverables[:package]) => @deliverables[:package])
-        disconnect
-
-        @log.info "Appliance #{@appliance_config.name} uploaded."
-      rescue => e
-        @log.error e
-        @log.error "An error occurred while uploading files."
-      end
-    end
+      sftp_opts={}
+      sftp_opts.merge!(:password => @plugin_config['password']) if @plugin_config['password']
+      sftp_opts.merge!(:keys => @identity.to_a) if @identity
 
-    def connect
-      @log.info "Connecting to #{@plugin_config['host']}..."
-      @ssh = Net::SSH.start(@plugin_config['host'], @plugin_config['username'], {:password => @plugin_config['password']})
-    end
-
-    def connected?
-      return true if !@ssh.nil? and !@ssh.closed?
-      false
-    end
+      @sftp_helper.connect(@plugin_config['host'], @plugin_config['username'], sftp_opts)
+      @sftp_helper.upload_files(@plugin_config['path'], @plugin_config['default_permissions'], @plugin_config['overwrite'], File.basename(@deliverables[:package]) => @deliverables[:package])
 
-    def disconnect
-      @log.info "Disconnecting from #{@plugin_config['host']}..."
-      @ssh.close if connected?
-      @ssh = nil
+      @log.info "Appliance #{@appliance_config.name} uploaded."
+    rescue => e
+      @log.error e
+      @log.error "An error occurred while uploading files."
+      raise
+    ensure
+      @sftp_helper.disconnect
     end
 
-    def upload_files(path, files = {})
-      return if files.size == 0
-
-      raise "You're not connected to server" unless connected?
-
-      @log.debug "Files to upload:"
-
-      files.each do |remote, local|
-        @log.debug "#{File.basename(local)} => #{path}/#{remote}"
-      end
-
-      global_size = 0
-
-      files.each_value do |file|
-        global_size += File.size(file)
-      end
-
-      global_size_kb = global_size / 1024
-      global_size_mb = global_size_kb / 1024
-
-      @log.info "#{files.size} files to upload (#{global_size_mb > 0 ? global_size_mb.to_s + "MB" : global_size_kb > 0 ? global_size_kb.to_s + "kB" : global_size.to_s})"
-
-      @ssh.sftp.connect do |sftp|
-        begin
-          sftp.stat!(path)
-        rescue Net::SFTP::StatusException => e
-          raise unless e.code == 2
-          @ssh.exec!("mkdir -p #{path}")
-        end
-
-        nb = 0
-
-        files.each do |key, local|
-          name       = File.basename(local)
-          remote     = "#{path}/#{key}"
-          size_b     = File.size(local)
-          size_kb    = size_b / 1024
-          nb_of      = "#{nb += 1}/#{files.size}"
-
-          begin
-            sftp.stat!(remote)
-
-            unless @plugin_config['overwrite']
-
-              local_md5_sum   = `md5sum #{local} | awk '{ print $1 }'`.strip
-              remote_md5_sum  = @ssh.exec!("md5sum #{remote} | awk '{ print $1 }'").strip
-
-              if (local_md5_sum.eql?(remote_md5_sum))
-                @log.info "#{nb_of} #{name}: files are identical (md5sum: #{local_md5_sum}), skipping..."
-                next
-              end
-            end
-
-          rescue Net::SFTP::StatusException => e
-            raise unless e.code == 2
-          end
-
-          @ssh.exec!("mkdir -p #{File.dirname(remote) }")
-
-          pbar = ProgressBar.new("#{nb_of} #{name}", size_b)
-          pbar.file_transfer_mode
-
-          sftp.upload!(local, remote) do |event, uploader, * args|
-            case event
-              when :open then
-              when :put then
-                pbar.set(args[1])
-              when :close then
-              when :mkdir then
-              when :finish then
-                pbar.finish
-            end
-          end
-
-          sftp.setstat(remote, :permissions => @plugin_config['default_permissions'])
-        end
-      end
-    end
   end
-end
-
-plugin :class => BoxGrinder::SFTPPlugin, :type => :delivery, :name => :sftp, :full_name  => "SSH File Transfer Protocol"
+end

data/lib/boxgrinder-build/plugins/os/centos/centos-plugin.rb

@@ -20,6 +20,7 @@ require 'boxgrinder-build/plugins/os/rhel/rhel-plugin'
 
 module BoxGrinder
   class CentOSPlugin < RHELPlugin
+    plugin :type => :os, :name => :centos, :full_name  => "CentOS", :versions => ["5", "6"]
 
     def after_init
       super
@@ -29,8 +30,8 @@ module BoxGrinder
     def execute(appliance_definition_file)
       repos = {}
 
-     @plugin_info[:versions].each do |version|
-      repos[version] = {
+      @plugin_info[:versions].each do |version|
+        repos[version] = {
           "base" => {
               "mirrorlist" => "http://mirrorlist.centos.org/?release=#OS_VERSION#&arch=#BASE_ARCH#&repo=os"
           },
@@ -45,4 +46,3 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::CentOSPlugin, :type => :os, :name => :centos, :full_name  => "CentOS", :versions   => ["5", "6"]

data/lib/boxgrinder-build/plugins/os/fedora/fedora-plugin.rb

@@ -20,6 +20,8 @@ require 'boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin'
 
 module BoxGrinder
   class FedoraPlugin < RPMBasedOSPlugin
+    plugin :type => :os, :name => :fedora, :full_name => "Fedora", :versions => ["13", "14", "15", "16", "rawhide"]
+
     def after_init
       super
       register_supported_os('fedora', ["13", "14", "15", "16", "rawhide"])
@@ -108,4 +110,3 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::FedoraPlugin, :type => :os, :name => :fedora, :full_name => "Fedora", :versions => ["13", "14", "15", "16", "rawhide"]

data/lib/boxgrinder-build/plugins/os/rhel/rhel-plugin.rb

@@ -20,6 +20,8 @@ require 'boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin'
 
 module BoxGrinder
   class RHELPlugin < RPMBasedOSPlugin
+    plugin :type => :os, :name => :rhel, :full_name => "Red Hat Enterprise Linux", :versions => ['5', '6']
+
     def after_init
       super
       register_supported_os('rhel', ['5', '6'])
@@ -57,4 +59,3 @@ module BoxGrinder
   end
 end
 
-plugin :class => BoxGrinder::RHELPlugin, :type => :os, :name => :rhel, :full_name => "Red Hat Enterprise Linux", :versions => ['5', '6']

data/lib/boxgrinder-build/plugins/os/rpm-based/rpm-based-os-plugin.rb

@@ -26,6 +26,8 @@ require 'boxgrinder-core/errors'
 module BoxGrinder
   class RPMBasedOSPlugin < BasePlugin
     def after_init
+      set_default_config_value('format', 'raw')
+
       register_deliverable(
           :disk => "#{@appliance_config.name}-sda.#{@plugin_config['format']}",
           :descriptor => "#{@appliance_config.name}.xml"
@@ -34,63 +36,6 @@ module BoxGrinder
       @linux_helper = LinuxHelper.new(:log => @log)
     end
 
-    def validate
-      set_default_config_value('format', 'raw')
-    end
-
-    def read_file(file)
-      read_kickstart(file) if File.extname(file).eql?('.ks')
-    end
-
-    def read_kickstart(file)
-      appliance_config = ApplianceConfig.new
-      appliance_config.name = File.basename(file, '.ks')
-
-      name = nil
-      version = nil
-
-      kickstart = File.read(file)
-
-      kickstart.each do |line|
-        n = line.scan(/^# bg_os_name: (.*)/).flatten.first
-        v = line.scan(/^# bg_os_version: (.*)/).flatten.first
-
-        name = n unless n.nil?
-        version = v unless v.nil?
-      end
-
-      raise "No operating system name specified, please add comment to you kickstrt file like this: # bg_os_name: fedora" if name.nil?
-      raise "No operating system version specified, please add comment to you kickstrt file like this: # bg_os_version: 14" if version.nil?
-
-      appliance_config.os.name = name
-      appliance_config.os.version = version
-
-      partitions = {}
-
-      kickstart.each do |line|
-        # Parse also the partition scheme
-        if line =~ /^part ([\/\w]+)/
-          root = $1
-          partitions[root] = {}
-
-          # size
-          partitions[root]['size'] = $1.to_f / 1024 if line =~ /--size[=\s]*(\d+)/
-          # fs type
-          partitions[root]['type'] = $1 if line =~ /--fstype[=\s]*(\w+)/
-          # fs options
-          partitions[root]['options'] = $1 if line =~ /--fsoptions[=\s]*([,\w]+)/
-
-          raise "Partition size not specified for #{root} partition in #{file}" if partitions[root]['size'].nil?
-        end
-      end
-
-      raise "No partitions specified in your kickstart file #{file}" if partitions.empty?
-
-      appliance_config.hardware.partitions = partitions
-
-      appliance_config
-    end
-
     # Add default repos (if present) to the list of additional repositories specified in appliance definition.
     def add_repos(repos)
       return if repos.empty?
@@ -116,13 +61,10 @@ module BoxGrinder
     def build_with_appliance_creator(appliance_definition_file, repos = {})
       @appliance_definition_file = appliance_definition_file
 
-      if File.extname(appliance_definition_file).eql?('.ks')
-        kickstart_file = appliance_definition_file
-      else
-        add_repos(repos) if @appliance_config.default_repos
-        kickstart_file = Kickstart.new(@config, @appliance_config, @dir, :log => @log).create
-        RPMDependencyValidator.new(@config, @appliance_config, @dir, :log => @log).resolve_packages
-      end
+      add_repos(repos) if @appliance_config.default_repos
+
+      kickstart_file = Kickstart.new(@config, @appliance_config, @dir, :log => @log).create
+      RPMDependencyValidator.new(@config, @appliance_config, @dir, :log => @log).resolve_packages
 
       @log.info "Building #{@appliance_config.name} appliance..."