vanagon 0.8.2 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94843aa8a37502b7e0548dc934267087f1ea5ed6
-  data.tar.gz: f8daf4b6a0e88855775c58f2a73fa8cdb354e322
+  metadata.gz: 66a9286e3d7f89a5d145e4a5745cbb4d08ac9276
+  data.tar.gz: 6e1cf377bd79799c726e583bd8401d81fc8e1475
 SHA512:
-  metadata.gz: 6212f217ffe247fecead7f8a105a22470e523d16dc87acc07e6df7ec013a806e9fad76f63eac876ce8b1b668a88c8ef3b3e7fef39a5cce7bb0d7fd21c2d0c9be
-  data.tar.gz: fea048b71df645483b7a9c988fc74784eb06897a520044d0e9b5c5e5ac3643c56ca94f968a4f489b5078b27aaf280afd485cc28011306a34a854f276484e4fdd
+  metadata.gz: 594d72e7506ae70075e34a690981f144b4d332204ceeee433462aaa82ec9611baf12f3e516d9283598532d4bc150b097b1d2fd2491f3d566feddb024cfb1e0bc
+  data.tar.gz: f2387a452a90af391f7cff1a5e498bb2322cc4ce8729b6fa26c6c43166173a0dd44285814bb1a2ab0009ec8d5b3b720e1b7dc7167a9f8707c2725689d3669fc1

data/README.md

@@ -68,6 +68,7 @@ the comma). If less targets are specified than platforms, the default engine
 (`pooler`) will be used for platforms without a target. If more targets are specified
 than platforms, the extra platforms will be ignored.
 
+Build machines should be cleaned between builds.
 
 #### Flagged arguments (can be anywhere in the command)
 

data/bin/build

@@ -2,7 +2,7 @@
 load File.expand_path(File.join(File.dirname(__FILE__), "..", "lib", "vanagon.rb"))
 
 optparse = Vanagon::OptParse.new("#{File.basename(__FILE__)} <project-name> <platform-name> [<target>] [options]",
-                                 [:workdir, :configdir, :engine, :preserve, :verbose, :skipcheck])
+                                 [:workdir, :configdir, :engine, :preserve, :verbose, :skipcheck, :only_build])
 options = optparse.parse! ARGV
 
 project = ARGV[0]

data/lib/vanagon/component/dsl.rb

@@ -39,8 +39,8 @@ class Vanagon
       # We only magically handle get_ methods, any other methods just get the
       # standard method_missing treatment.
       #
-      def method_missing(method, *args)
-        attribute_match = method.to_s.match(/get_(.*)/)
+      def method_missing(method_name, *args)
+        attribute_match = method_name.to_s.match(/get_(.*)/)
         if attribute_match
           attribute = attribute_match.captures.first
         else
@@ -50,6 +50,10 @@ class Vanagon
         @component.send(attribute)
       end
 
+      def respond_to_missing?(method_name, include_private = false)
+        method_name.to_s.start_with?('get_') || super
+      end
+
       # Set or add to the configure call for the component. The commands required to configure the component before building it.
       #
       # @param block [Proc] the command(s) required to configure the component
@@ -273,29 +277,15 @@ class Vanagon
         @component.url = the_url
       end
 
-      # Sets the md5 sum to verify the sum of the source
-      #
-      # @param md5 [String] md5 sum of the source for verification
-      def md5sum(md5)
-        @component.options[:sum] = md5
-        @component.options[:sum_type] = "md5"
-      end
-
-      # Sets the sha256 sum to verify the sum of the source
-      #
-      # @param sha256 [String] sha256 sum of the source for verification
-      def sha256sum(sha256)
-        @component.options[:sum] = sha256
-        @component.options[:sum_type] = "sha256"
-      end
-
-      # Sets the sha512 sum to verify the sum of the source
-      #
-      # @param sha512 [String] sha512 sum of the source for verification
-      def sha512sum(sha512)
-        @component.options[:sum] = sha512
-        @component.options[:sum_type] = "sha512"
+      def sum(value)
+        type = __callee__.to_s.gsub(/sum$/, '')
+        @component.options[:sum] = value
+        @component.options[:sum_type] = type
       end
+      alias_method :md5sum, :sum
+      alias_method :sha1sum, :sum
+      alias_method :sha256sum, :sum
+      alias_method :sha512sum, :sum
 
       # Sets the ref of the source for use in a git source
       #

data/lib/vanagon/component/source/git.rb

@@ -30,7 +30,7 @@ class Vanagon
         # Default options used when cloning; this may expand
         # or change over time.
         def default_options
-          @default_options ||= { ref: "refs/heads/master" }
+          @default_options ||= { ref: "HEAD" }
         end
         private :default_options
 
@@ -46,7 +46,6 @@ class Vanagon
           @url = URI.parse(url.to_s)
           @ref = opts[:ref]
           @workdir = workdir
-          @ref_name, @ref_type, = @ref.split('/', 3).reverse
 
           # We can test for Repo existence without cloning
           raise Vanagon::InvalidRepo, "#{url} not a valid Git repo" unless valid_remote?
@@ -62,10 +61,6 @@ class Vanagon
           update_submodules
         end
 
-        def ref
-          @ref_name || @ref
-        end
-
         # Return the correct incantation to cleanup the source directory for a given source
         #
         # @return [String] command to cleanup the source

data/lib/vanagon/component/source/http.rb

@@ -12,6 +12,9 @@ class Vanagon
         # Accessors :url, :file, :extension, :workdir, :cleanup are inherited from Local
         attr_accessor :sum, :sum_type
 
+        # Allowed checksum algorithms to use when validating files
+        CHECKSUM_TYPES = %w(md5 sha1 sha256 sha512).freeze
+
         class << self
           def valid_url?(target_url) # rubocop:disable Metrics/AbcSize
             uri = URI.parse(target_url.to_s)
@@ -51,6 +54,10 @@ class Vanagon
           unless sum_type
             fail "sum_type is required to validate the http source"
           end
+          unless CHECKSUM_TYPES.include? sum_type
+            fail %(checksum type "#{sum_type}" is invalid; please use #{CHECKSUM_TYPES.join(', ')})
+          end
+
           @url = url
           @sum = sum
           @workdir = workdir

data/lib/vanagon/component/source/local.rb

@@ -60,7 +60,7 @@ class Vanagon
         def copy
           puts "Copying file '#{url.basename}' to workdir"
 
-          FileUtils.cp(url, file)
+          FileUtils.cp_r(url, file)
         end
         alias_method :fetch, :copy
 
@@ -148,7 +148,13 @@ class Vanagon
         # @return [String] the directory that should be traversed into to build this source
         # @raise [RuntimeError] if the @extension for the @file isn't currently handled by the method
         def dirname
-          archive? ? File.basename(file, extension) : './'
+          # We are not treating file as a Pathname since other sources can inherit from this class
+          # which could cause file to be a URI instead of a string.
+          if archive? || File.directory?(file)
+            File.basename(file, extension)
+          else
+            './'
+          end
         end
 
         # Wrapper around the class method '.mangle'

data/lib/vanagon/driver.rb

@@ -13,12 +13,13 @@ class Vanagon
     attr_accessor :platform, :project, :target, :workdir, :verbose, :preserve
     attr_accessor :timeout, :retry_count
 
-    def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false, :verbose => false, :preserve => false }) # rubocop:disable Metrics/AbcSize
+    def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false, :verbose => false, :preserve => false, :only_build => nil }) # rubocop:disable Metrics/AbcSize
       @verbose = options[:verbose]
       @preserve = options[:preserve]
 
       @@configdir = options[:configdir] || File.join(Dir.pwd, "configs")
       components = options[:components] || []
+      only_build = options[:only_build]
       target = options[:target]
       engine = options[:engine] || 'pooler'
 
@@ -26,6 +27,7 @@ class Vanagon
       @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
       @project.settings[:verbose] = options[:verbose]
       @project.settings[:skipcheck] = options[:skipcheck]
+      filter_out_components(only_build) if only_build
       loginit('vanagon_hosts.log')
 
       load_engine(engine, @platform, target)
@@ -33,24 +35,42 @@ class Vanagon
       raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine}'")
     end
 
+    def filter_out_components(only_build)
+      # map each element in the only_build array to it's set of filtered components, then
+      # flatten all the results in to one array and set project.components to that.
+      @project.components = only_build.flat_map { |comp| @project.filter_component(comp) }.uniq
+      if @verbose
+        puts "Only building:"
+        @project.components.each { |comp| puts comp.name }
+      end
+    end
+
     def load_engine(engine_type, platform, target)
-      if platform.build_hosts
-        engine_type = 'hardware'
-      elsif platform.aws_ami
-        engine_type = 'ec2'
-      elsif platform.docker_image
-        engine_type = 'docker'
-      elsif target
-        engine_type = 'base'
+      if engine_type != 'always_be_scheduling'
+        if platform.build_hosts
+          engine_type = 'hardware'
+        elsif platform.aws_ami
+          engine_type = 'ec2'
+        elsif platform.docker_image
+          engine_type = 'docker'
+        elsif target
+          engine_type = 'base'
+        end
       end
       load_engine_object(engine_type, platform, target)
     end
 
     def load_engine_object(engine_type, platform, target)
       require "vanagon/engine/#{engine_type}"
-      @engine = Object::const_get("Vanagon::Engine::#{engine_type.capitalize}").new(platform, target)
+      @engine = Object::const_get("Vanagon::Engine::#{camelize(engine_type)}").new(platform, target)
     rescue
-      fail "No such engine '#{engine_type.capitalize}'"
+      fail "No such engine '#{camelize(engine_type)}'"
+    end
+
+    def camelize(string)
+      string.gsub(/(?:^|_)([a-z])?/) do |match|
+        (Regexp.last_match[1] || '').capitalize
+      end
     end
 
     def cleanup_workdir

data/lib/vanagon/engine/always_be_scheduling.rb

@@ -0,0 +1,92 @@
+require 'vanagon/engine/base'
+require 'json'
+
+class Vanagon
+  class Engine
+    # This engine allows build resources to be managed by the "Always be
+    # Scheduling" (ABS) scheduler (https://github.com/puppetlabs/always-be-scheduling)
+    #
+    # ABS expects to ask `build_host_info` for the needed resources for a build,
+    # and to have that return a platform name.  ABS will then acquire the
+    # desired build host resources and will later run a vanagon build, passing
+    # those resource hostnames in specifically.
+    #
+    # `build_host_info` will normally use the `hardware` engine when a hardware
+    # platform is queried. The `always_be_scheduling` engine's behavior will
+    # be invoked instead when:
+    #
+    # `build_host_info ... --engine always_be_scheduling` is specified on the
+    # command-line.
+    #
+    # Configuration:
+    #
+    # Project platform configurations can specify the platform name to be returned
+    # via the `abs_resource_name` attribute. If this is not set but `vmpooler_template`
+    # is set, then the `vmpooler_template` value will be used. Otherwise, the
+    # platform name will be returned unchanged.
+    #
+    # Example 1:
+    #
+    # platform 'ubuntu-10.04-amd64' do |plat|
+    #   plat.vmpooler_template 'ubuntu-1004-amd64'
+    # end
+    #
+    # $ build_host_info puppet-agent ubuntu-10.04-amd64
+    # {"name":"ubuntu-10.04-amd64","engine":"pooler"}
+    #
+    # $ build_host_info puppet-agent ubuntu-10.04-amd64 --engine always_be_scheduling
+    # {"name":"ubuntu-10.04-amd64","engine":"always_be_scheduling"}
+    #
+    #
+    # Example 2:
+    #
+    # platform 'aix-5.3-ppc' do |plat|
+    #   plat.build_host ['aix53-builder-1.example.com']
+    #   plat.abs_resource_name 'aix-53-ppc'
+    # end
+    #
+    # $ build_host_info puppet-agent aix-5.3-ppc
+    # {"name":"aix53-builder-1.example.com","engine":"hardware"}
+    #
+    # $ build_host_info puppet-agent aix-5.3-ppc --engine always_be_scheduling
+    # {"name":"aix-53-ppc","engine":"always_be_scheduling"}
+    #
+    #
+    # Example 3:
+    #
+    # platform 'aix-5.3-ppc' do |plat|
+    #   plat.build_host ['aix53-builder-1.example.com']
+    #   plat.vmpooler_template
+    #   plat.abs_resource_name 'aix-53-ppc'
+    # end
+    #
+    # $ build_host_info puppet-agent aix-5.3-ppc
+    # {"name":"aix53-builder-1.example.com","engine":"hardware"}
+    #
+    # $ build_host_info puppet-agent aix-5.3-ppc --engine always_be_scheduling
+    # {"name":"aix-53-ppc","engine":"always_be_scheduling"}
+    class AlwaysBeScheduling < Base
+      def initialize(platform, target)
+        super
+
+        Vanagon::Driver.logger.debug "AlwaysBeScheduling engine invoked."
+      end
+
+      # Get the engine name
+      def name
+        'always_be_scheduling'
+      end
+
+      # return the platform name as the "host" name
+      def build_host_name
+        if @platform.abs_resource_name
+          @platform.abs_resource_name
+        elsif @platform.vmpooler_template
+          @platform.vmpooler_template
+        else
+          @platform.name
+        end
+      end
+    end
+  end
+end

data/lib/vanagon/engine/base.rb

@@ -36,8 +36,7 @@ class Vanagon
 
       # Steps needed to tear down or clean up the system after the build is
       # complete
-      def teardown
-      end
+      def teardown; end
 
       # Applies the steps needed to extend the system to build packages against
       # the target system

data/lib/vanagon/engine/hardware.rb

@@ -27,7 +27,7 @@ class Vanagon
         host
       end
 
-      # Iterarte over the options and find a node open to lock.
+      # Iterate over the options and find a node open to lock.
       def node_lock(hosts)
         hosts.each do |h|
           Vanagon::Driver.logger.info "Attempting  to lock #{h}."

data/lib/vanagon/engine/pooler.rb

@@ -1,4 +1,5 @@
 require 'vanagon/engine/base'
+require 'yaml'
 
 class Vanagon
   class Engine
@@ -29,19 +30,50 @@ class Vanagon
         @build_host_template_name
       end
 
-      # This method loads the pooler token from one of two locations
+      # Retrieve the pooler token from an environment variable
+      # ("VMPOOLER_TOKEN") or from a number of potential configuration
+      # files (~/.vanagon-token or ~/.vmfloaty.yml).
       # @return [String, nil] token for use with the vmpooler
       def load_token
-        if ENV['VMPOOLER_TOKEN']
-          token = ENV['VMPOOLER_TOKEN']
-        else
-          token_file = File.expand_path("~/.vanagon-token")
-          if File.exist?(token_file)
-            token = File.open(token_file).read.chomp
-          end
-        end
-        token
+        ENV['VMPOOLER_TOKEN'] || token_from_file
+      end
+
+      # a wrapper method around retrieving a vmpooler token,
+      # with an explicitly ordered preference for a Vanagon-specific
+      # token file or a preexisting vmfoaty yaml file.
+      #
+      # @return [String, nil] token for use with the vmpooler
+      def token_from_file
+        read_vanagon_token || read_vmfloaty_token
+      end
+      private :token_from_file
+
+      # Read a vmpooler token from the plaintext vanagon-token file,
+      # as outlined in the project README.
+      #
+      # @return [String, nil] the vanagon vmpooler token value
+      def read_vanagon_token(path = "~/.vanagon-token")
+        absolute_path = File.expand_path(path)
+        return nil unless File.exist?(absolute_path)
+
+        puts "Reading vmpooler token from: #{path}"
+        File.read(absolute_path).chomp
+      end
+      private :read_vanagon_token
+
+      # Read a vmpooler token from the yaml formatted vmfloaty config,
+      # as outlined by the vmfloaty project:
+      # https://github.com/briancain/vmfloaty
+      #
+      # @return [String, nil] the vmfloaty vmpooler token value
+      def read_vmfloaty_token(path = "~/.vmfloaty.yml")
+        absolute_path = File.expand_path(path)
+        return nil unless File.exist?(absolute_path)
+
+        puts "Reading vmpooler token from: #{path}"
+        YAML.load_file(absolute_path)['token']
       end
+      private :read_vmfloaty_token
 
       # This method is used to obtain a vm to build upon using the Puppet Labs'
       # vmpooler (https://github.com/puppetlabs/vmpooler)

data/lib/vanagon/optparse.rb

@@ -10,6 +10,7 @@ class Vanagon
         :skipcheck => ['--skipcheck', 'Skip the `check` stage when building components'],
         :preserve => ['-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)'],
         :verbose => ['-v', '--verbose', 'Verbose output (does nothing)'],
+        :only_build => ["--only-build COMPONENT,COMPONENT,...", Array, 'Only build this array of components']
       }.freeze
 
     def initialize(banner, options = [])

data/lib/vanagon/platform.rb

@@ -2,13 +2,16 @@ require 'vanagon/platform/dsl'
 
 class Vanagon
   class Platform
-    attr_accessor :make, :servicedir, :defaultdir, :provisioning, :num_cores, :tar
-    attr_accessor :build_dependencies, :name, :vmpooler_template, :cflags, :ldflags, :settings
-    attr_accessor :servicetype, :patch, :architecture, :codename, :os_name, :os_version
-    attr_accessor :docker_image, :ssh_port, :rpmbuild, :install, :platform_triple
-    attr_accessor :target_user, :package_type, :find, :sort, :build_hosts, :copy, :cross_compiled
-    attr_accessor :aws_ami, :aws_user_data, :aws_shutdown_behavior, :aws_key_name, :aws_region, :aws_key
-    attr_accessor :aws_instance_type, :aws_vpc_id, :aws_subnet_id, :output_dir
+    attr_accessor :make, :servicedir, :defaultdir, :provisioning, :num_cores
+    attr_accessor :tar, :build_dependencies, :name, :vmpooler_template
+    attr_accessor :abs_resource_name, :cflags, :ldflags, :settings
+    attr_accessor :servicetype, :patch, :architecture, :codename, :os_name
+    attr_accessor :os_version, :docker_image, :ssh_port, :rpmbuild, :install
+    attr_accessor :platform_triple, :target_user, :package_type, :find, :sort
+    attr_accessor :build_hosts, :copy, :cross_compiled, :aws_ami
+    attr_accessor :aws_user_data, :aws_shutdown_behavior, :aws_key_name
+    attr_accessor :aws_region, :aws_key, :aws_instance_type, :aws_vpc_id
+    attr_accessor :aws_subnet_id, :output_dir
 
     # Platform names currently contain some information about the platform. Fields
     # within the name are delimited by the '-' character, and this regex can be used to