vanagon 0.4.1 → 0.5.0

data/lib/vanagon/component/source/http.rb

@@ -10,10 +10,10 @@ class Vanagon
         attr_accessor :url, :sum, :file, :extension, :workdir, :cleanup
 
         # Extensions for files we intend to unpack during the build
-        ARCHIVE_EXTENSIONS = '.tar.gz', '.tgz'
+        ARCHIVE_EXTENSIONS = '.tar.gz', '.tgz', '.zip'
 
         # Extensions for files we aren't going to unpack during the build
-        NON_ARCHIVE_EXTENSIONS = '.gem', '.ru', '.txt', '.conf', '.ini', '.gpg', '.rb', '.sh', '.csh', '.xml'
+        NON_ARCHIVE_EXTENSIONS = '.gem', '.ru', '.txt', '.conf', '.ini', '.gpg', '.rb', '.sh', '.csh', '.xml', '.vim', '.json', '.service'
 
         # Constructor for the Http source type
         #
@@ -100,7 +100,12 @@ class Vanagon
         # @raise [RuntimeError] an exception is raised if there is no known extraction method for @extension
         def extract(tar)
           if ARCHIVE_EXTENSIONS.include?(@extension)
-            return "gunzip -c '#{@file}' | '#{tar}' xf -"
+            case @extension
+            when ".tar.gz", ".tgz"
+              return "gunzip -c '#{@file}' | '#{tar}' xf -"
+            when ".zip"
+              return "unzip '#{@file}'"
+            end
           elsif NON_ARCHIVE_EXTENSIONS.include?(@extension)
             # Don't need to unpack gems, ru, txt, conf, ini, gpg
             return nil

data/lib/vanagon/driver.rb

@@ -12,7 +12,7 @@ class Vanagon
     include Vanagon::Utilities
     attr_accessor :platform, :project, :target, :workdir, :verbose, :preserve
 
-    def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil })
+    def initialize(platform, project, options = { :configdir => nil, :target => nil, :engine => nil, :components => nil, :skipcheck => false })
       @verbose = false
       @preserve = false
 
@@ -23,16 +23,19 @@ class Vanagon
 
       @platform = Vanagon::Platform.load_platform(platform, File.join(@@configdir, "platforms"))
       @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
+      @project.settings[:skipcheck] = options[:skipcheck]
+      @@logger = Logger.new('vanagon_hosts.log')
+      @@logger.progname = 'vanagon'
 
       # If a target has been given, we don't want to make any assumptions about how to tear it down.
       engine = 'base' if target
+      # Hardware has explicit teardown to unlock the node
+      engine = 'hardware' if @platform.build_hosts
       require "vanagon/engine/#{engine}"
       @engine = Object.const_get("Vanagon::Engine::#{engine.capitalize}").new(@platform, target)
 
-      @@logger = Logger.new('vanagon_hosts.log')
-      @@logger.progname = 'vanagon'
     rescue LoadError => e
-      raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{@engine_name}'.")
+      raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine}'")
     end
 
     def cleanup_workdir
@@ -88,7 +91,11 @@ class Vanagon
       puts e
       puts e.backtrace.join("\n")
       raise e
-    end
+    ensure
+      if @engine == "hardware"
+        @engine.teardown
+      end
+  end
 
     def prepare(workdir = nil)
       @workdir = workdir ? FileUtils.mkdir_p(workdir).first : Dir.mktmpdir

data/lib/vanagon/engine/hardware.rb

@@ -0,0 +1,64 @@
+require 'vanagon/engine/base'
+require 'json'
+require 'lock_manager'
+
+LOCK_MANAGER_HOST = ENV['LOCK_MANAGER_HOST'] || 'redis'
+LOCK_MANAGER_PORT = ENV['LOCK_MANAGER_PORT'] || 6379
+VANAGON_LOCK_USER = ENV['USER']
+
+
+class Vanagon
+  class Engine
+    # Class to use when building on a hardware device (e.g. AIX, Switch, etc)
+    #
+    class Hardware < Base
+
+      # This method is used to obtain a vm to build upon
+      # For the base class we just return the target that was passed in
+      def select_target
+        @target = node_lock(@build_hosts)
+      end
+
+      # Poll for a lock
+      def polling_lock(host)
+        Vanagon::Driver.logger.info "Polling for a lock on #{host}."
+        @lockman.polling_lock(host, VANAGON_LOCK_USER, "Vanagon automated lock")
+        Vanagon::Driver.logger.info "Lock acquired on #{host}."
+        puts "Lock acquired on #{host} for #{VANAGON_LOCK_USER}."
+        host
+      end
+
+      # Iterarte over the options and find a node open to lock.
+      def node_lock(hosts)
+        hosts.each do |h|
+          Vanagon::Driver.logger.info "Attempting  to lock #{h}."
+          if @lockman.lock(h, VANAGON_LOCK_USER, "Vanagon automated lock")
+            Vanagon::Driver.logger.info "Lock acquired on #{h}."
+            puts "Lock acquired on #{h} for #{VANAGON_LOCK_USER}."
+            return h
+          end
+        end
+        # If they are all locked, fall back to a polling lock on last item
+        polling_lock(hosts.pop)
+      end
+
+      # Steps needed to tear down or clean up the system after the build is
+      # complete. In this case, we'll attempt to unlock the hardware
+      def teardown
+        Vanagon::Driver.logger.info "Removing lock on #{@target}."
+        puts "Removing lock on #{@target}."
+        @lockman.unlock(@target, VANAGON_LOCK_USER)
+      end
+
+      def initialize(platform, target)
+        Vanagon::Driver.logger.debug "Hardware engine invoked."
+        @platform = platform
+        @build_hosts = platform.build_hosts
+        # Redis is the only backend supported in lock_manager currently
+        @lockman = LockManager.new(type: 'redis', server: LOCK_MANAGER_HOST)
+        super
+        @required_attributes << "build_hosts"
+      end
+    end
+  end
+end

data/lib/vanagon/optparse.rb

@@ -2,44 +2,27 @@ require 'optparse'
 
 class Vanagon
   class OptParse
+
+    FLAGS = {
+        :workdir => ['-w DIR', '--workdir DIR', "Working directory where build source should be put (defaults to a tmpdir)"],
+        :configdir => ['-c', '--configdir DIR', 'Configs dir (defaults to $pwd/configs)'],
+        :target => ['-t HOST', '--target HOST', 'Configure a target machine for build and packaging (defaults to grabbing one from the pooler)'],
+        :engine => ['-e ENGINE', '--engine ENGINE', "A custom engine to use (defaults to the pooler) [base, local, docker, pooler currently supported]"],
+        :skipcheck => ['--skipcheck', 'Ship the `check` stage when building components'],
+        :preserve => ['-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)'],
+        :verbose => ['-v', '--verbose', 'Verbose output (does nothing)'],
+      }
+
     def initialize(banner, options = [])
       @options = Hash[options.map { |v| [v, nil] }]
       @optparse = OptionParser.new do |opts|
         opts.banner = banner
 
-        if @options.include?(:workdir)
-          opts.on('-w DIR', '--workdir DIR', "Working directory where build source should be put (defaults to a tmpdir)") do |dir|
-            @options[:workdir] = dir
-          end
-        end
-
-        if @options.include?(:configdir)
-          opts.on('-c', '--configdir DIR', 'Configs dir (defaults to $pwd/configs)') do |dir|
-            @options[:configdir] = dir
-          end
-        end
-
-        if @options.include?(:target)
-          opts.on('-t HOST', '--target HOST', 'Configure a target machine for build and packaging (defaults to grabbing one from the pooler)') do |name|
-            @options[:target] = name
-          end
-        end
-
-        if @options.include?(:engine)
-          opts.on('-e ENGINE', '--engine ENGINE', "A custom engine to use (defaults to the pooler) [base, local, docker, pooler currently supported]") do |engine|
-            @options[:engine] = engine
-          end
-        end
-
-        if @options.include?(:preserve)
-          opts.on('-p', '--preserve', 'Whether to tear down the VM on success or not (defaults to false)') do |flag|
-            @options[:preserve] = flag
-          end
-        end
-
-        if @options.include?(:verbose)
-          opts.on('-v', '--verbose', 'Verbose output (does nothing)') do |flag|
-            @options[:verbose] = flag
+        FLAGS.each_pair do |name, args|
+          if @options.include?(name)
+            opts.on(*args) do |value|
+              @options[name] = value
+            end
           end
         end
 

data/lib/vanagon/patch.rb

@@ -0,0 +1,39 @@
+require 'vanagon/errors'
+
+class Vanagon
+  class Patch
+
+    # @!attribute [r] path
+    #   @return [String] The path to the patch
+    attr_reader :path
+
+    # @!attribute [r] strip
+    #   @return [Integer] the number of path components to strip from the patch path
+    attr_reader :strip
+
+    # @!attribute [r] fuzz
+    #   @return [Integer] The fuzz factor for applying the patch
+    attr_reader :fuzz
+
+    # @!attribute [r] after
+    #   @return [String] What step should this patch be applied to, one of ["unpack", "install"]
+    attr_reader :after
+
+    # @!attribute [r] destination
+    #   @return [String] The working directory where this patch will be applied. Only used for post-installation patches.
+    attr_reader :destination
+
+    def initialize(path, strip, fuzz, after, destination)
+      raise Vanagon::Error, "We can only apply patches after the source is unpacked or after installation" unless ['unpack', 'install'].include?(after)
+      @path = path
+      @strip = strip
+      @fuzz = fuzz
+      @after = after
+      @destination = destination
+    end
+
+    def cmd(platform)
+      "#{platform.patch} --strip=#{@strip} --fuzz=#{@fuzz} --ignore-whitespace < $(workdir)/patches/#{File.basename(@path)}"
+    end
+  end
+end

data/lib/vanagon/platform.rb

@@ -6,6 +6,7 @@ class Vanagon
     attr_accessor :build_dependencies, :name, :vmpooler_template, :cflags, :ldflags, :settings
     attr_accessor :servicetype, :patch, :architecture, :codename, :os_name, :os_version
     attr_accessor :docker_image, :ssh_port, :rpmbuild, :install, :platform_triple
+    attr_accessor :package_type, :build_hosts
 
     # Platform names currently contain some information about the platform. Fields
     # within the name are delimited by the '-' character, and this regex can be used to
@@ -137,7 +138,7 @@ class Vanagon
     #
     # @return [true, false] true if it is a debian variety, false otherwise
     def is_deb?
-      return !!@name.match(/^(debian|ubuntu|cumulus)-.*$/)
+      return !!@name.match(/^(debian|ubuntu|cumulus|huaweios)-.*$/)
     end
 
     # Utility matcher to determine is the platform is a redhat variety or
@@ -146,7 +147,7 @@ class Vanagon
     # @return [true, false] true if it is a redhat variety or uses rpm
     # under the hood, false otherwise
     def is_rpm?
-      return !!@name.match(/^(aix|cisco-wrlinux|el|eos|fedora|huaweios|sles)-.*$/)
+      return !!@name.match(/^(aix|cisco-wrlinux|el|eos|fedora|sles)-.*$/)
     end
 
     # Utility matcher to determine is the platform is an enterprise linux variety

data/lib/vanagon/platform/dsl.rb

@@ -3,7 +3,7 @@ require 'vanagon/platform/rpm'
 require 'vanagon/platform/rpm/aix'
 require 'vanagon/platform/rpm/sles'
 require 'vanagon/platform/rpm/wrl'
-require 'vanagon/platform/swix'
+require 'vanagon/platform/rpm/eos'
 require 'vanagon/platform/osx'
 require 'vanagon/platform/solaris_10'
 require 'vanagon/platform/solaris_11'
@@ -33,12 +33,10 @@ class Vanagon
                       Vanagon::Platform::RPM.new(@name)
                     when /^sles-/
                       Vanagon::Platform::RPM::SLES.new(@name)
-                    when /^huaweios-/
-                      Vanagon::Platform::RPM::WRL.new(@name)
-                    when /^(cumulus|debian|ubuntu)-/
+                    when /^(cumulus|debian|huaweios|ubuntu)-/
                       Vanagon::Platform::DEB.new(@name)
                     when /^eos-/
-                      Vanagon::Platform::RPM::Swix.new(@name)
+                      Vanagon::Platform::RPM::EOS.new(@name)
                     when /^osx-/
                       Vanagon::Platform::OSX.new(@name)
                     when /^solaris-10/
@@ -92,6 +90,13 @@ class Vanagon
         @platform.tar = tar_cmd
       end
 
+      # Set the type of package we are going to build for this platform
+      #
+      # @param pkg_type [String] The type of package we are going to build for this platform
+      def package_type(pkg_type)
+        @platform.package_type = pkg_type
+      end
+
       # Set the path to rpmbuild for the platform
       #
       # @param rpmbuild_cmd [String] Full path to rpmbuild with arguments to be used by default
@@ -156,6 +161,19 @@ class Vanagon
         @platform.servicetype = type
       end
 
+      # Set the list of possible host to perform a build on (when not using
+      # pooler or CLI flags)
+      #
+      # @param type [Array] the names of the hosts (must be resolvable)
+      # @rase ArgumentError if builds_hosts has no arguments
+      def build_hosts(*args)
+        raise ArgumentError, "build_hosts requires at least one host to be a build target." if args.empty?
+        @platform.build_hosts = Array(args).flatten
+      end
+
+      # Because single vs plural is annoying to remember
+      alias_method :build_host, :build_hosts
+
       # Set the name of this platform as the vm pooler expects it
       #
       # @param name [String] name of the target template to use from the vmpooler

data/lib/vanagon/platform/rpm/eos.rb

@@ -0,0 +1,83 @@
+class Vanagon
+  class Platform
+    class RPM
+      class EOS < Vanagon::Platform::RPM
+        # The specific bits used to generate an EOS package for a given project
+        #
+        # @param project [Vanagon::Project] project to build an EOS package of
+        # @return [Array] list of commands required to build the EOS package
+        # for the given project from an rpm or a swix
+        def generate_package(project)
+          # If nothing is passed in as platform type, default to building a swix package
+          if project.platform.package_type.nil? || project.platform.package_type.empty?
+            return generate_swix_package(project)
+          else
+            case project.platform.package_type
+            when "rpm"
+              return super(project)
+            when "swix"
+              return generate_swix_package(project)
+            else
+              fail "I don't know how to build package type '#{project.platform.package_type}' for EOS. Teach me?"
+            end
+          end
+        end
+
+        # Method to derive the package name for the project
+        #
+        # @param project [Vanagon::Project] project to name
+        # @return [String] name of the EOS package for this project
+        def package_name(project)
+          # If nothing is passed in as platform type, default to building a swix package
+          if project.platform.package_type.nil? || project.platform.package_type.empty?
+            return swix_package_name(project)
+          else
+            case project.platform.package_type
+            when "rpm"
+              return super(project)
+            when "swix"
+              return swix_package_name(project)
+            else
+              fail "I don't know how to name package type '#{project.platform.package_type}' for EOS. Teach me?"
+            end
+          end
+        end
+
+        # The specific bits used to generate an SWIX package for a given project
+        #
+        # @param project [Vanagon::Project] project to build a SWIX package of
+        # @return [Array] list of commands required to build the SWIX package
+        # for the given project from an rpm
+        def generate_swix_package(project)
+          target_dir = project.repo ? output_dir(project.repo) : output_dir
+          commands = ["bash -c 'mkdir -p $(tempdir)/rpmbuild/{SOURCES,SPECS,BUILD,RPMS,SRPMS}'",
+          "cp #{project.name}-#{project.version}.tar.gz $(tempdir)/rpmbuild/SOURCES",
+          "cp file-list-for-rpm $(tempdir)/rpmbuild/SOURCES",
+          "cp #{project.name}.spec $(tempdir)/rpmbuild/SPECS",
+          "PATH=/opt/freeware/bin:$$PATH #{@rpmbuild} -bb --target #{@architecture} #{rpm_defines} $(tempdir)/rpmbuild/SPECS/#{project.name}.spec",
+          "mkdir -p output/#{target_dir}",
+          "cp $(tempdir)/rpmbuild/*RPMS/**/*.rpm ./output/#{target_dir}"]
+
+
+          pkgname_swix = swix_package_name(project)
+          pkgname_rpm = pkgname_swix.sub(/swix$/, 'rpm')
+          commands += ["echo 'format: 1' > ./output/#{target_dir}/manifest.txt",
+          "echo \"primaryRpm: #{pkgname_rpm}\" >> ./output/#{target_dir}/manifest.txt",
+          "echo #{pkgname_rpm}-sha1: `sha1sum ./output/#{target_dir}/#{pkgname_rpm}`",
+          "cd ./output/#{target_dir}/ && zip #{pkgname_swix} manifest.txt #{pkgname_rpm}",
+          "rm ./output/#{target_dir}/manifest.txt ./output/#{target_dir}/#{pkgname_rpm}"]
+
+          commands
+        end
+
+        # Method to derive the package name for the project
+        #
+        # @param project [Vanagon::Project] project to name
+        # @return [String] name of the SWIX package for this project
+        def swix_package_name(project)
+          "#{project.name}-#{project.version}-#{project.release}.#{os_name}#{os_version}.#{project.noarch ? 'noarch' : @architecture}.swix"
+        end
+      end
+    end
+  end
+end

data/lib/vanagon/platform/solaris_10.rb

@@ -10,6 +10,19 @@ class Vanagon
         name_and_version = "#{project.name}-#{project.version}"
         pkg_name = package_name(project)
 
+        # Generate list of dirs in the package and create an exlplicit search
+        # string for AWK to use in order to explicitly define which directories
+        # to put in to the prototype file. Note that the Regexp object was avoided
+        # because the output from Regexp would create failures from AWK as the conversion
+        # from a Regexp obj to a string is formatted in a sub-optimal way that would have
+        # required more string manipulation anyway. the string should be formatted like so:
+        #         && ($$3 ~ /directory\/regex.*/ || $$3 ~ /another\/directory\/regex.*/)
+        # for as many iterations as there are directries in the package
+        pkgdirs = project.get_root_directories.map { |dir| dir.sub(/^\//, "").gsub(/([\/\.])+/, '\\\\\1') + '.*' }
+        explicit_search_string = pkgdirs.map do |dir_regex|
+          " $$3 ~ /" + dir_regex + "/ "
+        end.join("||")
+
         # Here we maintain backward compatibility with older vanagon versions
         # that did this by default.  This shim should get removed at some point
         # in favor of just letting the makefile deliver the bill-of-materials
@@ -42,14 +55,16 @@ class Vanagon
           # - The bin directory and all bin files are owned by root:bin instead of root:sys
           # - All files under lib are owned by root:bin instead of root:sys
           # - All .so files are owned by root:bin instead of root:sys
+          # - Explicity only include directories in the package contents
+          #   (this should exclude things like root/bin root/var and such)
           %((cd $(tempdir)/#{name_and_version}; pkgproto . | sort | awk ' \
-            $$1 ~ /^d$$/ {print "d",$$2,$$3,"0755 root sys";} \
+            $$1 ~ /^d$$/ && (#{explicit_search_string}) {print "d",$$2,$$3,"0755 root sys";} \
             $$1 ~ /^s$$/ {print;} \
             $$1 ~ /^f$$/ {print "f",$$2,$$3,$$4,"root sys";} \
             $$1 !~ /^[dfs]$$/ {print;} ' | /opt/csw/bin/gsed \
-               -e '/^[fd] [^ ]\\+ .*[/]s\\?bin/ {s/root sys$$/root bin/}' \
-               -e '/^[fd] [^ ]\\+ .*[/]lib[/][^/ ]\\+ / {s/root sys$$/root bin/}' \
-               -e '/^[fd] [^ ]\\+ .*[/][^ ]\\+[.]so / {s/root sys$$/root bin/}' >> ../packaging/proto) ),
+               -e '/^[fd] [^ ]\\+ .*[/]s\\?bin[^ ]\\+/ {s/root sys$$/root bin/}' \
+               -e '/^[fd] [^ ]\\+ .*[/]lib[/][^ ]\\+/ {s/root sys$$/root bin/}' \
+               -e '/^[fd] [^ ]\\+ .*[/][^ ]\\+[.]so/ {s/root sys$$/root bin/}' >> ../packaging/proto)),
           %((cd $(tempdir); #{project.get_directories.map { |dir| "/opt/csw/bin/ggrep -q 'd none #{dir.path.sub(/^\//, '')}' packaging/proto || echo 'd none #{dir.path.sub(/^\//, '')} #{dir.mode || '0755'} #{dir.owner || 'root'} #{dir.group || 'sys'}' >> packaging/proto" }.join('; ')})),
 
           # Actually build the package