vanagon 0.19.1 → 0.22.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5dc9e0c896685503aafc10ec2bae7531dd36299289a7b07850db2ec5547f32c
-  data.tar.gz: 8ec86222166a7d4ecd99026c3ca0e46793f2da2b3f284eafe024236bc2449d56
+  metadata.gz: 1cd74e5ec690b73ee51e201f896f81a448020c90509107ec229c603b19b578bb
+  data.tar.gz: 9584067cc8334e4b280f7105834ffa7efe08d62e5b418f89cc53ee90c6f436f6
 SHA512:
-  metadata.gz: 54be38ffc28ceb63f22de1dd5912c823881f7d5c0188733960ddfa2c170c959174b181491db92007f8319a762f22cef0ed417ecf44bf3ad7f1305c18e7b89e56
-  data.tar.gz: ea4ff7c8f3ed9ebafc4e9fefbb689464431ff22299fb7c4bc9d1582451c249dba8628c28426b2d9cd9c46526a4e9be9ded30c060bbe70acb5b2d6cb3021174b0
+  metadata.gz: 1637ab1e2874427d490551f7e166a6607aadf6fa4c452697d4eb857bd5735369ad3977f6e53efc1e6b99c3bbda6bb5a0422be4366ccebeab00eda49805a93d05
+  data.tar.gz: 2e476edabd0575c234b2b94a23947b2d0c9c5dced079648deea0a014e98b15bd69bee833e02d234ab11eb6949323ac4f9aa7743fcc07918ed8ef5833070de546

data/README.md

@@ -85,6 +85,24 @@ wheezy and build my project against it.
 For more detailed examples of the DSLs available, please see the
 [examples](examples) directory and the YARD documentation for Vanagon.
 
+### CLI commands
+
+The vanagon command line tool contains these commands: 
+
+* `build`               build a package given a project and platform
+* `build_host_info`     print information about build hosts
+* `build_requirements`  print external packages required to build project
+* `completion`          outputs path to tab completion script
+* `inspect`             a build dry-run, printing lots of information about the build
+* `list`                shows a list of available projects and platforms
+* `render`              create local versions of packaging artifacts for project
+* `sign`                sign a package
+* `ship`                upload a package to a distribution server
+* `help`                print this help
+
+Commands are called with a git-like pattern of `vanagon <subcommand>`. 
+For example: `vanagon list`
+
 ### CLI changes and deprecations (from version 0.16.0)
 
 Prior to 0.16.0, the vanagon command line contained these commands
@@ -197,8 +215,8 @@ Port of the system where redis is running. Defaults to *6379*.
 ##### `VANAGON_USE_MIRRORS`
 Controls whether component sources are downloaded directly from upstream URLs
 or from configured mirrors. Most Puppet projects using Vanagon default to
-fetching components from internal mirrors. Set this variable to `n` when
-building outside of the Puppet private network to download directly from
+fetching components from internal mirrors. Set this variable to `n` or `false`
+when building outside of the Puppet private network to download directly from
 upstream sources.
 
 ##### `VANAGON_RETRY_COUNT`
@@ -212,6 +230,13 @@ time. The default value is *7200* seconds(120 minutes) but setting to any
 integer value these components to fail after the `VANAGON_TIMEOUT` count is reached.
 Note that this value is expected to be in seconds.
 
+##### `VANAGON_FORCE_SIGNING`
+By default, Vanagon does not fail if extra files signing fails, it just logs an
+error and continues building the package. This is unwanted behavior in
+environments where we expect a hard failure when signing cannot proceed. To
+force Vanagon to fail if extra files signing fails, ensure this variable is set
+before starting a build.
+
 #### Example usage
 `vanagon build --preserve puppet-agent el-6-i386` will build the puppet-agent project
 on the el-6-i386 platform and leave the host intact afterward.

data/lib/vanagon/cli/list.rb

@@ -11,6 +11,7 @@ class Vanagon
         Options:
           -h, --help                       Display help
           -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
+          -d, --defaults                   Display the list of default platforms
           -l, --platforms                  Display a list of platforms
           -r, --projects                   Display a list of projects
           -s, --use-spaces                 Displays the list as space separated
@@ -36,12 +37,21 @@ class Vanagon
           exit 1
         end
 
+        default_list = Dir.children(File.join(File.dirname(__FILE__), '..', 'platform', 'defaults')).map do |platform|
+          File.basename(platform, File.extname(platform))
+        end.sort
+
         platform_list = Dir.children(File.join(options[:configdir], 'platforms')).map do |platform|
           File.basename(platform, File.extname(platform))
-        end
+        end.sort
 
         project_list = Dir.children(File.join(options[:configdir], 'projects')).map do |project|
           File.basename(project, File.extname(project))
+        end.sort
+
+        if options[:defaults]
+          puts "- Defaults", output(default_list, options[:use_spaces])
+          return
         end
 
         if options[:projects] == options[:platforms]
@@ -63,6 +73,7 @@ class Vanagon
       def options_translate(docopt_options)
         translations = {
           '--configdir' => :configdir,
+          '--defaults' => :defaults,
           '--platforms' => :platforms,
           '--projects' => :projects,
           '--use-spaces' => :use_spaces,

data/lib/vanagon/cli/ship.rb

@@ -22,16 +22,6 @@ class Vanagon
       def run(_)
         ENV['PROJECT_ROOT'] = Dir.pwd
 
-        artifactory_warning = <<-DOC
-          Unable to ship packages to artifactory. Please make sure you are pointing to a
-          recent version of packaging in your Gemfile. Please also make sure you include
-          the artifactory gem in your Gemfile.
-
-          Examples:
-            gem 'packaging', :github => 'puppetlabs/packaging', branch: '1.0.x'
-            gem 'artifactory'
-        DOC
-
         if Dir['output/**/*'].select { |entry| File.file?(entry) }.empty?
           VanagonLogger.error 'vanagon: Error: No packages to ship in the "output" directory. Maybe build some first?'
           exit 1
@@ -40,13 +30,7 @@ class Vanagon
         require 'packaging'
         Pkg::Util::RakeUtils.load_packaging_tasks
         Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship', 'artifacts', 'output')
-        begin
-          Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship_to_artifactory', 'output')
-        rescue LoadError
-          VanagonLogger.error artifactory_warning
-        rescue StandardError
-          VanagonLogger.error artifactory_warning
-        end
+        Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship_to_artifactory', 'output')
       end
     end
   end

data/lib/vanagon/component.rb

@@ -180,6 +180,7 @@ class Vanagon
       @preremove_actions = []
       @postremove_actions = []
       @install_only = false
+      @service = []
     end
 
     # Adds the given file to the list of files and returns @files.
@@ -303,9 +304,9 @@ class Vanagon
     #
     # @param workdir [String] working directory to put the source into
     def get_source(workdir) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
-      opts = options.merge({ workdir: workdir })
+      opts = options.merge({ workdir: workdir, dirname: dirname })
       if url || !mirrors.empty?
-        if ENV['VANAGON_USE_MIRRORS'] == 'n'
+        if ENV['VANAGON_USE_MIRRORS'] == 'n' or ENV['VANAGON_USE_MIRRORS'] == 'false'
           fetch_url(opts)
         else
           fetch_mirrors(opts) || fetch_url(opts)

data/lib/vanagon/component/dsl.rb

@@ -168,37 +168,42 @@ class Vanagon
       # @param service_file [String] path to the service file relative to the source
       # @param default_file [String] path to the default file relative to the source
       # @param service_name [String] name of the service
-      # @param service_type [String] type of the service (network, application, system, etc)
-      # @param link_target [String] executable service file should be linked to
-      def install_service(service_file, default_file = nil, service_name = @component.name, service_type: nil, link_target: nil) # rubocop:disable Metrics/AbcSize
-        case @component.platform.servicetype
+      # @param options optional extra parameters
+      #        service_type [String] type of the service (network, application, system, etc)
+      #        init_system [String] the init system on which to install service (sysv, systemd)
+      #        link_target [String] executable service file should be linked to
+      def install_service(service_file, default_file = nil, service_name = @component.name, **options) # rubocop:disable Metrics/AbcSize
+        init_system = options[:init_system] || @component.platform.servicetype
+        servicedir = @component.platform.get_service_dir(init_system)
+
+        case init_system
         when "sysv"
-          target_service_file = File.join(@component.platform.servicedir, service_name)
+          target_service_file = File.join(servicedir, service_name)
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0755'
           default_mode = '0644'
         when "systemd"
-          target_service_file = File.join(@component.platform.servicedir, "#{service_name}.service")
+          target_service_file = File.join(servicedir, "#{service_name}.service")
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0644'
           default_mode = '0644'
         when "launchd"
-          target_service_file = File.join(@component.platform.servicedir, "#{service_name}.plist")
+          target_service_file = File.join(servicedir, "#{service_name}.plist")
           target_mode = '0644'
           default_mode = '0644'
         when "smf"
           # modify version in smf manifest so service gets restarted after package upgrade
           @component.install << %{#{@component.platform.sed} -ri 's/(<service.*version=)(".*")/\\1"#{Time.now.to_i}"/' #{service_file}}
-          target_service_file = File.join(@component.platform.servicedir, service_type.to_s, "#{service_name}.xml")
+          target_service_file = File.join(servicedir, options[:service_type].to_s, "#{service_name}.xml")
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0644'
           default_mode = '0755'
         when "aix"
-          @component.service = OpenStruct.new(:name => service_name, :service_command => File.read(service_file).chomp)
+          @component.service << OpenStruct.new(:name => service_name, :service_command => File.read(service_file).chomp)
           # Return here because there is no file to install, just a string read in
           return
         when "windows"
-          @component.service = OpenStruct.new(\
+          @component.service << OpenStruct.new(\
             :bindir_id => "#{service_name.gsub(/[^A-Za-z0-9]/, '').upcase}BINDIR", \
             :service_file => service_file, \
             :component_group_id => "#{service_name.gsub(/[^A-Za-z0-9]/, '')}Component"\
@@ -206,13 +211,13 @@ class Vanagon
           # return here as we are just collecting the name of the service file to put into the harvest filter list.
           return
         else
-          fail "Don't know how to install the #{@component.platform.servicetype}. Please teach #install_service how to do this."
+          fail "Don't know how to install the #{init_system}. Please teach #install_service how to do this."
         end
 
         # Install the service and default files
-        if link_target
-          install_file(service_file, link_target, mode: target_mode)
-          link link_target, target_service_file
+        if options[:link_target]
+          install_file(service_file, options[:link_target], mode: target_mode)
+          link options[:link_target], target_service_file
         else
           install_file(service_file, target_service_file, mode: target_mode)
         end
@@ -223,7 +228,8 @@ class Vanagon
         end
 
         # Register the service for use in packaging
-        @component.service = OpenStruct.new(:name => service_name, :service_file => target_service_file, :type => service_type)
+        @component.service << OpenStruct.new(:name => service_name, :service_file => target_service_file,
+                                             :type => options[:service_type], :init_system => init_system)
       end
 
       # Copies a file from source to target during the install phase of the component

data/lib/vanagon/component/source.rb

@@ -37,6 +37,7 @@ class Vanagon
               sum: options[:sum],
               ref: options[:ref],
               workdir: options[:workdir],
+              dirname: options[:dirname],
               clone_options: options[:clone_options]
           end
 

data/lib/vanagon/component/source/git.rb

@@ -26,14 +26,38 @@ class Vanagon
           #        git command has failed. Useful in instances where a URL
           #        prompts for credentials despite not being a git remote
           # @return [Boolean] whether #url is a valid Git repo or not
+
+          # [RE-13837] This ought to be the way to do this. Unfortunately,
+          # there's a bug in Git.ls_remote that when ssh prints something like
+          #  Warning: Permanently added 'github.com,192.30.255.113' (RSA)
+          # Git.ls_remote attempts to parse it as actual git output and fails
+          # with: NoMethodError: undefined method `split' for nil:NilClass
+          #
+          # We'll work around that case by calling 'git ls-remote' directly ourselves.
+          #
+          # I'm leaving in the broken version here for a time when the ruby-git library
+          # is fixed.
+
+          #def valid_remote?(url, timeout = 0)
+          #  Timeout.timeout(timeout) do
+          #    !!::Git.ls_remote(url)
+          #  end
+          #rescue ::Git::GitExecuteError
+          #  false
+          #rescue Timeout::Error
+          #  false
+          #end
+
           def valid_remote?(url, timeout = 0)
             Timeout.timeout(timeout) do
-              !!::Git.ls_remote(url)
+              Vanagon::Utilities.local_command("git ls-remote #{url} > /dev/null 2>&1")
+              return false unless $?.exitstatus.zero?
+              return true
             end
-          rescue ::Git::GitExecuteError
-            false
           rescue Timeout::Error
-            false
+            return false
+          rescue RuntimeError
+            return false
           end
         end
 
@@ -55,6 +79,7 @@ class Vanagon
           # Ensure that #url returns a URI object
           @url = URI.parse(url.to_s)
           @ref = opts[:ref]
+          @dirname = opts[:dirname]
           @workdir = File.realpath(workdir)
           @clone_options = opts[:clone_options] ||= {}
 
@@ -89,7 +114,7 @@ class Vanagon
         #
         # @return [String] the directory where the repo was cloned
         def dirname
-          File.basename(url.path, ".git")
+          @dirname || File.basename(url.path, ".git")
         end
 
         # Use `git describe` to lazy-load a version for this component

data/lib/vanagon/engine/pooler.rb

@@ -15,7 +15,10 @@ class Vanagon
       def initialize(platform, target = nil, **opts)
         super
 
-        @available_poolers = ["https://vmpooler.delivery.puppetlabs.net", "https://nspooler-service-prod-1.delivery.puppetlabs.net"]
+        @available_poolers = %w[
+          https://vmpooler.delivery.puppetlabs.net
+          https://nspooler-prod.k8s.infracore.puppet.net
+        ]
         @token = load_token
         @required_attributes << "vmpooler_template"
       end

data/lib/vanagon/platform.rb

@@ -33,6 +33,9 @@ class Vanagon
     # Where does a given platform expect to find init scripts/service files?
     # e.g. /etc/init.d, /usr/lib/systemd/system
     attr_accessor :servicedir
+    # Array of OpenStructs containing the servicetype and the corresponding
+    # servicedir
+    attr_accessor :servicetypes
     # Where does a given platform's init system expect to find
     # something resembling 'defaults' files. Most likely to apply
     # to Linux systems that use SysV-ish, upstart, or systemd init systems.
@@ -245,6 +248,7 @@ class Vanagon
       # Our first attempt at defining metadata about a platform
       @cross_compiled ||= false
       @valid_operators ||= ['<', '>', '<=', '>=', '=']
+      @servicetypes = []
     end
 
     def shell # rubocop:disable Lint/DuplicateMethods
@@ -553,5 +557,33 @@ class Vanagon
     def validate_operator(operator_string)
       valid_operators.include?(operator_string)
     end
+
+    # Get all configured service types (added through plat.servicetype)
+    # @return array of service types, empty array if none have been configured
+    def get_service_types
+      if @servicetypes.any?
+        @servicetypes.flat_map(&:servicetype).compact
+      elsif @servicetype
+        [@servicetype]
+      else
+        []
+      end
+    end
+
+    # Get configured service dir (added through plat.servicedir, or plat.servicetype 'foo', servicedir: 'bar')
+    # @param servicetype the service type you want the service dir for (optional)
+    # @raises VanagonError if more than one service dir is found
+    def get_service_dir(servicetype = '')
+      if @servicetypes.empty?
+        return @servicedir
+      end
+      servicedir = @servicetypes.select { |s| s.servicetype.include?(servicetype) }.flat_map(&:servicedir).compact
+
+      if servicedir.size > 1
+        raise Vanagon::Error, "You can only have one service dir for each service type. Found '#{servicedir.join(',')}' for service type #{servicetype}"
+      end
+
+      servicedir.first
+    end
   end
 end

data/lib/vanagon/platform/defaults/debian-10-amd64.rb

@@ -0,0 +1,11 @@
+platform "debian-10-amd64" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "buster"
+
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot cmake)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends "
+  plat.vmpooler_template "debian-10-x86_64"
+end

data/lib/vanagon/platform/defaults/debian-11-amd64.rb

@@ -0,0 +1,11 @@
+platform "debian-11-amd64" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "bullseye"
+
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot cmake)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends "
+  plat.vmpooler_template "debian-11-x86_64"
+end

data/lib/vanagon/platform/defaults/debian-8-amd64.rb

@@ -0,0 +1,12 @@
+platform "debian-8-amd64" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "jessie"
+
+  plat.add_build_repository "http://pl-build-tools.delivery.puppetlabs.net/debian/pl-build-tools-release-#{plat.get_codename}.deb"
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends "
+  plat.vmpooler_template "debian-8-x86_64"
+end

data/lib/vanagon/platform/defaults/debian-8-i386.rb

@@ -0,0 +1,12 @@
+platform "debian-8-i386" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "jessie"
+
+  plat.add_build_repository "http://pl-build-tools.delivery.puppetlabs.net/debian/pl-build-tools-release-#{plat.get_codename}.deb"
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends"
+  plat.vmpooler_template "debian-8-i386"
+end

data/lib/vanagon/platform/defaults/debian-9-amd64.rb

@@ -0,0 +1,12 @@
+platform "debian-9-amd64" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "stretch"
+
+  plat.add_build_repository "http://pl-build-tools.delivery.puppetlabs.net/debian/pl-build-tools-release-#{plat.get_codename}.deb"
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends "
+  plat.vmpooler_template "debian-9-x86_64"
+end

data/lib/vanagon/platform/defaults/debian-9-i386.rb

@@ -0,0 +1,12 @@
+platform "debian-9-i386" do |plat|
+  plat.servicedir "/lib/systemd/system"
+  plat.defaultdir "/etc/default"
+  plat.servicetype "systemd"
+  plat.codename "stretch"
+
+  plat.add_build_repository "http://pl-build-tools.delivery.puppetlabs.net/debian/pl-build-tools-release-#{plat.get_codename}.deb"
+  packages = %w(build-essential devscripts make quilt pkg-config debhelper rsync fakeroot)
+  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+  plat.install_build_dependencies_with "DEBIAN_FRONTEND=noninteractive; apt-get install -qy --no-install-recommends "
+  plat.vmpooler_template "debian-9-i386"
+end