vanagon 0.17.0 → 0.20.0

data/lib/vanagon/cli/inspect.rb

@@ -1,5 +1,6 @@
 require 'docopt'
 require 'json'
+require 'vanagon/logger'
 
 class Vanagon
   class CLI
@@ -11,18 +12,26 @@ class Vanagon
         Options:
           -h, --help                       Display help
           -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
-          -e, --engine ENGINE              Custom engine to use [base, local, docker, pooler] [default: pooler]
+          -e, --engine ENGINE              Custom engine to use [default: always_be_scheduling]
 
           -p, --preserve [RULE]            Rule for VM preservation: never, on-failure, always
                                              [Default: on-failure]
           -w, --workdir DIRECTORY          Working directory on the local host
           -v, --verbose                    Only here for backwards compatibility. Does nothing.
+
+        Engines:
+          always_be_scheduling: default engine using Puppet's ABS infrastructure
+          docker: a docker container on the local host
+          ec2: an Amazon EC2 instance
+          hardware: a dedicated hardware device
+          local: the local machine, cannot be used with a target
+          pooler: [deprecated] Puppet's vmpooler
       DOCOPT
 
       def parse(argv)
         Docopt.docopt(DOCUMENTATION, { argv: argv })
       rescue Docopt::Exit => e
-        puts e.message
+        VanagonLogger.error e.message
         exit 1
       end
 
@@ -33,7 +42,7 @@ class Vanagon
         platforms.each do |platform|
           driver = Vanagon::Driver.new(platform, project, options)
           components = driver.project.components.map(&:to_hash)
-          $stdout.puts JSON.pretty_generate(components)
+          VanagonLogger.warn JSON.pretty_generate(components)
         end
       end
 

data/lib/vanagon/cli/list.rb

@@ -0,0 +1,74 @@
+require 'docopt'
+require 'vanagon/logger'
+
+class Vanagon
+  class CLI
+    class List < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+          list [options]
+
+        Options:
+          -h, --help                       Display help
+          -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
+          -l, --platforms                  Display a list of platforms
+          -r, --projects                   Display a list of projects
+          -s, --use-spaces                 Displays the list as space separated
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        VanagonLogger.error e.message
+        exit 1
+      end
+
+      def output(list, use_spaces)
+        return list.join(' ') if use_spaces
+        return list
+      end
+
+      def run(options) # rubocop:disable Metrics/AbcSize
+        if Dir.exist?(File.join(options[:configdir], 'platforms')) == false ||
+           Dir.exist?(File.join(options[:configdir], 'projects')) == false
+
+          VanagonLogger.error "Path to #{File.join(options[:configdir], 'platforms')} or #{File.join(options[:configdir], 'projects')} not found."
+          exit 1
+        end
+
+        platform_list = Dir.children(File.join(options[:configdir], 'platforms')).map do |platform|
+          File.basename(platform, File.extname(platform))
+        end
+
+        project_list = Dir.children(File.join(options[:configdir], 'projects')).map do |project|
+          File.basename(project, File.extname(project))
+        end
+
+        if options[:projects] == options[:platforms]
+          puts "- Projects", output(project_list, options[:use_spaces]), "\n", "- Platforms", output(platform_list, options[:use_spaces])
+          return
+        end
+
+        if options[:projects]
+          puts "- Projects", output(project_list, options[:use_spaces])
+          return
+        end
+
+        if options[:platforms]
+          puts "- Platforms", output(platform_list, options[:use_spaces])
+          return
+        end
+      end
+
+      def options_translate(docopt_options)
+        translations = {
+          '--configdir' => :configdir,
+          '--platforms' => :platforms,
+          '--projects' => :projects,
+          '--use-spaces' => :use_spaces,
+        }
+        return docopt_options.map { |k, v| [translations[k], v] }.to_h
+      end
+    end
+  end
+end

data/lib/vanagon/cli/render.rb

@@ -1,5 +1,6 @@
 require 'docopt'
 require 'json'
+require 'vanagon/logger'
 
 class Vanagon
   class CLI
@@ -11,15 +12,23 @@ class Vanagon
         Options:
           -h, --help                       Display help
           -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
-          -e, --engine ENGINE              Custom engine to use [base, local, docker, pooler] [default: pooler]
+          -e, --engine ENGINE              Custom engine to use [default: always_be_scheduling]
           -w, --workdir DIRECTORY          Working directory on the local host
           -v, --verbose                    Only here for backwards compatibility. Does nothing.
+
+        Engines:
+          always_be_scheduling: default engine using Puppet's ABS infrastructure
+          docker: a docker container on the local host
+          ec2: an Amazon EC2 instance
+          hardware: a dedicated hardware device
+          local: the local machine, cannot be used with a target
+          pooler: [deprecated] Puppet's vmpooler
       DOCOPT
 
       def parse(argv)
         Docopt.docopt(DOCUMENTATION, { argv: argv })
       rescue Docopt::Exit => e
-        puts e.message
+        VanagonLogger.error e.message
         exit 1
       end
 

data/lib/vanagon/cli/ship.rb

@@ -1,4 +1,5 @@
 require 'docopt'
+require 'vanagon/logger'
 
 class Vanagon
   class CLI
@@ -14,7 +15,7 @@ class Vanagon
       def parse(argv)
         Docopt.docopt(DOCUMENTATION, { argv: argv })
       rescue Docopt::Exit => e
-        puts e.message
+        VanagonLogger.error e.message
         exit 1
       end
 
@@ -32,7 +33,7 @@ class Vanagon
         DOC
 
         if Dir['output/**/*'].select { |entry| File.file?(entry) }.empty?
-          warn 'vanagon: Error: No packages to ship in the "output" directory. Maybe build some first?'
+          VanagonLogger.error 'vanagon: Error: No packages to ship in the "output" directory. Maybe build some first?'
           exit 1
         end
 
@@ -42,9 +43,9 @@ class Vanagon
         begin
           Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship_to_artifactory', 'output')
         rescue LoadError
-          warn artifactory_warning
+          VanagonLogger.error artifactory_warning
         rescue StandardError
-          warn artifactory_warning
+          VanagonLogger.error artifactory_warning
         end
       end
     end

data/lib/vanagon/cli/sign.rb

@@ -1,4 +1,5 @@
 require 'docopt'
+require 'vanagon/logger'
 
 class Vanagon
   class CLI
@@ -14,14 +15,14 @@ class Vanagon
       def parse(argv)
         Docopt.docopt(DOCUMENTATION, { argv: argv })
       rescue Docopt::Exit => e
-        puts e.message
+        VanagonLogger.error e.message
         exit 1
       end
 
       def run(_)
         ENV['PROJECT_ROOT'] = Dir.pwd
         if Dir['output/**/*'].select { |entry| File.file?(entry) }.empty?
-          warn 'sign: Error: No packages to sign in the "output" directory. Maybe build some first?'
+          VanagonLogger.error 'sign: Error: No packages to sign in the "output" directory. Maybe build some first?'
           exit 1
         end
 

data/lib/vanagon/component.rb

@@ -3,6 +3,8 @@ require 'vanagon/component/rules'
 require 'vanagon/component/source'
 require 'vanagon/component/source/rewrite'
 
+require 'vanagon/logger'
+
 class Vanagon
   class Component
     include Vanagon::Utilities
@@ -138,9 +140,9 @@ class Vanagon
       dsl.instance_eval(File.read(compfile), compfile, 1)
       dsl._component
     rescue StandardError => e
-      warn "Error loading project '#{name}' using '#{compfile}':"
-      warn e
-      warn e.backtrace.join("\n")
+      VanagonLogger.error "Error loading project '#{name}' using '#{compfile}':"
+      VanagonLogger.error e
+      VanagonLogger.error e.backtrace.join("\n")
       raise e
     end
 
@@ -178,6 +180,7 @@ class Vanagon
       @preremove_actions = []
       @postremove_actions = []
       @install_only = false
+      @service = []
     end
 
     # Adds the given file to the list of files and returns @files.
@@ -254,18 +257,18 @@ class Vanagon
     def fetch_mirrors(options)
       mirrors.to_a.shuffle.each do |mirror|
         begin
-          warn %(Attempting to fetch from mirror URL "#{mirror}")
+          VanagonLogger.info %(Attempting to fetch from mirror URL "#{mirror}")
           @source = Vanagon::Component::Source.source(mirror, options)
           return true if source.fetch
         rescue SocketError
           # SocketError means that there was no DNS/name resolution
           # for whatever remote protocol the mirror tried to use.
-          warn %(Unable to resolve mirror URL "#{mirror}")
+          VanagonLogger.error %(Unable to resolve mirror URL "#{mirror}")
         rescue RuntimeError
           # Source retrieval does not consistently return a meaningful
           # namespaced error message, which means we're brute-force rescuing
           # RuntimeError. Not a good look, and we should fix this.
-          warn %(Unable to retrieve mirror URL "#{mirror}")
+          VanagonLogger.error %(Unable to retrieve mirror URL "#{mirror}")
         end
       end
       false
@@ -277,7 +280,7 @@ class Vanagon
     # @return [Boolean] return True if the source can be retrieved,
     #   or False otherwise
     def fetch_url(options)
-      warn %(Attempting to fetch from canonical URL "#{url}")
+      VanagonLogger.info %(Attempting to fetch from canonical URL "#{url}")
       @source = Vanagon::Component::Source.source(url, options)
       # Explicitly coerce the return value of #source.fetch,
       # because each subclass of Vanagon::Component::Source returns
@@ -303,7 +306,7 @@ class Vanagon
     def get_source(workdir) # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
       opts = options.merge({ workdir: workdir })
       if url || !mirrors.empty?
-        if ENV['VANAGON_USE_MIRRORS'] == 'n'
+        if ENV['VANAGON_USE_MIRRORS'] == 'n' or ENV['VANAGON_USE_MIRRORS'] == 'false'
           fetch_url(opts)
         else
           fetch_mirrors(opts) || fetch_url(opts)
@@ -319,7 +322,7 @@ class Vanagon
           @version ||= source.version
         end
       else
-        warn "No source given for component '#{@name}'"
+        VanagonLogger.info "No source given for component '#{@name}'"
 
         # If there is no source, we don't want to try to change directories, so we just change to the current directory.
         @dirname = './'
@@ -407,7 +410,7 @@ class Vanagon
     # @return [String] environment suitable for inclusion in a Makefile
     # @deprecated
     def get_environment
-      warn <<-WARNING.undent
+      VanagonLogger.info <<-WARNING.undent
         #get_environment is deprecated; environment variables have been moved
         into the Makefile, and should not be used within a Makefile's recipe.
         The #get_environment method will be removed by Vanagon 1.0.0.

data/lib/vanagon/component/dsl.rb

@@ -1,4 +1,5 @@
 require 'vanagon/component'
+require 'vanagon/logger'
 require 'vanagon/patch'
 require 'ostruct'
 require 'json'
@@ -131,8 +132,8 @@ class Vanagon
       # component
       #
       # @param requirement [String] a package that is required at runtime for this component
-      def requires(requirement)
-        @component.requires << requirement
+      def requires(requirement, version = nil)
+        @component.requires << OpenStruct.new(:requirement => requirement, :version => version)
       end
 
       # Indicates that this component replaces a system level package. Replaces can be collected and used by the project and package.
@@ -167,37 +168,42 @@ class Vanagon
       # @param service_file [String] path to the service file relative to the source
       # @param default_file [String] path to the default file relative to the source
       # @param service_name [String] name of the service
-      # @param service_type [String] type of the service (network, application, system, etc)
-      # @param link_target [String] executable service file should be linked to
-      def install_service(service_file, default_file = nil, service_name = @component.name, service_type: nil, link_target: nil) # rubocop:disable Metrics/AbcSize
-        case @component.platform.servicetype
+      # @param options optional extra parameters
+      #        service_type [String] type of the service (network, application, system, etc)
+      #        init_system [String] the init system on which to install service (sysv, systemd)
+      #        link_target [String] executable service file should be linked to
+      def install_service(service_file, default_file = nil, service_name = @component.name, **options) # rubocop:disable Metrics/AbcSize
+        init_system = options[:init_system] || @component.platform.servicetype
+        servicedir = @component.platform.get_service_dir(init_system)
+
+        case init_system
         when "sysv"
-          target_service_file = File.join(@component.platform.servicedir, service_name)
+          target_service_file = File.join(servicedir, service_name)
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0755'
           default_mode = '0644'
         when "systemd"
-          target_service_file = File.join(@component.platform.servicedir, "#{service_name}.service")
+          target_service_file = File.join(servicedir, "#{service_name}.service")
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0644'
           default_mode = '0644'
         when "launchd"
-          target_service_file = File.join(@component.platform.servicedir, "#{service_name}.plist")
+          target_service_file = File.join(servicedir, "#{service_name}.plist")
           target_mode = '0644'
           default_mode = '0644'
         when "smf"
           # modify version in smf manifest so service gets restarted after package upgrade
           @component.install << %{#{@component.platform.sed} -ri 's/(<service.*version=)(".*")/\\1"#{Time.now.to_i}"/' #{service_file}}
-          target_service_file = File.join(@component.platform.servicedir, service_type.to_s, "#{service_name}.xml")
+          target_service_file = File.join(servicedir, options[:service_type].to_s, "#{service_name}.xml")
           target_default_file = File.join(@component.platform.defaultdir, service_name)
           target_mode = '0644'
           default_mode = '0755'
         when "aix"
-          @component.service = OpenStruct.new(:name => service_name, :service_command => File.read(service_file).chomp)
+          @component.service << OpenStruct.new(:name => service_name, :service_command => File.read(service_file).chomp)
           # Return here because there is no file to install, just a string read in
           return
         when "windows"
-          @component.service = OpenStruct.new(\
+          @component.service << OpenStruct.new(\
             :bindir_id => "#{service_name.gsub(/[^A-Za-z0-9]/, '').upcase}BINDIR", \
             :service_file => service_file, \
             :component_group_id => "#{service_name.gsub(/[^A-Za-z0-9]/, '')}Component"\
@@ -205,13 +211,13 @@ class Vanagon
           # return here as we are just collecting the name of the service file to put into the harvest filter list.
           return
         else
-          fail "Don't know how to install the #{@component.platform.servicetype}. Please teach #install_service how to do this."
+          fail "Don't know how to install the #{init_system}. Please teach #install_service how to do this."
         end
 
         # Install the service and default files
-        if link_target
-          install_file(service_file, link_target, mode: target_mode)
-          link link_target, target_service_file
+        if options[:link_target]
+          install_file(service_file, options[:link_target], mode: target_mode)
+          link options[:link_target], target_service_file
         else
           install_file(service_file, target_service_file, mode: target_mode)
         end
@@ -222,7 +228,8 @@ class Vanagon
         end
 
         # Register the service for use in packaging
-        @component.service = OpenStruct.new(:name => service_name, :service_file => target_service_file, :type => service_type)
+        @component.service << OpenStruct.new(:name => service_name, :service_file => target_service_file,
+                                             :type => options[:service_type], :init_system => init_system)
       end
 
       # Copies a file from source to target during the install phase of the component
@@ -237,7 +244,7 @@ class Vanagon
 
         if @component.platform.is_windows?
           unless mode.nil? && owner.nil? && group.nil?
-            warn "You're trying to set the mode, owner, or group for windows. I don't know how to do that, ignoring!"
+            VanagonLogger.info "You're trying to set the mode, owner, or group for windows. I don't know how to do that, ignoring!"
           end
         else
           mode ||= '0644'
@@ -407,7 +414,7 @@ class Vanagon
         install_flags = ['-d']
         if @component.platform.is_windows?
           unless mode.nil? && owner.nil? && group.nil?
-            warn "You're trying to set the mode, owner, or group for windows. I don't know how to do that, ignoring!"
+            VanagonLogger.info "You're trying to set the mode, owner, or group for windows. I don't know how to do that, ignoring!"
           end
         else
           install_flags << "-m '#{mode}'" unless mode.nil?
@@ -422,7 +429,7 @@ class Vanagon
       # @param env [Hash] mapping of keys to values to add to the environment for the component
       def environment(*env)
         if env.size == 1 && env.first.is_a?(Hash)
-          warn <<-WARNING.undent
+          VanagonLogger.info <<-WARNING.undent
             the component DSL method signature #environment({Key => Value}) is deprecated
             and will be removed by Vanagon 1.0.0.
 

data/lib/vanagon/component/source.rb

@@ -1,4 +1,5 @@
 require 'fustigit'
+require 'vanagon/logger'
 require 'vanagon/component/source/http'
 require 'vanagon/component/source/git'
 require 'vanagon/component/source/local'
@@ -66,7 +67,7 @@ class Vanagon
           timeout = 5
           if Vanagon::Component::Source::Git.valid_remote?(uri, timeout)
             if uri =~ /^http/
-              warn "Passing git URLs as http(s) addresses is deprecated! Please prefix your source URL with `git:`"
+              VanagonLogger.info "Passing git URLs as http(s) addresses is deprecated! Please prefix your source URL with `git:`"
             end
             return :git
           end

data/lib/vanagon/component/source/git.rb

@@ -1,5 +1,6 @@
 require 'vanagon/utilities'
 require 'vanagon/errors'
+require 'vanagon/logger'
 # This stupid library requires a capital 'E' in its name
 # but it provides a wealth of useful constants
 require 'English'
@@ -25,14 +26,38 @@ class Vanagon
           #        git command has failed. Useful in instances where a URL
           #        prompts for credentials despite not being a git remote
           # @return [Boolean] whether #url is a valid Git repo or not
+
+          # [RE-13837] This ought to be the way to do this. Unfortunately,
+          # there's a bug in Git.ls_remote that when ssh prints something like
+          #  Warning: Permanently added 'github.com,192.30.255.113' (RSA)
+          # Git.ls_remote attempts to parse it as actual git output and fails
+          # with: NoMethodError: undefined method `split' for nil:NilClass
+          #
+          # We'll work around that case by calling 'git ls-remote' directly ourselves.
+          #
+          # I'm leaving in the broken version here for a time when the ruby-git library
+          # is fixed.
+
+          #def valid_remote?(url, timeout = 0)
+          #  Timeout.timeout(timeout) do
+          #    !!::Git.ls_remote(url)
+          #  end
+          #rescue ::Git::GitExecuteError
+          #  false
+          #rescue Timeout::Error
+          #  false
+          #end
+
           def valid_remote?(url, timeout = 0)
             Timeout.timeout(timeout) do
-              !!::Git.ls_remote(url)
+              Vanagon::Utilities.local_command("git ls-remote #{url} > /dev/null 2>&1")
+              return false unless $?.exitstatus.zero?
+              return true
             end
-          rescue ::Git::GitExecuteError
-            false
           rescue Timeout::Error
-            false
+            return false
+          rescue RuntimeError
+            return false
           end
         end
 
@@ -81,7 +106,7 @@ class Vanagon
         # There is no md5 to manually verify here, so this is a noop.
         def verify
           # nothing to do here, so just tell users that and return
-          warn "Nothing to verify for '#{dirname}' (using Git reference '#{ref}')"
+          VanagonLogger.info "Nothing to verify for '#{dirname}' (using Git reference '#{ref}')"
         end
 
         # The dirname to reference when building from the repo
@@ -131,8 +156,8 @@ class Vanagon
         # Clone a remote repo, make noise about it, and fail entirely
         # if we're unable to retrieve the remote repo
         def clone!
-          warn "Cloning Git repo '#{url}'"
-          warn "Successfully cloned '#{dirname}'" if clone
+          VanagonLogger.info "Cloning Git repo '#{url}'"
+          VanagonLogger.info "Successfully cloned '#{dirname}'" if clone
         rescue ::Git::GitExecuteError
           raise Vanagon::InvalidRepo, "Unable to clone from '#{url}'"
         end
@@ -141,7 +166,7 @@ class Vanagon
         # Checkout desired ref/sha, make noise about it, and fail
         # entirely if we're unable to checkout that given ref/sha
         def checkout!
-          warn "Checking out '#{ref}' from Git repo '#{dirname}'"
+          VanagonLogger.info "Checking out '#{ref}' from Git repo '#{dirname}'"
           clone.checkout(ref)
         rescue ::Git::GitExecuteError
           raise Vanagon::CheckoutFailed, "unable to checkout #{ref} from '#{url}'"
@@ -151,7 +176,7 @@ class Vanagon
         # Attempt to update submodules, and do not panic
         # if there are no submodules to initialize
         def update_submodules
-          warn "Attempting to update submodules for repo '#{dirname}'"
+          VanagonLogger.info "Attempting to update submodules for repo '#{dirname}'"
           clone.update_submodules(init: true)
         end
         private :update_submodules
@@ -163,7 +188,7 @@ class Vanagon
         def describe
           clone.describe(ref, tags: true)
         rescue ::Git::GitExecuteError
-          warn "Directory '#{dirname}' cannot be versioned by Git. Maybe it hasn't been tagged yet?"
+          VanagonLogger.info "Directory '#{dirname}' cannot be versioned by Git. Maybe it hasn't been tagged yet?"
         end
         private :describe
       end