vanagon 0.15.35 → 0.16.1

data/lib/vanagon/cli/build_requirements.rb

@@ -0,0 +1,60 @@
+require 'docopt'
+require 'json'
+
+class Vanagon
+  class CLI
+    class BuildRequirements < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+        build_requirements [options] <project-name> <platform>
+
+        Options:
+          -h, --help                       Display help
+          -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
+          -e, --engine ENGINE              Custom engine to use [base, local, docker, pooler] [default: pooler]
+          -w, --workdir DIRECTORY          Working directory on the local host
+          -v, --verbose                    Only here for backwards compatibility. Does nothing.
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+
+      def run(options) # rubocop:disable Metrics/AbcSize
+        platform = options[:platform]
+        project = options[:project_name]
+        driver = Vanagon::Driver.new(platform, project)
+
+        components = driver.project.components
+        component_names = components.map(&:name)
+        build_requirements = []
+        components.each do |component|
+          build_requirements << component.build_requires.reject do |requirement|
+            # only include external requirements: i.e. those that do not match
+            # other components in the project
+            component_names.include?(requirement)
+          end
+        end
+
+        $stdout.puts
+        $stdout.puts "**** External packages required to build #{project} on #{platform}: ***"
+        $stdout.puts JSON.pretty_generate(build_requirements.flatten.uniq.sort)
+      end
+
+      def options_translate(docopt_options)
+        translations = {
+          '--verbose' => :verbose,
+          '--workdir' => :workdir,
+          '--configdir' => :configdir,
+          '--engine' => :engine,
+          '<project-name>' => :project_name,
+          '<platform>' => :platform,
+        }
+        return docopt_options.map { |k, v| [translations[k], v] }.to_h
+      end
+    end
+  end
+end

data/lib/vanagon/cli/inspect.rb

@@ -0,0 +1,65 @@
+require 'docopt'
+require 'json'
+
+class Vanagon
+  class CLI
+    class Inspect < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+        inspect [options] <project-name> <platforms>
+
+        Options:
+          -h, --help                       Display help
+          -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
+          -e, --engine ENGINE              Custom engine to use [base, local, docker, pooler] [default: pooler]
+
+          -p, --preserve [RULE]            Rule for VM preservation: never, on-failure, always
+                                             [Default: on-failure]
+          -w, --workdir DIRECTORY          Working directory on the local host
+          -v, --verbose                    Only here for backwards compatibility. Does nothing.
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+
+      def run(options)
+        platforms = options[:platforms].split(',')
+        project = options[:project_name]
+
+        platforms.each do |platform|
+          driver = Vanagon::Driver.new(platform, project, options)
+          components = driver.project.components.map(&:to_hash)
+          $stdout.puts JSON.pretty_generate(components)
+        end
+      end
+
+      def options_translate(docopt_options)
+        translations = {
+          '--verbose' => :verbose,
+          '--workdir' => :workdir,
+          '--configdir' => :configdir,
+          '--engine' => :engine,
+          '--preserve' => :preserve,
+          '<project-name>' => :project_name,
+          '<platforms>' => :platforms
+        }
+        return docopt_options.map { |k, v| [translations[k], v] }.to_h
+      end
+
+      def options_validate(options)
+        # Handle --preserve option checking
+        valid_preserves = %w[always never on-failure]
+        unless valid_preserves.include? options[:preserve]
+          raise InvalidArgument, "--preserve option can only be one of: " +
+                                 valid_preserves.join(', ')
+        end
+        options[:preserve] = options[:preserve].to_sym
+        return options
+      end
+    end
+  end
+end

data/lib/vanagon/cli/render.rb

@@ -0,0 +1,51 @@
+require 'docopt'
+require 'json'
+
+class Vanagon
+  class CLI
+    class Render < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+        render [options] <project-name> <platforms>
+
+        Options:
+          -h, --help                       Display help
+          -c, --configdir DIRECTORY        Configuration directory [default: #{Dir.pwd}/configs]
+          -e, --engine ENGINE              Custom engine to use [base, local, docker, pooler] [default: pooler]
+          -w, --workdir DIRECTORY          Working directory on the local host
+          -v, --verbose                    Only here for backwards compatibility. Does nothing.
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+
+      def run(options)
+        platforms = options[:platforms].split(',')
+        project = options[:project_name]
+        target_list = []
+
+        platforms.zip(target_list).each do |pair|
+          platform, target = pair
+          artifact = Vanagon::Driver.new(platform, project, options.merge({ :target => target }))
+          artifact.render
+        end
+      end
+
+      def options_translate(docopt_options)
+        translations = {
+          '--verbose' => :verbose,
+          '--workdir' => :workdir,
+          '--configdir' => :configdir,
+          '--engine' => :engine,
+          '<project-name>' => :project_name,
+          '<platforms>' => :platforms,
+        }
+        return docopt_options.map { |k, v| [translations[k], v] }.to_h
+      end
+    end
+  end
+end

data/lib/vanagon/cli/ship.rb

@@ -0,0 +1,52 @@
+require 'docopt'
+
+class Vanagon
+  class CLI
+    class Ship < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+          ship [--help]
+
+        Options:
+          -h, --help                       Display help
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+
+      def run(_)
+        ENV['PROJECT_ROOT'] = Dir.pwd
+
+        artifactory_warning = <<-DOC
+          Unable to ship packages to artifactory. Please make sure you are pointing to a
+          recent version of packaging in your Gemfile. Please also make sure you include
+          the artifactory gem in your Gemfile.
+
+          Examples:
+            gem 'packaging', :github => 'puppetlabs/packaging', branch: '1.0.x'
+            gem 'artifactory'
+        DOC
+
+        if Dir['output/**/*'].select { |entry| File.file?(entry) }.empty?
+          warn 'vanagon: Error: No packages to ship in the "output" directory. Maybe build some first?'
+          exit 1
+        end
+
+        require 'packaging'
+        Pkg::Util::RakeUtils.load_packaging_tasks
+        Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship', 'artifacts', 'output')
+        begin
+          Pkg::Util::RakeUtils.invoke_task('pl:jenkins:ship_to_artifactory', 'output')
+        rescue LoadError
+          warn artifactory_warning
+        rescue StandardError
+          warn artifactory_warning
+        end
+      end
+    end
+  end
+end

data/lib/vanagon/cli/sign.rb

@@ -0,0 +1,34 @@
+require 'docopt'
+
+class Vanagon
+  class CLI
+    class Sign < Vanagon::CLI
+      DOCUMENTATION = <<~DOCOPT.freeze
+        Usage:
+          sign [--help]
+
+        Options:
+          -h, --help                       Display help
+      DOCOPT
+
+      def parse(argv)
+        Docopt.docopt(DOCUMENTATION, { argv: argv })
+      rescue Docopt::Exit => e
+        puts e.message
+        exit 1
+      end
+
+      def run(_)
+        ENV['PROJECT_ROOT'] = Dir.pwd
+        if Dir['output/**/*'].select { |entry| File.file?(entry) }.empty?
+          warn 'sign: Error: No packages to sign in the "output" directory. Maybe build some first?'
+          exit 1
+        end
+
+        require 'packaging'
+        Pkg::Util::RakeUtils.load_packaging_tasks
+        Pkg::Util::RakeUtils.invoke_task('pl:jenkins:sign_all', 'output')
+      end
+    end
+  end
+end

data/lib/vanagon/driver.rb

@@ -29,7 +29,6 @@ class Vanagon
       components = options[:components] || []
       only_build = options[:only_build]
       target = options[:target]
-      engine = options[:engine] || 'pooler'
 
       @platform = Vanagon::Platform.load_platform(platform, File.join(@@configdir, "platforms"))
       @project = Vanagon::Project.load_project(project, File.join(@@configdir, "projects"), @platform, components)
@@ -40,9 +39,14 @@ class Vanagon
 
       @remote_workdir = options[:"remote-workdir"]
 
-      load_engine(engine, @platform, target)
-    rescue LoadError => e
-      raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine}'")
+      if options[:engine]
+        # Use the explicitly configured engine.
+        load_engine_object(options[:engine], @platform, target)
+      else
+        # Use 'pooler' as a default, but also apply selection logic that may
+        # choose something different based on platform configuration.
+        load_engine('pooler', @platform, target)
+      end
     end
 
     def filter_out_components(only_build)
@@ -73,8 +77,8 @@ class Vanagon
     def load_engine_object(engine_type, platform, target)
       require "vanagon/engine/#{engine_type}"
       @engine = Object::const_get("Vanagon::Engine::#{camelize(engine_type)}").new(platform, target, remote_workdir: remote_workdir)
-    rescue StandardError
-      fail "No such engine '#{camelize(engine_type)}'"
+    rescue StandardError, ScriptError => e
+      raise Vanagon::Error.wrap(e, "Could not load the desired engine '#{engine_type}'")
     end
 
     def camelize(string)
@@ -140,7 +144,7 @@ class Vanagon
       @project.make_bill_of_materials(workdir)
       # Don't generate packaging artifacts if no_packaging is set
       @project.generate_packaging_artifacts(workdir) unless @project.no_packaging
-      @project.save_manifest_json(@platform)
+      @project.save_manifest_json(@platform, workdir)
       @engine.ship_workdir(workdir)
       @engine.dispatch("(cd #{@engine.remote_workdir}; #{@platform.make} #{make_target})")
       @engine.retrieve_built_artifact(@project.artifacts_to_fetch, @project.no_packaging)

data/lib/vanagon/engine/docker.rb

@@ -10,6 +10,7 @@ class Vanagon
 
         @docker_cmd = Vanagon::Utilities.find_program_on_path('docker')
         @required_attributes << "docker_image"
+        @required_attributes.delete('ssh_port') if @platform.use_docker_exec
       end
 
       # Get the engine name
@@ -33,25 +34,16 @@ class Vanagon
       # This method is used to obtain a vm to build upon using
       # a docker container.
       # @raise [Vanagon::Error] if a target cannot be obtained
-      def select_target # rubocop:disable Metrics/AbcSize
+      def select_target
+        ssh_args = @platform.use_docker_exec ? '' : "-p #{@platform.ssh_port}:22"
         extra_args = @platform.docker_run_args.nil? ? [] : @platform.docker_run_args
 
-        Vanagon::Utilities.ex("#{@docker_cmd} run -d --name #{build_host_name}-builder -p #{@platform.ssh_port}:22 #{extra_args.join(' ')} #{@platform.docker_image}")
+        Vanagon::Utilities.ex("#{@docker_cmd} run -d --name #{build_host_name}-builder #{ssh_args} #{extra_args.join(' ')} #{@platform.docker_image}")
         @target = 'localhost'
 
-        # Wait for ssh to come up in the container. Retry 5 times with a 1
-        # second sleep between errors to account for network resets while SSHD
-        # is starting. Allow a maximum of 5 seconds for SSHD to start.
-        Vanagon::Utilities.retry_with_timeout(5, 5) do
-          begin
-            Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", 'exit', @platform.ssh_port)
-          rescue StandardError => e
-            sleep(1) # Give SSHD some time to start.
-            raise e
-          end
-        end
+        wait_for_ssh unless @platform.use_docker_exec
       rescue StandardError => e
-        raise Vanagon::Error.wrap(e, "Something went wrong getting a target vm to build on using docker. Ssh was not up in the container after 5 seconds.")
+        raise Vanagon::Error.wrap(e, "Something went wrong getting a target vm to build on using Docker.")
       end
 
       # This method is used to tell the vmpooler to delete the instance of the
@@ -62,6 +54,101 @@ class Vanagon
       rescue Vanagon::Error => e
         warn "There was a problem tearing down the docker container #{build_host_name}-builder (#{e.message})."
       end
+
+      def dispatch(command, return_output = false)
+        if @platform.use_docker_exec
+          docker_exec(command, return_output)
+        else
+          super
+        end
+      end
+
+      def ship_workdir(workdir)
+        if @platform.use_docker_exec
+          docker_cp_globs_to("#{workdir}/*", @remote_workdir)
+        else
+          super
+        end
+      end
+
+      def retrieve_built_artifact(artifacts_to_fetch, no_packaging)
+        if @platform.use_docker_exec
+          output_path = 'output/'
+          FileUtils.mkdir_p(output_path)
+          unless no_packaging
+            artifacts_to_fetch << "#{@remote_workdir}/output/*"
+          end
+
+          docker_cp_globs_from(artifacts_to_fetch, 'output/')
+        else
+          super
+        end
+      end
+
+      # Execute a command on a container via docker exec
+      def docker_exec(command, return_output = false)
+        command = command.gsub("'", "'\\\\''")
+        Vanagon::Utilities.local_command("#{@docker_cmd} exec #{build_host_name}-builder /bin/sh -c '#{command}'",
+                                         return_command_output: return_output)
+      end
+
+      # Copy files between a container and the host
+      def docker_cp(source, target)
+        Vanagon::Utilities.ex("#{@docker_cmd} cp '#{source}' '#{target}'")
+      end
+
+      # Copy files matching a glob pattern from the host to the container
+      def docker_cp_globs_to(globs, container_path)
+        Array(globs).each do |glob|
+          Dir.glob(glob).each do |path|
+            docker_cp(path, "#{build_host_name}-builder:#{container_path}")
+          end
+        end
+      end
+
+      # Copy files matching a glob pattern from the container to the host
+      #
+      # @note Globs are expanded by running `/bin/sh` in the container, which
+      #   may not support the same variety of expressions as Ruby's `Dir.glob`.
+      #   For example, `**` may not work.
+      def docker_cp_globs_from(globs, host_path)
+        Array(globs).each do |glob|
+          # Match the behavior of `rsync -r` when both paths are directories
+          # by copying the contents of the directory instead of the directory.
+          glob += '*' if glob.end_with?('/') && host_path.end_with?('/')
+
+          # TODO: This doesn't handle "interesting" paths. E.g. paths with
+          #   spaces or other special non-glob characters. This could be
+          #   fixed with a variant of `Shellwords.shellescape` that allows
+          #   glob characters to pass through.
+          paths = docker_exec(%(for file in #{glob};do [ -e "$file" ] && printf '%s\\0' "${file}";done), true).split("\0")
+
+          paths.each do |path|
+            docker_cp("#{build_host_name}-builder:#{path}", host_path)
+          end
+        end
+      end
+
+      # Wait for ssh to come up in the container
+      #
+      # Retry 5 times with a 1 second sleep between errors to account for
+      # network resets while SSHD is starting. Allow a maximum of 5 seconds for
+      # SSHD to start.
+      #
+      # @raise [Vanagon::Error] if a SSH connection cannot be established.
+      # @return [void]
+      def wait_for_ssh
+        Vanagon::Utilities.retry_with_timeout(5, 5) do
+          begin
+            Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", 'exit', @platform.ssh_port)
+          rescue StandardError => e
+            sleep(1) # Give SSHD some time to start.
+            raise e
+          end
+        end
+      rescue StandardError => e
+        raise Vanagon::Error.wrap(e, "SSH was not up in the container after 5 seconds.")
+      end
     end
   end
 end