vanagon 0.15.36 → 0.17.0

data/lib/vanagon/engine/docker.rb

@@ -10,6 +10,7 @@ class Vanagon
 
         @docker_cmd = Vanagon::Utilities.find_program_on_path('docker')
         @required_attributes << "docker_image"
+        @required_attributes.delete('ssh_port') if @platform.use_docker_exec
       end
 
       # Get the engine name
@@ -33,25 +34,16 @@ class Vanagon
       # This method is used to obtain a vm to build upon using
       # a docker container.
       # @raise [Vanagon::Error] if a target cannot be obtained
-      def select_target # rubocop:disable Metrics/AbcSize
+      def select_target
+        ssh_args = @platform.use_docker_exec ? '' : "-p #{@platform.ssh_port}:22"
         extra_args = @platform.docker_run_args.nil? ? [] : @platform.docker_run_args
 
-        Vanagon::Utilities.ex("#{@docker_cmd} run -d --name #{build_host_name}-builder -p #{@platform.ssh_port}:22 #{extra_args.join(' ')} #{@platform.docker_image}")
+        Vanagon::Utilities.ex("#{@docker_cmd} run -d --name #{build_host_name}-builder #{ssh_args} #{extra_args.join(' ')} #{@platform.docker_image}")
         @target = 'localhost'
 
-        # Wait for ssh to come up in the container. Retry 5 times with a 1
-        # second sleep between errors to account for network resets while SSHD
-        # is starting. Allow a maximum of 5 seconds for SSHD to start.
-        Vanagon::Utilities.retry_with_timeout(5, 5) do
-          begin
-            Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", 'exit', @platform.ssh_port)
-          rescue StandardError => e
-            sleep(1) # Give SSHD some time to start.
-            raise e
-          end
-        end
+        wait_for_ssh unless @platform.use_docker_exec
       rescue StandardError => e
-        raise Vanagon::Error.wrap(e, "Something went wrong getting a target vm to build on using docker. Ssh was not up in the container after 5 seconds.")
+        raise Vanagon::Error.wrap(e, "Something went wrong getting a target vm to build on using Docker.")
       end
 
       # This method is used to tell the vmpooler to delete the instance of the
@@ -62,6 +54,101 @@ class Vanagon
       rescue Vanagon::Error => e
         warn "There was a problem tearing down the docker container #{build_host_name}-builder (#{e.message})."
       end
+
+      def dispatch(command, return_output = false)
+        if @platform.use_docker_exec
+          docker_exec(command, return_output)
+        else
+          super
+        end
+      end
+
+      def ship_workdir(workdir)
+        if @platform.use_docker_exec
+          docker_cp_globs_to("#{workdir}/*", @remote_workdir)
+        else
+          super
+        end
+      end
+
+      def retrieve_built_artifact(artifacts_to_fetch, no_packaging)
+        if @platform.use_docker_exec
+          output_path = 'output/'
+          FileUtils.mkdir_p(output_path)
+          unless no_packaging
+            artifacts_to_fetch << "#{@remote_workdir}/output/*"
+          end
+
+          docker_cp_globs_from(artifacts_to_fetch, 'output/')
+        else
+          super
+        end
+      end
+
+      # Execute a command on a container via docker exec
+      def docker_exec(command, return_output = false)
+        command = command.gsub("'", "'\\\\''")
+        Vanagon::Utilities.local_command("#{@docker_cmd} exec #{build_host_name}-builder /bin/sh -c '#{command}'",
+                                         return_command_output: return_output)
+      end
+
+      # Copy files between a container and the host
+      def docker_cp(source, target)
+        Vanagon::Utilities.ex("#{@docker_cmd} cp '#{source}' '#{target}'")
+      end
+
+      # Copy files matching a glob pattern from the host to the container
+      def docker_cp_globs_to(globs, container_path)
+        Array(globs).each do |glob|
+          Dir.glob(glob).each do |path|
+            docker_cp(path, "#{build_host_name}-builder:#{container_path}")
+          end
+        end
+      end
+
+      # Copy files matching a glob pattern from the container to the host
+      #
+      # @note Globs are expanded by running `/bin/sh` in the container, which
+      #   may not support the same variety of expressions as Ruby's `Dir.glob`.
+      #   For example, `**` may not work.
+      def docker_cp_globs_from(globs, host_path)
+        Array(globs).each do |glob|
+          # Match the behavior of `rsync -r` when both paths are directories
+          # by copying the contents of the directory instead of the directory.
+          glob += '*' if glob.end_with?('/') && host_path.end_with?('/')
+
+          # TODO: This doesn't handle "interesting" paths. E.g. paths with
+          #   spaces or other special non-glob characters. This could be
+          #   fixed with a variant of `Shellwords.shellescape` that allows
+          #   glob characters to pass through.
+          paths = docker_exec(%(for file in #{glob};do [ -e "$file" ] && printf '%s\\0' "${file}";done), true).split("\0")
+
+          paths.each do |path|
+            docker_cp("#{build_host_name}-builder:#{path}", host_path)
+          end
+        end
+      end
+
+      # Wait for ssh to come up in the container
+      #
+      # Retry 5 times with a 1 second sleep between errors to account for
+      # network resets while SSHD is starting. Allow a maximum of 5 seconds for
+      # SSHD to start.
+      #
+      # @raise [Vanagon::Error] if a SSH connection cannot be established.
+      # @return [void]
+      def wait_for_ssh
+        Vanagon::Utilities.retry_with_timeout(5, 5) do
+          begin
+            Vanagon::Utilities.remote_ssh_command("#{@target_user}@#{@target}", 'exit', @platform.ssh_port)
+          rescue StandardError => e
+            sleep(1) # Give SSHD some time to start.
+            raise e
+          end
+        end
+      rescue StandardError => e
+        raise Vanagon::Error.wrap(e, "SSH was not up in the container after 5 seconds.")
+      end
     end
   end
 end

data/lib/vanagon/engine/pooler.rb

@@ -1,6 +1,10 @@
 require 'vanagon/engine/base'
 require 'yaml'
 
+### Note this class is deprecated in favor of using the ABS Engine. The pooler has changed it's API with regards to
+# getting VMs for ondemand provisioning and would need to be updated here.
+# See DIO-1066
+
 class Vanagon
   class Engine
     class Pooler < Base
@@ -10,7 +14,7 @@ class Vanagon
       def initialize(platform, target = nil, **opts)
         super
 
-        @available_poolers = ["http://vmpooler.delivery.puppetlabs.net", "https://nspooler-service-prod-1.delivery.puppetlabs.net"]
+        @available_poolers = ["https://vmpooler.delivery.puppetlabs.net", "https://nspooler-service-prod-1.delivery.puppetlabs.net"]
         @token = load_token
         @required_attributes << "vmpooler_template"
       end
@@ -63,7 +67,7 @@ class Vanagon
 
       # Read a vmpooler token from the yaml formatted vmfloaty config,
       # as outlined by the vmfloaty project:
-      # https://github.com/briancain/vmfloaty
+      # https://github.com/puppetlabs/vmfloaty
       #
       # @return [String, nil] the vmfloaty vmpooler token value
       def read_vmfloaty_token(path = "~/.vmfloaty.yml")
@@ -75,7 +79,7 @@ class Vanagon
       end
       private :read_vmfloaty_token
 
-      # This method is used to obtain a vm to build upon using the Puppet Labs'
+      # This method is used to obtain a vm to build upon using Puppet's internal
       # vmpooler (https://github.com/puppetlabs/vmpooler) or other pooler technologies
       # leveraging the same API
       # @raise [Vanagon::Error] if a target cannot be obtained

data/lib/vanagon/platform.rb

@@ -115,6 +115,7 @@ class Vanagon
     # Docker engine specific
     attr_accessor :docker_image
     attr_accessor :docker_run_args
+    attr_accessor :use_docker_exec
 
     # AWS engine specific
     attr_accessor :aws_ami
@@ -238,6 +239,8 @@ class Vanagon
       @copy ||= "cp"
       @shasum ||= "sha1sum"
 
+      @use_docker_exec = false
+
       # Our first attempt at defining metadata about a platform
       @cross_compiled ||= false
       @valid_operators ||= ['<', '>', '<=', '>=', '=']
@@ -486,9 +489,9 @@ class Vanagon
     def generate_compiled_archive(project)
       name_and_version = "#{project.name}-#{project.version}"
       name_and_version_and_platform = "#{name_and_version}.#{name}"
+      name_and_platform = "#{project.name}.#{name}"
       final_archive = "output/#{name_and_version_and_platform}.tar.gz"
       archive_directory = "#{project.name}-archive"
-      metadata = project.build_manifest_json(true)
 
       # previously, we weren't properly handling the case of custom BOM paths.
       # If we have a custom BOM path, during Makefile execution, the top-level
@@ -503,7 +506,6 @@ class Vanagon
         bill_of_materials_command = "mv .#{project.bill_of_materials.path}/bill-of-materials ../.."
       end
 
-      metadata.gsub!(/\n/, '\n')
       [
         "mkdir output",
         "mkdir #{archive_directory}",
@@ -511,7 +513,7 @@ class Vanagon
         "rm #{name_and_version}.tar.gz",
         "cd #{archive_directory}/#{name_and_version}; #{bill_of_materials_command}; #{tar} cf ../../#{name_and_version_and_platform}.tar *",
         "gzip -9c #{name_and_version_and_platform}.tar > #{name_and_version_and_platform}.tar.gz",
-        "echo -e \"#{metadata}\" > output/#{name_and_version_and_platform}.json",
+        "cp build_metadata.#{name_and_platform}.json output/#{name_and_version_and_platform}.json",
         "cp bill-of-materials output/#{name_and_version_and_platform}-bill-of-materials ||:",
         "cp #{name_and_version_and_platform}.tar.gz output",
         "#{shasum} #{final_archive} > #{final_archive}.sha1"

data/lib/vanagon/platform/deb.rb

@@ -140,7 +140,7 @@ class Vanagon
         @tar = "tar"
         @patch = "/usr/bin/patch"
         @num_cores = "/usr/bin/nproc"
-        @curl = "curl --silent --show-error --fail"
+        @curl = "curl --silent --show-error --fail --location"
         @valid_operators = ['<', '>', '<=', '>=', '=', '<<', '>>']
         super(name)
       end

data/lib/vanagon/platform/dsl.rb

@@ -284,6 +284,17 @@ class Vanagon
         @platform.docker_run_args = Array(args)
       end
 
+      # Specify whether to use Docker exec instead of SSH to run commands
+      #
+      # This also causes Vanagon to use `docker cp` instead of `rsync` when
+      # copying files.
+      #
+      # @param bool [Boolean] a boolean value indicating whether to use
+      #   `docker exec` and `docker cp` over `ssh` and `rsync`.
+      def use_docker_exec(bool)
+        @platform.use_docker_exec = bool
+      end
+
       # Set the ami for the platform to use
       #
       # @param ami [String] the ami id used.

data/lib/vanagon/platform/rpm.rb

@@ -103,7 +103,7 @@ class Vanagon
         @patch ||= "/usr/bin/patch"
         @num_cores ||= "/bin/grep -c 'processor' /proc/cpuinfo"
         @rpmbuild ||= "/usr/bin/rpmbuild"
-        @curl = "curl --silent --show-error --fail"
+        @curl = "curl --silent --show-error --fail --location"
         super(name)
       end
     end

data/lib/vanagon/platform/windows.rb

@@ -205,11 +205,36 @@ class Vanagon
         # "Misc Dir for versions.txt, License file and Icon file"
         misc_dir = "SourceDir/#{project.settings[:base_dir]}/#{project.settings[:company_id]}/#{project.settings[:product_id]}/misc"
         # Actual array of commands to be written to the Makefile
-        [
+        make_commands = [
           "mkdir -p output/#{target_dir}",
           "mkdir -p $(tempdir)/{SourceDir,wix/wixobj}",
           "#{@copy} -r wix/* $(tempdir)/wix/",
-          "gunzip -c #{project.name}-#{project.version}.tar.gz | '#{@tar}' -C '$(tempdir)/SourceDir' --strip-components 1 -xf -",
+          "gunzip -c #{project.name}-#{project.version}.tar.gz | '#{@tar}' -C '$(tempdir)/SourceDir' --strip-components 1 -xf -"
+        ]
+
+        unless project.extra_files_to_sign.empty?
+          begin
+            tempdir = nil
+            # Skip signing extra files if logging into the signing_host fails
+            # This enables things like CI being able to sign the additional files,
+            # but locally triggered builds by developers who don't have access to
+            # the signing host just print a message and skip the signing.
+            Vanagon::Utilities.retry_with_timeout(3, 5) do
+              tempdir = Vanagon::Utilities::remote_ssh_command("#{project.signing_username}@#{project.signing_hostname}", "#{@mktemp} 2>/dev/null", return_command_output: true)
+            end
+            project.extra_files_to_sign.each do |file|
+              file_location = File.join(tempdir, File.basename(file))
+              make_commands << [
+                "rsync -e '#{Vanagon::Utilities.ssh_command}' -rHlv --no-perms --no-owner --no-group #{File.join('$(tempdir)', 'SourceDir', file)} #{project.signing_username}@#{project.signing_hostname}:#{tempdir}",
+                "#{Vanagon::Utilities.ssh_command} #{project.signing_username}@#{project.signing_hostname} #{project.signing_command} #{file_location}",
+                "rsync -e '#{Vanagon::Utilities.ssh_command}' -rHlv -O --no-perms --no-owner --no-group #{project.signing_username}@#{project.signing_hostname}:#{file_location} #{File.join('$(tempdir)', 'SourceDir', file)}"
+              ]
+            end
+          rescue RuntimeError
+            warn "Unable to connect to #{project.signing_username}@#{project.signing_hostname}, skipping signing extra files: #{project.extra_files_to_sign.join(',')}"
+          end
+        end
+        make_commands << [
           "mkdir -p $(tempdir)/#{misc_dir}",
           # Need to use awk here to convert to DOS format so that notepad can display file correctly.
           "awk 'sub(\"$$\", \"\\r\")' $(tempdir)/SourceDir/bill-of-materials > $(tempdir)/#{misc_dir}/versions.txt",
@@ -225,6 +250,8 @@ class Vanagon
           # -loc is required for the UI localization it points to the actual localization .wxl
           "cd $(tempdir)/wix/wixobj; \"$$WIX/bin/light.exe\" #{light_flags} -b $$(cygpath -aw $(tempdir)) -loc $$(cygpath -aw $(tempdir)/wix/localization/puppet_en-us.wxl) -out $$(cygpath -aw $(workdir)/output/#{target_dir}/#{msi_package_name(project)}) *.wixobj",
         ]
+
+        make_commands.flatten
       end
 
       # Method to derive the msi (Windows Installer) package name for the project.

data/lib/vanagon/project.rb

@@ -108,6 +108,14 @@ class Vanagon
     # you just want to perform installation and pull down a file.
     attr_accessor :no_packaging
 
+    # Extra files to sign
+    # Right now just supported on windows, useful for signing powershell scripts
+    # that need to be signed between build and MSI creation
+    attr_accessor :extra_files_to_sign
+    attr_accessor :signing_hostname
+    attr_accessor :signing_username
+    attr_accessor :signing_command
+
     # Loads a given project from the configdir
     #
     # @param name [String] the name of the project
@@ -156,6 +164,10 @@ class Vanagon
       @upstream_metadata = {}
       @no_packaging = false
       @artifacts_to_fetch = []
+      @extra_files_to_sign = []
+      @signing_hostname = ''
+      @signing_username = ''
+      @signing_command = ''
     end
 
     # Magic getter to retrieve settings in the project
@@ -714,20 +726,27 @@ class Vanagon
     # Writes a json file at `ext/build_metadata.<project>.<platform>.json` containing information
     # about what went into a built artifact
     #
-    # @return [Hash] of build information
-    def save_manifest_json(platform)
+    # @param platform [String] platform we're writing metadata for
+    # @param additional_directories [String|Array[String]] additional
+    #        directories to write build_metadata.<project>.<platform>.json to
+    def save_manifest_json(platform, additional_directories = nil) # rubocop:disable Metrics/AbcSize
       manifest = build_manifest_json
       metadata = metadata_merge(manifest, @upstream_metadata)
 
       ext_directory = 'ext'
       FileUtils.mkdir_p ext_directory
 
+      directories = [ext_directory, additional_directories].compact
       metadata_file_name = "build_metadata.#{name}.#{platform.name}.json"
-      File.open(File.join(ext_directory, metadata_file_name), 'w') do |f|
-        f.write(JSON.pretty_generate(metadata))
+      directories.each do |directory|
+        File.open(File.join(directory, metadata_file_name), 'w') do |f|
+          f.write(JSON.pretty_generate(metadata))
+        end
       end
 
       ## VANAGON-132 Backwards compatibility: make a 'build_metadata.json' file
+      # No need to propagate this backwards compatibility to the new additional
+      # directories
       File.open(File.join(ext_directory, 'build_metadata.json'), 'w') do |f|
         f.write(JSON.pretty_generate(metadata))
       end

data/lib/vanagon/project/dsl.rb

@@ -363,6 +363,39 @@ class Vanagon
       def no_packaging(var)
         @project.no_packaging = var
       end
+
+      # Set to sign additional files during buildtime. Only implemented for
+      # windows. Can be specified more than once
+      #
+      # @param [String] file to sign
+      def extra_file_to_sign(file)
+        @project.extra_files_to_sign << file
+      end
+
+      # The hostname to sign additional files on. Only does anything when there
+      # are extra files to sign
+      #
+      # @param [String] hostname of the machine to run the extra file signing on
+      def signing_hostname(hostname)
+        @project.signing_hostname = hostname
+      end
+
+      # The username to log in to the signing_hostname as. Only does anything
+      # when there are extra files to sign
+      #
+      # @param [String] the username to log in to `signing_hostname` as
+      def signing_username(username)
+        @project.signing_username = username
+      end
+
+      # The command to run to sign additional files. The command should assume
+      # it will have the file path appended to the end of the command, since
+      # files end up in a temp directory.
+      #
+      # @param [String] the command to sign additional files
+      def signing_command(command)
+        @project.signing_command = command
+      end
     end
   end
 end

data/lib/vanagon/utilities.rb

@@ -39,17 +39,16 @@ class Vanagon
     end
 
     # Simple wrapper around Net::HTTP. Will make a request of the given type to
-    # the given url and return the body as parsed by JSON.
+    # the given url and return the response object
     #
     # @param url [String] The url to make the request against (needs to be parsable by URI
     # @param type [String] One of the supported request types (currently 'get', 'post', 'delete')
     # @param payload [String] The request body data payload used for POST and PUT
     # @param header [Hash] Send additional information in the HTTP request header
-    # @return [Hash] The response body is parsed by JSON and returned
+    # @return [Net::HTTPAccepted] The response object
     # @raise [RuntimeError, Vanagon::Error] an exception is raised if the
-    # action is not supported, or if there is a problem with the http request,
-    # or if the response is not JSON
-    def http_request(url, type, payload = {}.to_json, header = nil) # rubocop:disable Metrics/AbcSize
+    # action is not supported, or if there is a problem with the http request
+    def http_request_generic(url, type, payload = {}.to_json, header = nil) # rubocop:disable Metrics/AbcSize
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = true if uri.scheme == 'https'
@@ -77,14 +76,37 @@ class Vanagon
       end
 
       response = http.request(request)
-
-      JSON.parse(response.body)
+      response
     rescue Errno::ETIMEDOUT, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
       EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
       Net::ProtocolError => e
       raise Vanagon::Error.wrap(e, "Problem reaching #{url}. Is #{uri.host} down?")
+    end
+
+    # uses http_request_generic and returns the body as parsed by JSON.
+    # @param url [String] The url to make the request against (needs to be parsable by URI
+    # @param type [String] One of the supported request types (currently 'get', 'post', 'delete')
+    # @param payload [String] The request body data payload used for POST and PUT
+    # @param header [Hash] Send additional information in the HTTP request header
+    # @return [Hash] The response in JSON format
+    # @raise [RuntimeError, Vanagon::Error] an exception is raised if the response
+    # body cannot be parsed as JSON
+    def http_request(url, type, payload = {}.to_json, header = nil)
+      response = http_request_generic(url, type, payload, header)
+      JSON.parse(response.body)
     rescue JSON::ParserError => e
-      raise Vanagon::Error.wrap(e, "#{uri.host} handed us a response that doesn't look like JSON.")
+      raise Vanagon::Error.wrap(e, "#{url} handed us a response that doesn't look like JSON.")
+    end
+
+    # uses http_request_generic and returns the response code.
+    # @param url [String] The url to make the request against (needs to be parsable by URI
+    # @param type [String] One of the supported request types (currently 'get', 'post', 'delete')
+    # @param payload [String] The request body data payload used for POST and PUT
+    # @param header [Hash] Send additional information in the HTTP request header
+    # @return [String] The response code eg 202, 200 etc
+    def http_request_code(url, type, payload = {}.to_json, header = nil)
+      response = http_request_generic(url, type, payload, header)
+      response.code
     end
 
     # Similar to rake's sh, the passed command will be executed and an