bolt 3.23.1 → 3.26.1

data/lib/bolt/module_installer/specs/git_spec.rb

@@ -1,9 +1,15 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'net/http'
+require 'open3'
 require 'set'
 
 require_relative '../../../bolt/error'
+require_relative '../../../bolt/logger'
+require_relative '../../../bolt/module_installer/specs/id/gitclone'
+require_relative '../../../bolt/module_installer/specs/id/github'
+require_relative '../../../bolt/module_installer/specs/id/gitlab'
 
 # This class represents a Git module specification.
 #
@@ -17,12 +23,19 @@ module Bolt
 
         attr_reader :git, :ref, :resolve, :type
 
-        def initialize(init_hash)
-          @resolve    = init_hash.key?('resolve') ? init_hash['resolve'] : true
-          @name       = parse_name(init_hash['name'])
-          @git, @repo = parse_git(init_hash['git'])
-          @ref        = init_hash['ref']
-          @type       = :git
+        def initialize(init_hash, config = {})
+          @logger  = Bolt::Logger.logger(self)
+          @resolve = init_hash.key?('resolve') ? init_hash['resolve'] : true
+          @git     = init_hash['git']
+          @ref     = init_hash['ref']
+          @name    = parse_name(init_hash['name'])
+          @proxy   = config.dig('proxy')
+          @type    = :git
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+          end
 
           if @name.nil? && @resolve == false
             raise Bolt::ValidationError,
@@ -30,9 +43,9 @@ module Bolt
                   "must include a 'name' key when 'resolve' is false."
           end
 
-          unless @resolve == true || @resolve == false
+          unless valid_url?(@git)
             raise Bolt::ValidationError,
-                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+                  "Invalid URI #{@git}. Valid URIs must begin with 'git@', 'http://', or 'https://'."
           end
         end
 
@@ -57,22 +70,6 @@ module Bolt
           match[:name]
         end
 
-        # Gets the repo from the git URL.
-        #
-        private def parse_git(git)
-          return [git, nil] unless @resolve
-
-          repo = if git.start_with?('git@github.com:')
-                   git.split('git@github.com:').last.split('.git').first
-                 elsif git.start_with?('https://github.com')
-                   git.split('https://github.com/').last.split('.git').first
-                 else
-                   raise Bolt::ValidationError, invalid_git_msg(git)
-                 end
-
-          [git, repo]
-        end
-
         # Returns true if the specification is satisfied by the module.
         #
         def satisfied_by?(mod)
@@ -88,14 +85,6 @@ module Bolt
           }
         end
 
-        # Returns an error message that the provided repo is not a git repo or
-        # is private.
-        #
-        private def invalid_git_msg(repo_name)
-          "#{repo_name} is not a public GitHub repository. See https://pup.pt/no-resolve "\
-            "for information on how to install this module."
-        end
-
         # Returns a PuppetfileResolver::Model::GitModule object for resolving.
         #
         def to_resolver_module
@@ -107,90 +96,53 @@ module Bolt
           end
         end
 
-        # Resolves the module's title from the module metadata. This is lazily
-        # resolved since Bolt does not always need to know a Git module's name.
+        # Returns the module's name.
         #
         def name
-          @name ||= begin
-            url      = "https://raw.githubusercontent.com/#{@repo}/#{sha}/metadata.json"
-            response = make_request(:Get, url)
-
-            case response
-            when Net::HTTPOK
-              body = JSON.parse(response.body)
-
-              unless body.key?('name')
-                raise Bolt::Error.new(
-                  "Missing name in metadata.json at #{git}. This is not a valid module.",
-                  "bolt/missing-module-name-error"
-                )
-              end
-
-              parse_name(body['name'])
-            else
-              raise Bolt::Error.new(
-                "Missing metadata.json at #{git}. This is not a valid module.",
-                "bolt/missing-module-metadata-error"
-              )
-            end
-          end
+          @name ||= parse_name(id.name)
         end
 
-        # Resolves the SHA for the specified ref. This is lazily resolved since
-        # Bolt does not always need to know a Git module's SHA.
+        # Returns the SHA for the module's ref.
         #
         def sha
-          @sha ||= begin
-            url      = "https://api.github.com/repos/#{@repo}/commits/#{ref}"
-            headers  = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {}
-            response = make_request(:Get, url, headers)
-
-            case response
-            when Net::HTTPOK
-              body = JSON.parse(response.body)
-              body['sha']
-            when Net::HTTPUnauthorized
-              raise Bolt::Error.new(
-                "Invalid token at GITHUB_TOKEN, unable to resolve git modules.",
-                "bolt/invalid-git-token-error"
-              )
-            when Net::HTTPForbidden
-              message = "GitHub API rate limit exceeded, unable to resolve git modules. "
-
-              unless ENV['GITHUB_TOKEN']
-                message += "To increase your rate limit, set the GITHUB_TOKEN environment "\
-                          "variable with a GitHub personal access token."
-              end
-
-              raise Bolt::Error.new(message, 'bolt/github-api-rate-limit-error')
-            when Net::HTTPNotFound
-              raise Bolt::Error.new(invalid_git_msg(git), "bolt/missing-git-repository-error")
-            else
+          id.sha
+        end
+
+        # Gets the ID for the module based on the specified ref and git URL.
+        # This is lazily resolved since Bolt does not always need this information,
+        # and requesting it is expensive.
+        #
+        private def id
+          @id ||= begin
+            # The request methods here return an ID object if the module name and SHA
+            # were found and nil otherwise. This lets Bolt try multiple methods for
+            # finding the module name and SHA, and short circuiting as soon as it does.
+            module_id = Bolt::ModuleInstaller::Specs::ID::GitHub.request(@git, @ref, @proxy) ||
+                        Bolt::ModuleInstaller::Specs::ID::GitLab.request(@git, @ref, @proxy) ||
+                        Bolt::ModuleInstaller::Specs::ID::GitClone.request(@git, @ref, @proxy)
+
+            unless module_id
               raise Bolt::Error.new(
-                "Ref #{ref} at #{git} is not a commit, tag, or branch.",
-                "bolt/invalid-git-ref-error"
+                "Unable to locate metadata and calculate SHA for ref #{@ref} at #{@git}. This may "\
+                "not be a valid module. For more information about how Bolt attempted to locate "\
+                "this information, check the debugging logs.",
+                'bolt/missing-module-metadata-error'
               )
             end
+
+            module_id
           end
         end
 
-        # Makes a generic HTTP request.
+        # Returns true if the URL is valid.
         #
-        private def make_request(verb, url, headers = {})
-          require 'net/http'
-
-          uri      = URI.parse(url)
-          opts     = { use_ssl: uri.scheme == 'https' }
+        private def valid_url?(url)
+          return true if url.start_with?('git@')
 
-          Net::HTTP.start(uri.host, uri.port, opts) do |client|
-            request = Net::HTTP.const_get(verb).new(uri, headers)
-            client.request(request)
-          end
-        rescue StandardError => e
-          raise Bolt::Error.new(
-            "Failed to connect to #{uri}: #{e.message}",
-            "bolt/http-connect-error"
-          )
+          uri = URI.parse(url)
+          uri.is_a?(URI::HTTP) && uri.host
+        rescue URI::InvalidURIError
+          false
         end
       end
     end

data/lib/bolt/module_installer/specs/id/base.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+
+require_relative '../../../../bolt/error'
+require_relative '../../../../bolt/logger'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class Base
+          attr_reader :name, :sha
+
+          # @param name [String] The module's name.
+          # @param sha [String] The ref's SHA1.
+          #
+          def initialize(name, sha)
+            @name = name
+            @sha  = sha
+          end
+
+          # Request the name and SHA for a module and ref.
+          # This method must return either an ID object or nil. The GitSpec
+          # class relies on this class to return an ID object to indicate
+          # the module was found, or nil to indicate that it should try to
+          # find it another way (such as cloning the repo).
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param proxy [String] A proxy to use when making requests.
+          #
+          def self.request(git, ref, proxy)
+            name, sha = name_and_sha(git, ref, proxy)
+            name && sha ? new(name, sha) : nil
+          end
+
+          # Stub method for retrieving the module's name and SHA. Must
+          # be implemented by all sub classes.
+          #
+          private_class_method def self.name_and_sha(_git, _ref, _proxy)
+            raise NotImplementedError, 'Class does not implemented #name_and_sha'
+          end
+
+          # Makes a HTTP request.
+          #
+          # @param url [String] The URL to make the request to.
+          # @param proxy [String] A proxy to use when making the request.
+          # @param headers [Hash] Headers to send with the request.
+          #
+          private_class_method def self.make_request(url, proxy, headers = {})
+            uri  = URI.parse(url)
+            opts = { use_ssl: uri.scheme == 'https' }
+            args = [uri.host, uri.port]
+
+            if proxy
+              proxy = URI.parse(proxy)
+              args += [proxy.host, proxy.port, proxy.user, proxy.password]
+            end
+
+            Bolt::Logger.debug("Making request to #{loc(url, proxy)}")
+
+            Net::HTTP.start(*args, opts) do |client|
+              client.request(Net::HTTP::Get.new(uri, headers))
+            end
+          rescue StandardError => e
+            raise Bolt::Error.new(
+              "Failed to connect to #{loc(uri, proxy)}: #{e.message}",
+              "bolt/http-connect-error"
+            )
+          end
+
+          # Returns a formatted string describing the URL and proxy used when making
+          # a request.
+          #
+          # @param url [String, URI::HTTP] The URL used.
+          # @param proxy [String, URI::HTTP] The proxy used.
+          #
+          private_class_method def self.loc(url, proxy)
+            proxy ? "#{url} with proxy #{proxy}" : url.to_s
+          end
+
+          # Parses the metadata and validates that it is a Hash.
+          #
+          # @param metadata [String] The JSON data to parse.
+          #
+          private_class_method def self.parse_name_from_metadata(metadata)
+            metadata = JSON.parse(metadata)
+
+            unless metadata.is_a?(Hash)
+              raise Bolt::Error.new(
+                "Invalid metadata. Expected a Hash, got a #{metadata.class}: #{metadata}",
+                "bolt/invalid-module-metadata-error"
+              )
+            end
+
+            unless metadata.key?('name')
+              raise Bolt::Error.new(
+                "Invalid metadata. Metadata must include a 'name' key.",
+                "bolt/missing-module-name-error"
+              )
+            end
+
+            metadata['name']
+          rescue JSON::ParserError => e
+            raise Bolt::Error.new(
+              "Unable to parse metadata as JSON: #{e.message}",
+              "bolt/metadata-parse-error"
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/gitclone.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitClone < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param proxy [String] The proxy to use when cloning.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            require 'open3'
+
+            unless git?
+              Bolt::Logger.debug("'git' executable not found, unable to use git clone resolution.")
+              return nil
+            end
+
+            # Clone the repo into a temp directory that will be automatically cleaned up.
+            Dir.mktmpdir do |dir|
+              return nil unless clone_repo(git, ref, dir, proxy)
+
+              # Extract the name from the metadata file and calculate the SHA.
+              Dir.chdir(dir) do
+                [request_name(git, ref), request_sha(git, ref)]
+              end
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          #
+          private_class_method def self.request_name(git, ref)
+            command = %W[git show #{ref}:metadata.json]
+            Bolt::Logger.debug("Executing command '#{command.join(' ')}'")
+
+            out, err, status = Open3.capture3(*command)
+
+            unless status.success?
+              raise Bolt::Error.new(
+                "Unable to find metadata file at #{git}: #{err}",
+                "bolt/missing-metadata-file-error"
+              )
+            end
+
+            Bolt::Logger.debug("Found metadata file at #{git}")
+            parse_name_from_metadata(out)
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          #
+          private_class_method def self.request_sha(git, ref)
+            command = %W[git rev-parse #{ref}^{commit}]
+            Bolt::Logger.debug("Executing command '#{command.join(' ')}'")
+
+            out, err, status = Open3.capture3(*command)
+
+            if status.success?
+              out.strip
+            else
+              raise Bolt::Error.new(
+                "Unable to calculate SHA for ref #{ref} at #{git}: #{err}",
+                "bolt/invalid-ref-error"
+              )
+            end
+          end
+
+          # Clones the repository. First attempts to clone a bare repository
+          # and falls back to cloning the full repository if that fails. Cloning
+          # a bare repository is significantly faster for large modules, but
+          # cloning a bare repository using a commit is not supported.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param dir [String] The directory to clone the repo to.
+          # @param proxy [String] The proxy to use when cloning.
+          #
+          private_class_method def self.clone_repo(git, ref, dir, proxy)
+            clone  = %W[git clone #{git} #{dir}]
+            clone += %W[--config "http.proxy=#{proxy}" --config "https.proxy=#{proxy}"] if proxy
+
+            bare_clone = clone + %w[--bare --depth=1]
+            bare_clone.push("--branch=#{ref}") unless ref == 'HEAD'
+
+            # Attempt to clone a bare repository
+            Bolt::Logger.debug("Executing command '#{bare_clone.join(' ')}'")
+            _out, err, status = Open3.capture3(*bare_clone)
+            return true if status.success?
+            Bolt::Logger.debug("Unable to clone bare repository at #{loc(git, proxy)}: #{err}")
+
+            # Fall back to cloning the full repository
+            Bolt::Logger.debug("Executing command '#{clone.join(' ')}'")
+            _out, err, status = Open3.capture3(*clone)
+            Bolt::Logger.debug("Unable to clone repository at #{loc(git, proxy)}: #{err}") unless status.success?
+            status.success?
+          end
+
+          # Returns true if the 'git' executable is available.
+          #
+          private_class_method def self.git?
+            Open3.capture3('git', '--version')
+            true
+          rescue Errno::ENOENT
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/github.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitHub < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            repo = parse_repo(git)
+            return nil unless repo
+            [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)]
+          end
+
+          # Parses the repo path out of the URL.
+          #
+          # @param git [String] The URL to the git repo.
+          #
+          private_class_method def self.parse_repo(git)
+            if git.start_with?('git@github.com:')
+              git.split('git@github.com:').last.split('.git').first
+            elsif git.start_with?('https://github.com')
+              git.split('https://github.com/').last.split('.git').first
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_name(repo, ref, proxy)
+            metadata_url = "https://raw.githubusercontent.com/#{repo}/#{ref}/metadata.json"
+            response     = make_request(metadata_url, proxy)
+
+            case response
+            when Net::HTTPOK
+              Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}")
+              parse_name_from_metadata(response.body)
+            else
+              Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}")
+              nil
+            end
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to resolve.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_sha(repo, ref, proxy)
+            url      = "https://api.github.com/repos/#{repo}/commits/#{ref}"
+            headers  = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {}
+            response = make_request(url, proxy, headers)
+
+            case response
+            when Net::HTTPOK
+              JSON.parse(response.body).fetch('sha', nil)
+            when Net::HTTPUnauthorized
+              Bolt::Logger.debug("Invalid token at GITHUB_TOKEN, unable to calculate SHA.")
+              nil
+            when Net::HTTPForbidden
+              message = "GitHub API rate limit exceeded, unable to calculate SHA."
+
+              unless ENV['GITHUB_TOKEN']
+                message += " To increase your rate limit, set the GITHUB_TOKEN environment "\
+                          "variable with a GitHub personal access token."
+              end
+
+              Bolt::Logger.debug(message)
+              nil
+            else
+              Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}")
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/gitlab.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitLab < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            repo = parse_repo(git)
+            return nil unless repo
+            [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)]
+          end
+
+          # Parses the repo path out of the URL.
+          #
+          # @param git [String] The URL to the git repo.
+          #
+          private_class_method def self.parse_repo(git)
+            if git.start_with?('git@gitlab.com:')
+              git.split('git@gitlab.com:').last.split('.git').first
+            elsif git.start_with?('https://gitlab.com')
+              git.split('https://gitlab.com/').last.split('.git').first
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_name(repo, ref, proxy)
+            metadata_url = "https://gitlab.com/#{repo}/-/raw/#{ref}/metadata.json"
+            response     = make_request(metadata_url, proxy)
+
+            case response
+            when Net::HTTPOK
+              Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}")
+              parse_name_from_metadata(response.body)
+            else
+              Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}")
+              nil
+            end
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to resolve.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_sha(repo, ref, proxy)
+            require 'cgi'
+
+            url      = "https://gitlab.com/api/v4/projects/#{CGI.escape(repo)}/repository/commits/#{ref}"
+            headers  = ENV['GITLAB_TOKEN'] ? { "PRIVATE-TOKEN" => ENV['GITLAB_TOKEN'] } : {}
+            response = make_request(url, proxy, headers)
+
+            case response
+            when Net::HTTPOK
+              JSON.parse(response.body).fetch('id', nil)
+            when Net::HTTPUnauthorized
+              Bolt::Logger.debug("Invalid token at GITLAB_TOKEN, unable to calculate SHA.")
+              nil
+            when Net::HTTPForbidden
+              message = "GitLab API rate limit exceeded, unable to calculate SHA."
+
+              unless ENV['GITLAB_TOKEN']
+                message += " To increase your rate limit, set the GITLAB_TOKEN environment "\
+                          "variable with a GitLab personal access token."
+              end
+
+              Bolt::Logger.debug(message)
+              nil
+            else
+              Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}")
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs.rb

@@ -7,8 +7,10 @@ require_relative 'specs/git_spec'
 module Bolt
   class ModuleInstaller
     class Specs
-      def initialize(specs = [])
-        @specs = []
+      def initialize(specs = [], config = {})
+        @specs  = []
+        @config = config
+
         add_specs(specs)
         assert_unique_names
       end
@@ -49,7 +51,7 @@ module Bolt
       #
       private def spec_from_hash(hash)
         return ForgeSpec.new(hash) if ForgeSpec.implements?(hash)
-        return GitSpec.new(hash)   if GitSpec.implements?(hash)
+        return GitSpec.new(hash, @config) if GitSpec.implements?(hash)
 
         raise Bolt::ValidationError, <<~MESSAGE.chomp
           Invalid module specification:

data/lib/bolt/module_installer.rb

@@ -18,7 +18,7 @@ module Bolt
     # Adds a single module to the project.
     #
     def add(name, specs, puppetfile_path, moduledir, project_file, config)
-      project_specs = Specs.new(specs)
+      project_specs = Specs.new(specs, config)
 
       # Exit early if project config already includes a spec with this name.
       if project_specs.include?(name)
@@ -151,7 +151,7 @@ module Bolt
       @outputter.print_message("Installing project modules\n\n")
 
       if resolve != false
-        specs = Specs.new(specs)
+        specs = Specs.new(specs, config)
 
         # If forcibly installing or if there is no Puppetfile, resolve
         # and write a Puppetfile.