openbolt 5.0.0.rc1

data/lib/bolt/module_installer/specs/git_spec.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'open3'
+require 'set'
+
+require_relative '../../../bolt/error'
+require_relative '../../../bolt/logger'
+require_relative '../../../bolt/module_installer/specs/id/gitclone'
+require_relative '../../../bolt/module_installer/specs/id/github'
+require_relative '../../../bolt/module_installer/specs/id/gitlab'
+
+# This class represents a Git module specification.
+#
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class GitSpec
+        NAME_REGEX    = %r{\A(?:[a-zA-Z0-9]+[-/])?(?<name>[a-z][a-z0-9_]*)\z}.freeze
+        REQUIRED_KEYS = Set.new(%w[git ref]).freeze
+        KNOWN_KEYS    = Set.new(%w[git name ref resolve]).freeze
+
+        attr_reader :git, :ref, :resolve, :type
+
+        def initialize(init_hash, config = {})
+          @logger  = Bolt::Logger.logger(self)
+          @resolve = init_hash.key?('resolve') ? init_hash['resolve'] : true
+          @git     = init_hash['git']
+          @ref     = init_hash['ref']
+          @name    = parse_name(init_hash['name'])
+          @proxy   = config.dig('proxy')
+          @type    = :git
+
+          unless @resolve == true || @resolve == false
+            raise Bolt::ValidationError,
+                  "Option 'resolve' for module spec #{@git} must be a Boolean"
+          end
+
+          if @name.nil? && @resolve == false
+            raise Bolt::ValidationError,
+                  "Missing name for Git module specification: #{@git}. Git module specifications "\
+                  "must include a 'name' key when 'resolve' is false."
+          end
+
+          unless valid_url?(@git)
+            raise Bolt::ValidationError,
+                  "Invalid URI #{@git}. Valid URIs must begin with 'git@', 'http://', 'https://' or 'ssh://'."
+          end
+        end
+
+        def self.implements?(hash)
+          KNOWN_KEYS.superset?(hash.keys.to_set) && REQUIRED_KEYS.subset?(hash.keys.to_set)
+        end
+
+        # Parses the name into owner and name segments, and formats the full
+        # name.
+        #
+        private def parse_name(name)
+          return unless name
+
+          unless (match = name.match(NAME_REGEX))
+            raise Bolt::ValidationError,
+                  "Invalid name for Git module specification: #{name}. Name must match "\
+                  "'name' or 'owner/name'. Owner segment can only include letters or digits. "\
+                  "Name segment must start with a lowercase letter and can only include "\
+                  "lowercase letters, digits, and underscores."
+          end
+
+          match[:name]
+        end
+
+        # Returns true if the specification is satisfied by the module.
+        #
+        def satisfied_by?(mod)
+          @type == mod.type && @git == mod.git
+        end
+
+        # Returns a hash matching the module spec in bolt-project.yaml
+        #
+        def to_hash
+          {
+            'git' => @git,
+            'ref' => @ref
+          }
+        end
+
+        # Returns a PuppetfileResolver::Model::GitModule object for resolving.
+        #
+        def to_resolver_module
+          require 'puppetfile-resolver'
+
+          PuppetfileResolver::Puppetfile::GitModule.new(name).tap do |mod|
+            mod.remote = @git
+            mod.ref    = sha
+          end
+        end
+
+        # Returns the module's name.
+        #
+        def name
+          @name ||= parse_name(id.name)
+        end
+
+        # Returns the SHA for the module's ref.
+        #
+        def sha
+          id.sha
+        end
+
+        # Gets the ID for the module based on the specified ref and git URL.
+        # This is lazily resolved since Bolt does not always need this information,
+        # and requesting it is expensive.
+        #
+        private def id
+          @id ||= begin
+            # The request methods here return an ID object if the module name and SHA
+            # were found and nil otherwise. This lets Bolt try multiple methods for
+            # finding the module name and SHA, and short circuiting as soon as it does.
+            module_id = Bolt::ModuleInstaller::Specs::ID::GitHub.request(@git, @ref, @proxy) ||
+                        Bolt::ModuleInstaller::Specs::ID::GitLab.request(@git, @ref, @proxy) ||
+                        Bolt::ModuleInstaller::Specs::ID::GitClone.request(@git, @ref, @proxy)
+
+            unless module_id
+              raise Bolt::Error.new(
+                "Unable to locate metadata and calculate SHA for ref #{@ref} at #{@git}. This may "\
+                "not be a valid module. For more information about how Bolt attempted to locate "\
+                "this information, check the debugging logs.",
+                'bolt/missing-module-metadata-error'
+              )
+            end
+
+            module_id
+          end
+        end
+
+        # Returns true if the URL is valid.
+        #
+        private def valid_url?(url)
+          return true if url.start_with?('git@')
+
+          uri = URI.parse(url)
+          (uri.is_a?(URI::HTTP) || uri.scheme == "ssh") && uri.host
+        rescue URI::InvalidURIError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/base.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+
+require_relative '../../../../bolt/error'
+require_relative '../../../../bolt/logger'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class Base
+          attr_reader :name, :sha
+
+          # @param name [String] The module's name.
+          # @param sha [String] The ref's SHA1.
+          #
+          def initialize(name, sha)
+            @name = name
+            @sha  = sha
+          end
+
+          # Request the name and SHA for a module and ref.
+          # This method must return either an ID object or nil. The GitSpec
+          # class relies on this class to return an ID object to indicate
+          # the module was found, or nil to indicate that it should try to
+          # find it another way (such as cloning the repo).
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param proxy [String] A proxy to use when making requests.
+          #
+          def self.request(git, ref, proxy)
+            name, sha = name_and_sha(git, ref, proxy)
+            name && sha ? new(name, sha) : nil
+          end
+
+          # Stub method for retrieving the module's name and SHA. Must
+          # be implemented by all sub classes.
+          #
+          private_class_method def self.name_and_sha(_git, _ref, _proxy)
+            raise NotImplementedError, 'Class does not implemented #name_and_sha'
+          end
+
+          # Makes a HTTP request.
+          #
+          # @param url [String] The URL to make the request to.
+          # @param proxy [String] A proxy to use when making the request.
+          # @param headers [Hash] Headers to send with the request.
+          #
+          private_class_method def self.make_request(url, proxy, headers = {})
+            uri  = URI.parse(url)
+            opts = { use_ssl: uri.scheme == 'https' }
+            args = [uri.host, uri.port]
+
+            if proxy
+              proxy = URI.parse(proxy)
+              args += [proxy.host, proxy.port, proxy.user, proxy.password]
+            end
+
+            Bolt::Logger.debug("Making request to #{loc(url, proxy)}")
+
+            Net::HTTP.start(*args, opts) do |client|
+              client.request(Net::HTTP::Get.new(uri, headers))
+            end
+          rescue StandardError => e
+            raise Bolt::Error.new(
+              "Failed to connect to #{loc(uri, proxy)}: #{e.message}",
+              "bolt/http-connect-error"
+            )
+          end
+
+          # Returns a formatted string describing the URL and proxy used when making
+          # a request.
+          #
+          # @param url [String, URI::HTTP] The URL used.
+          # @param proxy [String, URI::HTTP] The proxy used.
+          #
+          private_class_method def self.loc(url, proxy)
+            proxy ? "#{url} with proxy #{proxy}" : url.to_s
+          end
+
+          # Parses the metadata and validates that it is a Hash.
+          #
+          # @param metadata [String] The JSON data to parse.
+          #
+          private_class_method def self.parse_name_from_metadata(metadata)
+            metadata = JSON.parse(metadata)
+
+            unless metadata.is_a?(Hash)
+              raise Bolt::Error.new(
+                "Invalid metadata. Expected a Hash, got a #{metadata.class}: #{metadata}",
+                "bolt/invalid-module-metadata-error"
+              )
+            end
+
+            unless metadata.key?('name')
+              raise Bolt::Error.new(
+                "Invalid metadata. Metadata must include a 'name' key.",
+                "bolt/missing-module-name-error"
+              )
+            end
+
+            metadata['name']
+          rescue JSON::ParserError => e
+            raise Bolt::Error.new(
+              "Unable to parse metadata as JSON: #{e.message}",
+              "bolt/metadata-parse-error"
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/gitclone.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitClone < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param proxy [String] The proxy to use when cloning.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            require 'open3'
+
+            unless git?
+              Bolt::Logger.debug("'git' executable not found, unable to use git clone resolution.")
+              return nil
+            end
+
+            # Clone the repo into a temp directory that will be automatically cleaned up.
+            Dir.mktmpdir do |dir|
+              return nil unless clone_repo(git, ref, dir, proxy)
+
+              # Extract the name from the metadata file and calculate the SHA.
+              Dir.chdir(dir) do
+                [request_name(git, ref), request_sha(git, ref)]
+              end
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          #
+          private_class_method def self.request_name(git, ref)
+            command = %W[git show #{ref}:metadata.json]
+            Bolt::Logger.debug("Executing command '#{command.join(' ')}'")
+
+            out, err, status = Open3.capture3(*command)
+
+            unless status.success?
+              raise Bolt::Error.new(
+                "Unable to find metadata file at #{git}: #{err}",
+                "bolt/missing-metadata-file-error"
+              )
+            end
+
+            Bolt::Logger.debug("Found metadata file at #{git}")
+            parse_name_from_metadata(out)
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          #
+          private_class_method def self.request_sha(git, ref)
+            command = %W[git rev-parse #{ref}^{commit}]
+            Bolt::Logger.debug("Executing command '#{command.join(' ')}'")
+
+            out, err, status = Open3.capture3(*command)
+
+            if status.success?
+              out.strip
+            else
+              raise Bolt::Error.new(
+                "Unable to calculate SHA for ref #{ref} at #{git}: #{err}",
+                "bolt/invalid-ref-error"
+              )
+            end
+          end
+
+          # Clones the repository. First attempts to clone a bare repository
+          # and falls back to cloning the full repository if that fails. Cloning
+          # a bare repository is significantly faster for large modules, but
+          # cloning a bare repository using a commit is not supported.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to checkout.
+          # @param dir [String] The directory to clone the repo to.
+          # @param proxy [String] The proxy to use when cloning.
+          #
+          private_class_method def self.clone_repo(git, ref, dir, proxy)
+            clone  = %W[git clone #{git} #{dir}]
+            clone += %W[--config "http.proxy=#{proxy}" --config "https.proxy=#{proxy}"] if proxy
+
+            bare_clone = clone + %w[--bare --depth=1]
+            bare_clone.push("--branch=#{ref}") unless ref == 'HEAD'
+
+            # Attempt to clone a bare repository
+            Bolt::Logger.debug("Executing command '#{bare_clone.join(' ')}'")
+            _out, err, status = Open3.capture3(*bare_clone)
+            return true if status.success?
+            Bolt::Logger.debug("Unable to clone bare repository at #{loc(git, proxy)}: #{err}")
+
+            # Fall back to cloning the full repository
+            Bolt::Logger.debug("Executing command '#{clone.join(' ')}'")
+            _out, err, status = Open3.capture3(*clone)
+            Bolt::Logger.debug("Unable to clone repository at #{loc(git, proxy)}: #{err}") unless status.success?
+            status.success?
+          end
+
+          # Returns true if the 'git' executable is available.
+          #
+          private_class_method def self.git?
+            Open3.capture3('git', '--version')
+            true
+          rescue Errno::ENOENT
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/github.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitHub < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            repo = parse_repo(git)
+            return nil unless repo
+            [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)]
+          end
+
+          # Parses the repo path out of the URL.
+          #
+          # @param git [String] The URL to the git repo.
+          #
+          private_class_method def self.parse_repo(git)
+            if git.start_with?('git@github.com:')
+              git.split('git@github.com:').last.split('.git').first
+            elsif git.start_with?('https://github.com')
+              git.split('https://github.com/').last.split('.git').first
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_name(repo, ref, proxy)
+            metadata_url = "https://raw.githubusercontent.com/#{repo}/#{ref}/metadata.json"
+            response     = make_request(metadata_url, proxy)
+
+            case response
+            when Net::HTTPOK
+              Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}")
+              parse_name_from_metadata(response.body)
+            else
+              Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}")
+              nil
+            end
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to resolve.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_sha(repo, ref, proxy)
+            url      = "https://api.github.com/repos/#{repo}/commits/#{ref}"
+            headers  = ENV['GITHUB_TOKEN'] ? { "Authorization" => "token #{ENV['GITHUB_TOKEN']}" } : {}
+            response = make_request(url, proxy, headers)
+
+            case response
+            when Net::HTTPOK
+              JSON.parse(response.body).fetch('sha', nil)
+            when Net::HTTPUnauthorized
+              Bolt::Logger.debug("Invalid token at GITHUB_TOKEN, unable to calculate SHA.")
+              nil
+            when Net::HTTPForbidden
+              message = "GitHub API rate limit exceeded, unable to calculate SHA."
+
+              unless ENV['GITHUB_TOKEN']
+                message += " To increase your rate limit, set the GITHUB_TOKEN environment "\
+                          "variable with a GitHub personal access token."
+              end
+
+              Bolt::Logger.debug(message)
+              nil
+            else
+              Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}")
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs/id/gitlab.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative '../../../../bolt/module_installer/specs/id/base'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      class ID
+        class GitLab < Base
+          # Returns the name and SHA for the module at the given ref.
+          #
+          # @param git [String] The URL to the git repo.
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.name_and_sha(git, ref, proxy)
+            repo = parse_repo(git)
+            return nil unless repo
+            [request_name(repo, ref, proxy), request_sha(repo, ref, proxy)]
+          end
+
+          # Parses the repo path out of the URL.
+          #
+          # @param git [String] The URL to the git repo.
+          #
+          private_class_method def self.parse_repo(git)
+            if git.start_with?('git@gitlab.com:')
+              git.split('git@gitlab.com:').last.split('.git').first
+            elsif git.start_with?('https://gitlab.com')
+              git.split('https://gitlab.com/').last.split('.git').first
+            end
+          end
+
+          # Requests a module's metadata and returns the name from it.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to use.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_name(repo, ref, proxy)
+            metadata_url = "https://gitlab.com/#{repo}/-/raw/#{ref}/metadata.json"
+            response     = make_request(metadata_url, proxy)
+
+            case response
+            when Net::HTTPOK
+              Bolt::Logger.debug("Found metadata file at #{loc(metadata_url, proxy)}")
+              parse_name_from_metadata(response.body)
+            else
+              Bolt::Logger.debug("Unable to find metadata file at #{loc(metadata_url, proxy)}")
+              nil
+            end
+          end
+
+          # Requests the SHA for the specified ref.
+          #
+          # @param repo [String] The repo ID, i.e. 'owner/repo'
+          # @param ref [String] The ref to resolve.
+          # @param proxy [String] The proxy to use when making requests.
+          #
+          private_class_method def self.request_sha(repo, ref, proxy)
+            require 'cgi'
+
+            url      = "https://gitlab.com/api/v4/projects/#{CGI.escape(repo)}/repository/commits/#{ref}"
+            headers  = ENV['GITLAB_TOKEN'] ? { "PRIVATE-TOKEN" => ENV['GITLAB_TOKEN'] } : {}
+            response = make_request(url, proxy, headers)
+
+            case response
+            when Net::HTTPOK
+              JSON.parse(response.body).fetch('id', nil)
+            when Net::HTTPUnauthorized
+              Bolt::Logger.debug("Invalid token at GITLAB_TOKEN, unable to calculate SHA.")
+              nil
+            when Net::HTTPForbidden
+              message = "GitLab API rate limit exceeded, unable to calculate SHA."
+
+              unless ENV['GITLAB_TOKEN']
+                message += " To increase your rate limit, set the GITLAB_TOKEN environment "\
+                          "variable with a GitLab personal access token."
+              end
+
+              Bolt::Logger.debug(message)
+              nil
+            else
+              Bolt::Logger.debug("Unable to calculate SHA for ref #{ref}")
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bolt/module_installer/specs.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require_relative '../../bolt/error'
+require_relative 'specs/forge_spec'
+require_relative 'specs/git_spec'
+
+module Bolt
+  class ModuleInstaller
+    class Specs
+      def initialize(specs = [], config = {})
+        @specs  = []
+        @config = config
+
+        add_specs(specs)
+        assert_unique_names
+      end
+
+      # Creates a list of specs from the modules in a Puppetfile object.
+      #
+      def self.from_puppetfile(puppetfile)
+        new(puppetfile.modules.map(&:to_hash))
+      end
+
+      # Returns a list of specs.
+      #
+      def specs
+        @specs.uniq(&:name)
+      end
+
+      # Returns true if the specs includes the given name.
+      #
+      def include?(name)
+        _owner, name = name.tr('-', '/').split('/', 2)
+        @specs.any? { |spec| spec.name == name }
+      end
+
+      # Adds a spec.
+      #
+      def add_specs(*specs)
+        specs.flatten.map do |spec|
+          case spec
+          when Hash
+            @specs.unshift spec_from_hash(spec)
+          else
+            @specs.unshift spec
+          end
+        end
+      end
+
+      # Parses a spec hash into a spec object.
+      #
+      private def spec_from_hash(hash)
+        return ForgeSpec.new(hash) if ForgeSpec.implements?(hash)
+        return GitSpec.new(hash, @config) if GitSpec.implements?(hash)
+
+        raise Bolt::ValidationError, <<~MESSAGE.chomp
+          Invalid module specification:
+          #{hash.to_yaml.lines.drop(1).join.chomp}
+
+          To read more about specifying modules, see https://pup.pt/bolt-module-specs
+        MESSAGE
+      end
+
+      # Returns true if all specs are satisfied by the modules in a Puppetfile.
+      #
+      def satisfied_by?(puppetfile)
+        @specs.all? do |spec|
+          puppetfile.modules.any? do |mod|
+            spec.satisfied_by?(mod)
+          end
+        end
+      end
+
+      # Asserts that all specs are unique by name. The puppetfile-resolver
+      # library also does this, but the error it raises isn't as helpful.
+      #
+      private def assert_unique_names
+        duplicates = @specs.group_by(&:name).select { |_name, specs| specs.count > 1 }
+
+        if duplicates.any?
+          message = String.new
+
+          duplicates.each do |name, duplicate_specs|
+            message << <<~MESSAGE
+              Detected multiple module specifications with name #{name}:
+              #{duplicate_specs.map(&:to_hash).to_yaml.lines.drop(1).join}
+            MESSAGE
+          end
+
+          raise Bolt::Error.new(message.chomp, "bolt/duplicate-spec-name-error")
+        end
+      end
+    end
+  end
+end