pdk 1.9.1 → 1.10.0

data/lib/pdk/module/update.rb

@@ -6,6 +6,8 @@ module PDK
       GIT_DESCRIBE_PATTERN = %r{\A(?<base>.+?)-(?<additional_commits>\d+)-g(?<sha>.+)\Z}
 
       def run
+        template_uri.git_ref = new_template_version
+
         stage_changes!
 
         if current_version == new_version
@@ -50,8 +52,8 @@ module PDK
         raise PDK::CLI::ExitWithError, e.message
       end
 
-      def template_url
-        @template_url ||= module_metadata.data['template-url']
+      def template_uri
+        @template_uri ||= PDK::Util::TemplateURI.new(module_metadata.data['template-url'])
       end
 
       def current_version
@@ -62,6 +64,16 @@ module PDK
         @new_version ||= fetch_remote_version(new_template_version)
       end
 
+      def new_template_version
+        return options[:'template-ref'] if options[:'template-ref']
+
+        if template_uri.default? && template_uri.ref_is_tag? && PDK::Util.package_install?
+          PDK::Util::TemplateURI.default_template_ref
+        else
+          template_uri.git_ref
+        end
+      end
+
       private
 
       def current_template_version
@@ -73,27 +85,23 @@ module PDK
 
         return data if data.nil?
 
-        if data[:base].start_with?('heads/')
-          "#{data[:base].gsub(%r{^heads/}, '')}@#{data[:sha]}"
+        if data[:base] =~ %r{^(?:heads|remotes)/}
+          "#{data[:base].gsub(%r{^(heads/|remotes/\w+?/)}, '')}@#{data[:sha]}"
         else
           data[:base]
         end
       end
 
-      def new_template_version
-        PDK::Util.default_template_ref
-      end
-
-      def fetch_remote_version(version)
-        return version unless version.include?('/')
+      def fetch_remote_version(template_ref)
+        return template_ref unless current_template_version.is_a?(String)
+        return template_ref if template_ref == PDK::TEMPLATE_REF
 
-        branch = version.partition('/').last
         sha_length = GIT_DESCRIBE_PATTERN.match(current_template_version)[:sha].length - 1
-        "#{branch}@#{PDK::Util::Git.ls_remote(template_url, "refs/heads/#{branch}")[0..sha_length]}"
+        "#{template_ref}@#{PDK::Util::Git.ls_remote(template_uri.git_remote, template_ref)[0..sha_length]}"
       end
 
       def update_message
-        format_string = if template_url == PDK::Util.puppetlabs_template_url
+        format_string = if template_uri.default?
                           _('Updating %{module_name} using the default template, from %{current_version} to %{new_version}')
                         else
                           _('Updating %{module_name} using the template at %{template_url}, from %{current_version} to %{new_version}')
@@ -101,7 +109,7 @@ module PDK
 
         format_string % {
           module_name:     module_metadata.data['name'],
-          template_url:    template_url,
+          template_url:    template_uri.git_remote,
           current_version: current_version,
           new_version:     new_version,
         }

data/lib/pdk/util.rb

@@ -5,6 +5,7 @@ require 'pdk/util/version'
 require 'pdk/util/windows'
 require 'pdk/util/vendored_file'
 require 'pdk/util/filesystem'
+require 'pdk/util/template_uri'
 
 module PDK
   module Util
@@ -197,63 +198,6 @@ module PDK
     end
     module_function :targets_relative_to_pwd
 
-    def default_template_url
-      answer_file_url = PDK.answers['template-url']
-
-      return puppetlabs_template_url if answer_file_url.nil?
-
-      # Ignore answer file template-url if the value is the old or new default.
-      return puppetlabs_template_url if answer_file_url == 'https://github.com/puppetlabs/pdk-module-template'
-      return puppetlabs_template_url if answer_file_url == puppetlabs_template_url
-
-      if File.directory?(answer_file_url)
-        # Instantiating a new TemplateDir object pointing to the directory
-        # will cause the directory contents to be validated, raising
-        # ArgumentError if it does not appear to be a valid template.
-        PDK::Module::TemplateDir.new(answer_file_url) {}
-        return answer_file_url
-      end
-
-      raise ArgumentError unless PDK::Util::Git.repo?(answer_file_url)
-
-      answer_file_url
-    rescue ArgumentError
-      PDK.logger.warn(
-        _("Unable to access the previously used template '%{template}', using the default template instead.") % {
-          template: answer_file_url,
-        },
-      )
-      PDK.answers.update!('template-url' => nil)
-      return puppetlabs_template_url
-    end
-    module_function :default_template_url
-
-    def puppetlabs_template_url
-      if package_install?
-        'file://' + File.join(package_cachedir, 'pdk-templates.git')
-      else
-        'https://github.com/puppetlabs/pdk-templates'
-      end
-    end
-    module_function :puppetlabs_template_url
-
-    def default_template_ref
-      # TODO: This should respect a --template-ref option if we add that
-      return 'origin/master' if default_template_url != puppetlabs_template_url
-
-      puppetlabs_template_ref
-    end
-    module_function :default_template_ref
-
-    def puppetlabs_template_ref
-      if PDK::Util.development_mode?
-        'origin/master'
-      else
-        PDK::TEMPLATE_REF
-      end
-    end
-    module_function :puppetlabs_template_ref
-
     # TO-DO: Refactor replacement of lib/pdk/module/build.rb:metadata to use this function instead
     def module_metadata
       PDK::Module::Metadata.from_file(File.join(module_root, 'metadata.json')).data

data/lib/pdk/util/filesystem.rb

@@ -4,7 +4,13 @@ module PDK
       def write_file(path, content)
         raise ArgumentError unless path.is_a?(String) || path.respond_to?(:to_path)
 
-        File.open(path, 'wb') { |f| f.write(content.encode(universal_newline: true)) }
+        # Harmonize newlines across platforms.
+        content = content.encode(universal_newline: true)
+
+        # Make sure all written files have a trailing newline.
+        content += "\n" unless content[-1] == "\n"
+
+        File.open(path, 'wb') { |f| f.write(content) }
       end
       module_function :write_file
     end

data/lib/pdk/util/git.rb

@@ -1,5 +1,21 @@
 module PDK
   module Util
+    class GitError < StandardError
+      attr_reader :stdout
+      attr_reader :stderr
+      attr_reader :exit_code
+      attr_reader :args
+
+      def initialze(args, result)
+        @args = args
+        @stdout = result[:stdout]
+        @stderr = result[:stderr]
+        @exit_code = result[:exit_code]
+
+        super(_('Git command failed: git %{args}' % { args: args.join(' ') }))
+      end
+    end
+
     module Git
       def self.git_bindir
         @git_dir ||= File.join('private', 'git', Gem.win_platform? ? 'cmd' : 'bin')
@@ -55,7 +71,27 @@ module PDK
         git('ls-remote', '--exit-code', maybe_repo)[:exit_code].zero?
       end
 
+      def self.work_tree?(path)
+        return false unless File.directory?(path)
+
+        Dir.chdir(path) do
+          rev_parse = git('rev-parse', '--is-inside-work-tree')
+          rev_parse[:exit_code].zero? && rev_parse[:stdout].strip == 'true'
+        end
+      end
+
+      def self.work_dir_clean?(repo)
+        raise PDK::CLI::ExitWithError, _('Unable to locate git work dir "%{workdir}') % { workdir: repo } unless File.directory?(repo)
+        raise PDK::CLI::ExitWithError, _('Unable to locate git dir "%{gitdir}') % { gitdir: repo } unless File.directory?(File.join(repo, '.git'))
+
+        git('--work-tree', repo, '--git-dir', File.join(repo, '.git'), 'status', '--untracked-files=no', '--porcelain', repo)[:stdout].empty?
+      end
+
       def self.ls_remote(repo, ref)
+        if File.directory?(repo)
+          repo = 'file://' + repo
+        end
+
         output = git('ls-remote', '--refs', repo, ref)
 
         unless output[:exit_code].zero?
@@ -67,7 +103,16 @@ module PDK
         end
 
         matching_refs = output[:stdout].split("\n").map { |r| r.split("\t") }
-        matching_refs.find { |_sha, remote_ref| remote_ref == ref }.first
+        matching_ref = matching_refs.find { |_sha, remote_ref| remote_ref == "refs/tags/#{ref}" || remote_ref == "refs/remotes/origin/#{ref}" || remote_ref == "refs/heads/#{ref}" }
+        raise PDK::CLI::ExitWithError, _('Unable to find a branch or tag named "%{ref}" in %{repo}') % { ref: ref, repo: repo } if matching_ref.nil?
+        matching_ref.first
+      end
+
+      def self.describe(path, ref = nil)
+        args = ['--git-dir', path, 'describe', '--all', '--long', '--always', ref].compact
+        result = git(*args)
+        raise PDK::Util::GitError, args, result unless result[:exit_code].zero?
+        result[:stdout].strip
       end
     end
   end

data/lib/pdk/util/template_uri.rb

@@ -0,0 +1,231 @@
+require 'pdk/util'
+
+module PDK
+  module Util
+    class TemplateURI
+      SCP_PATTERN = %r{\A(?!\w+://)(?:(?<user>.+?)@)?(?<host>[^:/]+):(?<path>.+)\z}
+
+      # XXX Previously
+      # - template_uri used to get the string form of the uri when generating the module and written to pdk answers and metadata
+      # - template_path or deuri_path used for humans to see and commands to run
+      # - uri_path used only internally by the template selection code; move out
+      # - template_ref used by git checkout
+      attr_reader :uri
+
+      # input/output formats:
+      #
+      # file:///c:/foo (git clone location)
+      # c:/foo (shell paths)
+      # file:///c:/foo#master (only for metadata)
+      # c:/foo#master (only for metadata)
+      #
+      # non output formats:
+      #
+      # /c:/foo (internal use only)
+      # /c:/foo#master (internal use only)
+      #
+      def initialize(opts_or_uri)
+        # If a uri string is passed, skip the valid uri finding code.
+        @uri = if opts_or_uri.is_a?(String) || opts_or_uri.is_a?(self.class)
+                 begin
+                   Addressable::URI.parse(opts_or_uri)
+                 rescue Addressable::URI::InvalidURIError
+                   raise PDK::CLI::FatalError, _('PDK::Util::TemplateURI attempted initialization with a non-uri string: {string}') % { string: opts_or_uri }
+                 end
+               elsif opts_or_uri.is_a?(Addressable::URI)
+                 opts_or_uri.dup
+               else
+                 self.class.first_valid_uri(self.class.templates(opts_or_uri))
+               end
+      end
+
+      def ==(other)
+        @uri == other.uri
+      end
+
+      # This is the URI represented in a format suitable for writing to
+      # metadata.
+      #
+      # @returns String
+      def metadata_format
+        self.class.human_readable(@uri.to_s)
+      end
+      alias to_s metadata_format
+      alias to_str metadata_format
+
+      # This is the url without a fragment, suitable for git clones.
+      #
+      # @returns String
+      def git_remote
+        self.class.git_remote(@uri)
+      end
+
+      def self.git_remote(uri)
+        if uri.is_a?(Addressable::URI) && uri.fragment
+          human_readable(uri.to_s.chomp('#' + uri.fragment))
+        else
+          human_readable(uri.to_s)
+        end
+      end
+
+      # This is the path of the URI, suitable for accessing directly from the shell.
+      # @returns String
+      def shell_path
+        self.class.human_readable(@uri.path)
+      end
+
+      # @returns String
+      def git_ref
+        if @uri.fragment
+          @uri.fragment
+        else
+          self.class.default_template_ref
+        end
+      end
+
+      def git_ref=(ref)
+        @uri.fragment = ref
+      end
+
+      # @returns PDK::Util::TemplateURI
+      def self.default_template_uri
+        if PDK::Util.package_install?
+          PDK::Util::TemplateURI.new(Addressable::URI.new(scheme: 'file', host: '', path: File.join(PDK::Util.package_cachedir, 'pdk-templates.git')))
+        else
+          PDK::Util::TemplateURI.new('https://github.com/puppetlabs/pdk-templates')
+        end
+      end
+
+      def default?
+        git_remote == self.class.default_template_uri.git_remote
+      end
+
+      def ref_is_tag?
+        PDK::Util::Git.git('ls-remote', '--tags', '--exit-code', git_remote, git_ref)[:exit_code].zero?
+      end
+
+      # `C:...` urls are not URI-safe. They should be of the form `/C:...` to
+      # be URI-safe. scp-like urls like `user@host:/path` are not URI-safe
+      # either but are not handled here. Should they be?
+      #
+      # @returns String
+      def self.uri_safe(string)
+        (Gem.win_platform? && string =~ %r{^[a-zA-Z][\|:]}) ? "/#{string}" : string
+      end
+
+      # If the passed value is a URI-safe windows path such as `/C:...` then it
+      # should be changed to a human-friendly `C:...` form. Otherwise the
+      # passed value is left alone.
+      #
+      # @returns String
+      def self.human_readable(string)
+        (Gem.win_platform? && string =~ %r{^\/[a-zA-Z][\|:]}) ? string[1..-1] : string
+      end
+
+      # @return [Array<Hash{Symbol => Object}>] an array of hashes. Each hash
+      #   contains 3 keys: :type contains a String that describes the template
+      #   directory, :url contains a String with the URL to the template
+      #   directory, and :allow_fallback contains a Boolean that specifies if
+      #   the lookup process should proceed to the next template directory if
+      #   the template file is not in this template directory.
+      #
+      def self.templates(opts)
+        explicit_url = opts.fetch(:'template-url', nil)
+        explicit_ref = opts.fetch(:'template-ref', nil)
+
+        # 1. Get the CLI, metadata (or answers if no metadata), and default URIs
+        # 2. Construct the hash
+        if explicit_url
+          # Valid URIs to avoid catching:
+          # - absolute local paths
+          # - have :'s in paths when preceeded by a slash
+          # - have only digits following the : and preceeding a / or end-of-string that is 0-65535
+          # The last item is ambiguous in the case of scp/git paths vs. URI port
+          # numbers, but can be made unambiguous by making the form to
+          # ssh://git@github.com/1234/repo.git or
+          # ssh://git@github.com:1234/user/repo.git
+          scp_url = explicit_url.match(SCP_PATTERN)
+          if Pathname.new(uri_safe(explicit_url)).relative? && scp_url
+            explicit_uri = Addressable::URI.new(scheme: 'ssh', user: scp_url[:user], host: scp_url[:host], path: scp_url[:path])
+            PDK.logger.warn _('%{scp_uri} appears to be an SCP style URL; it will be converted to an RFC compliant URI: %{rfc_uri}') % {
+              scp_uri: explicit_url,
+              rfc_uri: explicit_uri.to_s,
+            }
+          end
+          explicit_uri ||= Addressable::URI.parse(uri_safe(explicit_url))
+          explicit_uri.fragment = explicit_ref || default_template_ref
+        else
+          explicit_uri = nil
+        end
+        metadata_uri = if PDK::Util.module_root && File.file?(File.join(PDK::Util.module_root, 'metadata.json'))
+                         Addressable::URI.parse(uri_safe(PDK::Util.module_metadata['template-url']))
+                       else
+                         nil
+                       end
+        answers_uri = if PDK.answers['template-url'] == 'https://github.com/puppetlabs/pdk-module-template'
+                        # use the new github template-url if it is still the old one.
+                        Addressable::URI.parse(default_template_uri)
+                      elsif PDK.answers['template-url']
+                        Addressable::URI.parse(uri_safe(PDK.answers['template-url']))
+                      else
+                        nil
+                      end
+        default_uri = Addressable::URI.parse(default_template_uri)
+        default_uri.fragment = default_template_ref
+
+        ary = []
+        ary << { type: _('--template-url'), uri: explicit_uri, allow_fallback: false } if explicit_url
+        ary << { type: _('metadata.json'), uri: metadata_uri, allow_fallback: true } if metadata_uri
+        ary << { type: _('PDK answers'), uri: answers_uri, allow_fallback: true } if answers_uri
+        ary << { type: _('default'), uri: default_uri, allow_fallback: false }
+        ary
+      end
+
+      # @returns String
+      def self.default_template_ref
+        if PDK::Util.development_mode?
+          'master'
+        else
+          PDK::TEMPLATE_REF
+        end
+      end
+
+      # @returns Addressable::URI
+      def self.first_valid_uri(templates_array)
+        # 1. Get the four sources of URIs
+        # 2. Pick the first non-nil URI
+        # 3. Error if the URI is not a valid git repo (missing directory or http 404)
+        # 4. Leave updating answers/metadata to other code
+        found_template = templates_array.find { |t| valid_template?(t) }
+
+        raise PDK::CLI::FatalError, _('Unable to find a valid module template to use.') if found_template.nil?
+        found_template[:uri]
+      end
+
+      def self.valid_template?(template)
+        return false if template.nil? || !template.is_a?(Hash)
+        return false if template[:uri].nil? || !template[:uri].is_a?(Addressable::URI)
+
+        return true if PDK::Util::Git.repo?(git_remote(template[:uri]))
+
+        path = human_readable(template[:uri].path)
+        if File.directory?(path)
+          begin
+            PDK::Module::TemplateDir.new(path) {}
+            return true
+          rescue ArgumentError
+            nil
+          end
+        end
+
+        unless template[:allow_fallback]
+          raise PDK::CLI::FatalError, _('Unable to find a valid template at %{uri}') % {
+            uri: template[:uri].to_s,
+          }
+        end
+
+        false
+      end
+    end
+  end
+end