pdk 1.17.0 → 1.18.0

data/lib/pdk/template/renderer/v1/legacy_template_dir.rb

@@ -0,0 +1,116 @@
+require 'pdk'
+
+module PDK
+  module Template
+    module Renderer
+      module V1
+        # The old templating code in the PDK passed through a TemplateDir object. This class mimics the methods
+        # of that old class so that existing custom templates will still function with the newer refactor templating code.
+        # Methods which have no use in custom templates exist but do no nothing, for example `def render; end`
+        #
+        # @see https://raw.githubusercontent.com/puppetlabs/pdk/4ffd58062c77ad1e54d2fe16b16015f7207bcee8/lib/pdk/module/template_dir/base.rb
+        # :nocov: This class is tested in the packaging and acceptance testing suites
+        class LegacyTemplateDir
+          attr_accessor :module_metadata
+          attr_reader :uri
+
+          def initialize(context, uri, path, module_metadata = {})
+            @uri = uri
+            @module_metadata = module_metadata
+            @context = context
+            @path = path
+          end
+
+          def metadata; end
+
+          def render; end
+
+          def object_template_for; end
+
+          def object_config
+            config_for(nil)
+          end
+
+          # Generate a hash of data to be used when rendering the specified
+          # template.
+          #
+          # @param dest_path [String] The destination path of the file that the
+          # data is for, relative to the root of the module.
+          #
+          # @return [Hash] The data that will be available to the template via the
+          # `@configs` instance variable.
+          #
+          # @api private
+          def config_for(dest_path, sync_config_path = nil)
+            require 'pdk/util'
+            require 'pdk/analytics'
+
+            module_root = PDK::Util.module_root
+            sync_config_path ||= File.join(module_root, '.sync.yml') unless module_root.nil?
+            config_path = File.join(@path, 'config_defaults.yml')
+
+            if @config.nil?
+              require 'deep_merge'
+              conf_defaults = read_config(config_path)
+              @sync_config = read_config(sync_config_path) unless sync_config_path.nil?
+              @config = conf_defaults
+              @config.deep_merge!(@sync_config, knockout_prefix: '---') unless @sync_config.nil?
+            end
+            file_config = @config.fetch(:global, {})
+            file_config['module_metadata'] = @module_metadata
+            file_config.merge!(@config.fetch(dest_path, {})) unless dest_path.nil?
+            file_config.merge!(@config).tap do |c|
+              if uri.default?
+                file_value = if c['unmanaged']
+                               'unmanaged'
+                             elsif c['delete']
+                               'deleted'
+                             elsif @sync_config && @sync_config.key?(dest_path)
+                               'customized'
+                             else
+                               'default'
+                             end
+
+                PDK.analytics.event('TemplateDir', 'file', label: dest_path, value: file_value)
+              end
+            end
+          end
+
+          # Generates a hash of data from a given yaml file location.
+          #
+          # @param loc [String] The path of the yaml config file.
+          #
+          # @warn If the specified path is not a valid yaml file. Returns an empty Hash
+          # if so.
+          #
+          # @return [Hash] The data that has been read in from the given yaml file.
+          #
+          # @api private
+          def read_config(loc)
+            if PDK::Util::Filesystem.file?(loc) && PDK::Util::Filesystem.readable?(loc)
+              require 'yaml'
+
+              begin
+                YAML.safe_load(PDK::Util::Filesystem.read_file(loc), [], [], true)
+              rescue Psych::SyntaxError => e
+                PDK.logger.warn _("'%{file}' is not a valid YAML file: %{problem} %{context} at line %{line} column %{column}") % {
+                  file:    loc,
+                  problem: e.problem,
+                  context: e.context,
+                  line:    e.line,
+                  column:  e.column,
+                }
+                {}
+              end
+            else
+              {}
+            end
+          end
+
+          def template_path(_uri); end
+        end
+        # :nocov:
+      end
+    end
+  end
+end

data/lib/pdk/template/renderer/v1/renderer.rb

@@ -0,0 +1,132 @@
+require 'pdk'
+require 'pdk/template/renderer'
+
+module PDK
+  module Template
+    module Renderer
+      module V1
+        class Renderer < PDK::Template::Renderer::AbstractRenderer
+          # @see PDK::Template::Renderer::AbstractRenderer.render
+          def render(template_type, _name, options = {})
+            render_module(options) { |*args| yield(*args) } if template_type == PDK::Template::MODULE_TEMPLATE_TYPE
+          end
+
+          # @see PDK::Template::Renderer::AbstractRenderer.has_single_item?
+          def has_single_item?(item_path) # rubocop:disable Naming/PredicateName
+            PDK::Util::Filesystem.exist?(single_item_path(item_path))
+          end
+
+          # @see PDK::Template::Renderer::AbstractRenderer.render_single_item
+          def render_single_item(relative_file_path, template_data_hash)
+            template_file = single_item_path(relative_file_path)
+            return nil unless PDK::Util::Filesystem.file?(template_file) && PDK::Util::Filesystem.readable?(template_file)
+
+            PDK.logger.debug(_("Rendering '%{template}'...") % { template: template_file })
+            new_template_file(template_file, template_data_hash).render
+          end
+
+          # Returns the full path for a single item
+          #
+          # @param item_path [String] The path of the single item to render
+          # @return [String]
+          # @api private
+          #:nocov:
+          def single_item_path(item_path)
+            File.join(template_root, 'object_templates', item_path)
+          end
+          #:nocov:
+
+          # Helper method used during testing
+          #:nocov:
+          # @api private
+          def new_template_file(template_file, template_data_hash)
+            TemplateFile.new(template_file, template_data_hash)
+          end
+          #:nocov:
+
+          # Helper method used during testing
+          #:nocov:
+          # @api private
+          def new_legacy_template_dir(context, uri, path, module_metadata = {})
+            LegacyTemplateDir.new(context, uri, path, module_metadata)
+          end
+          #:nocov:
+
+          # Renders a new module
+          #
+          # @param options [Hash{Object => Object}] A list of options to pass through to the renderer. See PDK::Template::TemplateDir helper methods for other options
+          # @see #render
+          # @api private
+          #:nocov: This is tested in acceptance and packaging tests
+          def render_module(options = {})
+            require 'pdk/template/renderer/v1/template_file'
+
+            moduleroot_dir = File.join(template_root, 'moduleroot')
+            moduleroot_init = File.join(template_root, 'moduleroot_init')
+
+            dirs = [moduleroot_dir]
+            dirs << moduleroot_init if options[:include_first_time]
+
+            legacy_template_dir = new_legacy_template_dir(context, template_uri, template_root, options[:module_metadata] || {})
+
+            files_in_template(dirs).each do |template_file, template_loc|
+              template_file = template_file.to_s
+              PDK.logger.debug(_("Rendering '%{template}'...") % { template: template_file })
+              dest_path = template_file.sub(%r{\.erb\Z}, '')
+              config = legacy_template_dir.config_for(dest_path)
+
+              dest_status = if template_loc.start_with?(moduleroot_init)
+                              :init
+                            else
+                              :manage
+                            end
+
+              if config['unmanaged']
+                dest_status = :unmanage
+              elsif config['delete']
+                dest_status = :delete
+              else
+                begin
+                  dest_content = new_template_file(File.join(template_loc, template_file), configs: config, template_dir: legacy_template_dir).render
+                rescue => error
+                  error_msg = _(
+                    "Failed to render template '%{template}'\n" \
+                    '%{exception}: %{message}',
+                  ) % { template: template_file, exception: error.class, message: error.message }
+                  raise PDK::CLI::FatalError, error_msg
+                end
+              end
+
+              yield dest_path, dest_content, dest_status
+            end
+          end
+          #:nocov:
+
+          # Returns all files in the given template directories
+          #
+          # @param dirs [Array[String]] Directories to search in
+          # @param glob_suffix [Array[String]] File glob to use when searching for files. Defaults to ['**', '*']
+          #
+          # @return [Hash{String => String}] Key is the template file relative path and the value is the absolute path to the template directory
+          # @api private
+          def files_in_template(dirs, glob_suffix = ['**', '*'])
+            temp_paths = []
+            dirlocs = []
+            dirs.each do |dir|
+              raise ArgumentError, _("The directory '%{dir}' doesn't exist") % { dir: dir } unless PDK::Util::Filesystem.directory?(dir)
+              temp_paths += PDK::Util::Filesystem.glob(File.join(dir, *glob_suffix), File::FNM_DOTMATCH).select do |template_path|
+                if PDK::Util::Filesystem.file?(template_path) && !PDK::Util::Filesystem.symlink?(template_path)
+                  dirlocs << dir
+                end
+              end
+              temp_paths.map do |template_path|
+                template_path.sub!(%r{\A#{Regexp.escape(dir)}#{Regexp.escape(File::SEPARATOR)}}, '')
+              end
+            end
+            Hash[temp_paths.zip dirlocs]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/template/renderer/v1/template_file.rb

@@ -0,0 +1,102 @@
+require 'pdk'
+require 'ostruct'
+
+module PDK
+  module Template
+    module Renderer
+      module V1
+        class TemplateFile < OpenStruct
+          # Initialises the TemplateFile object with the path to the template file
+          # and the data to be used when rendering the template.
+          #
+          # @param template_file [String] The path on disk to the template file.
+          # @param data [Hash{Symbol => Object}] The data that should be provided to
+          # the template when rendering.
+          # @option data [Object] :configs The value of this key will be provided to
+          # the template as an instance variable `@configs` in order to maintain
+          # compatibility with modulesync.
+          #
+          # @api public
+          def initialize(template_file, data = {})
+            @template_file = template_file
+
+            if data.key?(:configs)
+              @configs = data[:configs]
+            end
+
+            super(data)
+          end
+
+          # Renders the template by calling the appropriate engine based on the file
+          # extension.
+          #
+          # If the template has an `.erb` extension, the content of the template
+          # file will be treated as an ERB template. All other extensions are treated
+          # as plain text.
+          #
+          # @return [String] The rendered template
+          #
+          # @raise (see #template_content)
+          #
+          # @api public
+          def render
+            case File.extname(@template_file)
+            when '.erb'
+              render_erb
+            else
+              render_plain
+            end
+          end
+
+          def config_for(path)
+            return unless respond_to?(:template_dir)
+
+            template_dir.config_for(path)
+          end
+
+          private
+
+          # Reads the content of the template file into memory.
+          #
+          # @return [String] The content of the template file.
+          #
+          # @raise [ArgumentError] If the template file does not exist or can not be
+          # read.
+          #
+          # @api private
+          def template_content
+            if PDK::Util::Filesystem.file?(@template_file) && PDK::Util::Filesystem.readable?(@template_file)
+              return PDK::Util::Filesystem.read_file(@template_file)
+            end
+
+            raise ArgumentError, _("'%{template}' is not a readable file") % { template: @template_file }
+          end
+
+          # Renders the content of the template file as an ERB template.
+          #
+          # @return [String] The rendered template.
+          #
+          # @raise (see #template_content)
+          #
+          # @api private
+          def render_erb
+            renderer = ERB.new(template_content, nil, '-')
+            renderer.filename = @template_file
+            renderer.result(binding)
+          end
+
+          # Renders the content of the template file as plain text.
+          #
+          # @return [String] The rendered template.
+          #
+          # @raise (see #template_content)
+          #
+          # @api private
+          def render_plain
+            template_content
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pdk/template/renderer/v1.rb

@@ -0,0 +1,25 @@
+require 'pdk'
+
+module PDK
+  module Template
+    module Renderer
+      module V1
+        autoload :LegacyTemplateDir, 'pdk/template/renderer/v1/legacy_template_dir'
+        autoload :Renderer, 'pdk/template/renderer/v1/renderer'
+        autoload :TemplateFile, 'pdk/template/renderer/v1/template_file'
+
+        # Whether the template directory and context are valid for the V1 renderer
+        # @see PDK::Template::Renderer.instance
+        def self.compatible?(template_root, _context)
+          %w[moduleroot moduleroot_init].all? { |dir| PDK::Util::Filesystem.directory?(File.join(template_root, dir)) }
+        end
+
+        # Creates an instance of the V1 Renderer
+        # @see PDK::Template::Renderer.instance
+        def self.instance(template_root, template_uri, context)
+          PDK::Template::Renderer::V1::Renderer.new(template_root, template_uri, context)
+        end
+      end
+    end
+  end
+end

data/lib/pdk/template/renderer.rb

@@ -0,0 +1,96 @@
+require 'pdk'
+
+module PDK
+  module Template
+    module Renderer
+      autoload :V1, 'pdk/template/renderer/v1'
+
+      # Returns the most appropriate renderer for the given Template Directory and Context
+      #
+      # @param template_root [String] The path to where the template exists on disk
+      # @param template_uri [PDK::Util::TemplateUri] A URI which points to the source location of the Template
+      # @param context [PDK::Context] The context in which the redering will occur in
+      #
+      # @return [AbstractRenderer, nil] An instance of an AbstractRenderer subclass. Returns nil if no renderer could be found
+      def self.instance(template_uri, template_root, context)
+        return V1.instance(template_root, template_uri, context) if V1.compatible?(template_root, context)
+        nil
+      end
+
+      # An abstract class which all Template Renderers should subclass and implement. This class is responsible for
+      # rendering a template or a single item within a template directory
+      #
+      # To implement a new renderer:
+      # 1. Create a new class which subclasses AbstractRenderer and implements the public methods (has_single_item?, render and render_single_item)
+      # 2. Add class methods .compatible? and .instance which are used to detect if a template is compatible with the new renderer
+      #    and create an instance of the new renderer respectively
+      # 3. Update the PDK::Template::Renderer.instance method to detect and create an instance of the new renderer (using the .compatible? and .instance methods
+      #    created in step 2).
+      #
+      # See the PDK::Template::Renderer::V1 module and classes for an example on how to to this.
+      #
+      # @api private
+      # @abstract
+      class AbstractRenderer
+        # @return [String] The path to where the template exists on disk
+        attr_reader :template_root
+
+        # @return [PDK::Util::TemplateURI] The URI which points to the source location of the Template
+        attr_reader :template_uri
+
+        # @return context [PDK::Context] The context in which the redering will occur in
+        attr_reader :context
+
+        # @param template_root [String] The path to where the template exists on disk
+        # @param template_uri [PDK::Util::TemplateUri] A URI which points to the source location of the Template
+        # @param context [PDK::Context] The context in which the redering will occur in
+        def initialize(template_root, template_uri, context)
+          @template_root = template_root
+          @template_uri = template_uri
+          @context = context
+        end
+
+        # Whether the renderer supports rendering the a single item called 'item_path'. This is used when rendering things like a new Task or a new Puppet Classes.
+        # Rendering a single item is different than redering an entire project, like a entire Puppet Module or Control Repo.
+        #
+        # @param item_path [String] The path to the item to check
+        #
+        # @return [Boolean] Whether the renderer can render the item
+        # @abstract
+        def has_single_item?(_item_path) # rubocop:disable Naming/PredicateName Changing the method name to `single_item?` will convey the wrong intent
+          false
+        end
+
+        # Loop through the files in the template type, yielding each rendered file to the supplied block.
+        #
+        # @param template_type [PDK::Template::*_TEMPLATE_TYPE] The type of template to render
+        # @param name [String] The name to use in the rendering process
+        # @param options [Hash{Object => Object}] A list of options to pass through to the renderer. See PDK::Template::TemplateDir helper methods for other options
+        # @option options [Boolean] :include_first_time Whether to include "first time" items when rendering the project. While it is up to the renderer to implement this
+        #                                               the expected behavior is that if the item already exists, it will not be rendererd again.  Unlike normal items which
+        #                                               are always rendered to keep them in-sync
+        #
+        # @yieldparam dest_path [String] The path of the destination file, relative to the root of the context.
+        # @yieldparam dest_content [String] The rendered content of the destination file.
+        # @yieldparam dest_status [Symbol] :unmanage, :delete, :init, :manage
+        #
+        # @see PDK::Template::TemplateDir.render
+        #
+        # @return [void]
+        # @abstract
+        def render(template_type, name, options = {}); end
+
+        # Render a single item and return the resulting string. This is used when rendering things like a new Task or a new Puppet Classes.
+        # Rendering a single item is different than redering an entire project, like a entire Puppet Module or Control Repo. This method is
+        # used in conjunction with .has_single_item?
+        #
+        # @param item_path [String] The path of the single item to render
+        # @param template_data_hash [Hash{Object => Object}] A hash of information which will be used in the rendering process
+        #
+        # @return [String, Nil] The rendered content, or nil of the file could not be rendered
+        # @abstract
+        def render_single_item(item_path, template_data_hash = {}); end
+      end
+    end
+  end
+end

data/lib/pdk/template/template_dir.rb

@@ -0,0 +1,67 @@
+require 'pdk'
+require 'forwardable'
+
+module PDK
+  module Template
+    # A helper class representing an already fetched template on disk, with an appropriate renderer instance.
+    # @see PDK::Template.with
+    class TemplateDir
+      # Creates an instance of TemplateDir object
+      # @see TemplateDir.new
+      def self.instance(uri, path, context, renderer = nil)
+        new(uri, path, context, renderer)
+      end
+
+      extend Forwardable
+
+      # Helper methods for rendering
+      def_delegators :@renderer, :render, :render_single_item, :has_single_item?
+
+      # @return [PDK::Util::TemplateURI] The URI which points to the source location of the Template
+      attr_accessor :uri
+
+      # @return [String] The path to where the template exists on disk
+      attr_accessor :path
+
+      # @return [Hash{String => String}] A hash of information about the template
+      attr_accessor :metadata
+
+      # @param template_uri [PDK::Util::TemplateUri] A URI which points to the source location of the Template
+      # @param path [String] The path to where the template exists on disk
+      # @param context [PDK::Context] The context in which the redering will occur in
+      # @param renderer [PDK::Template::Renderer::AbstractRenderer] The an instance of a rendering class. If nil, a renderer will be created that's appropriate for the template and context
+      def initialize(uri, path, context, renderer = nil)
+        @uri = uri
+        @path = path
+        @metadata = {}
+
+        @renderer = renderer.nil? ? Renderer.instance(uri, path, context) : renderer
+        raise _('Could not find a compatible template renderer for %{path}') % { path: path } if @renderer.nil?
+      end
+
+      # Later additions may include Control Repo rendering, for example
+      #
+      # def render_control_repo(name, options = {})
+      #   render(CONTROL_REPO_TEMPLATE_TYPE, name, options.merge(include_first_time: false)) { |*args| yield(*args) }
+      # end
+      #
+      # def render_new_control_repo(name, repo_metadata = {}, options = {})
+      #   render(CONTROL_REPO_TEMPLATE_TYPE, name, options.merge(include_first_time: true, control_repo_metadata: repo_metadata)) { |*args| yield(*args) }
+      # end
+      #:nocov: These are just helper methods and are tested elsewhere.
+
+      # Render an existing module
+      # @see PDK::Template::Renderer::AbstractRenderer.render
+      def render_module(module_name, options = {})
+        @renderer.render(MODULE_TEMPLATE_TYPE, module_name, options.merge(include_first_time: false)) { |*args| yield(*args) }
+      end
+
+      # Render a new module
+      # @see PDK::Template::Renderer::AbstractRenderer.render
+      def render_new_module(module_name, module_metadata = {}, options = {})
+        @renderer.render(MODULE_TEMPLATE_TYPE, module_name, options.merge(include_first_time: true, module_metadata: module_metadata)) { |*args| yield(*args) }
+      end
+      #:nocov:
+    end
+  end
+end

data/lib/pdk/template.rb

@@ -0,0 +1,59 @@
+require 'pdk'
+
+module PDK
+  module Template
+    autoload :Fetcher, 'pdk/template/fetcher'
+    autoload :Renderer, 'pdk/template/renderer'
+    autoload :TemplateDir, 'pdk/template/template_dir'
+
+    MODULE_TEMPLATE_TYPE = :module_template
+
+    # Creates a TemplateDir object with the path or URL to the template
+    # and the block of code to run to be run while the template is available.
+    #
+    # The template directory is only guaranteed to be available on disk
+    # within the scope of the block passed to this method.
+    #
+    # @param uri [PDK::Util::TemplateURI] The path to a directory to use as the
+    # template or a URI to a git repository.
+    #
+    # @param context [PDK::Context::AbstractContext] The context in which the template will render to
+    #
+    # @yieldparam self [PDK::Template::TemplateDir] The initialised object with
+    # the template available on disk.
+    #
+    # @example Using a git repository as a template
+    #   PDK::Template.with('https://github.com/puppetlabs/pdk-templates') do |t|
+    #     t.render_module('module, PDK.context) do |filename, content, status|
+    #       File.open(filename, 'w') do |file|
+    #         ...
+    #       end
+    #     end
+    #   end
+    #
+    # @raise [ArgumentError] If no block is given to this method.
+    # @raise [PDK::CLI::FatalError]
+    # @raise [ArgumentError]
+    #
+    # @api public
+    def self.with(uri, context)
+      unless block_given?
+        raise ArgumentError, _('%{class_name}.with must be passed a block.') % { class_name: name }
+      end
+      unless uri.is_a? PDK::Util::TemplateURI
+        raise ArgumentError, _('%{class_name}.with must be passed a PDK::Util::TemplateURI, got a %{uri_type}') % { uri_type: uri.class, class_name: name }
+      end
+
+      Fetcher.with(uri) do |fetcher|
+        template_dir = TemplateDir.instance(uri, fetcher.path, context)
+        template_dir.metadata = fetcher.metadata
+
+        template_type = uri.default? ? 'default' : 'custom'
+        PDK.analytics.event('TemplateDir', 'initialize', label: template_type)
+
+        yield template_dir
+      end
+      nil
+    end
+  end
+end

data/lib/pdk/tests/unit.rb

@@ -89,6 +89,11 @@ module PDK
         setup
 
         tests = options[:tests]
+        # Due to how rake handles paths in the command line options, any backslashed path (Windows platforms) needs to be converted
+        # to forward slash. We can't use File.expand_path as the files aren't guaranteed to be on-disk
+        #
+        # Ref - https://github.com/puppetlabs/pdk/issues/828
+        tests.tr!('\\', '/') unless tests.nil?
 
         environment = { 'CI_SPEC_OPTIONS' => '--format j' }
         environment['PUPPET_GEM_VERSION'] = options[:puppet] if options[:puppet]