pdk-akerl 1.8.0.1

data/lib/pdk/module/templatedir.rb

@@ -0,0 +1,313 @@
+require 'yaml'
+require 'deep_merge'
+require 'pdk/util'
+require 'pdk/util/git'
+require 'pdk/cli/errors'
+require 'pdk/template_file'
+
+module PDK
+  module Module
+    class TemplateDir
+      attr_accessor :module_metadata
+
+      # Initialises the TemplateDir object with the path or URL to the template
+      # and the block of code to run to be run while the template is available.
+      #
+      # The template directory is only guaranteed to be available on disk
+      # within the scope of the block passed to this method.
+      #
+      # @param path_or_url [String] The path to a directory to use as the
+      # template or a URL to a git repository.
+      # @param module_metadata [Hash] A Hash containing the module metadata.
+      # Defaults to an empty Hash.
+      # @yieldparam self [PDK::Module::TemplateDir] The initialised object with
+      # the template available on disk.
+      #
+      # @example Using a git repository as a template
+      #   PDK::Module::TemplateDir.new('https://github.com/puppetlabs/pdk-templates') do |t|
+      #     t.render do |filename, content|
+      #       File.open(filename, 'w') do |file|
+      #         file.write(content)
+      #       end
+      #     end
+      #   end
+      #
+      # @raise [ArgumentError] If no block is given to this method.
+      # @raise [PDK::CLI::FatalError] (see #clone_repo)
+      # @raise [ArgumentError] (see #validate_module_template!)
+      #
+      # @api public
+      def initialize(path_or_url, module_metadata = {}, init = false)
+        unless block_given?
+          raise ArgumentError, _('%{class_name} must be initialized with a block.') % { class_name: self.class.name }
+        end
+
+        if PDK::Util::Git.repo?(path_or_url)
+          @path = self.class.clone_template_repo(path_or_url)
+          @repo = path_or_url
+        else
+          @path = path_or_url
+        end
+
+        @init = init
+        @moduleroot_dir = File.join(@path, 'moduleroot')
+        @moduleroot_init = File.join(@path, 'moduleroot_init')
+        @dirs = [@moduleroot_dir]
+        @dirs << @moduleroot_init if @init
+        @object_dir = File.join(@path, 'object_templates')
+
+        validate_module_template!
+
+        @module_metadata = module_metadata
+
+        yield self
+      ensure
+        # If we cloned a git repo to get the template, remove the clone once
+        # we're done with it.
+        if @repo
+          FileUtils.remove_dir(@path)
+        end
+      end
+
+      # Retrieve identifying metadata for the template.
+      #
+      # For git repositories, this will return the URL to the repository and
+      # a reference to the HEAD.
+      #
+      # @return [Hash{String => String}] A hash of identifying metadata.
+      #
+      # @api public
+      def metadata
+        result = {
+          'pdk-version' => PDK::Util::Version.version_string,
+        }
+
+        result['template-url'] = @repo ? @repo : @path
+
+        ref_result = PDK::Util::Git.git('--git-dir', File.join(@path, '.git'), 'describe', '--all', '--long', '--always')
+        result['template-ref'] = ref_result[:stdout].strip if ref_result[:exit_code].zero?
+
+        result
+      end
+
+      # Loop through the files in the template, yielding each rendered file to
+      # the supplied block.
+      #
+      # @yieldparam dest_path [String] The path of the destination file,
+      # relative to the root of the module.
+      # @yieldparam dest_content [String] The rendered content of the
+      # destination file.
+      #
+      # @raise [PDK::CLI::FatalError] If the template fails to render.
+      #
+      # @return [void]
+      #
+      # @api public
+      def render
+        PDK::Module::TemplateDir.files_in_template(@dirs).each do |template_file, template_loc|
+          template_file = template_file.to_s
+          PDK.logger.debug(_("Rendering '%{template}'...") % { template: template_file })
+          dest_path = template_file.sub(%r{\.erb\Z}, '')
+          config = config_for(dest_path)
+          dest_status = :manage
+
+          if config['unmanaged']
+            dest_status = :unmanage
+          elsif config['delete']
+            dest_status = :delete
+          else
+            begin
+              dest_content = PDK::TemplateFile.new(File.join(template_loc, template_file), configs: config).render
+            rescue => e
+              error_msg = _(
+                "Failed to render template '%{template}'\n" \
+                '%{exception}: %{message}',
+              ) % { template: template_file, exception: e.class, message: e.message }
+              raise PDK::CLI::FatalError, error_msg
+            end
+          end
+
+          yield dest_path, dest_content, dest_status
+        end
+      end
+
+      # Searches the template directory for template files that can be used to
+      # render files for the specified object type.
+      #
+      # @param object_type [Symbol] The object type, e.g. (`:class`,
+      # `:defined_type`, `:fact`, etc).
+      #
+      # @return [Hash{Symbol => String}] if the templates are available in the
+      # template dir, otherwise `nil`. The returned hash can contain two keys,
+      # :object contains the path on disk to the template for the object, :spec
+      # contains the path on disk to the template for the object's spec file
+      # (if available).
+      #
+      # @api public
+      def object_template_for(object_type)
+        object_path = File.join(@object_dir, "#{object_type}.erb")
+        type_path = File.join(@object_dir, "#{object_type}_type.erb")
+        spec_path = File.join(@object_dir, "#{object_type}_spec.erb")
+        type_spec_path = File.join(@object_dir, "#{object_type}_type_spec.erb")
+
+        if File.file?(object_path) && File.readable?(object_path)
+          result = { object: object_path }
+          result[:type] = type_path if File.file?(type_path) && File.readable?(type_path)
+          result[:spec] = spec_path if File.file?(spec_path) && File.readable?(spec_path)
+          result[:type_spec] = type_spec_path if File.file?(type_spec_path) && File.readable?(type_spec_path)
+          result
+        else
+          nil
+        end
+      end
+
+      # Generate a hash of data to be used when rendering object templates.
+      #
+      # Read `config_defaults.yml` from the root of the template directory (if
+      # it exists) build a hash of values from the value of the `:global`
+      # key.
+      #
+      # @return [Hash] The data that will be available to the template via the
+      # `@configs` instance variable.
+      #
+      # @api private
+      def object_config
+        config_for(nil)
+      end
+
+      # Validate the content of the template directory.
+      #
+      # @raise [ArgumentError] If the specified path is not a directory.
+      # @raise [ArgumentError] If the template directory does not contain
+      # a directory called 'moduleroot'.
+      #
+      # @return [void]
+      #
+      # @api private
+      def validate_module_template!
+        # rubocop:disable Style/GuardClause
+        unless File.directory?(@path)
+          if PDK::Util.package_install? && File.fnmatch?(File.join(PDK::Util.package_cachedir, '*'), @path)
+            raise ArgumentError, _('The built-in template has substantially changed. Please run "pdk convert" on your module to continue.')
+          else
+            raise ArgumentError, _("The specified template '%{path}' is not a directory.") % { path: @path }
+          end
+        end
+
+        unless File.directory?(@moduleroot_dir)
+          raise ArgumentError, _("The template at '%{path}' does not contain a 'moduleroot/' directory.") % { path: @path }
+        end
+
+        unless File.directory?(@moduleroot_init)
+          # rubocop:disable Metrics/LineLength
+          raise ArgumentError, _("The template at '%{path}' does not contain a 'moduleroot_init/' directory, which indicates you are using an older style of template. Before continuing please use the --template-url flag when running the pdk new commands to pass a new style template.") % { path: @path }
+          # rubocop:enable Metrics/LineLength Style/GuardClause
+        end
+      end
+
+      # Get a list of template files in the template directory.
+      #
+      # @return [Hash{String=>String}] A hash of key file names and
+      # value locations.
+      #
+      # @api public
+      def self.files_in_template(dirs)
+        temp_paths = []
+        dirlocs = []
+        dirs.each do |dir|
+          raise ArgumentError, _("The directory '%{dir}' doesn't exist") % { dir: dir } unless Dir.exist?(dir)
+          temp_paths += Dir.glob(File.join(dir, '**', '*'), File::FNM_DOTMATCH).select do |template_path|
+            if File.file?(template_path) && !File.symlink?(template_path)
+              dirlocs << dir
+            end
+          end
+          temp_paths.map do |template_path|
+            template_path.sub!(%r{\A#{Regexp.escape(dir)}#{Regexp.escape(File::SEPARATOR)}}, '')
+          end
+        end
+        Hash[temp_paths.zip dirlocs]
+      end
+
+      # Generate a hash of data to be used when rendering the specified
+      # template.
+      #
+      # @param dest_path [String] The destination path of the file that the
+      # data is for, relative to the root of the module.
+      #
+      # @return [Hash] The data that will be available to the template via the
+      # `@configs` instance variable.
+      #
+      # @api private
+      def config_for(dest_path, sync_config_path = nil)
+        module_root = PDK::Util.module_root
+        sync_config_path ||= File.join(module_root, '.sync.yml') unless module_root.nil?
+        config_path = File.join(@path, 'config_defaults.yml')
+
+        if @config.nil?
+          conf_defaults = read_config(config_path)
+          sync_config = read_config(sync_config_path) unless sync_config_path.nil?
+          @config = conf_defaults
+          @config.deep_merge!(sync_config, knockout_prefix: '---') unless sync_config.nil?
+        end
+        file_config = @config.fetch(:global, {})
+        file_config['module_metadata'] = @module_metadata
+        file_config.merge!(@config.fetch(dest_path, {})) unless dest_path.nil?
+        file_config.merge!(@config)
+      end
+
+      # Generates a hash of data from a given yaml file location.
+      #
+      # @param loc [String] The path of the yaml config file.
+      #
+      # @warn If the specified path is not a valid yaml file. Returns an empty Hash
+      # if so.
+      #
+      # @return [Hash] The data that has been read in from the given yaml file.
+      #
+      # @api private
+      def read_config(loc)
+        if File.file?(loc) && File.readable?(loc)
+          begin
+            YAML.safe_load(File.read(loc), [], [], true)
+          rescue StandardError => e
+            PDK.logger.warn(_("'%{file}' is not a valid YAML file: %{message}") % { file: loc, message: e.message })
+            {}
+          end
+        else
+          {}
+        end
+      end
+
+      # @return [String] Path to working directory into which template repo has been cloned and reset
+      #
+      # @raise [PDK::CLI::FatalError] If unable to clone the given origin_repo into a tempdir.
+      # @raise [PDK::CLI::FatalError] If reset HEAD of the cloned repo to desired ref.
+      #
+      # @api private
+      def self.clone_template_repo(origin_repo)
+        # @todo When switching this over to using rugged, cache the cloned
+        # template repo in `%AppData%` or `$XDG_CACHE_DIR` and update before
+        # use.
+        temp_dir = PDK::Util.make_tmpdir_name('pdk-templates')
+        git_ref = (origin_repo == PDK::Util.default_template_url) ? PDK::Util.default_template_ref : 'origin/master'
+
+        clone_result = PDK::Util::Git.git('clone', origin_repo, temp_dir)
+
+        if clone_result[:exit_code].zero?
+          reset_result = PDK::Util::Git.git('-C', temp_dir, 'reset', '--hard', git_ref)
+          unless reset_result[:exit_code].zero?
+            PDK.logger.error reset_result[:stdout]
+            PDK.logger.error reset_result[:stderr]
+            raise PDK::CLI::FatalError, _("Unable to set HEAD of git repository at '%{repo}' to ref:'%{ref}'.") % { repo: temp_dir, ref: git_ref }
+          end
+        else
+          PDK.logger.error clone_result[:stdout]
+          PDK.logger.error clone_result[:stderr]
+          raise PDK::CLI::FatalError, _("Unable to clone git repository at '%{repo}' into '%{dest}'.") % { repo: origin_repo, dest: temp_dir }
+        end
+
+        PDK::Util.canonical_path(temp_dir)
+      end
+    end
+  end
+end

data/lib/pdk/module/update.rb

@@ -0,0 +1,111 @@
+require 'pdk/module/convert'
+
+module PDK
+  module Module
+    class Update < Convert
+      GIT_DESCRIBE_PATTERN = %r{\A(?<base>.+?)-(?<additional_commits>\d+)-g(?<sha>.+)\Z}
+
+      def run
+        stage_changes!
+
+        if current_version == new_version
+          PDK.logger.debug _('This module is already up to date with version %{version} of the template.') % {
+            version: new_version,
+          }
+        end
+
+        unless update_manager.changes?
+          PDK::Report.default_target.puts(_('No changes required.'))
+          return
+        end
+
+        PDK.logger.info(update_message)
+
+        print_summary
+        full_report('update_report.txt') unless update_manager.changes[:modified].empty?
+
+        return if noop?
+
+        unless force?
+          message = _('Do you want to continue and make these changes to your module?')
+          return unless PDK::CLI::Util.prompt_for_yes(message)
+        end
+
+        # Remove these files straight away as these changes are not something that the user needs to review.
+        if needs_bundle_update?
+          update_manager.unlink_file('Gemfile.lock')
+          update_manager.unlink_file(File.join('.bundle', 'config'))
+        end
+
+        update_manager.sync_changes!
+
+        PDK::Util::Bundler.ensure_bundle! if needs_bundle_update?
+
+        print_result 'Update completed'
+      end
+
+      def module_metadata
+        @module_metadata ||= PDK::Module::Metadata.from_file('metadata.json')
+      rescue ArgumentError => e
+        raise PDK::CLI::ExitWithError, e.message
+      end
+
+      def template_url
+        @template_url ||= module_metadata.data['template-url']
+      end
+
+      def current_version
+        @current_version ||= describe_ref_to_s(current_template_version)
+      end
+
+      def new_version
+        @new_version ||= fetch_remote_version(new_template_version)
+      end
+
+      private
+
+      def current_template_version
+        @current_template_version ||= module_metadata.data['template-ref']
+      end
+
+      def describe_ref_to_s(describe_ref)
+        data = GIT_DESCRIBE_PATTERN.match(describe_ref)
+
+        return data if data.nil?
+
+        if data[:base].start_with?('heads/')
+          "#{data[:base].gsub(%r{^heads/}, '')}@#{data[:sha]}"
+        else
+          data[:base]
+        end
+      end
+
+      def new_template_version
+        PDK::Util.default_template_ref
+      end
+
+      def fetch_remote_version(version)
+        return version unless version.include?('/')
+
+        branch = version.partition('/').last
+        sha_length = GIT_DESCRIBE_PATTERN.match(current_template_version)[:sha].length - 1
+        "#{branch}@#{PDK::Util::Git.ls_remote(template_url, "refs/heads/#{branch}")[0..sha_length]}"
+      end
+
+      def update_message
+        format_string = if template_url == PDK::Util.puppetlabs_template_url
+                          _('Updating %{module_name} using the default template, from %{current_version} to %{new_version}')
+                        else
+                          _('Updating %{module_name} using the template at %{template_url}, from %{current_version} to %{new_version}')
+                        end
+
+        format_string % {
+          module_name:     module_metadata.data['name'],
+          template_url:    template_url,
+          current_version: current_version,
+          new_version:     new_version,
+        }
+      end
+    end
+  end
+end

data/lib/pdk/module/update_manager.rb

@@ -0,0 +1,210 @@
+require 'diff/lcs'
+require 'diff/lcs/hunk'
+require 'English'
+require 'fileutils'
+require 'set'
+require 'pdk/util/filesystem'
+
+module PDK
+  module Module
+    class UpdateManager
+      # Initialises a blank UpdateManager object, which is used to store and
+      # process file additions/removals/modifications.
+      def initialize
+        @modified_files = Set.new
+        @added_files = Set.new
+        @removed_files = Set.new
+        @diff_cache = {}
+      end
+
+      # Store a pending modification to an existing file.
+      #
+      # @param path [String] The path to the file to be modified.
+      # @param content [String] The new content of the file.
+      def modify_file(path, content)
+        @modified_files << { path: path, content: content }
+      end
+
+      # Store a pending file addition.
+      #
+      # @param path [String] The path where the file will be created.
+      # @param content [String] The content of the new file.
+      def add_file(path, content)
+        @added_files << { path: path, content: content }
+      end
+
+      # Store a pending file removal.
+      #
+      # @param path [String] The path to the file to be removed.
+      def remove_file(path)
+        @removed_files << path
+      end
+
+      # Generate a summary of the changes that will be applied to the module.
+      #
+      # @raise (see #calculate_diffs)
+      # @return [Hash{Symbol => Set,Hash}] the summary of the pending changes.
+      def changes
+        calculate_diffs
+
+        {
+          added:    @added_files,
+          removed:  @removed_files.select { |f| File.exist?(f) },
+          modified: @diff_cache.reject { |_, value| value.nil? },
+        }
+      end
+
+      # Check if there are any pending changes to apply to the module.
+      #
+      # @raise (see #changes)
+      # @return [Boolean] true if there are changes to apply to the module.
+      def changes?
+        !changes[:added].empty? ||
+          !changes[:removed].empty? ||
+          changes[:modified].any? { |_, value| !value.nil? }
+      end
+
+      # Check if the update manager will change the specified file upon sync.
+      #
+      # @param path [String] The path to the file.
+      #
+      # @raise (see #changes)
+      # @return [Boolean] true if the file will be changed.
+      def changed?(path)
+        changes[:added].any? { |add| add[:path] == path } ||
+          changes[:removed].include?(path) ||
+          changes[:modified].keys.include?(path)
+      end
+
+      # Apply any pending changes stored in the UpdateManager to the module.
+      #
+      # @raise (see #calculate_diffs)
+      # @raise (see #write_file)
+      # @raise (see #unlink_file)
+      def sync_changes!
+        calculate_diffs
+
+        files_to_write = @added_files
+        files_to_write += @modified_files.reject { |file| @diff_cache[file[:path]].nil? }
+
+        @removed_files.each do |file|
+          unlink_file(file)
+        end
+
+        files_to_write.each do |file|
+          write_file(file[:path], file[:content])
+        end
+      end
+
+      # Remove a file from disk.
+      #
+      # Like FileUtils.rm_f, this method will not fail if the file does not
+      # exist. Unlike FileUtils.rm_f, this method will not blindly swallow all
+      # exceptions.
+      #
+      # @param path [String] The path to the file to be removed.
+      #
+      # @raise [PDK::CLI::ExitWithError] if the file could not be removed.
+      def unlink_file(path)
+        if File.file?(path)
+          PDK.logger.debug(_("unlinking '%{path}'") % { path: path })
+          FileUtils.rm(path)
+        else
+          PDK.logger.debug(_("'%{path}': already gone") % { path: path })
+        end
+      rescue => e
+        raise PDK::CLI::ExitWithError, _("Unable to remove '%{path}': %{message}") % {
+          path:    path,
+          message: e.message,
+        }
+      end
+
+      private
+
+      # Loop through all the files to be modified and cache of unified diff of
+      # the changes to be made to each file.
+      #
+      # @raise [PDK::CLI::ExitWithError] if a file being modified isn't
+      #   readable.
+      def calculate_diffs
+        @modified_files.each do |file|
+          next if @diff_cache.key?(file[:path])
+
+          unless File.readable?(file[:path])
+            raise PDK::CLI::ExitWithError, _("Unable to open '%{path}' for reading") % { path: file[:path] }
+          end
+
+          old_content = File.read(file[:path])
+          file_diff = unified_diff(file[:path], old_content, file[:content])
+          @diff_cache[file[:path]] = file_diff
+        end
+      end
+
+      # Write or overwrite a file with the specified content.
+      #
+      # @param path [String] The path to be written to.
+      # @param content [String] The data to be written to the file.
+      #
+      # @raise [PDK::CLI::ExitWithError] if the file is not writeable.
+      def write_file(path, content)
+        FileUtils.mkdir_p(File.dirname(path))
+        PDK.logger.debug(_("writing '%{path}'") % { path: path })
+        PDK::Util::Filesystem.write_file(path, content)
+      rescue Errno::EACCES
+        raise PDK::CLI::ExitWithError, _("You do not have permission to write to '%{path}'") % { path: path }
+      end
+
+      # Generate a unified diff of the changes to be made to a file.
+      #
+      # @param path [String] The path to the file being diffed (only used to
+      #   generate the diff header).
+      # @param old_content [String] The current content of the file.
+      # @param new_content [String] The new content of the file if the pending
+      #   change is applied.
+      # @param lines_of_context [Integer] The maximum number of lines of
+      #   context to include around the changed lines in the diff output
+      #   (default: 3).
+      #
+      # @return [String] The unified diff of the pending changes to the file.
+      def unified_diff(path, old_content, new_content, lines_of_context = 3)
+        output = []
+
+        old_lines = old_content.split($INPUT_RECORD_SEPARATOR).map(&:chomp)
+        new_lines = new_content.split($INPUT_RECORD_SEPARATOR).map(&:chomp)
+
+        diffs = Diff::LCS.diff(old_lines, new_lines)
+
+        return nil if diffs.empty?
+
+        file_mtime = File.stat(path).mtime.localtime.strftime('%Y-%m-%d %H:%M:%S.%N %z')
+        now = Time.now.localtime.strftime('%Y-%m-%d %H:%M:%S.%N %z')
+
+        output << "--- #{path}\t#{file_mtime}"
+        output << "+++ #{path}.pdknew\t#{now}"
+
+        oldhunk = hunk = nil
+        file_length_difference = 0
+
+        diffs.each do |piece|
+          begin
+            hunk = Diff::LCS::Hunk.new(old_lines, new_lines, piece, lines_of_context, file_length_difference)
+            file_length_difference = hunk.file_length_difference
+
+            next unless oldhunk
+
+            # If the hunk overlaps with the oldhunk, merge them.
+            next if lines_of_context > 0 && hunk.merge(oldhunk)
+
+            output << oldhunk.diff(:unified)
+          ensure
+            oldhunk = hunk
+          end
+        end
+
+        output << oldhunk.diff(:unified)
+
+        output.join($INPUT_RECORD_SEPARATOR)
+      end
+    end
+  end
+end