pdk 1.2.1 → 1.3.0

data/lib/pdk/{generators → generate}/puppet_class.rb

@@ -1,4 +1,4 @@
-require 'pdk/generators/puppet_object'
+require 'pdk/generate/puppet_object'
 
 module PDK
   module Generate

data/lib/pdk/{generators → generate}/puppet_object.rb

@@ -223,7 +223,7 @@ module PDK
         @templates ||= [
           { type: 'CLI', url: @options[:'template-url'], allow_fallback: false },
           { type: 'metadata', url: module_metadata.data['template-url'], allow_fallback: true },
-          { type: 'default', url: PDK::Generate::Module.default_template_url, allow_fallback: false },
+          { type: 'default', url: PDK::Util.default_template_url, allow_fallback: false },
         ]
       end
 

data/lib/pdk/{generators → generate}/task.rb

@@ -1,4 +1,4 @@
-require 'pdk/generators/puppet_object'
+require 'pdk/generate/puppet_object'
 
 module PDK
   module Generate

data/lib/pdk/module/convert.rb

@@ -0,0 +1,163 @@
+require 'pdk/generate/module'
+require 'pdk/module/update_manager'
+require 'pdk/util'
+require 'pdk/report'
+
+module PDK
+  module Module
+    class Convert
+      def self.invoke(options)
+        update_manager = PDK::Module::UpdateManager.new
+        template_url = options.fetch(:'template-url', PDK::Util.default_template_url)
+
+        PDK::Module::TemplateDir.new(template_url, nil, false) do |templates|
+          new_metadata = update_metadata('metadata.json', templates.metadata, options)
+
+          if options[:noop] && new_metadata.nil?
+            update_manager.add_file('metadata.json', '')
+          elsif File.file?('metadata.json')
+            update_manager.modify_file('metadata.json', new_metadata)
+          else
+            update_manager.add_file('metadata.json', new_metadata)
+          end
+
+          templates.render do |file_path, file_content|
+            if File.exist? file_path
+              update_manager.modify_file(file_path, file_content)
+            else
+              update_manager.add_file(file_path, file_content)
+            end
+          end
+        end
+
+        unless update_manager.changes?
+          PDK::Report.default_target.puts(_('No changes required.'))
+          return
+        end
+
+        # Print the summary to the default target of reports
+        summary = get_summary(update_manager)
+        print_summary(summary)
+
+        # Generates the full convert report
+        full_report(update_manager) unless update_manager.changes[:modified].empty?
+
+        return if options[:noop]
+
+        unless options[:force]
+          PDK.logger.info _(
+            'Module conversion is a potentially destructive action. ' \
+            'Ensure that you have committed your module to a version control ' \
+            'system or have a backup, and review the changes above before continuing.',
+          )
+          continue = PDK::CLI::Util.prompt_for_yes(_('Do you want to continue and make these changes to your module?'))
+          return unless continue
+        end
+
+        # Mark these files for removal after generating the report as these
+        # changes are not something that the user needs to review.
+        if update_manager.changed?('Gemfile')
+          update_manager.remove_file('Gemfile.lock')
+          update_manager.remove_file(File.join('.bundle', 'config'))
+        end
+
+        update_manager.sync_changes!
+
+        PDK::Util::Bundler.ensure_bundle! if update_manager.changed?('Gemfile')
+
+        print_result(summary)
+      end
+
+      def self.update_metadata(metadata_path, template_metadata, options = {})
+        if File.file?(metadata_path)
+          if File.readable?(metadata_path)
+            begin
+              metadata = PDK::Module::Metadata.from_file(metadata_path)
+              new_values = PDK::Module::Metadata::DEFAULTS.reject { |key, _| metadata.data.key?(key) }
+              metadata.update!(new_values)
+            rescue ArgumentError
+              metadata = PDK::Generate::Module.prepare_metadata(options) unless options[:noop] # rubocop:disable Metrics/BlockNesting
+            end
+          else
+            raise PDK::CLI::ExitWithError, _('Unable to convert module metadata; %{path} exists but it is not readable.') % {
+              path: metadata_path,
+            }
+          end
+        elsif File.exist?(metadata_path)
+          raise PDK::CLI::ExitWithError, _('Unable to convert module metadata; %{path} exists but it is not a file.') % {
+            path: metadata_path,
+          }
+        else
+          return nil if options[:noop]
+
+          project_dir = File.basename(Dir.pwd)
+          options[:module_name] = project_dir.split('-', 2).compact[-1]
+          options[:prompt] = false
+          options[:'skip-interview'] = true if options[:force]
+
+          metadata = PDK::Generate::Module.prepare_metadata(options)
+        end
+
+        metadata.update!(template_metadata)
+        metadata.to_json
+      end
+
+      def self.get_summary(update_manager)
+        summary = {}
+        update_manager.changes.each do |category, update_category|
+          updated_files = if update_category.respond_to?(:keys)
+                            update_category.keys
+                          else
+                            update_category.map { |file| file[:path] }
+                          end
+
+          summary[category] = updated_files
+        end
+
+        summary
+      end
+
+      def self.print_summary(summary)
+        footer = false
+
+        summary.keys.each do |category|
+          next if summary[category].empty?
+
+          PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner("Files to be #{category}", 40) })
+          PDK::Report.default_target.puts(summary[category])
+          footer = true
+        end
+
+        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner('', 40) }) if footer
+      end
+
+      def self.print_result(summary)
+        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner('Convert completed', 40) })
+        summary_to_print = summary.map { |k, v| "#{v.length} files #{k}" unless v.empty? }.compact
+        PDK::Report.default_target.puts(_("\n%{summary}\n\n") % { summary: "#{summary_to_print.join(', ')}." })
+      end
+
+      def self.full_report(update_manager)
+        File.open('convert_report.txt', 'w') do |f|
+          f.write("/* Convert Report generated by PDK at #{Time.now} */")
+          update_manager.changes[:modified].each do |_, diff|
+            f.write("\n\n\n" + diff)
+          end
+        end
+        PDK::Report.default_target.puts(_("\nYou can find a report of differences in convert_report.txt.\n\n"))
+      end
+
+      def self.generate_banner(text, width = 80)
+        padding = width - text.length
+        banner = ''
+        padding_char = '-'
+
+        (padding / 2.0).ceil.times { banner << padding_char }
+        banner << text
+        (padding / 2.0).floor.times { banner << padding_char }
+
+        banner
+      end
+    end
+  end
+end

data/lib/pdk/module/metadata.rb

@@ -7,14 +7,14 @@ module PDK
 
       DEFAULTS = {
         'name'          => nil,
-        'version'       => nil,
+        'version'       => '0.1.0',
         'author'        => nil,
         'summary'       => '',
         'license'       => 'Apache-2.0',
         'source'        => '',
         'project_page'  => nil,
         'issues_url'    => nil,
-        'dependencies'  => Set.new.freeze,
+        'dependencies'  => [],
         'data_provider' => nil,
         'operatingsystem_support' => [
           {
@@ -34,7 +34,9 @@ module PDK
             'operatingsystemrelease' => ['2012 R2'],
           },
         ],
-        'requirements' => Set.new.freeze,
+        'requirements' => [
+          { 'name' => 'puppet', 'version_requirement' => '>= 4.7.0 < 6.0.0' },
+        ],
       }.freeze
 
       def initialize(params = {})
@@ -71,6 +73,12 @@ module PDK
         JSON.pretty_generate(@data.dup.delete_if { |_key, value| value.nil? })
       end
 
+      def write!(path)
+        File.open(path, 'w') do |file|
+          file.puts to_json
+        end
+      end
+
       private
 
       # Do basic validation and parsing of the name parameter.

data/lib/pdk/module/templatedir.rb

@@ -1,6 +1,7 @@
 require 'yaml'
+require 'deep_merge'
 require 'pdk/util'
-require 'pdk/cli/exec'
+require 'pdk/util/git'
 require 'pdk/cli/errors'
 require 'pdk/template_file'
 
@@ -21,7 +22,7 @@ module PDK
       # the template available on disk.
       #
       # @example Using a git repository as a template
-      #   PDK::Module::TemplateDir.new('https://github.com/puppetlabs/pdk-module-template') do |t|
+      #   PDK::Module::TemplateDir.new('https://github.com/puppetlabs/pdk-templates') do |t|
       #     t.render do |filename, content|
       #       File.open(filename, 'w') do |file|
       #         file.write(content)
@@ -36,7 +37,7 @@ module PDK
       # @raise [ArgumentError] (see #validate_module_template!)
       #
       # @api public
-      def initialize(path_or_url, module_metadata = {})
+      def initialize(path_or_url, module_metadata = {}, init = false)
         if File.directory?(path_or_url)
           @path = path_or_url
         else
@@ -46,10 +47,19 @@ module PDK
           # @todo When switching this over to using rugged, cache the cloned
           # template repo in `%AppData%` or `$XDG_CACHE_DIR` and update before
           # use.
-          temp_dir = PDK::Util.make_tmpdir_name('pdk-module-template')
+          temp_dir = PDK::Util.make_tmpdir_name('pdk-templates')
+          git_ref = PDK::Util.default_template_ref
 
-          clone_result = PDK::CLI::Exec.git('clone', path_or_url, temp_dir)
-          unless clone_result[:exit_code].zero?
+          clone_result = PDK::Util::Git.git('clone', path_or_url, temp_dir)
+
+          if clone_result[:exit_code].zero?
+            reset_result = PDK::Util::Git.git('-C', temp_dir, 'reset', '--hard', git_ref)
+            unless reset_result[:exit_code].zero?
+              PDK.logger.error reset_result[:stdout]
+              PDK.logger.error reset_result[:stderr]
+              raise PDK::CLI::FatalError, _("Unable to set git repository '%{repo}' to ref:'%{ref}'.") % { repo: temp_dir, ref: git_ref }
+            end
+          else
             PDK.logger.error clone_result[:stdout]
             PDK.logger.error clone_result[:stderr]
             raise PDK::CLI::FatalError, _("Unable to clone git repository '%{repo}' to '%{dest}'.") % { repo: path_or_url, dest: temp_dir }
@@ -59,7 +69,11 @@ module PDK
           @repo = path_or_url
         end
 
+        @init = init
         @moduleroot_dir = File.join(@path, 'moduleroot')
+        @moduleroot_init = File.join(@path, 'moduleroot_init')
+        @dirs = [@moduleroot_dir]
+        @dirs << @moduleroot_init if @init
         @object_dir = File.join(@path, 'object_templates')
         validate_module_template!
 
@@ -85,7 +99,7 @@ module PDK
       def metadata
         return {} unless @repo
 
-        ref_result = PDK::CLI::Exec.git('--git-dir', File.join(@path, '.git'), 'describe', '--all', '--long', '--always')
+        ref_result = PDK::Util::Git.git('--git-dir', File.join(@path, '.git'), 'describe', '--all', '--long', '--always')
         if ref_result[:exit_code].zero?
           { 'template-url' => @repo, 'template-ref' => ref_result[:stdout].strip }
         else
@@ -107,12 +121,12 @@ module PDK
       #
       # @api public
       def render
-        files_in_template.each do |template_file|
+        PDK::Module::TemplateDir.files_in_template(@dirs).each do |template_file, template_loc|
+          template_file = template_file.to_s
           PDK.logger.debug(_("Rendering '%{template}'...") % { template: template_file })
           dest_path = template_file.sub(%r{\.erb\Z}, '')
-
           begin
-            dest_content = PDK::TemplateFile.new(File.join(@moduleroot_dir, template_file), configs: config_for(dest_path)).render
+            dest_content = PDK::TemplateFile.new(File.join(template_loc, template_file), configs: config_for(dest_path)).render
           rescue => e
             error_msg = _(
               "Failed to render template '%{template}'\n" \
@@ -120,7 +134,6 @@ module PDK
             ) % { template: template_file, exception: e.class, message: e.message }
             raise PDK::CLI::FatalError, error_msg
           end
-
           yield dest_path, dest_content
         end
       end
@@ -165,8 +178,6 @@ module PDK
         config_for(nil)
       end
 
-      private
-
       # Validate the content of the template directory.
       #
       # @raise [ArgumentError] If the specified path is not a directory.
@@ -181,36 +192,46 @@ module PDK
           raise ArgumentError, _("The specified template '%{path}' is not a directory.") % { path: @path }
         end
 
-        unless File.directory?(@moduleroot_dir) # rubocop:disable Style/GuardClause
+        unless File.directory?(@moduleroot_dir)
           raise ArgumentError, _("The template at '%{path}' does not contain a 'moduleroot/' directory.") % { path: @path }
         end
+
+        unless File.directory?(@moduleroot_init) # rubocop:disable Style/GuardClause
+          # rubocop:disable Metrics/LineLength
+          raise ArgumentError, _("The template at '%{path}' does not contain a 'moduleroot_init/' directory, which indicates you are using an older style of template. Before continuing please use the --template-url flag when running the pdk new commands to pass a new style template.") % { path: @path }
+          # rubocop:enable Metrics/LineLength
+        end
       end
 
       # Get a list of template files in the template directory.
       #
-      # @return [Array[String]] An array of file names, relative to the
-      # `moduleroot` directory.
+      # @return [Hash{String=>String}] A hash of key file names and
+      # value locations.
       #
-      # @api private
-      def files_in_template
-        @files ||= begin
-          template_paths = Dir.glob(File.join(@moduleroot_dir, '**', '*'), File::FNM_DOTMATCH).select do |template_path|
+      # @api public
+      def self.files_in_template(dirs)
+        temp_paths = []
+        dirlocs = []
+        dirs.each do |dir|
+          raise ArgumentError, _("The directory '%{dir}' doesn't exist") unless Dir.exist?(dir)
+          temp_paths += Dir.glob(File.join(dir, '**', '*'), File::FNM_DOTMATCH).select do |template_path|
             File.file?(template_path) && !File.symlink?(template_path)
+            dirlocs << dir
           end
-
-          template_paths.map do |template_path|
-            template_path.sub(%r{\A#{Regexp.escape(@moduleroot_dir)}#{Regexp.escape(File::SEPARATOR)}}, '')
+          temp_paths.map do |template_path|
+            template_path.sub!(%r{\A#{Regexp.escape(dir)}#{Regexp.escape(File::SEPARATOR)}}, '')
           end
         end
+        template_paths = Hash[temp_paths.zip dirlocs]
+        template_paths.delete('.')
+        template_paths.delete('spec')
+        template_paths.delete('spec/.')
+        template_paths
       end
 
       # Generate a hash of data to be used when rendering the specified
       # template.
       #
-      # Read `config_defaults.yml` from the root of the template directory (if
-      # it exists) build a hash of values by merging the value of the `:global`
-      # key with the value of the key that matches `dest_path`.
-      #
       # @param dest_path [String] The destination path of the file that the
       # data is for, relative to the root of the module.
       #
@@ -218,26 +239,44 @@ module PDK
       # `@configs` instance variable.
       #
       # @api private
-      def config_for(dest_path)
+      def config_for(dest_path, sync_config_path = nil)
+        module_root = PDK::Util.module_root
+        sync_config_path ||= File.join(module_root, '.sync.yml') unless module_root.nil?
+        config_path = File.join(@path, 'config_defaults.yml')
+
         if @config.nil?
-          config_path = File.join(@path, 'config_defaults.yml')
-
-          if File.file?(config_path) && File.readable?(config_path)
-            begin
-              @config = YAML.safe_load(File.read(config_path), [], [], true)
-            rescue StandardError => e
-              PDK.logger.warn(_("'%{file}' is not a valid YAML file: %{message}") % { file: config_path, message: e.message })
-              @config = {}
-            end
-          else
-            @config = {}
-          end
+          conf_defaults = read_config(config_path)
+          sync_config = read_config(sync_config_path) unless sync_config_path.nil?
+          @config = conf_defaults
+          @config.deep_merge!(sync_config) unless sync_config.nil?
         end
-
         file_config = @config.fetch(:global, {})
         file_config['module_metadata'] = @module_metadata
         file_config.merge!(@config.fetch(dest_path, {})) unless dest_path.nil?
-        file_config
+        file_config.merge!(@config)
+      end
+
+      # Generates a hash of data from a given yaml file location.
+      #
+      # @param loc [String] The path of the yaml config file.
+      #
+      # @warn If the specified path is not a valid yaml file. Returns an empty Hash
+      # if so.
+      #
+      # @return [Hash] The data that has been read in from the given yaml file.
+      #
+      # @api private
+      def read_config(loc)
+        if File.file?(loc) && File.readable?(loc)
+          begin
+            YAML.safe_load(File.read(loc), [], [], true)
+          rescue StandardError => e
+            PDK.logger.warn(_("'%{file}' is not a valid YAML file: %{message}") % { file: config_path, message: e.message })
+            {}
+          end
+        else
+          {}
+        end
       end
     end
   end

data/lib/pdk/module/update_manager.rb

@@ -0,0 +1,203 @@
+require 'set'
+require 'diff/lcs'
+require 'diff/lcs/hunk'
+require 'English'
+require 'fileutils'
+
+module PDK
+  module Module
+    class UpdateManager
+      # Initialises a blank UpdateManager object, which is used to store and
+      # process file additions/removals/modifications.
+      def initialize
+        @modified_files = Set.new
+        @added_files = Set.new
+        @removed_files = Set.new
+        @diff_cache = {}
+      end
+
+      # Store a pending modification to an existing file.
+      #
+      # @param path [String] The path to the file to be modified.
+      # @param content [String] The new content of the file.
+      def modify_file(path, content)
+        @modified_files << { path: path, content: content }
+      end
+
+      # Store a pending file addition.
+      #
+      # @param path [String] The path where the file will be created.
+      # @param content [String] The content of the new file.
+      def add_file(path, content)
+        @added_files << { path: path, content: content }
+      end
+
+      # Store a pending file removal.
+      #
+      # @param path [String] The path to the file to be removed.
+      def remove_file(path)
+        @removed_files << path
+      end
+
+      # Generate a summary of the changes that will be applied to the module.
+      #
+      # @raise (see #calculate_diffs)
+      # @return [Hash{Symbol => Set,Hash}] the summary of the pending changes.
+      def changes
+        calculate_diffs
+
+        {
+          added:    @added_files,
+          removed:  @removed_files,
+          modified: @diff_cache.reject { |_, value| value.nil? },
+        }
+      end
+
+      # Check if there are any pending changes to apply to the module.
+      #
+      # @raise (see #changes)
+      # @return [Boolean] true if there are changes to apply to the module.
+      def changes?
+        !changes[:added].empty? ||
+          !changes[:removed].empty? ||
+          changes[:modified].any? { |_, value| !value.nil? }
+      end
+
+      # Check if the update manager will change the specified file upon sync.
+      #
+      # @param path [String] The path to the file.
+      #
+      # @raise (see #changes)
+      # @return [Boolean] true if the file will be changed.
+      def changed?(path)
+        changes[:added].any? { |add| add[:path] == path } ||
+          changes[:removed].include?(path) ||
+          changes[:modified].keys.include?(path)
+      end
+
+      # Apply any pending changes stored in the UpdateManager to the module.
+      #
+      # @raise (see #calculate_diffs)
+      # @raise (see #write_file)
+      # @raise (see #unlink_file)
+      def sync_changes!
+        calculate_diffs
+
+        files_to_write = @added_files
+        files_to_write += @modified_files.reject { |file| @diff_cache[file[:path]].nil? }
+
+        files_to_write.each do |file|
+          write_file(file[:path], file[:content])
+        end
+
+        @removed_files.each do |file|
+          unlink_file(file)
+        end
+      end
+
+      private
+
+      # Loop through all the files to be modified and cache of unified diff of
+      # the changes to be made to each file.
+      #
+      # @raise [PDK::CLI::ExitWithError] if a file being modified isn't
+      #   readable.
+      def calculate_diffs
+        @modified_files.each do |file|
+          next if @diff_cache.key?(file[:path])
+
+          unless File.readable?(file[:path])
+            raise PDK::CLI::ExitWithError, _("Unable to open '%{path}' for reading") % { path: file[:path] }
+          end
+
+          old_content = File.read(file[:path])
+          file_diff = unified_diff(file[:path], old_content, file[:content])
+          @diff_cache[file[:path]] = file_diff
+        end
+      end
+
+      # Write or overwrite a file with the specified content.
+      #
+      # @param path [String] The path to be written to.
+      # @param content [String] The data to be written to the file.
+      #
+      # @raise [PDK::CLI::ExitWithError] if the file is not writeable.
+      def write_file(path, content)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, 'w') { |f| f.puts content }
+      rescue Errno::EACCES
+        raise PDK::CLI::ExitWithError, _("You do not have permission to write to '%{path}'") % { path: path }
+      end
+
+      # Remove a file from disk.
+      #
+      # Like FileUtils.rm_f, this method will not fail if the file does not
+      # exist. Unlink FileUtils.rm_f, this method will not blindly swallow all
+      # exceptions.
+      #
+      # @param path [String] The path to the file to be removed.
+      #
+      # @raise [PDK::CLI::ExitWithError] if the file could not be removed.
+      def unlink_file(path)
+        FileUtils.rm(path) if File.file?(path)
+      rescue => e
+        raise PDK::CLI::ExitWithError, _("Unable to remove '%{path}': %{message}") % {
+          path:    path,
+          message: e.message,
+        }
+      end
+
+      # Generate a unified diff of the changes to be made to a file.
+      #
+      # @param path [String] The path to the file being diffed (only used to
+      #   generate the diff header).
+      # @param old_content [String] The current content of the file.
+      # @param new_content [String] The new content of the file if the pending
+      #   change is applied.
+      # @param lines_of_context [Integer] The maximum number of lines of
+      #   context to include around the changed lines in the diff output
+      #   (default: 3).
+      #
+      # @return [String] The unified diff of the pending changes to the file.
+      def unified_diff(path, old_content, new_content, lines_of_context = 3)
+        output = []
+
+        old_lines = old_content.split($INPUT_RECORD_SEPARATOR).map(&:chomp)
+        new_lines = new_content.split($INPUT_RECORD_SEPARATOR).map(&:chomp)
+
+        diffs = Diff::LCS.diff(old_lines, new_lines)
+
+        return nil if diffs.empty?
+
+        file_mtime = File.stat(path).mtime.localtime.strftime('%Y-%m-%d %H:%M:%S.%N %z')
+        now = Time.now.localtime.strftime('%Y-%m-%d %H:%M:%S.%N %z')
+
+        output << "--- #{path}\t#{file_mtime}"
+        output << "+++ #{path}.pdknew\t#{now}"
+
+        oldhunk = hunk = nil
+        file_length_difference = 0
+
+        diffs.each do |piece|
+          begin
+            hunk = Diff::LCS::Hunk.new(old_lines, new_lines, piece, lines_of_context, file_length_difference)
+            file_length_difference = hunk.file_length_difference
+
+            next unless oldhunk
+
+            # If the hunk overlaps with the oldhunk, merge them.
+            next if lines_of_context > 0 && hunk.merge(oldhunk)
+
+            output << oldhunk.diff(:unified)
+          ensure
+            oldhunk = hunk
+          end
+        end
+
+        output << oldhunk.diff(:unified)
+
+        output.join($INPUT_RECORD_SEPARATOR)
+      end
+    end
+  end
+end