pdk 1.9.0 → 3.2.0

data/lib/pdk/module/build.rb

@@ -1,10 +1,4 @@
-require 'fileutils'
-require 'minitar'
-require 'zlib'
-require 'pathspec'
-require 'find'
-require 'pdk/module'
-require 'pdk/tests/unit'
+require 'pdk'
 
 module PDK
   module Module
@@ -13,12 +7,11 @@ module PDK
         new(options).build
       end
 
-      attr_reader :module_dir
-      attr_reader :target_dir
+      attr_reader :module_dir, :target_dir
 
       def initialize(options = {})
-        @module_dir = File.expand_path(options[:module_dir] || Dir.pwd)
-        @target_dir = File.expand_path(options[:'target-dir'] || File.join(module_dir, 'pkg'))
+        @module_dir = PDK::Util::Filesystem.expand_path(options[:module_dir] || Dir.pwd)
+        @target_dir = PDK::Util::Filesystem.expand_path(options[:'target-dir'] || File.join(module_dir, 'pkg'))
       end
 
       # Read and parse the values from metadata.json for the module that is
@@ -26,6 +19,8 @@ module PDK
       #
       # @return [Hash{String => Object}] The hash of metadata values.
       def metadata
+        require 'pdk/module/metadata'
+
         @metadata ||= PDK::Module::Metadata.from_file(File.join(module_dir, 'metadata.json')).data
       end
 
@@ -51,7 +46,7 @@ module PDK
       # Verify if there is an existing package in the target directory and prompts
       # the user if they want to overwrite it.
       def package_already_exists?
-        File.exist? package_file
+        PDK::Util::Filesystem.exist?(package_file)
       end
 
       # Check if the module is PDK Compatible. If not, then prompt the user if
@@ -73,14 +68,14 @@ module PDK
       def create_build_dir
         cleanup_build_dir
 
-        FileUtils.mkdir_p(build_dir)
+        PDK::Util::Filesystem.mkdir_p(build_dir)
       end
 
       # Remove the temporary build directory and all its contents from disk.
       #
       # @return nil.
       def cleanup_build_dir
-        FileUtils.rm_rf(build_dir, secure: true)
+        PDK::Util::Filesystem.rm_rf(build_dir, secure: true)
       end
 
       # Combine the module name and version into a Forge-compatible dash
@@ -90,7 +85,7 @@ module PDK
       def release_name
         @release_name ||= [
           metadata['name'],
-          metadata['version'],
+          metadata['version']
         ].join('-')
       end
 
@@ -99,6 +94,8 @@ module PDK
       #
       # @return nil
       def stage_module_in_build_dir
+        require 'find'
+
         Find.find(module_dir) do |path|
           next if path == module_dir
 
@@ -112,16 +109,24 @@ module PDK
       #
       # @return nil.
       def stage_path(path)
+        require 'pathname'
+
         relative_path = Pathname.new(path).relative_path_from(Pathname.new(module_dir))
         dest_path = File.join(build_dir, relative_path)
 
-        if File.directory?(path)
-          FileUtils.mkdir_p(dest_path, mode: File.stat(path).mode)
-        elsif File.symlink?(path)
+        validate_path_encoding!(relative_path.to_path)
+
+        if PDK::Util::Filesystem.directory?(path)
+          PDK::Util::Filesystem.mkdir_p(dest_path, mode: PDK::Util::Filesystem.stat(path).mode)
+        elsif PDK::Util::Filesystem.symlink?(path)
           warn_symlink(path)
         else
-          FileUtils.cp(path, dest_path, preserve: true)
+          validate_ustar_path!(relative_path.to_path)
+          PDK::Util::Filesystem.cp(path, dest_path, preserve: true)
         end
+      rescue ArgumentError => e
+        raise PDK::CLI::ExitWithError, format('%{message} Rename the file or exclude it from the package ' \
+                                              'by adding it to the .pdkignore file in your module.', message: e.message)
       end
 
       # Check if the given path matches one of the patterns listed in the
@@ -131,7 +136,7 @@ module PDK
       #
       # @return [Boolean] true if the path matches and should be ignored.
       def ignored_path?(path)
-        path = path.to_s + '/' if File.directory?(path)
+        path = "#{path}/" if PDK::Util::Filesystem.directory?(path)
 
         !ignored_files.match_paths([path], module_dir).empty?
       end
@@ -143,13 +148,80 @@ module PDK
       #
       # @return nil.
       def warn_symlink(path)
+        require 'pathname'
+
         symlink_path = Pathname.new(path)
         module_path = Pathname.new(module_dir)
 
-        PDK.logger.warn _('Symlinks in modules are not supported and will not be included in the package. Please investigate symlink %{from} -> %{to}.') % {
-          from: symlink_path.relative_path_from(module_path),
-          to:   symlink_path.realpath.relative_path_from(module_path),
-        }
+        PDK.logger.warn format('Symlinks in modules are not supported and will not be included in the package. Please investigate symlink %{from} -> %{to}.',
+                               from: symlink_path.relative_path_from(module_path), to: symlink_path.realpath.relative_path_from(module_path))
+      end
+
+      # Checks if the path length will fit into the POSIX.1-1998 (ustar) tar
+      # header format.
+      #
+      # POSIX.1-2001 (which allows paths of infinite length) was adopted by GNU
+      # tar in 2004 and is supported by minitar 0.7 and above. Unfortunately
+      # much of the Puppet ecosystem still uses minitar 0.6.1.
+      #
+      # POSIX.1-1998 tar format does not allow for paths greater than 256 bytes,
+      # or paths that can't be split into a prefix of 155 bytes (max) and
+      # a suffix of 100 bytes (max).
+      #
+      # This logic was pretty much copied from the private method
+      # {Archive::Tar::Minitar::Writer#split_name}.
+      #
+      # @param path [String] the relative path to be added to the tar file.
+      #
+      # @raise [ArgumentError] if the path is too long or could not be split.
+      #
+      # @return [nil]
+      def validate_ustar_path!(path)
+        raise ArgumentError, format("The path '%{path}' is longer than 256 bytes.", path: path) if path.bytesize > 256
+
+        if path.bytesize <= 100
+          prefix = ''
+        else
+          parts = path.split(File::SEPARATOR)
+          newpath = parts.pop
+          nxt = ''
+
+          loop do
+            nxt = parts.pop || ''
+            break if newpath.bytesize + 1 + nxt.bytesize >= 100
+
+            newpath = File.join(nxt, newpath)
+          end
+
+          prefix = File.join(*parts, nxt)
+          path = newpath
+        end
+
+        return unless path.bytesize > 100 || prefix.bytesize > 155
+
+        raise ArgumentError,
+              format("'%{path}' could not be split at a directory separator into two " \
+                     'parts, the first having a maximum length of 155 bytes and the ' \
+                     'second having a maximum length of 100 bytes.', path: path)
+      end
+
+      # Checks if the path contains any non-ASCII characters.
+      #
+      # Java will throw an error when it encounters a path containing
+      # characters that are not supported by the hosts locale. In order to
+      # maximise compatibility we limit the paths to contain only ASCII
+      # characters, which should be part of any locale character set.
+      #
+      # @param path [String] the relative path to be added to the tar file.
+      #
+      # @raise [ArgumentError] if the path contains non-ASCII characters.
+      #
+      # @return [nil]
+      def validate_path_encoding!(path)
+        return unless /[^\x00-\x7F]/.match?(path)
+
+        raise ArgumentError, format("'%{path}' can only include ASCII characters in its path or " \
+                                    'filename in order to be compatible with a wide range of hosts.', path: path)
       end
 
       # Creates a gzip compressed tarball of the build directory.
@@ -159,12 +231,31 @@ module PDK
       #
       # @return nil.
       def build_package
-        FileUtils.rm_f(package_file)
+        require 'zlib'
+        require 'minitar'
+        require 'find'
+
+        PDK::Util::Filesystem.rm_f(package_file)
 
         Dir.chdir(target_dir) do
-          Zlib::GzipWriter.open(package_file) do |package_fd|
-            Minitar.pack(release_name, package_fd)
+          gz = Zlib::GzipWriter.new(File.open(package_file, 'wb')) # rubocop:disable PDK/FileOpen
+          tar = Minitar::Output.new(gz)
+          Find.find(release_name) do |entry|
+            entry_meta = {
+              name: entry
+            }
+
+            orig_mode = PDK::Util::Filesystem.stat(entry).mode
+            min_mode = Minitar.dir?(entry) ? 0o755 : 0o644
+
+            entry_meta[:mode] = orig_mode | min_mode
+
+            PDK.logger.debug(format('Updated permissions of packaged \'%{entry}\' to %{new_mode}', entry: entry, new_mode: (entry_meta[:mode] & 0o7777).to_s(8))) if entry_meta[:mode] != orig_mode
+
+            Minitar.pack_file(entry_meta, tar)
           end
+        ensure
+          tar.close
         end
       end
 
@@ -179,8 +270,8 @@ module PDK
         @ignore_file ||= [
           File.join(module_dir, '.pdkignore'),
           File.join(module_dir, '.pmtignore'),
-          File.join(module_dir, '.gitignore'),
-        ].find { |file| File.file?(file) && File.readable?(file) }
+          File.join(module_dir, '.gitignore')
+        ].find { |file| PDK::Util::Filesystem.file?(file) && PDK::Util::Filesystem.readable?(file) }
       end
 
       # Instantiate a new PathSpec class and populate it with the pattern(s) of
@@ -188,21 +279,18 @@ module PDK
       #
       # @return [PathSpec] The populated ignore path matcher.
       def ignored_files
+        require 'pdk/module'
+        require 'pathspec'
+
         @ignored_files ||=
           begin
             ignored = if ignore_file.nil?
                         PathSpec.new
                       else
-                        fd = File.open(ignore_file, 'rb:UTF-8')
-                        data = fd.read
-                        fd.close
-
-                        PathSpec.new(data)
+                        PathSpec.new(PDK::Util::Filesystem.read_file(ignore_file, open_args: 'rb:UTF-8'))
                       end
 
-            if File.realdirpath(target_dir).start_with?(File.realdirpath(module_dir))
-              ignored = ignored.add("\/#{File.basename(target_dir)}\/")
-            end
+            ignored = ignored.add("/#{File.basename(target_dir)}/") if File.realdirpath(target_dir).start_with?(File.realdirpath(module_dir))
 
             PDK::Module::DEFAULT_IGNORED.each { |r| ignored.add(r) }
 

data/lib/pdk/module/convert.rb

@@ -1,26 +1,36 @@
-require 'pdk/generate/module'
-require 'pdk/module/update_manager'
-require 'pdk/util'
-require 'pdk/report'
+require 'pdk'
 
 module PDK
   module Module
     class Convert
-      def self.invoke(options)
-        new(options).run
+      def self.invoke(module_dir, options)
+        new(module_dir, options).run
       end
 
-      attr_reader :options
+      attr_reader :module_dir, :options
 
-      def initialize(options = {})
+      def initialize(module_dir, options = {})
+        @module_dir = module_dir
         @options = options
       end
 
+      def convert?
+        instance_of?(PDK::Module::Convert)
+      end
+
       def run
         stage_changes!
 
         unless update_manager.changes?
-          PDK::Report.default_target.puts(_('No changes required.'))
+          if adding_tests?
+            add_tests!
+            print_result 'Convert completed'
+          else
+            require 'pdk/report'
+
+            PDK::Report.default_target.puts('No changes required.')
+          end
+
           return
         end
 
@@ -31,24 +41,26 @@ module PDK
         return if noop?
 
         unless force?
-          PDK.logger.info _(
-            'Module conversion is a potentially destructive action. ' \
-            'Ensure that you have committed your module to a version control ' \
-            'system or have a backup, and review the changes above before continuing.',
-          )
-          continue = PDK::CLI::Util.prompt_for_yes(_('Do you want to continue and make these changes to your module?'))
+          require 'pdk/cli/util'
+
+          PDK.logger.info 'Module conversion is a potentially destructive action. ' \
+                          'Ensure that you have committed your module to a version control ' \
+                          'system or have a backup, and review the changes above before continuing.'
+          continue = PDK::CLI::Util.prompt_for_yes('Do you want to continue and make these changes to your module?')
           return unless continue
         end
 
-        # Remove these files straight away as these changes are not something that the user needs to review.
-        if needs_bundle_update?
-          update_manager.unlink_file('Gemfile.lock')
-          update_manager.unlink_file(File.join('.bundle', 'config'))
-        end
+        # Remove these files straight away as these changes are not something
+        # that the user needs to review.
+        update_manager.unlink_file('Gemfile.lock')
+        update_manager.unlink_file(File.join('.bundle', 'config'))
 
         update_manager.sync_changes!
 
-        PDK::Util::Bundler.ensure_bundle! if needs_bundle_update?
+        require 'pdk/util/bundler'
+        PDK::Util::Bundler.ensure_bundle!
+
+        add_tests! if adding_tests?
 
         print_result 'Convert completed'
       end
@@ -61,35 +73,100 @@ module PDK
         options[:force]
       end
 
-      def needs_bundle_update?
-        update_manager.changed?('Gemfile')
+      def add_tests?
+        options[:'add-tests']
+      end
+
+      def adding_tests?
+        add_tests? && missing_tests?
       end
 
-      def stage_changes!
-        metadata_path = 'metadata.json'
+      def missing_tests?
+        !available_test_generators.empty?
+      end
+
+      def available_test_generators
+        # Only select generators which can run and have no pre-existing files
+        test_generators.select do |gen|
+          if gen.can_run?
+            gen.template_files.none? { |_, dst_path| PDK::Util::Filesystem.exist?(File.join(gen.context.root_path, dst_path)) }
+          else
+            false
+          end
+        end
+      end
 
-        PDK::Module::TemplateDir.new(template_url, nil, false) do |templates|
-          new_metadata = update_metadata(metadata_path, templates.metadata)
-          templates.module_metadata = new_metadata.data unless new_metadata.nil?
+      def test_generators(context = PDK.context)
+        return @test_generators unless @test_generators.nil?
 
+        require 'pdk/util/puppet_strings'
+
+        test_gens = PDK::Util::PuppetStrings.all_objects.map do |generator, objects|
+          (objects || []).map do |obj|
+            generator.new(context, obj['name'], spec_only: true)
+          end
+        end
+
+        @test_generators = test_gens.flatten
+      end
+
+      def stage_tests!(manager)
+        available_test_generators.each do |gen|
+          gen.stage_changes(manager)
+        end
+        manager
+      end
+
+      def add_tests!
+        update_manager.clear!
+        stage_tests!(update_manager)
+
+        if update_manager.changes?
+          update_manager.sync_changes!
+          print_summary
+        else
+          PDK::Report.default_target.puts('No test changes required.')
+        end
+      end
+
+      def stage_changes!(context = PDK.context)
+        require 'pdk/util/filesystem'
+
+        metadata_path = File.join(module_dir, 'metadata.json')
+
+        PDK::Template.with(template_uri, context) do |template_dir|
+          new_metadata = update_metadata(metadata_path, template_dir.metadata)
           if options[:noop] && new_metadata.nil?
             update_manager.add_file(metadata_path, '')
-          elsif File.file?(metadata_path)
+          elsif PDK::Util::Filesystem.file?(metadata_path)
             update_manager.modify_file(metadata_path, new_metadata.to_json)
           else
             update_manager.add_file(metadata_path, new_metadata.to_json)
           end
 
-          templates.render do |file_path, file_content, file_status|
-            if file_status == :unmanage
-              PDK.logger.debug(_("skipping '%{path}'") % { path: file_path })
-            elsif file_status == :delete
-              update_manager.remove_file(file_path)
-            elsif file_status == :manage
-              if File.exist? file_path
-                update_manager.modify_file(file_path, file_content)
+          # new_metadata == nil when creating a new module but with --noop@
+          module_name = new_metadata.nil? ? 'new-module' : new_metadata.data['name']
+          metadata_for_render = new_metadata.nil? ? {} : new_metadata.data
+
+          template_dir.render_new_module(module_name, metadata_for_render) do |relative_file_path, file_content, file_status, file_executable|
+            absolute_file_path = File.join(module_dir, relative_file_path)
+            case file_status
+            when :unmanage
+              PDK.logger.debug(format("skipping '%{path}'", path: absolute_file_path))
+            when :delete
+              update_manager.remove_file(absolute_file_path)
+            when :init
+              if convert? && !PDK::Util::Filesystem.exist?(absolute_file_path)
+                update_manager.add_file(absolute_file_path, file_content)
+                update_manager.make_file_executable(absolute_file_path) if file_executable
+              end
+            when :manage
+              if PDK::Util::Filesystem.exist?(absolute_file_path)
+                update_manager.modify_file(absolute_file_path, file_content)
+                update_manager.make_file_executable(absolute_file_path) if file_executable && !PDK::Util::Filesystem.executable?(absolute_file_path)
               else
-                update_manager.add_file(file_path, file_content)
+                update_manager.add_file(absolute_file_path, file_content)
+                update_manager.make_file_executable(absolute_file_path) if file_executable
               end
             end
           end
@@ -99,20 +176,24 @@ module PDK
       end
 
       def update_manager
+        require 'pdk/module/update_manager'
+
         @update_manager ||= PDK::Module::UpdateManager.new
       end
 
-      def template_url
-        @template_url ||= options.fetch(:'template-url', PDK::Util.default_template_url)
+      def template_uri
+        require 'pdk/util/template_uri'
+
+        @template_uri ||= PDK::Util::TemplateURI.new(options)
       end
 
       def update_metadata(metadata_path, template_metadata)
-        if File.file?(metadata_path)
-          unless File.readable?(metadata_path)
-            raise PDK::CLI::ExitWithError, _('Unable to update module metadata; %{path} exists but it is not readable.') % {
-              path: metadata_path,
-            }
-          end
+        require 'pdk/generate/module'
+        require 'pdk/util/filesystem'
+        require 'pdk/module/metadata'
+
+        if PDK::Util::Filesystem.file?(metadata_path)
+          raise PDK::CLI::ExitWithError, format('Unable to update module metadata; %{path} exists but it is not readable.', path: metadata_path) unless PDK::Util::Filesystem.readable?(metadata_path)
 
           begin
             metadata = PDK::Module::Metadata.from_file(metadata_path)
@@ -124,12 +205,10 @@ module PDK
           rescue ArgumentError
             metadata = PDK::Generate::Module.prepare_metadata(options) unless options[:noop]
           end
-        elsif File.exist?(metadata_path)
-          raise PDK::CLI::ExitWithError, _('Unable to update module metadata; %{path} exists but it is not a file.') % {
-            path: metadata_path,
-          }
+        elsif PDK::Util::Filesystem.exist?(metadata_path)
+          raise PDK::CLI::ExitWithError, format('Unable to update module metadata; %{path} exists but it is not a file.', path: metadata_path)
         else
-          return nil if options[:noop]
+          return if options[:noop]
 
           project_dir = File.basename(Dir.pwd)
           options[:module_name] = project_dir.split('-', 2).compact[-1]
@@ -163,34 +242,36 @@ module PDK
       end
 
       def print_summary
+        require 'pdk/report'
+
         footer = false
 
-        summary.keys.each do |category|
+        summary.each_key do |category|
           next if summary[category].empty?
 
-          PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner("Files to be #{category}", 40) })
+          PDK::Report.default_target.puts(format("\n%{banner}", banner: generate_banner("Files to be #{category}", 40)))
           PDK::Report.default_target.puts(summary[category])
           footer = true
         end
 
-        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner('', 40) }) if footer
+        PDK::Report.default_target.puts(format("\n%{banner}", banner: generate_banner('', 40))) if footer
       end
 
       def print_result(banner_text)
-        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner(banner_text, 40) })
-        summary_to_print = summary.map { |k, v| "#{v.length} files #{k}" unless v.empty? }.compact
-        PDK::Report.default_target.puts(_("\n%{summary}\n\n") % { summary: "#{summary_to_print.join(', ')}." })
+        require 'pdk/report'
+
+        PDK::Report.default_target.puts(format("\n%{banner}", banner: generate_banner(banner_text, 40)))
+        summary_to_print = summary.filter_map { |k, v| "#{v.length} files #{k}" unless v.empty? }
+        PDK::Report.default_target.puts(format("\n%{summary}\n\n", summary: "#{summary_to_print.join(', ')}."))
       end
 
       def full_report(path)
-        File.open(path, 'w') do |f|
-          f.write("/* Report generated by PDK at #{Time.now} */")
-          update_manager.changes[:modified].each do |_, diff|
-            f.write("\n\n\n" + diff)
-          end
-          f.write("\n")
-        end
-        PDK::Report.default_target.puts(_("\nYou can find a report of differences in %{path}.\n\n") % { path: path })
+        require 'pdk/report'
+
+        report = ["/* Report generated by PDK at #{Time.now} */"]
+        report.concat(update_manager.changes[:modified].map { |_, diff| "\n\n\n#{diff}" })
+        PDK::Util::Filesystem.write_file(path, report.join)
+        PDK::Report.default_target.puts(format("\nYou can find a report of differences in %{path}.\n\n", path: path))
       end
 
       def generate_banner(text, width = 80)