pdk-akerl 1.8.0.1

data/lib/pdk/module.rb

@@ -0,0 +1,21 @@
+require 'pathspec'
+
+module PDK
+  module Module
+    DEFAULT_IGNORED = [
+      '/pkg/',
+      '.*',
+      '~*',
+      '/coverage',
+      '/checksums.json',
+      '/REVISION',
+      '/spec/fixtures/modules/',
+      '/vendor/',
+    ].freeze
+
+    def default_ignored_pathspec
+      @default_ignored_pathspec ||= PathSpec.new(DEFAULT_IGNORED)
+    end
+    module_function :default_ignored_pathspec
+  end
+end

data/lib/pdk/module/build.rb

@@ -0,0 +1,214 @@
+require 'fileutils'
+require 'minitar'
+require 'zlib'
+require 'pathspec'
+require 'find'
+require 'pdk/module'
+require 'pdk/tests/unit'
+
+module PDK
+  module Module
+    class Build
+      def self.invoke(options = {})
+        new(options).build
+      end
+
+      attr_reader :module_dir
+      attr_reader :target_dir
+
+      def initialize(options = {})
+        @module_dir = File.expand_path(options[:module_dir] || Dir.pwd)
+        @target_dir = File.expand_path(options[:'target-dir'] || File.join(module_dir, 'pkg'))
+      end
+
+      # Read and parse the values from metadata.json for the module that is
+      # being built.
+      #
+      # @return [Hash{String => Object}] The hash of metadata values.
+      def metadata
+        @metadata ||= PDK::Module::Metadata.from_file(File.join(module_dir, 'metadata.json')).data
+      end
+
+      # Return the path where the built package file will be written to.
+      def package_file
+        @package_file ||= File.join(target_dir, "#{release_name}.tar.gz")
+      end
+
+      # Build a module package from a module directory.
+      #
+      # @return [String] The path to the built package file.
+      def build
+        create_build_dir
+
+        stage_module_in_build_dir
+        build_package
+
+        package_file
+      ensure
+        cleanup_build_dir
+      end
+
+      # Verify if there is an existing package in the target directory and prompts
+      # the user if they want to overwrite it.
+      def package_already_exists?
+        File.exist? package_file
+      end
+
+      # Check if the module is PDK Compatible. If not, then prompt the user if
+      # they want to run PDK Convert.
+      def module_pdk_compatible?
+        ['pdk-version', 'template-url'].any? { |key| metadata.key?(key) }
+      end
+
+      # Return the path to the temporary build directory, which will be placed
+      # inside the target directory and match the release name (see #release_name).
+      def build_dir
+        @build_dir ||= File.join(target_dir, release_name)
+      end
+
+      # Create a temporary build directory where the files to be included in
+      # the package will be staged before building the tarball.
+      #
+      # If the directory already exists, remove it first.
+      def create_build_dir
+        cleanup_build_dir
+
+        FileUtils.mkdir_p(build_dir)
+      end
+
+      # Remove the temporary build directory and all its contents from disk.
+      #
+      # @return nil.
+      def cleanup_build_dir
+        FileUtils.rm_rf(build_dir, secure: true)
+      end
+
+      # Combine the module name and version into a Forge-compatible dash
+      # separated string.
+      #
+      # @return [String] The module name and version, joined by a dash.
+      def release_name
+        @release_name ||= [
+          metadata['name'],
+          metadata['version'],
+        ].join('-')
+      end
+
+      # Iterate through all the files and directories in the module and stage
+      # them into the temporary build directory (unless ignored).
+      #
+      # @return nil
+      def stage_module_in_build_dir
+        Find.find(module_dir) do |path|
+          next if path == module_dir
+
+          ignored_path?(path) ? Find.prune : stage_path(path)
+        end
+      end
+
+      # Stage a file or directory from the module into the build directory.
+      #
+      # @param path [String] The path to the file or directory.
+      #
+      # @return nil.
+      def stage_path(path)
+        relative_path = Pathname.new(path).relative_path_from(Pathname.new(module_dir))
+        dest_path = File.join(build_dir, relative_path)
+
+        if File.directory?(path)
+          FileUtils.mkdir_p(dest_path, mode: File.stat(path).mode)
+        elsif File.symlink?(path)
+          warn_symlink(path)
+        else
+          FileUtils.cp(path, dest_path, preserve: true)
+        end
+      end
+
+      # Check if the given path matches one of the patterns listed in the
+      # ignore file.
+      #
+      # @param path [String] The path to be checked.
+      #
+      # @return [Boolean] true if the path matches and should be ignored.
+      def ignored_path?(path)
+        path = path.to_s + '/' if File.directory?(path)
+
+        !ignored_files.match_paths([path], module_dir).empty?
+      end
+
+      # Warn the user about a symlink that would have been included in the
+      # built package.
+      #
+      # @param path [String] The relative or absolute path to the symlink.
+      #
+      # @return nil.
+      def warn_symlink(path)
+        symlink_path = Pathname.new(path)
+        module_path = Pathname.new(module_dir)
+
+        PDK.logger.warn _('Symlinks in modules are not supported and will not be included in the package. Please investigate symlink %{from} -> %{to}.') % {
+          from: symlink_path.relative_path_from(module_path),
+          to:   symlink_path.realpath.relative_path_from(module_path),
+        }
+      end
+
+      # Creates a gzip compressed tarball of the build directory.
+      #
+      # If the destination package already exists, it will be removed before
+      # creating the new tarball.
+      #
+      # @return nil.
+      def build_package
+        FileUtils.rm_f(package_file)
+
+        Dir.chdir(target_dir) do
+          Zlib::GzipWriter.open(package_file) do |package_fd|
+            Minitar.pack(release_name, package_fd)
+          end
+        end
+      end
+
+      # Select the most appropriate ignore file in the module directory.
+      #
+      # In order of preference, we first try `.pdkignore`, then `.pmtignore`
+      # and finally `.gitignore`.
+      #
+      # @return [String] The path to the file containing the patterns of file
+      #   paths to ignore.
+      def ignore_file
+        @ignore_file ||= [
+          File.join(module_dir, '.pdkignore'),
+          File.join(module_dir, '.pmtignore'),
+          File.join(module_dir, '.gitignore'),
+        ].find { |file| File.file?(file) && File.readable?(file) }
+      end
+
+      # Instantiate a new PathSpec class and populate it with the pattern(s) of
+      # files to be ignored.
+      #
+      # @return [PathSpec] The populated ignore path matcher.
+      def ignored_files
+        @ignored_files ||=
+          begin
+            ignored = if ignore_file.nil?
+                        PathSpec.new
+                      else
+                        fd = File.open(ignore_file, 'rb:UTF-8')
+                        data = fd.read
+                        fd.close
+
+                        PathSpec.new(data)
+                      end
+
+            if File.realdirpath(target_dir).start_with?(File.realdirpath(module_dir))
+              ignored = ignored.add("\/#{File.basename(target_dir)}\/")
+            end
+
+            PDK::Module::DEFAULT_IGNORED.each { |r| ignored.add(r) }
+
+            ignored
+          end
+      end
+    end
+  end
+end

data/lib/pdk/module/convert.rb

@@ -0,0 +1,209 @@
+require 'pdk/generate/module'
+require 'pdk/module/update_manager'
+require 'pdk/util'
+require 'pdk/report'
+
+module PDK
+  module Module
+    class Convert
+      def self.invoke(options)
+        new(options).run
+      end
+
+      attr_reader :options
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      def run
+        stage_changes!
+
+        unless update_manager.changes?
+          PDK::Report.default_target.puts(_('No changes required.'))
+          return
+        end
+
+        print_summary
+
+        full_report('convert_report.txt') unless update_manager.changes[:modified].empty?
+
+        return if noop?
+
+        unless force?
+          PDK.logger.info _(
+            'Module conversion is a potentially destructive action. ' \
+            'Ensure that you have committed your module to a version control ' \
+            'system or have a backup, and review the changes above before continuing.',
+          )
+          continue = PDK::CLI::Util.prompt_for_yes(_('Do you want to continue and make these changes to your module?'))
+          return unless continue
+        end
+
+        # Remove these files straight away as these changes are not something that the user needs to review.
+        if needs_bundle_update?
+          update_manager.unlink_file('Gemfile.lock')
+          update_manager.unlink_file(File.join('.bundle', 'config'))
+        end
+
+        update_manager.sync_changes!
+
+        PDK::Util::Bundler.ensure_bundle! if needs_bundle_update?
+
+        print_result 'Convert completed'
+      end
+
+      def noop?
+        options[:noop]
+      end
+
+      def force?
+        options[:force]
+      end
+
+      def needs_bundle_update?
+        update_manager.changed?('Gemfile')
+      end
+
+      def stage_changes!
+        metadata_path = 'metadata.json'
+
+        PDK::Module::TemplateDir.new(template_url, nil, false) do |templates|
+          new_metadata = update_metadata(metadata_path, templates.metadata)
+          templates.module_metadata = new_metadata.data unless new_metadata.nil?
+
+          if options[:noop] && new_metadata.nil?
+            update_manager.add_file(metadata_path, '')
+          elsif File.file?(metadata_path)
+            update_manager.modify_file(metadata_path, new_metadata.to_json)
+          else
+            update_manager.add_file(metadata_path, new_metadata.to_json)
+          end
+
+          templates.render do |file_path, file_content, file_status|
+            if file_status == :unmanage
+              PDK.logger.debug(_("skipping '%{path}'") % { path: file_path })
+            elsif file_status == :delete
+              update_manager.remove_file(file_path)
+            elsif file_status == :manage
+              if File.exist? file_path
+                update_manager.modify_file(file_path, file_content)
+              else
+                update_manager.add_file(file_path, file_content)
+              end
+            end
+          end
+        end
+      rescue ArgumentError => e
+        raise PDK::CLI::ExitWithError, e
+      end
+
+      def update_manager
+        @update_manager ||= PDK::Module::UpdateManager.new
+      end
+
+      def template_url
+        @template_url ||= options.fetch(:'template-url', PDK::Util.default_template_url)
+      end
+
+      def update_metadata(metadata_path, template_metadata)
+        if File.file?(metadata_path)
+          unless File.readable?(metadata_path)
+            raise PDK::CLI::ExitWithError, _('Unable to update module metadata; %{path} exists but it is not readable.') % {
+              path: metadata_path,
+            }
+          end
+
+          begin
+            metadata = PDK::Module::Metadata.from_file(metadata_path)
+            new_values = PDK::Module::Metadata::DEFAULTS.select do |key, _|
+              !metadata.data.key?(key) || metadata.data[key].nil? ||
+                (key == 'requirements' && metadata.data[key].empty?)
+            end
+            metadata.update!(new_values)
+          rescue ArgumentError
+            metadata = PDK::Generate::Module.prepare_metadata(options) unless options[:noop]
+          end
+        elsif File.exist?(metadata_path)
+          raise PDK::CLI::ExitWithError, _('Unable to update module metadata; %{path} exists but it is not a file.') % {
+            path: metadata_path,
+          }
+        else
+          return nil if options[:noop]
+
+          project_dir = File.basename(Dir.pwd)
+          options[:module_name] = project_dir.split('-', 2).compact[-1]
+          options[:prompt] = false
+          options[:'skip-interview'] = true if options[:force]
+
+          metadata = PDK::Generate::Module.prepare_metadata(options)
+        end
+
+        metadata.update!(template_metadata)
+        metadata
+      end
+
+      def summary
+        summary = {}
+        update_manager.changes.each do |category, update_category|
+          if update_category.respond_to?(:keys)
+            updated_files = update_category.keys
+          else
+            begin
+              updated_files = update_category.map { |file| file[:path] }
+            rescue TypeError
+              updated_files = update_category.to_a
+            end
+          end
+
+          summary[category] = updated_files
+        end
+
+        summary
+      end
+
+      def print_summary
+        footer = false
+
+        summary.keys.each do |category|
+          next if summary[category].empty?
+
+          PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner("Files to be #{category}", 40) })
+          PDK::Report.default_target.puts(summary[category])
+          footer = true
+        end
+
+        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner('', 40) }) if footer
+      end
+
+      def print_result(banner_text)
+        PDK::Report.default_target.puts(_("\n%{banner}") % { banner: generate_banner(banner_text, 40) })
+        summary_to_print = summary.map { |k, v| "#{v.length} files #{k}" unless v.empty? }.compact
+        PDK::Report.default_target.puts(_("\n%{summary}\n\n") % { summary: "#{summary_to_print.join(', ')}." })
+      end
+
+      def full_report(path)
+        File.open(path, 'w') do |f|
+          f.write("/* Report generated by PDK at #{Time.now} */")
+          update_manager.changes[:modified].each do |_, diff|
+            f.write("\n\n\n" + diff)
+          end
+          f.write("\n")
+        end
+        PDK::Report.default_target.puts(_("\nYou can find a report of differences in %{path}.\n\n") % { path: path })
+      end
+
+      def generate_banner(text, width = 80)
+        padding = width - text.length
+        banner = ''
+        padding_char = '-'
+
+        (padding / 2.0).ceil.times { banner << padding_char }
+        banner << text
+        (padding / 2.0).floor.times { banner << padding_char }
+
+        banner
+      end
+    end
+  end
+end

data/lib/pdk/module/metadata.rb

@@ -0,0 +1,193 @@
+require 'json'
+require 'pdk/util/filesystem'
+
+module PDK
+  module Module
+    class Metadata
+      attr_accessor :data
+
+      include PDK::Util::Filesystem
+
+      OPERATING_SYSTEMS = {
+        'RedHat based Linux' => [
+          {
+            'operatingsystem'        => 'CentOS',
+            'operatingsystemrelease' => ['7'],
+          },
+          {
+            'operatingsystem'        => 'OracleLinux',
+            'operatingsystemrelease' => ['7'],
+          },
+          {
+            'operatingsystem'        => 'RedHat',
+            'operatingsystemrelease' => ['7'],
+          },
+          {
+            'operatingsystem'        => 'Scientific',
+            'operatingsystemrelease' => ['7'],
+          },
+        ],
+        'Debian based Linux' => [
+          {
+            'operatingsystem'        => 'Debian',
+            'operatingsystemrelease' => ['8'],
+          },
+          {
+            'operatingsystem'        => 'Ubuntu',
+            'operatingsystemrelease' => ['16.04'],
+          },
+        ],
+        'Fedora' => {
+          'operatingsystem'        => 'Fedora',
+          'operatingsystemrelease' => ['25'],
+        },
+        'OSX' => {
+          'operatingsystem'        => 'Darwin',
+          'operatingsystemrelease' => ['16'],
+        },
+        'SLES' => {
+          'operatingsystem'        => 'SLES',
+          'operatingsystemrelease' => ['12'],
+        },
+        'Solaris' => {
+          'operatingsystem'        => 'Solaris',
+          'operatingsystemrelease' => ['11'],
+        },
+        'Windows' => {
+          'operatingsystem'        => 'windows',
+          'operatingsystemrelease' => ['2008 R2', '2012 R2', '10'],
+        },
+      }.freeze
+
+      DEFAULT_OPERATING_SYSTEMS = [
+        'RedHat based Linux',
+        'Debian based Linux',
+        'Windows',
+      ].freeze
+
+      DEFAULTS = {
+        'name'          => nil,
+        'version'       => '0.1.0',
+        'author'        => nil,
+        'summary'       => '',
+        'license'       => 'Apache-2.0',
+        'source'        => '',
+        'project_page'  => nil,
+        'issues_url'    => nil,
+        'dependencies'  => [],
+        'data_provider' => nil,
+        'operatingsystem_support' => DEFAULT_OPERATING_SYSTEMS.map { |os_name|
+          OPERATING_SYSTEMS[os_name]
+        }.flatten,
+        'requirements' => [
+          { 'name' => 'puppet', 'version_requirement' => '>= 4.10.0 < 7.0.0' },
+        ],
+      }.freeze
+
+      def initialize(params = {})
+        @data = DEFAULTS.dup
+        update!(params) if params
+      end
+
+      def self.from_file(metadata_json_path)
+        if metadata_json_path.nil?
+          raise ArgumentError, _('Cannot read metadata from file: no path to file was given.')
+        end
+
+        unless File.file?(metadata_json_path)
+          raise ArgumentError, _("'%{file}' does not exist or is not a file.") % { file: metadata_json_path }
+        end
+
+        unless File.readable?(metadata_json_path)
+          raise ArgumentError, _("Unable to open '%{file}' for reading.") % { file: metadata_json_path }
+        end
+
+        begin
+          data = JSON.parse(File.read(metadata_json_path))
+        rescue JSON::JSONError => e
+          raise ArgumentError, _('Invalid JSON in metadata.json: %{msg}') % { msg: e.message }
+        end
+
+        new(data)
+      end
+
+      def update!(data)
+        # TODO: validate all data
+        process_name(data) if data['name']
+        @data.merge!(data)
+        self
+      end
+
+      def to_json
+        JSON.pretty_generate(@data.dup.delete_if { |_key, value| value.nil? })
+      end
+
+      def write!(path)
+        write_file(path, to_json)
+      end
+
+      def forge_ready?
+        missing_fields.empty?
+      end
+
+      def interview_for_forge!
+        PDK::Generate::Module.module_interview(self, only_ask: missing_fields)
+      end
+
+      def validate_puppet_version_requirement!
+        msgs = {
+          no_reqs:       _('Module metadata does not contain any requirements.'),
+          no_puppet_req: _('Module metadata does not contain a "puppet" requirement.'),
+          no_puppet_ver: _('The "puppet" requirement in module metadata does not specify a "version_requirement".'),
+        }
+
+        raise ArgumentError, msgs[:no_reqs] unless @data.key?('requirements')
+        raise ArgumentError, msgs[:no_puppet_req] if puppet_requirement.nil?
+        raise ArgumentError, msgs[:no_puppet_ver] unless puppet_requirement.key?('version_requirement')
+        raise ArgumentError, msgs[:no_puppet_ver] if puppet_requirement['version_requirement'].empty?
+      end
+
+      def puppet_requirement
+        @data['requirements'].find do |r|
+          r.key?('name') && r['name'] == 'puppet'
+        end
+      end
+
+      private
+
+      def missing_fields
+        fields = DEFAULTS.keys - %w[data_provider requirements dependencies]
+        fields.select { |key| @data[key].nil? || @data[key].empty? }
+      end
+
+      # Do basic validation and parsing of the name parameter.
+      def process_name(data)
+        validate_name(data['name'])
+        author, _modname = data['name'].split(%r{[-/]}, 2)
+
+        data['author'] ||= author if @data['author'] == DEFAULTS['author']
+      end
+
+      # Validates that the given module name is both namespaced and well-formed.
+      def validate_name(name)
+        return if name =~ %r{\A[a-z0-9]+[-\/][a-z][a-z0-9_]*\Z}i
+
+        namespace, modname = name.split(%r{[-/]}, 2)
+        modname = :namespace_missing if namespace == ''
+
+        err = case modname
+              when nil, '', :namespace_missing
+                _('Field must be a dash-separated user name and module name.')
+              when %r{[^a-z0-9_]}i
+                _('Module name must contain only alphanumeric or underscore characters.')
+              when %r{^[^a-z]}i
+                _('Module name must begin with a letter.')
+              else
+                _('Namespace must contain only alphanumeric characters.')
+              end
+
+        raise ArgumentError, _("Invalid 'name' field in metadata.json: %{err}") % { err: err }
+      end
+    end
+  end
+end