pdk 1.9.0 → 3.2.0

data/lib/pdk/module/metadata.rb

@@ -1,87 +1,86 @@
-require 'json'
-require 'pdk/util/filesystem'
+require 'pdk'
 
 module PDK
   module Module
     class Metadata
       attr_accessor :data
 
-      include PDK::Util::Filesystem
-
       OPERATING_SYSTEMS = {
         'RedHat based Linux' => [
           {
-            'operatingsystem'        => 'CentOS',
-            'operatingsystemrelease' => ['7'],
+            'operatingsystem' => 'CentOS',
+            'operatingsystemrelease' => ['7']
           },
           {
-            'operatingsystem'        => 'OracleLinux',
-            'operatingsystemrelease' => ['7'],
+            'operatingsystem' => 'OracleLinux',
+            'operatingsystemrelease' => ['7']
           },
           {
-            'operatingsystem'        => 'RedHat',
-            'operatingsystemrelease' => ['7'],
+            'operatingsystem' => 'RedHat',
+            'operatingsystemrelease' => ['8']
           },
           {
-            'operatingsystem'        => 'Scientific',
-            'operatingsystemrelease' => ['7'],
-          },
+            'operatingsystem' => 'Scientific',
+            'operatingsystemrelease' => ['7']
+          }
         ],
         'Debian based Linux' => [
           {
-            'operatingsystem'        => 'Debian',
-            'operatingsystemrelease' => ['8'],
+            'operatingsystem' => 'Debian',
+            'operatingsystemrelease' => ['10']
           },
           {
-            'operatingsystem'        => 'Ubuntu',
-            'operatingsystemrelease' => ['16.04'],
-          },
+            'operatingsystem' => 'Ubuntu',
+            'operatingsystemrelease' => ['18.04']
+          }
         ],
         'Fedora' => {
-          'operatingsystem'        => 'Fedora',
-          'operatingsystemrelease' => ['25'],
+          'operatingsystem' => 'Fedora',
+          'operatingsystemrelease' => ['29']
         },
         'OSX' => {
-          'operatingsystem'        => 'Darwin',
-          'operatingsystemrelease' => ['16'],
+          'operatingsystem' => 'Darwin',
+          'operatingsystemrelease' => ['16']
         },
         'SLES' => {
-          'operatingsystem'        => 'SLES',
-          'operatingsystemrelease' => ['12'],
+          'operatingsystem' => 'SLES',
+          'operatingsystemrelease' => ['15']
         },
         'Solaris' => {
-          'operatingsystem'        => 'Solaris',
-          'operatingsystemrelease' => ['11'],
+          'operatingsystem' => 'Solaris',
+          'operatingsystemrelease' => ['11']
         },
         'Windows' => {
-          'operatingsystem'        => 'windows',
-          'operatingsystemrelease' => ['2008 R2', '2012 R2', '10'],
+          'operatingsystem' => 'windows',
+          'operatingsystemrelease' => ['2019', '10']
         },
+        'AIX' => {
+          'operatingsystem' => 'AIX',
+          'operatingsystemrelease' => ['6.1', '7.1', '7.2']
+        }
       }.freeze
 
       DEFAULT_OPERATING_SYSTEMS = [
         'RedHat based Linux',
         'Debian based Linux',
-        'Windows',
+        'Windows'
       ].freeze
 
       DEFAULTS = {
-        'name'          => nil,
-        'version'       => '0.1.0',
-        'author'        => nil,
-        'summary'       => '',
-        'license'       => 'Apache-2.0',
-        'source'        => '',
-        'project_page'  => nil,
-        'issues_url'    => nil,
-        'dependencies'  => [],
+        'name' => nil,
+        'version' => '0.1.0',
+        'author' => nil,
+        'summary' => '',
+        'license' => 'Apache-2.0',
+        'source' => '',
+        'project_page' => nil,
+        'issues_url' => nil,
+        'dependencies' => [],
         'data_provider' => nil,
-        'operatingsystem_support' => DEFAULT_OPERATING_SYSTEMS.map { |os_name|
-          OPERATING_SYSTEMS[os_name]
-        }.flatten,
+        'operatingsystem_support' => DEFAULT_OPERATING_SYSTEMS.dup,
         'requirements' => [
-          { 'name' => 'puppet', 'version_requirement' => '>= 4.10.0 < 7.0.0' },
-        ],
+          { 'name' => 'puppet', 'version_requirement' => '>= 7.24 < 9.0.0' }
+        ]
       }.freeze
 
       def initialize(params = {})
@@ -90,24 +89,21 @@ module PDK
       end
 
       def self.from_file(metadata_json_path)
-        if metadata_json_path.nil?
-          raise ArgumentError, _('Cannot read metadata from file: no path to file was given.')
-        end
+        raise ArgumentError, 'Cannot read metadata from file: no path to file was given.' if metadata_json_path.nil?
 
-        unless File.file?(metadata_json_path)
-          raise ArgumentError, _("'%{file}' does not exist or is not a file.") % { file: metadata_json_path }
-        end
+        raise ArgumentError, format("'%{file}' does not exist or is not a file.", file: metadata_json_path) unless PDK::Util::Filesystem.file?(metadata_json_path)
 
-        unless File.readable?(metadata_json_path)
-          raise ArgumentError, _("Unable to open '%{file}' for reading.") % { file: metadata_json_path }
-        end
+        raise ArgumentError, format("Unable to open '%{file}' for reading.", file: metadata_json_path) unless PDK::Util::Filesystem.readable?(metadata_json_path)
 
+        require 'json'
         begin
-          data = JSON.parse(File.read(metadata_json_path))
+          data = JSON.parse(PDK::Util::Filesystem.read_file(metadata_json_path))
         rescue JSON::JSONError => e
-          raise ArgumentError, _('Invalid JSON in metadata.json: %{msg}') % { msg: e.message }
+          raise ArgumentError, format('Invalid JSON in metadata.json: %{msg}', msg: e.message)
         end
 
+        require 'pdk/util'
+        data['template-url'] = PDK::Util::TemplateURI.default_template_uri.metadata_format if PDK::Util.package_install? && data['template-url'] == PDK::Util::TemplateURI::PACKAGED_TEMPLATE_KEYWORD
         new(data)
       end
 
@@ -118,12 +114,14 @@ module PDK
         self
       end
 
-      def to_json
+      def to_json(*_args)
+        require 'json'
+
         JSON.pretty_generate(@data.dup.delete_if { |_key, value| value.nil? })
       end
 
       def write!(path)
-        write_file(path, to_json)
+        PDK::Util::Filesystem.write_file(path, to_json)
       end
 
       def forge_ready?
@@ -131,14 +129,16 @@ module PDK
       end
 
       def interview_for_forge!
+        require 'pdk/generate/module'
+
         PDK::Generate::Module.module_interview(self, only_ask: missing_fields)
       end
 
       def validate_puppet_version_requirement!
         msgs = {
-          no_reqs:       _('Module metadata does not contain any requirements.'),
-          no_puppet_req: _('Module metadata does not contain a "puppet" requirement.'),
-          no_puppet_ver: _('The "puppet" requirement in module metadata does not specify a "version_requirement".'),
+          no_reqs: 'Module metadata does not contain any requirements.',
+          no_puppet_req: 'Module metadata does not contain a "puppet" requirement.',
+          no_puppet_ver: 'The "puppet" requirement in module metadata does not specify a "version_requirement".'
         }
 
         raise ArgumentError, msgs[:no_reqs] unless @data.key?('requirements')
@@ -153,40 +153,41 @@ module PDK
         end
       end
 
-      private
-
       def missing_fields
-        fields = DEFAULTS.keys - %w[data_provider requirements dependencies]
+        fields = DEFAULTS.keys - ['data_provider', 'requirements', 'dependencies']
         fields.select { |key| @data[key].nil? || @data[key].empty? }
       end
 
+      private
+
       # Do basic validation and parsing of the name parameter.
       def process_name(data)
         validate_name(data['name'])
-        author, _modname = data['name'].split(%r{[-/]}, 2)
+        author, modname = data['name'].split(%r{[-/]}, 2)
+        data['name'] = [author, modname].join('-')
 
         data['author'] ||= author if @data['author'] == DEFAULTS['author']
       end
 
       # Validates that the given module name is both namespaced and well-formed.
       def validate_name(name)
-        return if name =~ %r{\A[a-z0-9]+[-\/][a-z][a-z0-9_]*\Z}i
+        return if %r{\A[a-z0-9]+[-/][a-z][a-z0-9_]*\Z}i.match?(name)
 
         namespace, modname = name.split(%r{[-/]}, 2)
         modname = :namespace_missing if namespace == ''
 
         err = case modname
               when nil, '', :namespace_missing
-                _('Field must be a dash-separated user name and module name.')
-              when %r{[^a-z0-9_]}i
-                _('Module name must contain only alphanumeric or underscore characters.')
-              when %r{^[^a-z]}i
-                _('Module name must begin with a letter.')
+                'Field must be a dash-separated user name and module name.'
+              when /[^a-z0-9_]/i
+                'Module name must contain only alphanumeric or underscore characters.'
+              when /^[^a-z]/i
+                'Module name must begin with a letter.'
               else
-                _('Namespace must contain only alphanumeric characters.')
+                'Namespace must contain only alphanumeric characters.'
               end
 
-        raise ArgumentError, _("Invalid 'name' field in metadata.json: %{err}") % { err: err }
+        raise ArgumentError, format("Invalid 'name' field in metadata.json: %{err}", err: err)
       end
     end
   end

data/lib/pdk/module/release.rb

@@ -0,0 +1,255 @@
+require 'pdk'
+require 'uri'
+
+module PDK
+  module Module
+    class Release
+      def self.invoke(module_path, options = {})
+        new(module_path, options).run
+      end
+
+      attr_reader :options, :module_path
+
+      def initialize(module_path, options = {})
+        @options = options
+
+        # TODO: Currently the release process can ONLY be run if the working directory IS the module root. However, in the future
+        # this WILL change, so we have the API arguments for it, but only accept `nil` for the first parameter
+        raise PDK::CLI::ExitWithError, 'Running the release process outside of the working directory is not supported' unless module_path.nil?
+
+        if module_path.nil?
+          module_path = PDK::Util.module_root
+          raise PDK::CLI::ExitWithError, 'The module release process requires a valid module path' if module_path.nil?
+        end
+        raise PDK::CLI::ExitWithError, format('%{module_path} is not a valid module', module_path: module_path) unless PDK::Util.in_module_root?(module_path)
+
+        @module_path = module_path
+      end
+
+      def run
+        # Pre-release checks
+        unless force?
+          raise PDK::CLI::ExitWithError, 'The module is not PDK compatible' if requires_pdk_compatibility? && !pdk_compatible?
+          raise PDK::CLI::ExitWithError, 'The module is not Forge compatible' if requires_forge_compatibility? && !forge_compatible?
+        end
+
+        # Note that these checks are duplicated in the run_publish method, however it's a much better
+        # experience to fail early, than going through the whole process, only to error at the end knowing full well
+        # it'll fail anyway.
+        validate_publish_options!
+
+        run_validations(options) unless skip_validation?
+
+        PDK.logger.info format('Releasing %{module_name} - from version %{module_version}', module_name: module_metadata.data['name'], module_version: module_metadata.data['version'])
+
+        PDK::Util::ChangelogGenerator.generate_changelog unless skip_changelog?
+
+        # Calculate the new module version
+        new_version = specified_version
+        new_version = PDK::Util::ChangelogGenerator.compute_next_version(module_metadata.data['version']) if new_version.nil? && !skip_changelog?
+        new_version = module_metadata.data['version'] if new_version.nil?
+
+        if new_version != module_metadata.data['version']
+          PDK.logger.info format('Updating version to %{module_version}', module_version: new_version)
+
+          # Set the new version in metadata file
+          module_metadata.data['version'] = new_version
+          write_module_metadata!
+
+          # Update the changelog with the correct version
+          PDK::Util::ChangelogGenerator.generate_changelog unless skip_changelog?
+
+          # Check if the versions match
+          latest_version = PDK::Util::ChangelogGenerator.latest_version
+          if !latest_version && (new_version != latest_version)
+            raise PDK::CLI::ExitWithError, format('%{new_version} does not match %{latest_version}', new_version: new_version, latest_version: latest_version)
+          end
+        end
+
+        run_documentation(options) unless skip_documentation?
+
+        run_dependency_checker(options) unless skip_dependency?
+
+        if skip_build?
+          # Even if we're skipping the build, we still need the name of the tarball
+          # Use the specified package path if set
+          package_file = specified_package if package_file.nil?
+          # Use the default as a last resort
+          package_file = default_package_filename if package_file.nil?
+        else
+          package_file = run_build(options)
+        end
+
+        run_publish(options.dup, package_file) unless skip_publish?
+      end
+
+      def module_metadata
+        @module_metada ||= PDK::Module::Metadata.from_file(File.join(module_path, 'metadata.json'))
+      end
+
+      def write_module_metadata!
+        module_metadata.write!(File.join(module_path, 'metadata.json'))
+        clear_cached_data
+      end
+
+      def default_package_filename
+        return @default_tarball_filename unless @default_tarball_filename.nil?
+
+        builder = PDK::Module::Build.new(module_dir: module_path)
+        @default_tarball_filename = builder.package_file
+      end
+
+      def run_validations(opts)
+        # TODO: Surely I can use a pre-existing class for this?
+        PDK::CLI::Util.validate_puppet_version_opts(opts)
+
+        PDK::CLI::Util.module_version_check
+
+        puppet_env = PDK::CLI::Util.puppet_from_opts_or_env(opts)
+        PDK::Util::PuppetVersion.fetch_puppet_dev if opts[:'puppet-dev']
+        PDK::Util::RubyVersion.use(puppet_env[:ruby_version])
+
+        PDK::Util::Bundler.ensure_bundle!(puppet_env[:gemset])
+
+        validator_exit_code, = PDK::Validate.invoke_validators_by_name(PDK.context, PDK::Validate.validator_names, false, options)
+        raise PDK::CLI::ExitWithError, 'An error occured during validation' unless validator_exit_code.zero?
+      end
+
+      def run_documentation(_opts)
+        PDK.logger.info 'Updating documentation using puppet strings'
+        docs_command = PDK::CLI::Exec::InteractiveCommand.new(PDK::CLI::Exec.bundle_bin, 'exec', 'puppet', 'strings', 'generate', '--format', 'markdown', '--out', 'REFERENCE.md')
+        docs_command.context = :module
+        result = docs_command.execute!
+        raise PDK::CLI::ExitWithError, format('An error occured generating the module documentation: %{stdout}', stdout: result[:stdout]) unless result[:exit_code].zero?
+      end
+
+      def run_dependency_checker(_opts)
+        # run dependency-checker and output dependent modules list
+        PDK.logger.info 'Running dependency checks'
+
+        dep_command = PDK::CLI::Exec::Command.new('dependency-checker', 'metadata.json')
+        dep_command.context = :module
+        result = dep_command.execute!
+
+        raise PDK::CLI::ExitWithError, format('An error occured checking the module dependencies: %{stdout}', stdout: result[:stdout]) unless result[:exit_code].zero?
+      end
+
+      # @return [String] Path to the built tarball
+      def run_build(opts)
+        PDK::Module::Build.invoke(opts.dup)
+      end
+
+      def run_publish(_opts, tarball_path)
+        validate_publish_options!
+        raise PDK::CLI::ExitWithError, format('Module tarball %{tarball_path} does not exist', tarball_path: tarball_path) unless PDK::Util::Filesystem.file?(tarball_path)
+
+        # TODO: Replace this code when the upload functionality is added to the forge ruby gem
+        require 'base64'
+        file_data = Base64.encode64(PDK::Util::Filesystem.read_file(tarball_path, open_args: 'rb'))
+
+        PDK.logger.info 'Uploading tarball to puppet forge...'
+        uri = URI(forge_upload_url)
+        require 'net/http'
+        request = Net::HTTP::Post.new(uri.path)
+        request['Authorization'] = "Bearer #{forge_token}"
+        request['Content-Type'] = 'application/json'
+        data = { file: file_data }
+
+        request.body = data.to_json
+
+        require 'openssl'
+        use_ssl = uri.instance_of?(URI::HTTPS)
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+          http.request(request)
+        end
+
+        raise PDK::CLI::ExitWithError, format('Error uploading to Puppet Forge: %{result}', result: response.body) unless response.is_a?(Net::HTTPSuccess)
+
+        PDK.logger.info 'Publish to Forge was successful'
+      end
+
+      def validate_publish_options!
+        return if skip_publish?
+        raise PDK::CLI::ExitWithError, 'Missing forge-upload-url option' unless forge_upload_url
+        raise PDK::CLI::ExitWithError, 'Missing forge-token option' unless forge_token
+      end
+
+      def force?
+        options[:force]
+      end
+
+      def skip_build?
+        options[:'skip-build']
+      end
+
+      def skip_changelog?
+        options[:'skip-changelog']
+      end
+
+      def skip_dependency?
+        options[:'skip-dependency']
+      end
+
+      def skip_documentation?
+        options[:'skip-documentation']
+      end
+
+      def skip_publish?
+        options[:'skip-publish']
+      end
+
+      def skip_validation?
+        options[:'skip-validation']
+      end
+
+      def specified_version
+        options[:version]
+      end
+
+      def specified_package
+        options[:file]
+      end
+
+      def forge_token
+        options[:'forge-token']
+      end
+
+      def forge_upload_url
+        options[:'forge-upload-url']
+      end
+
+      def requires_pdk_compatibility?
+        # Validation, Changelog and Dependency checks require the
+        # module to be PDK Compatible
+        !(skip_validation? && skip_changelog? && skip_dependency?)
+      end
+
+      def requires_forge_compatibility?
+        # Pushing to the for requires the metadata to be forge compatible
+        !skip_publish?
+      end
+
+      # :nocov:
+      # These are just convenience methods and are tested elsewhere
+      def forge_compatible?
+        module_metadata.forge_ready?
+      end
+
+      def pdk_compatible?
+        return @pdk_compatible unless @pdk_compatible.nil?
+
+        builder = PDK::Module::Build.new(module_dir: module_path)
+        @pdk_compatible = builder.module_pdk_compatible?
+      end
+      # :nocov:
+
+      private
+
+      def clear_cached_data
+        @module_metadata = nil
+        @pdk_compatible = nil
+        @default_tarball_filename = nil
+      end
+    end
+  end
+end

data/lib/pdk/module/update.rb

@@ -1,21 +1,19 @@
-require 'pdk/module/convert'
+require 'pdk'
 
 module PDK
   module Module
     class Update < Convert
-      GIT_DESCRIBE_PATTERN = %r{\A(?<base>.+?)-(?<additional_commits>\d+)-g(?<sha>.+)\Z}
+      GIT_DESCRIBE_PATTERN = /\A(?<base>.+?)-(?<additional_commits>\d+)-g(?<sha>.+)\Z/.freeze
 
       def run
+        template_uri.uri_fragment = new_template_version
+
         stage_changes!
 
-        if current_version == new_version
-          PDK.logger.debug _('This module is already up to date with version %{version} of the template.') % {
-            version: new_version,
-          }
-        end
+        PDK.logger.debug format('This module is already up to date with version %{version} of the template.', version: new_version) if current_version == new_version
 
         unless update_manager.changes?
-          PDK::Report.default_target.puts(_('No changes required.'))
+          PDK::Report.default_target.puts('No changes required.')
           return
         end
 
@@ -27,31 +25,31 @@ module PDK
         return if noop?
 
         unless force?
-          message = _('Do you want to continue and make these changes to your module?')
+          message = 'Do you want to continue and make these changes to your module?'
           return unless PDK::CLI::Util.prompt_for_yes(message)
         end
 
-        # Remove these files straight away as these changes are not something that the user needs to review.
-        if needs_bundle_update?
-          update_manager.unlink_file('Gemfile.lock')
-          update_manager.unlink_file(File.join('.bundle', 'config'))
-        end
+        # Remove these files straight away as these changes are not something
+        # that the user needs to review.
+        update_manager.unlink_file('Gemfile.lock')
+        update_manager.unlink_file(File.join('.bundle', 'config'))
 
         update_manager.sync_changes!
 
-        PDK::Util::Bundler.ensure_bundle! if needs_bundle_update?
+        require 'pdk/util/bundler'
+        PDK::Util::Bundler.ensure_bundle!
 
         print_result 'Update completed'
       end
 
       def module_metadata
-        @module_metadata ||= PDK::Module::Metadata.from_file('metadata.json')
+        @module_metadata ||= PDK::Module::Metadata.from_file(File.join(module_dir, 'metadata.json'))
       rescue ArgumentError => e
         raise PDK::CLI::ExitWithError, e.message
       end
 
-      def template_url
-        @template_url ||= module_metadata.data['template-url']
+      def template_uri
+        @template_uri ||= PDK::Util::TemplateURI.new(module_metadata.data['template-url'])
       end
 
       def current_version
@@ -62,8 +60,30 @@ module PDK
         @new_version ||= fetch_remote_version(new_template_version)
       end
 
+      def new_template_version
+        return options[:'template-ref'] if options[:'template-ref']
+
+        if template_uri.default? && PDK::Util::Git.tag?(template_uri.bare_uri, template_uri.uri_fragment) && PDK::Util.package_install?
+          PDK::Util::TemplateURI.default_template_ref
+        else
+          template_uri.uri_fragment
+        end
+      end
+
+      def pinned_to_puppetlabs_template_tag?
+        return false unless template_uri.puppetlabs_template?
+        return false unless PDK::Util::Git.tag?(template_uri.bare_uri, template_uri.uri_fragment)
+        return false if latest_template?
+
+        template_uri.uri_fragment == new_template_version
+      end
+
       private
 
+      def latest_template?
+        [PDK::TEMPLATE_REF, 'master', 'main'].include?(template_uri.uri_fragment)
+      end
+
       def current_template_version
         @current_template_version ||= module_metadata.data['template-ref']
       end
@@ -73,38 +93,29 @@ module PDK
 
         return data if data.nil?
 
-        if data[:base].start_with?('heads/')
-          "#{data[:base].gsub(%r{^heads/}, '')}@#{data[:sha]}"
+        if %r{^(?:heads|remotes)/}.match?(data[:base])
+          "#{data[:base].gsub(%r{^(heads/|remotes/\w+?/)}, '')}@#{data[:sha]}"
         else
           data[:base]
         end
       end
 
-      def new_template_version
-        PDK::Util.default_template_ref
-      end
-
-      def fetch_remote_version(version)
-        return version unless version.include?('/')
+      def fetch_remote_version(template_ref)
+        return template_ref unless current_template_version.is_a?(String)
+        return template_ref if template_ref == PDK::TEMPLATE_REF
 
-        branch = version.partition('/').last
         sha_length = GIT_DESCRIBE_PATTERN.match(current_template_version)[:sha].length - 1
-        "#{branch}@#{PDK::Util::Git.ls_remote(template_url, "refs/heads/#{branch}")[0..sha_length]}"
+        "#{template_ref}@#{PDK::Util::Git.ls_remote(template_uri.bare_uri, template_ref)[0..sha_length]}"
       end
 
       def update_message
-        format_string = if template_url == PDK::Util.puppetlabs_template_url
-                          _('Updating %{module_name} using the default template, from %{current_version} to %{new_version}')
+        format_string = if template_uri.default?
+                          'Updating %{module_name} using the default template, from %{current_version} to %{new_version}'
                         else
-                          _('Updating %{module_name} using the template at %{template_url}, from %{current_version} to %{new_version}')
+                          'Updating %{module_name} using the template at %{template_url}, from %{current_version} to %{new_version}'
                         end
 
-        format_string % {
-          module_name:     module_metadata.data['name'],
-          template_url:    template_url,
-          current_version: current_version,
-          new_version:     new_version,
-        }
+        format(format_string, module_name: module_metadata.data['name'], template_url: template_uri.bare_uri, current_version: current_version, new_version: new_version)
       end
     end
   end