pdk 1.16.0 → 2.1.0

data/lib/pdk/util/bundler.rb

@@ -44,7 +44,7 @@ module PDK
         # If there are missing dependencies after updating the lockfile, let `bundle install`
         # go out and get them. If the specified puppet gem version points to a remote location
         # or local filepath, then run bundle install as well.
-        if !bundle.installed? || (gem_overrides[:puppet] && gem_overrides[:puppet].start_with?('file://', 'git://', 'https://'))
+        if !bundle.installed?(gem_overrides) || (gem_overrides[:puppet] && gem_overrides[:puppet].start_with?('file://', 'git://', 'https://'))
           bundle.install!(gem_overrides)
         end
 
@@ -210,6 +210,7 @@ module PDK
         end
 
         def binstubs!(gems)
+          raise PDK::CLI::FatalError, _('Unable to install requested binstubs as the Gemfile is missing') if gemfile.nil?
           binstub_dir = File.join(File.dirname(gemfile), 'bin')
           return true if gems.all? { |gem| PDK::Util::Filesystem.file?(File.join(binstub_dir, gem)) }
 

data/lib/pdk/util/changelog_generator.rb

@@ -5,11 +5,20 @@ module PDK
     module ChangelogGenerator
       # Taken from the version regex in https://forgeapi.puppet.com/schemas/module.json
       VERSION_REGEX = %r{^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$}
+      GEM = 'github_changelog_generator'.freeze
 
       # Raises if the github_changelog_generator is not available
       def self.github_changelog_generator_available!
-        require 'bundler'
-        raise PDK::CLI::ExitWithError, _('Unable to generate the changelog as the github_changelog_generator gem is not installed') unless Bundler.rubygems.find_name('github_changelog_generator').any?
+        check_command = PDK::CLI::Exec::InteractiveCommand.new(PDK::CLI::Exec.bundle_bin, 'show', 'github_changelog_generator')
+        check_command.context = :module
+
+        result = check_command.execute!
+
+        return if result[:exit_code].zero?
+
+        raise PDK::CLI::ExitWithError, _(
+          'Unable to generate the changelog as the %{gem} gem is not included in this module\'s Gemfile',
+        ) % { gem: GEM }
       end
 
       # Runs the Changelog Generator gem (in the module's context) to automatically create a CHANGLELOG.MD file

data/lib/pdk/util/json_finder.rb

@@ -0,0 +1,84 @@
+module PDK
+  module Util
+    # Processes a given string, looking for JSON objects and parsing them.
+    #
+    # @example A string with a JSON object and some junk characters
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}').objects
+    #   => [{ 'bar' => 1 }]
+    #
+    # @example A string with mulitple JSON objects
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}baz{"gronk":2}').objects
+    #   => [{ 'bar' => 1 }, { 'gronk' => 2 }]
+    class JSONFinder
+      # Creates a new instance of PDK::Util::JSONFinder.
+      #
+      # @param string [String] the string to find JSON objects inside of.
+      #
+      # @return [PDK::Util::JSONFinder] a new PDK::Util::JSONFinder object.
+      def initialize(string)
+        require 'strscan'
+
+        @scanner = StringScanner.new(string)
+      end
+
+      # Returns the parsed JSON objects from the string.
+      #
+      # @return [Array[Hash]] the parsed JSON objects present in the string.
+      def objects
+        return @objects unless @objects.nil?
+
+        require 'json'
+
+        until @scanner.eos?
+          @scanner.getch until @scanner.peek(1) == '{' || @scanner.eos?
+
+          (@objects ||= []) << begin
+                                 JSON.parse(read_object(true) || '')
+                               rescue JSON::ParserError
+                                 nil
+                               end
+        end
+
+        @objects = @objects.compact
+      end
+
+      private
+
+      # Recursively process the string to extract a complete JSON object.
+      #
+      # @param new_object [Boolean] Set to true if processing a new object to
+      #   capture the opening brace. Set to false if being called recursively
+      #   where the opening brace has already been captured.
+      #
+      # @return [String] The matched substring containing a JSON object.
+      def read_object(new_object = false)
+        matched_text = new_object ? @scanner.getch : ''
+
+        until @scanner.eos?
+          text = @scanner.scan_until(%r{(?:(?<!\\)"|\{|\})})
+          unless text
+            @scanner.terminate
+            return nil
+          end
+          matched_text += text
+
+          case @scanner.matched
+          when '}'
+            break
+          when '"'
+            text = @scanner.scan_until(%r{(?<!\\)"})
+            unless text
+              @scanner.terminate
+              return nil
+            end
+            matched_text += text
+          else
+            matched_text += read_object
+          end
+        end
+
+        matched_text
+      end
+    end
+  end
+end

data/lib/pdk/util/puppet_strings.rb

@@ -92,12 +92,12 @@ module PDK
         require 'pdk/generate'
 
         generators = PDK::Generate.generators.select do |gen|
-          gen.respond_to?(:puppet_strings_type) && !gen.puppet_strings_type.nil?
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && !gen::PUPPET_STRINGS_TYPE.nil?
         end
 
         known_objects = generate_hash
 
-        generators.map { |gen| [gen, known_objects[gen.puppet_strings_type]] }.reject do |_, obj|
+        generators.map { |gen| [gen, known_objects[gen::PUPPET_STRINGS_TYPE]] }.reject do |_, obj|
           obj.nil? || obj.empty?
         end
       end
@@ -117,7 +117,7 @@ module PDK
         require 'pdk/generate'
 
         PDK::Generate.generators.find do |gen|
-          gen.respond_to?(:puppet_strings_type) && gen.puppet_strings_type == type
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && gen::PUPPET_STRINGS_TYPE == type
         end
       end
     end

data/lib/pdk/util/puppet_version.rb

@@ -17,7 +17,7 @@ module PDK
 
       PE_VERSIONS_URL = 'https://forgeapi.puppet.com/private/versions/pe'.freeze
       DEFAULT_PUPPET_DEV_URL = 'https://github.com/puppetlabs/puppet'.freeze
-      DEFAULT_PUPPET_DEV_BRANCH = 'master'.freeze
+      DEFAULT_PUPPET_DEV_BRANCH = 'main'.freeze
 
       def puppet_dev_env
         require 'pdk/util/ruby_version'
@@ -80,7 +80,7 @@ module PDK
         end
 
         # Reset local repo to latest
-        reset_result = PDK::Util::Git.git('-C', puppet_dev_path, 'reset', '--hard', 'origin/master')
+        reset_result = PDK::Util::Git.git('-C', puppet_dev_path, 'reset', '--hard', "origin/#{DEFAULT_PUPPET_DEV_BRANCH}")
 
         @puppet_dev_fetched = true
         return if reset_result[:exit_code].zero?

data/lib/pdk/util/ruby_version.rb

@@ -60,7 +60,11 @@ module PDK
                                       #
                                       # PDK::Util::PuppetVersion.find_gem_for('5.5.10')[:ruby_version]
                                       #
-                                      PDK::Util::PuppetVersion.latest_available[:ruby_version]
+                                      # For using the latest puppet gem:
+                                      # PDK::Util::PuppetVersion.latest_available[:ruby_version]
+                                      #
+                                      # Temporarily lock to Ruby 2.5.x as default until 2.7.x ecosystem is sorted
+                                      versions.keys.detect { |v| v =~ %r{^2\.5} }
                                     else
                                       # TODO: may not be a safe assumption that highest available version should be default
                                       # WARNING Do NOT use PDK::Util::PuppetVersion.*** methods as it can recurse into this

data/lib/pdk/util/template_uri.rb

@@ -26,13 +26,13 @@ module PDK
       #
       # file:///c:/foo (git clone location)
       # c:/foo (shell paths)
-      # file:///c:/foo#master (only for metadata)
-      # c:/foo#master (only for metadata)
+      # file:///c:/foo#main (only for metadata)
+      # c:/foo#main (only for metadata)
       #
       # non output formats:
       #
       # /c:/foo (internal use only)
-      # /c:/foo#master (internal use only)
+      # /c:/foo#main (internal use only)
       #
       def initialize(opts_or_uri)
         require 'addressable'
@@ -192,7 +192,6 @@ module PDK
       #   directory, and :allow_fallback contains a Boolean that specifies if
       #   the lookup process should proceed to the next template directory if
       #   the template file is not in this template directory.
-      #
       def self.templates(opts)
         require 'pdk/answer_file'
         require 'pdk/util'
@@ -218,10 +217,11 @@ module PDK
                        else
                          nil
                        end
-        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(PDK.answers['template-url'])
+        default_template_url = PDK.config.get_within_scopes('module_defaults.template-url')
+        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(default_template_url)
                         Addressable::URI.parse(default_template_uri)
-                      elsif PDK.answers['template-url']
-                        new(uri_safe(PDK.answers['template-url'])).uri
+                      elsif default_template_url
+                        new(uri_safe(default_template_url)).uri
                       else
                         nil
                       end
@@ -241,11 +241,11 @@ module PDK
         require 'pdk/util'
         require 'pdk/version'
 
-        return 'master' if PDK::Util.development_mode?
+        return 'main' if PDK::Util.development_mode?
         return PDK::TEMPLATE_REF if uri.nil?
 
         uri = new(uri) unless uri.is_a?(self)
-        uri.default? ? PDK::TEMPLATE_REF : 'master'
+        uri.default? ? PDK::TEMPLATE_REF : 'main'
       end
 
       # @returns Addressable::URI
@@ -260,21 +260,20 @@ module PDK
         found_template[:uri]
       end
 
-      def self.valid_template?(template)
+      def self.valid_template?(template, context = PDK.context)
         require 'addressable'
 
         return false if template.nil? || !template.is_a?(Hash)
         return false if template[:uri].nil? || !template[:uri].is_a?(Addressable::URI)
 
         return true if PDK::Util::Git.repo?(bare_uri(template[:uri]))
-
         path = human_readable(template[:uri].path)
         if PDK::Util::Filesystem.directory?(path)
           # We know that it's not a git repository, but it's a valid path on disk
           begin
-            PDK::Module::TemplateDir.validate_module_template!(path)
-            return true
-          rescue ArgumentError
+            renderer = PDK::Template::Renderer.instance(path, template[:uri], context)
+            return !renderer.nil?
+          rescue StandardError
             nil
           end
         end

data/lib/pdk/util/vendored_file.rb

@@ -55,8 +55,7 @@ module PDK
         uri = URI.parse(url)
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = true
-        # TODO: Get rid of this, possible workaround:
-        # https://github.com/glennsarti/dev-tools/blob/master/RubyCerts.ps1
+        # TODO: Get rid of this
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE if Gem.win_platform?
         request = Net::HTTP::Get.new(uri.request_uri)
         response = http.request(request)

data/lib/pdk/validate.rb

@@ -2,37 +2,91 @@ require 'pdk'
 
 module PDK
   module Validate
-    # TODO: Fix validator namespacing
-    autoload :BaseValidator, 'pdk/validate/base_validator'
-    autoload :MetadataJSONLint, 'pdk/validate/metadata/metadata_json_lint'
-    autoload :MetadataSyntax, 'pdk/validate/metadata/metadata_syntax'
-    autoload :MetadataValidator, 'pdk/validate/metadata_validator'
-    autoload :PuppetEPP, 'pdk/validate/puppet/puppet_epp'
-    autoload :PuppetLint, 'pdk/validate/puppet/puppet_lint'
-    autoload :PuppetSyntax, 'pdk/validate/puppet/puppet_syntax'
-    autoload :PuppetValidator, 'pdk/validate/puppet_validator'
-    autoload :Rubocop, 'pdk/validate/ruby/rubocop'
-    autoload :RubyValidator, 'pdk/validate/ruby_validator'
-    autoload :TasksValidator, 'pdk/validate/tasks_validator'
-    autoload :YAMLValidator, 'pdk/validate/yaml_validator'
+    autoload :ExternalCommandValidator, 'pdk/validate/external_command_validator'
+    autoload :InternalRubyValidator, 'pdk/validate/internal_ruby_validator'
+    autoload :InvokableValidator, 'pdk/validate/invokable_validator'
+    autoload :Validator, 'pdk/validate/validator'
+    autoload :ValidatorGroup, 'pdk/validate/validator_group'
 
-    class Tasks
-      autoload :Name, 'pdk/validate/tasks/name'
-      autoload :MetadataLint, 'pdk/validate/tasks/metadata_lint'
+    module ControlRepo
+      autoload :ControlRepoValidatorGroup, 'pdk/validate/control_repo/control_repo_validator_group'
+      autoload :EnvironmentConfValidator, 'pdk/validate/control_repo/environment_conf_validator'
     end
 
-    class YAML
-      autoload :Syntax, 'pdk/validate/yaml/syntax'
+    module Metadata
+      autoload :MetadataJSONLintValidator, 'pdk/validate/metadata/metadata_json_lint_validator'
+      autoload :MetadataSyntaxValidator, 'pdk/validate/metadata/metadata_syntax_validator'
+      autoload :MetadataValidatorGroup, 'pdk/validate/metadata/metadata_validator_group'
+    end
+
+    module Puppet
+      autoload :PuppetEPPValidator, 'pdk/validate/puppet/puppet_epp_validator'
+      autoload :PuppetLintValidator, 'pdk/validate/puppet/puppet_lint_validator'
+      autoload :PuppetSyntaxValidator, 'pdk/validate/puppet/puppet_syntax_validator'
+      autoload :PuppetValidatorGroup, 'pdk/validate/puppet/puppet_validator_group'
+    end
+
+    module Ruby
+      autoload :RubyRubocopValidator, 'pdk/validate/ruby/ruby_rubocop_validator'
+      autoload :RubyValidatorGroup, 'pdk/validate/ruby/ruby_validator_group'
+    end
+
+    module Tasks
+      autoload :TasksMetadataLintValidator, 'pdk/validate/tasks/tasks_metadata_lint_validator'
+      autoload :TasksNameValidator, 'pdk/validate/tasks/tasks_name_validator'
+      autoload :TasksValidatorGroup, 'pdk/validate/tasks/tasks_validator_group'
+    end
+
+    module YAML
+      autoload :YAMLSyntaxValidator, 'pdk/validate/yaml/yaml_syntax_validator'
+      autoload :YAMLValidatorGroup, 'pdk/validate/yaml/yaml_validator_group'
     end
 
     def self.validators
-      @validators ||= [
-        MetadataValidator,
-        YAMLValidator,
-        PuppetValidator,
-        RubyValidator,
-        TasksValidator,
-      ].freeze
+      validator_hash.values
+    end
+
+    def self.validator_names
+      validator_hash.keys
+    end
+
+    # @api private
+    def self.validator_hash
+      # TODO: This isn't the most performant... But with only 6 items, it's fine
+      @validator_hash ||= [
+        ControlRepo::ControlRepoValidatorGroup,
+        Metadata::MetadataValidatorGroup,
+        Puppet::PuppetValidatorGroup,
+        Ruby::RubyValidatorGroup,
+        Tasks::TasksValidatorGroup,
+        YAML::YAMLValidatorGroup,
+      ].map { |klass| [klass.new.name, klass] }.to_h.freeze
+    end
+
+    def self.invoke_validators_by_name(context, names, parallel = false, options = {})
+      instances = names.select { |name| validator_names.include?(name) }
+                       .map { |name| validator_hash[name].new(context, options) }
+                       .select { |instance| instance.valid_in_context? }
+                       .each { |instance| instance.prepare_invoke! }
+      report = PDK::Report.new
+
+      # Nothing to validate then nothing to do.
+      return [0, report] if instances.empty?
+
+      require 'pdk/cli/exec_group'
+      exec_group = PDK::CLI::ExecGroup.create(
+        _('Validating module using %{num_of_threads} threads' % { num_of_threads: instances.count }),
+        { parallel: parallel },
+        options,
+      )
+
+      instances.each do |validator|
+        exec_group.register do
+          validator.invoke(report)
+        end
+      end
+
+      [exec_group.exit_code, report]
     end
 
     class ParseOutputError < StandardError; end

data/lib/pdk/validate/control_repo/control_repo_validator_group.rb

@@ -0,0 +1,23 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module ControlRepo
+      class ControlRepoValidatorGroup < ValidatorGroup
+        def name
+          'control-repo'
+        end
+
+        def valid_in_context?
+          context.is_a?(PDK::Context::ControlRepo)
+        end
+
+        def validators
+          [
+            EnvironmentConfValidator,
+          ].freeze
+        end
+      end
+    end
+  end
+end

data/lib/pdk/validate/control_repo/environment_conf_validator.rb

@@ -0,0 +1,98 @@
+require 'pdk'
+
+module PDK
+  module Validate
+    module ControlRepo
+      class EnvironmentConfValidator < InternalRubyValidator
+        ALLOWED_SETTINGS = %w[modulepath manifest config_version environment_timeout].freeze
+
+        def name
+          'environment-conf'
+        end
+
+        def valid_in_context?
+          context.is_a?(PDK::Context::ControlRepo)
+        end
+
+        def pattern
+          ['environment.conf']
+        end
+
+        def spinner_text
+          _('Checking Puppet Environment settings (%{patterns}).') % {
+            patterns: pattern.join(' '),
+          }
+        end
+
+        def validate_target(report, target)
+          unless PDK::Util::Filesystem.readable?(target)
+            report.add_event(
+              file: target,
+              source: name,
+              state: :failure,
+              severity: 'error',
+              message: _('Could not be read.'),
+            )
+            return 1
+          end
+
+          is_valid = true
+          begin
+            env_conf = PDK::ControlRepo.environment_conf_as_config(target)
+
+            env_conf.resolve.each do |setting_name, setting_value|
+              # Remove the 'environment.' setting_name prefix
+              setting_name = setting_name.slice(12..-1)
+              next if ALLOWED_SETTINGS.include?(setting_name)
+              # A hash indicates that the ini file has a section in it.
+              message = if setting_value.is_a?(Hash)
+                          _("Invalid section '%{name}'") % { name: setting_name }
+                        else
+                          _("Invalid setting '%{name}'") % { name: setting_name }
+                        end
+
+              report.add_event(
+                file:     target,
+                source:   name,
+                state:    :failure,
+                severity: 'error',
+                message:  message,
+              )
+              is_valid = false
+            end
+
+            timeout = env_conf.fetch('environment_timeout', nil)
+            unless timeout.nil? || timeout == '0' || timeout == 'unlimited'
+              report.add_event(
+                file:     target,
+                source:   name,
+                state:    :failure,
+                severity: 'error',
+                message:  _("environment_timeout is set to '%{timeout}' but should be 0, 'unlimited' or not set.") % { timeout: timeout },
+              )
+              is_valid = false
+            end
+
+            return 1 unless is_valid
+            report.add_event(
+              file:     target,
+              source:   name,
+              state:    :passed,
+              severity: 'ok',
+            )
+            return 0
+          rescue StandardError => e
+            report.add_event(
+              file:     target,
+              source:   name,
+              state:    :failure,
+              severity: 'error',
+              message:  e.message,
+            )
+            return 1
+          end
+        end
+      end
+    end
+  end
+end