pdk 1.15.0 → 2.0.0

data/lib/pdk/util/bundler.rb

@@ -44,7 +44,7 @@ module PDK
         # If there are missing dependencies after updating the lockfile, let `bundle install`
         # go out and get them. If the specified puppet gem version points to a remote location
         # or local filepath, then run bundle install as well.
-        if !bundle.installed? || (gem_overrides[:puppet] && gem_overrides[:puppet].start_with?('file://', 'git://', 'https://'))
+        if !bundle.installed?(gem_overrides) || (gem_overrides[:puppet] && gem_overrides[:puppet].start_with?('file://', 'git://', 'https://'))
           bundle.install!(gem_overrides)
         end
 
@@ -210,6 +210,7 @@ module PDK
         end
 
         def binstubs!(gems)
+          raise PDK::CLI::FatalError, _('Unable to install requested binstubs as the Gemfile is missing') if gemfile.nil?
           binstub_dir = File.join(File.dirname(gemfile), 'bin')
           return true if gems.all? { |gem| PDK::Util::Filesystem.file?(File.join(binstub_dir, gem)) }
 

data/lib/pdk/util/changelog_generator.rb

@@ -5,11 +5,20 @@ module PDK
     module ChangelogGenerator
       # Taken from the version regex in https://forgeapi.puppet.com/schemas/module.json
       VERSION_REGEX = %r{^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$}
+      GEM = 'github_changelog_generator'.freeze
 
       # Raises if the github_changelog_generator is not available
       def self.github_changelog_generator_available!
-        require 'bundler'
-        raise PDK::CLI::ExitWithError, _('Unable to generate the changelog as the github_changelog_generator gem is not installed') unless Bundler.rubygems.find_name('github_changelog_generator').any?
+        check_command = PDK::CLI::Exec::InteractiveCommand.new(PDK::CLI::Exec.bundle_bin, 'show', 'github_changelog_generator')
+        check_command.context = :module
+
+        result = check_command.execute!
+
+        return if result[:exit_code].zero?
+
+        raise PDK::CLI::ExitWithError, _(
+          'Unable to generate the changelog as the %{gem} gem is not included in this module\'s Gemfile',
+        ) % { gem: GEM }
       end
 
       # Runs the Changelog Generator gem (in the module's context) to automatically create a CHANGLELOG.MD file

data/lib/pdk/util/env.rb

@@ -4,25 +4,42 @@ require 'forwardable'
 module PDK
   module Util
     class Env
-      class << self
+      class WindowsEnv
         extend Forwardable
 
-        def_delegators :env_hash, :[], :key?, :fetch, :select, :reject
+        # Note, these delegators may not have case insensitive keys
+        def_delegators :env_hash, :fetch, :select, :reject
 
         def []=(key, value)
-          if Gem.win_platform?
-            PDK::Util::Windows::Process.set_environment_variable(key, value)
-          else
-            ENV[key] = value
+          PDK::Util::Windows::Process.set_environment_variable(key, value)
+        end
+
+        def key?(key)
+          !env_hash.keys.find { |item| key.casecmp(item).zero? }.nil?
+        end
+
+        def [](key)
+          env_hash.each do |item, value|
+            next unless key.casecmp(item).zero?
+            return value
           end
+          nil
         end
 
+        private
+
         def env_hash
-          if Gem.win_platform?
-            PDK::Util::Windows::Process.environment_hash
-          else
-            ENV
-          end
+          PDK::Util::Windows::Process.environment_hash
+        end
+      end
+
+      class << self
+        extend Forwardable
+
+        def_delegators :implementation, :key?, :[], :[]=, :fetch, :select, :reject
+
+        def implementation
+          @implementation ||= Gem.win_platform? ? WindowsEnv.new : ENV
         end
       end
     end

data/lib/pdk/util/json_finder.rb

@@ -0,0 +1,84 @@
+module PDK
+  module Util
+    # Processes a given string, looking for JSON objects and parsing them.
+    #
+    # @example A string with a JSON object and some junk characters
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}').objects
+    #   => [{ 'bar' => 1 }]
+    #
+    # @example A string with mulitple JSON objects
+    #   PDK::Util::JSONFinder.new('foo{"bar":1}baz{"gronk":2}').objects
+    #   => [{ 'bar' => 1 }, { 'gronk' => 2 }]
+    class JSONFinder
+      # Creates a new instance of PDK::Util::JSONFinder.
+      #
+      # @param string [String] the string to find JSON objects inside of.
+      #
+      # @return [PDK::Util::JSONFinder] a new PDK::Util::JSONFinder object.
+      def initialize(string)
+        require 'strscan'
+
+        @scanner = StringScanner.new(string)
+      end
+
+      # Returns the parsed JSON objects from the string.
+      #
+      # @return [Array[Hash]] the parsed JSON objects present in the string.
+      def objects
+        return @objects unless @objects.nil?
+
+        require 'json'
+
+        until @scanner.eos?
+          @scanner.getch until @scanner.peek(1) == '{' || @scanner.eos?
+
+          (@objects ||= []) << begin
+                                 JSON.parse(read_object(true) || '')
+                               rescue JSON::ParserError
+                                 nil
+                               end
+        end
+
+        @objects = @objects.compact
+      end
+
+      private
+
+      # Recursively process the string to extract a complete JSON object.
+      #
+      # @param new_object [Boolean] Set to true if processing a new object to
+      #   capture the opening brace. Set to false if being called recursively
+      #   where the opening brace has already been captured.
+      #
+      # @return [String] The matched substring containing a JSON object.
+      def read_object(new_object = false)
+        matched_text = new_object ? @scanner.getch : ''
+
+        until @scanner.eos?
+          text = @scanner.scan_until(%r{(?:(?<!\\)"|\{|\})})
+          unless text
+            @scanner.terminate
+            return nil
+          end
+          matched_text += text
+
+          case @scanner.matched
+          when '}'
+            break
+          when '"'
+            text = @scanner.scan_until(%r{(?<!\\)"})
+            unless text
+              @scanner.terminate
+              return nil
+            end
+            matched_text += text
+          else
+            matched_text += read_object
+          end
+        end
+
+        matched_text
+      end
+    end
+  end
+end

data/lib/pdk/util/puppet_strings.rb

@@ -92,12 +92,12 @@ module PDK
         require 'pdk/generate'
 
         generators = PDK::Generate.generators.select do |gen|
-          gen.respond_to?(:puppet_strings_type) && !gen.puppet_strings_type.nil?
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && !gen::PUPPET_STRINGS_TYPE.nil?
         end
 
         known_objects = generate_hash
 
-        generators.map { |gen| [gen, known_objects[gen.puppet_strings_type]] }.reject do |_, obj|
+        generators.map { |gen| [gen, known_objects[gen::PUPPET_STRINGS_TYPE]] }.reject do |_, obj|
           obj.nil? || obj.empty?
         end
       end
@@ -117,7 +117,7 @@ module PDK
         require 'pdk/generate'
 
         PDK::Generate.generators.find do |gen|
-          gen.respond_to?(:puppet_strings_type) && gen.puppet_strings_type == type
+          gen.const_defined?(:PUPPET_STRINGS_TYPE) && gen::PUPPET_STRINGS_TYPE == type
         end
       end
     end

data/lib/pdk/util/puppet_version.rb

@@ -17,7 +17,7 @@ module PDK
 
       PE_VERSIONS_URL = 'https://forgeapi.puppet.com/private/versions/pe'.freeze
       DEFAULT_PUPPET_DEV_URL = 'https://github.com/puppetlabs/puppet'.freeze
-      DEFAULT_PUPPET_DEV_BRANCH = 'master'.freeze
+      DEFAULT_PUPPET_DEV_BRANCH = 'main'.freeze
 
       def puppet_dev_env
         require 'pdk/util/ruby_version'
@@ -80,7 +80,7 @@ module PDK
         end
 
         # Reset local repo to latest
-        reset_result = PDK::Util::Git.git('-C', puppet_dev_path, 'reset', '--hard', 'origin/master')
+        reset_result = PDK::Util::Git.git('-C', puppet_dev_path, 'reset', '--hard', "origin/#{DEFAULT_PUPPET_DEV_BRANCH}")
 
         @puppet_dev_fetched = true
         return if reset_result[:exit_code].zero?

data/lib/pdk/util/ruby_version.rb

@@ -60,7 +60,11 @@ module PDK
                                       #
                                       # PDK::Util::PuppetVersion.find_gem_for('5.5.10')[:ruby_version]
                                       #
-                                      PDK::Util::PuppetVersion.latest_available[:ruby_version]
+                                      # For using the latest puppet gem:
+                                      # PDK::Util::PuppetVersion.latest_available[:ruby_version]
+                                      #
+                                      # Temporarily lock to Ruby 2.5.x as default until 2.7.x ecosystem is sorted
+                                      versions.keys.detect { |v| v =~ %r{^2\.5} }
                                     else
                                       # TODO: may not be a safe assumption that highest available version should be default
                                       # WARNING Do NOT use PDK::Util::PuppetVersion.*** methods as it can recurse into this
@@ -115,9 +119,7 @@ module PDK
         else
           # This allows the subprocess to find the 'bundler' gem, which isn't
           # in GEM_HOME for gem installs.
-          # TODO: There must be a better way to do this than shelling out to
-          # gem... Perhaps can be replaced with "Gem.path"?
-          [File.absolute_path(File.join(`gem which bundler`, '..', '..', '..', '..'))]
+          [File.absolute_path(File.join(bundler_basedir, '..', '..', '..'))]
         end
       end
 
@@ -128,6 +130,7 @@ module PDK
       def gem_home
         require 'pdk/util'
 
+        # TODO: bundle install --path is deprecated
         # `bundle install --path` ignores all "system" installed gems and
         # causes unnecessary package installs. `bundle install` (without
         # --path) installs into GEM_HOME, which by default is non-user
@@ -167,6 +170,10 @@ module PDK
       def versions
         self.class.versions
       end
+
+      def bundler_basedir
+        Gem::Specification.latest_specs.find { |spec| spec.name.eql?('bundler') }.base_dir
+      end
     end
   end
 end

data/lib/pdk/util/template_uri.rb

@@ -26,13 +26,13 @@ module PDK
       #
       # file:///c:/foo (git clone location)
       # c:/foo (shell paths)
-      # file:///c:/foo#master (only for metadata)
-      # c:/foo#master (only for metadata)
+      # file:///c:/foo#main (only for metadata)
+      # c:/foo#main (only for metadata)
       #
       # non output formats:
       #
       # /c:/foo (internal use only)
-      # /c:/foo#master (internal use only)
+      # /c:/foo#main (internal use only)
       #
       def initialize(opts_or_uri)
         require 'addressable'
@@ -94,6 +94,10 @@ module PDK
         bare_uri == PDK::Util::TemplateURI.bare_uri(PDK::Util::TemplateURI.default_template_addressable_uri)
       end
 
+      def default_ref?
+        uri_fragment == self.class.default_template_ref(self)
+      end
+
       def puppetlabs_template?
         self.class.packaged_template?(bare_uri) || bare_uri == PDK_TEMPLATE_URL
       end
@@ -188,7 +192,6 @@ module PDK
       #   directory, and :allow_fallback contains a Boolean that specifies if
       #   the lookup process should proceed to the next template directory if
       #   the template file is not in this template directory.
-      #
       def self.templates(opts)
         require 'pdk/answer_file'
         require 'pdk/util'
@@ -214,10 +217,11 @@ module PDK
                        else
                          nil
                        end
-        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(PDK.answers['template-url'])
+        default_template_url = PDK.config.get_within_scopes('module_defaults.template-url')
+        answers_uri = if [PACKAGED_TEMPLATE_KEYWORD, DEPRECATED_TEMPLATE_URL].include?(default_template_url)
                         Addressable::URI.parse(default_template_uri)
-                      elsif PDK.answers['template-url']
-                        new(uri_safe(PDK.answers['template-url'])).uri
+                      elsif default_template_url
+                        new(uri_safe(default_template_url)).uri
                       else
                         nil
                       end
@@ -237,11 +241,11 @@ module PDK
         require 'pdk/util'
         require 'pdk/version'
 
-        return 'master' if PDK::Util.development_mode?
+        return 'main' if PDK::Util.development_mode?
         return PDK::TEMPLATE_REF if uri.nil?
 
         uri = new(uri) unless uri.is_a?(self)
-        uri.default? ? PDK::TEMPLATE_REF : 'master'
+        uri.default? ? PDK::TEMPLATE_REF : 'main'
       end
 
       # @returns Addressable::URI
@@ -256,21 +260,20 @@ module PDK
         found_template[:uri]
       end
 
-      def self.valid_template?(template)
+      def self.valid_template?(template, context = PDK.context)
         require 'addressable'
 
         return false if template.nil? || !template.is_a?(Hash)
         return false if template[:uri].nil? || !template[:uri].is_a?(Addressable::URI)
 
         return true if PDK::Util::Git.repo?(bare_uri(template[:uri]))
-
         path = human_readable(template[:uri].path)
         if PDK::Util::Filesystem.directory?(path)
           # We know that it's not a git repository, but it's a valid path on disk
           begin
-            PDK::Module::TemplateDir.validate_module_template!(path)
-            return true
-          rescue ArgumentError
+            renderer = PDK::Template::Renderer.instance(path, template[:uri], context)
+            return !renderer.nil?
+          rescue StandardError
             nil
           end
         end

data/lib/pdk/util/vendored_file.rb

@@ -55,8 +55,7 @@ module PDK
         uri = URI.parse(url)
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = true
-        # TODO: Get rid of this, possible workaround:
-        # https://github.com/glennsarti/dev-tools/blob/master/RubyCerts.ps1
+        # TODO: Get rid of this
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE if Gem.win_platform?
         request = Net::HTTP::Get.new(uri.request_uri)
         response = http.request(request)

data/lib/pdk/validate.rb

@@ -2,37 +2,91 @@ require 'pdk'
 
 module PDK
   module Validate
-    # TODO: Fix validator namespacing
-    autoload :BaseValidator, 'pdk/validate/base_validator'
-    autoload :MetadataJSONLint, 'pdk/validate/metadata/metadata_json_lint'
-    autoload :MetadataSyntax, 'pdk/validate/metadata/metadata_syntax'
-    autoload :MetadataValidator, 'pdk/validate/metadata_validator'
-    autoload :PuppetEPP, 'pdk/validate/puppet/puppet_epp'
-    autoload :PuppetLint, 'pdk/validate/puppet/puppet_lint'
-    autoload :PuppetSyntax, 'pdk/validate/puppet/puppet_syntax'
-    autoload :PuppetValidator, 'pdk/validate/puppet_validator'
-    autoload :Rubocop, 'pdk/validate/ruby/rubocop'
-    autoload :RubyValidator, 'pdk/validate/ruby_validator'
-    autoload :TasksValidator, 'pdk/validate/tasks_validator'
-    autoload :YAMLValidator, 'pdk/validate/yaml_validator'
+    autoload :ExternalCommandValidator, 'pdk/validate/external_command_validator'
+    autoload :InternalRubyValidator, 'pdk/validate/internal_ruby_validator'
+    autoload :InvokableValidator, 'pdk/validate/invokable_validator'
+    autoload :Validator, 'pdk/validate/validator'
+    autoload :ValidatorGroup, 'pdk/validate/validator_group'
 
-    class Tasks
-      autoload :Name, 'pdk/validate/tasks/name'
-      autoload :MetadataLint, 'pdk/validate/tasks/metadata_lint'
+    module ControlRepo
+      autoload :ControlRepoValidatorGroup, 'pdk/validate/control_repo/control_repo_validator_group'
+      autoload :EnvironmentConfValidator, 'pdk/validate/control_repo/environment_conf_validator'
     end
 
-    class YAML
-      autoload :Syntax, 'pdk/validate/yaml/syntax'
+    module Metadata
+      autoload :MetadataJSONLintValidator, 'pdk/validate/metadata/metadata_json_lint_validator'
+      autoload :MetadataSyntaxValidator, 'pdk/validate/metadata/metadata_syntax_validator'
+      autoload :MetadataValidatorGroup, 'pdk/validate/metadata/metadata_validator_group'
+    end
+
+    module Puppet
+      autoload :PuppetEPPValidator, 'pdk/validate/puppet/puppet_epp_validator'
+      autoload :PuppetLintValidator, 'pdk/validate/puppet/puppet_lint_validator'
+      autoload :PuppetSyntaxValidator, 'pdk/validate/puppet/puppet_syntax_validator'
+      autoload :PuppetValidatorGroup, 'pdk/validate/puppet/puppet_validator_group'
+    end
+
+    module Ruby
+      autoload :RubyRubocopValidator, 'pdk/validate/ruby/ruby_rubocop_validator'
+      autoload :RubyValidatorGroup, 'pdk/validate/ruby/ruby_validator_group'
+    end
+
+    module Tasks
+      autoload :TasksMetadataLintValidator, 'pdk/validate/tasks/tasks_metadata_lint_validator'
+      autoload :TasksNameValidator, 'pdk/validate/tasks/tasks_name_validator'
+      autoload :TasksValidatorGroup, 'pdk/validate/tasks/tasks_validator_group'
+    end
+
+    module YAML
+      autoload :YAMLSyntaxValidator, 'pdk/validate/yaml/yaml_syntax_validator'
+      autoload :YAMLValidatorGroup, 'pdk/validate/yaml/yaml_validator_group'
     end
 
     def self.validators
-      @validators ||= [
-        MetadataValidator,
-        YAMLValidator,
-        PuppetValidator,
-        RubyValidator,
-        TasksValidator,
-      ].freeze
+      validator_hash.values
+    end
+
+    def self.validator_names
+      validator_hash.keys
+    end
+
+    # @api private
+    def self.validator_hash
+      # TODO: This isn't the most performant... But with only 6 items, it's fine
+      @validator_hash ||= [
+        ControlRepo::ControlRepoValidatorGroup,
+        Metadata::MetadataValidatorGroup,
+        Puppet::PuppetValidatorGroup,
+        Ruby::RubyValidatorGroup,
+        Tasks::TasksValidatorGroup,
+        YAML::YAMLValidatorGroup,
+      ].map { |klass| [klass.new.name, klass] }.to_h.freeze
+    end
+
+    def self.invoke_validators_by_name(context, names, parallel = false, options = {})
+      instances = names.select { |name| validator_names.include?(name) }
+                       .map { |name| validator_hash[name].new(context, options) }
+                       .select { |instance| instance.valid_in_context? }
+                       .each { |instance| instance.prepare_invoke! }
+      report = PDK::Report.new
+
+      # Nothing to validate then nothing to do.
+      return [0, report] if instances.empty?
+
+      require 'pdk/cli/exec_group'
+      exec_group = PDK::CLI::ExecGroup.create(
+        _('Validating module using %{num_of_threads} threads' % { num_of_threads: instances.count }),
+        { parallel: parallel },
+        options,
+      )
+
+      instances.each do |validator|
+        exec_group.register do
+          validator.invoke(report)
+        end
+      end
+
+      [exec_group.exit_code, report]
     end
 
     class ParseOutputError < StandardError; end