pdk 1.16.0 → 1.17.0

data/lib/pdk/cli/get/config.rb

@@ -0,0 +1,24 @@
+module PDK::CLI
+  @get_config_cmd = @get_cmd.define_command do
+    name 'config'
+    usage _('config [name]')
+    summary _('Retrieve the configuration for <name>. If not specified, retrieve all configuration settings')
+
+    run do |_opts, args, _cmd|
+      item_name = args[0]
+      resolved_config = PDK.config.resolve(item_name)
+      # If the user wanted to know a setting but it doesn't exist, raise an error
+      if resolved_config.empty? && !item_name.nil?
+        PDK.logger.error(_("Configuration item '%{name}' does not exist") % { name: item_name })
+        exit 1
+      end
+      # If the user requested a setting and it's the only one resolved, then just output the value
+      if resolved_config.count == 1 && resolved_config.keys[0] == item_name
+        puts _('%{value}') % { value: resolved_config.values[0] }
+        exit 0
+      end
+      # Otherwise just output everything
+      resolved_config.keys.sort.each { |key| puts _('%{name}=%{value}') % { name: key, value: resolved_config[key] } }
+    end
+  end
+end

data/lib/pdk/cli/util.rb

@@ -129,13 +129,14 @@ module PDK
       end
       module_function :check_for_deprecated_puppet
 
-      # @param opts [Hash] - the pdk options ot use, defaults to empty hash
+      # @param opts [Hash] - the pdk options to use, defaults to empty hash
       # @option opts [String] :'puppet-dev' Use the puppet development version, default to PDK_PUPPET_DEV env
       # @option opts [String] :'puppet-version' Puppet version to use, default to PDK_PUPPET_VERSION env
       # @option opts [String] :'pe-version' PE Puppet version to use, default to PDK_PE_VERSION env
       # @param logging_disabled [Boolean] - disable logging of PDK info
+      # @param context [PDK::Context::AbstractContext] - The context the PDK is running in
       # @return [Hash] - return hash of { gemset: <>, ruby_version: 2.x.x }
-      def puppet_from_opts_or_env(opts, logging_disabled = false)
+      def puppet_from_opts_or_env(opts, logging_disabled = false, context = PDK.context)
         opts ||= {}
         use_puppet_dev = opts.fetch(:'puppet-dev', PDK::Util::Env['PDK_PUPPET_DEV'])
         desired_puppet_version = opts.fetch(:'puppet-version', PDK::Util::Env['PDK_PUPPET_VERSION'])
@@ -150,8 +151,10 @@ module PDK
               PDK::Util::PuppetVersion.find_gem_for(desired_puppet_version)
             elsif desired_pe_version
               PDK::Util::PuppetVersion.from_pe_version(desired_pe_version)
-            else
+            elsif context.is_a?(PDK::Context::Module)
               PDK::Util::PuppetVersion.from_module_metadata || PDK::Util::PuppetVersion.latest_available
+            else
+              PDK::Util::PuppetVersion.latest_available
             end
         rescue ArgumentError => e
           raise PDK::CLI::ExitWithError, e.message

data/lib/pdk/cli/validate.rb

@@ -19,6 +19,9 @@ module PDK::CLI
     flag nil, :parallel, _('Run validations in parallel.')
 
     run do |opts, args, _cmd|
+      # Write the context information to the debug log
+      PDK.context.to_debug_log
+
       if args == ['help']
         PDK::CLI.run(['validate', '--help'])
         exit 0
@@ -26,42 +29,42 @@ module PDK::CLI
 
       require 'pdk/validate'
 
-      validator_names = PDK::Validate.validators.map { |v| v.name }
-      validators = PDK::Validate.validators
-      targets = []
-
       if opts[:list]
         PDK::CLI::Util.analytics_screen_view('validate', opts)
-        PDK.logger.info(_('Available validators: %{validator_names}') % { validator_names: validator_names.join(', ') })
+        PDK.logger.info(_('Available validators: %{validator_names}') % { validator_names: PDK::Validate.validator_names.join(', ') })
         exit 0
       end
 
       PDK::CLI::Util.validate_puppet_version_opts(opts)
+      unless PDK.feature_flag?('controlrepo') || PDK.context.is_a?(PDK::Context::Module)
+        raise PDK::CLI::ExitWithError.new(_('Code validation can only be run from inside a valid module directory'), log_level: :error)
+      end
 
-      PDK::CLI::Util.ensure_in_module!(
-        message:   _('Code validation can only be run from inside a valid module directory'),
-        log_level: :info,
-      )
+      PDK::CLI::Util.module_version_check if PDK.context.is_a?(PDK::Context::Module)
 
-      PDK::CLI::Util.module_version_check
+      # Set the ruby version we're going to use early. Must be set before the validators are created.
+      # Note that this is a bit of code-smell and should be fixed
+      puppet_env = PDK::CLI::Util.puppet_from_opts_or_env(opts)
+      PDK::Util::RubyVersion.use(puppet_env[:ruby_version])
 
+      targets = []
+      validators_to_run = nil
       if args[0]
         # This may be a single validator, a list of validators, or a target.
         if Util::OptionValidator.comma_separated_list?(args[0])
           # This is a comma separated list. Treat each item as a validator.
-
           vals = Util::OptionNormalizer.comma_separated_list_to_array(args[0])
-          validators = PDK::Validate.validators.select { |v| vals.include?(v.name) }
+          validators_to_run = PDK::Validate.validator_names.select { |name| vals.include?(name) }
 
-          invalid = vals.reject { |v| validator_names.include?(v) }
-          invalid.each do |v|
-            PDK.logger.warn(_("Unknown validator '%{v}'. Available validators: %{validators}.") % { v: v, validators: validator_names.join(', ') })
+          vals.reject { |v| PDK::Validate.validator_names.include?(v) }
+              .each do |v|
+            PDK.logger.warn(_("Unknown validator '%{v}'. Available validators: %{validators}.") % { v: v, validators: PDK::Validate.validator_names.join(', ') })
           end
         else
           # This is a single item. Check if it's a known validator, or otherwise treat it as a target.
-          val = PDK::Validate.validators.find { |v| args[0] == v.name }
+          val = PDK::Validate.validator_names.find { |name| args[0] == name }
           if val
-            validators = [val]
+            validators_to_run = [val]
           else
             targets = [args[0]]
             # We now know that no validators were passed, so let the user know we're using all of them by default.
@@ -71,11 +74,12 @@ module PDK::CLI
       else
         PDK.logger.info(_('Running all available validators...'))
       end
+      validators_to_run = PDK::Validate.validator_names if validators_to_run.nil?
 
-      if validators == PDK::Validate.validators
+      if validators_to_run.sort == PDK::Validate.validator_names.sort
         PDK::CLI::Util.analytics_screen_view('validate', opts)
       else
-        PDK::CLI::Util.analytics_screen_view(['validate', validators.map(&:name).sort].flatten.join('_'), opts)
+        PDK::CLI::Util.analytics_screen_view(['validate', validators_to_run.sort].flatten.join('_'), opts)
       end
 
       # Subsequent arguments are targets.
@@ -93,36 +97,14 @@ module PDK::CLI
 
       options = targets.empty? ? {} : { targets: targets }
       options[:auto_correct] = true if opts[:'auto-correct']
-
-      # Ensure that the bundled gems are up to date and correct Ruby is activated before running any validations.
-      puppet_env = PDK::CLI::Util.puppet_from_opts_or_env(opts)
-      PDK::Util::RubyVersion.use(puppet_env[:ruby_version])
-
       options.merge!(puppet_env[:gemset])
 
+      # Ensure that the bundled gems are up to date and correct Ruby is activated before running any validations.
+      # Note that if no Gemfile exists, then ensure_bundle! will log a debug message and exit gracefully
       require 'pdk/util/bundler'
-
       PDK::Util::Bundler.ensure_bundle!(puppet_env[:gemset])
 
-      exit_code = 0
-      if opts[:parallel]
-        require 'pdk/cli/exec_group'
-
-        exec_group = PDK::CLI::ExecGroup.new(_('Validating module using %{num_of_threads} threads' % { num_of_threads: validators.count }), opts)
-
-        validators.each do |validator|
-          exec_group.register do
-            validator.invoke(report, options.merge(exec_group: exec_group))
-          end
-        end
-
-        exit_code = exec_group.exit_code
-      else
-        validators.each do |validator|
-          validator_exit_code = validator.invoke(report, options.dup)
-          exit_code = validator_exit_code if validator_exit_code != 0
-        end
-      end
+      exit_code, report = PDK::Validate.invoke_validators_by_name(PDK.context, validators_to_run, opts.fetch(:parallel, false), options)
 
       report_formats.each do |format|
         report.send(format[:method], format[:target])

data/lib/pdk/config.rb

@@ -2,6 +2,8 @@ require 'pdk'
 
 module PDK
   class Config
+    autoload :IniFile, 'pdk/config/ini_file'
+    autoload :IniFileSetting, 'pdk/config/ini_file_setting'
     autoload :JSON, 'pdk/config/json'
     autoload :JSONSchemaNamespace, 'pdk/config/json_schema_namespace'
     autoload :JSONSchemaSetting, 'pdk/config/json_schema_setting'
@@ -11,16 +13,58 @@ module PDK
     autoload :Validator, 'pdk/config/validator'
     autoload :YAML, 'pdk/config/yaml'
 
+    # Create a new instance of the PDK Configuration
+    # @param options [Hash[String => Object]] Optional hash to override configuration options
+    # @option options [String]                        'system.path'                 Path to the system PDK configuration file
+    # @option options [String]                        'system.module_defaults.path' Path to the system module answers PDK configuration file
+    # @option options [String]                        'user.path'                   Path to the user PDK configuration file
+    # @option options [String]                        'user.module_defaults.path'   Path to the user module answers PDK configuration file
+    # @option options [String]                        'user.analytics.path'         Path to the user analytics PDK configuration file
+    # @option options [PDK::Context::AbstractContext] 'context'                     The context that the configuration should be created in
+    def initialize(options = nil)
+      options = {} if options.nil?
+      @config_options = {
+        'system.path'                 => PDK::Config.system_config_path,
+        'system.module_defaults.path' => PDK::Config.system_answers_path,
+        'user.path'                   => PDK::Config.user_config_path,
+        'user.module_defaults.path'   => PDK::AnswerFile.default_answer_file_path,
+        'user.analytics.path'         => PDK::Config.analytics_config_path,
+        'context'                     => PDK.context,
+      }.merge(options)
+    end
+
+    # The user configuration settings.
+    # @deprecated This method is only provided as a courtesy until the `pdk set config` CLI and associated changes in this class, are completed.
+    #             Any read-only operations should be using `.get` or `.get_within_scopes`
+    # @return [PDK::Config::Namespace]
     def user
-      @user ||= PDK::Config::JSON.new('user', file: PDK::Config.user_config_path) do
-        mount :module_defaults, PDK::Config::JSON.new(file: PDK.answers.answer_file_path)
+      user_config
+    end
+
+    # The system level configuration settings.
+    # @return [PDK::Config::Namespace]
+    # @api private
+    def system_config
+      local_options = @config_options
+      @system ||= PDK::Config::JSON.new('system', file: local_options['system.path']) do
+        mount :module_defaults, PDK::Config::JSON.new(file: local_options['system.module_defaults.path'])
+      end
+    end
+
+    # The user level configuration settings.
+    # @return [PDK::Config::Namespace]
+    # @api private
+    def user_config
+      local_options = @config_options
+      @user ||= PDK::Config::JSON.new('user', file: local_options['user.path']) do
+        mount :module_defaults, PDK::Config::JSON.new(file: local_options['user.module_defaults.path'])
 
         # Due to the json-schema gem having issues with Windows based paths, and only supporting Draft 05 (or less) do
         # not use JSON validation yet.  Once PDK drops support for EOL rubies, we will be able to use the json_schemer gem
         # Which has much more modern support
         # Reference - https://github.com/puppetlabs/pdk/pull/777
         # Reference - https://tickets.puppetlabs.com/browse/PDK-1526
-        mount :analytics, PDK::Config::YAML.new(file: PDK::Config.analytics_config_path, persistent_defaults: true) do
+        mount :analytics, PDK::Config::YAML.new(file: local_options['user.analytics.path'], persistent_defaults: true) do
           setting :disabled do
             validate PDK::Config::Validator.boolean
             default_to { PDK::Config.bolt_analytics_config.fetch('disabled', true) }
@@ -35,6 +79,29 @@ module PDK
             end
           end
         end
+
+        # Display the feature flags
+        mount :pdk_feature_flags, PDK::Config::Namespace.new('pdk_feature_flags') do
+          setting 'available' do
+            default_to { PDK.available_feature_flags }
+          end
+
+          setting 'requested' do
+            default_to { PDK.requested_feature_flags }
+          end
+        end
+      end
+    end
+
+    # The project level configuration settings.
+    # @return [PDK::Config::Namespace]
+    # @api private
+    def project_config
+      context = @config_options['context']
+      @project ||= PDK::Config::Namespace.new('project') do
+        if context.is_a?(PDK::Context::ControlRepo)
+          mount :environment, PDK::ControlRepo.environment_conf_as_config(File.join(context.root_path, 'environment.conf'))
+        end
       end
     end
 
@@ -43,7 +110,63 @@ module PDK
     # @param filter [String] Only resolve setting names which match the filter. See PDK::Config::Namespace.be_resolved? for matching rules
     # @return [Hash{String => Object}] All resolved settings for example {'user.module_defaults.author' => 'johndoe'}
     def resolve(filter = nil)
-      user.resolve(filter)
+      all_scopes.values.reverse.reduce({}) do |result, method_name|
+        result.merge(send(method_name).resolve(filter))
+      end
+    end
+
+    # Returns a configuration setting by name. This name can either be a String, Array or parameters e.g. These are equivalent
+    # - PDK.config.get('user.a.b.c')
+    # - PDK.config.get(['user', 'a', 'b', 'c'])
+    # - PDK.config.get('user', 'a', 'b', 'c')
+    # @return [PDK::Config::Namespace, Object, nil] The value of the configuration setting. Returns nil if it does no exist
+    def get(root, *keys)
+      return nil if root.nil? || root.empty?
+
+      if keys.empty?
+        if root.is_a?(Array)
+          name = root
+        elsif root.is_a?(String)
+          name = split_key_string(root)
+        else
+          return nil
+        end
+      else
+        name = [root].concat(keys)
+      end
+
+      get_within_scopes(name[1..-1], [name[0]])
+    end
+
+    # Returns a configuration setting by name, using scope precedence rules. If no scopes are passed, then all scopes are queried using the default precedence rules
+    # @setting_name [String, Array[String]] The setting name to retrieve without the leading scope name e.g. Use 'setting' instead of 'system.setting'
+    # @scopes [Nil, Array[String]] The list of scopes, in order, to query in turn for the setting_name. Invalid or missing scopes are ignored.
+    # @return [PDK::Config::Namespace, Object, nil] The value of the configuration setting. Returns nil if it does no exist
+    def get_within_scopes(setting_name, scopes = nil)
+      raise ArgumentError, _('Expected an Array but got \'%{klass}\' for scopes') % { klass: scopes.class } unless scopes.nil? || scopes.is_a?(Array)
+      raise ArgumentError, _('Expected an Array or String but got \'%{klass}\' for setting_name') % { klass: setting_name.class } unless setting_name.is_a?(Array) || setting_name.is_a?(String)
+
+      setting_arr = setting_name.is_a?(String) ? split_key_string(setting_name) : setting_name
+      all_scope_names = all_scopes.keys
+
+      # Use only valid scope names
+      scopes = scopes.nil? ? all_scope_names : scopes & all_scope_names
+
+      scopes.each do |scope_name|
+        value = traverse_object(send(all_scopes[scope_name]), *setting_arr)
+        return value unless value.nil?
+      end
+      nil
+    end
+
+    # Yields a configuration setting value by name, using scope precedence rules. If no scopes are passed, then all scopes are queried using the default precedence rules
+    # @setting_name [String, Array[String]] The setting name to retrieve without the leading scope name e.g. Use 'setting' instead of 'system.setting'
+    # @scopes [Nil, Array[String]] The list of scopes, in order, to query in turn for the setting_name. Invalid or missing scopes are ignored.
+    # @yield [PDK::Config::Namespace, Object] The value of the configuration setting. Does not yield if the setting does not exist or is nil
+    def with_scoped_value(setting_name, scopes = nil)
+      raise ArgumentError, _('must be passed a block') unless block_given?
+      value = get_within_scopes(setting_name, scopes)
+      yield value unless value.nil?
     end
 
     def self.bolt_analytics_config
@@ -65,6 +188,14 @@ module PDK
       File.join(PDK::Util.configdir, 'user_config.json')
     end
 
+    def self.system_config_path
+      File.join(PDK::Util.system_configdir, 'system_config.json')
+    end
+
+    def self.system_answers_path
+      File.join(PDK::Util.system_configdir, 'answers.json')
+    end
+
     def self.json_schemas_path
       File.join(__dir__, 'config')
     end
@@ -123,5 +254,48 @@ module PDK
 
       PDK.logger.info(text: post_message, wrap: true)
     end
+
+    private
+
+    #:nocov: This is a private method and is tested elsewhere
+    def traverse_object(object, *names)
+      return nil if object.nil? || !object.respond_to?(:[])
+      return nil if names.nil? || names.empty?
+
+      name = names.shift
+      value = object[name]
+      if names.empty?
+        return value if value.is_a?(PDK::Config::Namespace)
+        # Duplicate arrays and hashes so that they are isolated from changes being made
+        (value.is_a?(Hash) || value.is_a?(Array)) ? value.dup : value
+      else
+        traverse_object(value, *names)
+      end
+    end
+    #:nocov:
+
+    #:nocov: This is a private method and is tested elsewhere
+    # Takes a string representation of a setting and splits into its constituent setting parts e.g.
+    # 'user.a.b.c' becomes ['user', 'a', 'b', 'c']
+    # @return [Array[String]] The string split into each setting name as an array
+    def split_key_string(key)
+      raise ArgumentError, _('Expected a String but got \'%{klass}\'') % { klass: key.class } unless key.is_a?(String)
+      key.split('.')
+    end
+    #:nocov:
+
+    #:nocov:
+    # Returns all known scope names and their associated method name to call, to query the scope
+    # Note - Order is important. This dictates the resolution precedence order (topmost is processed first)
+    # @return [Hash[String, Symbol]] A hash of the scope name then method name to call to query the scope (as a Symbol)
+    def all_scopes
+      # Note - Order is important. This dictates the resolution precedence order (topmost is processed first)
+      {
+        'project' => :project_config,
+        'user'    => :user_config,
+        'system'  => :system_config,
+      }.freeze
+    end
+    #:nocov:
   end
 end

data/lib/pdk/config/ini_file.rb

@@ -0,0 +1,183 @@
+require 'pdk'
+
+module PDK
+  class Config
+    # Represents a configuration file using the INI file format
+    class IniFile < Namespace
+      # Ini Files have very strict valdiation rules which are set in the IniFileSetting class
+      # @see PDK::Config::Namespace.default_setting_class
+      def default_setting_class
+        PDK::Config::IniFileSetting
+      end
+
+      # Parses an IniFile document.
+      #
+      # @see PDK::Config::Namespace.parse_file
+      def parse_file(filename)
+        raise unless block_given?
+        data = load_data(filename)
+        return if data.nil? || data.empty?
+
+        ini_file = IniFileImpl.parse(data)
+        ini_file.to_hash.each do |name, value|
+          begin
+            new_setting = PDK::Config::IniFileSetting.new(name, self, value)
+          rescue StandardError
+            # We just ignore invalid initial settings
+            new_setting = PDK::Config::IniFileSetting.new(name, self, nil)
+          end
+
+          yield name, new_setting
+        end
+      end
+
+      # Serializes object data into an INI file string.
+      #
+      # @see PDK::Config::Namespace.serialize_data
+      def serialize_data(data)
+        default_lines = ''
+        lines = ''
+        data.each do |name, value|
+          next if value.nil?
+          if value.is_a?(Hash)
+            # Hashes are an INI section
+            lines += "\n[#{name}]\n"
+            value.each do |child_name, child_value|
+              next if child_value.nil?
+              lines += "#{child_name} = #{munge_serialized_value(child_value)}\n"
+            end
+          else
+            default_lines += "#{name} = #{munge_serialized_value(value)}\n"
+          end
+        end
+
+        default_lines + lines
+      end
+
+      private
+
+      def munge_serialized_value(value)
+        value = value.to_s unless value.is_a?(String)
+        # Add enclosing quotes if there's a space in the value
+        value = '"' + value + '"' if value.include?(' ')
+        value
+      end
+
+      # Adapted from https://raw.githubusercontent.com/puppetlabs/puppet/6c257fc7827989c2af2901f974666f0f23611153/lib/puppet/settings/ini_file.rb
+      # rubocop:disable Style/RegexpLiteral
+      # rubocop:disable Style/PerlBackrefs
+      # rubocop:disable Style/RedundantSelf
+      # rubocop:disable Style/StringLiterals
+      class IniFileImpl
+        DEFAULT_SECTION_NAME = 'default_section_name'.freeze
+
+        def self.parse(config_fh)
+          config = new([DefaultSection.new])
+          config_fh.each_line do |line|
+            case line.chomp
+            when /^(\s*)\[([[:word:]]+)\](\s*)$/
+              config.append(SectionLine.new($1, $2, $3))
+            when /^(\s*)([[:word:]]+)(\s*=\s*)(.*?)(\s*)$/
+              config.append(SettingLine.new($1, $2, $3, $4, $5))
+            else
+              config.append(Line.new(line))
+            end
+          end
+
+          config
+        end
+
+        def initialize(lines = [])
+          @lines = lines
+        end
+
+        def to_hash
+          result = {}
+
+          current_section_name = nil
+          @lines.each do |line|
+            if line.instance_of?(SectionLine)
+              current_section_name = line.name
+              result[current_section_name] = {}
+            elsif line.instance_of?(SettingLine)
+              if current_section_name.nil?
+                result[line.name] = munge_value(line.value)
+              else
+                result[current_section_name][line.name] = munge_value(line.value)
+              end
+            end
+          end
+
+          result
+        end
+
+        def munge_value(value)
+          value = value.to_s unless value.is_a?(String)
+          # Strip enclosing quotes
+          value = value.slice(1...-1) if value.start_with?('"') && value.end_with?('"')
+          value
+        end
+
+        def append(line)
+          line.previous = @lines.last
+          @lines << line
+        end
+
+        module LineNumber
+          attr_accessor :previous
+
+          def line_number
+            line = 0
+            previous_line = previous
+            while previous_line
+              line += 1
+              previous_line = previous_line.previous
+            end
+            line
+          end
+        end
+
+        Line = Struct.new(:text) do
+          include LineNumber
+
+          def to_s
+            text
+          end
+        end
+
+        SettingLine = Struct.new(:prefix, :name, :infix, :value, :suffix) do
+          include LineNumber
+
+          def to_s
+            "#{prefix}#{name}#{infix}#{value}#{suffix}"
+          end
+
+          def ==(other)
+            super(other) && self.line_number == other.line_number
+          end
+        end
+
+        SectionLine = Struct.new(:prefix, :name, :suffix) do
+          include LineNumber
+
+          def to_s
+            "#{prefix}[#{name}]#{suffix}"
+          end
+        end
+
+        class DefaultSection < SectionLine
+          attr_accessor :write_sectionline
+
+          def initialize
+            @write_sectionline = false
+            super("", DEFAULT_SECTION_NAME, "")
+          end
+        end
+      end
+      # rubocop:enable Style/StringLiterals
+      # rubocop:enable Style/RedundantSelf
+      # rubocop:enable Style/PerlBackrefs
+      # rubocop:enable Style/RegexpLiteral
+    end
+  end
+end