puppet 4.3.2 → 4.4.0

data/lib/puppet/agent.rb

@@ -41,7 +41,7 @@ class Puppet::Agent
       result = run_in_fork(should_fork) do
         with_client do |client|
           begin
-            client_args = client_options.merge(:pluginsync => Puppet[:pluginsync])
+            client_args = client_options.merge(:pluginsync => Puppet::Configurer.should_pluginsync?)
             lock { client.run(client_args) }
           rescue Puppet::LockError
             Puppet.notice "Run of #{client_class} already in progress; skipping  (#{lockfile_path} exists)"

data/lib/puppet/application/agent.rb

@@ -155,7 +155,7 @@ specify '--server <servername>' as an argument. Boolean settings translate into
 '--setting' and '--no-setting' pairs.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable settings. A commented list of all settings can also be
 generated by running puppet agent with '--genconfig'.
 
@@ -404,7 +404,9 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
     # we want the last report to be persisted locally
     Puppet::Transaction::Report.indirection.cache_class = :yaml
 
-    if Puppet[:catalog_cache_terminus]
+    if Puppet[:noop]
+      Puppet::Resource::Catalog.indirection.cache_class = nil
+    elsif Puppet[:catalog_cache_terminus]
       Puppet::Resource::Catalog.indirection.cache_class = Puppet[:catalog_cache_terminus]
     end
 

data/lib/puppet/application/apply.rb

@@ -74,7 +74,7 @@ valid setting, so you can specify '--tags <class>,<tag>'
 as an argument.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable parameters. A commented list of all
 configuration options can also be generated by running puppet with
 '--genconfig'.
@@ -175,7 +175,7 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
     if options[:catalog] == "-"
       text = $stdin.read
     else
-      text = ::File.read(options[:catalog])
+      text = Puppet::FileSystem.read(options[:catalog], :encoding => 'utf-8')
     end
     catalog = read_catalog(text)
     apply_catalog(catalog)
@@ -239,7 +239,7 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
             $stderr.puts "#{file} is not readable"
             exit(63)
           end
-          node.classes = ::File.read(file).split(/[\s\n]+/)
+          node.classes = Puppet::FileSystem.read(file, :encoding => 'utf-8').split(/[\s\n]+/)
         end
       end
 
@@ -315,7 +315,10 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
 
     Puppet.settings.use :main, :agent, :ssl
 
-    if Puppet[:catalog_cache_terminus]
+
+    if Puppet[:noop]
+      Puppet::Resource::Catalog.indirection.cache_class = nil
+    elsif Puppet[:catalog_cache_terminus]
       Puppet::Resource::Catalog.indirection.cache_class = Puppet[:catalog_cache_terminus]
     end
 

data/lib/puppet/application/cert.rb

@@ -164,7 +164,7 @@ setting, so you can specify '--ssldir <directory>' as an
 argument.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable parameters. A commented list of all
 configuration options can also be generated by running puppet cert with
 '--genconfig'.

data/lib/puppet/application/device.rb

@@ -199,7 +199,7 @@ Licensed under the Apache 2.0 License
 
         require 'puppet/configurer'
         configurer = Puppet::Configurer.new
-        configurer.run(:network_device => true, :pluginsync => Puppet[:pluginsync])
+        configurer.run(:network_device => true, :pluginsync => Puppet::Configurer.should_pluginsync?)
       rescue => detail
         Puppet.log_exception(detail)
         # If we rescued an error, then we return 1 as the exit code

data/lib/puppet/application/filebucket.rb

@@ -73,7 +73,7 @@ setting, so you can specify '--ssldir <directory>' as an
 argument.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable parameters. A commented list of all
 configuration options can also be generated by running puppet with
 '--genconfig'.

data/lib/puppet/application/inspect.rb

@@ -48,7 +48,7 @@ OPTIONS
 Any configuration setting which is valid in the configuration file is
 also a valid long argument, e.g. '--server=master.domain.com'. See the
 configuration file documentation at
-http://docs.puppetlabs.com/references/latest/configuration.html for
+https://docs.puppetlabs.com/references/latest/configuration.html for
 the full list of acceptable settings.
 
 * --archive_files:

data/lib/puppet/application/lookup.rb

@@ -214,7 +214,7 @@ the puppet lookup function linked to above.
 
 EXAMPLE
 -------
-  If you wanted to lookup 'key_name' within the scope of the master, you would 
+  If you wanted to lookup 'key_name' within the scope of the master, you would
   call lookup like this:
   $ puppet lookup key_name
 
@@ -302,11 +302,11 @@ Copyright (c) 2015 Puppet Labs, LLC Licensed under the Apache 2.0 License
     renderer = Puppet::Network::FormatHandler.format(format == :json ? :pson : format)
     raise "Unknown rendering format '#{format}'" if renderer.nil?
 
-    type = options.include?(:type) ? Puppet::Pops::Types::TypeParser.new.parse(options[:type]) : nil
 
     generate_scope do |scope|
       lookup_invocation = Puppet::Pops::Lookup::Invocation.new(scope, {}, {}, explain ? Puppet::Pops::Lookup::Explainer.new(explain_options, only_explain_options) : nil)
       begin
+        type = options.include?(:type) ? Puppet::Pops::Types::TypeParser.new.parse(options[:type], scope) : nil
         result = Puppet::Pops::Lookup.lookup(keys, type, options[:default_value], use_default_value, merge_options, lookup_invocation)
         puts renderer.render(result) unless explain
       rescue Puppet::DataBinding::LookupError
@@ -335,9 +335,9 @@ Copyright (c) 2015 Puppet Labs, LLC Licensed under the Apache 2.0 License
     if fact_file
       original_facts = node.parameters
       if fact_file.end_with?("json")
-        given_facts = JSON.parse(File.read(fact_file))
+        given_facts = JSON.parse(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
       else
-        given_facts = YAML.load(File.read(fact_file))
+        given_facts = YAML.load(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
       end
 
       unless given_facts.instance_of?(Hash)

data/lib/puppet/application/master.rb

@@ -61,7 +61,7 @@ specify '--server <servername>' as an argument. Boolean settings translate into
 '--setting' and '--no-setting' pairs.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable settings. A commented list of all settings can also be
 generated by running puppet master with '--genconfig'.
 
@@ -92,7 +92,7 @@ generated by running puppet master with '--genconfig'.
   valid JSON.
 
 * --masterport:
-  The port on which to listen for traffic.
+  The port on which to listen for traffic. The default port is 8140.
   (This is a Puppet setting, and can go in puppet.conf.)
 
 * --verbose:

data/lib/puppet/application/resource.rb

@@ -72,7 +72,7 @@ setting, so you can specify '--ssldir <directory>' as an
 argument.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/references/stable/configuration.html for the
 full list of acceptable parameters. A commented list of all
 configuration options can also be generated by running puppet with
 '--genconfig'.

data/lib/puppet/configurer.rb

@@ -22,6 +22,18 @@ class Puppet::Configurer
     "Puppet configuration client"
   end
 
+  def self.should_pluginsync?
+    if Puppet.settings.set_by_cli?(:pluginsync) || Puppet.settings.set_by_config?(:pluginsync)
+      Puppet[:pluginsync]
+    else
+      if Puppet[:use_cached_catalog]
+        false
+      else
+        true
+      end
+    end
+  end
+
   def execute_postrun_command
     execute_from_setting(:postrun_command)
   end
@@ -47,26 +59,46 @@ class Puppet::Configurer
   def initialize(factory = Puppet::Configurer::DownloaderFactory.new)
     @running = false
     @splayed = false
+    @cached_catalog_status = 'not_used'
     @environment = Puppet[:environment]
     @transaction_uuid = SecureRandom.uuid
+    @static_catalog = true
+    @checksum_type = Puppet[:supported_checksum_types]
     @handler = Puppet::Configurer::PluginHandler.new(factory)
   end
 
   # Get the remote catalog, yo.  Returns nil if no catalog can be found.
   def retrieve_catalog(query_options)
     query_options ||= {}
-    # First try it with no cache, then with the cache.
-    unless (Puppet[:use_cached_catalog] and result = retrieve_catalog_from_cache(query_options)) or result = retrieve_new_catalog(query_options)
-      if ! Puppet[:usecacheonfailure]
-        Puppet.warning "Not using cache on failed catalog"
-        return nil
+    if (Puppet[:use_cached_catalog] && result = retrieve_catalog_from_cache(query_options))
+      @cached_catalog_status = 'explicitly_requested'
+
+      Puppet.info "Using cached catalog from environment '#{result.environment}'"
+    else
+      result = retrieve_new_catalog(query_options)
+
+      if !result
+        if !Puppet[:usecacheonfailure]
+          Puppet.warning "Not using cache on failed catalog"
+          return nil
+        end
+
+        result = retrieve_catalog_from_cache(query_options)
+
+        if result
+          # don't use use cached catalog if it doesn't match server specified environment
+          if @node_environment && result.environment != @environment
+            Puppet.err "Not using cached catalog because its environment '#{result.environment}' does not match '#{@environment}'"
+            return nil
+          end
+
+          @cached_catalog_status = 'on_failure'
+          Puppet.info "Using cached catalog from environment '#{result.environment}'"
+        end
       end
-      result = retrieve_catalog_from_cache(query_options)
     end
 
-    return nil unless result
-
-    convert_catalog(result, @duration)
+    result
   end
 
   # Convert a plain resource catalog into our full host catalog.
@@ -100,12 +132,32 @@ class Puppet::Configurer
   def prepare_and_retrieve_catalog(options, query_options)
     # set report host name now that we have the fact
     options[:report].host = Puppet[:node_name_value]
+    query_options[:transaction_uuid] = @transaction_uuid
+    query_options[:static_catalog] = @static_catalog
+
+    # Query params don't enforce ordered evaluation, so munge this list into a
+    # dot-separated string.
+    query_options[:checksum_type] = @checksum_type.join('.')
+
+    # apply passes in ral catalog
+    catalog = options.delete(:catalog)
+    return catalog if catalog
 
-    unless catalog = (options.delete(:catalog) || retrieve_catalog(query_options))
-      Puppet.err "Could not retrieve catalog; skipping run"
-      return
+    # retrieve_catalog returns json catalog
+    catalog = retrieve_catalog(query_options)
+    return convert_catalog(catalog, @duration) if catalog
+
+    Puppet.err "Could not retrieve catalog; skipping run"
+    nil
+  end
+
+  def prepare_and_retrieve_catalog_from_cache
+    result = retrieve_catalog_from_cache({:transaction_uuid => @transaction_uuid, :static_catalog => @static_catalog})
+    if result
+      Puppet.info "Using cached catalog from environment '#{result.environment}'"
+      return convert_catalog(result, @duration)
     end
-    catalog
+    nil
   end
 
   # Retrieve (optionally) and apply a catalog. If a catalog is passed in
@@ -149,6 +201,28 @@ class Puppet::Configurer
 
     Puppet::Util::Log.newdestination(report)
 
+    # If a cached catalog is explicitly requested, attempt to retrieve it. Skip the node request,
+    # don't pluginsync and switch to the catalog's environment if we successfully retrieve it.
+    if Puppet[:use_cached_catalog]
+      if catalog = prepare_and_retrieve_catalog_from_cache
+        options[:catalog] = catalog
+        @cached_catalog_status = 'explicitly_requested'
+
+        if @environment != catalog.environment && !Puppet[:strict_environment_mode]
+          Puppet.notice "Local environment: '#{@environment}' doesn't match the environment of the cached catalog '#{catalog.environment}', switching agent to '#{catalog.environment}'."
+          @environment = catalog.environment
+        end
+
+        report.environment = @environment
+      else
+        # Don't try to retrieve a catalog from the cache again after we've already
+        # failed to do so the first time.
+        Puppet[:use_cached_catalog] = false
+        Puppet[:usecacheonfailure] = false
+        options[:pluginsync] = Puppet::Configurer.should_pluginsync?
+      end
+    end
+
     begin
       unless Puppet[:node_name_fact].empty?
         query_options = get_facts(options)
@@ -157,7 +231,7 @@ class Puppet::Configurer
       configured_environment = Puppet[:environment] if Puppet.settings.set_by_config?(:environment)
 
       # We only need to find out the environment to run in if we don't already have a catalog
-      unless options[:catalog]
+      unless (options[:catalog] || Puppet[:strict_environment_mode])
         begin
           if node = Puppet::Node.indirection.find(Puppet[:node_name_value],
               :environment => Puppet::Node::Environment.remote(@environment),
@@ -172,6 +246,8 @@ class Puppet::Configurer
               node.environment = Puppet::Node::Environment.remote(node.environment_name)
             end
 
+            @node_environment = node.environment.to_s
+
             if node.environment.to_s != @environment
               Puppet.notice "Local environment: '#{@environment}' doesn't match server specified node environment '#{node.environment}', switching agent to '#{node.environment}'."
               @environment = node.environment.to_s
@@ -200,13 +276,17 @@ class Puppet::Configurer
       Puppet.push_context({:current_environment => local_node_environment}, "Local node environment for configurer transaction")
 
       query_options = get_facts(options) unless query_options
-      query_options[:transaction_uuid] = @transaction_uuid
       query_options[:configured_environment] = configured_environment
 
       unless catalog = prepare_and_retrieve_catalog(options, query_options)
         return nil
       end
 
+      if Puppet[:strict_environment_mode] && catalog.environment != @environment
+        Puppet.err "Not using catalog because its environment '#{catalog.environment}' does not match agent specified environment '#{@environment}' and strict_environment_mode is set"
+        return nil
+      end
+
       # Here we set the local environment based on what we get from the
       # catalog. Since a change in environment means a change in facts, and
       # facts may be used to determine which catalog we get, we need to
@@ -221,7 +301,6 @@ class Puppet::Configurer
         report.environment = @environment
 
         query_options = get_facts(options)
-        query_options[:transaction_uuid] = @transaction_uuid
         query_options[:configured_environment] = configured_environment
 
         return nil unless catalog = prepare_and_retrieve_catalog(options, query_options)
@@ -229,6 +308,10 @@ class Puppet::Configurer
       end
 
       execute_prerun_command or return nil
+
+      options[:report].code_id = catalog.code_id
+      options[:report].catalog_uuid = catalog.catalog_uuid
+      options[:report].cached_catalog_status = @cached_catalog_status
       apply_catalog(catalog, options)
       report.exit_status
     rescue => detail
@@ -238,11 +321,7 @@ class Puppet::Configurer
       execute_postrun_command or return nil
     end
   ensure
-    # Between Puppet runs we need to forget the cached values.  This lets us
-    # pick up on new functions installed by gems or new modules being added
-    # without the daemon being restarted.
-    $env_module_directories = nil
-
+    report.cached_catalog_status ||= @cached_catalog_status
     Puppet::Util::Log.close(report)
     send_report(report)
     Puppet.pop_context
@@ -286,7 +365,6 @@ class Puppet::Configurer
       result = Puppet::Resource::Catalog.indirection.find(Puppet[:node_name_value],
         query_options.merge(:ignore_terminus => true, :environment => Puppet::Node::Environment.remote(@environment)))
     end
-    Puppet.notice "Using cached catalog"
     result
   rescue => detail
     Puppet.log_exception(detail, "Could not retrieve catalog from cache: #{detail}")

data/lib/puppet/data_providers/hiera_config.rb

@@ -27,6 +27,8 @@ module Puppet::DataProviders
         Hash[struct.map { |k,v| [k.to_s, symkeys_to_string(v)] }]
       when Array
         struct.map { |v| symkeys_to_string(v) }
+      when nil
+        {}
       else
         struct
       end
@@ -70,6 +72,18 @@ module Puppet::DataProviders
     end
 
     def create_data_providers(lookup_invocation)
+      Puppet.deprecation_warning('Method HieraConfig#create_data_providers is deprecated. Use create_configured_data_providers instead')
+      create_configured_data_providers(lookup_invocation, nil)
+    end
+
+    # Creates the data providers for this config
+    #
+    # @param lookup_invocation [Puppet::Pops::Lookup::Invocation] Invocation data containing scope, overrides, and defaults
+    # @param parent_data_provider [Puppet::Plugins::DataProviders::DataProvider] The data provider that loaded this configuration
+    # @return [Array[DataProvider]] the created providers
+    #
+    # @api private
+    def create_configured_data_providers(lookup_invocation, parent_data_provider)
       injector = Puppet.lookup(:injector)
       service_type = Registry.hash_of_path_based_data_provider_factories
       default_datadir = @config['datadir']
@@ -92,14 +106,25 @@ module Puppet::DataProviders
         raise Puppet::DataBinding::LookupError, "#{@config_path}: No data provider is registered for backend '#{provider_name}' " unless provider_factory
 
         datadir = @config_root + (he['datadir'] || default_datadir)
-        resolved_paths = provider_factory.resolve_paths(datadir, original_paths, paths, lookup_invocation)
-        data_providers[name] = provider_factory.create(name, resolved_paths)
+        data_providers[name] = create_data_provider(parent_data_provider, provider_factory, name,
+          provider_factory.resolve_paths(datadir, original_paths, paths, lookup_invocation))
       end
       data_providers.values
     end
 
+    def create_data_provider(parent_data_provider, provider_factory, name, resolved_paths)
+      provider_factory_version = provider_factory.respond_to?(:version) ? provider_factory.version : 1
+      if provider_factory_version == 1
+        # Version 1 is not aware of the parent provider
+        provider_factory.create(name, resolved_paths)
+      else
+        provider_factory.create(name, resolved_paths, parent_data_provider)
+      end
+    end
+    private :create_data_provider
+
     def validate_config(hiera_config)
-      Puppet::Pops::Types::TypeAsserter.assert_instance_of(@config_path, self.class.config_type, hiera_config)
+      Puppet::Pops::Types::TypeAsserter.assert_instance_of(["The Hiera Configuration at '%s'", @config_path], self.class.config_type, hiera_config)
     end
     private :validate_config
   end

data/lib/puppet/data_providers/hiera_interpolate.rb

@@ -5,31 +5,46 @@ require_relative 'hiera_config'
 #
 module Puppet::DataProviders::HieraInterpolate
   def interpolate(subject, lookup_invocation, allow_methods)
-    return subject unless subject.is_a?(String) && !subject.index('%{').nil?
+    case subject
+    when String
+      subject.index('%{').nil? ? subject : interpolate_string(subject, lookup_invocation, allow_methods)
+    when Array
+      subject.map { |element| interpolate(element, lookup_invocation, allow_methods) }
+    when Hash
+      Hash[subject.map { |k, v| [k, interpolate(v, lookup_invocation, allow_methods)] }]
+    else
+      subject
+    end
+  end
 
-    lookup_invocation.with(:interpolate, subject) do
-      subject.gsub(/%\{([^\}]+)\}/) do |match|
-        method_key, key = get_method_and_data($1, allow_methods)
-        is_alias = method_key == 'alias'
+  private
 
-        # Alias is only permitted if the entire string is equal to the interpolate expression
-        raise Puppet::DataBinding::LookupError, "'alias' interpolation is only permitted if the expression is equal to the entire string" if is_alias && subject != match
+  def interpolate_string(subject, lookup_invocation, allow_methods)
+    lookup_invocation.with(:interpolate, subject) do
+      subject.gsub(/%\{([^\}]*)\}/) do |match|
+        expr = $1
+        # Leading and trailing spaces inside an interpolation expression are insignificant
+        expr.strip!
+        unless expr.empty? || expr == '::'
+          method_key, key = get_method_and_data(expr, allow_methods)
+          is_alias = method_key == 'alias'
 
-        segments = key.split('.')
-        value = interpolate_method(method_key).call(segments[0], lookup_invocation)
-        value = qualified_lookup(segments.drop(1), value) if segments.size > 1
-        value = lookup_invocation.check(key) { interpolate(value, lookup_invocation, allow_methods) } if value.is_a?(String)
+          # Alias is only permitted if the entire string is equal to the interpolate expression
+          raise Puppet::DataBinding::LookupError, "'alias' interpolation is only permitted if the expression is equal to the entire string" if is_alias && subject != match
 
-        # break gsub and return value immediately if this was an alias substitution. The value might be something other than a String
-        return value if is_alias
+          segments = key.split('.')
+          value = interpolate_method(method_key).call(segments[0], lookup_invocation)
+          value = qualified_lookup(segments.drop(1), value) if segments.size > 1
+          value = lookup_invocation.check(key) { interpolate(value, lookup_invocation, allow_methods) }
 
+          # break gsub and return value immediately if this was an alias substitution. The value might be something other than a String
+          return value if is_alias
+        end
         value || ''
       end
     end
   end
 
-  private
-
   def interpolate_method(method_key)
     @@interpolate_methods ||= begin
       global_lookup = lambda { |key, lookup_invocation| Puppet::Pops::Lookup.lookup(key, nil, '', true, nil, lookup_invocation) }