puppet 4.10.0 → 4.10.1

data/Gemfile

@@ -1,5 +1,7 @@
 source ENV['GEM_SOURCE'] || "https://rubygems.org"
 
+gemspec
+
 def location_for(place, fake_version = nil)
   if place =~ /^(git[:@][^#]*)#(.*)/
     [fake_version, { :git => $1, :branch => $2, :require => false }].compact
@@ -13,7 +15,6 @@ end
 # C Ruby (MRI) or Rubinius, but NOT Windows
 platforms :ruby do
   gem 'pry', :group => :development
-  gem 'yard', :group => :development
   gem 'redcarpet', '~> 2.0', :group => :development
   gem "racc", "1.4.9", :group => :development
 
@@ -23,20 +24,18 @@ platforms :ruby do
   #gem 'ruby-augeas', :group => :development
 end
 
-gem "puppet", :path => File.dirname(__FILE__), :require => false
-gem "facter", *location_for(ENV['FACTER_LOCATION'] || ['> 2.0', '< 4'])
-gem "hiera", *location_for(ENV['HIERA_LOCATION'] || ['>= 2.0', '< 4'])
+# override .gemspec deps - may issue warning depending on Bundler version
+gem "facter", *location_for(ENV['FACTER_LOCATION']) if ENV.has_key?('FACTER_LOCATION')
+gem "hiera", *location_for(ENV['HIERA_LOCATION']) if ENV.has_key?('HIERA_LOCATION')
 # PUP-7115 - return to a gem dependency in Puppet 5
 # gem "semantic_puppet", *location_for(ENV['SEMANTIC_PUPPET_LOCATION'] || ['>= 0.1.3', '< 2'])
-gem "rake", "10.1.1", :require => false
-# Hiera has an unbound dependency on json_pure
-# json_pure 2.0.2+ officially requires Ruby >= 2.0, but should have specified that in 2.0
-gem 'json_pure', '~> 1.8', :require => false
-# i18n support (gettext-setup and dependencies)
-gem 'gettext-setup', '>= 0.10', '< 1.0', :require => false
-gem 'locale', '~> 2.1', :require => false
 
 group(:development, :test) do
+  # rake is in .gemspec as a development dependency but cannot
+  # be removed here *yet* due to TravisCI / AppVeyor which call:
+  # bundle install --without development
+  # PUP-7433 describes work necessary to restructure this
+  gem "rake", "10.1.1", :require => false
   gem "rspec", "~> 3.1", :require => false
   gem "rspec-its", "~> 1.1", :require => false
   gem "rspec-collection_matchers", "~> 1.1", :require => false
@@ -57,6 +56,7 @@ group(:development, :test) do
   gem "rubocop", "~> 0.39.0", :platforms => [:ruby]
 
   gem 'rdoc', "~> 4.1", :platforms => [:ruby]
+  gem 'yard'
 
   # webmock requires addressable as as of 2.5.0 addressable started
   # requiring the public_suffix gem which requires Ruby 2
@@ -75,34 +75,10 @@ end
 
 group(:extra) do
   gem "rack", "~> 1.4", :require => false
-  gem "net-ssh", '~> 2.1', :require => false
   gem "puppetlabs_spec_helper", :require => false
-  gem "tzinfo", :require => false
   gem "msgpack", :require => false
 end
 
-require 'yaml'
-data = YAML.load_file(File.join(File.dirname(__FILE__), 'ext', 'project_data.yaml'))
-bundle_platforms = data['bundle_platforms']
-x64_platform = Gem::Platform.local.cpu == 'x64'
-data['gem_platform_dependencies'].each_pair do |gem_platform, info|
-  next if gem_platform == 'x86-mingw32' && x64_platform
-  next if gem_platform == 'x64-mingw32' && !x64_platform
-  if bundle_deps = info['gem_runtime_dependencies']
-    bundle_platform = bundle_platforms[gem_platform] or raise "Missing bundle_platform"
-    if bundle_platform == "all"
-      bundle_deps.each_pair do |name, version|
-        gem(name, version, :require => false)
-      end
-    else
-      platform(bundle_platform.intern) do
-        bundle_deps.each_pair do |name, version|
-          gem(name, version, :require => false)
-        end
-      end
-    end
-  end
-end
 
 if File.exists? "#{__FILE__}.local"
   eval(File.read("#{__FILE__}.local"), binding)

data/lib/puppet.rb

@@ -1,6 +1,6 @@
 require 'puppet/version'
 
-if RUBY_VERSION < "1.9.3"
+if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("1.9.3")
   raise LoadError, "Puppet #{Puppet.version} requires ruby 1.9.3 or greater."
 end
 
@@ -164,7 +164,7 @@ module Puppet
 
   # Now that settings are loaded we have the code loaded to be able to issue
   # deprecation warnings. Warn if we're on a deprecated ruby version.
-  if RUBY_VERSION < Puppet::OLDEST_RECOMMENDED_RUBY_VERSION
+  if Gem::Version.new(RUBY_VERSION) < Gem::Version.new(Puppet::OLDEST_RECOMMENDED_RUBY_VERSION)
     Puppet.deprecation_warning("Support for ruby version #{RUBY_VERSION} is deprecated and will be removed in a future release. See https://docs.puppet.com/puppet/latest/system_requirements.html#ruby for a list of supported ruby versions.")
   end
 

data/lib/puppet/defaults.rb

@@ -407,7 +407,29 @@ deprecated and has been replaced by 'always_retry_plugins'."
     :node_terminus => {
       :type       => :terminus,
       :default    => "plain",
-      :desc       => "Where to find information about nodes.",
+      :desc       => <<-'EOT',
+        Which node data plugin to use when compiling node catalogs.
+
+        When Puppet compiles a catalog, it combines two primary sources of info: the main manifest,
+        and a node data plugin (often called a "node terminus," for historical reasons). Node data
+        plugins provide three things for a given node name:
+
+        1. A list of classes to add to that node's catalog (and, optionally, values for their
+           parameters).
+        2. Which Puppet environment the node should use.
+        3. A list of additional top-scope variables to set.
+
+        The three main node data plugins are:
+
+        * `plain` --- Returns no data, so that the main manifest controls all node configuration.
+        * `exec` --- Uses an
+          [external node classifier (ENC)](https://docs.puppet.com/puppet/latest/nodes_external.html),
+          configured by the `external_nodes` setting. This lets you pull a list of Puppet classes
+          from any external system, using a small glue script to perform the request and format the
+          result as YAML.
+        * `classifier` (formerly `console`) --- Specific to Puppet Enterprise. Uses the PE console
+          for node data."
+      EOT
     },
     :node_cache_terminus => {
       :type       => :terminus,
@@ -1802,14 +1824,24 @@ EOT
     :main,
     :external_nodes => {
         :default  => "none",
-        :desc     => "An external command that can produce node information.  The command's output
-          must be a YAML dump of a hash, and that hash must have a `classes` key and/or
-          a `parameters` key, where `classes` is an array or hash and
-          `parameters` is a hash.  For unknown nodes, the command should
-          exit with a non-zero exit code.
-
-          This command makes it straightforward to store your node mapping
-          information in other data sources like databases.",
+        :desc     => "The external node classifier (ENC) script to use for node data.
+          Puppet combines this data with the main manifest to produce node catalogs.
+
+          To enable this setting, set the `node_terminus` setting to `exec`.
+
+          This setting's value must be the path to an executable command that
+          can produce node information. The command must:
+
+          * Take the name of a node as a command-line argument.
+          * Return a YAML hash with up to three keys:
+            * `classes` --- A list of classes, as an array or hash.
+            * `environment` --- A string.
+            * `parameters` --- A list of top-scope variables to set, as a hash.
+          * For unknown nodes, exit with a non-zero exit code.
+
+          Generally, an ENC script makes requests to an external data source.
+
+          For more info, see [the ENC documentation](https://docs.puppet.com/puppet/latest/nodes_external.html).",
     }
     )
 
@@ -1968,10 +2000,10 @@ EOT
         envs.clear_all unless envs.nil?
       end,
       :desc     => <<-'EOT'
-        Enables having extended data in the catalog by adding the key `ext_parameters` to serialized
-        resources. When enabled, resource containing values of the data types `Binary`, `Regexp`,
+        Enables having extended data in the catalog by storing them as a hash with the special key
+        `__pcore_type__`. When enabled, resource containing values of the data types `Binary`, `Regexp`,
         `SemVer`, `SemVerRange`, `Timespan` and `Timestamp`, as well as instances of types derived
-        from `Object` retain their data type and are serialized using Pcore in `ext_parameters`.
+        from `Object` retain their data type.
       EOT
     }
   )

data/lib/puppet/indirector/catalog/compiler.rb

@@ -25,9 +25,11 @@ class Puppet::Resource::Catalog::Compiler < Puppet::Indirector::Code
       # in Network::HTTP::Handler will automagically deserialize the value.
       if text_facts.is_a?(Puppet::Node::Facts)
         facts = text_facts
-      else
+      elsif format == 'pson'
         # We unescape here because the corresponding code in Puppet::Configurer::FactHandler escapes
-        facts = Puppet::Node::Facts.convert_from(format, CGI.unescape(text_facts))
+        facts = Puppet::Node::Facts.convert_from('pson', CGI.unescape(text_facts))
+      else
+        raise ArgumentError, "Unsupported facts format"
       end
 
       unless facts.name == request.key

data/lib/puppet/module_tool/tar.rb

@@ -6,10 +6,10 @@ module Puppet::ModuleTool::Tar
   require 'puppet/module_tool/tar/mini'
 
   def self.instance
-    if Puppet::Util.which('tar') && ! Puppet::Util::Platform.windows?
-      Gnu.new
-    elsif Puppet.features.minitar? && Puppet.features.zlib?
+    if Puppet.features.minitar? && Puppet.features.zlib?
       Mini.new
+    elsif Puppet::Util.which('tar') && ! Puppet::Util::Platform.windows?
+      Gnu.new
     else
       raise RuntimeError, 'No suitable tar implementation found'
     end

data/lib/puppet/parser/functions/new.rb

@@ -635,7 +635,7 @@ Defaults to `s` at top level and `p` inside array or hash.
 
 | Format    | Regexp Formats
 | ----      | --------------
-| s         | Delimiters `/ /`, alternate flag `#` replaces `/` delimiters with quotes.
+| s         | No delimiters, quoted if alternative flag `#` is used.
 | p         | Delimiters `/ /`.
 
 ### Undef to String
@@ -775,6 +775,19 @@ Accepts a single value as argument:
 Conversion to a `Struct` works exactly as conversion to a `Hash`, only that the constructed hash is
 asserted against the given struct type.
 
+Conversion to a Regexp
+---
+A `String` can be converted into a `Regexp`
+
+**Example**: Converting a String into a Regexp
+```puppet
+$s = '[a-z]+\.com'
+$r = Regexp($s)
+if('foo.com' =~ $r) {
+  ...
+}
+```
+
 Creating a SemVer
 ---
 

data/lib/puppet/pops/lookup.rb

@@ -28,7 +28,7 @@ module Lookup
     override_values = lookup_invocation.override_values
     result_with_name = names.reduce([nil, not_found]) do |memo, key|
       value = override_values.include?(key) ? assert_type(["Value found for key '%s' in override hash", key], value_type, override_values[key]) : not_found
-      catch(:no_such_key) { value = search_and_merge(key, lookup_invocation, merge) } if value.equal?(not_found)
+      catch(:no_such_key) { value = search_and_merge(key, lookup_invocation, merge, false) } if value.equal?(not_found)
       break [key, assert_type('Found value', value_type, value)] unless value.equal?(not_found)
       memo
     end
@@ -72,9 +72,9 @@ module Lookup
   end
 
   # @api private
-  def self.search_and_merge(name, lookup_invocation, merge)
+  def self.search_and_merge(name, lookup_invocation, merge, apl = true)
     answer = lookup_invocation.lookup_adapter.lookup(name, lookup_invocation, merge)
-    lookup_invocation.emit_debug_info("Automatic Parameter Lookup of '#{name}") if Puppet[:debug]
+    lookup_invocation.emit_debug_info("Automatic Parameter Lookup of '#{name}'") if apl && Puppet[:debug]
     answer
   end
 

data/lib/puppet/pops/lookup/global_data_provider.rb

@@ -15,16 +15,13 @@ class GlobalDataProvider < ConfiguredDataProvider
       # Hiera version 3 needs access to special scope variables
       scope = lookup_invocation.scope
       unless scope.is_a?(Hiera::Scope)
-        hiera_invocation = Invocation.new(
-          Hiera::Scope.new(scope),
-          lookup_invocation.override_values,
-          lookup_invocation.default_values,
-          lookup_invocation.explainer)
+        return lookup_invocation.with_scope(Hiera::Scope.new(scope)) do |hiera_invocation|
 
-        # Confine to global scope unless an environment data provider has been defined (same as for hiera_xxx functions)
-        adapter = lookup_invocation.lookup_adapter
-        hiera_invocation.set_global_only unless adapter.global_only? || adapter.has_environment_data_provider?(lookup_invocation)
-        lookup_invocation = hiera_invocation
+          # Confine to global scope unless an environment data provider has been defined (same as for hiera_xxx functions)
+          adapter = lookup_invocation.lookup_adapter
+          hiera_invocation.set_global_only unless adapter.global_only? || adapter.has_environment_data_provider?(lookup_invocation)
+          hiera_invocation.lookup(key, lookup_invocation.module_name) { unchecked_key_lookup(key , hiera_invocation, merge) }
+        end
       end
 
       merge = MergeStrategy.strategy(merge)

data/lib/puppet/pops/lookup/interpolation.rb

@@ -70,15 +70,14 @@ module Interpolation
   def interpolate_method(method_key)
     @@interpolate_methods ||= begin
       global_lookup = lambda do |key, lookup_invocation, _|
-        if lookup_invocation.scope.is_a?(Hiera::Scope) && !lookup_invocation.global_only?
+        scope = lookup_invocation.scope
+        if scope.is_a?(Hiera::Scope) && !lookup_invocation.global_only?
           # "unwrap" the Hiera::Scope
-          lookup_invocation = Invocation.new(
-            lookup_invocation.scope.real,
-            lookup_invocation.override_values,
-            lookup_invocation.default_values,
-            lookup_invocation.explainer)
+          scope = scope.real
+        end
+        lookup_invocation.with_scope(scope) do |sub_invocation|
+          sub_invocation.lookup(key) {  Lookup.lookup(key, nil, '', true, nil, sub_invocation) }
         end
-        Lookup.lookup(key, nil, '', true, nil, lookup_invocation)
       end
       scope_lookup = lambda do |key, lookup_invocation, subject|
         segments = split_key(key) { |problem| Puppet::DataBinding::LookupError.new("#{problem} in string: #{subject}") }

data/lib/puppet/pops/lookup/invocation.rb

@@ -10,6 +10,15 @@ class Invocation
     @current
   end
 
+  # Creates a new instance with same settings as this instance but with a new given scope
+  # and yields with that scope.
+  #
+  # @param scope [Puppet::Parser::Scope] The new scope
+  # @return [Invocation] the new instance
+  def with_scope(scope)
+    yield(Invocation.new(scope, override_values, default_values, explainer))
+  end
+
   # Creates a context object for a lookup invocation. The object contains the current scope, overrides, and default
   # values and may optionally contain an {ExplanationAcceptor} instance that will receive book-keeping information
   # about the progress of the lookup.
@@ -53,9 +62,10 @@ class Invocation
     @explainer = explainer
   end
 
-  def lookup(key, module_name)
+  def lookup(key, module_name = nil)
+    key = LookupKey.new(key) unless key.is_a?(LookupKey)
     @top_key = key
-    @module_name = module_name
+    @module_name = module_name.nil? ? key.module_name : module_name
     save_current = self.class.current
     if save_current.equal?(self)
       yield

data/lib/puppet/pops/lookup/lookup_adapter.rb

@@ -289,13 +289,17 @@ class LookupAdapter < DataAdapter
           compile_patterns(global_lookup_options(meta_invocation, merge_strategy))
         else
           opts = env_lookup_options(meta_invocation, merge_strategy)
-          catch(:no_such_key) do
-            module_opts = validate_lookup_options(lookup_in_module(LookupKey::LOOKUP_OPTIONS, meta_invocation, merge_strategy), module_name)
-            opts = if opts.nil?
-              module_opts
-            else
-              merge_strategy.lookup([GLOBAL_ENV_MERGE, "Module #{lookup_invocation.module_name}"], meta_invocation) do |n|
-                meta_invocation.with(:scope, n) { meta_invocation.report_found(LOOKUP_OPTIONS,  n == GLOBAL_ENV_MERGE ? opts : module_opts) }
+          unless module_name.nil?
+            # Store environment options at key nil. This removes the need for an additional lookup for keys that are not prefixed.
+            @lookup_options[nil] = compile_patterns(opts) unless @lookup_options.include?(nil)
+            catch(:no_such_key) do
+              module_opts = validate_lookup_options(lookup_in_module(LookupKey::LOOKUP_OPTIONS, meta_invocation, merge_strategy), module_name)
+              opts = if opts.nil?
+                module_opts
+              else
+                merge_strategy.lookup([GLOBAL_ENV_MERGE, "Module #{lookup_invocation.module_name}"], meta_invocation) do |n|
+                  meta_invocation.with(:scope, n) { meta_invocation.report_found(LOOKUP_OPTIONS,  n == GLOBAL_ENV_MERGE ? opts : module_opts) }
+                end
               end
             end
           end

data/lib/puppet/pops/serialization/object.rb

@@ -52,7 +52,7 @@ class ObjectWriter
       args.pop
     end
 
-    if type.name.start_with?('Pcore::')
+    if type.name.start_with?('Pcore::') || serializer.type_by_reference?
       serializer.push_written(value)
       serializer.start_pcore_object(type.name, args.size)
     else

data/lib/puppet/pops/serialization/serializer.rb

@@ -10,11 +10,14 @@ module Serialization
     # @api private
     attr_reader :writer
 
-    # @param [AbstractWriter] writer the writer that is used for writing primitive values
+    # @param writer [AbstractWriter] the writer that is used for writing primitive values
+    # @param options [{String, Object}] serialization options
+    # @option options [Boolean] :type_by_reference `true` if Object types are serialized by name only.
     # @api public
-    def initialize(writer)
+    def initialize(writer, options = EMPTY_HASH)
       @written = {}
       @writer = writer
+      @options = options
     end
 
     # Tell the underlying writer to finish
@@ -81,6 +84,10 @@ module Serialization
       @writer.write(Extension::SensitiveStart::INSTANCE)
     end
 
+    def type_by_reference?
+      @options[:type_by_reference] == true
+    end
+
     # First time write of a tabulated object. This means that the object is written and then remembered. Subsequent writes
     # of the same object will yield a write of a tabulation index instead.
     # @param [Object] value the value to write

data/lib/puppet/pops/types/string_converter.rb

@@ -51,7 +51,7 @@ class StringConverter
 
   # Format represents one format specification that is textually represented by %<flags><width>.<precision><format>
   # Format parses and makes the individual parts available when an instance is created.
-  # 
+  #
   # @api private
   #
   class Format
@@ -172,7 +172,7 @@ class StringConverter
     # Sorts format based on generality of types - most specific types before general
     #
     def self.sort_formats(format_map)
-      format_map = format_map.sort do |(a,_),(b,_)| 
+      format_map = format_map.sort do |(a,_),(b,_)|
         ab = b.assignable?(a)
         ba = a.assignable?(b)
         if a == b
@@ -219,7 +219,7 @@ class StringConverter
     end
     # Returns an array with a delimiter pair derived from the format.
     # If format does not contain a delimiter specification the given default is returned
-    # 
+    #
     # @param [Array<String>] the default delimiters
     # @returns [Array<String>] a tuple with left, right delimiters
     #
@@ -289,7 +289,7 @@ class StringConverter
   # When converting to string it is possible to use a set of built in conversion rules.
   #
   # A format is specified on the form:
-  # 
+  #
   # ´´´
   # %[Flags][Width][.Precision]Format
   # ´´´
@@ -348,9 +348,9 @@ class StringConverter
   # Defaults to `s` at top level and `p` inside array or hash.
   #
   # ### Boolean
-  # 
+  #
   # | Format    | Boolean Formats
-  # | ----      | -------------------   
+  # | ----      | -------------------
   # | t T       | 'true'/'false' or 'True'/'False' , first char if alternate form is used (i.e. 't'/'f' or 'T'/'F').
   # | y Y       | 'yes'/'no', 'Yes'/'No', 'y'/'n' or 'Y'/'N' if alternative flag # is used
   # | dxXobB    | numeric value 0/1 in accordance with the given format which must be valid integer format
@@ -379,7 +379,7 @@ class StringConverter
   # | u         | 'undef', or 'undefined' if alternative # flag is used
   #
   # ### Default (value)
-  # 
+  #
   # | Format    | Default formats
   # | ------    | ---------------
   # | d D       | 'default' or 'Default', alternative form # causes value to be quoted
@@ -420,18 +420,18 @@ class StringConverter
   #
   # | Format    | Hash/Struct Formats
   # | ------    | -------------
-  # | h         | formats with `{ }` delimiters, `,` element separator and ` => ` inner element separator unless overridden by flags 
+  # | h         | formats with `{ }` delimiters, `,` element separator and ` => ` inner element separator unless overridden by flags
   # | s         | same as h
   # | p         | same as h
   # | a         | converts the hash to an array of [k,v] tuples and formats it using array rule(s)
-  # 
+  #
   # See "Flags" `<[({\|` for formatting of delimiters, and "Additional parameters for containers; Array and Hash" for
   # more information about options.
   #
   # The alternate form flag `#` will format each hash key/value entry indented on a separate line.
   #
   # ### Type
-  # 
+  #
   # | Format    | Array/Tuple Formats
   # | ------    | -------------
   # | s        | The same as p, quoted if alternative flag # is used
@@ -439,7 +439,7 @@ class StringConverter
   #
   # ### Flags
   #
-  # | Flag     | Effect 
+  # | Flag     | Effect
   # | ------   | ------
   # | (space)  | space instead of + for numeric output (- is shown), for containers skips delimiters
   # | #        | alternate format; prefix 0x/0x, 0 (octal) and 0b/0B for binary, Floats force decimal '.'. For g/G keep trailing 0.
@@ -448,7 +448,7 @@ class StringConverter
   # | 0        | pad with 0 instead of space for widths larger than value
   # | <[({\|   | defines an enclosing pair <> [] () {} or \| \| when used with a container type
   #
-  # 
+  #
   # ### Additional parameters for containers; Array and Hash
   #
   # For containers (Array and Hash), the format is specified by a hash where the following keys can be set:
@@ -456,7 +456,7 @@ class StringConverter
   # * `'separator'` - the separator string to use between elements, should not contain padding space at the end
   # * `'separator2'` - the separator string to use between association of hash entries key/value
   # * `'string_formats'´ - a map of type to format for elements contained in the container
-  # 
+  #
   # Note that the top level format applies to Array and Hash objects contained/nested in an Array or a Hash.
   #
   # Given format mappings are merged with (default) formats and a format specified for a narrower type
@@ -538,8 +538,9 @@ class StringConverter
   end
   private :validate_container_input
 
-  def string_PRuntimeType(val_type, val, format_map)
-    case (f = get_format(val_type, format_map))
+  def string_PRuntimeType(val_type, val, format_map, _)
+    f = get_format(val_type, format_map)
+    case f.format
     when :s
       val.to_s
     when :q
@@ -550,7 +551,7 @@ class StringConverter
       converted = convert(o, PNumericType) # rest is default
       "%#{f}" % converted
     else
-      throw(:failed_conversion, [o, PStringType::DEFAULT, f])
+      raise FormatError.new('Runtime', f.format, 'sqidxof')
     end
   end
 
@@ -838,13 +839,15 @@ class StringConverter
     f = get_format(val_type, format_map)
     case f.format
     when :p
-      Kernel.format(f.orig_fmt, val)
+      str_regexp = '/'
+      str_regexp << (val.options == 0 ? val.source : val.to_s) << '/'
+      f.orig_fmt == '%p' ? str_regexp : Kernel.format(f.orig_fmt.gsub('p', 's'), str_regexp)
     when :s
-      str_regexp = val.inspect
-      str_regexp = f.alt? ? "\"#{str_regexp[1..-2]}\"" : str_regexp
-      Kernel.format(f.orig_fmt, str_regexp)
+      str_regexp = val.options == 0 ? val.source : val.to_s
+      str_regexp = puppet_quote(str_regexp) if f.alt?
+      f.orig_fmt == '%s' ? str_regexp : Kernel.format(f.orig_fmt, str_regexp)
     else
-      raise FormatError.new('Regexp', f.format, 'rsp')
+      raise FormatError.new('Regexp', f.format, 'sp')
     end
   end