puppet 4.2.0 → 4.3.0

data/lib/puppet/data_providers.rb

@@ -4,28 +4,27 @@ module Puppet::DataProviders
     unless @loaded
       require 'puppet/pops'
       require 'puppet/data_providers/data_adapter'
+      require 'puppet/data_providers/lookup_adapter'
     end
     @loaded = true
   end
 
-  def self.lookup_in_environment(name, scope, merge)
-    assert_loaded()
-    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
-    adapter.env_provider.lookup(name, scope, merge)
+  # @deprecated use `lookup_adapter(lookup_invocation).lookup` instead
+  def self.lookup_in_environment(name, lookup_invocation, merge)
+    Puppet.deprecation_warning('The method Puppet::DataProviders.lookup_in_environment is deprecated and will be removed in the next major release of Puppet.')
+    lookup_adapter(lookup_invocation).lookup_in_environment(name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(merge))
   end
 
   MODULE_NAME = 'module_name'.freeze
 
-  def self.lookup_in_module(name, scope, merge)
-    # Do not attempt to do a lookup in a module unless the name is qualified.
-    qual_index = name.index('::')
-    throw :no_such_key if qual_index.nil?
-    module_name = name[0..qual_index-1]
+  # @deprecated use `adapter(lookup_invocation).lookup` instead
+  def self.lookup_in_module(name, lookup_invocation, merge)
+    Puppet.deprecation_warning('The method Puppet::DataProviders.lookup_in_module is deprecated and will be removed in the next major release of Puppet.')
+    lookup_adapter(lookup_invocation).lookup_in_module(name, lookup_invocation, Puppet::Pops::MergeStrategy.strategy(merge))
+  end
 
+  def self.lookup_adapter(lookup_invocation)
     assert_loaded()
-    adapter = Puppet::DataProviders::DataAdapter.adapt(Puppet.lookup(:current_environment))
-    data_provider = adapter.module_provider(module_name)
-    throw :no_such_key if data_provider.nil?
-    data_provider.lookup(name, scope, merge)
+    LookupAdapter.adapt(lookup_invocation.scope.environment)
   end
 end

data/lib/puppet/defaults.rb

@@ -20,7 +20,9 @@ module Puppet
     :cfacter => {
         :default => false,
         :type    => :boolean,
-        :desc    => 'Whether or not to use the native facter (cfacter) implementation instead of the Ruby one (facter). Defaults to false.',
+        :desc    => 'Whether to enable a pre-Facter 3.0 release of native Facter (distributed as
+          the "cfacter" package). This is not necessary if Facter 3.0 or later is installed.
+          This setting is deprecated, as Facter 3 is now the default in puppet-agent.',
         :hook    => proc do |value|
           return unless value
           raise ArgumentError, 'facter has already evaluated facts.' if Facter.instance_variable_get(:@collection)
@@ -525,13 +527,22 @@ module Puppet
       :desc    => "The directory where catalog previews per node are generated."
     }
   )
+
+  define_settings(:main,
+      :app_management => {
+          :default  => false,
+          :type     => :boolean,
+          :desc     => "Whether the application management feature is on or off. You must restart Puppet Server after changing this setting.",
+      }
+  )
+
   Puppet.define_settings(:module_tool,
     :module_repository  => {
       :default  => 'https://forgeapi.puppetlabs.com',
       :desc     => "The module repository",
     },
     :module_working_dir => {
-        :default  => '$vardir/puppet-module',
+        :default  => (Puppet.features.microsoft_windows? ? Dir.tmpdir() : '$vardir/puppet-module'),
         :desc     => "The directory into which module tool data is stored",
     },
     :module_skeleton_dir => {
@@ -544,7 +555,7 @@ module Puppet
     },
     :module_groups => {
         :default  => nil,
-        :desc     => "Extra module groups to request from the Puppet Forge",
+        :desc     => "Extra module groups to request from the Puppet Forge. This is an internal setting, and users should never change it.",
     }
   )
 

data/lib/puppet/environments.rb

@@ -38,6 +38,9 @@ module Puppet::Environments
         raise EnvironmentNotFound, name
       end
     end
+
+    def clear_all
+    end
   end
 
   # @!macro [new] loader_search_paths
@@ -193,12 +196,13 @@ module Puppet::Environments
     def create_environment(name)
       env_symbol = name.intern
       setting_values = Puppet.settings.values(env_symbol, Puppet.settings.preferred_run_mode)
-      Puppet::Node::Environment.create(
+      env = Puppet::Node::Environment.create(
         env_symbol,
         Puppet::Node::Environment.split_path(setting_values.interpolate(:modulepath)),
         setting_values.interpolate(:manifest),
         setting_values.interpolate(:config_version)
       )
+      env
     end
 
     def valid_directory?(envdir)
@@ -257,6 +261,9 @@ module Puppet::Environments
       nil
     end
 
+    def clear_all
+      @loaders.each {|loader| loader.clear_all}
+    end
   end
 
   class Cached
@@ -282,10 +289,29 @@ module Puppet::Environments
       @cache_expiration_service || DefaultCacheExpirationService.new
     end
 
+    # Returns the end of time (the next Mesoamerican Long Count cycle-end after 2012 (5125+2012) = 7137,
+    # of for a 32 bit machine using Ruby < 1.9.3, the year 2038.
+    def self.end_of_time
+      begin
+        Time.gm(7137)
+      rescue ArgumentError
+        Time.gm(2038)
+      end
+    end
+
+    END_OF_TIME = end_of_time
+    START_OF_TIME = Time.gm(1)
+
     def initialize(loader)
       @loader = loader
-      @cache = {}
       @cache_expiration_service = Puppet::Environments::Cached.cache_expiration_service
+      @cache = {}
+
+      # Holds expiration times in sorted order - next to expire is first
+      @expirations = SortedSet.new
+
+      # Infinity since it there are no entries, this is a cache of the first to expire time
+      @next_expiration = END_OF_TIME
     end
 
     # @!macro loader_list
@@ -300,15 +326,34 @@ module Puppet::Environments
 
     # @!macro loader_get
     def get(name)
-      evict_if_expired(name)
+      # Aggressively evict all that has expired
+      # This strategy favors smaller memory footprint over environment
+      # retrieval time.
+      clear_all_expired
       if result = @cache[name]
+        # found in cache
         return result.value
       elsif (result = @loader.get(name))
-        @cache[name] = entry(result)
+        # environment loaded, cache it
+        cache_entry = entry(result)
+        @cache_expiration_service.created(result)
+        add_entry(name, cache_entry)
         result
       end
     end
 
+    # Adds a cache entry to the cache
+    def add_entry(name, cache_entry)
+      Puppet.debug {"Caching environment '#{name}' #{cache_entry.label}"}
+      @cache[name] = cache_entry
+      expires = cache_entry.expires
+      @expirations.add(expires)
+      if @next_expiration > expires
+        @next_expiration = expires
+      end
+    end
+    private :add_entry
+
     # Clears the cache of the environment with the given name.
     # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear(name)
@@ -318,7 +363,27 @@ module Puppet::Environments
     # Clears all cached environments.
     # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear_all()
+      super
       @cache = {}
+      @expirations.clear
+      @next_expiration = END_OF_TIME
+    end
+
+    # Clears all environments that have expired, either by exceeding their time to live, or
+    # through an explicit eviction determined by the cache expiration service.
+    #
+    def clear_all_expired()
+      t = Time.now
+      return if t < @next_expiration && ! @cache.any? {|name, _| @cache_expiration_service.expired?(name.to_sym) }
+      to_expire = @cache.select { |name, entry| entry.expires < t || @cache_expiration_service.expired?(name.to_sym) }
+      to_expire.each do |name, entry|
+        Puppet.debug {"Evicting cache entry for environment '#{name}'"}
+        @cache_expiration_service.evicted(name)
+        clear(name)
+        @expirations.delete(entry.expires)
+        Puppet.settings.clear_environment_settings(name)
+      end
+      @next_expiration = @expirations.first || END_OF_TIME
     end
 
     # This implementation evicts the cache, and always gets the current
@@ -337,9 +402,7 @@ module Puppet::Environments
     # Creates a suitable cache entry given the time to live for one environment
     #
     def entry(env)
-      @cache_expiration_service.created(env)
       ttl = (conf = get_conf(env.name)) ? conf.environment_timeout : Puppet.settings.value(:environment_timeout)
-      Puppet.debug {"Caching environment '#{env.name}' (cache ttl: #{ttl})"}
       case ttl
       when 0
         NotCachedEntry.new(env)     # Entry that is always expired (avoids syscall to get time)
@@ -373,6 +436,14 @@ module Puppet::Environments
       def expired?
         false
       end
+
+      def label
+        ""
+      end
+
+      def expires
+        END_OF_TIME
+      end
     end
 
     # Always evicting entry
@@ -380,6 +451,14 @@ module Puppet::Environments
       def expired?
         true
       end
+
+      def label
+        "(ttl = 0 sec)"
+      end
+
+      def expires
+        START_OF_TIME
+      end
     end
 
     # Time to Live eviction policy entry
@@ -387,11 +466,20 @@ module Puppet::Environments
       def initialize(value, ttl_seconds)
         super value
         @ttl = Time.now + ttl_seconds
+        @ttl_seconds = ttl_seconds
       end
 
       def expired?
         Time.now > @ttl
       end
+
+      def label
+        "(ttl = #{@ttl_seconds} sec)"
+      end
+
+      def expires
+        @ttl
+      end
     end
   end
 end

data/lib/puppet/error.rb

@@ -93,4 +93,8 @@ module Puppet
   class MissingCommand < Puppet::Error
   end
 
+  # Raised when we failed to aquire a lock
+  class LockError < Puppet::Error
+  end
+
 end

data/lib/puppet/face/module/changes.rb

@@ -4,7 +4,8 @@ Puppet::Face.define(:module, '1.0.0') do
     description <<-EOT
       Shows any files in a module that have been modified since it was
       installed. This action compares the files on disk to the md5 checksums
-      included in the module's metadata.
+      included in the module's checksums.json or, if that is missing, in
+      metadata.json.
     EOT
 
     returns "Array of strings representing paths of modified files."

data/lib/puppet/feature/cfacter.rb

@@ -2,6 +2,7 @@ require 'facter'
 require 'puppet/util/feature'
 
 Puppet.features.add :cfacter do
+  Puppet.deprecation_warning("The cfacter setting is deprecated. You can use Facter 3 and higher without this setting.")
   begin
     require 'cfacter'
 

data/lib/puppet/file_bucket/dipper.rb

@@ -2,6 +2,8 @@ require 'pathname'
 require 'puppet/file_bucket'
 require 'puppet/file_bucket/file'
 require 'puppet/indirector/request'
+require 'puppet/util/diff'
+require 'tempfile'
 
 class Puppet::FileBucket::Dipper
   include Puppet::Util::Checksums
@@ -22,7 +24,7 @@ class Puppet::FileBucket::Dipper
       @rest_path  = nil
     else
       @local_path = nil
-      @rest_path = "https://#{server}:#{port}/#{environment}/file_bucket_file/"
+      @rest_path = "filebucket://#{server}:#{port}/"
     end
     @checksum_type = Puppet[:digest_algorithm].to_sym
     @digest = method(@checksum_type)
@@ -56,6 +58,46 @@ class Puppet::FileBucket::Dipper
     end
   end
 
+  # Diffs two filebucket files identified by their sums
+  def diff(checksum_a, checksum_b, file_a, file_b)
+    raise RuntimeError, "Diff is not supported on this platform" if Puppet[:diff] == ""
+    if checksum_a
+      source_path = "#{@rest_path}#{@checksum_type}/#{checksum_a}"
+      if checksum_b
+        file_diff = Puppet::FileBucket::File.indirection.find(
+          source_path,
+          :bucket_path => @local_path,
+          :diff_with => checksum_b)
+      elsif file_b
+        tmp_file = ::Tempfile.new('diff')
+        begin
+          restore(tmp_file.path, checksum_a)
+          file_diff = Puppet::Util::Diff.diff(tmp_file.path, file_b)
+        ensure
+          tmp_file.close
+          tmp_file.unlink
+        end
+      else
+        raise Puppet::Error, "Please provide a file or checksum do diff with"
+      end
+    elsif file_a
+      if checksum_b
+        tmp_file = ::Tempfile.new('diff')
+        begin
+          restore(tmp_file.path, checksum_b)
+          file_diff = Puppet::Util::Diff.diff(file_a, tmp_file.path)
+        ensure
+          tmp_file.close
+          tmp_file.unlink
+        end
+      elsif file_b
+        file_diff = Puppet::Util::Diff.diff(file_a, file_b)
+      end
+    end
+    raise Puppet::Error, "Failed to diff files" unless file_diff
+    file_diff.to_s
+  end
+
   # Retrieves a file by sum.
   def getfile(sum)
     get_bucket_file(sum).to_s
@@ -109,10 +151,24 @@ class Puppet::FileBucket::Dipper
     end
   end
 
+  # List Filebucket content.
+  def list(fromdate, todate)
+    raise Puppet::Error, "Listing remote file buckets is not allowed" unless local?
+
+    source_path = "#{@rest_path}#{@checksum_type}/"
+    file_bucket_list = Puppet::FileBucket::File.indirection.find(
+      source_path,
+      :bucket_path => @local_path,
+      :list_all => true,
+      :fromdate => fromdate,
+      :todate => todate)
+    raise Puppet::Error, "File not found" unless file_bucket_list
+    file_bucket_list.to_s
+  end
+
   private
   def absolutize_path( path )
     Pathname.new(path).realpath
   end
 
 end
-

data/lib/puppet/file_system/windows.rb

@@ -62,7 +62,9 @@ class Puppet::FileSystem::Windows < Puppet::FileSystem::Posix
       stat = Puppet::Util::Windows::File.stat(file_name) rescue nil
 
       # sigh, Ruby + Windows :(
-      if stat && stat.ftype == 'directory'
+      if !stat
+        ::File.unlink(file_name) rescue Dir.rmdir(file_name)
+      elsif stat.ftype == 'directory'
         if Puppet::Util::Windows::File.symlink?(file_name)
           Dir.rmdir(file_name)
         else

data/lib/puppet/functions/assert_type.rb

@@ -1,18 +1,54 @@
-# Returns the given value if it is an instance of the given type, and raises an error otherwise.
-# Optionally, if a block is given (accepting two parameters), it will be called instead of raising
-# an error. This to enable giving the user richer feedback, or to supply a default value.
+# Returns the given value if it is of the given
+# [data type](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html), or
+# otherwise either raises an error or executes an optional two-parameter
+# [lambda](https://docs.puppetlabs.com/puppet/latest/reference/lang_lambdas.html).
 #
-# @example how to assert type
-#   # assert that `$b` is a non empty `String` and assign to `$a`
-#   $a = assert_type(String[1], $b)
+# The function takes two mandatory arguments, in this order:
 #
-# @example using custom error message
-#   $a = assert_type(String[1], $b) |$expected, $actual| { fail("The name cannot be empty") }
+# 1. The expected data type.
+# 2. A value to compare against the expected data type.
 #
-# @example using a warning and a default
-#   $a = assert_type(String[1], $b) |$expected, $actual| { warning("Name is empty, using default") 'anonymous' }
+# @example Using `assert_type`
 #
-# See the documentation for "The Puppet Type System" for more information about types.
+# ~~~ puppet
+# $raw_username = 'Amy Berry'
+#
+# # Assert that $raw_username is a non-empty string and assign it to $valid_username.
+# $valid_username = assert_type(String[1], $raw_username)
+#
+# # $valid_username contains "Amy Berry".
+# # If $raw_username was an empty string or a different data type, the Puppet run would
+# # fail with an "Expected type does not match actual" error.
+# ~~~
+#
+# You can use an optional lambda to provide enhanced feedback. The lambda takes two
+# mandatory parameters, in this order:
+#
+# 1. The expected data type as described in the function's first argument.
+# 2. The actual data type of the value.
+#
+# @example Using `assert_type` with a warning and default value
+#
+# ~~~ puppet
+# $raw_username = 'Amy Berry'
+#
+# # Assert that $raw_username is a non-empty string and assign it to $valid_username.
+# # If it isn't, output a warning describing the problem and use a default value.
+# $valid_username = assert_type(String[1], $raw_username) |$expected, $actual| {
+#   warning( "The username should be \'${expected}\', not \'${actual}\'. Using 'anonymous'." )
+#   'anonymous'
+# }
+#
+# # $valid_username contains "Amy Berry".
+# # If $raw_username was an empty string, the Puppet run would set $valid_username to
+# # "anonymous" and output a warning: "The username should be 'String[1, default]', not
+# # 'String[0, 0]'. Using 'anonymous'."
+# ~~~
+#
+# For more information about data types, see the
+# [documentation](https://docs.puppetlabs.com/puppet/latest/reference/lang_data.html).
+#
+# @since 4.0.0
 #
 Puppet::Functions.create_function(:assert_type) do
   dispatch :assert_type do
@@ -42,7 +78,7 @@ Puppet::Functions.create_function(:assert_type) do
         # Do not give all the details - i.e. format as Integer, instead of Integer[n, n] for exact value, which
         # is just confusing. (OTOH: may need to revisit, or provide a better "type diff" output.
         #
-        actual = Puppet::Pops::Types::TypeCalculator.generalize!(inferred_type)
+        actual = Puppet::Pops::Types::TypeCalculator.generalize(inferred_type)
         raise Puppet::Pops::Types::TypeAssertionError.new("assert_type(): Expected type #{type} does not match actual: #{actual}", type,actual)
       end
     end

data/lib/puppet/functions/defined.rb

@@ -1,70 +1,101 @@
-# Determines whether
-# a given class or resource type is defined. This function can also determine whether a
-# specific resource has been declared, or whether a variable has been assigned a value
-# (including undef...as opposed to never having been assigned anything). Returns true
-# or false. Accepts class names, type names, resource references, and variable
-# reference strings of the form '$name'.  When more than one argument is
-# supplied, defined() returns true if any are defined.
+# Determines whether a given class or resource type is defined and returns a Boolean
+# value. You can also use `defined` to determine whether a specific resource is defined,
+# or whether a variable has a value (including `undef`, as opposed to the variable never
+# being declared or assigned).
+#
+# This function takes at least one string argument, which can be a class name, type name,
+# resource reference, or variable reference of the form `'$name'`.
 #
 # The `defined` function checks both native and defined types, including types
-# provided as plugins via modules. Types and classes are both checked using their names:
+# provided by modules. Types and classes are matched by their names. The function matches
+# resource declarations by using resource references.
+#
+# **Examples**: Different types of `defined` function matches
+#
+# ~~~ puppet
+# # Matching resource types
+# defined("file")
+# defined("customtype")
+#
+# # Matching defines and classes
+# defined("foo")
+# defined("foo::bar")
+#
+# # Matching variables
+# defined('$name')
+#
+# # Matching declared resources
+# defined(File['/tmp/file'])
+# ~~~
+#
+# Puppet depends on the configuration's evaluation order when checking whether a resource
+# is declared.
 #
-#     defined("file")
-#     defined("customtype")
-#     defined("foo")
-#     defined("foo::bar")
-#     defined('$name')
+# @example Importance of evaluation order when using `defined`
 #
-# Resource declarations are checked using resource references, e.g.
-# `defined( File['/tmp/myfile'] )`, or `defined( Class[myclass] )`.
-# Checking whether a given resource
-# has been declared is, unfortunately, dependent on the evaluation order of
-# the configuration, and the following code will not work:
+# ~~~ puppet
+# # Assign values to $is_defined_before and $is_defined_after using identical `defined`
+# # functions.
 #
-#     if defined(File['/tmp/foo']) {
-#         notify { "This configuration includes the /tmp/foo file.":}
-#     }
-#     file { "/tmp/foo":
-#         ensure => present,
-#     }
+# $is_defined_before = defined(File['/tmp/file'])
 #
-# However, this order requirement refers to evaluation order only, and ordering of
-# resources in the configuration graph (e.g. with `before` or `require`) does not
-# affect the behavior of `defined`.
+# file { "/tmp/file":
+#   ensure => present,
+# }
 #
-# You may also search using types:
+# $is_defined_after = defined(File['/tmp/file'])
 #
-#     defined(Resource['file','/some/file'])
-#     defined(File['/some/file'])
-#     defined(Class['foo'])
+# # $is_defined_before returns false, but $is_defined_after returns true.
+# ~~~
 #
-# The `defined` function does not answer if data types (e.g. `Integer`) are defined. If
-# given the string 'integer' the result is false, and if given a non CatalogEntry type,
-# an error is raised.
+# This order requirement only refers to evaluation order. The order of resources in the
+# configuration graph (e.g. with `before` or `require`) does not affect the `defined`
+# function's behavior.
 #
-# The rules for asking for undef, empty strings, and the main class are different from 3.x
-# (non future parser) and 4.x (with future parser or in Puppet 4.0.0 and later):
+# > **Warning:** Avoid relying on the result of the `defined` function in modules, as you
+# > might not be able to guarantee the evaluation order well enough to produce consistent
+# > results. This can cause other code that relies on the function's result to behave
+# > inconsistently or fail.
 #
-#     defined('')     # 3.x => true,  4.x => false
-#     defined(undef)  # 3.x => true,  4.x => error
-#     defined('main') # 3.x => false, 4.x => true
+# If you pass more than one argument to `defined`, the function returns `true` if _any_
+# of the arguments are defined. You can also match resources by type, allowing you to
+# match conditions of different levels of specificity, such as whether a specific resource
+# is of a specific data type.
 #
-# With the future parser, it is also possible to ask specifically if a name is
-# a resource type (built in or defined), or a class, by giving its type:
+# @example Matching multiple resources and resources by different types with `defined`
 #
-#     defined(Type[Class['foo']])
-#     defined(Type[Resource['foo']])
+# ~~~ puppet
+# file { "/tmp/file1":
+#   ensure => file,
+# }
 #
-# Which is different from asking:
+# $tmp_file = file { "/tmp/file2":
+#   ensure => file,
+# }
 #
-#     defined('foo')
+# # Each of these statements return `true` ...
+# defined(File['/tmp/file1'])
+# defined(File['/tmp/file1'],File['/tmp/file2'])
+# defined(File['/tmp/file1'],File['/tmp/file2'],File['/tmp/file3'])
+# # ... but this returns `false`.
+# defined(File['/tmp/file3'])
 #
-# Since the later returns true if 'foo' is either a class, a built-in resource type, or a user defined
-# resource type, and a specific request like `Type[Class['foo']]` only returns true if `'foo'` is a class.
+# # Each of these statements returns `true` ...
+# defined(Type[Resource['file','/tmp/file2']])
+# defined(Resource['file','/tmp/file2'])
+# defined(File['/tmp/file2'])
+# defined('$tmp_file')
+# # ... but each of these returns `false`.
+# defined(Type[Resource['exec','/tmp/file2']])
+# defined(Resource['exec','/tmp/file2'])
+# defined(File['/tmp/file3'])
+# defined('$tmp_file2')
+# ~~~
 #
 # @since 2.7.0
-# @since 3.6.0 variable reference and future parser types")
+# @since 3.6.0 variable reference and future parser types
 # @since 3.8.1 type specific requests with future parser
+# @since 4.0.0
 #
 Puppet::Functions.create_function(:'defined', Puppet::Functions::InternalFunction) do