puppet 5.3.4 → 6.29.0

data/lib/puppet/type/file.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require 'digest/md5'
 require 'cgi'
 require 'etc'
@@ -12,7 +13,6 @@ require 'puppet/util/backups'
 require 'puppet/util/symbolic_file_mode'
 
 Puppet::Type.newtype(:file) do
-  include Puppet::Util::MethodHelper
   include Puppet::Util::Checksums
   include Puppet::Util::Backups
   include Puppet::Util::SymbolicFileMode
@@ -31,7 +31,12 @@ Puppet::Type.newtype(:file) do
 
     **Autorequires:** If Puppet is managing the user or group that owns a
     file, the file resource will autorequire them. If Puppet is managing any
-    parent directories of a file, the file resource will autorequire them."
+    parent directories of a file, the file resource autorequires them.
+
+    Warning: Enabling `recurse` on directories containing large numbers of
+    files slows agent runs. To manage file attributes for many files,
+    consider using alternative methods such as the `chmod_r`, `chown_r`,
+     or `recursive_file_permissions` modules from the Forge."
 
   feature :manages_symlinks,
     "The provider can manage symbolic links."
@@ -74,7 +79,7 @@ Puppet::Type.newtype(:file) do
       (`File { backup => main }`), so it can affect all file resources.
 
       * If set to `false`, file content won't be backed up.
-      * If set to a string beginning with `.` (e.g., `.puppet-bak`), Puppet will
+      * If set to a string beginning with `.`, such as `.puppet-bak`, Puppet will
         use copy the file in the same directory with that value as the extension
         of the backup. (A value of `true` is a synonym for `.puppet-bak`.)
       * If set to any other string, Puppet will try to back up to a filebucket
@@ -88,23 +93,36 @@ Puppet::Type.newtype(:file) do
 
       Backing up to a local filebucket isn't particularly useful. If you want
       to make organized use of backups, you will generally want to use the
-      puppet master server's filebucket service. This requires declaring a
+      primary Puppet server's filebucket service. This requires declaring a
       filebucket resource and a resource default for the `backup` attribute
       in site.pp:
 
           # /etc/puppetlabs/puppet/manifests/site.pp
           filebucket { 'main':
             path   => false,                # This is required for remote filebuckets.
-            server => 'puppet.example.com', # Optional; defaults to the configured puppet master.
+            server => 'puppet.example.com', # Optional; defaults to the configured primary Puppet server.
           }
 
           File { backup => main, }
 
-      If you are using multiple puppet master servers, you will want to
+      If you are using multiple primary servers, you will want to
       centralize the contents of the filebucket. Either configure your load
-      balancer to direct all filebucket traffic to a single master, or use
+      balancer to direct all filebucket traffic to a single primary server, or use
       something like an out-of-band rsync task to synchronize the content on all
-      masters.
+      primary servers.
+
+      > **Note**: Enabling and using the backup option, and by extension the 
+        filebucket resource, requires appropriate planning and management to ensure 
+        that sufficient disk space is available for the file backups. Generally, you 
+        can implement this using one of the following two options:
+        - Use a `find` command and `crontab` entry to retain only the last X days 
+        of file backups. For example:
+        
+        ```
+        find /opt/puppetlabs/server/data/puppetserver/bucket -type f -mtime +45 -atime +45 -print0 | xargs -0 rm
+        ```
+        
+        - Restrict the directory to a maximum size after which the oldest items are removed.
     EOT
 
     defaultto "puppet"
@@ -187,8 +205,7 @@ Puppet::Type.newtype(:file) do
       Setting `recurselimit => 2` will manage the direct contents of the
       directory, as well as the contents of the _first_ level of subdirectories.
 
-      And so on --- 3 will manage the contents of the second level of
-      subdirectories, etc."
+      This pattern continues for each incremental value of `recurselimit`."
 
     newvalues(/^[0-9]+$/)
 
@@ -203,12 +220,29 @@ Puppet::Type.newtype(:file) do
     end
   end
 
+  newparam(:max_files) do
+    desc "In case the resource is a directory and the recursion is enabled, puppet will
+      generate a new resource for each file file found, possible leading to
+      an excessive number of resources generated without any control.
+
+      Setting `max_files` will check the number of file resources that
+      will eventually be created and will raise a resource argument error if the
+      limit will be exceeded.
+
+      Use value `0` to log a warning instead of raising an error.
+
+      Use value `-1` to disable errors and warnings due to max files."
+
+    defaultto 0
+    newvalues(/^[0-9]+$/, /^-1$/)
+  end
+
   newparam(:replace, :boolean => true, :parent => Puppet::Parameter::Boolean) do
     desc "Whether to replace a file or symlink that already exists on the local system but
       whose content doesn't match what the `source` or `content` attribute
       specifies.  Setting this to false allows file resources to initialize files
       without overwriting future changes.  Note that this only affects content;
-      Puppet will still manage ownership and permissions. Defaults to `true`."
+      Puppet will still manage ownership and permissions."
     defaultto :true
   end
 
@@ -225,9 +259,9 @@ Puppet::Type.newtype(:file) do
   newparam(:ignore) do
     desc "A parameter which omits action on files matching
       specified patterns during recursion.  Uses Ruby's builtin globbing
-      engine, so shell metacharacters are fully supported, e.g. `[a-z]*`.
+      engine, so shell metacharacters such as `[a-z]*` are fully supported.
       Matches that would descend into the directory structure are ignored,
-      e.g., `*/*`."
+      such as `*/*`."
 
     validate do |value|
       unless value.is_a?(Array) or value.is_a?(String) or value == false
@@ -294,6 +328,30 @@ Puppet::Type.newtype(:file) do
     defaultto :true
   end
 
+  newparam(:staging_location) do
+    desc "When rendering a file first render it to this location. The default
+         location is the same path as the desired location with a unique filename.
+         This parameter is useful in conjuction with validate_cmd to test a
+         file before moving the file to it's final location.
+         WARNING: File replacement is only guaranteed to be atomic if the staging
+         location is on the same filesystem as the final location."
+
+    validate do |value|
+      unless Puppet::Util.absolute_path?(value)
+        fail Puppet::Error, "File paths must be fully qualified, not '#{value}'"
+      end
+    end
+
+    munge do |value|
+      if value.start_with?('//') and ::File.basename(value) == "/"
+        # This is a UNC path pointing to a share, so don't add a trailing slash
+        ::File.expand_path(value)
+      else
+        ::File.join(::File.split(::File.expand_path(value)))
+      end
+    end
+  end
+
   newparam(:validate_cmd) do
     desc "A command for validating the file's syntax before replacing it. If
       Puppet would need to rewrite a file due to new `source` or `content`, it
@@ -303,7 +361,7 @@ Puppet::Type.newtype(:file) do
       This command must have a fully qualified path, and should contain a
       percent (`%`) token where it would expect an input file. It must exit `0`
       if the syntax is correct, and non-zero otherwise. The command will be
-      run on the target system while applying the catalog, not on the puppet master.
+      run on the target system while applying the catalog, not on the primary Puppet server.
 
       Example:
 
@@ -321,7 +379,7 @@ Puppet::Type.newtype(:file) do
 
   newparam(:validate_replacement) do
     desc "The replacement string in a `validate_cmd` that will be replaced
-      with an input file name. Defaults to: `%`"
+      with an input file name."
 
     defaultto '%'
   end
@@ -333,7 +391,8 @@ Puppet::Type.newtype(:file) do
     if !path.root?
       # Start at our parent, to avoid autorequiring ourself
       parents = path.parent.enum_for(:ascend)
-      if found = parents.find { |p| catalog.resource(:file, p.to_s) }
+      found = parents.find { |p| catalog.resource(:file, p.to_s) }
+      if found
         req << found.to_s
       end
     end
@@ -347,7 +406,8 @@ Puppet::Type.newtype(:file) do
     autorequire(type) do
       if @parameters.include?(property)
         # The user/group property automatically converts to IDs
-        next unless should = @parameters[property].shouldorig
+        should = @parameters[property].shouldorig
+        next unless should
         val = should[0]
         if val.is_a?(Integer) or val =~ /^\d+$/
           nil
@@ -358,8 +418,12 @@ Puppet::Type.newtype(:file) do
     end
   end
 
-  CREATORS = [:content, :source, :target]
-  SOURCE_ONLY_CHECKSUMS = [:none, :ctime, :mtime]
+  # mutually exclusive ways to create files
+  CREATORS = [:content, :source, :target].freeze
+
+  # This is both "checksum types that can't be used with the content property"
+  # and "checksum types that are not digest based"
+  SOURCE_ONLY_CHECKSUMS = [:none, :ctime, :mtime].freeze
 
   validate do
     creator_count = 0
@@ -385,7 +449,7 @@ Puppet::Type.newtype(:file) do
       @parameters[:content].value = @parameters[:checksum].sum(@parameters[:content].actual_content)
     end
 
-    if self[:checksum] && self[:checksum_value] && !send("#{self[:checksum]}?", self[:checksum_value])
+    if self[:checksum] && self[:checksum_value] && !valid_checksum?(self[:checksum], self[:checksum_value])
       self.fail _("Checksum value '%{value}' is not a valid checksum type %{checksum}") % { value: self[:checksum_value], checksum: self[:checksum] }
     end
 
@@ -430,7 +494,8 @@ Puppet::Type.newtype(:file) do
       fail _("Can not find filebucket for backups without a catalog")
     end
 
-    unless catalog and filebucket = catalog.resource(:filebucket, backup) or backup == "puppet"
+    filebucket = catalog.resource(:filebucket, backup) if catalog
+    if !catalog || (!filebucket && backup != 'puppet')
       fail _("Could not find filebucket %{backup} specified in backup") % { backup: backup }
     end
 
@@ -528,7 +593,7 @@ Puppet::Type.newtype(:file) do
     options = @original_parameters.merge(:path => full_path).reject { |param, value| value.nil? }
 
     # These should never be passed to our children.
-    [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].each do |param|
+    [:parent, :ensure, :recurse, :recurselimit, :max_files, :target, :alias, :source].each do |param|
       options.delete(param) if options.include?(param)
     end
 
@@ -541,7 +606,8 @@ Puppet::Type.newtype(:file) do
   def pathbuilder
     # We specifically need to call the method here, so it looks
     # up our parent in the catalog graph.
-    if parent = parent()
+    parent = parent()
+    if parent
       # We only need to behave specially when our parent is also
       # a file
       if parent.is_a?(self.class)
@@ -573,8 +639,7 @@ Puppet::Type.newtype(:file) do
     # remote system.
     mark_children_for_purging(children) if self.purge?
 
-    # REVISIT: sort_by is more efficient?
-    result = children.values.sort { |a, b| a[:path] <=> b[:path] }
+    result = children.values.sort_by { |a| a[:path] }
     remove_less_specific_files(result)
   end
 
@@ -664,14 +729,16 @@ Puppet::Type.newtype(:file) do
     total = self[:source].collect do |source|
       # For each inlined file resource, the catalog contains a hash mapping
       # source path to lists of metadata returned by a server-side search.
-      if recursive_metadata = catalog.recursive_metadata[title]
+      recursive_metadata = catalog.recursive_metadata[title]
+      if recursive_metadata
         result = recursive_metadata[source]
       else
         result = perform_recursion(source)
       end
 
       next unless result
-      return [] if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
+      top = result.find { |r| r.relative_path == "." }
+      return [] if top && top.ftype != "directory"
       result.each do |data|
         if data.relative_path == '.'
           data.source = source
@@ -703,6 +770,7 @@ Puppet::Type.newtype(:file) do
       :links => self[:links],
       :recurse => (self[:recurse] == :remote ? true : self[:recurse]),
       :recurselimit => self[:recurselimit],
+      :max_files => self[:max_files],
       :source_permissions => self[:source_permissions],
       :ignore => self[:ignore],
       :checksum_type => (self[:source] || self[:content]) ? self[:checksum] : :none,
@@ -750,7 +818,7 @@ Puppet::Type.newtype(:file) do
     # This check is done in retrieve to ensure it happens before we try to use
     # metadata in `copy_source_values`, but so it only fails the resource and not
     # catalog validation (because that would be a breaking change from Puppet 4).
-    if Puppet.features.microsoft_windows? && parameter(:source) &&
+    if Puppet::Util::Platform.windows? && parameter(:source) &&
       [:use, :use_when_creating].include?(self[:source_permissions])
       #TRANSLATORS "source_permissions => ignore" should not be translated
       err_msg = _("Copying owner/mode/group from the source file on Windows is not supported; use source_permissions => ignore.")
@@ -795,12 +863,12 @@ Puppet::Type.newtype(:file) do
     return true if self[:ensure] == :file
 
     # I.e., it's set to something like "directory"
-    return false if e = self[:ensure] and e != :present
+    return false if self[:ensure] && self[:ensure] != :present
 
     # The user doesn't really care, apparently
     if self[:ensure] == :present
-      return true unless s = stat
-      return(s.ftype == "file" ? true : false)
+      return true unless stat
+      return(stat.ftype == "file" ? true : false)
     end
 
     # If we've gotten here, then :ensure isn't set
@@ -829,13 +897,16 @@ Puppet::Type.newtype(:file) do
 
     @stat = begin
       Puppet::FileSystem.send(method, self[:path])
-    rescue Errno::ENOENT => error
+    rescue Errno::ENOENT
       nil
-    rescue Errno::ENOTDIR => error
+    rescue Errno::ENOTDIR
       nil
-    rescue Errno::EACCES => error
+    rescue Errno::EACCES
       warning _("Could not stat; permission denied")
       nil
+    rescue Errno::EINVAL
+      warning _("Could not stat; invalid pathname")
+      nil
     end
   end
 
@@ -855,7 +926,16 @@ Puppet::Type.newtype(:file) do
     mode_int = mode ? symbolic_mode_to_int(mode, Puppet::Util::DEFAULT_POSIX_MODE) : nil
 
     if write_temporary_file?
-      Puppet::Util.replace_file(self[:path], mode_int) do |file|
+      if self[:validate_cmd]
+        validate_callback = proc { |path|
+          output = Puppet::Util::Execution.execute(self[:validate_cmd].gsub(self[:validate_replacement], path), :failonfail => true, :combine => true)
+          output.split(/\n/).each { |line|
+            self.debug(line)
+          }
+        }
+      end
+
+      Puppet::Util.replace_file(self[:path], mode_int, staging_location: self[:staging_location], validate_callback: validate_callback) do |file|
         file.binmode
         devfail 'a property should have been provided if write_temporary_file? returned true' if property.nil?
         content_checksum = property.write(file)
@@ -872,13 +952,7 @@ Puppet::Type.newtype(:file) do
           # that out.
         end
 
-        fail_if_checksum_is_wrong(file.path, content_checksum) if validate_checksum?
-        if self[:validate_cmd]
-          output = Puppet::Util::Execution.execute(self[:validate_cmd].gsub(self[:validate_replacement], file.path), :failonfail => true, :combine => true)
-          output.split(/\n/).each { |line|
-            self.debug(line)
-          }
-        end
+        fail_if_checksum_is_wrong(property, file.path, content_checksum)
       end
     else
       umask = mode ? 000 : 022
@@ -988,17 +1062,38 @@ Puppet::Type.newtype(:file) do
     end
   end
 
-  # Should we validate the checksum of the file we're writing?
-  def validate_checksum?
-    self[:checksum] !~ /time/
-  end
-
   # Make sure the file we wrote out is what we think it is.
-  def fail_if_checksum_is_wrong(path, content_checksum)
-    newsum = parameter(:checksum).sum_file(path)
-    return if [:absent, nil, content_checksum].include?(newsum)
+  # @param [Puppet::Parameter] property the param or property that wrote the file, or nil
+  # @param [String] path to the file
+  # @param [String] the checksum for the local file
+  #
+  # @api private
+  #
+  def fail_if_checksum_is_wrong(property, path, content_checksum)
+    desired_checksum = desired_checksum(property, path)
 
-    self.fail _("File written to disk did not match checksum; discarding changes (%{content_checksum} vs %{newsum})") % { content_checksum: content_checksum, newsum: newsum }
+    if desired_checksum && content_checksum != desired_checksum
+      self.fail _("File written to disk did not match desired checksum; discarding changes (%{content_checksum} vs %{desired_checksum})") % { content_checksum: content_checksum, desired_checksum: desired_checksum }
+    end
+  end
+
+  # Return the desired checksum or nil
+  def desired_checksum(property, path)
+    return if SOURCE_ONLY_CHECKSUMS.include?(self[:checksum])
+
+    if self[:checksum] && self[:checksum_value]
+      "{#{self[:checksum]}}#{self[:checksum_value]}"
+    elsif property && property.name == :source
+      meta = property.metadata
+      return unless meta
+
+      # due to HttpMetadata the checksum type may fallback to mtime, so recheck
+      return if SOURCE_ONLY_CHECKSUMS.include?(meta.checksum_type)
+      meta.checksum
+    elsif property && property.name == :content
+      str = property.actual_content
+      str ? parameter(:checksum).sum(str) : nil
+    end
   end
 
   def write_temporary_file?

data/lib/puppet/type/filebucket.rb

@@ -4,8 +4,8 @@ module Puppet
   Type.newtype(:filebucket) do
     @doc = <<-EOT
       A repository for storing and retrieving file content by MD5 checksum. Can
-      be local to each agent node, or centralized on a puppet master server. All
-      puppet masters provide a filebucket service that agent nodes can access
+      be local to each agent node, or centralized on a primary Puppet server. All
+      puppet servers provide a filebucket service that agent nodes can access
       via HTTP, but you must declare a filebucket resource before any agents
       will do so.
 
@@ -25,7 +25,7 @@ module Puppet
           # /etc/puppetlabs/puppet/manifests/site.pp
           filebucket { 'main':
             path   => false,                # This is required for remote filebuckets.
-            server => 'puppet.example.com', # Optional; defaults to the configured puppet master.
+            server => 'puppet.example.com', # Optional; defaults to the configured primary server.
           }
 
           File { backup => main, }
@@ -42,19 +42,23 @@ module Puppet
     end
 
     newparam(:server) do
-      desc "The server providing the remote filebucket service. Defaults to the
-        value of the `server` setting (that is, the currently configured
-        puppet master server).
+      desc "The server providing the remote filebucket service.
 
-        This setting is _only_ consulted if the `path` attribute is set to `false`."
-      defaultto { Puppet[:server] }
+        This setting is _only_ consulted if the `path` attribute is set to `false`.
+
+        If this attribute is not specified, the first entry in the `server_list`
+        configuration setting is used, followed by the value of the `server` setting
+        if `server_list` is not set."
     end
 
     newparam(:port) do
-      desc "The port on which the remote server is listening. Defaults to the
-        value of the `masterport` setting, which is usually %s." % Puppet[:masterport]
+      desc "The port on which the remote server is listening.
+
+        This setting is _only_ consulted if the `path` attribute is set to `false`.
 
-      defaultto { Puppet[:masterport] }
+        If this attribute is not specified, the first entry in the `server_list`
+        configuration setting is used, followed by the value of the `serverport`
+        setting if `server_list` is not set."
     end
 
     newparam(:path) do

data/lib/puppet/type/group.rb

@@ -1,5 +1,4 @@
 require 'etc'
-require 'facter'
 require 'puppet/property/keyvalue'
 require 'puppet/parameter/boolean'
 
@@ -22,9 +21,9 @@ module Puppet
     feature :system_groups,
       "The provider allows you to create system groups with lower GIDs."
 
-    feature :libuser,
+    feature :manages_local_users_and_groups,
       "Allows local groups to be managed on systems that also use some other
-       remote NSS method of managing accounts."
+       remote Name Switch Service (NSS) method of managing accounts."
 
     ensurable do
       desc "Create or remove the group."
@@ -36,6 +35,9 @@ module Puppet
       newvalue(:absent) do
         provider.delete
       end
+
+      defaultto :present
+
     end
 
     newproperty(:gid) do
@@ -84,6 +86,8 @@ module Puppet
         behavior can be configured with `auth_membership`."
 
       def change_to_s(currentvalue, newvalue)
+        newvalue = actual_should(currentvalue, newvalue)
+
         currentvalue = currentvalue.join(",") if currentvalue != :absent
         newvalue = newvalue.join(",")
         super(currentvalue, newvalue)
@@ -107,7 +111,33 @@ module Puppet
 
         super(currentvalue)
       end
-      alias :should_to_s :is_to_s
+
+      def should_to_s(newvalue)
+        is_to_s(actual_should(retrieve, newvalue))
+      end
+
+      # Calculates the actual should value given the current and
+      # new values. This is only used in should_to_s and change_to_s
+      # to fix the change notification issue reported in PUP-6542.
+      def actual_should(currentvalue, newvalue)
+        currentvalue = munge_members_value(currentvalue)
+        newvalue = munge_members_value(newvalue)
+
+        if @resource[:auth_membership]
+          newvalue.uniq.sort 
+        else
+          (currentvalue + newvalue).uniq.sort
+        end
+      end
+
+      # Useful helper to handle the possible property value types that we can
+      # both pass-in and return. It munges the value into an array
+      def munge_members_value(value)
+        return [] if value == :absent
+        return value.split(',') if value.is_a?(String)
+
+        value
+      end
 
       validate do |value|
         if provider.respond_to?(:member_valid?)
@@ -138,19 +168,22 @@ module Puppet
     end
 
     newparam(:allowdupe, :boolean => true, :parent => Puppet::Parameter::Boolean) do
-      desc "Whether to allow duplicate GIDs. Defaults to `false`."
+      desc "Whether to allow duplicate GIDs."
 
       defaultto false
     end
 
     newparam(:ia_load_module, :required_features => :manages_aix_lam) do
-      desc "The name of the I&A module to use to manage this user"
+      desc "The name of the I&A module to use to manage this group.
+        This should be set to `files` if managing local groups."
     end
 
     newproperty(:attributes, :parent => Puppet::Property::KeyValue, :required_features => :manages_aix_lam) do
       desc "Specify group AIX attributes, as an array of `'key=value'` strings. This
         parameter's behavior can be configured with `attribute_membership`."
 
+      self.log_only_changed_or_new_keys = true
+
       def membership
         :attribute_membership
       end
@@ -158,10 +191,6 @@ module Puppet
       def delimiter
         " "
       end
-
-      validate do |value|
-        raise ArgumentError, _("Attributes value pairs must be separated by an =") unless value.include?("=")
-      end
     end
 
     newparam(:attribute_membership) do
@@ -184,10 +213,12 @@ module Puppet
     end
 
     newparam(:forcelocal, :boolean => true,
-             :required_features => :libuser,
+             :required_features => :manages_local_users_and_groups,
              :parent => Puppet::Parameter::Boolean) do
       desc "Forces the management of local accounts when accounts are also
-            being managed by some other NSS"
+            being managed by some other Name Switch Service (NSS). For AIX, refer to the `ia_load_module` parameter.
+            
+            This option relies on your operating system's implementation of `luser*` commands, such as `luseradd` , `lgroupadd`, and `lusermod`. The `forcelocal` option could behave unpredictably in some circumstances. If the tools it depends on are not available, it might have no effect at all."
       defaultto false
     end
 

data/lib/puppet/type/notify.rb

@@ -4,18 +4,19 @@
 
 module Puppet
   Type.newtype(:notify) do
-    @doc = "Sends an arbitrary message to the agent run-time log."
+    @doc = "Sends an arbitrary message, specified as a string, to the agent run-time log. It's important to note that the notify resource type is not idempotent. As a result, notifications are shown as a change on every Puppet run."
 
     apply_to_all
 
     newproperty(:message, :idempotent => false) do
-      desc "The message to be sent to the log."
+      desc "The message to be sent to the log. Note that the value specified must be a string."
       def sync
+        message = @sensitive ? 'Sensitive [value redacted]' : self.should
         case @resource["withpath"]
         when :true
-          send(@resource[:loglevel], self.should)
+          send(@resource[:loglevel], message)
         else
-          Puppet.send(@resource[:loglevel], self.should)
+          Puppet.send(@resource[:loglevel], message)
         end
         return
       end
@@ -32,7 +33,7 @@ module Puppet
     end
 
     newparam(:withpath) do
-      desc "Whether to show the full object path. Defaults to false."
+      desc "Whether to show the full object path."
       defaultto :false
 
       newvalues(:true, :false)