puppet 4.0.0 → 4.9.1

data/lib/puppet/type/file/content.rb

@@ -1,18 +1,14 @@
 require 'net/http'
 require 'uri'
 require 'tempfile'
-require 'date'
 
 require 'puppet/util/checksums'
-require 'puppet/network/http'
-require 'puppet/network/http/api/indirected_routes'
-require 'puppet/network/http/compression'
+require 'puppet/type/file/data_sync.rb'
 
 module Puppet
   Puppet::Type.type(:file).newproperty(:content) do
-    include Puppet::Util::Diff
     include Puppet::Util::Checksums
-    include Puppet::Network::HTTP::Compression.module
+    include Puppet::DataSync
 
     attr_reader :actual_content
 
@@ -27,20 +23,20 @@ module Puppet
       the manifest...
 
           define resolve(nameserver1, nameserver2, domain, search) {
-              $str = "search $search
-                  domain $domain
-                  nameserver $nameserver1
-                  nameserver $nameserver2
+              $str = "search ${search}
+                  domain ${domain}
+                  nameserver ${nameserver1}
+                  nameserver ${nameserver2}
                   "
 
-              file { "/etc/resolv.conf":
-                content => "$str",
+              file { '/etc/resolv.conf':
+                content => $str,
               }
           }
 
       ...but for larger files, this attribute is more useful when combined with the
-      [template](http://docs.puppetlabs.com/references/latest/function.html#template)
-      or [file](http://docs.puppetlabs.com/references/latest/function.html#file)
+      [template](https://docs.puppetlabs.com/puppet/latest/reference/function.html#template)
+      or [file](https://docs.puppetlabs.com/puppet/latest/reference/function.html#file)
       function.
     EOT
 
@@ -49,13 +45,13 @@ module Puppet
     munge do |value|
       if value == :absent
         value
-      elsif checksum?(value)
+      elsif value.is_a?(String) && checksum?(value)
         # XXX This is potentially dangerous because it means users can't write a file whose
-        # entire contents are a plain checksum
+        # entire contents are a plain checksum unless it is a Binary content.
         value
       else
-        @actual_content = value
-        resource.parameter(:checksum).sum(value)
+        @actual_content = value.is_a?(Puppet::Pops::Types::PBinaryType::Binary) ? value.binary_buffer : value
+        resource.parameter(:checksum).sum(@actual_content)
       end
     end
 
@@ -86,53 +82,29 @@ module Puppet
     # Also, fix #872: when content is used, and replace is true, the file
     # should be insync when it exists
     def insync?(is)
-      if resource.should_be_file?
-        return false if is == :absent
-      else
-        if resource[:ensure] == :present and resource[:content] and s = resource.stat
-          resource.warning "Ensure set to :present but file type is #{s.ftype} so no content will be synced"
-        end
-        return true
+      if resource[:source] && resource[:checksum_value]
+        # Asserts that nothing has changed since validate ran.
+        devfail "content property should not exist if source and checksum_value are specified"
       end
 
-      return true if ! @resource.replace?
-
-      result = super
-
-      if ! result and Puppet[:show_diff] and resource.show_diff?
-        write_temporarily do |path|
-          send @resource[:loglevel], "\n" + diff(@resource[:path], path)
-        end
-      end
-      result
+      contents_prop = resource.parameter(:source) || self
+      checksum_insync?(contents_prop, is, !resource[:content].nil?) {|_is| super(_is)}
     end
 
     def property_matches?(current, desired)
-      basic = super
-      # The inherited equality is always accepted, so use it if valid.
-      time_types = [:mtime, :ctime]
+      # If checksum_value is specified, it overrides comparing the content field.
       checksum_type = resource.parameter(:checksum).value
-      return basic if basic || !time_types.include?(checksum_type)
-      return false unless current && desired
-      begin
-        raise if !time_types.include?(sumtype(current).to_sym) || !time_types.include?(sumtype(desired).to_sym)
-        DateTime.parse(sumdata(current)) >= DateTime.parse(sumdata(desired))
-      rescue => detail
-        self.fail Puppet::Error, "Resource with checksum_type #{checksum_type} didn't contain a date in #{current} or #{desired}", detail.backtrace
+      if checksum_value = resource.parameter(:checksum_value)
+        desired = "{#{checksum_type}}#{checksum_value.value}"
       end
+
+      # The inherited equality is always accepted, so use it if valid.
+      return true if super(current, desired)
+      return date_matches?(checksum_type, current, desired)
     end
 
     def retrieve
-      return :absent unless stat = @resource.stat
-      ftype = stat.ftype
-      # Don't even try to manage the content on directories or links
-      return nil if ["directory","link"].include?(ftype)
-
-      begin
-        resource.parameter(:checksum).sum_file(resource[:path])
-      rescue => detail
-        raise Puppet::Error, "Could not read #{ftype} #{@resource.title}: #{detail}", detail.backtrace
-      end
+      retrieve_checksum(resource)
     end
 
     # Make sure we're also managing the checksum property.
@@ -146,93 +118,36 @@ module Puppet
 
     # Just write our content out to disk.
     def sync
-      return_event = @resource.stat ? :file_changed : :file_created
-
-      # We're safe not testing for the 'source' if there's no 'should'
-      # because we wouldn't have gotten this far if there weren't at least
-      # one valid value somewhere.
-      @resource.write(:content)
-
-      return_event
-    end
-
-    def write_temporarily
-      tempfile = Tempfile.new("puppet-file")
-      tempfile.open
-
-      write(tempfile)
-
-      tempfile.close
-
-      yield tempfile.path
-
-      tempfile.delete
+      contents_sync(resource.parameter(:source) || self)
     end
 
     def write(file)
       resource.parameter(:checksum).sum_stream { |sum|
-        each_chunk_from(actual_content || resource.parameter(:source)) { |chunk|
+        each_chunk_from { |chunk|
           sum << chunk
           file.print chunk
         }
       }
     end
 
+    private
+
     # the content is munged so if it's a checksum source_or_content is nil
     # unless the checksum indirectly comes from source
-    def each_chunk_from(source_or_content)
-      if source_or_content.is_a?(String)
-        yield source_or_content
-      elsif content_is_really_a_checksum? && source_or_content.nil?
+    def each_chunk_from
+      if actual_content.is_a?(String)
+        yield actual_content
+      elsif content_is_really_a_checksum? && actual_content.nil?
         yield read_file_from_filebucket
-      elsif source_or_content.nil?
+      elsif actual_content.nil?
         yield ''
-      elsif Puppet[:default_file_terminus] == :file_server
-        yield source_or_content.content
-      elsif source_or_content.local?
-        chunk_file_from_disk(source_or_content) { |chunk| yield chunk }
-      else
-        chunk_file_from_source(source_or_content) { |chunk| yield chunk }
       end
     end
 
-    private
-
     def content_is_really_a_checksum?
       checksum?(should)
     end
 
-    def chunk_file_from_disk(source_or_content)
-      File.open(source_or_content.full_path, "rb") do |src|
-        while chunk = src.read(8192)
-          yield chunk
-        end
-      end
-    end
-
-    def get_from_source(source_or_content, &block)
-      source = source_or_content.metadata.source
-      request = Puppet::Indirector::Request.new(:file_content, :find, source, nil, :environment => resource.catalog.environment_instance)
-
-      request.do_request(:fileserver) do |req|
-        connection = Puppet::Network::HttpPool.http_instance(req.server, req.port)
-        connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => "binary"}), &block)
-      end
-    end
-
-
-    def chunk_file_from_source(source_or_content)
-      get_from_source(source_or_content) do |response|
-        case response.code
-        when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
-        else
-          # Raise the http error if we didn't get a 'success' of some kind.
-          message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
-          raise Net::HTTPError.new(message, response)
-        end
-      end
-    end
-
     def read_file_from_filebucket
       raise "Could not get filebucket from file" unless dipper = resource.bucket
       sum = should.sub(/\{\w+\}/, '')

data/lib/puppet/type/file/data_sync.rb

@@ -0,0 +1,92 @@
+require 'puppet/util/checksums'
+require 'puppet/util/diff'
+require 'date'
+require 'tempfile'
+
+module Puppet
+  module DataSync
+    include Puppet::Util::Checksums
+    include Puppet::Util::Diff
+
+    def write_temporarily(param)
+      tempfile = Tempfile.new("puppet-file")
+      tempfile.open
+
+      param.write(tempfile)
+
+      tempfile.close
+
+      yield tempfile.path
+    ensure
+      tempfile.delete if tempfile
+    end
+
+    def checksum_insync?(param, is, has_contents, &block)
+      resource = param.resource
+      if resource.should_be_file?
+        return false if is == :absent
+      else
+        if resource[:ensure] == :present && has_contents && (s = resource.stat)
+          resource.warning "Ensure set to :present but file type is #{s.ftype} so no content will be synced"
+        end
+        return true
+      end
+
+      return true if ! resource.replace?
+
+      is_insync = yield(is)
+
+      if show_diff?(!is_insync)
+        if param.sensitive
+          send resource[:loglevel], "[diff redacted]"
+        else
+          write_temporarily(param) do |path|
+            send resource[:loglevel], "\n" + diff(resource[:path], path)
+          end
+        end
+      end
+      is_insync
+    end
+
+    def show_diff?(has_changes)
+      has_changes && Puppet[:show_diff] && resource.show_diff?
+    end
+
+    def date_matches?(checksum_type, current, desired)
+      time_types = [:mtime, :ctime]
+      return false if !time_types.include?(checksum_type)
+      return false unless current && desired
+
+      begin
+        if checksum?(current) || checksum?(desired)
+          raise if !time_types.include?(sumtype(current).to_sym) || !time_types.include?(sumtype(desired).to_sym)
+          current = sumdata(current)
+          desired = sumdata(desired)
+        end
+        DateTime.parse(current) >= DateTime.parse(desired)
+      rescue => detail
+        self.fail Puppet::Error, "Resource with checksum_type #{checksum_type} didn't contain a date in #{current} or #{desired}", detail.backtrace
+      end
+    end
+
+    def retrieve_checksum(resource)
+      return :absent unless stat = resource.stat
+      ftype = stat.ftype
+      # Don't even try to manage the content on directories or links
+      return nil if ["directory","link"].include?(ftype)
+
+      begin
+        resource.parameter(:checksum).sum_file(resource[:path])
+      rescue => detail
+        raise Puppet::Error, "Could not read #{ftype} #{resource.title}: #{detail}", detail.backtrace
+      end
+    end
+
+    def contents_sync(param)
+      return_event = param.resource.stat ? :file_changed : :file_created
+      resource.write(param)
+      return_event
+    end
+
+  end
+end

data/lib/puppet/type/file/ensure.rb

@@ -9,22 +9,21 @@ module Puppet
       Whether the file should exist, and if so what kind of file it should be.
       Possible values are `present`, `absent`, `file`, `directory`, and `link`.
 
-      * `present` will accept any form of file existence, and will create a
+      * `present` accepts any form of file existence, and creates a
         normal file if the file is missing. (The file will have no content
         unless the `content` or `source` attribute is used.)
-      * `absent` will make sure the file doesn't exist, deleting it
-        if necessary.
-      * `file` will make sure it's a normal file, and enables use of the
-        `content` or `source` attribute.
-      * `directory` will make sure it's a directory, and enables use of the
-        `source`, `recurse`, `recurselimit`, `ignore`, and `purge` attributes.
-      * `link` will make sure the file is a symlink, and **requires** that you
-        also set the `target` attribute. Symlinks are supported on all Posix
+      * `absent` ensures the file doesn't exist, and deletes it if necessary.
+      * `file` ensures it's a normal file, and enables use of the `content` or
+        `source` attribute.
+      * `directory` ensures it's a directory, and enables use of the `source`,
+        `recurse`, `recurselimit`, `ignore`, and `purge` attributes.
+      * `link` ensures the file is a symlink, and **requires** that you also
+        set the `target` attribute. Symlinks are supported on all Posix
         systems and on Windows Vista / 2008 and higher. On Windows, managing
-        symlinks requires puppet agent's user account to have the "Create
+        symlinks requires Puppet agent's user account to have the "Create
         Symbolic Links" privilege; this can be configured in the "User Rights
-        Assignment" section in the Windows policy editor. By default, puppet
-        agent runs as the Administrator account, which does have this privilege.
+        Assignment" section in the Windows policy editor. By default, Puppet
+        agent runs as the Administrator account, which has this privilege.
 
       Puppet avoids destroying directories unless the `force` attribute is set
       to `true`. This means that if a file is currently a directory, setting
@@ -37,17 +36,19 @@ module Puppet
 
           # Equivalent resources:
 
-          file { "/etc/inetd.conf":
-            ensure => "/etc/inet/inetd.conf",
+          file { '/etc/inetd.conf':
+            ensure => '/etc/inet/inetd.conf',
           }
 
-          file { "/etc/inetd.conf":
+          file { '/etc/inetd.conf':
             ensure => link,
-            target => "/etc/inet/inetd.conf",
+            target => '/etc/inet/inetd.conf',
           }
 
       However, we recommend using `link` and `target` explicitly, since this
-      behavior can be harder to read.
+      behavior can be harder to read and is
+      [deprecated](https://docs.puppetlabs.com/puppet/4.3/reference/deprecated_language.html)
+      as of Puppet 4.3.0.
     EOT
 
     # Most 'ensure' properties have a default, but with files we, um, don't.
@@ -63,8 +64,10 @@ module Puppet
       # Make sure we're not managing the content some other way
       if property = @resource.property(:content)
         property.sync
+      elsif property = @resource.property(:checksum_value)
+        property.sync
       else
-        @resource.write(:ensure)
+        @resource.write
         @resource.should(:mode)
       end
     end
@@ -115,7 +118,7 @@ module Puppet
     end
 
     def change_to_s(currentvalue, newvalue)
-      return super unless newvalue.to_s == "file"
+      return super unless [:file, :present].include?(newvalue)
 
       return super unless property = @resource.property(:content)
 

data/lib/puppet/type/file/selcontext.rb

@@ -17,7 +17,7 @@
 # command.  This allows the user to specify a subset of the three
 # values while leaving the others alone.
 #
-# See http://www.nsa.gov/selinux/ for complete docs on SELinux.
+# See https://www.nsa.gov/selinux/ for complete docs on SELinux.
 
 
 module Puppet

data/lib/puppet/type/file/source.rb

@@ -1,5 +1,11 @@
 require 'puppet/file_serving/content'
 require 'puppet/file_serving/metadata'
+require 'puppet/file_serving/terminus_helper'
+
+require 'puppet/util/http_proxy'
+require 'puppet/network/http'
+require 'puppet/network/http/api/indirected_routes'
+require 'puppet/network/http/compression'
 
 module Puppet
   # Copy files from a local or remote source.  This state *only* does any work
@@ -8,41 +14,49 @@ module Puppet
   # this state, during retrieval, modifies the appropriate other states
   # so that things get taken care of appropriately.
   Puppet::Type.type(:file).newparam(:source) do
-    include Puppet::Util::Diff
+    include Puppet::Network::HTTP::Compression.module
 
     attr_accessor :source, :local
     desc <<-'EOT'
-      A source file, which will be copied into place on the local system.
-      Values can be URIs pointing to remote files, or fully qualified paths to
-      files available on the local system (including files on NFS shares or
-      Windows mapped drives). This attribute is mutually exclusive with
-      `content` and `target`.
+      A source file, which will be copied into place on the local system. This
+      attribute is mutually exclusive with `content` and `target`. Allowed
+      values are:
+
+      * `puppet:` URIs, which point to files in modules or Puppet file server
+      mount points.
+      * Fully qualified paths to locally available files (including files on NFS
+      shares or Windows mapped drives).
+      * `file:` URIs, which behave the same as local file paths.
+      * `http:` URIs, which point to files served by common web servers
 
-      The available URI schemes are *puppet* and *file*. *Puppet*
-      URIs will retrieve files from Puppet's built-in file server, and are
-      usually formatted as:
+      The normal form of a `puppet:` URI is:
 
-      `puppet:///modules/name_of_module/filename`
+      `puppet:///modules/<MODULE NAME>/<FILE PATH>`
 
-      This will fetch a file from a module on the puppet master (or from a
-      local module when using puppet apply). Given a `modulepath` of
+      This will fetch a file from a module on the Puppet master (or from a
+      local module when using Puppet apply). Given a `modulepath` of
       `/etc/puppetlabs/code/modules`, the example above would resolve to
-      `/etc/puppetlabs/code/modules/name_of_module/files/filename`.
+      `/etc/puppetlabs/code/modules/<MODULE NAME>/files/<FILE PATH>`.
 
       Unlike `content`, the `source` attribute can be used to recursively copy
       directories if the `recurse` attribute is set to `true` or `remote`. If
       a source directory contains symlinks, use the `links` attribute to
       specify whether to recreate links or follow them.
 
+      *HTTP* URIs cannot be used to recursively synchronize whole directory
+      trees. It is also not possible to use `source_permissions` values other
+      than `ignore`. That's because HTTP servers do not transfer any metadata
+      that translates to ownership or permission details.
+
       Multiple `source` values can be specified as an array, and Puppet will
       use the first source that exists. This can be used to serve different
       files to different system types:
 
-          file { "/etc/nfs.conf":
+          file { '/etc/nfs.conf':
             source => [
-              "puppet:///modules/nfs/conf.$host",
-              "puppet:///modules/nfs/conf.$operatingsystem",
-              "puppet:///modules/nfs/conf"
+              "puppet:///modules/nfs/conf.${host}",
+              "puppet:///modules/nfs/conf.${operatingsystem}",
+              'puppet:///modules/nfs/conf'
             ]
           }
 
@@ -63,7 +77,9 @@ module Puppet
 
         self.fail "Cannot use relative URLs '#{source}'" unless uri.absolute?
         self.fail "Cannot use opaque URLs '#{source}'" unless uri.hierarchical?
-        self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving" unless %w{file puppet}.include?(uri.scheme)
+        unless %w{file puppet http https}.include?(uri.scheme)
+          self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving"
+        end
       end
     end
 
@@ -72,7 +88,7 @@ module Puppet
     munge do |sources|
       sources = [sources] unless sources.is_a?(Array)
       sources.map do |source|
-        source = source.sub(/[#{SEPARATOR_REGEX}]+$/, '')
+        source = self.class.normalize(source)
 
         if Puppet::Util.absolute_path?(source)
           URI.unescape(Puppet::Util.path_to_uri(source).to_s)
@@ -82,6 +98,10 @@ module Puppet
       end
     end
 
+    def self.normalize(source)
+      source.sub(/[#{SEPARATOR_REGEX}]+$/, '')
+    end
+
     def change_to_s(currentvalue, newvalue)
       # newvalue = "{md5}#{@metadata.checksum}"
       if resource.property(:ensure).retrieve == :absent
@@ -121,21 +141,6 @@ module Puppet
         next if metadata_method == :owner and !Puppet.features.root?
         next if metadata_method == :group and !Puppet.features.root?
 
-        if Puppet.features.microsoft_windows?
-          # Warn on Windows if source permissions are being used and the file resource
-          # does not have mode owner and group all set (which would take precedence).
-          if [:use, :use_when_creating].include?(resource[:source_permissions]) &&
-            (resource[:owner] == nil || resource[:group] == nil || resource[:mode] == nil)
-
-            err_msg = "Copying %s from the source" <<
-                      " file on Windows is not supported;" <<
-                      " use source_permissions => ignore."
-            self.fail Puppet::Error, err_msg % 'owner/mode/group'
-          end
-          # But never try to copy remote owner/group on Windows
-          next if [:owner, :group].include?(metadata_method) && !local?
-        end
-
         case resource[:source_permissions]
         when :ignore
           next
@@ -165,6 +170,11 @@ module Puppet
     # problems in our query.
     def metadata
       return @metadata if @metadata
+
+      if @metadata = resource.catalog.metadata[resource.title]
+        return @metadata
+      end
+
       return nil unless value
       value.each do |source|
         begin
@@ -212,6 +222,15 @@ module Puppet
       @uri ||= URI.parse(URI.escape(metadata.source))
     end
 
+    def write(file)
+      resource.parameter(:checksum).sum_stream { |sum|
+        each_chunk_from { |chunk|
+          sum << chunk
+          file.print chunk
+        }
+      }
+    end
+
     private
 
     def scheme
@@ -225,10 +244,78 @@ module Puppet
     def copy_source_value(metadata_method)
       param_name = (metadata_method == :checksum) ? :content : metadata_method
       if resource[param_name].nil? or resource[param_name] == :absent
+        if Puppet.features.microsoft_windows? && [:owner, :group, :mode].include?(metadata_method)
+          devfail "Should not have tried to use source owner/mode/group on Windows"
+        end
+
         value = metadata.send(metadata_method)
         # Force the mode value in file resources to be a string containing octal.
         value = value.to_s(8) if param_name == :mode && value.is_a?(Numeric)
         resource[param_name] = value
+
+        if (metadata_method == :checksum)
+          # If copying checksum, also copy checksum_type
+          resource[:checksum] = metadata.checksum_type
+        end
+      end
+    end
+
+    def each_chunk_from
+      if Puppet[:default_file_terminus] == :file_server
+        yield content
+      elsif local?
+        chunk_file_from_disk { |chunk| yield chunk }
+      else
+        chunk_file_from_source { |chunk| yield chunk }
+      end
+    end
+
+    def chunk_file_from_disk
+      File.open(full_path, "rb") do |src|
+        while chunk = src.read(8192)
+          yield chunk
+        end
+      end
+    end
+
+    def get_from_puppet_source(source_uri, content_uri, &block)
+      options = { :environment => resource.catalog.environment_instance }
+      if content_uri
+        options[:code_id] = resource.catalog.code_id
+        request = Puppet::Indirector::Request.new(:static_file_content, :find, content_uri, nil, options)
+      else
+        request = Puppet::Indirector::Request.new(:file_content, :find, source_uri, nil, options)
+      end
+
+      request.do_request(:fileserver) do |req|
+        connection = Puppet::Network::HttpPool.http_instance(req.server, req.port)
+        connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => "binary"}), &block)
+      end
+    end
+
+    def get_from_http_source(source_uri, &block)
+      Puppet::Util::HttpProxy.request_with_redirects(URI(source_uri), :get, &block)
+    end
+
+    def get_from_source(&block)
+      source_uri = metadata.source
+      if source_uri =~ /^https?:/
+        get_from_http_source(source_uri, &block)
+      else
+        get_from_puppet_source(source_uri, metadata.content_uri, &block)
+      end
+    end
+
+
+    def chunk_file_from_source
+      get_from_source do |response|
+        case response.code
+        when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
+        else
+          # Raise the http error if we didn't get a 'success' of some kind.
+          message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
+          raise Net::HTTPError.new(message, response)
+        end
       end
     end
   end

data/lib/puppet/type/file/target.rb

@@ -7,9 +7,9 @@ module Puppet
       Symlink targets can be relative, as well as absolute:
 
           # (Useful on Solaris)
-          file { \"/etc/inetd.conf\":
+          file { '/etc/inetd.conf':
             ensure => link,
-            target => \"inet/inetd.conf\",
+            target => 'inet/inetd.conf',
           }
 
       Directories of symlinks can be served recursively by instead using the