puppet 4.0.0 → 4.5.0

data/lib/puppet/type/file/source.rb

@@ -1,5 +1,11 @@
 require 'puppet/file_serving/content'
 require 'puppet/file_serving/metadata'
+require 'puppet/file_serving/terminus_helper'
+
+require 'puppet/util/http_proxy'
+require 'puppet/network/http'
+require 'puppet/network/http/api/indirected_routes'
+require 'puppet/network/http/compression'
 
 module Puppet
   # Copy files from a local or remote source.  This state *only* does any work
@@ -8,41 +14,49 @@ module Puppet
   # this state, during retrieval, modifies the appropriate other states
   # so that things get taken care of appropriately.
   Puppet::Type.type(:file).newparam(:source) do
-    include Puppet::Util::Diff
+    include Puppet::Network::HTTP::Compression.module
 
     attr_accessor :source, :local
     desc <<-'EOT'
-      A source file, which will be copied into place on the local system.
-      Values can be URIs pointing to remote files, or fully qualified paths to
-      files available on the local system (including files on NFS shares or
-      Windows mapped drives). This attribute is mutually exclusive with
-      `content` and `target`.
+      A source file, which will be copied into place on the local system. This
+      attribute is mutually exclusive with `content` and `target`. Allowed
+      values are:
+
+      * `puppet:` URIs, which point to files in modules or Puppet file server
+      mount points.
+      * Fully qualified paths to locally available files (including files on NFS
+      shares or Windows mapped drives).
+      * `file:` URIs, which behave the same as local file paths.
+      * `http:` URIs, which point to files served by common web servers
 
-      The available URI schemes are *puppet* and *file*. *Puppet*
-      URIs will retrieve files from Puppet's built-in file server, and are
-      usually formatted as:
+      The normal form of a `puppet:` URI is:
 
-      `puppet:///modules/name_of_module/filename`
+      `puppet:///modules/<MODULE NAME>/<FILE PATH>`
 
-      This will fetch a file from a module on the puppet master (or from a
-      local module when using puppet apply). Given a `modulepath` of
+      This will fetch a file from a module on the Puppet master (or from a
+      local module when using Puppet apply). Given a `modulepath` of
       `/etc/puppetlabs/code/modules`, the example above would resolve to
-      `/etc/puppetlabs/code/modules/name_of_module/files/filename`.
+      `/etc/puppetlabs/code/modules/<MODULE NAME>/files/<FILE PATH>`.
 
       Unlike `content`, the `source` attribute can be used to recursively copy
       directories if the `recurse` attribute is set to `true` or `remote`. If
       a source directory contains symlinks, use the `links` attribute to
       specify whether to recreate links or follow them.
 
+      *HTTP* URIs cannot be used to recursively synchronize whole directory
+      trees. It is also not possible to use `source_permissions` values other
+      than `ignore`. That's because HTTP servers do not transfer any metadata
+      that translates to ownership or permission details.
+
       Multiple `source` values can be specified as an array, and Puppet will
       use the first source that exists. This can be used to serve different
       files to different system types:
 
-          file { "/etc/nfs.conf":
+          file { '/etc/nfs.conf':
             source => [
-              "puppet:///modules/nfs/conf.$host",
-              "puppet:///modules/nfs/conf.$operatingsystem",
-              "puppet:///modules/nfs/conf"
+              "puppet:///modules/nfs/conf.${host}",
+              "puppet:///modules/nfs/conf.${operatingsystem}",
+              'puppet:///modules/nfs/conf'
             ]
           }
 
@@ -63,7 +77,9 @@ module Puppet
 
         self.fail "Cannot use relative URLs '#{source}'" unless uri.absolute?
         self.fail "Cannot use opaque URLs '#{source}'" unless uri.hierarchical?
-        self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving" unless %w{file puppet}.include?(uri.scheme)
+        unless %w{file puppet http https}.include?(uri.scheme)
+          self.fail "Cannot use URLs of type '#{uri.scheme}' as source for fileserving"
+        end
       end
     end
 
@@ -72,7 +88,7 @@ module Puppet
     munge do |sources|
       sources = [sources] unless sources.is_a?(Array)
       sources.map do |source|
-        source = source.sub(/[#{SEPARATOR_REGEX}]+$/, '')
+        source = self.class.normalize(source)
 
         if Puppet::Util.absolute_path?(source)
           URI.unescape(Puppet::Util.path_to_uri(source).to_s)
@@ -82,6 +98,10 @@ module Puppet
       end
     end
 
+    def self.normalize(source)
+      source.sub(/[#{SEPARATOR_REGEX}]+$/, '')
+    end
+
     def change_to_s(currentvalue, newvalue)
       # newvalue = "{md5}#{@metadata.checksum}"
       if resource.property(:ensure).retrieve == :absent
@@ -121,21 +141,6 @@ module Puppet
         next if metadata_method == :owner and !Puppet.features.root?
         next if metadata_method == :group and !Puppet.features.root?
 
-        if Puppet.features.microsoft_windows?
-          # Warn on Windows if source permissions are being used and the file resource
-          # does not have mode owner and group all set (which would take precedence).
-          if [:use, :use_when_creating].include?(resource[:source_permissions]) &&
-            (resource[:owner] == nil || resource[:group] == nil || resource[:mode] == nil)
-
-            err_msg = "Copying %s from the source" <<
-                      " file on Windows is not supported;" <<
-                      " use source_permissions => ignore."
-            self.fail Puppet::Error, err_msg % 'owner/mode/group'
-          end
-          # But never try to copy remote owner/group on Windows
-          next if [:owner, :group].include?(metadata_method) && !local?
-        end
-
         case resource[:source_permissions]
         when :ignore
           next
@@ -165,6 +170,11 @@ module Puppet
     # problems in our query.
     def metadata
       return @metadata if @metadata
+
+      if @metadata = resource.catalog.metadata[resource.title]
+        return @metadata
+      end
+
       return nil unless value
       value.each do |source|
         begin
@@ -212,6 +222,15 @@ module Puppet
       @uri ||= URI.parse(URI.escape(metadata.source))
     end
 
+    def write(file)
+      resource.parameter(:checksum).sum_stream { |sum|
+        each_chunk_from { |chunk|
+          sum << chunk
+          file.print chunk
+        }
+      }
+    end
+
     private
 
     def scheme
@@ -225,10 +244,78 @@ module Puppet
     def copy_source_value(metadata_method)
       param_name = (metadata_method == :checksum) ? :content : metadata_method
       if resource[param_name].nil? or resource[param_name] == :absent
+        if Puppet.features.microsoft_windows? && [:owner, :group, :mode].include?(metadata_method)
+          devfail "Should not have tried to use source owner/mode/group on Windows"
+        end
+
         value = metadata.send(metadata_method)
         # Force the mode value in file resources to be a string containing octal.
         value = value.to_s(8) if param_name == :mode && value.is_a?(Numeric)
         resource[param_name] = value
+
+        if (metadata_method == :checksum)
+          # If copying checksum, also copy checksum_type
+          resource[:checksum] = metadata.checksum_type
+        end
+      end
+    end
+
+    def each_chunk_from
+      if Puppet[:default_file_terminus] == :file_server
+        yield content
+      elsif local?
+        chunk_file_from_disk { |chunk| yield chunk }
+      else
+        chunk_file_from_source { |chunk| yield chunk }
+      end
+    end
+
+    def chunk_file_from_disk
+      File.open(full_path, "rb") do |src|
+        while chunk = src.read(8192)
+          yield chunk
+        end
+      end
+    end
+
+    def get_from_puppet_source(source_uri, content_uri, &block)
+      options = { :environment => resource.catalog.environment_instance }
+      if content_uri
+        options[:code_id] = resource.catalog.code_id
+        request = Puppet::Indirector::Request.new(:static_file_content, :find, content_uri, nil, options)
+      else
+        request = Puppet::Indirector::Request.new(:file_content, :find, source_uri, nil, options)
+      end
+
+      request.do_request(:fileserver) do |req|
+        connection = Puppet::Network::HttpPool.http_instance(req.server, req.port)
+        connection.request_get(Puppet::Network::HTTP::API::IndirectedRoutes.request_to_uri(req), add_accept_encoding({"Accept" => "binary"}), &block)
+      end
+    end
+
+    def get_from_http_source(source_uri, &block)
+      Puppet::Util::HttpProxy.request_with_redirects(URI(source_uri), :get, &block)
+    end
+
+    def get_from_source(&block)
+      source_uri = metadata.source
+      if source_uri =~ /^https?:/
+        get_from_http_source(source_uri, &block)
+      else
+        get_from_puppet_source(source_uri, metadata.content_uri, &block)
+      end
+    end
+
+
+    def chunk_file_from_source
+      get_from_source do |response|
+        case response.code
+        when /^2/;  uncompress(response) { |uncompressor| response.read_body { |chunk| yield uncompressor.uncompress(chunk) } }
+        else
+          # Raise the http error if we didn't get a 'success' of some kind.
+          message = "Error #{response.code} on SERVER: #{(response.body||'').empty? ? response.message : uncompress_body(response)}"
+          raise Net::HTTPError.new(message, response)
+        end
       end
     end
   end

data/lib/puppet/type/file/target.rb

@@ -7,9 +7,9 @@ module Puppet
       Symlink targets can be relative, as well as absolute:
 
           # (Useful on Solaris)
-          file { \"/etc/inetd.conf\":
+          file { '/etc/inetd.conf':
             ensure => link,
-            target => \"inet/inetd.conf\",
+            target => 'inet/inetd.conf',
           }
 
       Directories of symlinks can be served recursively by instead using the

data/lib/puppet/type/file.rb

@@ -236,11 +236,9 @@ Puppet::Type.newtype(:file) do
 
   newparam(:links) do
     desc "How to handle links during file actions.  During file copying,
-      `follow` will copy the target file instead of the link, `manage`
-      will copy the link itself, and `ignore` will just pass it by.
-      When not copying, `manage` and `ignore` behave equivalently
-      (because you cannot really ignore links entirely during local
-      recursion), and `follow` will manage the file to which the link points."
+      `follow` will copy the target file instead of the link and `manage`
+      will copy the link itself. When not copying, `manage` will manage
+      the link, and `follow` will manage the file to which the link points."
 
     newvalues(:follow, :manage)
 
@@ -367,9 +365,10 @@ Puppet::Type.newtype(:file) do
       creator_count += 1 if self.should(param)
     end
     creator_count += 1 if @parameters.include?(:source)
+
     self.fail "You cannot specify more than one of #{CREATORS.collect { |p| p.to_s}.join(", ")}" if creator_count > 1
 
-    self.fail "You cannot specify a remote recursion without a source" if !self[:source] and self[:recurse] == :remote
+    self.fail "You cannot specify a remote recursion without a source" if !self[:source] && self[:recurse] == :remote
 
     self.fail "You cannot specify source when using checksum 'none'" if self[:checksum] == :none && !self[:source].nil?
 
@@ -377,13 +376,19 @@ Puppet::Type.newtype(:file) do
       self.fail "You cannot specify content when using checksum '#{checksum_type}'" if self[:checksum] == checksum_type && !self[:content].nil?
     end
 
-    self.warning "Possible error: recurselimit is set but not recurse, no recursion will happen" if !self[:recurse] and self[:recurselimit]
+    self.warning "Possible error: recurselimit is set but not recurse, no recursion will happen" if !self[:recurse] && self[:recurselimit]
 
     if @parameters[:content] && @parameters[:content].actual_content
       # Now that we know the checksum, update content (in case it was created before checksum was known).
       @parameters[:content].value = @parameters[:checksum].sum(@parameters[:content].actual_content)
     end
 
+    if self[:checksum] && self[:checksum_value] && !send("#{self[:checksum]}?", self[:checksum_value])
+      self.fail "Checksum value '#{self[:checksum_value]}' is not a valid checksum type #{self[:checksum]}"
+    end
+
+    self.warning "Checksum value is ignored unless content or source are specified" if self[:checksum_value] && !self[:content] && !self[:source]
+
     provider.validate if provider.respond_to?(:validate)
   end
 
@@ -398,7 +403,7 @@ Puppet::Type.newtype(:file) do
 
   # Determine the user to write files as.
   def asuser
-    if self.should(:owner) and ! self.should(:owner).is_a?(Symbol)
+    if self.should(:owner) && ! self.should(:owner).is_a?(Symbol)
       writeable = Puppet::Util::SUIDManager.asuser(self.should(:owner)) {
         FileTest.writable?(::File.dirname(self[:path]))
       }
@@ -571,22 +576,29 @@ Puppet::Type.newtype(:file) do
     remove_less_specific_files(result)
   end
 
+  def remove_less_specific_files(files)
+    existing_files = catalog.vertices.select { |r| r.is_a?(self.class) }
+    self.class.remove_less_specific_files(files, self[:path], existing_files) do |file|
+      file[:path]
+    end
+  end
+
   # This is to fix bug #2296, where two files recurse over the same
   # set of files.  It's a rare case, and when it does happen you're
   # not likely to have many actual conflicts, which is good, because
   # this is a pretty inefficient implementation.
-  def remove_less_specific_files(files)
+  def self.remove_less_specific_files(files, parent_path, existing_files, &block)
     # REVISIT: is this Windows safe?  AltSeparator?
-    mypath = self[:path].split(::File::Separator)
-    other_paths = catalog.vertices.
-      select  { |r| r.is_a?(self.class) and r[:path] != self[:path] }.
-      collect { |r| r[:path].split(::File::Separator) }.
+    mypath = parent_path.split(::File::Separator)
+    other_paths = existing_files.
+      select { |r| (yield r) != parent_path}.
+      collect { |r| (yield r).split(::File::Separator) }.
       select  { |p| p[0,mypath.length]  == mypath }
 
     return files if other_paths.empty?
 
     files.reject { |file|
-      path = file[:path].split(::File::Separator)
+      path = (yield file).split(::File::Separator)
       other_paths.any? { |p| path[0,p.length] == p }
       }
   end
@@ -629,12 +641,43 @@ Puppet::Type.newtype(:file) do
 
   # Recurse against our remote file.
   def recurse_remote(children)
+    recurse_remote_metadata.each do |meta|
+      if meta.relative_path == "."
+        self[:checksum] = meta.checksum_type
+        parameter(:source).metadata = meta
+        next
+      end
+      children[meta.relative_path] ||= newchild(meta.relative_path)
+      children[meta.relative_path][:source] = meta.source
+      children[meta.relative_path][:checksum] = meta.checksum_type
+      children[meta.relative_path].parameter(:source).metadata = meta
+    end
+
+    children
+  end
+
+  def recurse_remote_metadata
     sourceselect = self[:sourceselect]
 
     total = self[:source].collect do |source|
-      next unless result = perform_recursion(source)
-      return if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
-      result.each { |data| data.source = "#{source}/#{data.relative_path}" }
+      # For each inlined file resource, the catalog contains a hash mapping
+      # source path to lists of metadata returned by a server-side search.
+      if recursive_metadata = catalog.recursive_metadata[title]
+        result = recursive_metadata[source]
+      else
+        result = perform_recursion(source)
+      end
+
+      next unless result
+      return [] if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
+      result.each do |data|
+        if data.relative_path == '.'
+          data.source = source
+        else
+          # REMIND: appending file paths to URL may not be safe, e.g. foo+bar
+          data.source = "#{source}/#{data.relative_path}"
+        end
+      end
       break result if result and ! result.empty? and sourceselect == :first
       result
     end.flatten.compact
@@ -644,22 +687,12 @@ Puppet::Type.newtype(:file) do
       found = []
       total.reject! do |data|
         result = found.include?(data.relative_path)
-        found << data.relative_path unless found.include?(data.relative_path)
+        found << data.relative_path unless result
         result
       end
     end
 
-    total.each do |meta|
-      if meta.relative_path == "."
-        parameter(:source).metadata = meta
-        next
-      end
-      children[meta.relative_path] ||= newchild(meta.relative_path)
-      children[meta.relative_path][:source] = meta.source
-      children[meta.relative_path].parameter(:source).metadata = meta
-    end
-
-    children
+    total
   end
 
   def perform_recursion(path)
@@ -668,6 +701,7 @@ Puppet::Type.newtype(:file) do
       :links => self[:links],
       :recurse => (self[:recurse] == :remote ? true : self[:recurse]),
       :recurselimit => self[:recurselimit],
+      :source_permissions => self[:source_permissions],
       :ignore => self[:ignore],
       :checksum_type => (self[:source] || self[:content]) ? self[:checksum] : :none,
       :environment => catalog.environment_instance
@@ -706,7 +740,27 @@ Puppet::Type.newtype(:file) do
   end
 
   def retrieve
-    if source = parameter(:source)
+    # This check is done in retrieve to ensure it happens before we try to use
+    # metadata in `copy_source_values`, but so it only fails the resource and not
+    # catalog validation (because that would be a breaking change from Puppet 4).
+    if Puppet.features.microsoft_windows? && parameter(:source) &&
+      [:use, :use_when_creating].include?(self[:source_permissions])
+      err_msg = "Copying owner/mode/group from the source file on Windows" <<
+      " is not supported; use source_permissions => ignore."
+
+      if self[:owner] == nil || self[:group] == nil || self[:mode] == nil
+        # Fail on Windows if source permissions are being used and the file resource
+        # does not have mode owner, group, and mode all set (which would take precedence).
+        self.fail err_msg
+      else
+        # Warn if use source permissions is specified on Windows
+        self.warning err_msg
+      end
+    end
+
+    # `checksum_value` implies explicit management of all metadata, so skip metadata
+    # retrieval. Otherwise, if source is set, retrieve metadata for source.
+    if (source = parameter(:source)) && property(:checksum_value).nil?
       source.copy_source_values
     end
     super
@@ -785,9 +839,10 @@ Puppet::Type.newtype(:file) do
     resource
   end
 
-  # Write out the file.  Requires the property name for logging.
-  # Write will be done by the content property, along with checksum computation
-  def write(property)
+  # Write out the file. To write content, pass the property as an argument
+  # to delegate writing to; must implement a #write method that takes the file
+  # as an argument.
+  def write(property = nil)
     remove_existing(:file)
 
     mode = self.should(:mode) # might be nil
@@ -796,8 +851,21 @@ Puppet::Type.newtype(:file) do
     if write_temporary_file?
       Puppet::Util.replace_file(self[:path], mode_int) do |file|
         file.binmode
-        content_checksum = write_content(file)
+        devfail 'a property should have been provided if write_temporary_file? returned true' if property.nil?
+        content_checksum = property.write(file)
         file.flush
+        begin
+          file.fsync
+        rescue NotImplementedError
+          # fsync may not be implemented by Ruby on all platforms, but
+          # there is absolutely no recovery path if we detect that.  So, we just
+          # ignore the return code.
+          #
+          # However, don't be fooled: that is accepting that we are running in
+          # an unsafe fashion.  If you are porting to a new platform don't stub
+          # that out.
+        end
+
         fail_if_checksum_is_wrong(file.path, content_checksum) if validate_checksum?
         if self[:validate_cmd]
           output = Puppet::Util::Execution.execute(self[:validate_cmd].gsub(self[:validate_replacement], file.path), :failonfail => true, :combine => true)
@@ -808,7 +876,7 @@ Puppet::Type.newtype(:file) do
       end
     else
       umask = mode ? 000 : 022
-      Puppet::Util.withumask(umask) { ::File.open(self[:path], 'wb', mode_int ) { |f| write_content(f) } }
+      Puppet::Util.withumask(umask) { ::File.open(self[:path], 'wb', mode_int ) { |f| property.write(f) if property } }
     end
 
     # make sure all of the modes are actually correct
@@ -888,16 +956,9 @@ Puppet::Type.newtype(:file) do
     self.fail "File written to disk did not match checksum; discarding changes (#{content_checksum} vs #{newsum})"
   end
 
-  # write the current content. Note that if there is no content property
-  # simply opening the file with 'w' as done in write is enough to truncate
-  # or write an empty length file.
-  def write_content(file)
-    (content = property(:content)) && content.write(file)
-  end
-
   def write_temporary_file?
-    # unfortunately we don't know the source file size before fetching it
-    # so let's assume the file won't be empty
+    # Unfortunately we don't know the source file size before fetching it so
+    # let's assume the file won't be empty. Why isn't it part of the metadata?
     (c = property(:content) and c.length) || @parameters[:source]
   end
 
@@ -913,6 +974,7 @@ Puppet::Type.newtype(:file) do
       thing.sync unless thing.safe_insync?(currentvalue)
     end
   end
+
 end
 
 # We put all of the properties in separate files, because there are so many
@@ -921,6 +983,7 @@ end
 require 'puppet/type/file/checksum'
 require 'puppet/type/file/content'     # can create the file
 require 'puppet/type/file/source'      # can create the file
+require 'puppet/type/file/checksum_value' # can create the file, in place of content
 require 'puppet/type/file/target'      # creates a different type of file
 require 'puppet/type/file/ensure'      # can create the file
 require 'puppet/type/file/owner'

data/lib/puppet/type/filebucket.rb

@@ -21,7 +21,7 @@ module Puppet
         puppet master's filebucket with the _desired_ content for each file,
         then instructs the agent to retrieve the content for a specific
         checksum. For more details,
-        [see the `static_compiler` section in the catalog indirection docs](http://docs.puppetlabs.com/references/latest/indirection.html#catalog).
+        [see the `static_compiler` section in the catalog indirection docs](https://docs.puppetlabs.com/puppet/latest/reference/indirection.html#catalog).
 
       To use a central filebucket for backups, you will usually want to declare
       a filebucket resource and a resource default for the `backup` attribute

data/lib/puppet/type/group.rb

@@ -80,7 +80,9 @@ module Puppet
 
     newproperty(:members, :array_matching => :all, :required_features => :manages_members) do
       desc "The members of the group. For directory services where group
-      membership is stored in the group objects, not the users."
+      membership is stored in the group objects, not the users. Use
+      with auth_membership to determine whether the specified members
+      are inclusive or the minimum."
 
       def change_to_s(currentvalue, newvalue)
         currentvalue = currentvalue.join(",") if currentvalue != :absent
@@ -99,16 +101,26 @@ module Puppet
       def is_to_s(currentvalue)
         if provider.respond_to?(:members_to_s)
           currentvalue = '' if currentvalue.nil?
-          return provider.members_to_s(currentvalue.split(','))
+          currentvalue = currentvalue.is_a?(Array) ? currentvalue : currentvalue.split(',')
+
+          return provider.members_to_s(currentvalue)
         end
 
         super(currentvalue)
       end
       alias :should_to_s :is_to_s
+
+      validate do |value|
+        if provider.respond_to?(:member_valid?)
+          return provider.member_valid?(value)
+        end
+      end
     end
 
     newparam(:auth_membership, :boolean => true, :parent => Puppet::Parameter::Boolean) do
-      desc "whether the provider is authoritative for group membership."
+      desc "Whether the provider is authoritative for group membership. This
+        must be set to true to allow setting the group to no members with
+        `members => [],`."
       defaultto false
     end
 

data/lib/puppet/type/interface.rb

@@ -45,19 +45,24 @@ Puppet::Type.newtype(:interface) do
       newvalues(:auto, :full, :half)
     end
 
+    newproperty(:access_vlan) do
+      desc "Interface static access vlan."
+      newvalues(/^\d+/)
+    end
+
     newproperty(:native_vlan) do
-      desc "Interface native vlan (for access mode only)."
+      desc "Interface native vlan when trunking."
       newvalues(/^\d+/)
     end
 
     newproperty(:encapsulation) do
       desc "Interface switchport encapsulation."
-      newvalues(:none, :dot1q, :isl )
+      newvalues(:none, :dot1q, :isl, :negotiate)
     end
 
     newproperty(:mode) do
       desc "Interface switchport mode."
-      newvalues(:access, :trunk)
+      newvalues(:access, :trunk, 'dynamic auto', 'dynamic desirable')
     end
 
     newproperty(:allowed_trunk_vlans) do

data/lib/puppet/type/macauthorization.rb

@@ -1,7 +1,7 @@
 Puppet::Type.newtype(:macauthorization) do
 
   @doc = "Manage the Mac OS X authorization database. See the
-    [Apple developer site](http://developer.apple.com/documentation/Security/Conceptual/Security_Overview/Security_Services/chapter_4_section_5.html)
+    [Apple developer site](https://developer.apple.com/documentation/Security/Conceptual/Security_Overview/Security_Services/chapter_4_section_5.html)
     for more information.
 
     Note that authorization store directives with hyphens in their names have