puppet 4.1.0 → 4.10.0

data/lib/puppet/util/http_proxy.rb

@@ -1,4 +1,16 @@
+require 'uri'
+require 'openssl'
+
 module Puppet::Util::HttpProxy
+  def self.proxy(uri)
+    if self.no_proxy?(uri)
+      proxy_class = Net::HTTP::Proxy(nil)
+    else
+      proxy_class = Net::HTTP::Proxy(self.http_proxy_host, self.http_proxy_port, self.http_proxy_user, self.http_proxy_password)
+    end
+
+    return proxy_class.new(uri.host, uri.port)
+  end
 
   def self.http_proxy_env
     # Returns a URI object if proxy is set, or nil
@@ -11,6 +23,57 @@ module Puppet::Util::HttpProxy
     return nil
   end
 
+  # The documentation around the format of the no_proxy variable seems
+  # inconsistent.  Some suggests the use of the * as a way of matching any
+  # hosts under a domain, e.g.:
+  #   *.example.com
+  # Other documentation suggests that just a leading '.' indicates a domain
+  # level exclusion, e.g.:
+  #   .example.com
+  # We'll accommodate both here.
+  def self.no_proxy?(dest)
+    unless no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
+      return false
+    end
+
+    unless dest.is_a? URI
+      begin
+        dest = URI.parse(dest)
+      rescue URI::InvalidURIError
+        return false
+      end
+    end
+
+    no_proxy_env.split(/\s*,\s*/).each do |d|
+      host, port = d.split(':')
+      host = Regexp.escape(host).gsub('\*', '.*')
+
+      #If the host of this no_proxy value starts with '.', this entry is
+      #a domain level entry. Don't pin the regex to the beginning of the entry.
+      #If it does not start with a '.' then it is a host specific entry and
+      #should be matched to the destination starting at the beginning.
+      unless host =~ /^\\\./
+        host = "^#{host}"
+      end
+
+      #If this no_proxy entry specifies a port, we want to match it against
+      #the destination port.  Otherwise just match hosts.
+      if port
+        no_proxy_regex  = %r(#{host}:#{port}$)
+        dest_string     = "#{dest.host}:#{dest.port}"
+      else
+        no_proxy_regex  = %r(#{host}$)
+        dest_string     = "#{dest.host}"
+      end
+
+      if no_proxy_regex.match(dest_string)
+        return true
+      end
+    end
+
+    return false
+  end
+
   def self.http_proxy_host
     env = self.http_proxy_env
 
@@ -62,4 +125,73 @@ module Puppet::Util::HttpProxy
 
     return Puppet.settings[:http_proxy_password]
   end
+
+  # Return a Net::HTTP::Proxy object.
+  #
+  # This method optionally configures SSL correctly if the URI scheme is
+  # 'https', including setting up the root certificate store so remote server
+  # SSL certificates can be validated.
+  #
+  # @param [URI] uri The URI that is to be accessed.
+  # @return [Net::HTTP::Proxy] object constructed tailored for the passed URI
+  def self.get_http_object(uri)
+    proxy = proxy(uri)
+
+    if uri.scheme == 'https'
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.set_default_paths
+
+      proxy.use_ssl = true
+      proxy.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      proxy.cert_store = cert_store
+    end
+
+    if Puppet[:http_debug]
+      proxy.set_debug_output($stderr)
+    end
+
+    proxy.open_timeout = Puppet[:http_connect_timeout]
+    proxy.read_timeout = Puppet[:http_read_timeout]
+
+    proxy
+  end
+
+  # Retrieve a document through HTTP(s), following redirects if necessary.
+  # 
+  # Based on the the client implementation in the HTTP pool.
+  #
+  # @see Puppet::Network::HTTP::Connection#request_with_redirects
+  #
+  # @param [URI] uri The address of the resource to retrieve.
+  # @param [symbol] method The name of the Net::HTTP method to use, typically :get, :head, :post etc.
+  # @param [FixNum] redirect_limit The number of redirections that can be followed.
+  # @return [Net::HTTPResponse] a response object
+  def self.request_with_redirects(uri, method, redirect_limit = 10, &block)
+    current_uri = uri
+    response = nil
+
+    0.upto(redirect_limit) do |redirection|
+      proxy = get_http_object(current_uri)
+      response = proxy.send(:head, current_uri.path)
+
+      if [301, 302, 307].include?(response.code.to_i)
+        # handle the redirection
+        current_uri = URI.parse(response['location'])
+        next
+      end
+
+      if block_given?
+        headers = {'Accept' => 'binary', 'accept-encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'}
+        response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
+      else
+        response = proxy.send(method, current_uri.path)
+      end
+
+      Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")
+
+      return response
+    end
+
+    raise RedirectionLimitExceededException, "Too many HTTP redirections for #{uri}"
+  end
 end

data/lib/puppet/util/inifile.rb

@@ -56,7 +56,7 @@ module Puppet::Util::IniConfig
     end
 
     # Set the entry 'key=value'. If no entry with the
-    # given key exists, one is appended to teh end of the section
+    # given key exists, one is appended to the end of the section
     def []=(key, value)
       entry = find_entry(key)
       @dirty = true

data/lib/puppet/util/instance_loader.rb

@@ -60,7 +60,7 @@ module Puppet::Util::InstanceLoader
     @autoloaders[type.intern]
   end
 
-  # Retrieve an alread-loaded instance, or attempt to load our instance.
+  # Retrieve an already-loaded instance, or attempt to load our instance.
   def loaded_instance(type, name)
     name = name.intern
     return nil unless instances = instance_hash(type)

data/lib/puppet/util/log.rb

@@ -1,5 +1,6 @@
 require 'puppet/util/tagging'
 require 'puppet/util/classgen'
+require 'puppet/util/psych_support'
 require 'puppet/network/format_support'
 require 'facter'
 
@@ -9,6 +10,7 @@ require 'facter'
 class Puppet::Util::Log
   include Puppet::Util
   extend Puppet::Util::ClassGen
+  include Puppet::Util::PsychSupport
   include Puppet::Util::Tagging
   include Puppet::Network::FormatSupport
 
@@ -243,6 +245,31 @@ class Puppet::Util::Log
     obj
   end
 
+  # Log output using scope and level
+  #
+  # @param [Puppet::Parser::Scope] scope
+  # @param [Symbol] level log level
+  # @param [Array<Object>] vals the values to log (will be converted to string and joined with space)
+  #
+  def self.log_func(scope, level, vals)
+    # NOTE: 3x, does this: vals.join(" ")
+    # New implementation uses the evaluator to get proper formatting per type
+    vals = vals.map { |v| Puppet::Pops::Evaluator::EvaluatorImpl.new.string(v, scope) }
+
+    # Bypass Puppet.<level> call since it picks up source from "self" which is not applicable in the 4x
+    # Function API.
+    # TODO: When a function can obtain the file, line, pos of the call merge those in (3x supports
+    #       options :file, :line. (These were never output when calling the 3x logging functions since
+    #       3x scope does not know about the calling location at that detailed level, nor do they
+    #       appear in a report to stdout/error when included). Now, the output simply uses scope (like 3x)
+    #       as this is good enough, but does not reflect the true call-stack, but is a rough estimate
+    #       of where the logging call originates from).
+    #
+    Puppet::Util::Log.create({:level => level, :source => scope, :message => vals.join(" ")})
+    nil
+  end
+
+
   attr_accessor :time, :remote, :file, :line, :pos, :source, :issue_code, :environment, :node, :backtrace
   attr_reader :level, :message
 
@@ -335,7 +362,7 @@ class Puppet::Util::Log
   # If they pass a source in to us, we make sure it is a string, and
   # we retrieve any tags we can.
   def source=(source)
-    if source.respond_to?(:path)
+    if defined?(Puppet::Type) && source.is_a?(Puppet::Type)
       @source = source.path
       source.tags.each { |t| tag(t) }
       self.file = source.file

data/lib/puppet/util/logging.rb

@@ -1,9 +1,12 @@
 # A module to make logging a bit easier.
 require 'puppet/util/log'
 require 'puppet/error'
+require 'puppet/util/multi_match.rb'
+
 require 'facter'
 
-module Puppet::Util::Logging
+module Puppet::Util
+module Logging
 
   def send_log(level, message)
     Puppet::Util::Log.create({:level => level, :source => log_source, :message => message}.merge(log_metadata))
@@ -52,7 +55,7 @@ module Puppet::Util::Logging
       backtrace = []
       build_exception_trace(backtrace, exception, trace)
       Puppet::Util::Log.create({
-          :level => :err,
+          :level => options[:level] || :err,
           :source => log_source,
           :message => exception.basic_message,
           :issue_code => exception.issue_code,
@@ -148,6 +151,43 @@ module Puppet::Util::Logging
     key ||= "#{file}:#{line}"
     issue_deprecation_warning(message, key, file, line, false)
   end
+  MM = MultiMatch
+  FILE_AND_LINE = MM::TUPLE
+  FILE_NO_LINE  = MM.new(MM::NOT_NIL, nil).freeze
+  NO_FILE_LINE  = MM.new(nil, MM::NOT_NIL).freeze
+
+  # Logs a (non deprecation) warning once for a given key.
+  #
+  # @param kind [String] The kind of warning. The
+  #   kind must be one of the defined kinds for the Puppet[:disable_warnings] setting.
+  # @param message [String] The message to log (logs via warning)
+  # @param key [String] Key used to make this warning unique
+  # @param file [String,nil] the File related to the warning
+  # @param line [Integer,nil] the Line number related to the warning
+  #   warning as unique
+  #
+  # Either :file and :line and/or :key must be passed.
+  def warn_once(kind, key, message, file = nil, line = nil)
+    return if Puppet[:disable_warnings].include?(kind)
+    $unique_warnings ||= {}
+    if $unique_warnings.length < 100 then
+      if (! $unique_warnings.has_key?(key)) then
+        $unique_warnings[key] = message
+        call_trace =
+        case MM.new(file, line)
+        when FILE_AND_LINE
+          "\n   (at #{file}:#{line})"
+        when FILE_NO_LINE
+          "\n   (in #{file})"
+        when NO_FILE_LINE
+          "\n   (in unknown file, line #{line})"
+        else
+          "\n   (file & line not available)"
+        end
+        warning("#{message}#{call_trace}")
+      end
+    end
+  end
 
   def get_deprecation_offender()
     # we have to put this in its own method to simplify testing; we need to be able to mock the offender results in
@@ -163,6 +203,7 @@ module Puppet::Util::Logging
   end
 
   def clear_deprecation_warnings
+    $unique_warnings.clear if $unique_warnings
     $deprecation_warnings.clear if $deprecation_warnings
   end
 
@@ -183,7 +224,8 @@ module Puppet::Util::Logging
     # find the same offender, and we'd end up logging it again.
     $logged_deprecation_warnings ||= {}
 
-    File.open(deprecations_file, "a") do |f|
+    # Deprecation messages are UTF-8 as they are produced by Ruby
+    Puppet::FileSystem.open(deprecations_file, nil, "a:UTF-8") do |f|
       if ($deprecation_warnings) then
         $deprecation_warnings.each do |offender, message|
           if (! $logged_deprecation_warnings.has_key?(offender)) then
@@ -266,3 +308,4 @@ module Puppet::Util::Logging
     to_s
   end
 end
+end

data/lib/puppet/util/monkey_patches.rb

@@ -20,8 +20,16 @@ end
 
 class Symbol
   def <=> (other)
+    if (other.class != Symbol)
+      case Puppet[:strict]
+      when :warning
+        Puppet.warn_once('deprecation', 'symbol_comparison', 'Comparing Symbols to non-Symbol values is deprecated')
+      when :error
+        raise ArgumentError.new("Comparing Symbols to non-Symbol values is no longer allowed")
+      end
+    end
     self.to_s <=> other.to_s
-  end unless method_defined? '<=>'
+  end
 
   def intern
     self
@@ -114,15 +122,20 @@ if Puppet::Util::Platform.windows?
   require 'openssl'
 
   class OpenSSL::X509::Store
+    @puppet_certs_loaded = false
     alias __original_set_default_paths set_default_paths
     def set_default_paths
       # This can be removed once openssl integrates with windows
-      # cert store, see http://rt.openssl.org/Ticket/Display.html?id=2158
-      Puppet::Util::Windows::RootCerts.instance.to_a.uniq.each do |x509|
-        begin
-          add_cert(x509)
-        rescue OpenSSL::X509::StoreError => e
-          warn "Failed to add #{x509.subject.to_s}"
+      # cert store, see https://rt.openssl.org/Ticket/Display.html?id=2158
+      unless @puppet_certs_loaded
+        @puppet_certs_loaded = true
+
+        Puppet::Util::Windows::RootCerts.instance.to_a.uniq { |cert| cert.to_der }.each do |x509|
+          begin
+            add_cert(x509)
+          rescue OpenSSL::X509::StoreError => e
+            warn "Failed to add #{x509.subject.to_s}"
+          end
         end
       end
 

data/lib/puppet/util/multi_match.rb

@@ -0,0 +1,51 @@
+# MultiMatch allows multiple values to be tested at once in a case expression.
+# This class is needed since Array does not implement the === operator to mean
+# "each v === other.each v". 
+#
+# This class is useful in situations when the Puppet Type System cannot be used
+# (e.g. in Logging, since it needs to be able to log very early in the initialization
+# cycle of puppet)
+#
+# Typically used with the constants
+# NOT_NIL
+# TUPLE
+# TRIPLE
+# 
+# which test against single NOT_NIL value, Array with two NOT_NIL, and Array with three NOT_NIL
+#
+module Puppet::Util
+class MultiMatch
+  attr_reader :values
+
+  def initialize(*values)
+    @values = values
+  end
+
+  def ===(other)
+    lv = @values  # local var is faster than instance var
+    case other
+    when MultiMatch
+      return false unless other.values.size == values.size
+      other.values.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    when Array
+      return false unless other.size == values.size
+      other.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    else
+      false
+    end
+    true
+  end
+
+  # Matches any value that is not nil using the === operator.
+  #
+  class MatchNotNil
+    def ===(v)
+      !v.nil?
+    end
+  end
+
+  NOT_NIL = MatchNotNil.new().freeze
+  TUPLE = MultiMatch.new(NOT_NIL, NOT_NIL).freeze
+  TRIPLE = MultiMatch.new(NOT_NIL, NOT_NIL, NOT_NIL).freeze
+end
+end

data/lib/puppet/util/network_device/cisco/device.rb

@@ -47,7 +47,13 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
   end
 
   def execute(cmd)
-    transport.command(cmd)
+    transport.command(cmd) do |out|
+      if out =~ /^%/mo or out =~ /^Command rejected:/mo
+        # strip off the command just sent
+        error = out.sub(cmd,'')
+        Puppet.err "Error while executing '#{cmd}', device returned: #{error}"
+      end
+    end
   end
 
   def login
@@ -209,6 +215,12 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
       return
     end
 
+    # Cisco VLANs are supposed to be alphanumeric only
+    if should[:description] =~ /[^\w]/
+      Puppet.err "Invalid VLAN name '#{should[:description]}' for Cisco device.\nVLAN name must be alphanumeric, no spaces or special characters."
+      return
+    end
+    
     # We're creating or updating an entry
     execute("conf t")
     execute("vlan #{id}")
@@ -234,20 +246,26 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
           trunking[:mode] = :trunk
         when "static access"
           trunking[:mode] = :access
+        when "dynamic auto"
+          trunking[:mode] = 'dynamic auto'
+        when "dynamic desirable"
+          trunking[:mode] = 'dynamic desirable'
         else
           raise "Unknown switchport mode: #{$1} for #{interface}"
         end
       when /^Administrative Trunking Encapsulation:\s+(.*)$/
         case $1
         when "dot1q","isl"
-          trunking[:encapsulation] = $1.to_sym if trunking[:mode] == :trunk
+          trunking[:encapsulation] = $1.to_sym if trunking[:mode] != :access
+        when "negotiate"
+          trunking[:encapsulation] = :negotiate
         else
           raise "Unknown switchport encapsulation: #{$1} for #{interface}"
         end
-      when /^Access Mode VLAN:\s+(.*) \(\(Inactive\)\)$/
-        # nothing
-      when /^Access Mode VLAN:\s+(.*) \(.*\)$/
-        trunking[:native_vlan] = $1 if trunking[:mode] == :access
+      when /^Access Mode VLAN:\s+(.*) \((.*)\)$/
+        trunking[:access_vlan] = $1 if $2 != '(Inactive)'
+      when /^Trunking Native Mode VLAN:\s+(.*) \(.*\)$/
+        trunking[:native_vlan] = $1
       when /^Trunking VLANs Enabled:\s+(.*)$/
         next if trunking[:mode] == :access
         vlans = $1

data/lib/puppet/util/network_device/cisco/interface.rb

@@ -19,17 +19,18 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
     :description   => [1, "description %s"],
     :speed         => [2, "speed %s"],
     :duplex        => [3, "duplex %s"],
-    :native_vlan   => [4, "switchport access vlan %s"],
-    :encapsulation => [5, "switchport trunk encapsulation %s"],
-    :mode          => [6, "switchport mode %s"],
-    :allowed_trunk_vlans => [7, "switchport trunk allowed vlan %s"],
-    :etherchannel  => [8, ["channel-group %s", "port group %s"]],
-    :ipaddress     => [9,
+    :encapsulation => [4, "switchport trunk encapsulation %s"],
+    :mode          => [5, "switchport mode %s"],
+    :access_vlan   => [6, "switchport access vlan %s"],
+    :native_vlan   => [7, "switchport trunk native vlan %s"],
+    :allowed_trunk_vlans => [8, "switchport trunk allowed vlan %s"],
+    :etherchannel  => [9, ["channel-group %s", "port group %s"]],
+    :ipaddress     => [10,
       lambda do |prefix,ip,option|
         ip.ipv6? ? "ipv6 address #{ip.to_s}/#{prefix} #{option}" :
                    "ip address #{ip.to_s} #{netmask(Socket::AF_INET,prefix)}"
       end],
-      :ensure        => [10, lambda { |value| value == :present ? "no shutdown" : "shutdown" } ]
+    :ensure        => [11, lambda { |value| value == :present ? "no shutdown" : "shutdown" } ]
   }
 
   def update(is={}, should={})
@@ -37,8 +38,15 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
     command("conf t")
     command("interface #{name}")
 
-    # apply changes in a defined orders for cisco IOS devices
+    # apply changes in a defined order for cisco IOS devices
     [is.keys, should.keys].flatten.uniq.sort {|a,b| COMMANDS[a][0] <=> COMMANDS[b][0] }.each do |property|
+      # Work around for old documentation which shows :native_vlan used for access vlan
+      if property == :access_vlan and should[:mode] != :trunk and should[:access_vlan].nil?
+        should[:access_vlan] = should[:native_vlan]
+      end
+
+      Puppet.debug("comparing #{property}: #{is[property]} == #{should[property]}")
+
       # They're equal, so do nothing.
       next if is[property] == should[property]
 
@@ -76,7 +84,11 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
 
   def command(command)
     transport.command(command) do |out|
-      Puppet.err "Error while executing #{command}, device returned #{out}" if out =~ /^%/mo
+      if out =~ /^%/mo or out =~ /^Command rejected:/mo
+        # strip off the command just sent
+        error = out.sub(command,'')
+        Puppet.err "Error while executing '#{command}', device returned: #{error}"
+      end
     end
   end
 end

data/lib/puppet/util/network_device/config.rb

@@ -59,7 +59,7 @@ class Puppet::Util::NetworkDevice::Config
             device.options = { :debug => false }
             Puppet.debug "found device: #{device.name} at #{device.line}"
             devices[name] = device
-          when /^\s*(type|url|debug)(\s+(.+))*$/
+          when /^\s*(type|url|debug)(\s+(.+)\s*)*$/
             parse_directive(device, $1, $3, count)
           else
             raise Puppet::Error, "Invalid line #{count}: #{line}"

data/lib/puppet/util/network_device/transport/ssh.rb

@@ -68,9 +68,13 @@ class Puppet::Util::NetworkDevice::Transport::Ssh < Puppet::Util::NetworkDevice:
   end
 
   def close
-    @channel.close if @channel
-    @channel = nil
-    @ssh.close if @ssh
+    begin
+      @channel.close if @channel
+      @channel = nil
+      @ssh.close if @ssh
+    rescue IOError
+      Puppet.debug "device terminated ssh session impolitely"
+    end
   end
 
   def expect(prompt)

data/lib/puppet/util/network_device/transport/telnet.rb

@@ -1,44 +1,47 @@
 require 'puppet/util/network_device'
 require 'puppet/util/network_device/transport'
 require 'puppet/util/network_device/transport/base'
-require 'net/telnet'
-
-class Puppet::Util::NetworkDevice::Transport::Telnet < Puppet::Util::NetworkDevice::Transport::Base
-    def initialize(verbose = false)
-      super()
-      @verbose = verbose
-    end
-
-    def handles_login?
-      false
-    end
-
-    def connect
-      @telnet = Net::Telnet::new("Host" => host, "Port" => port || 23,
-                                 "Timeout" => 10,
-                                 "Prompt" => default_prompt)
-    end
-
-    def close
-      @telnet.close if @telnet
-      @telnet = nil
-    end
-
-    def expect(prompt)
-      @telnet.waitfor(prompt) do |out|
-        yield out if block_given?
+
+if Puppet.features.telnet?
+  require 'net/telnet'
+
+  class Puppet::Util::NetworkDevice::Transport::Telnet < Puppet::Util::NetworkDevice::Transport::Base
+      def initialize(verbose = false)
+        super()
+        @verbose = verbose
+      end
+
+      def handles_login?
+        false
+      end
+
+      def connect
+        @telnet = Net::Telnet::new("Host" => host, "Port" => port || 23,
+                                   "Timeout" => 10,
+                                   "Prompt" => default_prompt)
       end
-    end
 
-    def command(cmd, options = {})
-      send(cmd)
-      expect(options[:prompt] || default_prompt) do |output|
-        yield output if block_given?
+      def close
+        @telnet.close if @telnet
+        @telnet = nil
       end
-    end
 
-    def send(line)
-      Puppet.debug("telnet: send #{line}") if @verbose
-      @telnet.puts(line)
-    end
+      def expect(prompt)
+        @telnet.waitfor(prompt) do |out|
+          yield out if block_given?
+        end
+      end
+
+      def command(cmd, options = {})
+        send(cmd)
+        expect(options[:prompt] || default_prompt) do |output|
+          yield output if block_given?
+        end
+      end
+
+      def send(line)
+        Puppet.debug("telnet: send #{line}") if @verbose
+        @telnet.puts(line)
+      end
+  end
 end