puppet 4.0.0 → 4.5.0

data/lib/puppet/util/http_proxy.rb

@@ -1,4 +1,16 @@
+require 'uri'
+require 'openssl'
+
 module Puppet::Util::HttpProxy
+  def self.proxy(uri)
+    if self.no_proxy?(uri)
+      proxy_class = Net::HTTP::Proxy(nil)
+    else
+      proxy_class = Net::HTTP::Proxy(self.http_proxy_host, self.http_proxy_port, self.http_proxy_user, self.http_proxy_password)
+    end
+
+    return proxy_class.new(uri.host, uri.port)
+  end
 
   def self.http_proxy_env
     # Returns a URI object if proxy is set, or nil
@@ -11,6 +23,57 @@ module Puppet::Util::HttpProxy
     return nil
   end
 
+  # The documentation around the format of the no_proxy variable seems
+  # inconsistent.  Some suggests the use of the * as a way of matching any
+  # hosts under a domain, e.g.:
+  #   *.example.com
+  # Other documentation suggests that just a leading '.' indicates a domain
+  # level exclusion, e.g.:
+  #   .example.com
+  # We'll accomodate both here.
+  def self.no_proxy?(dest)
+    unless no_proxy_env = ENV["no_proxy"] || ENV["NO_PROXY"]
+      return false
+    end
+
+    unless dest.is_a? URI
+      begin
+        dest = URI.parse(dest)
+      rescue URI::InvalidURIError
+        return false
+      end
+    end
+
+    no_proxy_env.split(/\s*,\s*/).each do |d|
+      host, port = d.split(':')
+      host = Regexp.escape(host).gsub('\*', '.*')
+
+      #If the host of this no_proxy value starts with '.', this entry is
+      #a domain level entry. Don't pin the regex to the beginning of the entry.
+      #If it does not start with a '.' then it is a host specific entry and
+      #should be matched to the destination starting at the beginning.
+      unless host =~ /^\\\./
+        host = "^#{host}"
+      end
+
+      #If this no_proxy entry specifies a port, we want to match it against
+      #the destination port.  Otherwise just match hosts.
+      if port
+        no_proxy_regex  = %r(#{host}:#{port}$)
+        dest_string     = "#{dest.host}:#{dest.port}"
+      else
+        no_proxy_regex  = %r(#{host}$)
+        dest_string     = "#{dest.host}"
+      end
+
+      if no_proxy_regex.match(dest_string)
+        return true
+      end
+    end
+
+    return false
+  end
+
   def self.http_proxy_host
     env = self.http_proxy_env
 
@@ -62,4 +125,73 @@ module Puppet::Util::HttpProxy
 
     return Puppet.settings[:http_proxy_password]
   end
+
+  # Return a Net::HTTP::Proxy object.
+  #
+  # This method optionally configures SSL correctly if the URI scheme is
+  # 'https', including setting up the root certificate store so remote server
+  # SSL certificates can be validated.
+  #
+  # @param [URI] uri The URI that is to be accessed.
+  # @return [Net::HTTP::Proxy] object constructed tailored for the passed URI
+  def self.get_http_object(uri)
+    proxy = proxy(uri)
+
+    if uri.scheme == 'https'
+      cert_store = OpenSSL::X509::Store.new
+      cert_store.set_default_paths
+
+      proxy.use_ssl = true
+      proxy.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      proxy.cert_store = cert_store
+    end
+
+    if Puppet[:http_debug]
+      proxy.set_debug_output($stderr)
+    end
+
+    proxy.open_timeout = Puppet[:http_connect_timeout]
+    proxy.read_timeout = Puppet[:http_read_timeout]
+
+    proxy
+  end
+
+  # Retrieve a document through HTTP(s), following redirects if necessary.
+  # 
+  # Based on the the client implementation in the HTTP pool.
+  #
+  # @see Puppet::Network::HTTP::Connection#request_with_redirects
+  #
+  # @param [URI] uri The address of the resource to retrieve.
+  # @param [symbol] method The name of the Net::HTTP method to use, typically :get, :head, :post etc.
+  # @param [FixNum] redirect_limit The number of redirections that can be followed.
+  # @return [Net::HTTPResponse] a response object
+  def self.request_with_redirects(uri, method, redirect_limit = 10, &block)
+    current_uri = uri
+    response = nil
+
+    0.upto(redirect_limit) do |redirection|
+      proxy = get_http_object(current_uri)
+      response = proxy.send(:head, current_uri.path)
+
+      if [301, 302, 307].include?(response.code.to_i)
+        # handle the redirection
+        current_uri = URI.parse(response['location'])
+        next
+      end
+
+      if block_given?
+        headers = {'Accept' => 'binary', 'accept-encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3'}
+        response = proxy.send("request_#{method}".to_sym, current_uri.path, headers, &block)
+      else
+        response = proxy.send(method, current_uri.path)
+      end
+
+      Puppet.debug("HTTP #{method.to_s.upcase} request to #{current_uri} returned #{response.code} #{response.message}")
+
+      return response
+    end
+
+    raise RedirectionLimitExceededException, "Too many HTTP redirections for #{uri}"
+  end
 end

data/lib/puppet/util/log/destinations.rb

@@ -65,6 +65,8 @@ Puppet::Util::Log.newdesttype :file do
 
   def initialize(path)
     @name = path
+    @json = path.end_with?('.json') ? 1 : 0
+
     # first make sure the directory exists
     # We can't just use 'Config.use' here, because they've
     # specified a "special" destination.
@@ -74,7 +76,21 @@ Puppet::Util::Log.newdesttype :file do
     end
 
     # create the log file, if it doesn't already exist
-    file = File.open(path, File::WRONLY|File::CREAT|File::APPEND)
+    need_array_start = false
+    if @json == 1
+      need_array_start = true
+      if File.exists?(path)
+        sz = File.size(path)
+        need_array_start = sz == 0
+
+        # Assume that entries have been written and that a comma
+        # is needed before next entry
+        @json = 2 if sz > 2
+      end
+    end
+
+    file = File.open(path,  File::WRONLY|File::CREAT|File::APPEND)
+    file.puts('[') if need_array_start
 
     # Give ownership to the user and group puppet will run as
     if Puppet.features.root? && !Puppet::Util::Platform.windows?
@@ -91,7 +107,12 @@ Puppet::Util::Log.newdesttype :file do
   end
 
   def handle(msg)
-    @file.puts("#{msg.time} #{msg.source} (#{msg.level}): #{msg}")
+    if @json > 0
+      @json > 1 ? @file.puts(',') : @json = 2
+      JSON.dump(msg.to_structured_hash, @file)
+    else
+      @file.puts("#{msg.time} #{msg.source} (#{msg.level}): #{msg}")
+    end
 
     @file.flush if @autoflush
   end

data/lib/puppet/util/log.rb

@@ -1,5 +1,6 @@
 require 'puppet/util/tagging'
 require 'puppet/util/classgen'
+require 'puppet/util/psych_support'
 require 'puppet/network/format_support'
 require 'facter'
 
@@ -9,6 +10,7 @@ require 'facter'
 class Puppet::Util::Log
   include Puppet::Util
   extend Puppet::Util::ClassGen
+  include Puppet::Util::PsychSupport
   include Puppet::Util::Tagging
   include Puppet::Network::FormatSupport
 
@@ -243,7 +245,7 @@ class Puppet::Util::Log
     obj
   end
 
-  attr_accessor :time, :remote, :file, :line, :source
+  attr_accessor :time, :remote, :file, :line, :pos, :source, :issue_code, :environment, :node, :backtrace
   attr_reader :level, :message
 
   def initialize(args)
@@ -257,9 +259,10 @@ class Puppet::Util::Log
       tags.each { |t| self.tag(t) }
     end
 
-    [:file, :line].each do |attr|
+    # Don't add these unless defined (preserve 3.x API as much as possible)
+    [:file, :line, :pos, :issue_code, :environment, :node, :backtrace].each do |attr|
       next unless value = args[attr]
-      send(attr.to_s + "=", value)
+      send(attr.to_s + '=', value)
     end
 
     Log.newmessage(self)
@@ -274,8 +277,11 @@ class Puppet::Util::Log
     if @time.is_a? String
       @time = Time.parse(@time)
     end
-    @file = data['file'] if data['file']
-    @line = data['line'] if data['line']
+    # Don't add these unless defined (preserve 3.x API as much as possible)
+    %w(file line pos issue_code environment node backtrace).each do |name|
+      next unless value = data[name]
+      send(name + '=', value)
+    end
   end
 
   def to_hash
@@ -285,15 +291,30 @@ class Puppet::Util::Log
   def to_data_hash
     {
       'level' => @level,
-      'message' => @message,
+      'message' => to_s,
       'source' => @source,
-      'tags' => @tags,
+      'tags' => @tags.to_a,
       'time' => @time.iso8601(9),
       'file' => @file,
       'line' => @line,
     }
   end
 
+  def to_structured_hash
+    hash = {
+      'level' => @level,
+      'message' => @message,
+      'source' => @source,
+      'tags' => @tags.to_a,
+      'time' => @time.iso8601(9),
+    }
+    %w(file line pos issue_code environment node backtrace).each do |name|
+      attr_name = "@#{name}"
+      hash[name] = instance_variable_get(attr_name) if instance_variable_defined?(attr_name)
+    end
+    hash
+  end
+
   def to_pson(*args)
     to_data_hash.to_pson(*args)
   end
@@ -316,7 +337,7 @@ class Puppet::Util::Log
   # If they pass a source in to us, we make sure it is a string, and
   # we retrieve any tags we can.
   def source=(source)
-    if source.respond_to?(:path)
+    if defined?(Puppet::Type) && source.is_a?(Puppet::Type)
       @source = source.path
       source.tags.each { |t| tag(t) }
       self.file = source.file
@@ -331,7 +352,30 @@ class Puppet::Util::Log
   end
 
   def to_s
-    message
+    msg = message
+
+    # Issue based messages do not have details in the message. It
+    # must be appended here
+    unless issue_code.nil?
+      msg = "Could not parse for environment #{environment}: #{msg}" unless environment.nil?
+      if file && line && pos
+        msg = "#{msg} at #{file}:#{line}:#{pos}"
+      elsif file and line
+        msg = "#{msg}  at #{file}:#{line}"
+      elsif line && pos
+        msg = "#{msg}  at line #{line}:#{pos}"
+      elsif line
+        msg = "#{msg}  at line #{line}"
+      elsif file
+        msg = "#{msg}  in #{file}"
+      end
+      msg = "#{msg} on node #{node}" unless node.nil?
+      if @backtrace.is_a?(Array)
+        msg += "\n"
+        msg += @backtrace.join("\n")
+      end
+    end
+    msg
   end
 
 end

data/lib/puppet/util/logging.rb

@@ -1,9 +1,12 @@
 # A module to make logging a bit easier.
 require 'puppet/util/log'
 require 'puppet/error'
+require 'puppet/util/multi_match.rb'
+
 require 'facter'
 
-module Puppet::Util::Logging
+module Puppet::Util
+module Logging
 
   def send_log(level, message)
     Puppet::Util::Log.create({:level => level, :source => log_source, :message => message}.merge(log_metadata))
@@ -46,9 +49,45 @@ module Puppet::Util::Logging
   #    wish to log a message at all; in this case it is likely that you are only calling this method in order
   #    to take advantage of the backtrace logging.
   def log_exception(exception, message = :default, options = {})
-    err(format_exception(exception, message, Puppet[:trace] || options[:trace]))
+    trace = Puppet[:trace] || options[:trace]
+    if message == :default && exception.is_a?(Puppet::ParseErrorWithIssue)
+      # Retain all detailed info and keep plain message and stacktrace separate
+      backtrace = []
+      build_exception_trace(backtrace, exception, trace)
+      Puppet::Util::Log.create({
+          :level => :err,
+          :source => log_source,
+          :message => exception.basic_message,
+          :issue_code => exception.issue_code,
+          :backtrace => backtrace.empty? ? nil : backtrace,
+          :file => exception.file,
+          :line => exception.line,
+          :pos => exception.pos,
+          :environment => exception.environment,
+          :node => exception.node
+        }.merge(log_metadata))
+    else
+      err(format_exception(exception, message, trace))
+    end
   end
 
+  def build_exception_trace(arr, exception, trace = true)
+    if trace and exception.backtrace
+      exception.backtrace.each do |line|
+        arr << line =~ /^(.+):(\d+.*)$/ ? ("#{Pathname($1).realpath}:#{$2}" rescue line) : line
+      end
+    end
+    if exception.respond_to?(:original)
+      original =  exception.original
+      unless original.nil?
+        arr << 'Wrapped exception:'
+        arr << original.message
+        build_exception_trace(arr, original, trace)
+      end
+    end
+  end
+  private :build_exception_trace
+
   def format_exception(exception, message = :default, trace = true)
     arr = []
     case message
@@ -112,6 +151,43 @@ module Puppet::Util::Logging
     key ||= "#{file}:#{line}"
     issue_deprecation_warning(message, key, file, line, false)
   end
+  MM = MultiMatch
+  FILE_AND_LINE = MM::TUPLE
+  FILE_NO_LINE  = MM.new(MM::NOT_NIL, nil).freeze
+  NO_FILE_LINE  = MM.new(nil, MM::NOT_NIL).freeze
+
+  # Logs a (non deprecation) warning once for a given key.
+  #
+  # @param kind [String] The kind of warning. The
+  #   kind must be one of the defined kinds for the Puppet[:disable_warnings] setting.
+  # @param message [String] The message to log (logs via warning)
+  # @param key [String] Key used to make this warning unique
+  # @param file [String,nil] the File related to the warning
+  # @param line [Integer,nil] the Line number related to the warning
+  #   warning as unique
+  #
+  # Either :file and :line and/or :key must be passed.
+  def warn_once(kind, key, message, file = nil, line = nil)
+    return if Puppet[:disable_warnings].include?(kind)
+    $unique_warnings ||= {}
+    if $unique_warnings.length < 100 then
+      if (! $unique_warnings.has_key?(key)) then
+        $unique_warnings[key] = message
+        call_trace =
+        case MM.new(file, line)
+        when FILE_AND_LINE
+          "\n   (at #{file}:#{line})"
+        when FILE_NO_LINE
+          "\n   (in #{file})"
+        when NO_FILE_LINE
+          "\n   (in unknown file, line #{line})"
+        else
+          "\n   (file & line not available)"
+        end
+        warning("#{message}#{call_trace}")
+      end
+    end
+  end
 
   def get_deprecation_offender()
     # we have to put this in its own method to simplify testing; we need to be able to mock the offender results in
@@ -127,6 +203,7 @@ module Puppet::Util::Logging
   end
 
   def clear_deprecation_warnings
+    $unique_warnings.clear if $unique_warnings
     $deprecation_warnings.clear if $deprecation_warnings
   end
 
@@ -230,3 +307,4 @@ module Puppet::Util::Logging
     to_s
   end
 end
+end

data/lib/puppet/util/monkey_patches.rb

@@ -20,8 +20,16 @@ end
 
 class Symbol
   def <=> (other)
+    if (other.class != Symbol)
+      case Puppet[:strict]
+      when :warning
+        Puppet.warn_once('deprecation', 'symbol_comparison', 'Comparing Symbols to non-Symbol values is deprecated')
+      when :error
+        raise ArgumentError.new("Comparing Symbols to non-Symbol values is no longer allowed")
+      end
+    end
     self.to_s <=> other.to_s
-  end unless method_defined? '<=>'
+  end
 
   def intern
     self
@@ -114,15 +122,20 @@ if Puppet::Util::Platform.windows?
   require 'openssl'
 
   class OpenSSL::X509::Store
+    @puppet_certs_loaded = false
     alias __original_set_default_paths set_default_paths
     def set_default_paths
       # This can be removed once openssl integrates with windows
-      # cert store, see http://rt.openssl.org/Ticket/Display.html?id=2158
-      Puppet::Util::Windows::RootCerts.instance.to_a.uniq.each do |x509|
-        begin
-          add_cert(x509)
-        rescue OpenSSL::X509::StoreError => e
-          warn "Failed to add #{x509.subject.to_s}"
+      # cert store, see https://rt.openssl.org/Ticket/Display.html?id=2158
+      unless @puppet_certs_loaded
+        @puppet_certs_loaded = true
+
+        Puppet::Util::Windows::RootCerts.instance.to_a.uniq { |cert| cert.to_der }.each do |x509|
+          begin
+            add_cert(x509)
+          rescue OpenSSL::X509::StoreError => e
+            warn "Failed to add #{x509.subject.to_s}"
+          end
         end
       end
 

data/lib/puppet/util/multi_match.rb

@@ -0,0 +1,51 @@
+# MultiMatch allows multiple values to be tested at once in a case expression.
+# This class is needed since Array does not implement the === operator to mean
+# "each v === other.each v". 
+#
+# This class is useful in situations when the Puppet Type System cannot be used
+# (e.g. in Logging, since it needs to be able to log very early i the intialization
+# cycle of puppet)
+#
+# Typically used with the constants
+# NOT_NIL
+# TUPLE
+# TRIPLE
+# 
+# which test against single NOT_NIL value, Array with two NOT_NIL, and Array with three NOT_NIL
+#
+module Puppet::Util
+class MultiMatch
+  attr_reader :values
+
+  def initialize(*values)
+    @values = values
+  end
+
+  def ===(other)
+    lv = @values  # local var is faster than instance var
+    case other
+    when MultiMatch
+      return false unless other.values.size == values.size
+      other.values.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    when Array
+      return false unless other.size == values.size
+      other.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    else
+      false
+    end
+    true
+  end
+
+  # Matches any value that is not nil using the === operator.
+  #
+  class MatchNotNil
+    def ===(v)
+      !v.nil?
+    end
+  end
+
+  NOT_NIL = MatchNotNil.new().freeze
+  TUPLE = MultiMatch.new(NOT_NIL, NOT_NIL).freeze
+  TRIPLE = MultiMatch.new(NOT_NIL, NOT_NIL, NOT_NIL).freeze
+end
+end

data/lib/puppet/util/network_device/cisco/device.rb

@@ -47,7 +47,13 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
   end
 
   def execute(cmd)
-    transport.command(cmd)
+    transport.command(cmd) do |out|
+      if out =~ /^%/mo or out =~ /^Command rejected:/mo
+        # strip off the command just sent
+        error = out.sub(cmd,'')
+        Puppet.err "Error while executing '#{cmd}', device returned: #{error}"
+      end
+    end
   end
 
   def login
@@ -209,6 +215,12 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
       return
     end
 
+    # Cisco VLANs are supposed to be alphanumeric only
+    if should[:description] =~ /[^\w]/
+      Puppet.err "Invalid VLAN name '#{should[:description]}' for Cisco device.\nVLAN name must be alphanumeric, no spaces or special characters."
+      return
+    end
+    
     # We're creating or updating an entry
     execute("conf t")
     execute("vlan #{id}")
@@ -234,20 +246,26 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
           trunking[:mode] = :trunk
         when "static access"
           trunking[:mode] = :access
+        when "dynamic auto"
+          trunking[:mode] = 'dynamic auto'
+        when "dynamic desirable"
+          trunking[:mode] = 'dynamic desirable'
         else
           raise "Unknown switchport mode: #{$1} for #{interface}"
         end
       when /^Administrative Trunking Encapsulation:\s+(.*)$/
         case $1
         when "dot1q","isl"
-          trunking[:encapsulation] = $1.to_sym if trunking[:mode] == :trunk
+          trunking[:encapsulation] = $1.to_sym if trunking[:mode] != :access
+        when "negotiate"
+          trunking[:encapsulation] = :negotiate
         else
           raise "Unknown switchport encapsulation: #{$1} for #{interface}"
         end
-      when /^Access Mode VLAN:\s+(.*) \(\(Inactive\)\)$/
-        # nothing
-      when /^Access Mode VLAN:\s+(.*) \(.*\)$/
-        trunking[:native_vlan] = $1 if trunking[:mode] == :access
+      when /^Access Mode VLAN:\s+(.*) \((.*)\)$/
+        trunking[:access_vlan] = $1 if $2 != '(Inactive)'
+      when /^Trunking Native Mode VLAN:\s+(.*) \(.*\)$/
+        trunking[:native_vlan] = $1
       when /^Trunking VLANs Enabled:\s+(.*)$/
         next if trunking[:mode] == :access
         vlans = $1

data/lib/puppet/util/network_device/cisco/interface.rb

@@ -19,17 +19,18 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
     :description   => [1, "description %s"],
     :speed         => [2, "speed %s"],
     :duplex        => [3, "duplex %s"],
-    :native_vlan   => [4, "switchport access vlan %s"],
-    :encapsulation => [5, "switchport trunk encapsulation %s"],
-    :mode          => [6, "switchport mode %s"],
-    :allowed_trunk_vlans => [7, "switchport trunk allowed vlan %s"],
-    :etherchannel  => [8, ["channel-group %s", "port group %s"]],
-    :ipaddress     => [9,
+    :encapsulation => [4, "switchport trunk encapsulation %s"],
+    :mode          => [5, "switchport mode %s"],
+    :access_vlan   => [6, "switchport access vlan %s"],
+    :native_vlan   => [7, "switchport trunk native vlan %s"],
+    :allowed_trunk_vlans => [8, "switchport trunk allowed vlan %s"],
+    :etherchannel  => [9, ["channel-group %s", "port group %s"]],
+    :ipaddress     => [10,
       lambda do |prefix,ip,option|
         ip.ipv6? ? "ipv6 address #{ip.to_s}/#{prefix} #{option}" :
                    "ip address #{ip.to_s} #{netmask(Socket::AF_INET,prefix)}"
       end],
-      :ensure        => [10, lambda { |value| value == :present ? "no shutdown" : "shutdown" } ]
+    :ensure        => [11, lambda { |value| value == :present ? "no shutdown" : "shutdown" } ]
   }
 
   def update(is={}, should={})
@@ -37,8 +38,15 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
     command("conf t")
     command("interface #{name}")
 
-    # apply changes in a defined orders for cisco IOS devices
+    # apply changes in a defined order for cisco IOS devices
     [is.keys, should.keys].flatten.uniq.sort {|a,b| COMMANDS[a][0] <=> COMMANDS[b][0] }.each do |property|
+      # Work around for old documentation which shows :native_vlan used for access vlan
+      if property == :access_vlan and should[:mode] != :trunk and should[:access_vlan].nil?
+        should[:access_vlan] = should[:native_vlan]
+      end
+
+      Puppet.debug("comparing #{property}: #{is[property]} == #{should[property]}")
+
       # They're equal, so do nothing.
       next if is[property] == should[property]
 
@@ -76,7 +84,11 @@ class Puppet::Util::NetworkDevice::Cisco::Interface
 
   def command(command)
     transport.command(command) do |out|
-      Puppet.err "Error while executing #{command}, device returned #{out}" if out =~ /^%/mo
+      if out =~ /^%/mo or out =~ /^Command rejected:/mo
+        # strip off the command just sent
+        error = out.sub(command,'')
+        Puppet.err "Error while executing '#{command}', device returned: #{error}"
+      end
     end
   end
 end

data/lib/puppet/util/network_device/transport/ssh.rb

@@ -68,9 +68,13 @@ class Puppet::Util::NetworkDevice::Transport::Ssh < Puppet::Util::NetworkDevice:
   end
 
   def close
-    @channel.close if @channel
-    @channel = nil
-    @ssh.close if @ssh
+    begin
+      @channel.close if @channel
+      @channel = nil
+      @ssh.close if @ssh
+    rescue IOError
+      Puppet.debug "device terminated ssh session impolitely"
+    end
   end
 
   def expect(prompt)