puppet 4.0.0 → 5.0.0

data/lib/puppet/util/log.rb

@@ -1,5 +1,6 @@
 require 'puppet/util/tagging'
 require 'puppet/util/classgen'
+require 'puppet/util/psych_support'
 require 'puppet/network/format_support'
 require 'facter'
 
@@ -9,6 +10,7 @@ require 'facter'
 class Puppet::Util::Log
   include Puppet::Util
   extend Puppet::Util::ClassGen
+  include Puppet::Util::PsychSupport
   include Puppet::Util::Tagging
   include Puppet::Network::FormatSupport
 
@@ -168,6 +170,17 @@ class Puppet::Util::Log
   def Log.newmessage(msg)
     return if @levels.index(msg.level) < @loglevel
 
+    if msg.message.valid_encoding?
+      message = Puppet::Util::CharacterEncoding.convert_to_utf_8(msg.message)
+    else
+      message = _("Received a log message with invalid encoding:")
+      message << Puppet::Util::CharacterEncoding.convert_to_utf_8(msg.message.dump) << "\n"
+      # We only select the last 10 callers in the stack to avoid being spammy
+      message << _("Backtrace:") << "\n" << caller[0..10].join("\n")
+    end
+
+    msg.message = message
+
     queuemessage(msg) if @destinations.length == 0
 
     @destinations.each do |name, dest|
@@ -207,7 +220,7 @@ class Puppet::Util::Log
 
   # Reopen all of our logs.
   def Log.reopen
-    Puppet.notice "Reopening log files"
+    Puppet.notice _("Reopening log files")
     types = @destinations.keys
     @destinations.each { |type, dest|
       dest.close if dest.respond_to?(:close)
@@ -243,7 +256,32 @@ class Puppet::Util::Log
     obj
   end
 
-  attr_accessor :time, :remote, :file, :line, :source
+  # Log output using scope and level
+  #
+  # @param [Puppet::Parser::Scope] scope
+  # @param [Symbol] level log level
+  # @param [Array<Object>] vals the values to log (will be converted to string and joined with space)
+  #
+  def self.log_func(scope, level, vals)
+    # NOTE: 3x, does this: vals.join(" ")
+    # New implementation uses the evaluator to get proper formatting per type
+    vals = vals.map { |v| Puppet::Pops::Evaluator::EvaluatorImpl.new.string(v, scope) }
+
+    # Bypass Puppet.<level> call since it picks up source from "self" which is not applicable in the 4x
+    # Function API.
+    # TODO: When a function can obtain the file, line, pos of the call merge those in (3x supports
+    #       options :file, :line. (These were never output when calling the 3x logging functions since
+    #       3x scope does not know about the calling location at that detailed level, nor do they
+    #       appear in a report to stdout/error when included). Now, the output simply uses scope (like 3x)
+    #       as this is good enough, but does not reflect the true call-stack, but is a rough estimate
+    #       of where the logging call originates from).
+    #
+    Puppet::Util::Log.create({:level => level, :source => scope, :message => vals.join(" ")})
+    nil
+  end
+
+
+  attr_accessor :time, :remote, :file, :line, :pos, :source, :issue_code, :environment, :node, :backtrace
   attr_reader :level, :message
 
   def initialize(args)
@@ -257,9 +295,10 @@ class Puppet::Util::Log
       tags.each { |t| self.tag(t) }
     end
 
-    [:file, :line].each do |attr|
+    # Don't add these unless defined (preserve 3.x API as much as possible)
+    [:file, :line, :pos, :issue_code, :environment, :node, :backtrace].each do |attr|
       next unless value = args[attr]
-      send(attr.to_s + "=", value)
+      send(attr.to_s + '=', value)
     end
 
     Log.newmessage(self)
@@ -274,8 +313,11 @@ class Puppet::Util::Log
     if @time.is_a? String
       @time = Time.parse(@time)
     end
-    @file = data['file'] if data['file']
-    @line = data['line'] if data['line']
+    # Don't add these unless defined (preserve 3.x API as much as possible)
+    %w(file line pos issue_code environment node backtrace).each do |name|
+      next unless value = data[name]
+      send(name + '=', value)
+    end
   end
 
   def to_hash
@@ -284,18 +326,29 @@ class Puppet::Util::Log
 
   def to_data_hash
     {
-      'level' => @level,
-      'message' => @message,
+      'level' => @level.to_s,
+      'message' => to_s,
       'source' => @source,
-      'tags' => @tags,
+      'tags' => @tags.to_a,
       'time' => @time.iso8601(9),
       'file' => @file,
       'line' => @line,
     }
   end
 
-  def to_pson(*args)
-    to_data_hash.to_pson(*args)
+  def to_structured_hash
+    hash = {
+      'level' => @level,
+      'message' => @message,
+      'source' => @source,
+      'tags' => @tags.to_a,
+      'time' => @time.iso8601(9),
+    }
+    %w(file line pos issue_code environment node backtrace).each do |name|
+      attr_name = "@#{name}"
+      hash[name] = instance_variable_get(attr_name) if instance_variable_defined?(attr_name)
+    end
+    hash
   end
 
   def message=(msg)
@@ -316,7 +369,7 @@ class Puppet::Util::Log
   # If they pass a source in to us, we make sure it is a string, and
   # we retrieve any tags we can.
   def source=(source)
-    if source.respond_to?(:path)
+    if defined?(Puppet::Type) && source.is_a?(Puppet::Type)
       @source = source.path
       source.tags.each { |t| tag(t) }
       self.file = source.file
@@ -331,12 +384,35 @@ class Puppet::Util::Log
   end
 
   def to_s
-    message
+    msg = message
+
+    # Issue based messages do not have details in the message. It
+    # must be appended here
+    unless issue_code.nil?
+      msg = _("Could not parse for environment %{env}: %{msg}") % { env: environment, msg: msg } unless environment.nil?
+      if file && line && pos
+        msg = _("%{msg} at %{file}:%{line}:%{pos}") % { msg: msg, file: file, line: line, pos: pos }
+      elsif file and line
+        msg = _("%{msg}  at %{file}:%{line}") % { msg: msg, file: file, line: line }
+      elsif line && pos
+        msg = _("%{msg}  at line %{line}:%{pos}") % { msg: msg, line: line, pos: pos }
+      elsif line
+        msg = _("%{msg}  at line %{line}") % { msg: msg, line: line }
+      elsif file
+        msg = _("%{msg}  in %{file}") % { msg: msg, file: file }
+      end
+      msg = _("%{msg} on node %{node}") % { msg: msg, node: node } unless node.nil?
+      if @backtrace.is_a?(Array)
+        msg += "\n"
+        msg += @backtrace.join("\n")
+      end
+    end
+    msg
   end
 
 end
 
-# This is for backward compatibility from when we changed the constant to Puppet::Util::Log
-# because the reports include the constant name.  Apparently the alias was created in
-# March 2007, should could probably be removed soon.
+# This is for backward compatibility from when we changed the constant to
+# Puppet::Util::Log because the reports include the constant name. It was
+# considered for removal but left in due to risk of breakage (PUP-7502).
 Puppet::Log = Puppet::Util::Log

data/lib/puppet/util/logging.rb

@@ -1,9 +1,12 @@
 # A module to make logging a bit easier.
 require 'puppet/util/log'
 require 'puppet/error'
+require 'puppet/util/multi_match.rb'
+
 require 'facter'
 
-module Puppet::Util::Logging
+module Puppet::Util
+module Logging
 
   def send_log(level, message)
     Puppet::Util::Log.create({:level => level, :source => log_source, :message => message}.merge(log_metadata))
@@ -46,9 +49,45 @@ module Puppet::Util::Logging
   #    wish to log a message at all; in this case it is likely that you are only calling this method in order
   #    to take advantage of the backtrace logging.
   def log_exception(exception, message = :default, options = {})
-    err(format_exception(exception, message, Puppet[:trace] || options[:trace]))
+    trace = Puppet[:trace] || options[:trace]
+    if message == :default && exception.is_a?(Puppet::ParseErrorWithIssue)
+      # Retain all detailed info and keep plain message and stacktrace separate
+      backtrace = []
+      build_exception_trace(backtrace, exception, trace)
+      Puppet::Util::Log.create({
+          :level => options[:level] || :err,
+          :source => log_source,
+          :message => exception.basic_message,
+          :issue_code => exception.issue_code,
+          :backtrace => backtrace.empty? ? nil : backtrace,
+          :file => exception.file,
+          :line => exception.line,
+          :pos => exception.pos,
+          :environment => exception.environment,
+          :node => exception.node
+        }.merge(log_metadata))
+    else
+      err(format_exception(exception, message, trace))
+    end
   end
 
+  def build_exception_trace(arr, exception, trace = true)
+    if trace and exception.backtrace
+      exception.backtrace.each do |line|
+        arr << line =~ /^(.+):(\d+.*)$/ ? ("#{Pathname($1).realpath}:#{$2}" rescue line) : line
+      end
+    end
+    if exception.respond_to?(:original)
+      original =  exception.original
+      unless original.nil?
+        arr << _('Wrapped exception:')
+        arr << original.message
+        build_exception_trace(arr, original, trace)
+      end
+    end
+  end
+  private :build_exception_trace
+
   def format_exception(exception, message = :default, trace = true)
     arr = []
     case message
@@ -64,7 +103,7 @@ module Puppet::Util::Logging
       arr << Puppet::Util.pretty_backtrace(exception.backtrace)
     end
     if exception.respond_to?(:original) and exception.original
-      arr << "Wrapped exception:"
+      arr << _("Wrapped exception:")
       arr << format_exception(exception.original, :default, trace)
     end
     arr.flatten.join("\n")
@@ -112,6 +151,43 @@ module Puppet::Util::Logging
     key ||= "#{file}:#{line}"
     issue_deprecation_warning(message, key, file, line, false)
   end
+  MM = MultiMatch
+  FILE_AND_LINE = MM::TUPLE
+  FILE_NO_LINE  = MM.new(MM::NOT_NIL, nil).freeze
+  NO_FILE_LINE  = MM.new(nil, MM::NOT_NIL).freeze
+
+  # Logs a (non deprecation) warning once for a given key.
+  #
+  # @param kind [String] The kind of warning. The
+  #   kind must be one of the defined kinds for the Puppet[:disable_warnings] setting.
+  # @param message [String] The message to log (logs via warning)
+  # @param key [String] Key used to make this warning unique
+  # @param file [String,nil] the File related to the warning
+  # @param line [Integer,nil] the Line number related to the warning
+  #   warning as unique
+  #
+  # Either :file and :line and/or :key must be passed.
+  def warn_once(kind, key, message, file = nil, line = nil)
+    return if Puppet[:disable_warnings].include?(kind)
+    $unique_warnings ||= {}
+    if $unique_warnings.length < 100 then
+      if (! $unique_warnings.has_key?(key)) then
+        $unique_warnings[key] = message
+        call_trace =
+        case MM.new(file, line)
+        when FILE_AND_LINE
+          _("\n   (at %{file}:%{line})") % { file: file, line: line }
+        when FILE_NO_LINE
+          _("\n   (in %{file})") % { file: file }
+        when NO_FILE_LINE
+          _("\n   (in unknown file, line %{line})") % { line: line }
+        else
+          _("\n   (file & line not available)")
+        end
+        warning("#{message}#{call_trace}")
+      end
+    end
+  end
 
   def get_deprecation_offender()
     # we have to put this in its own method to simplify testing; we need to be able to mock the offender results in
@@ -127,6 +203,7 @@ module Puppet::Util::Logging
   end
 
   def clear_deprecation_warnings
+    $unique_warnings.clear if $unique_warnings
     $deprecation_warnings.clear if $deprecation_warnings
   end
 
@@ -147,7 +224,8 @@ module Puppet::Util::Logging
     # find the same offender, and we'd end up logging it again.
     $logged_deprecation_warnings ||= {}
 
-    File.open(deprecations_file, "a") do |f|
+    # Deprecation messages are UTF-8 as they are produced by Ruby
+    Puppet::FileSystem.open(deprecations_file, nil, "a:UTF-8") do |f|
       if ($deprecation_warnings) then
         $deprecation_warnings.each do |offender, message|
           if (! $logged_deprecation_warnings.has_key?(offender)) then
@@ -201,10 +279,13 @@ module Puppet::Util::Logging
       key ||= (offender = get_deprecation_offender)
       if (! $deprecation_warnings.has_key?(key)) then
         $deprecation_warnings[key] = message
+        # split out to allow translation
+        unknown = _('unknown')
         call_trace = use_caller ?
           (offender || get_deprecation_offender).join('; ') :
-          "#{file || 'unknown'}:#{line || 'unknown'}"
-        warning("#{message}\n   (at #{call_trace})")
+          "#{file || unknown}:#{line || unknown}"
+        #TRANSLATORS error message with origin location
+        warning(_("%{message}\n   (at %{call_trace})") % { message: message, call_trace: call_trace })
       end
     end
   end
@@ -230,3 +311,4 @@ module Puppet::Util::Logging
     to_s
   end
 end
+end

data/lib/puppet/util/metric.rb

@@ -4,17 +4,24 @@ require 'puppet/network/format_support'
 
 # A class for handling metrics.  This is currently ridiculously hackish.
 class Puppet::Util::Metric
+  include Puppet::Util::PsychSupport
   include Puppet::Network::FormatSupport
 
   attr_accessor :type, :name, :value, :label
   attr_writer :values
 
   def self.from_data_hash(data)
-    metric = new(data['name'], data['label'])
-    metric.values = data['values']
+    metric = allocate
+    metric.initialize_from_hash(data)
     metric
   end
 
+  def initialize_from_hash(data)
+    @name = data['name']
+    @label = data['label'] || self.class.labelize(@name)
+    @values = data['values']
+  end
+
   def to_data_hash
     {
       'name' => @name,
@@ -23,10 +30,6 @@ class Puppet::Util::Metric
     }
   end
 
-  def to_pson(*args)
-    to_data_hash.to_pson(*args)
-  end
-
   # Return a specific value
   def [](name)
     if value = @values.find { |v| v[0] == name }

data/lib/puppet/util/monkey_patches.rb

@@ -18,16 +18,6 @@ module RDoc
   end
 end
 
-class Symbol
-  def <=> (other)
-    self.to_s <=> other.to_s
-  end unless method_defined? '<=>'
-
-  def intern
-    self
-  end unless method_defined? 'intern'
-end
-
 class Object
   # ActiveSupport 2.3.x mixes in a dangerous method
   # that can cause rspec to fork bomb
@@ -37,54 +27,6 @@ class Object
   end
 end
 
-require 'fcntl'
-class IO
-  def self.binwrite(name, string, offset = nil)
-    # Determine if we should truncate or not.  Since the truncate method on a
-    # file handle isn't implemented on all platforms, safer to do this in what
-    # looks like the libc / POSIX flag - which is usually pretty robust.
-    # --daniel 2012-03-11
-    mode = Fcntl::O_CREAT | Fcntl::O_WRONLY | (offset.nil? ? Fcntl::O_TRUNC : 0)
-
-    # We have to duplicate the mode because Ruby on Windows is a bit precious,
-    # and doesn't actually carry over the mode.  It won't work to just use
-    # open, either, because that doesn't like our system modes and the default
-    # open bits don't do what we need, which is awesome. --daniel 2012-03-30
-    IO.open(IO::sysopen(name, mode), mode) do |f|
-      # ...seek to our desired offset, then write the bytes.  Don't try to
-      # seek past the start of the file, eh, because who knows what platform
-      # would legitimately blow up if we did that.
-      #
-      # Double-check the positioning, too, since destroying data isn't my idea
-      # of a good time. --daniel 2012-03-11
-      target = [0, offset.to_i].max
-      unless (landed = f.sysseek(target, IO::SEEK_SET)) == target
-        raise "unable to seek to target offset #{target} in #{name}: got to #{landed}"
-      end
-
-      f.syswrite(string)
-    end
-  end unless singleton_methods.include?(:binwrite)
-end
-
-class Range
-  def intersection(other)
-    raise ArgumentError, 'value must be a Range' unless other.kind_of?(Range)
-    return unless other === self.first || self === other.first
-
-    start = [self.first, other.first].max
-    if self.exclude_end? && self.last <= other.last
-      start ... self.last
-    elsif other.exclude_end? && self.last >= other.last
-      start ... other.last
-    else
-      start .. [ self.last, other.last ].min
-    end
-  end unless method_defined? :intersection
-
-  alias_method :&, :intersection unless method_defined? :&
-end
-
 # (#19151) Reject all SSLv2 ciphers and handshakes
 require 'openssl'
 class OpenSSL::SSL::SSLContext
@@ -93,7 +35,9 @@ class OpenSSL::SSL::SSLContext
   else
     DEFAULT_PARAMS[:options] = OpenSSL::SSL::OP_NO_SSLv2 | OpenSSL::SSL::OP_NO_SSLv3
   end
-  DEFAULT_PARAMS[:ciphers] << ':!SSLv2'
+  if DEFAULT_PARAMS[:ciphers]
+    DEFAULT_PARAMS[:ciphers] << ':!SSLv2'
+  end
 
   alias __original_initialize initialize
   private :__original_initialize
@@ -114,15 +58,20 @@ if Puppet::Util::Platform.windows?
   require 'openssl'
 
   class OpenSSL::X509::Store
+    @puppet_certs_loaded = false
     alias __original_set_default_paths set_default_paths
     def set_default_paths
       # This can be removed once openssl integrates with windows
-      # cert store, see http://rt.openssl.org/Ticket/Display.html?id=2158
-      Puppet::Util::Windows::RootCerts.instance.to_a.uniq.each do |x509|
-        begin
-          add_cert(x509)
-        rescue OpenSSL::X509::StoreError => e
-          warn "Failed to add #{x509.subject.to_s}"
+      # cert store, see https://rt.openssl.org/Ticket/Display.html?id=2158
+      unless @puppet_certs_loaded
+        @puppet_certs_loaded = true
+
+        Puppet::Util::Windows::RootCerts.instance.to_a.uniq { |cert| cert.to_der }.each do |x509|
+          begin
+            add_cert(x509)
+          rescue OpenSSL::X509::StoreError => e
+            warn "Failed to add #{x509.subject.to_s}"
+          end
         end
       end
 
@@ -130,3 +79,30 @@ if Puppet::Util::Platform.windows?
     end
   end
 end
+
+# The Enumerable#uniq method was added in Ruby 2.4.0 (https://bugs.ruby-lang.org/issues/11090)
+# This is a backport to earlier Ruby versions.
+#
+unless Enumerable.instance_methods.include?(:uniq)
+  module Enumerable
+    def uniq
+      result = []
+      uniq_map = {}
+      if block_given?
+        each do |value|
+          key = yield value
+          next if uniq_map.has_key?(key)
+          uniq_map[key] = true
+          result << value
+        end
+      else
+        each do |value|
+          next if uniq_map.has_key?(value)
+          uniq_map[value] = true
+          result << value
+        end
+      end
+      result
+    end
+  end
+end

data/lib/puppet/util/multi_match.rb

@@ -0,0 +1,51 @@
+# MultiMatch allows multiple values to be tested at once in a case expression.
+# This class is needed since Array does not implement the === operator to mean
+# "each v === other.each v". 
+#
+# This class is useful in situations when the Puppet Type System cannot be used
+# (e.g. in Logging, since it needs to be able to log very early in the initialization
+# cycle of puppet)
+#
+# Typically used with the constants
+# NOT_NIL
+# TUPLE
+# TRIPLE
+# 
+# which test against single NOT_NIL value, Array with two NOT_NIL, and Array with three NOT_NIL
+#
+module Puppet::Util
+class MultiMatch
+  attr_reader :values
+
+  def initialize(*values)
+    @values = values
+  end
+
+  def ===(other)
+    lv = @values  # local var is faster than instance var
+    case other
+    when MultiMatch
+      return false unless other.values.size == values.size
+      other.values.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    when Array
+      return false unless other.size == values.size
+      other.each_with_index {|v, i| return false unless lv[i] === v || v === lv[i]}
+    else
+      false
+    end
+    true
+  end
+
+  # Matches any value that is not nil using the === operator.
+  #
+  class MatchNotNil
+    def ===(v)
+      !v.nil?
+    end
+  end
+
+  NOT_NIL = MatchNotNil.new().freeze
+  TUPLE = MultiMatch.new(NOT_NIL, NOT_NIL).freeze
+  TRIPLE = MultiMatch.new(NOT_NIL, NOT_NIL, NOT_NIL).freeze
+end
+end

data/lib/puppet/util/network_device/cisco/device.rb

@@ -47,7 +47,13 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
   end
 
   def execute(cmd)
-    transport.command(cmd)
+    transport.command(cmd) do |out|
+      if out =~ /^%/mo or out =~ /^Command rejected:/mo
+        # strip off the command just sent
+        error = out.sub(cmd,'')
+        Puppet.err _("Error while executing '%{cmd}', device returned: %{error}") % { cmd: cmd, error: error }
+      end
+    end
   end
 
   def login
@@ -61,7 +67,7 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
   end
 
   def enable
-    raise "Can't issue \"enable\" to enter privileged, no enable password set" unless enable_password
+    raise _("Can't issue \"enable\" to enter privileged, no enable password set") unless enable_password
     transport.command("enable", :prompt => /^Password:/)
     transport.command(enable_password)
   end
@@ -190,7 +196,7 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
         end
         vlans[vlan[:name]] = vlan
       when /^\s+([a-zA-Z0-9,\/. ]+)\s*$/
-        raise "invalid sh vlan summary output" unless vlan
+        raise _("invalid sh vlan summary output") unless vlan
         if $1.strip.length > 0
           vlan[:interfaces] += $1.strip.split(/\s*,\s*/).map{ |ifn| canonalize_ifname(ifn) }
         end
@@ -202,13 +208,19 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
 
   def update_vlan(id, is = {}, should = {})
     if should[:ensure] == :absent
-      Puppet.info "Removing #{id} from device vlan"
+      Puppet.info _("Removing %{id} from device vlan") % { id: id }
       execute("conf t")
       execute("no vlan #{id}")
       execute("exit")
       return
     end
 
+    # Cisco VLANs are supposed to be alphanumeric only
+    if should[:description] =~ /[^\w]/
+      Puppet.err _("Invalid VLAN name '%{name}' for Cisco device.\nVLAN name must be alphanumeric, no spaces or special characters.") % { name: should[:description] }
+      return
+    end
+    
     # We're creating or updating an entry
     execute("conf t")
     execute("vlan #{id}")
@@ -234,20 +246,26 @@ class Puppet::Util::NetworkDevice::Cisco::Device < Puppet::Util::NetworkDevice::
           trunking[:mode] = :trunk
         when "static access"
           trunking[:mode] = :access
+        when "dynamic auto"
+          trunking[:mode] = 'dynamic auto'
+        when "dynamic desirable"
+          trunking[:mode] = 'dynamic desirable'
         else
-          raise "Unknown switchport mode: #{$1} for #{interface}"
+          raise _("Unknown switchport mode: %{mode} for %{interface}") % { mode: $1, interface: interface }
         end
       when /^Administrative Trunking Encapsulation:\s+(.*)$/
         case $1
         when "dot1q","isl"
-          trunking[:encapsulation] = $1.to_sym if trunking[:mode] == :trunk
+          trunking[:encapsulation] = $1.to_sym if trunking[:mode] != :access
+        when "negotiate"
+          trunking[:encapsulation] = :negotiate
         else
-          raise "Unknown switchport encapsulation: #{$1} for #{interface}"
+          raise _("Unknown switchport encapsulation: %{value} for %{interface}") % { value: $1, interface: interface }
         end
-      when /^Access Mode VLAN:\s+(.*) \(\(Inactive\)\)$/
-        # nothing
-      when /^Access Mode VLAN:\s+(.*) \(.*\)$/
-        trunking[:native_vlan] = $1 if trunking[:mode] == :access
+      when /^Access Mode VLAN:\s+(.*) \((.*)\)$/
+        trunking[:access_vlan] = $1 if $2 != '(Inactive)'
+      when /^Trunking Native Mode VLAN:\s+(.*) \(.*\)$/
+        trunking[:native_vlan] = $1
       when /^Trunking VLANs Enabled:\s+(.*)$/
         next if trunking[:mode] == :access
         vlans = $1