puppet 4.4.2 → 4.5.0

data/lib/puppet/pops/validation.rb

@@ -126,7 +126,7 @@ module Validation
     #
     def []=(issue, level)
       raise Puppet::DevError.new("Attempt to set validation severity for something that is not an Issue. (Got #{issue.class})") unless issue.is_a? Issues::Issue
-      raise Puppet::DevError.new("Illegal severity level: #{option}") unless @@severity_hash[level]
+      raise Puppet::DevError.new("Illegal severity level: #{level} for '#{issue.issue_code}'") unless @@severity_hash[level]
       raise Puppet::DevError.new("Attempt to demote the hard issue '#{issue.issue_code}' to #{level}") unless issue.demotable? || level == :error
       @severities[issue] = level
     end

data/lib/puppet/pops/validation/checker4_0.rb

@@ -11,7 +11,7 @@ module Validation
 #       Model objects via their metamodel. (I.e an extra call to multiplicity check in polymorph check).
 #       This is however mostly valuable when validating model to model transformations, and is therefore T.B.D
 #
-class Checker4_0
+class Checker4_0 < Evaluator::LiteralEvaluator
   attr_reader :acceptor
   attr_reader :migration_checker
 
@@ -19,6 +19,7 @@ class Checker4_0
   # `:will_accept?` and `:accept`.
   #
   def initialize(diagnostics_producer)
+    super()
     @@check_visitor       ||= Visitor.new(nil, "check", 0, 0)
     @@rvalue_visitor      ||= Visitor.new(nil, "rvalue", 0, 0)
     @@hostname_visitor    ||= Visitor.new(nil, "hostname", 1, 2)
@@ -235,7 +236,13 @@ class Checker4_0
   end
 
   def check_CallNamedFunctionExpression(o)
-    case o.functor_expr
+    functor = o.functor_expr
+    if functor.is_a?(Model::QualifiedReference) || 
+      functor.is_a?(Model::AccessExpression) && functor.left_expr.is_a?(Model::QualifiedReference)
+      # ok (a call to a type)
+      return nil
+    end
+    case functor
     when Model::QualifiedName
       # ok
       nil
@@ -355,6 +362,50 @@ class Checker4_0
     internal_check_type_ref(o, o.type_expr)
   end
 
+  def check_TypeMapping(o)
+    top(o.eContainer, o)
+    lhs = o.type_expr
+    lhs_type = 0 # Not Runtime
+    if lhs.is_a?(Model::AccessExpression)
+      left = lhs.left_expr
+      if left.is_a?(Model::QualifiedReference) && left.cased_value == 'Runtime'
+        lhs_type = 1 # Runtime
+        keys = lhs.keys
+
+        # Must be a literal string or pattern replacement
+        lhs_type = 2 if keys.size == 2 && pattern_with_replacement?(keys[1])
+      end
+    end
+
+    if lhs_type == 0
+      # This is not a TypeMapping. Something other than Runtime[] on LHS
+      acceptor.accept(Issues::UNSUPPORTED_EXPRESSION, o)
+    else
+      rhs = o.mapping_expr
+      if pattern_with_replacement?(rhs)
+        acceptor.accept(Issues::ILLEGAL_SINGLE_TYPE_MAPPING, o) if lhs_type == 1
+      elsif type_ref?(rhs)
+        acceptor.accept(Issues::ILLEGAL_REGEXP_TYPE_MAPPING, o) if lhs_type == 2
+      else
+        acceptor.accept(lhs_type == 1 ? Issues::ILLEGAL_SINGLE_TYPE_MAPPING : Issues::ILLEGAL_REGEXP_TYPE_MAPPING, o)
+      end
+    end
+  end
+
+  def pattern_with_replacement?(o)
+    if o.is_a?(Model::LiteralList)
+      v = o.values
+      v.size == 2 && v[0].is_a?(Model::LiteralRegularExpression) && v[1].is_a?(Model::LiteralString)
+    else
+      false
+    end
+  end
+
+  def type_ref?(o)
+    o = o.left_expr if o.is_a?(Model::AccessExpression)
+    o.is_a?(Model::QualifiedReference)
+  end
+
   def check_TypeDefinition(o)
     top(o.eContainer, o)
     internal_check_reserved_type_name(o, o.name)
@@ -467,6 +518,17 @@ class Checker4_0
     o.values.each {|v| rvalue(v) }
   end
 
+  def check_LiteralHash(o)
+    # the keys of a literal hash may be non-literal expressions. They cannot be checked.
+    unique = Set.new
+    o.entries.each do |entry|
+      catch(:not_literal) do
+        literal_key = literal(entry.key)
+        acceptor.accept(Issues::DUPLICATE_KEY, entry, {:key => literal_key}) if unique.add?(literal_key).nil?
+      end
+    end
+  end
+
   def check_NodeDefinition(o)
     # Check that hostnames are valid hostnames (or regular expressions)
     hostname(o.host_matches, o)
@@ -491,8 +553,8 @@ class Checker4_0
   # DOH: QualifiedReferences are created with LOWER CASE NAMES at parse time
   def check_QualifiedReference(o)
     # Is this a valid qualified name?
-    if o.value !~ Patterns::CLASSREF
-      acceptor.accept(Issues::ILLEGAL_CLASSREF, o, {:name=>o.value})
+    if o.cased_value !~ Patterns::CLASSREF_EXT
+      acceptor.accept(Issues::ILLEGAL_CLASSREF, o, {:name=>o.cased_value})
     end
   end
 

data/lib/puppet/pops/validation/validator_factory_4_0.rb

@@ -27,6 +27,7 @@ class ValidatorFactory_4_0 < Factory
 
     p[Issues::FUTURE_RESERVED_WORD]          = :deprecation
 
+    p[Issues::DUPLICATE_KEY]                 = Puppet[:strict] == :off ? :ignore : Puppet[:strict]
     p[Issues::NAME_WITH_HYPHEN]              = :error
     p[Issues::EMPTY_RESOURCE_SPECIALIZATION] = :ignore
     p

data/lib/puppet/pops/visitor.rb

@@ -40,7 +40,9 @@ class Puppet::Pops::Visitor
       return receiver.send(method_name, thing, *args)
     else
       thing.class.ancestors().each do |ancestor|
-        method_name = :"#{@message}_#{ancestor.name.split(DOUBLE_COLON).last}"
+        name = ancestor.name
+        next if name.nil?
+        method_name = :"#{@message}_#{name.split(DOUBLE_COLON).last}"
         next unless receiver.respond_to?(method_name, true)
         @cache[thing.class] = method_name
         return receiver.send(method_name, thing, *args)

data/lib/puppet/property.rb

@@ -521,7 +521,7 @@ class Puppet::Property < Puppet::Parameter
     if features = self.class.value_option(self.class.value_name(value), :required_features)
       features = Array(features)
       needed_features = features.collect { |f| f.to_s }.join(", ")
-      raise ArgumentError, "Provider must have features '#{needed_features}' to set '#{self.class.name}' to '#{value}'" unless provider.satisfies?(features)
+      raise ArgumentError, "Provider #{provider.class.name} must have features '#{needed_features}' to set '#{self.class.name}' to '#{value}'" unless provider.satisfies?(features)
     end
   end
 

data/lib/puppet/provider/augeas/augeas.rb

@@ -425,7 +425,7 @@ Puppet::Type.type(:augeas).provide(:augeas) do
       end
 
       unless force
-        # If we have a verison of augeas which is at least 0.3.6 then we
+        # If we have a version of augeas which is at least 0.3.6 then we
         # can make the changes now and see if changes were made.
         if return_value and versioncmp(get_augeas_version, "0.3.6") >= 0
           debug("Will attempt to save and only run if files changed")

data/lib/puppet/provider/package/pip.rb

@@ -30,7 +30,7 @@ Puppet::Type.type(:package).provide :pip,
   # that's managed by `pip` or an empty array if `pip` is not available.
   def self.instances
     packages = []
-    pip_cmd = cmd.map { |c| which(c) }.find { |c| c != nil }
+    pip_cmd = self.pip_cmd
     return [] unless pip_cmd
     execpipe "#{pip_cmd} freeze" do |process|
       process.collect do |line|
@@ -38,6 +38,13 @@ Puppet::Type.type(:package).provide :pip,
         packages << new(options)
       end
     end
+
+    # Pip can also upgrade pip, but it's not listed in freeze so need to special case it
+    # Pip list would also show pip installed version, but "pip list" doesn't exist for older versions of pip (E.G v1.0)
+    if version = self.pip_version
+      packages << new({:ensure => version, :name => File.basename(pip_cmd), :provider => name})
+    end
+
     packages
   end
 
@@ -45,6 +52,21 @@ Puppet::Type.type(:package).provide :pip,
     ["pip", "pip-python"]
   end
 
+  def self.pip_cmd
+    self.cmd.map { |c| which(c) }.find { |c| c != nil }
+  end
+
+  def self.pip_version
+    pip_cmd = self.pip_cmd
+    return nil unless pip_cmd
+
+    execpipe [pip_cmd, '--version'] do |process|
+      process.collect do |line|
+        return line.strip.match(/^pip (\d+\.\d+\.?\d*).*$/)[1]
+      end
+    end
+  end
+
   # Return structured information about a particular package or `nil` if
   # it is not installed or `pip` itself is not available.
   def query
@@ -54,26 +76,13 @@ Puppet::Type.type(:package).provide :pip,
     return nil
   end
 
-  # Ask the PyPI API for the latest version number.  There is no local
-  # cache of PyPI's package list so this operation will always have to
-  # ask the web service.
+  # Use pip CLI to look up versions from PyPI repositories, honoring local pip config such as custom repositories
   def latest
-    http_proxy_host = Puppet::Util::HttpProxy.http_proxy_host
-    http_proxy_port = Puppet::Util::HttpProxy.http_proxy_port
-    if http_proxy_host && http_proxy_port
-      proxy = "#{http_proxy_host}:#{http_proxy_port}"
-    else
-      # nil is acceptable
-      proxy = http_proxy_host
+    return nil unless self.class.pip_cmd
+    if Puppet::Util::Package.versioncmp(self.class.pip_version, '1.5.4') == -1 # a < b
+      return latest_with_old_pip
     end
-
-    client = XMLRPC::Client.new2("http://pypi.python.org/pypi", proxy)
-    client.http_header_extra = {"Content-Type" => "text/xml"}
-    client.timeout = 10
-    result = client.call("package_releases", @resource[:name])
-    result.first
-  rescue Timeout::Error => detail
-    raise Puppet::Error, "Timeout while contacting pypi.python.org: #{detail}", detail.backtrace
+    latest_with_new_pip
   end
 
   # Install a package.  The ensure parameter may specify installed,
@@ -120,8 +129,14 @@ Puppet::Type.type(:package).provide :pip,
   def lazy_pip(*args)
     pip *args
   rescue NoMethodError => e
+    # Ensure pip can upgrade pip, which usually puts pip into a new path /usr/local/bin/pip (compared to /usr/bin/pip)
+    # The path to pip needs to be looked up again in the subsequent request. Using the preferred approach as noted
+    # in provider.rb ensures this (copied below for reference)
+    #
+    # @note From provider.rb; It is preferred if the commands are not entered with absolute paths as this allows puppet
+    # to search for them using the PATH variable.
     if pathname = self.class.cmd.map { |c| which(c) }.find { |c| c != nil }
-      self.class.commands :pip => pathname
+      self.class.commands :pip => File.basename(pathname)
       pip *args
     else
       raise e, "Could not locate command #{self.class.cmd.join(' and ')}.", e.backtrace
@@ -131,4 +146,33 @@ Puppet::Type.type(:package).provide :pip,
   def install_options
     join_options(@resource[:install_options])
   end
+
+  def latest_with_new_pip
+    # Less resource intensive approach for pip version 1.5.4 and above
+    execpipe ["#{self.class.pip_cmd}", "install", "#{@resource[:name]}==versionplease"] do |process|
+      process.collect do |line|
+        # PIP OUTPUT: Could not find a version that satisfies the requirement Django==versionplease (from versions: 1.1.3, 1.8rc1)
+        if line =~ /from versions: /
+          textAfterLastMatch = $'
+          versionList = textAfterLastMatch.chomp(")\n").split(', ')
+          return versionList.last
+        end
+      end
+      return nil
+    end
+  end
+
+  def latest_with_old_pip
+    Dir.mktmpdir("puppet_pip") do |dir|
+      execpipe ["#{self.class.pip_cmd}", "install", "#{@resource[:name]}", "-d", "#{dir}", "-v"] do |process|
+        process.collect do |line|
+          # PIP OUTPUT: Using version 0.10.1 (newest of versions: 0.10.1, 0.10, 0.9, 0.8.1, 0.8, 0.7.2, 0.7.1, 0.7, 0.6.1, 0.6, 0.5.2, 0.5.1, 0.5, 0.4, 0.3.1, 0.3, 0.2, 0.1)
+          if line =~ /Using version (.+?) \(newest of versions/
+            return $1
+          end
+        end
+        return nil
+      end
+    end
+  end
 end

data/lib/puppet/provider/package/rpm.rb

@@ -22,6 +22,31 @@ Puppet::Type.type(:package).provide :rpm, :source => :rpm, :parent => Puppet::Pr
   self::NEVRA_REGEX  = %r{^(\S+) (\S+) (\S+) (\S+) (\S+)$}
   self::NEVRA_FIELDS = [:name, :epoch, :version, :release, :arch]
 
+  ARCH_LIST = [
+    'noarch',
+    'i386',
+    'i686',
+    'ppc',
+    'ppc64',
+    'armv3l',
+    'armv4b',
+    'armv4l',
+    'armv4tl',
+    'armv5tel',
+    'armv5tejl',
+    'armv6l',
+    'armv7l',
+    'm68kmint',
+    's390',
+    's390x',
+    'ia64',
+    'x86_64',
+    'sh3',
+    'sh4',
+  ]
+
+  ARCH_REGEX = Regexp.new(ARCH_LIST.join('|\.'))
+
   commands :rpm => "rpm"
 
   if command('rpm')
@@ -268,6 +293,92 @@ Puppet::Type.type(:package).provide :rpm, :source => :rpm, :parent => Puppet::Pr
     end
   end
 
+  def insync?(is)
+    return false if [:purged, :absent].include?(is)
+    should = resource[:ensure]
+    0 == rpm_compareEVR(rpm_parse_evr(should), rpm_parse_evr(is))
+  end
+
+  # parse a rpm "version" specification
+  # this re-implements rpm's
+  # rpmUtils.miscutils.stringToVersion() in ruby
+  def rpm_parse_evr(s)
+    ei = s.index(':')
+    if ei
+      e = s[0,ei]
+      s = s[ei+1,s.length]
+    else
+      e = nil
+    end
+    begin
+      e = String(Integer(e))
+    rescue
+      # If there are non-digits in the epoch field, default to nil
+      e = nil
+    end
+    ri = s.index('-')
+    if ri
+      v = s[0,ri]
+      r = s[ri+1,s.length]
+      if arch = r.scan(ARCH_REGEX)[0]
+        a = arch.gsub(/\./, '')
+	r.gsub!(ARCH_REGEX, '')
+      end
+    else
+      v = s
+      r = nil
+    end
+    return { :epoch => e, :version => v, :release => r, :arch => a }
+  end
+
+  # how rpm compares two package versions:
+  # rpmUtils.miscutils.compareEVR(), which massages data types and then calls
+  # rpm.labelCompare(), found in rpm.git/python/header-py.c, which
+  # sets epoch to 0 if null, then compares epoch, then ver, then rel
+  # using compare_values() and returns the first non-0 result, else 0.
+  # This function combines the logic of compareEVR() and labelCompare().
+  #
+  # "version_should" can be v, v-r, or e:v-r.
+  # "version_is" will always be at least v-r, can be e:v-r
+  def rpm_compareEVR(should_hash, is_hash)
+    # pass on to rpm labelCompare
+
+    if !should_hash[:epoch].nil?
+      rc = compare_values(should_hash[:epoch], is_hash[:epoch])
+      return rc unless rc == 0
+    end
+
+    rc = compare_values(should_hash[:version], is_hash[:version])
+    return rc unless rc == 0
+
+    # here is our special case, PUP-1244.
+    # if should_hash[:release] is nil (not specified by the user),
+    # and comparisons up to here are equal, return equal. We need to
+    # evaluate to whatever level of detail the user specified, so we
+    # don't end up upgrading or *downgrading* when not intended.
+    #
+    # This should NOT be triggered if we're trying to ensure latest.
+    return 0 if should_hash[:release].nil?
+
+    rc = compare_values(should_hash[:release], is_hash[:release])
+
+    return rc
+  end
+
+  # this method is a native implementation of the
+  # compare_values function in rpm's python bindings,
+  # found in python/header-py.c, as used by rpm.
+  def compare_values(s1, s2)
+    if s1.nil? && s2.nil?
+      return 0
+    elsif ( not s1.nil? ) && s2.nil?
+      return 1
+    elsif s1.nil? && (not s2.nil?)
+      return -1
+    end
+    return rpmvercmp(s1, s2)
+  end
+
   private
   # @param line [String] one line of rpm package query information
   # @return [Hash] of NEVRA_FIELDS strings parsed from package info
@@ -281,6 +392,7 @@ Puppet::Type.type(:package).provide :rpm, :source => :rpm, :parent => Puppet::Pr
       self::NEVRA_FIELDS.zip(match.captures) { |f, v| hash[f] = v }
       hash[:provider] = self.name
       hash[:ensure] = "#{hash[:version]}-#{hash[:release]}"
+      hash[:ensure].prepend("#{hash[:epoch]}:") if hash[:epoch] != '0'
     else
       Puppet.debug("Failed to match rpm line #{line}")
     end

data/lib/puppet/provider/package/yum.rb

@@ -149,8 +149,13 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
     else
       # Add the package version
       wanted += "-#{should}"
+      if wanted.scan(ARCH_REGEX)
+        self.debug "Detected Arch argument in package! - Moving arch to end of version string"
+        wanted.gsub!(/(.+)(#{ARCH_REGEX})(.+)/,'\1\3\2')
+      end
+
       is = self.query
-      if is && yum_compareEVR(yum_parse_evr(should), yum_parse_evr(is[:ensure])) < 0
+      if is && rpm_compareEVR(rpm_parse_evr(should), rpm_parse_evr(is[:ensure])) < 0
         self.debug "Downgrading package #{@resource[:name]} from version #{is[:ensure]} to #{should}"
         operation = :downgrade
       end
@@ -173,7 +178,7 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
 
       # FIXME: Should we raise an exception even if should == :latest
       # and yum updated us to a version other than @param_hash[:ensure] ?
-      vercmp_result = yum_compareEVR(yum_parse_evr(should), yum_parse_evr(is[:ensure]))
+      vercmp_result = rpm_compareEVR(rpm_parse_evr(should), rpm_parse_evr(is[:ensure]))
       raise Puppet::Error, "Failed to update to version #{should}, got version #{is[:ensure]} instead" if vercmp_result != 0
     end
   end
@@ -202,72 +207,6 @@ Puppet::Type.type(:package).provide :yum, :parent => :rpm, :source => :rpm do
     execute([command(:cmd), "-y", :erase, @resource[:name]])
   end
 
-  # parse a yum "version" specification
-  # this re-implements yum's
-  # rpmUtils.miscutils.stringToVersion() in ruby
-  def yum_parse_evr(s)
-    ei = s.index(':')
-    if ei
-      e = s[0,ei]
-      s = s[ei+1,s.length]
-    else
-      e = nil
-    end
-    e = String(Bignum(e)) rescue '0'
-    ri = s.index('-')
-    if ri
-      v = s[0,ri]
-      r = s[ri+1,s.length]
-    else
-      v = s
-      r = nil
-    end
-    return { :epoch => e, :version => v, :release => r }
-  end
-
-  # how yum compares two package versions:
-  # rpmUtils.miscutils.compareEVR(), which massages data types and then calls
-  # rpm.labelCompare(), found in rpm.git/python/header-py.c, which
-  # sets epoch to 0 if null, then compares epoch, then ver, then rel
-  # using compare_values() and returns the first non-0 result, else 0.
-  # This function combines the logic of compareEVR() and labelCompare().
-  #
-  # "version_should" can be v, v-r, or e:v-r.
-  # "version_is" will always be at least v-r, can be e:v-r
-  def yum_compareEVR(should_hash, is_hash)
-    # pass on to rpm labelCompare
-    rc = compare_values(should_hash[:epoch], is_hash[:epoch])
-    return rc unless rc == 0
-    rc = compare_values(should_hash[:version], is_hash[:version])
-    return rc unless rc == 0
-
-    # here is our special case, PUP-1244.
-    # if should_hash[:release] is nil (not specified by the user),
-    # and comparisons up to here are equal, return equal. We need to
-    # evaluate to whatever level of detail the user specified, so we
-    # don't end up upgrading or *downgrading* when not intended.
-    #
-    # This should NOT be triggered if we're trying to ensure latest.
-    return 0 if should_hash[:release].nil?
-
-    rc = compare_values(should_hash[:release], is_hash[:release])
-    return rc
-  end
-
-  # this method is a native implementation of the
-  # compare_values function in rpm's python bindings,
-  # found in python/header-py.c, as used by yum.
-  def compare_values(s1, s2)
-    if s1.nil? && s2.nil?
-      return 0
-    elsif ( not s1.nil? ) && s2.nil?
-      return 1
-    elsif s1.nil? && (not s2.nil?)
-      return -1
-    end
-    return rpmvercmp(s1, s2)
-  end
-
   private
 
   def enablerepo