puppet 4.0.0 → 4.10.0

data/lib/puppet/environments.rb

@@ -38,6 +38,14 @@ module Puppet::Environments
         raise EnvironmentNotFound, name
       end
     end
+
+    def clear_all
+      root = Puppet.lookup(:root_environment) { nil }
+      unless root.nil?
+        root.instance_variable_set(:@static_catalogs, nil)
+        root.instance_variable_set(:@rich_data, nil)
+      end
+    end
   end
 
   # @!macro [new] loader_search_paths
@@ -107,7 +115,7 @@ module Puppet::Environments
     def get_conf(name)
       env = get(name)
       if env
-        Puppet::Settings::EnvironmentConf.static_for(env)
+        Puppet::Settings::EnvironmentConf.static_for(env, 0, Puppet[:static_catalogs], Puppet[:rich_data])
       else
         nil
       end
@@ -142,8 +150,10 @@ module Puppet::Environments
     include EnvironmentLoader
 
     def initialize(environment_dir, global_module_path)
-      @environment_dir = environment_dir
-      @global_module_path = global_module_path
+      @environment_dir =  Puppet::FileSystem.expand_path(environment_dir)
+      @global_module_path = global_module_path ?
+        global_module_path.map { |p| Puppet::FileSystem.expand_path(p) } :
+        nil
     end
 
     # Generate an array of directory loaders from a path string.
@@ -190,15 +200,16 @@ module Puppet::Environments
 
     private
 
-    def create_environment(name, setting_values = nil)
+    def create_environment(name)
       env_symbol = name.intern
       setting_values = Puppet.settings.values(env_symbol, Puppet.settings.preferred_run_mode)
-      Puppet::Node::Environment.create(
+      env = Puppet::Node::Environment.create(
         env_symbol,
         Puppet::Node::Environment.split_path(setting_values.interpolate(:modulepath)),
         setting_values.interpolate(:manifest),
         setting_values.interpolate(:config_version)
       )
+      env
     end
 
     def valid_directory?(envdir)
@@ -257,6 +268,9 @@ module Puppet::Environments
       nil
     end
 
+    def clear_all
+      @loaders.each {|loader| loader.clear_all}
+    end
   end
 
   class Cached
@@ -282,10 +296,29 @@ module Puppet::Environments
       @cache_expiration_service || DefaultCacheExpirationService.new
     end
 
+    # Returns the end of time (the next Mesoamerican Long Count cycle-end after 2012 (5125+2012) = 7137,
+    # of for a 32 bit machine using Ruby < 1.9.3, the year 2038.
+    def self.end_of_time
+      begin
+        Time.gm(7137)
+      rescue ArgumentError
+        Time.gm(2038)
+      end
+    end
+
+    END_OF_TIME = end_of_time
+    START_OF_TIME = Time.gm(1)
+
     def initialize(loader)
       @loader = loader
-      @cache = {}
       @cache_expiration_service = Puppet::Environments::Cached.cache_expiration_service
+      @cache = {}
+
+      # Holds expiration times in sorted order - next to expire is first
+      @expirations = SortedSet.new
+
+      # Infinity since it there are no entries, this is a cache of the first to expire time
+      @next_expiration = END_OF_TIME
     end
 
     # @!macro loader_list
@@ -300,15 +333,34 @@ module Puppet::Environments
 
     # @!macro loader_get
     def get(name)
-      evict_if_expired(name)
+      # Aggressively evict all that has expired
+      # This strategy favors smaller memory footprint over environment
+      # retrieval time.
+      clear_all_expired
       if result = @cache[name]
+        # found in cache
         return result.value
       elsif (result = @loader.get(name))
-        @cache[name] = entry(result)
+        # environment loaded, cache it
+        cache_entry = entry(result)
+        @cache_expiration_service.created(result)
+        add_entry(name, cache_entry)
         result
       end
     end
 
+    # Adds a cache entry to the cache
+    def add_entry(name, cache_entry)
+      Puppet.debug {"Caching environment '#{name}' #{cache_entry.label}"}
+      @cache[name] = cache_entry
+      expires = cache_entry.expires
+      @expirations.add(expires)
+      if @next_expiration > expires
+        @next_expiration = expires
+      end
+    end
+    private :add_entry
+
     # Clears the cache of the environment with the given name.
     # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear(name)
@@ -318,7 +370,27 @@ module Puppet::Environments
     # Clears all cached environments.
     # (The intention is that this could be used from a MANUAL cache eviction command (TBD)
     def clear_all()
+      super
       @cache = {}
+      @expirations.clear
+      @next_expiration = END_OF_TIME
+    end
+
+    # Clears all environments that have expired, either by exceeding their time to live, or
+    # through an explicit eviction determined by the cache expiration service.
+    #
+    def clear_all_expired()
+      t = Time.now
+      return if t < @next_expiration && ! @cache.any? {|name, _| @cache_expiration_service.expired?(name.to_sym) }
+      to_expire = @cache.select { |name, entry| entry.expires < t || @cache_expiration_service.expired?(name.to_sym) }
+      to_expire.each do |name, entry|
+        Puppet.debug {"Evicting cache entry for environment '#{name}'"}
+        @cache_expiration_service.evicted(name)
+        clear(name)
+        @expirations.delete(entry.expires)
+        Puppet.settings.clear_environment_settings(name)
+      end
+      @next_expiration = @expirations.first || END_OF_TIME
     end
 
     # This implementation evicts the cache, and always gets the current
@@ -337,9 +409,7 @@ module Puppet::Environments
     # Creates a suitable cache entry given the time to live for one environment
     #
     def entry(env)
-      @cache_expiration_service.created(env)
       ttl = (conf = get_conf(env.name)) ? conf.environment_timeout : Puppet.settings.value(:environment_timeout)
-      Puppet.debug {"Caching environment '#{env.name}' (cache ttl: #{ttl})"}
       case ttl
       when 0
         NotCachedEntry.new(env)     # Entry that is always expired (avoids syscall to get time)
@@ -373,6 +443,14 @@ module Puppet::Environments
       def expired?
         false
       end
+
+      def label
+        ""
+      end
+
+      def expires
+        END_OF_TIME
+      end
     end
 
     # Always evicting entry
@@ -380,6 +458,14 @@ module Puppet::Environments
       def expired?
         true
       end
+
+      def label
+        "(ttl = 0 sec)"
+      end
+
+      def expires
+        START_OF_TIME
+      end
     end
 
     # Time to Live eviction policy entry
@@ -387,11 +473,20 @@ module Puppet::Environments
       def initialize(value, ttl_seconds)
         super value
         @ttl = Time.now + ttl_seconds
+        @ttl_seconds = ttl_seconds
       end
 
       def expired?
         Time.now > @ttl
       end
+
+      def label
+        "(ttl = #{@ttl_seconds} sec)"
+      end
+
+      def expires
+        @ttl
+      end
     end
   end
 end

data/lib/puppet/error.rb

@@ -54,8 +54,34 @@ module Puppet
     include ExternalFileError
   end
 
+  # Contains an issue code and can be annotated with an environment and a node
+  class ParseErrorWithIssue < Puppet::ParseError
+    attr_reader :issue_code, :basic_message
+    attr_accessor :environment, :node
+
+    # @param message [String] The error message
+    # @param file [String] The path to the file where the error was found
+    # @param line [Integer] The line in the file
+    # @param pos [Integer] The position on the line
+    # @param original [Exception] Original exception
+    # @param issue_code [Symbol] The issue code
+    #
+    def initialize(message, file=nil, line=nil, pos=nil, original=nil, issue_code= nil)
+      super(message, file, line, pos, original)
+      @issue_code = issue_code
+      @basic_message = message
+    end
+
+    def to_s
+      msg = super
+      msg = "Could not parse for environment #{environment}: #{msg}" if environment
+      msg = "#{msg} on node #{node}" if node
+      msg
+    end
+  end
+
   # An error that already contains location information in the message text
-  class PreformattedError < Puppet::ParseError
+  class PreformattedError < Puppet::ParseErrorWithIssue
   end
 
   # An error class for when I don't know what happened.  Automatically
@@ -67,4 +93,8 @@ module Puppet
   class MissingCommand < Puppet::Error
   end
 
+  # Raised when we failed to acquire a lock
+  class LockError < Puppet::Error
+  end
+
 end

data/lib/puppet/etc.rb

@@ -0,0 +1,125 @@
+require 'puppet/util/character_encoding'
+# Wrapper around Ruby Etc module allowing us to manage encoding in a single
+# place.
+# This represents a subset of Ruby's Etc module, only the methods required by Puppet.
+# Etc returns strings in variable encoding depending on
+# environment. For Puppet we specifically want UTF-8 as our input from the Etc
+# module - which is our input for many resource instance 'is' values. The
+# associated 'should' value will basically always be coming from Puppet in
+# UTF-8. Etc is defined for Windows but calls to it return nil.
+#
+# We only use Etc for retrieving existing property values from the system. For
+# setting property values, providers leverage system tools (i.e., `useradd`)
+#
+# @api private
+module Puppet::Etc
+  class << self
+    # Etc::getgrent returns an Etc::Group struct object
+    # On first call opens /etc/group and returns parse of first entry. Each subsquent call
+    # returns new struct the next entry or nil if EOF. Call ::endgrent to close file.
+    def getgrent
+      convert_field_values_to_utf8!(::Etc.getgrent)
+    end
+
+    # closes handle to /etc/group file
+    def endgrent
+      ::Etc.endgrent
+    end
+
+    # effectively equivalent to IO#rewind of /etc/group
+    def setgrent
+      ::Etc.setgrent
+    end
+
+    # Etc::getpwent returns an Etc::Passwd struct object
+    # On first call opens /etc/passwd and returns parse of first entry. Each subsquent call
+    # returns new struct for the next entry or nil if EOF. Call ::endgrent to close file.
+    def getpwent
+      convert_field_values_to_utf8!(::Etc.getpwent)
+    end
+
+    # closes handle to /etc/passwd file
+    def endpwent
+      ::Etc.endpwent
+    end
+
+    #effectively equivalent to IO#rewind of /etc/passwd
+    def setpwent
+      ::Etc.setpwent
+    end
+
+    # Etc::getpwnam searches /etc/passwd file for an entry corresponding to
+    # username.
+    # returns an Etc::Passwd struct corresponding to the entry or raises
+    # ArgumentError if none
+    def getpwnam(username)
+      convert_field_values_to_utf8!(::Etc.getpwnam(username))
+    end
+
+    # Etc::getgrnam searches /etc/group file for an entry corresponding to groupname.
+    # returns an Etc::Group struct corresponding to the entry or raises
+    # ArgumentError if none
+    def getgrnam(groupname)
+      convert_field_values_to_utf8!(::Etc.getgrnam(groupname))
+    end
+
+    # Etc::getgrid searches /etc/group file for an entry corresponding to id.
+    # returns an Etc::Group struct corresponding to the entry or raises
+    # ArgumentError if none
+    def getgrgid(id)
+      convert_field_values_to_utf8!(::Etc.getgrgid(id))
+    end
+
+    # Etc::getpwuid searches /etc/passwd file for an entry corresponding to id.
+    # returns an Etc::Passwd struct corresponding to the entry or raises
+    # ArgumentError if none
+    def getpwuid(id)
+      convert_field_values_to_utf8!(::Etc.getpwuid(id))
+    end
+
+    private
+    # Utility method for converting the String values of a struct returned by
+    # the Etc module to UTF-8. Structs returned by the ruby Etc module contain
+    # members with fields of type String, Integer, or Array of Strings, so we
+    # handle these types. Otherwise ignore fields.
+    #
+    # NOTE: If a string cannot be converted to UTF-8, this leaves the original
+    # string string intact in the Struct.
+    #
+    # Warning! This is a destructive method - the struct passed is modified!
+    #
+    # @api private
+    # @param [Etc::Passwd or Etc::Group struct]
+    # @return [Etc::Passwd or Etc::Group struct] the original struct with values
+    #   converted to UTF-8 if possible, or the original value intact if not
+    def convert_field_values_to_utf8!(struct)
+      return nil if struct.nil?
+      struct.each_with_index do |value, index|
+        if value.is_a?(String)
+            converted = Puppet::Util::CharacterEncoding.convert_to_utf_8!(value)
+            struct[index] = converted if !converted.nil?
+        elsif value.is_a?(Array)
+          struct[index] = convert_array_values_to_utf8!(value)
+        end
+      end
+    end
+
+    # Helper method for ::convert_field_values_to_utf8!
+    #
+    # Warning! This is a destructive method - the array passed is modified!
+    #
+    # @api private
+    # @param [Array] object containing String values to convert to UTF-8
+    # @return [Array] original Array with String values converted to UTF-8 if
+    #   convertible, or original, unmodified values if not.
+    def convert_array_values_to_utf8!(string_array)
+      string_array.map! do |elem|
+        converted = Puppet::Util::CharacterEncoding.convert_to_utf_8!(elem)
+        converted.nil? ? elem : converted
+      end
+    end
+  end
+end
+
+
+

data/lib/puppet/external/dot.rb

@@ -8,7 +8,7 @@
 
 module DOT
 
-  # These glogal vars are used to make nice graph source.
+  # These global vars are used to make nice graph source.
 
   $tab  = '    '
   $tab2 = $tab * 2

data/lib/puppet/face/ca.rb

@@ -178,7 +178,10 @@ Puppet::Face.define(:ca, '0.1.0') do
       Puppet::SSL::Host.ca_location = :only
 
       begin
-        ca.sign(host, options[:allow_dns_alt_names])
+        signing_options = options.select { |k,_|
+          [:allow_dns_alt_names, :allow_authorization_extensions].include?(k)
+        }
+        ca.sign(host, signing_options)
       rescue ArgumentError => e
         if e.to_s =~ /Could not find certificate request/
           e.to_s
@@ -251,4 +254,6 @@ Puppet::Face.define(:ca, '0.1.0') do
       end
     end
   end
+
+  deprecate
 end

data/lib/puppet/face/catalog.rb

@@ -123,7 +123,7 @@ Puppet::Indirector::Face.define(:catalog, '0.0.1') do
 
       Puppet::Resource::Catalog.indirection.terminus_class = :yaml
       Puppet::Face[:catalog, "0.0.1"].save(catalog)
-      Puppet.notice "Saved catalog for #{Puppet[:certname]} to yaml"
+      Puppet.notice "Saved catalog for #{Puppet[:certname]} to #{Puppet::Resource::Catalog.indirection.terminus.path(Puppet[:certname])}"
       nil
     end
   end

data/lib/puppet/face/certificate.rb

@@ -77,6 +77,10 @@ Puppet::Indirector::Face.define(:certificate, '0.0.1') do
       # them. Otherwise, they will default to the config file setting iff this
       # cert is for the host we're running on.
 
+      unless Puppet::FileSystem.exist?(Puppet[:hostcert])
+        Puppet.push_context({:ssl_host => host})
+      end
+
       host.generate_certificate_request(:dns_alt_names => options[:dns_alt_names])
     end
   end
@@ -129,7 +133,9 @@ Puppet::Indirector::Face.define(:certificate, '0.0.1') do
           raise ArgumentError, "This process is not configured as a certificate authority"
         end
 
-        ca.sign(name, options[:allow_dns_alt_names])
+        signing_options = {allow_dns_alt_names: options[:allow_dns_alt_names]}
+
+        ca.sign(name, signing_options)
       end
     end
   end

data/lib/puppet/face/certificate_request.rb

@@ -51,4 +51,6 @@ Puppet::Indirector::Face.define(:certificate_request, '0.0.1') do
 
   get_action(:save).summary "API only: submit a certificate signing request."
   get_action(:save).arguments "<x509_CSR>"
+
+  deprecate
 end

data/lib/puppet/face/certificate_revocation_list.rb

@@ -51,4 +51,6 @@ Puppet::Indirector::Face.define(:certificate_revocation_list, '0.0.1') do
 
   deactivate_action(:search)
   deactivate_action(:save)
+
+  deprecate
 end

data/lib/puppet/face/config.rb

@@ -9,7 +9,7 @@ Puppet::Face.define(:config, '0.0.1') do
 
   description "This subcommand can inspect and modify settings from Puppet's
     'puppet.conf' configuration file. For documentation about individual settings,
-    see http://docs.puppetlabs.com/references/latest/configuration.html."
+    see https://docs.puppetlabs.com/puppet/latest/reference/configuration.html."
 
   option "--section SECTION_NAME" do
     default_to { "main" }
@@ -108,7 +108,7 @@ Puppet::Face.define(:config, '0.0.1') do
     when_invoked do |name, value, options|
       path = Puppet::FileSystem.pathname(Puppet.settings.which_configuration_file)
       Puppet::FileSystem.touch(path)
-      Puppet::FileSystem.open(path, nil, 'r+') do |file|
+      Puppet::FileSystem.open(path, nil, 'r+:UTF-8') do |file|
         Puppet::Settings::IniFile.update(file) do |config|
           config.set(options[:section], name, value)
         end

data/lib/puppet/face/epp.rb

@@ -62,6 +62,8 @@ Puppet::Face.define(:epp, '0.0.1') do
     EOT
     when_invoked do |*args|
       options = args.pop
+      # pass a dummy node, as facts are not needed for validation
+      options[:node] = node = Puppet::Node.new("testnode", :facts => Puppet::Node::Facts.new("facts", {}))
       compiler = create_compiler(options)
 
       status = true # no validation error yet
@@ -72,7 +74,7 @@ Puppet::Face.define(:epp, '0.0.1') do
           status &&= tmp
         else
           # This is not an error since a validate of all files in an empty
-          # directry should not be treated as a failed validation.
+          # directory should not be treated as a failed validation.
           Puppet.notice "No template specified. No action taken"
         end
       end
@@ -115,10 +117,10 @@ Puppet::Face.define(:epp, '0.0.1') do
       If no arguments are given, the stdin is read (unless it is attached to a terminal)
 
       If multiple templates are given, they are separated with a header indicating the
-      name of the template. This can be surpressed with the option --no-header.
-      The option --[no-]header has no effect whe a single template is dumped.
+      name of the template. This can be suppressed with the option --no-header.
+      The option --[no-]header has no effect when a single template is dumped.
 
-      When debugging the epp parser itself, it may be useful to surpress the valiation
+      When debugging the epp parser itself, it may be useful to suppress the validation
       step with the `--no-validate` option to observe what the parser produced from the
       given source.
 
@@ -141,6 +143,8 @@ Puppet::Face.define(:epp, '0.0.1') do
     when_invoked do |*args|
       require 'puppet/pops'
       options = args.pop
+      # pass a dummy node, as facts are not needed for dump
+      options[:node] = node = Puppet::Node.new("testnode", :facts => Puppet::Node::Facts.new("facts", {}))
       options[:header] = options[:header].nil? ? true : options[:header]
       options[:validate] = options[:validate].nil? ? true : options[:validate]
 
@@ -172,7 +176,7 @@ Puppet::Face.define(:epp, '0.0.1') do
 
         show_filename = templates.count > 1
         dumps = templates.each do |file|
-          buffer.print dump_parse(File.read(file), file, options, show_filename)
+          buffer.print dump_parse(Puppet::FileSystem.read(file, :encoding => 'utf-8'), file, options, show_filename)
         end
 
         if !missing_files.empty?
@@ -224,8 +228,12 @@ Puppet::Face.define(:epp, '0.0.1') do
       showing the name of the template before the output. The header output can be turned off with
       `--no-header`. This also concatenates the template results without any added newline separators.
 
-      Facts for the simulated node can be feed to the rendering process by referencing a .yaml file
-      with facts using the --facts option. (Values can be obtained in yaml format directly from
+      Facts from the node where the command is being run are used by default.args Facts can be obtained
+      for other nodes if they have called in, and reported their facts by using the `--node <nodename>`
+      flag.
+
+      Overriding node facts as well as additional facts can be given in a .yaml or .json file and referencing
+      it with the --facts option. (Values can be obtained in yaml format directly from
       `facter`, or from puppet for a given node). Note that it is not possible to simulate the
       reserved variable name `$facts` in any other way.
 
@@ -238,7 +246,7 @@ Puppet::Face.define(:epp, '0.0.1') do
 
       When designing EPP templates, it is strongly recommended to define all template arguments
       in the template, and to give them in a hash when calling `epp` or `inline_epp` and to use
-      as few global variables as possible, preferrably only the $facts hash. This makes templates
+      as few global variables as possible, preferably only the $facts hash. This makes templates
       more free standing and are easier to reuse, and to test.
     EOT
 
@@ -278,6 +286,10 @@ Puppet::Face.define(:epp, '0.0.1') do
           $ puppet epp render -e '<% $facts[osfamily] %>' --facts data.yaml
     EOT
 
+    option("--node <node_name>") do
+      summary "The name of the node for which facts are obtained. Defaults to facts for the local node."
+    end
+
     option "--e <source>" do
       default_to { nil }
       summary "Render one inline epp template given on the command line."
@@ -291,8 +303,8 @@ Puppet::Face.define(:epp, '0.0.1') do
       summary "A .pp or .yaml file that is processed to produce a hash of values for the template."
     end
 
-    option("--facts <yaml_file>") do
-      summary "A .yaml file containing a hash of facts made available in $facts"
+    option("--facts <facts_file>") do
+      summary "A .yaml or .json file containing a hash of facts made available in $facts and $trusted"
     end
 
     option("--[no-]header") do
@@ -444,16 +456,50 @@ Puppet::Face.define(:epp, '0.0.1') do
 
   # @api private
   def create_compiler(options)
-    fact_values = options[:facts] ? YAML.load_file(options[:facts]) : {}
-    node = Puppet::Node.new("testnode", :facts => Puppet::Node::Facts.new("facts", fact_values))
+    if options[:node]
+      node = options[:node]
+    else
+      node = Puppet[:node_name_value]
+
+      # If we want to lookup the node we are currently on
+      # we must returning these settings to their default values
+      Puppet.settings[:facts_terminus] = 'facter'
+      Puppet.settings[:node_cache_terminus] = nil
+    end
+
+    unless node.is_a?(Puppet::Node)
+      node = Puppet::Node.indirection.find(node)
+      # Found node must be given the environment to use in some cases, use the one configured
+      # or given on the command line
+      node.environment = Puppet[:environment]
+    end
+
+    fact_file = options[:facts]
+
+    if fact_file
+      if fact_file.is_a?(Hash) # when used via the Face API
+        given_facts = fact_file
+      elsif fact_file.end_with?("json")
+        given_facts = JSON.parse(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
+      else
+        given_facts = YAML.load(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
+      end
+
+      unless given_facts.instance_of?(Hash)
+        raise "Incorrect formatted data in #{fact_file} given via the --facts flag"
+      end
+      # It is difficult to add to or modify the set of facts once the node is created
+      # as changes does not show up in parameters. Rather than manually patching up
+      # a node and risking future regressions, a new node is created from scratch
+      node = Puppet::Node.new(node.name, :facts => Puppet::Node::Facts.new("facts", node.facts.values.merge(given_facts)))
+      node.environment = Puppet[:environment]
+      node.merge(node.facts.values)
+    end
+
     compiler = Puppet::Parser::Compiler.new(node)
     # configure compiler with facts and node related data
     # Set all global variables from facts
-    fact_values.each {|param, value| compiler.topscope[param] = value }
-    # Configured trusted data (even if there are none)
-    compiler.topscope.set_trusted(node.trusted_data)
-    # Set the facts hash
-    compiler.topscope.set_facts(fact_values)
+    compiler.send(:set_node_parameters)
 
     # pretend that the main class (named '') has been evaluated
     # since it is otherwise not possible to resolve top scope variables

data/lib/puppet/face/file.rb

@@ -44,4 +44,7 @@ Puppet::Indirector::Face.define(:file, '0.0.1') do
   deactivate_action(:destroy)
 
   set_indirection_name :file_bucket_file
+
+  # The file face is deprecated
+  deprecate
 end