puppet 3.5.0.rc1-x86-mingw32

data/lib/puppet/graph/title_hash_prioritizer.rb

@@ -0,0 +1,16 @@
+# Prioritize keys, which must be Puppet::Resources, based on a static hash of
+# the key's ref. This prioritizer does not take containment into account.
+#
+# @api private
+require 'digest/sha1'
+
+class Puppet::Graph::TitleHashPrioritizer < Puppet::Graph::Prioritizer
+  def generate_priority_for(resource)
+    record_priority_for(resource,
+                        Digest::SHA1.hexdigest("NaCl, MgSO4 (salts) and then #{resource.ref}"))
+  end
+
+  def generate_priority_contained_in(container, resource)
+    generate_priority_for(resource)
+  end
+end

data/lib/puppet/indirector.rb

@@ -0,0 +1,61 @@
+# Manage indirections to termini.  They are organized in terms of indirections -
+# - e.g., configuration, node, file, certificate -- and each indirection has one
+# or more terminus types defined.  The indirection is configured via the
+# +indirects+ method, which will be called by the class extending itself
+# with this module.
+module Puppet::Indirector
+  # LAK:FIXME We need to figure out how to handle documentation for the
+  # different indirection types.
+
+  require 'puppet/indirector/indirection'
+  require 'puppet/indirector/terminus'
+  require 'puppet/indirector/code'
+  require 'puppet/indirector/envelope'
+  require 'puppet/network/format_support'
+
+  def self.configure_routes(application_routes)
+    application_routes.each do |indirection_name, termini|
+      indirection_name = indirection_name.to_sym
+      terminus_name = termini["terminus"]
+      cache_name    = termini["cache"]
+
+      Puppet::Indirector::Terminus.terminus_class(indirection_name, terminus_name || cache_name)
+
+      indirection = Puppet::Indirector::Indirection.instance(indirection_name)
+      raise "Indirection #{indirection_name} does not exist" unless indirection
+
+      indirection.terminus_class = terminus_name if terminus_name
+      indirection.cache_class = cache_name if cache_name
+    end
+  end
+
+  # Declare that the including class indirects its methods to
+  # this terminus.  The terminus name must be the name of a Puppet
+  # default, not the value -- if it's the value, then it gets
+  # evaluated at parse time, which is before the user has had a chance
+  # to override it.
+  def indirects(indirection, options = {})
+    raise(ArgumentError, "Already handling indirection for #{@indirection.name}; cannot also handle #{indirection}") if @indirection
+    # populate this class with the various new methods
+    extend ClassMethods
+    include Puppet::Indirector::Envelope
+    include Puppet::Network::FormatSupport
+
+    # record the indirected class name for documentation purposes
+    options[:indirected_class] = name
+
+    # instantiate the actual Terminus for that type and this name (:ldap, w/ args :node)
+    # & hook the instantiated Terminus into this class (Node: @indirection = terminus)
+    @indirection = Puppet::Indirector::Indirection.new(self, indirection, options)
+  end
+
+  module ClassMethods
+    attr_reader :indirection
+  end
+
+  # Helper definition for indirections that handle filenames.
+  BadNameRegexp = Regexp.union(/^\.\./,
+                               %r{[\\/]},
+                               "\0",
+                               /(?i)^[a-z]:/)
+end

data/lib/puppet/indirector/active_record.rb

@@ -0,0 +1,29 @@
+require 'puppet/rails'
+require 'puppet/indirector'
+
+class Puppet::Indirector::ActiveRecord < Puppet::Indirector::Terminus
+  class << self
+    attr_accessor :ar_model
+  end
+
+  def self.use_ar_model(klass)
+    self.ar_model = klass
+  end
+
+  def ar_model
+    self.class.ar_model
+  end
+
+  def initialize
+    Puppet::Rails.init
+  end
+
+  def find(request)
+    return nil unless instance = ar_model.find_by_name(request.key)
+    instance.to_puppet
+  end
+
+  def save(request)
+    ar_model.from_puppet(request.instance).save
+  end
+end

data/lib/puppet/indirector/catalog/active_record.rb

@@ -0,0 +1,39 @@
+require 'puppet/rails/host'
+require 'puppet/indirector/active_record'
+require 'puppet/resource/catalog'
+
+class Puppet::Resource::Catalog::ActiveRecord < Puppet::Indirector::ActiveRecord
+  use_ar_model Puppet::Rails::Host
+
+  desc "A component of ActiveRecord storeconfigs. ActiveRecord-based storeconfigs
+    and inventory are deprecated. See http://links.puppetlabs.com/activerecord-deprecation"
+
+  def initialize
+    Puppet.deprecation_warning "ActiveRecord-based storeconfigs and inventory are deprecated. See http://links.puppetlabs.com/activerecord-deprecation"
+    super
+  end
+
+  # We don't retrieve catalogs from storeconfigs
+  def find(request)
+    nil
+  end
+
+  # Save the values from a Facts instance as the facts on a Rails Host instance.
+  def save(request)
+    catalog = request.instance
+
+    host = ar_model.find_by_name(catalog.name) || ar_model.create(:name => catalog.name)
+
+    host.railsmark "Saved catalog to database" do
+      host.merge_resources(catalog.vertices)
+      host.last_compile = Time.now
+
+      if node = Puppet::Node.indirection.find(catalog.name)
+        host.ip = node.parameters["ipaddress"]
+        host.environment = node.environment.to_s
+      end
+
+      host.save
+    end
+  end
+end

data/lib/puppet/indirector/catalog/compiler.rb

@@ -0,0 +1,175 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/code'
+require 'puppet/util/profiler'
+require 'yaml'
+
+class Puppet::Resource::Catalog::Compiler < Puppet::Indirector::Code
+  desc "Compiles catalogs on demand using Puppet's compiler."
+
+  include Puppet::Util
+
+  attr_accessor :code
+
+  def extract_facts_from_request(request)
+    return unless text_facts = request.options[:facts]
+    unless format = request.options[:facts_format]
+      raise ArgumentError, "Facts but no fact format provided for #{request.key}"
+    end
+
+    Puppet::Util::Profiler.profile("Found facts") do
+      # If the facts were encoded as yaml, then the param reconstitution system
+      # in Network::HTTP::Handler will automagically deserialize the value.
+      if text_facts.is_a?(Puppet::Node::Facts)
+        facts = text_facts
+      else
+        # We unescape here because the corresponding code in Puppet::Configurer::FactHandler escapes
+        facts = Puppet::Node::Facts.convert_from(format, CGI.unescape(text_facts))
+      end
+
+      unless facts.name == request.key
+        raise Puppet::Error, "Catalog for #{request.key.inspect} was requested with fact definition for the wrong node (#{facts.name.inspect})."
+      end
+
+      facts.add_timestamp
+      Puppet::Node::Facts.indirection.save(facts)
+    end
+  end
+
+  # Compile a node's catalog.
+  def find(request)
+    extract_facts_from_request(request)
+
+    node = node_from_request(request)
+    node.trusted_data = Puppet.lookup(:trusted_information) { Puppet::Context::TrustedInformation.local(node) }.to_h
+
+    if catalog = compile(node)
+      return catalog
+    else
+      # This shouldn't actually happen; we should either return
+      # a config or raise an exception.
+      return nil
+    end
+  end
+
+  # filter-out a catalog to remove exported resources
+  def filter(catalog)
+    return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter)
+    catalog
+  end
+
+  def initialize
+    Puppet::Util::Profiler.profile("Setup server facts for compiling") do
+      set_server_facts
+    end
+  end
+
+  # Is our compiler part of a network, or are we just local?
+  def networked?
+    Puppet.run_mode.master?
+  end
+
+  private
+
+  # Add any extra data necessary to the node.
+  def add_node_data(node)
+    # Merge in our server-side facts, so they can be used during compilation.
+    node.merge(@server_facts)
+  end
+
+  # Compile the actual catalog.
+  def compile(node)
+    str = "Compiled catalog for #{node.name}"
+    str += " in environment #{node.environment}" if node.environment
+    config = nil
+
+    benchmark(:notice, str) do
+      Puppet::Util::Profiler.profile(str) do
+        begin
+          config = Puppet::Parser::Compiler.compile(node)
+        rescue Puppet::Error => detail
+          Puppet.err(detail.to_s) if networked?
+          raise
+        end
+      end
+    end
+
+    config
+  end
+
+  # Turn our host name into a node object.
+  def find_node(name, environment)
+    Puppet::Util::Profiler.profile("Found node information") do
+      node = nil
+      begin
+        node = Puppet::Node.indirection.find(name, :environment => environment)
+      rescue => detail
+        message = "Failed when searching for node #{name}: #{detail}"
+        Puppet.log_exception(detail, message)
+        raise Puppet::Error, message, detail.backtrace
+      end
+
+
+      # Add any external data to the node.
+      if node
+        add_node_data(node)
+      end
+      node
+    end
+  end
+
+  # Extract the node from the request, or use the request
+  # to find the node.
+  def node_from_request(request)
+    if node = request.options[:use_node]
+      if request.remote?
+        raise Puppet::Error, "Invalid option use_node for a remote request"
+      else
+        return node
+      end
+    end
+
+    # We rely on our authorization system to determine whether the connected
+    # node is allowed to compile the catalog's node referenced by key.
+    # By default the REST authorization system makes sure only the connected node
+    # can compile his catalog.
+    # This allows for instance monitoring systems or puppet-load to check several
+    # node's catalog with only one certificate and a modification to auth.conf
+    # If no key is provided we can only compile the currently connected node.
+    name = request.key || request.node
+    if node = find_node(name, request.environment)
+      return node
+    end
+
+    raise ArgumentError, "Could not find node '#{name}'; cannot compile"
+  end
+
+  # Initialize our server fact hash; we add these to each client, and they
+  # won't change while we're running, so it's safe to cache the values.
+  def set_server_facts
+    @server_facts = {}
+
+    # Add our server version to the fact list
+    @server_facts["serverversion"] = Puppet.version.to_s
+
+    # And then add the server name and IP
+    {"servername" => "fqdn",
+      "serverip" => "ipaddress"
+    }.each do |var, fact|
+      if value = Facter.value(fact)
+        @server_facts[var] = value
+      else
+        Puppet.warning "Could not retrieve fact #{fact}"
+      end
+    end
+
+    if @server_facts["servername"].nil?
+      host = Facter.value(:hostname)
+      if domain = Facter.value(:domain)
+        @server_facts["servername"] = [host, domain].join(".")
+      else
+        @server_facts["servername"] = host
+      end
+    end
+  end
+end

data/lib/puppet/indirector/catalog/json.rb

@@ -0,0 +1,6 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/json'
+
+class Puppet::Resource::Catalog::Json < Puppet::Indirector::JSON
+  desc "Store catalogs as flat files, serialized using JSON."
+end

data/lib/puppet/indirector/catalog/msgpack.rb

@@ -0,0 +1,6 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/msgpack'
+
+class Puppet::Resource::Catalog::Msgpack < Puppet::Indirector::Msgpack
+  desc "Store catalogs as flat files, serialized using MessagePack."
+end

data/lib/puppet/indirector/catalog/queue.rb

@@ -0,0 +1,9 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/queue'
+
+class Puppet::Resource::Catalog::Queue < Puppet::Indirector::Queue
+
+  desc "Part of async storeconfigs, requiring the puppet queue daemon. ActiveRecord-based storeconfigs
+    and inventory are deprecated. See http://links.puppetlabs.com/activerecord-deprecation"
+
+end

data/lib/puppet/indirector/catalog/rest.rb

@@ -0,0 +1,6 @@
+require 'puppet/resource/catalog'
+require 'puppet/indirector/rest'
+
+class Puppet::Resource::Catalog::Rest < Puppet::Indirector::REST
+  desc "Find resource catalogs over HTTP via REST."
+end

data/lib/puppet/indirector/catalog/static_compiler.rb

@@ -0,0 +1,214 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/catalog/compiler'
+
+class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Compiler
+
+  desc %q{Compiles catalogs on demand using the optional static compiler. This
+    functions similarly to the normal compiler, but it replaces puppet:/// file
+    URLs with explicit metadata and file content hashes, expecting puppet agent
+    to fetch the exact specified content from the filebucket. This guarantees
+    that a given catalog will always result in the same file states. It also
+    decreases catalog application time and fileserver load, at the cost of
+    increased compilation time.
+
+    This terminus works today, but cannot be used without additional
+    configuration. Specifically:
+
+    * You must create a special filebucket resource --- with the title `puppet`
+      and the `path` attribute set to `false` --- in site.pp or somewhere else
+      where it will be added to every node's catalog. Using `puppet` as the title
+      is mandatory; the static compiler treats this title as magical.
+
+          filebucket { puppet:
+            path => false,
+          }
+
+    * You must set `catalog_terminus = static_compiler` in the puppet
+      master's puppet.conf.
+    * The puppet master's auth.conf must allow authenticated nodes to access the
+      `file_bucket_file` endpoint. This is enabled by default (see the
+      `path /file` rule), but if you have made your auth.conf more restrictive,
+      you may need to re-enable it.)
+    * If you are using multiple puppet masters, you must configure load balancer
+      affinity for agent nodes. This is because puppet masters other than the one
+      that compiled a given catalog may not have stored the required file contents
+      in their filebuckets.}
+
+  def find(request)
+    return nil unless catalog = super
+
+    raise "Did not get catalog back" unless catalog.is_a?(model)
+
+    catalog.resources.find_all { |res| res.type == "File" }.each do |resource|
+      next unless source = resource[:source]
+      next unless source =~ /^puppet:/
+
+      file = resource.to_ral
+
+      if file.recurse?
+        add_children(request.key, catalog, resource, file)
+      else
+        find_and_replace_metadata(request.key, resource, file)
+      end
+    end
+
+    catalog
+  end
+
+  # Take a resource with a fileserver based file source remove the source
+  # parameter, and insert the file metadata into the resource.
+  #
+  # This method acts to do the fileserver metadata retrieval in advance, while
+  # the file source is local and doesn't require an HTTP request. It retrieves
+  # the file metadata for a given file resource, removes the source parameter
+  # from the resource, inserts the metadata into the file resource, and uploads
+  # the file contents of the source to the file bucket.
+  #
+  # @param host [String] The host name of the node requesting this catalog
+  # @param resource [Puppet::Resource] The resource to replace the metadata in
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  def find_and_replace_metadata(host, resource, file)
+    # We remove URL info from it, so it forces a local copy
+    # rather than routing through the network.
+    # Weird, but true.
+    newsource = file[:source][0].sub("puppet:///", "")
+    file[:source][0] = newsource
+
+    raise "Could not get metadata for #{resource[:source]}" unless metadata = file.parameter(:source).metadata
+
+    replace_metadata(host, resource, metadata)
+  end
+
+  # Rewrite a given file resource with the metadata from a fileserver based file
+  #
+  # This performs the actual metadata rewrite for the given file resource and
+  # uploads the content of the source file to the filebucket.
+  #
+  # @param host [String] The host name of the node requesting this catalog
+  # @param resource [Puppet::Resource] The resource to add the metadata to
+  # @param metadata [Puppet::FileServing::Metadata] The metadata of the given fileserver based file
+  def replace_metadata(host, resource, metadata)
+    [:mode, :owner, :group].each do |param|
+      resource[param] ||= metadata.send(param)
+    end
+
+    resource[:ensure] = metadata.ftype
+    if metadata.ftype == "file"
+      unless resource[:content]
+        resource[:content] = metadata.checksum
+        resource[:checksum] = metadata.checksum_type
+      end
+    end
+
+    store_content(resource) if resource[:ensure] == "file"
+    old_source = resource.delete(:source)
+    Puppet.info "Metadata for #{resource} in catalog for '#{host}' added from '#{old_source}'"
+  end
+
+  # Generate children resources for a recursive file and add them to the catalog.
+  #
+  # @param host [String] The host name of the node requesting this catalog
+  # @param catalog [Puppet::Resource::Catalog]
+  # @param resource [Puppet::Resource]
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  def add_children(host, catalog, resource, file)
+    file = resource.to_ral
+
+    children = get_child_resources(host, catalog, resource, file)
+
+    remove_existing_resources(children, catalog)
+
+    children.each do |name, res|
+      catalog.add_resource res
+      catalog.add_edge(resource, res)
+    end
+  end
+
+  # Given a recursive file resource, recursively generate its children resources
+  #
+  # @param host [String] The host name of the node requesting this catalog
+  # @param catalog [Puppet::Resource::Catalog]
+  # @param resource [Puppet::Resource]
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  #
+  # @return [Array<Puppet::Resource>] The recursively generated File resources for the given resource
+  def get_child_resources(host, catalog, resource, file)
+    sourceselect = file[:sourceselect]
+    children = {}
+
+    source = resource[:source]
+
+    # This is largely a copy of recurse_remote in File
+    total = file[:source].collect do |source|
+      next unless result = file.perform_recursion(source)
+      return if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
+      result.each { |data| data.source = "#{source}/#{data.relative_path}" }
+      break result if result and ! result.empty? and sourceselect == :first
+      result
+    end.flatten.compact
+
+    # This only happens if we have sourceselect == :all
+    unless sourceselect == :first
+      found = []
+      total.reject! do |data|
+        result = found.include?(data.relative_path)
+        found << data.relative_path unless found.include?(data.relative_path)
+        result
+      end
+    end
+
+    total.each do |meta|
+      # This is the top-level parent directory
+      if meta.relative_path == "."
+        replace_metadata(host, resource, meta)
+        next
+      end
+      children[meta.relative_path] ||= Puppet::Resource.new(:file, File.join(file[:path], meta.relative_path))
+
+      # I think this is safe since it's a URL, not an actual file
+      children[meta.relative_path][:source] = source + "/" + meta.relative_path
+      resource.each do |param, value|
+        # These should never be passed to our children.
+        unless [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].include? param
+          children[meta.relative_path][param] = value
+        end
+      end
+      replace_metadata(host, children[meta.relative_path], meta)
+    end
+
+    children
+  end
+
+  # Remove any file resources in the catalog that will be duplicated by the
+  # given file resources.
+  #
+  # @param children [Array<Puppet::Resource>]
+  # @param catalog [Puppet::Resource::Catalog]
+  def remove_existing_resources(children, catalog)
+    existing_names = catalog.resources.collect { |r| r.to_s }
+    both = (existing_names & children.keys).inject({}) { |hash, name| hash[name] = true; hash }
+    both.each { |name| children.delete(name) }
+  end
+
+  # Retrieve the source of a file resource using a fileserver based source and
+  # upload it to the filebucket.
+  #
+  # @param resource [Puppet::Resource]
+  def store_content(resource)
+    @summer ||= Object.new
+    @summer.extend(Puppet::Util::Checksums)
+
+    type = @summer.sumtype(resource[:content])
+    sum = @summer.sumdata(resource[:content])
+
+    if Puppet::FileBucket::File.indirection.find("#{type}/#{sum}")
+      Puppet.info "Content for '#{resource[:source]}' already exists"
+    else
+      Puppet.info "Storing content for source '#{resource[:source]}'"
+      content = Puppet::FileServing::Content.indirection.find(resource[:source])
+      file = Puppet::FileBucket::File.new(content.content)
+      Puppet::FileBucket::File.indirection.save(file)
+    end
+  end
+end