puppet-retrospec 0.12.2 → 1.0.0

data/vendor/gems/puppet-4.5.2/lib/puppet/indirector/catalog/compiler.rb

@@ -0,0 +1,367 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/code'
+require 'puppet/util/profiler'
+require 'puppet/util/checksums'
+require 'yaml'
+require 'uri'
+
+class Puppet::Resource::Catalog::Compiler < Puppet::Indirector::Code
+  desc "Compiles catalogs on demand using Puppet's compiler."
+
+  include Puppet::Util
+  include Puppet::Util::Checksums
+
+  attr_accessor :code
+
+  def extract_facts_from_request(request)
+    return unless text_facts = request.options[:facts]
+    unless format = request.options[:facts_format]
+      raise ArgumentError, "Facts but no fact format provided for #{request.key}"
+    end
+
+    Puppet::Util::Profiler.profile("Found facts", [:compiler, :find_facts]) do
+      # If the facts were encoded as yaml, then the param reconstitution system
+      # in Network::HTTP::Handler will automagically deserialize the value.
+      if text_facts.is_a?(Puppet::Node::Facts)
+        facts = text_facts
+      else
+        # We unescape here because the corresponding code in Puppet::Configurer::FactHandler escapes
+        facts = Puppet::Node::Facts.convert_from(format, CGI.unescape(text_facts))
+      end
+
+      unless facts.name == request.key
+        raise Puppet::Error, "Catalog for #{request.key.inspect} was requested with fact definition for the wrong node (#{facts.name.inspect})."
+      end
+
+      options = {
+        :environment => request.environment,
+        :transaction_uuid => request.options[:transaction_uuid],
+      }
+
+      Puppet::Node::Facts.indirection.save(facts, nil, options)
+    end
+  end
+
+  # Compile a node's catalog.
+  def find(request)
+    extract_facts_from_request(request)
+
+    node = node_from_request(request)
+    node.trusted_data = Puppet.lookup(:trusted_information) { Puppet::Context::TrustedInformation.local(node) }.to_h
+
+    if catalog = compile(node, request.options)
+      return catalog
+    else
+      # This shouldn't actually happen; we should either return
+      # a config or raise an exception.
+      return nil
+    end
+  end
+
+  # filter-out a catalog to remove exported resources
+  def filter(catalog)
+    return catalog.filter { |r| r.virtual? } if catalog.respond_to?(:filter)
+    catalog
+  end
+
+  def initialize
+    Puppet::Util::Profiler.profile("Setup server facts for compiling", [:compiler, :init_server_facts]) do
+      set_server_facts
+    end
+  end
+
+  # Is our compiler part of a network, or are we just local?
+  def networked?
+    Puppet.run_mode.master?
+  end
+
+  private
+
+  # Add any extra data necessary to the node.
+  def add_node_data(node)
+    # Merge in our server-side facts, so they can be used during compilation.
+    node.add_server_facts(@server_facts)
+  end
+
+  # Determine which checksum to use; if agent_checksum_type is not nil,
+  # use the first entry in it that is also in known_checksum_types.
+  # If no match is found, return nil.
+  def common_checksum_type(agent_checksum_type)
+    if agent_checksum_type
+      agent_checksum_types = agent_checksum_type.split('.').map {|type| type.to_sym}
+      checksum_type = agent_checksum_types.drop_while do |type|
+        not known_checksum_types.include? type
+      end.first
+    end
+    checksum_type
+  end
+
+  def get_content_uri(metadata, source, environment_path)
+    # The static file content server doesn't know how to expand mountpoints, so
+    # we need to do that ourselves from the actual system path of the source file.
+    # This does that, while preserving any user-specified server or port.
+    source_path = Pathname.new(metadata.full_path)
+    path = source_path.relative_path_from(environment_path).to_s
+    source_as_uri = URI.parse(URI.escape(source))
+    server = source_as_uri.host
+    port = ":#{source_as_uri.port}" if source_as_uri.port
+    return "puppet://#{server}#{port}/#{path}"
+  end
+
+  # Helper method to decide if a file resource's metadata can be inlined.
+  # Also used to profile/log reasons for not inlining.
+  def inlineable?(resource, sources)
+    case
+      when resource[:ensure] == 'absent'
+        return Puppet::Util::Profiler.profile("Not inlining absent resource", [:compiler, :static_compile_inlining, :skipped_file_metadata, :absent]) { false }
+      when sources.empty?
+        return Puppet::Util::Profiler.profile("Not inlining resource without sources", [:compiler, :static_compile_inlining, :skipped_file_metadata, :no_sources]) { false }
+      when (not (sources.all? {|source| source =~ /^puppet:/}))
+        return Puppet::Util::Profiler.profile("Not inlining unsupported source scheme", [:compiler, :static_compile_inlining, :skipped_file_metadata, :unsupported_scheme]) { false }
+      else
+        return true
+    end
+  end
+
+  # Return true if metadata is inlineable, meaning the request's source is
+  # for the 'modules' mount and the resolved path is of the form:
+  #   $codedir/environments/$environment/*/*/files/**
+  def inlineable_metadata?(metadata, source, environment_path)
+    source_as_uri = URI.parse(URI.escape(source))
+    location = Puppet::Module::FILETYPES['files']
+
+    !!(source_as_uri.path =~ /^\/modules\// &&
+       metadata.full_path =~ /#{environment_path}[^\/]+\/[^\/]+\/#{location}\/.+/)
+  end
+
+  # Helper method to log file resources that could not be inlined because they
+  # fall outside of an environment.
+  def log_file_outside_environment
+    Puppet::Util::Profiler.profile("Not inlining file outside environment", [:compiler, :static_compile_inlining, :skipped_file_metadata, :file_outside_environment]) { true }
+  end
+
+  # Helper method to log file resources that were successfully inlined.
+  def log_metadata_inlining
+    Puppet::Util::Profiler.profile("Inlining file metadata", [:compiler, :static_compile_inlining, :inlined_file_metadata]) { true }
+  end
+
+  # Inline file metadata for static catalogs
+  # Initially restricted to files sourced from codedir via puppet:/// uri.
+  def inline_metadata(catalog, checksum_type)
+    environment_path = Pathname.new File.join(Puppet[:environmentpath], catalog.environment, "")
+    list_of_resources = catalog.resources.find_all { |res| res.type == "File" }
+
+    # TODO: get property/parameter defaults if entries are nil in the resource
+    # For now they're hard-coded to match the File type.
+
+    file_metadata = {}
+    list_of_resources.each do |resource|
+      sources = [resource[:source]].flatten.compact
+      next unless inlineable?(resource, sources)
+
+      # both need to handle multiple sources
+      if resource[:recurse] == true || resource[:recurse] == 'true' || resource[:recurse] == 'remote'
+        # Construct a hash mapping sources to arrays (list of files found recursively) of metadata
+        options = {
+          :environment        => catalog.environment_instance,
+          :links              => resource[:links] ? resource[:links].to_sym : :manage,
+          :checksum_type      => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym,
+          :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore,
+          :recurse            => true,
+          :recurselimit       => resource[:recurselimit],
+          :ignore             => resource[:ignore],
+        }
+
+        sources_in_environment = true
+
+        source_to_metadatas = {}
+        sources.each do |source|
+          source = Puppet::Type.type(:file).attrclass(:source).normalize(source)
+
+          if list_of_data = Puppet::FileServing::Metadata.indirection.search(source, options)
+            basedir_meta = list_of_data.find {|meta| meta.relative_path == '.'}
+            devfail "FileServing::Metadata search should always return the root search path" if basedir_meta.nil?
+
+            if ! inlineable_metadata?(basedir_meta, source,  environment_path)
+              # If any source is not in the environment path, skip inlining this resource.
+              log_file_outside_environment
+              sources_in_environment = false
+              break
+            end
+
+            base_content_uri = get_content_uri(basedir_meta, source, environment_path)
+            list_of_data.each do |metadata|
+              if metadata.relative_path == '.'
+                metadata.content_uri = base_content_uri
+              else
+                metadata.content_uri = "#{base_content_uri}/#{metadata.relative_path}"
+              end
+            end
+
+            source_to_metadatas[source] = list_of_data
+            # Optimize for returning less data if sourceselect is first
+            if resource[:sourceselect] == 'first' || resource[:sourceselect].nil?
+              break
+            end
+          end
+        end
+
+        if sources_in_environment && !source_to_metadatas.empty?
+          log_metadata_inlining
+          catalog.recursive_metadata[resource.title] = source_to_metadatas
+        end
+      else
+        options = {
+          :environment        => catalog.environment_instance,
+          :links              => resource[:links] ? resource[:links].to_sym : :manage,
+          :checksum_type      => resource[:checksum] ? resource[:checksum].to_sym : checksum_type.to_sym,
+          :source_permissions => resource[:source_permissions] ? resource[:source_permissions].to_sym : :ignore
+        }
+
+        metadata = nil
+        sources.each do |source|
+          source = Puppet::Type.type(:file).attrclass(:source).normalize(source)
+
+          if data = Puppet::FileServing::Metadata.indirection.find(source, options)
+            metadata = data
+            metadata.source = source
+            break
+          end
+        end
+
+        raise "Could not get metadata for #{resource[:source]}" unless metadata
+
+        if inlineable_metadata?(metadata, metadata.source,  environment_path)
+          metadata.content_uri = get_content_uri(metadata, metadata.source, environment_path)
+          log_metadata_inlining
+
+          # If the file is in the environment directory, we can safely inline
+          catalog.metadata[resource.title] = metadata
+        else
+          # Log a profiler event that we skipped this file because it is not in an environment.
+          log_file_outside_environment
+        end
+      end
+    end
+  end
+
+  # Compile the actual catalog.
+  def compile(node, options)
+    if node.environment && node.environment.static_catalogs? && options[:static_catalog] && options[:code_id]
+      # Check for errors before compiling the catalog
+      checksum_type = common_checksum_type(options[:checksum_type])
+      raise Puppet::Error, "Unable to find a common checksum type between agent '#{options[:checksum_type]}' and master '#{known_checksum_types}'." unless checksum_type
+    end
+
+    str = "Compiled %s for " % [checksum_type ? 'static catalog' : 'catalog']
+    str += node.name
+    str += " in environment #{node.environment}" if node.environment
+    config = nil
+
+    benchmark(:notice, str) do
+      compile_type = checksum_type ? :static_compile : :compile
+      Puppet::Util::Profiler.profile(str, [:compiler, compile_type, node.environment, node.name]) do
+        begin
+          config = Puppet::Parser::Compiler.compile(node, options[:code_id])
+        rescue Puppet::Error => detail
+          Puppet.err(detail.to_s) if networked?
+          raise
+        ensure
+          Puppet::Type.clear_misses unless Puppet[:always_retry_plugins]
+        end
+
+        if checksum_type && config.is_a?(model)
+          str = "Inlined resource metadata into static catalog for #{node.name}"
+          str += " in environment #{node.environment}" if node.environment
+          benchmark(:notice, str) do
+            Puppet::Util::Profiler.profile(str, [:compiler, :static_compile_postprocessing, node.environment, node.name]) do
+              inline_metadata(config, checksum_type)
+            end
+          end
+        end
+      end
+    end
+
+
+    config
+  end
+
+  # Turn our host name into a node object.
+  def find_node(name, environment, transaction_uuid, configured_environment)
+    Puppet::Util::Profiler.profile("Found node information", [:compiler, :find_node]) do
+      node = nil
+      begin
+        node = Puppet::Node.indirection.find(name, :environment => environment,
+                                             :transaction_uuid => transaction_uuid,
+                                             :configured_environment => configured_environment)
+      rescue => detail
+        message = "Failed when searching for node #{name}: #{detail}"
+        Puppet.log_exception(detail, message)
+        raise Puppet::Error, message, detail.backtrace
+      end
+
+
+      # Add any external data to the node.
+      if node
+        add_node_data(node)
+      end
+      node
+    end
+  end
+
+  # Extract the node from the request, or use the request
+  # to find the node.
+  def node_from_request(request)
+    if node = request.options[:use_node]
+      if request.remote?
+        raise Puppet::Error, "Invalid option use_node for a remote request"
+      else
+        return node
+      end
+    end
+
+    # We rely on our authorization system to determine whether the connected
+    # node is allowed to compile the catalog's node referenced by key.
+    # By default the REST authorization system makes sure only the connected node
+    # can compile his catalog.
+    # This allows for instance monitoring systems or puppet-load to check several
+    # node's catalog with only one certificate and a modification to auth.conf
+    # If no key is provided we can only compile the currently connected node.
+    name = request.key || request.node
+    if node = find_node(name, request.environment, request.options[:transaction_uuid], request.options[:configured_environment])
+      return node
+    end
+
+    raise ArgumentError, "Could not find node '#{name}'; cannot compile"
+  end
+
+  # Initialize our server fact hash; we add these to each client, and they
+  # won't change while we're running, so it's safe to cache the values.
+  def set_server_facts
+    @server_facts = {}
+
+    # Add our server version to the fact list
+    @server_facts["serverversion"] = Puppet.version.to_s
+
+    # And then add the server name and IP
+    {"servername" => "fqdn",
+      "serverip" => "ipaddress"
+    }.each do |var, fact|
+      if value = Facter.value(fact)
+        @server_facts[var] = value
+      else
+        Puppet.warning "Could not retrieve fact #{fact}"
+      end
+    end
+
+    if @server_facts["servername"].nil?
+      host = Facter.value(:hostname)
+      if domain = Facter.value(:domain)
+        @server_facts["servername"] = [host, domain].join(".")
+      else
+        @server_facts["servername"] = host
+      end
+    end
+  end
+end

data/vendor/gems/puppet-4.5.2/lib/puppet/indirector/catalog/static_compiler.rb

@@ -0,0 +1,229 @@
+require 'puppet/node'
+require 'puppet/resource/catalog'
+require 'puppet/indirector/catalog/compiler'
+
+class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Compiler
+
+  desc %q{Compiles catalogs on demand using the optional static compiler. This
+    functions similarly to the normal compiler, but it replaces puppet:/// file
+    URLs with explicit metadata and file content hashes, expecting puppet agent
+    to fetch the exact specified content from the filebucket. This guarantees
+    that a given catalog will always result in the same file states. It also
+    decreases catalog application time and fileserver load, at the cost of
+    increased compilation time.
+
+    This terminus works today, but cannot be used without additional
+    configuration. Specifically:
+
+    * You must create a special filebucket resource --- with the title `puppet`
+      and the `path` attribute set to `false` --- in site.pp or somewhere else
+      where it will be added to every node's catalog. Using `puppet` as the title
+      is mandatory; the static compiler treats this title as magical.
+
+          filebucket { puppet:
+            path => false,
+          }
+
+    * You must set `catalog_terminus = static_compiler` in the puppet
+      master's puppet.conf.
+    * The puppet master's auth.conf must allow authenticated nodes to access the
+      `file_bucket_file` endpoint. This is enabled by default (see the
+      `path /file` rule), but if you have made your auth.conf more restrictive,
+      you may need to re-enable it.)
+    * If you are using multiple puppet masters, you must configure load balancer
+      affinity for agent nodes. This is because puppet masters other than the one
+      that compiled a given catalog may not have stored the required file contents
+      in their filebuckets.}
+
+  def find(request)
+    return nil unless catalog = super
+
+    raise "Did not get catalog back" unless catalog.is_a?(model)
+
+    catalog.resources.find_all { |res| res.type == "File" }.each do |resource|
+      next if resource[:ensure] == 'absent'
+
+      next unless source = resource[:source]
+      next unless source =~ /^puppet:/
+
+      file = resource.to_ral
+
+      if file.recurse?
+        add_children(request, catalog, resource, file)
+      else
+        find_and_replace_metadata(request, resource, file)
+      end
+    end
+
+    catalog
+  end
+
+  # Take a resource with a fileserver based file source remove the source
+  # parameter, and insert the file metadata into the resource.
+  #
+  # This method acts to do the fileserver metadata retrieval in advance, while
+  # the file source is local and doesn't require an HTTP request. It retrieves
+  # the file metadata for a given file resource, removes the source parameter
+  # from the resource, inserts the metadata into the file resource, and uploads
+  # the file contents of the source to the file bucket.
+  #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
+  # @param resource [Puppet::Resource] The resource to replace the metadata in
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  def find_and_replace_metadata(request, resource, file)
+    # We remove URL info from it, so it forces a local copy
+    # rather than routing through the network.
+    # Weird, but true.
+    newsource = file[:source][0].sub("puppet:///", "")
+    file[:source][0] = newsource
+
+    raise "Could not get metadata for #{resource[:source]}" unless metadata = file.parameter(:source).metadata
+
+    replace_metadata(request, resource, metadata)
+  end
+
+  # Rewrite a given file resource with the metadata from a fileserver based file
+  #
+  # This performs the actual metadata rewrite for the given file resource and
+  # uploads the content of the source file to the filebucket.
+  #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
+  # @param resource [Puppet::Resource] The resource to add the metadata to
+  # @param metadata [Puppet::FileServing::Metadata] The metadata of the given fileserver based file
+  def replace_metadata(request, resource, metadata)
+    [:owner, :group].each do |param|
+      resource[param] ||= metadata.send(param).to_s
+    end
+    resource[:mode] ||= metadata.send(:mode).to_s(8)
+
+    resource[:ensure] = metadata.ftype
+    if metadata.ftype == "file"
+      unless resource[:content]
+        resource[:content] = metadata.checksum
+        resource[:checksum] = metadata.checksum_type
+      end
+    end
+
+    store_content(request, resource) if resource[:ensure] == "file"
+    old_source = resource.delete(:source)
+    Puppet.info "Metadata for #{resource} in catalog for '#{request.key}' added from '#{old_source}'"
+  end
+
+  # Generate children resources for a recursive file and add them to the catalog.
+  #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
+  # @param catalog [Puppet::Resource::Catalog]
+  # @param resource [Puppet::Resource]
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  def add_children(request, catalog, resource, file)
+    children = get_child_resources(request, catalog, resource, file)
+    # get_child_resources() returned early because source is not
+    # a directory, but we still need to replace the metadata of the
+    # resource, so we do it here before returning.
+    if children.nil?
+      find_and_replace_metadata(request, resource, file)
+      return
+    end
+
+    remove_existing_resources(children, catalog)
+
+    children.each do |name, res|
+      catalog.add_resource res
+      catalog.add_edge(resource, res)
+    end
+  end
+
+  # Given a recursive file resource, recursively generate its children resources
+  #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
+  # @param catalog [Puppet::Resource::Catalog]
+  # @param resource [Puppet::Resource]
+  # @param file [Puppet::Type::File] The file RAL associated with the resource
+  #
+  # @return [Array<Puppet::Resource>] The recursively generated File resources for the given resource
+  def get_child_resources(request, catalog, resource, file)
+    sourceselect = file[:sourceselect]
+    children = {}
+
+    source = resource[:source]
+
+    # This is largely a copy of recurse_remote in File
+    total = file[:source].collect do |src|
+      next unless result = file.perform_recursion(src)
+      return if top = result.find { |r| r.relative_path == "." } and top.ftype != "directory"
+      result.each { |data| data.source = "#{src}/#{data.relative_path}" }
+      break result if result and ! result.empty? and sourceselect == :first
+      result
+    end.flatten.compact
+
+    # This only happens if we have sourceselect == :all
+    unless sourceselect == :first
+      found = []
+      total.reject! do |data|
+        result = found.include?(data.relative_path)
+        found << data.relative_path unless found.include?(data.relative_path)
+        result
+      end
+    end
+
+    parent_meta = nil
+    total.each do |meta|
+      # This is the top-level parent directory
+      if meta.relative_path == "."
+        parent_meta = meta
+        next
+      end
+      child = children[meta.relative_path] ||=
+        Puppet::Resource.new(:file, File.join(file[:path], meta.relative_path))
+
+      # I think this is safe since it's a URL, not an actual file
+      child[:source] = source + "/" + meta.relative_path
+      resource.each do |param, value|
+        # These should never be passed to our children.
+        unless [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].include? param
+          child[param] = value
+        end
+      end
+      replace_metadata(request, child, meta)
+    end
+    replace_metadata(request, resource, parent_meta)
+
+    children
+  end
+
+  # Remove any recursed file resources already in the catalog
+  #
+  # @param children [Array<Puppet::Resource>]
+  # @param catalog [Puppet::Resource::Catalog]
+  def remove_existing_resources(children, catalog)
+    relative_paths = children.keys
+    relative_paths.each do |relative_path|
+      child = children[relative_path]
+      if catalog.resource(child.ref)
+        Puppet.debug("Resource #{child.ref} already managed, removing from recursed children")
+        children.delete(relative_path)
+      end
+    end
+  end
+
+  # Retrieve the source of a file resource using a fileserver based source and
+  # upload it to the filebucket.
+  #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
+  # @param resource [Puppet::Resource]
+  def store_content(request, resource)
+    @summer ||= Puppet::Util::Checksums
+
+    type = @summer.sumtype(resource[:content])
+    sum = @summer.sumdata(resource[:content])
+
+    if Puppet::FileBucket::File.indirection.find("#{type}/#{sum}")
+      Puppet.info "Content for '#{resource[:source]}' already exists"
+    else
+      Puppet.info "Storing content for source '#{resource[:source]}'"
+      content = Puppet::FileServing::Content.indirection.find(resource[:source], {:environment => request.environment})
+      file = Puppet::FileBucket::File.new(content.content)
+      Puppet::FileBucket::File.indirection.save(file)
+    end
+  end
+end