puppet 4.2.0 → 4.3.0

data/lib/puppet/indirector/catalog/static_compiler.rb

@@ -41,15 +41,17 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
     raise "Did not get catalog back" unless catalog.is_a?(model)
 
     catalog.resources.find_all { |res| res.type == "File" }.each do |resource|
+      next if resource[:ensure] == 'absent'
+
       next unless source = resource[:source]
       next unless source =~ /^puppet:/
 
       file = resource.to_ral
 
       if file.recurse?
-        add_children(request.key, catalog, resource, file)
+        add_children(request, catalog, resource, file)
       else
-        find_and_replace_metadata(request.key, resource, file)
+        find_and_replace_metadata(request, resource, file)
       end
     end
 
@@ -65,10 +67,10 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
   # from the resource, inserts the metadata into the file resource, and uploads
   # the file contents of the source to the file bucket.
   #
-  # @param host [String] The host name of the node requesting this catalog
+  # @param request [Puppet::Indirector::Request] The request for the catalog
   # @param resource [Puppet::Resource] The resource to replace the metadata in
   # @param file [Puppet::Type::File] The file RAL associated with the resource
-  def find_and_replace_metadata(host, resource, file)
+  def find_and_replace_metadata(request, resource, file)
     # We remove URL info from it, so it forces a local copy
     # rather than routing through the network.
     # Weird, but true.
@@ -77,7 +79,7 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
 
     raise "Could not get metadata for #{resource[:source]}" unless metadata = file.parameter(:source).metadata
 
-    replace_metadata(host, resource, metadata)
+    replace_metadata(request, resource, metadata)
   end
 
   # Rewrite a given file resource with the metadata from a fileserver based file
@@ -85,13 +87,14 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
   # This performs the actual metadata rewrite for the given file resource and
   # uploads the content of the source file to the filebucket.
   #
-  # @param host [String] The host name of the node requesting this catalog
+  # @param request [Puppet::Indirector::Request] The request for the catalog
   # @param resource [Puppet::Resource] The resource to add the metadata to
   # @param metadata [Puppet::FileServing::Metadata] The metadata of the given fileserver based file
-  def replace_metadata(host, resource, metadata)
-    [:mode, :owner, :group].each do |param|
-      resource[param] ||= metadata.send(param)
+  def replace_metadata(request, resource, metadata)
+    [:owner, :group].each do |param|
+      resource[param] ||= metadata.send(param).to_s
     end
+    resource[:mode] ||= metadata.send(:mode).to_s(8)
 
     resource[:ensure] = metadata.ftype
     if metadata.ftype == "file"
@@ -101,21 +104,26 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
       end
     end
 
-    store_content(resource) if resource[:ensure] == "file"
+    store_content(request, resource) if resource[:ensure] == "file"
     old_source = resource.delete(:source)
-    Puppet.info "Metadata for #{resource} in catalog for '#{host}' added from '#{old_source}'"
+    Puppet.info "Metadata for #{resource} in catalog for '#{request.key}' added from '#{old_source}'"
   end
 
   # Generate children resources for a recursive file and add them to the catalog.
   #
-  # @param host [String] The host name of the node requesting this catalog
+  # @param request [Puppet::Indirector::Request] The request for the catalog
   # @param catalog [Puppet::Resource::Catalog]
   # @param resource [Puppet::Resource]
   # @param file [Puppet::Type::File] The file RAL associated with the resource
-  def add_children(host, catalog, resource, file)
-    file = resource.to_ral
-
-    children = get_child_resources(host, catalog, resource, file)
+  def add_children(request, catalog, resource, file)
+    children = get_child_resources(request, catalog, resource, file)
+    # get_child_resources() returned early because source is not
+    # a directory, but we still need to replace the metadata of the
+    # resource, so we do it here before returning.
+    if children.nil?
+      find_and_replace_metadata(request, resource, file)
+      return
+    end
 
     remove_existing_resources(children, catalog)
 
@@ -127,13 +135,13 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
 
   # Given a recursive file resource, recursively generate its children resources
   #
-  # @param host [String] The host name of the node requesting this catalog
+  # @param request [Puppet::Indirector::Request] The request for the catalog
   # @param catalog [Puppet::Resource::Catalog]
   # @param resource [Puppet::Resource]
   # @param file [Puppet::Type::File] The file RAL associated with the resource
   #
   # @return [Array<Puppet::Resource>] The recursively generated File resources for the given resource
-  def get_child_resources(host, catalog, resource, file)
+  def get_child_resources(request, catalog, resource, file)
     sourceselect = file[:sourceselect]
     children = {}
 
@@ -158,44 +166,52 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
       end
     end
 
+    parent_meta = nil
     total.each do |meta|
       # This is the top-level parent directory
       if meta.relative_path == "."
-        replace_metadata(host, resource, meta)
+        parent_meta = meta
         next
       end
-      children[meta.relative_path] ||= Puppet::Resource.new(:file, File.join(file[:path], meta.relative_path))
+      child = children[meta.relative_path] ||=
+        Puppet::Resource.new(:file, File.join(file[:path], meta.relative_path))
 
       # I think this is safe since it's a URL, not an actual file
-      children[meta.relative_path][:source] = source + "/" + meta.relative_path
+      child[:source] = source + "/" + meta.relative_path
       resource.each do |param, value|
         # These should never be passed to our children.
         unless [:parent, :ensure, :recurse, :recurselimit, :target, :alias, :source].include? param
-          children[meta.relative_path][param] = value
+          child[param] = value
         end
       end
-      replace_metadata(host, children[meta.relative_path], meta)
+      replace_metadata(request, child, meta)
     end
+    replace_metadata(request, resource, parent_meta)
 
     children
   end
 
-  # Remove any file resources in the catalog that will be duplicated by the
-  # given file resources.
+  # Remove any recursed file resources already in the catalog
   #
   # @param children [Array<Puppet::Resource>]
   # @param catalog [Puppet::Resource::Catalog]
   def remove_existing_resources(children, catalog)
-    existing_names = catalog.resources.collect { |r| r.to_s }
-    both = (existing_names & children.keys).inject({}) { |hash, name| hash[name] = true; hash }
-    both.each { |name| children.delete(name) }
+    relative_paths = children.keys
+    relative_paths.each do |relative_path|
+      child = children[relative_path]
+      if catalog.resource(child.ref)
+        Puppet.debug("Resource #{child.ref} already managed, removing from recursed children")
+        children.delete(relative_path)
+      end
+    end
   end
 
   # Retrieve the source of a file resource using a fileserver based source and
   # upload it to the filebucket.
   #
+  # @param request [Puppet::Indirector::Request] The request for the catalog
   # @param resource [Puppet::Resource]
-  def store_content(resource)
+  def store_content(request, resource)
     @summer ||= Puppet::Util::Checksums
 
     type = @summer.sumtype(resource[:content])
@@ -205,7 +221,7 @@ class Puppet::Resource::Catalog::StaticCompiler < Puppet::Resource::Catalog::Com
       Puppet.info "Content for '#{resource[:source]}' already exists"
     else
       Puppet.info "Storing content for source '#{resource[:source]}'"
-      content = Puppet::FileServing::Content.indirection.find(resource[:source])
+      content = Puppet::FileServing::Content.indirection.find(resource[:source], {:environment => request.environment})
       file = Puppet::FileBucket::File.new(content.content)
       Puppet::FileBucket::File.indirection.save(file)
     end

data/lib/puppet/indirector/data_binding/none.rb

@@ -1,5 +1,8 @@
 require 'puppet/indirector/none'
 
 class Puppet::DataBinding::None < Puppet::Indirector::None
-  desc "A Dummy terminus that always returns nil for data lookups."
+  desc "A Dummy terminus that always throws :no_such_key for data lookups."
+  def find(request)
+    throw :no_such_key
+  end
 end

data/lib/puppet/indirector/file_bucket_file/file.rb

@@ -1,15 +1,23 @@
 require 'puppet/indirector/code'
 require 'puppet/file_bucket/file'
 require 'puppet/util/checksums'
+require 'puppet/util/diff'
 require 'fileutils'
 
 module Puppet::FileBucketFile
   class File < Puppet::Indirector::Code
     include Puppet::Util::Checksums
+    include Puppet::Util::Diff
 
     desc "Store files in a directory set based on their checksums."
 
     def find(request)
+      request.options[:bucket_path] ||= Puppet[:bucketdir]
+      # If filebucket mode is 'list'
+      if request.options[:list_all]
+        return nil unless ::File.exists?(request.options[:bucket_path])
+        return list(request)
+      end
       checksum, files_original_path = request_to_checksum_and_path(request)
       contents_file = path_for(request.options[:bucket_path], checksum, 'contents')
       paths_file = path_for(request.options[:bucket_path], checksum, 'paths')
@@ -18,7 +26,8 @@ module Puppet::FileBucketFile
         if request.options[:diff_with]
           other_contents_file = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents')
           raise "could not find diff_with #{request.options[:diff_with]}" unless Puppet::FileSystem.exist?(other_contents_file)
-          return `diff #{Puppet::FileSystem.path_string(contents_file).inspect} #{Puppet::FileSystem.path_string(other_contents_file).inspect}`
+          raise "Unable to diff on this platform" unless Puppet[:diff] != ""
+          return diff(Puppet::FileSystem.path_string(contents_file), Puppet::FileSystem.path_string(other_contents_file))
         else
           Puppet.info "FileBucket read #{checksum}"
           model.new(Puppet::FileSystem.binread(contents_file))
@@ -28,6 +37,54 @@ module Puppet::FileBucketFile
       end
     end
 
+    def list(request)
+      if request.remote?
+        raise Puppet::Error, "Listing remote file buckets is not allowed"
+      end
+
+      fromdate = request.options[:fromdate] || "0:0:0 1-1-1970"
+      todate = request.options[:todate] || Time.now.strftime("%F %T")
+      begin
+        to = Time.parse(todate)
+      rescue ArgumentError
+        raise Puppet::Error, "Error while parsing 'todate'"
+      end
+      begin
+        from = Time.parse(fromdate)
+      rescue ArgumentError
+        raise Puppet::Error, "Error while parsing 'fromdate'"
+      end
+      # Setting hash's default value to [], needed by the following loop
+      bucket = Hash.new {[]}
+      msg = ""
+      # Get all files with mtime between 'from' and 'to'
+      Pathname.new(request.options[:bucket_path]).find { |item|
+        if item.file? and item.basename.to_s == "paths"
+          filenames = item.read.strip.split("\n")
+          filestat = Time.parse(item.stat.mtime.to_s)
+          if from <= filestat and filestat <= to
+            filenames.each do |filename|
+              bucket[filename] += [[ item.stat.mtime , item.parent.basename ]]
+            end
+          end
+        end
+      }
+      # Sort the results
+      bucket.each { |filename, contents|
+        contents.sort_by! do |item|
+          item[0]
+        end
+      }
+      # Build the output message. Sorted by names then by dates
+      bucket.sort.each { |filename,contents|
+        contents.each { |mtime, chksum|
+          date = mtime.strftime("%F %T")
+          msg += "#{chksum} #{date} #{filename}\n"
+        }
+      }
+      return model.new(msg)
+    end
+
     def head(request)
       checksum, files_original_path = request_to_checksum_and_path(request)
       contents_file = path_for(request.options[:bucket_path], checksum, 'contents')

data/lib/puppet/indirector/hiera.rb

@@ -18,6 +18,7 @@ class Puppet::Indirector::Hiera < Puppet::Indirector::Terminus
   def find(request)
     not_found = Object.new
     options = request.options
+    Puppet.debug { "Performing a hiera indirector lookup of #{request.key} with options #{options.inspect}" }
     value = hiera.lookup(request.key, not_found, Hiera::Scope.new(options[:variables]), nil, convert_merge(options[:merge]))
     throw :no_such_key if value.equal?(not_found)
     value
@@ -34,8 +35,11 @@ class Puppet::Indirector::Hiera < Puppet::Indirector::Terminus
   def convert_merge(merge)
     case merge
     when nil
+    when 'first'
       # Nil is OK. Defaults to Hiera :priority
       nil
+    when Puppet::Pops::MergeStrategy
+      convert_merge(merge.configuration)
     when 'unique'
       # Equivalent to Hiera :array
       :array

data/lib/puppet/indirector/indirection.rb

@@ -196,7 +196,12 @@ class Puppet::Indirector::Indirection
         result.expiration ||= self.expiration if result.respond_to?(:expiration)
         if cache?
           Puppet.info "Caching #{self.name} for #{request.key}"
-          cache.save request(:save, key, result, options)
+          begin
+            cache.save request(:save, key, result, options)
+          rescue => detail
+            Puppet.log_exception(detail)
+            raise detail
+          end
         end
 
         filtered = result

data/lib/puppet/indirector/json.rb

@@ -17,7 +17,7 @@ class Puppet::Indirector::JSON < Puppet::Indirector::Terminus
 
     Puppet::Util.replace_file(filename, 0660) {|f| f.print to_json(request.instance) }
   rescue TypeError => detail
-    Puppet.log_exception "Could not save #{self.name} #{request.key}: #{detail}"
+    Puppet.log_exception(detail, "Could not save #{self.name} #{request.key}: #{detail}")
   end
 
   def destroy(request)

data/lib/puppet/indirector/msgpack.rb

@@ -23,7 +23,7 @@ class Puppet::Indirector::Msgpack < Puppet::Indirector::Terminus
 
     Puppet::Util.replace_file(filename, 0660) {|f| f.print to_msgpack(request.instance) }
   rescue TypeError => detail
-    Puppet.log_exception "Could not save #{self.name} #{request.key}: #{detail}"
+    Puppet.log_exception(detail, "Could not save #{self.name} #{request.key}: #{detail}")
   end
 
   def destroy(request)

data/lib/puppet/indirector/request.rb

@@ -243,15 +243,14 @@ class Puppet::Indirector::Request
       @port = uri.port.to_i
     end
 
-    @protocol = uri.scheme
-
-    if uri.scheme == 'puppet'
-      @key = URI.unescape(uri.path.sub(/^\//, ''))
-      return
+    # filebucket:// is only used internally to pass request details
+    # from Dipper objects to the indirector. The wire always uses HTTPS.
+    if uri.scheme == 'filebucket'
+      @protocol = 'https'
+    else
+      @protocol = uri.scheme
     end
 
-    env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3)
-    @key ||= ''
-    self.environment = env unless env == ''
+    @key = URI.unescape(uri.path.sub(/^\//, ''))
   end
 end

data/lib/puppet/indirector/resource_type/parser.rb

@@ -27,7 +27,7 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
       krt = resource_types_in(request.environment)
 
       # This is a bit ugly.
-      [:hostclass, :definition, :node].each do |type|
+      [:hostclass, :definition, :application, :node].each do |type|
         # We have to us 'find_<type>' here because it will
         # load any missing types from disk, whereas the plain
         # '<type>' method only returns from memory.
@@ -66,14 +66,16 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
             krt.hostclasses.values
           when "defined_type"
             krt.definitions.values
+          when "application"
+            krt.applications.values
           when "node"
             krt.nodes.values
           when nil
-            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.nodes.values]
+            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.applications.values, krt.nodes.values]
           else
             raise ArgumentError, "Unrecognized kind filter: " +
                       "'#{request.options[:kind]}', expected one " +
-                      " of 'class', 'defined_type', or 'node'."
+                      " of 'class', 'defined_type', 'application', or 'node'."
         end
 
       result = result_candidates.flatten.reject { |t| t.name == "" }

data/lib/puppet/info_service/class_information_service.rb

@@ -0,0 +1,111 @@
+require 'puppet'
+require 'puppet/pops'
+require 'puppet/pops/evaluator/literal_evaluator'
+
+class Puppet::InfoService::ClassInformationService
+
+  def initialize
+    @file_to_result = {}
+    @parser = Puppet::Pops::Parser::EvaluatingParser.new()
+  end
+
+  def classes_per_environment(env_file_hash)
+    # In this version of puppet there is only one way to parse manifests, as feature switches per environment
+    # are added or removed, this logic needs to change to compute the result per environment with the correct
+    # feature flags in effect.
+
+    unless env_file_hash.is_a?(Hash)
+      raise ArgumentError, 'Given argument must be a Hash'
+    end
+
+    result = {}
+
+    # for each environment
+    #   for each file
+    #     if file already processed, use last result or error
+    #
+    env_file_hash.each do |env, files|
+      env_result = result[env] = {}
+      files.each do |f|
+        env_result[f] = result_of(f)
+      end
+    end
+    result
+  end
+
+  private
+
+  def type_parser
+    # Safe to cache this as it would otherwise constantly build up the visitor cache
+    @@type_parser ||= Puppet::Pops::Types::TypeParser.new
+  end
+
+  def literal_evaluator
+    @@literal_evaluator ||= Puppet::Pops::Evaluator::LiteralEvaluator.new
+  end
+
+  def result_of(f)
+    entry =  @file_to_result[f]
+    if entry.nil?
+      @file_to_result[f] = entry = parse_file(f)
+    end
+    entry
+  end
+
+  def parse_file(f)
+    begin
+      parse_result = @parser.parse_file(f)
+      parse_result.definitions.select {|d| d.is_a?(Puppet::Pops::Model::HostClassDefinition)}.map do |d|
+        {:name   => d.name,
+         :params => params = d.parameters.map {|p| extract_param(p) }
+        }
+      end
+    rescue StandardError => e
+      {:error => e.message }
+    end
+  end
+
+  def extract_param(p)
+    extract_default(extract_type({:name => p.name}, p), p)
+  end
+
+  def extract_type(structure, p)
+    return structure if p.type_expr.nil?
+    structure[:type] = typeexpr_to_string(p.type_expr)
+    structure
+  end
+
+  def extract_default(structure, p)
+    value_expr = p.value
+    return structure if value_expr.nil?
+    default_value = value_as_literal(value_expr)
+    if !default_value.nil?
+      structure[:default_literal] = default_value
+      return structure
+    end
+    structure[:default_source] = extract_value_source(value_expr)
+    structure
+  end
+
+  def typeexpr_to_string(type_expr)
+    begin
+      type_parser.interpret_any(type_expr).to_s
+    rescue Puppet::ParseError
+      # type is to complex - contains expressions that are not literal
+      nil
+    end
+  end
+
+  def value_as_literal(value_expr)
+    catch(:not_literal) do
+      return literal_evaluator.literal(value_expr)
+    end
+    nil
+  end
+
+  # Extracts the source for the expression
+  def extract_value_source(value_expr)
+    position = Puppet::Pops::Adapters::SourcePosAdapter.adapt(value_expr)
+    position.extract_tree_text
+  end
+end

data/lib/puppet/info_service.rb

@@ -0,0 +1,7 @@
+
+module Puppet::InfoService
+  require 'puppet/info_service/class_information_service'
+  def self.classes_per_environment(env_file_hash)
+    Puppet::InfoService::ClassInformationService.new.classes_per_environment(env_file_hash)
+  end
+end

data/lib/puppet/module_tool/applications/unpacker.rb

@@ -93,7 +93,7 @@ module Puppet::ModuleTool
       # @api private
       # @return [String] path to temporary unpacking location
       def tmpdir
-        @dir ||= Dir.mktmpdir('tmp-unpacker', Puppet::Forge::Cache.base_path)
+        @dir ||= Dir.mktmpdir('tmp', Puppet::Forge::Cache.base_path)
       end
     end
   end

data/lib/puppet/module_tool/metadata.rb

@@ -15,15 +15,16 @@ module Puppet::ModuleTool
     attr_accessor :module_name
 
     DEFAULTS = {
-      'name'         => nil,
-      'version'      => nil,
-      'author'       => nil,
-      'summary'      => nil,
-      'license'      => 'Apache-2.0',
-      'source'       => '',
-      'project_page' => nil,
-      'issues_url'   => nil,
-      'dependencies' => Set.new.freeze,
+      'name'          => nil,
+      'version'       => nil,
+      'author'        => nil,
+      'summary'       => nil,
+      'license'       => 'Apache-2.0',
+      'source'        => '',
+      'project_page'  => nil,
+      'issues_url'    => nil,
+      'dependencies'  => Set.new.freeze,
+      'data_provider' => nil,
     }
 
     def initialize
@@ -52,6 +53,7 @@ module Puppet::ModuleTool
       process_name(data) if data['name']
       process_version(data) if data['version']
       process_source(data) if data['source']
+      process_data_provider(data) if data['data_provider']
       merge_dependencies(data) if data['dependencies']
 
       @data.merge!(data)
@@ -128,6 +130,10 @@ module Puppet::ModuleTool
       validate_version(data['version'])
     end
 
+    def process_data_provider(data)
+      validate_data_provider(data['data_provider'])
+    end
+
     # Do basic parsing of the source parameter.  If the source is hosted on
     # GitHub, we can predict sensible defaults for both project_page and
     # issues_url.
@@ -188,6 +194,23 @@ module Puppet::ModuleTool
       raise ArgumentError, "Invalid 'version' field in metadata.json: #{err}"
     end
 
+    # Validates that the given _value_ is a symbolic name that starts with a letter
+    # and then contains only letters, digits, or underscore. Will raise an ArgumentError
+    # if that's not the case.
+    #
+    # @param value [Object] The value to be tested
+    def validate_data_provider(value)
+      err = nil
+      if value.is_a?(String)
+        unless value =~ /^[a-zA-Z][a-zA-Z0-9_]*$/
+          err = value =~ /^[a-zA-Z]/ ? 'contains non-alphanumeric characters' : 'must begin with a letter'
+        end
+      else
+        err = 'must be a string'
+      end
+      raise ArgumentError, "field 'data_provider' #{err}" if err
+    end
+
     # Validates that the version range can be parsed by Semantic.
     def validate_version_range(version_range)
       Semantic::VersionRange.parse(version_range)