puppet 4.2.3 → 4.3.0

data/lib/puppet/indirector/data_binding/none.rb

@@ -1,5 +1,8 @@
 require 'puppet/indirector/none'
 
 class Puppet::DataBinding::None < Puppet::Indirector::None
-  desc "A Dummy terminus that always returns nil for data lookups."
+  desc "A Dummy terminus that always throws :no_such_key for data lookups."
+  def find(request)
+    throw :no_such_key
+  end
 end

data/lib/puppet/indirector/file_bucket_file/file.rb

@@ -1,15 +1,23 @@
 require 'puppet/indirector/code'
 require 'puppet/file_bucket/file'
 require 'puppet/util/checksums'
+require 'puppet/util/diff'
 require 'fileutils'
 
 module Puppet::FileBucketFile
   class File < Puppet::Indirector::Code
     include Puppet::Util::Checksums
+    include Puppet::Util::Diff
 
     desc "Store files in a directory set based on their checksums."
 
     def find(request)
+      request.options[:bucket_path] ||= Puppet[:bucketdir]
+      # If filebucket mode is 'list'
+      if request.options[:list_all]
+        return nil unless ::File.exists?(request.options[:bucket_path])
+        return list(request)
+      end
       checksum, files_original_path = request_to_checksum_and_path(request)
       contents_file = path_for(request.options[:bucket_path], checksum, 'contents')
       paths_file = path_for(request.options[:bucket_path], checksum, 'paths')
@@ -18,7 +26,8 @@ module Puppet::FileBucketFile
         if request.options[:diff_with]
           other_contents_file = path_for(request.options[:bucket_path], request.options[:diff_with], 'contents')
           raise "could not find diff_with #{request.options[:diff_with]}" unless Puppet::FileSystem.exist?(other_contents_file)
-          return `diff #{Puppet::FileSystem.path_string(contents_file).inspect} #{Puppet::FileSystem.path_string(other_contents_file).inspect}`
+          raise "Unable to diff on this platform" unless Puppet[:diff] != ""
+          return diff(Puppet::FileSystem.path_string(contents_file), Puppet::FileSystem.path_string(other_contents_file))
         else
           Puppet.info "FileBucket read #{checksum}"
           model.new(Puppet::FileSystem.binread(contents_file))
@@ -28,6 +37,54 @@ module Puppet::FileBucketFile
       end
     end
 
+    def list(request)
+      if request.remote?
+        raise Puppet::Error, "Listing remote file buckets is not allowed"
+      end
+
+      fromdate = request.options[:fromdate] || "0:0:0 1-1-1970"
+      todate = request.options[:todate] || Time.now.strftime("%F %T")
+      begin
+        to = Time.parse(todate)
+      rescue ArgumentError
+        raise Puppet::Error, "Error while parsing 'todate'"
+      end
+      begin
+        from = Time.parse(fromdate)
+      rescue ArgumentError
+        raise Puppet::Error, "Error while parsing 'fromdate'"
+      end
+      # Setting hash's default value to [], needed by the following loop
+      bucket = Hash.new {[]}
+      msg = ""
+      # Get all files with mtime between 'from' and 'to'
+      Pathname.new(request.options[:bucket_path]).find { |item|
+        if item.file? and item.basename.to_s == "paths"
+          filenames = item.read.strip.split("\n")
+          filestat = Time.parse(item.stat.mtime.to_s)
+          if from <= filestat and filestat <= to
+            filenames.each do |filename|
+              bucket[filename] += [[ item.stat.mtime , item.parent.basename ]]
+            end
+          end
+        end
+      }
+      # Sort the results
+      bucket.each { |filename, contents|
+        contents.sort_by! do |item|
+          item[0]
+        end
+      }
+      # Build the output message. Sorted by names then by dates
+      bucket.sort.each { |filename,contents|
+        contents.each { |mtime, chksum|
+          date = mtime.strftime("%F %T")
+          msg += "#{chksum} #{date} #{filename}\n"
+        }
+      }
+      return model.new(msg)
+    end
+
     def head(request)
       checksum, files_original_path = request_to_checksum_and_path(request)
       contents_file = path_for(request.options[:bucket_path], checksum, 'contents')

data/lib/puppet/indirector/hiera.rb

@@ -18,6 +18,7 @@ class Puppet::Indirector::Hiera < Puppet::Indirector::Terminus
   def find(request)
     not_found = Object.new
     options = request.options
+    Puppet.debug { "Performing a hiera indirector lookup of #{request.key} with options #{options.inspect}" }
     value = hiera.lookup(request.key, not_found, Hiera::Scope.new(options[:variables]), nil, convert_merge(options[:merge]))
     throw :no_such_key if value.equal?(not_found)
     value
@@ -34,8 +35,11 @@ class Puppet::Indirector::Hiera < Puppet::Indirector::Terminus
   def convert_merge(merge)
     case merge
     when nil
+    when 'first'
       # Nil is OK. Defaults to Hiera :priority
       nil
+    when Puppet::Pops::MergeStrategy
+      convert_merge(merge.configuration)
     when 'unique'
       # Equivalent to Hiera :array
       :array

data/lib/puppet/indirector/json.rb

@@ -17,7 +17,7 @@ class Puppet::Indirector::JSON < Puppet::Indirector::Terminus
 
     Puppet::Util.replace_file(filename, 0660) {|f| f.print to_json(request.instance) }
   rescue TypeError => detail
-    Puppet.log_exception "Could not save #{self.name} #{request.key}: #{detail}"
+    Puppet.log_exception(detail, "Could not save #{self.name} #{request.key}: #{detail}")
   end
 
   def destroy(request)

data/lib/puppet/indirector/msgpack.rb

@@ -23,7 +23,7 @@ class Puppet::Indirector::Msgpack < Puppet::Indirector::Terminus
 
     Puppet::Util.replace_file(filename, 0660) {|f| f.print to_msgpack(request.instance) }
   rescue TypeError => detail
-    Puppet.log_exception "Could not save #{self.name} #{request.key}: #{detail}"
+    Puppet.log_exception(detail, "Could not save #{self.name} #{request.key}: #{detail}")
   end
 
   def destroy(request)

data/lib/puppet/indirector/request.rb

@@ -243,15 +243,14 @@ class Puppet::Indirector::Request
       @port = uri.port.to_i
     end
 
-    @protocol = uri.scheme
-
-    if uri.scheme == 'puppet'
-      @key = URI.unescape(uri.path.sub(/^\//, ''))
-      return
+    # filebucket:// is only used internally to pass request details
+    # from Dipper objects to the indirector. The wire always uses HTTPS.
+    if uri.scheme == 'filebucket'
+      @protocol = 'https'
+    else
+      @protocol = uri.scheme
     end
 
-    env, indirector, @key = URI.unescape(uri.path.sub(/^\//, '')).split('/',3)
-    @key ||= ''
-    self.environment = env unless env == ''
+    @key = URI.unescape(uri.path.sub(/^\//, ''))
   end
 end

data/lib/puppet/indirector/resource_type/parser.rb

@@ -27,7 +27,7 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
       krt = resource_types_in(request.environment)
 
       # This is a bit ugly.
-      [:hostclass, :definition, :node].each do |type|
+      [:hostclass, :definition, :application, :node].each do |type|
         # We have to us 'find_<type>' here because it will
         # load any missing types from disk, whereas the plain
         # '<type>' method only returns from memory.
@@ -66,14 +66,16 @@ class Puppet::Indirector::ResourceType::Parser < Puppet::Indirector::Code
             krt.hostclasses.values
           when "defined_type"
             krt.definitions.values
+          when "application"
+            krt.applications.values
           when "node"
             krt.nodes.values
           when nil
-            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.nodes.values]
+            result_candidates = [krt.hostclasses.values, krt.definitions.values, krt.applications.values, krt.nodes.values]
           else
             raise ArgumentError, "Unrecognized kind filter: " +
                       "'#{request.options[:kind]}', expected one " +
-                      " of 'class', 'defined_type', or 'node'."
+                      " of 'class', 'defined_type', 'application', or 'node'."
         end
 
       result = result_candidates.flatten.reject { |t| t.name == "" }

data/lib/puppet/info_service.rb

@@ -0,0 +1,7 @@
+
+module Puppet::InfoService
+  require 'puppet/info_service/class_information_service'
+  def self.classes_per_environment(env_file_hash)
+    Puppet::InfoService::ClassInformationService.new.classes_per_environment(env_file_hash)
+  end
+end

data/lib/puppet/info_service/class_information_service.rb

@@ -0,0 +1,111 @@
+require 'puppet'
+require 'puppet/pops'
+require 'puppet/pops/evaluator/literal_evaluator'
+
+class Puppet::InfoService::ClassInformationService
+
+  def initialize
+    @file_to_result = {}
+    @parser = Puppet::Pops::Parser::EvaluatingParser.new()
+  end
+
+  def classes_per_environment(env_file_hash)
+    # In this version of puppet there is only one way to parse manifests, as feature switches per environment
+    # are added or removed, this logic needs to change to compute the result per environment with the correct
+    # feature flags in effect.
+
+    unless env_file_hash.is_a?(Hash)
+      raise ArgumentError, 'Given argument must be a Hash'
+    end
+
+    result = {}
+
+    # for each environment
+    #   for each file
+    #     if file already processed, use last result or error
+    #
+    env_file_hash.each do |env, files|
+      env_result = result[env] = {}
+      files.each do |f|
+        env_result[f] = result_of(f)
+      end
+    end
+    result
+  end
+
+  private
+
+  def type_parser
+    # Safe to cache this as it would otherwise constantly build up the visitor cache
+    @@type_parser ||= Puppet::Pops::Types::TypeParser.new
+  end
+
+  def literal_evaluator
+    @@literal_evaluator ||= Puppet::Pops::Evaluator::LiteralEvaluator.new
+  end
+
+  def result_of(f)
+    entry =  @file_to_result[f]
+    if entry.nil?
+      @file_to_result[f] = entry = parse_file(f)
+    end
+    entry
+  end
+
+  def parse_file(f)
+    begin
+      parse_result = @parser.parse_file(f)
+      parse_result.definitions.select {|d| d.is_a?(Puppet::Pops::Model::HostClassDefinition)}.map do |d|
+        {:name   => d.name,
+         :params => params = d.parameters.map {|p| extract_param(p) }
+        }
+      end
+    rescue StandardError => e
+      {:error => e.message }
+    end
+  end
+
+  def extract_param(p)
+    extract_default(extract_type({:name => p.name}, p), p)
+  end
+
+  def extract_type(structure, p)
+    return structure if p.type_expr.nil?
+    structure[:type] = typeexpr_to_string(p.type_expr)
+    structure
+  end
+
+  def extract_default(structure, p)
+    value_expr = p.value
+    return structure if value_expr.nil?
+    default_value = value_as_literal(value_expr)
+    if !default_value.nil?
+      structure[:default_literal] = default_value
+      return structure
+    end
+    structure[:default_source] = extract_value_source(value_expr)
+    structure
+  end
+
+  def typeexpr_to_string(type_expr)
+    begin
+      type_parser.interpret_any(type_expr).to_s
+    rescue Puppet::ParseError
+      # type is to complex - contains expressions that are not literal
+      nil
+    end
+  end
+
+  def value_as_literal(value_expr)
+    catch(:not_literal) do
+      return literal_evaluator.literal(value_expr)
+    end
+    nil
+  end
+
+  # Extracts the source for the expression
+  def extract_value_source(value_expr)
+    position = Puppet::Pops::Adapters::SourcePosAdapter.adapt(value_expr)
+    position.extract_tree_text
+  end
+end

data/lib/puppet/module_tool/metadata.rb

@@ -15,15 +15,16 @@ module Puppet::ModuleTool
     attr_accessor :module_name
 
     DEFAULTS = {
-      'name'         => nil,
-      'version'      => nil,
-      'author'       => nil,
-      'summary'      => nil,
-      'license'      => 'Apache-2.0',
-      'source'       => '',
-      'project_page' => nil,
-      'issues_url'   => nil,
-      'dependencies' => Set.new.freeze,
+      'name'          => nil,
+      'version'       => nil,
+      'author'        => nil,
+      'summary'       => nil,
+      'license'       => 'Apache-2.0',
+      'source'        => '',
+      'project_page'  => nil,
+      'issues_url'    => nil,
+      'dependencies'  => Set.new.freeze,
+      'data_provider' => nil,
     }
 
     def initialize
@@ -52,6 +53,7 @@ module Puppet::ModuleTool
       process_name(data) if data['name']
       process_version(data) if data['version']
       process_source(data) if data['source']
+      process_data_provider(data) if data['data_provider']
       merge_dependencies(data) if data['dependencies']
 
       @data.merge!(data)
@@ -128,6 +130,10 @@ module Puppet::ModuleTool
       validate_version(data['version'])
     end
 
+    def process_data_provider(data)
+      validate_data_provider(data['data_provider'])
+    end
+
     # Do basic parsing of the source parameter.  If the source is hosted on
     # GitHub, we can predict sensible defaults for both project_page and
     # issues_url.
@@ -188,6 +194,23 @@ module Puppet::ModuleTool
       raise ArgumentError, "Invalid 'version' field in metadata.json: #{err}"
     end
 
+    # Validates that the given _value_ is a symbolic name that starts with a letter
+    # and then contains only letters, digits, or underscore. Will raise an ArgumentError
+    # if that's not the case.
+    #
+    # @param value [Object] The value to be tested
+    def validate_data_provider(value)
+      err = nil
+      if value.is_a?(String)
+        unless value =~ /^[a-zA-Z][a-zA-Z0-9_]*$/
+          err = value =~ /^[a-zA-Z]/ ? 'contains non-alphanumeric characters' : 'must begin with a letter'
+        end
+      else
+        err = 'must be a string'
+      end
+      raise ArgumentError, "field 'data_provider' #{err}" if err
+    end
+
     # Validates that the version range can be parsed by Semantic.
     def validate_version_range(version_range)
       Semantic::VersionRange.parse(version_range)

data/lib/puppet/module_tool/skeleton/templates/generator/README.md.erb

@@ -2,78 +2,82 @@
 
 #### Table of Contents
 
-1. [Overview](#overview)
-2. [Module Description - What the module does and why it is useful](#module-description)
-3. [Setup - The basics of getting started with <%= metadata.name %>](#setup)
+1. [Description](#description)
+1. [Setup - The basics of getting started with <%= metadata.name %>](#setup)
     * [What <%= metadata.name %> affects](#what-<%= metadata.name %>-affects)
     * [Setup requirements](#setup-requirements)
     * [Beginning with <%= metadata.name %>](#beginning-with-<%= metadata.name %>)
-4. [Usage - Configuration options and additional functionality](#usage)
-5. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
-5. [Limitations - OS compatibility, etc.](#limitations)
-6. [Development - Guide for contributing to the module](#development)
+1. [Usage - Configuration options and additional functionality](#usage)
+1. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
+1. [Limitations - OS compatibility, etc.](#limitations)
+1. [Development - Guide for contributing to the module](#development)
 
-## Overview
+## Description
 
-A one-maybe-two sentence summary of what the module does/what problem it solves.
-This is your 30 second elevator pitch for your module. Consider including
-OS/Puppet version it works with.
+Start with a one- or two-sentence summary of what the module does and/or what
+problem it solves. This is your 30-second elevator pitch for your module.
+Consider including OS/Puppet version it works with.
 
-## Module Description
-
-If applicable, this section should have a brief description of the technology
-the module integrates with and what that integration enables. This section
+You can give more descriptive information in a second paragraph. This paragraph
 should answer the questions: "What does this module *do*?" and "Why would I use
-it?"
-
-If your module has a range of functionality (installation, configuration,
-management, etc.) this is the time to mention it.
+it?" If your module has a range of functionality (installation, configuration,
+management, etc.), this is the time to mention it.
 
 ## Setup
 
-### What <%= metadata.name %> affects
+### What <%= metadata.name %> affects **OPTIONAL**
+
+If it's obvious what your module touches, you can skip this section. For
+example, folks can probably figure out that your mysql_instance module affects
+their MySQL instances.
+
+If there's more that they should know about, though, this is the place to mention:
 
 * A list of files, packages, services, or operations that the module will alter,
-  impact, or execute on the system it's installed on.
-* This is a great place to stick any warnings.
-* Can be in list or paragraph form.
+  impact, or execute.
+* Dependencies that your module automatically installs.
+* Warnings or other important notices.
 
 ### Setup Requirements **OPTIONAL**
 
 If your module requires anything extra before setting up (pluginsync enabled,
 etc.), mention it here.
 
-### Beginning with <%= metadata.name %>
+If your most recent release breaks compatibility or requires particular steps
+for upgrading, you might want to include an additional "Upgrading" section
+here.
 
-The very basic steps needed for a user to get the module up and running.
+### Beginning with <%= metadata.name %>
 
-If your most recent release breaks compatibility or requires particular steps
-for upgrading, you may wish to include an additional section here: Upgrading
-(For an example, see http://forge.puppetlabs.com/puppetlabs/firewall).
+The very basic steps needed for a user to get the module up and running. This
+can include setup steps, if necessary, or it can be an example of the most
+basic use of the module.
 
 ## Usage
 
-Put the classes, types, and resources for customizing, configuring, and doing
-the fancy stuff with your module here.
+This section is where you describe how to customize, configure, and do the
+fancy stuff with your module here. It's especially helpful if you include usage
+examples and code samples for doing things with your module.
 
 ## Reference
 
-Here, list the classes, types, providers, facts, etc contained in your module.
-This section should include all of the under-the-hood workings of your module so
-people know what the module is touching on their system but don't need to mess
-with things. (We are working on automating this section!)
+Here, include a complete list of your module's classes, types, providers,
+facts, along with the parameters for each. Users refer to this section (thus
+the name "Reference") to find specific details; most users don't read it per
+se.
 
 ## Limitations
 
-This is where you list OS compatibility, version compatibility, etc.
+This is where you list OS compatibility, version compatibility, etc. If there
+are Known Issues, you might want to include them under their own heading here.
 
 ## Development
 
 Since your module is awesome, other users will want to play with it. Let them
 know what the ground rules for contributing are.
 
-## Release Notes/Contributors/Etc **Optional**
+## Release Notes/Contributors/Etc. **Optional**
 
 If you aren't using changelog, put your release notes here (though you should
-consider using changelog). You may also add any additional sections you feel are
-necessary or important to include here. Please use the `## ` header.
+consider using changelog). You can also add any additional sections you feel
+are necessary or important to include here. Please use the `## ` header.