puppet 4.0.0 → 4.5.0

data/lib/puppet/info_service/class_information_service.rb

@@ -0,0 +1,112 @@
+require 'puppet'
+require 'puppet/pops'
+require 'puppet/pops/evaluator/json_strict_literal_evaluator'
+
+class Puppet::InfoService::ClassInformationService
+
+  def initialize
+    @file_to_result = {}
+    @parser = Puppet::Pops::Parser::EvaluatingParser.new()
+  end
+
+  def classes_per_environment(env_file_hash)
+    # In this version of puppet there is only one way to parse manifests, as feature switches per environment
+    # are added or removed, this logic needs to change to compute the result per environment with the correct
+    # feature flags in effect.
+
+    unless env_file_hash.is_a?(Hash)
+      raise ArgumentError, 'Given argument must be a Hash'
+    end
+
+    result = {}
+
+    # for each environment
+    #   for each file
+    #     if file already processed, use last result or error
+    #
+    env_file_hash.each do |env, files|
+      env_result = result[env] = {}
+      files.each do |f|
+        env_result[f] = result_of(f)
+      end
+    end
+    result
+  end
+
+  private
+
+  def type_parser
+    # Safe to cache this as it would otherwise constantly build up the visitor cache
+    @@type_parser ||= Puppet::Pops::Types::TypeParser.new
+  end
+
+  def literal_evaluator
+    @@literal_evaluator ||= Puppet::Pops::Evaluator::JsonStrictLiteralEvaluator.new
+  end
+
+  def result_of(f)
+    entry =  @file_to_result[f]
+    if entry.nil?
+      @file_to_result[f] = entry = parse_file(f)
+    end
+    entry
+  end
+
+  def parse_file(f)
+    return {:error => "The file #{f} does not exist"} unless Puppet::FileSystem.exist?(f)
+
+    begin
+      parse_result = @parser.parse_file(f)
+      {:classes =>
+        parse_result.definitions.select {|d| d.is_a?(Puppet::Pops::Model::HostClassDefinition)}.map do |d|
+          {:name   => d.name,
+           :params => params = d.parameters.map {|p| extract_param(p) }
+          }
+        end
+      }
+    rescue StandardError => e
+      {:error => e.message }
+    end
+  end
+
+  def extract_param(p)
+    extract_default(extract_type({:name => p.name}, p), p)
+  end
+
+  def extract_type(structure, p)
+    return structure if p.type_expr.nil?
+    structure[:type] = typeexpr_to_string(p.type_expr)
+    structure
+  end
+
+  def extract_default(structure, p)
+    value_expr = p.value
+    return structure if value_expr.nil?
+    default_value = value_as_literal(value_expr)
+    structure[:default_literal] = default_value unless default_value.nil?
+    structure[:default_source] = extract_value_source(value_expr)
+    structure
+  end
+
+  def typeexpr_to_string(type_expr)
+    begin
+      type_parser.interpret_any(type_expr, nil).to_s
+    rescue Puppet::ParseError
+      # type is to complex - contains expressions that are not literal
+      nil
+    end
+  end
+
+  def value_as_literal(value_expr)
+    catch(:not_literal) do
+      return literal_evaluator.literal(value_expr)
+    end
+    nil
+  end
+
+  # Extracts the source for the expression
+  def extract_value_source(value_expr)
+    position = Puppet::Pops::Adapters::SourcePosAdapter.adapt(value_expr)
+    position.extract_tree_text
+  end
+end

data/lib/puppet/info_service.rb

@@ -0,0 +1,7 @@
+
+module Puppet::InfoService
+  require 'puppet/info_service/class_information_service'
+  def self.classes_per_environment(env_file_hash)
+    Puppet::InfoService::ClassInformationService.new.classes_per_environment(env_file_hash)
+  end
+end

data/lib/puppet/loaders.rb

@@ -10,9 +10,13 @@ module Puppet
       require 'puppet/pops/loader/dependency_loader'
       require 'puppet/pops/loader/null_loader'
       require 'puppet/pops/loader/static_loader'
+      require 'puppet/pops/loader/runtime3_type_loader'
       require 'puppet/pops/loader/ruby_function_instantiator'
+      require 'puppet/pops/loader/puppet_function_instantiator'
+      require 'puppet/pops/loader/type_definition_instantiator'
       require 'puppet/pops/loader/loader_paths'
       require 'puppet/pops/loader/simple_environment_loader'
+      require 'puppet/pops/loader/predefined_loader'
     end
   end
 

data/lib/puppet/metatype/manager.rb

@@ -20,6 +20,16 @@ module Manager
     }
   end
 
+  # Clears any types that were used but absent when types were last loaded.
+  # @note Used after each catalog compile when always_retry_plugins is false
+  # @api private
+  #
+  def clear_misses
+    unless @types.nil?
+      @types.delete_if {|_, v| v.nil? }
+    end
+  end
+
   # Iterates over all already loaded Type subclasses.
   # @yield [t] a block receiving each type
   # @yieldparam t [Puppet::Type] each defined type
@@ -116,6 +126,10 @@ module Manager
     klass.providerloader.loadall Puppet.lookup(:current_environment)
     klass.providify unless klass.providers.empty?
 
+    loc = block_given? ? block.source_location : nil
+    uri = loc.nil? ? nil : URI("#{Puppet::Util.path_to_uri(loc[0])}?line=#{loc[1]}")
+    Puppet::Pops::Loaders.register_runtime3_type(name, uri)
+
     klass
   end
 
@@ -146,16 +160,19 @@ module Manager
     # We are overwhelmingly symbols here, which usually match, so it is worth
     # having this special-case to return quickly.  Like, 25K symbols vs. 300
     # strings in this method. --daniel 2012-07-17
-    return @types[name] if @types[name]
+    return @types[name] if @types.include? name
 
     # Try mangling the name, if it is a string.
     if name.is_a? String
       name = name.downcase.intern
-      return @types[name] if @types[name]
+      return @types[name] if @types.include? name
     end
     # Try loading the type.
     if typeloader.load(name, Puppet.lookup(:current_environment))
       Puppet.warning "Loaded puppet/type/#{name} but no class was created" unless @types.include? name
+    elsif !Puppet[:always_retry_plugins]
+      # PUP-5482 - Only look for a type once if plugin retry is disabled
+      @types[name] = nil
     end
 
     # ...and I guess that is that, eh.

data/lib/puppet/module.rb

@@ -48,7 +48,7 @@ class Puppet::Module
 
   def self.is_module_namespaced_name?(name)
     # it must match the full module name according to forge validator
-    return true if name =~ /^[a-zA-Z0-9]+[-][a-z][a-z0-9_]*$/   
+    return true if name =~ /^[a-zA-Z0-9]+[-][a-z][a-z0-9_]*$/
     return false
   end
 
@@ -56,7 +56,7 @@ class Puppet::Module
   attr_writer :environment
 
   attr_accessor :dependencies, :forge_name
-  attr_accessor :source, :author, :version, :license, :puppetversion, :summary, :description, :project_page
+  attr_accessor :source, :author, :version, :license, :summary, :description, :project_page
 
   def initialize(name, path, environment)
     @name = name
@@ -67,18 +67,30 @@ class Puppet::Module
 
     load_metadata if has_metadata?
 
-    validate_puppet_version
-
     @absolute_path_to_manifests = Puppet::FileSystem::PathPattern.absolute(manifests)
   end
 
+  # @deprecated The puppetversion module metadata field is no longer used.
+  def puppetversion
+    nil
+  end
+
+  # @deprecated The puppetversion module metadata field is no longer used.
+  def puppetversion=(something)
+  end
+
+  # @deprecated The puppetversion module metadata field is no longer used.
+  def validate_puppet_version
+    return
+  end
+
   def has_metadata?
     return false unless metadata_file
 
     return false unless Puppet::FileSystem.exist?(metadata_file)
 
     begin
-      metadata =  JSON.parse(File.read(metadata_file))
+      metadata =  JSON.parse(File.read(metadata_file, :encoding => 'utf-8'))
     rescue JSON::JSONError => e
       Puppet.debug("#{name} has an invalid and unparsable metadata.json file.  The parse error: #{e.message}")
       return false
@@ -133,18 +145,21 @@ class Puppet::Module
   end
 
   def load_metadata
-    @metadata = data = JSON.parse(File.read(metadata_file))
+    @metadata = data = JSON.parse(File.read(metadata_file, :encoding => 'utf-8'))
     @forge_name = data['name'].gsub('-', '/') if data['name']
 
-    [:source, :author, :version, :license, :puppetversion, :dependencies].each do |attr|
+    [:source, :author, :version, :license, :dependencies].each do |attr|
       unless value = data[attr.to_s]
-        unless attr == :puppetversion
-          raise MissingMetadata, "No #{attr} module metadata provided for #{self.name}"
-        end
+        raise MissingMetadata, "No #{attr} module metadata provided for #{self.name}"
       end
 
       if attr == :dependencies
+        unless value.is_a?(Array)
+          raise MissingMetadata, "The value for the key dependencies in the file metadata.json of the module #{self.name} must be an array, not: '#{value}'"
+        end
         value.each do |dep|
+          name = dep['name']
+          dep['name'] = name.tr('-', '/') unless name.nil?
           dep['version_requirement'] ||= '>= 0.0.0'
         end
       end
@@ -259,11 +274,10 @@ class Puppet::Module
 
     dependencies.each do |dependency|
       name = dependency['name']
-      forge_name = name.tr('-', '/')
       version_string = dependency['version_requirement'] || '>= 0.0.0'
 
       dep_mod = begin
-        environment.module_by_forge_name(forge_name)
+        environment.module_by_forge_name(name)
       rescue
         nil
       end
@@ -307,11 +321,6 @@ class Puppet::Module
     unmet_dependencies
   end
 
-  def validate_puppet_version
-    return unless puppetversion and puppetversion != Puppet.version
-    raise IncompatibleModule, "Module #{self.name} is only compatible with Puppet version #{puppetversion}, not #{Puppet.version}"
-  end
-
   def ==(other)
     self.name == other.name &&
     self.version == other.version &&

data/lib/puppet/module_tool/applications/application.rb

@@ -41,7 +41,7 @@ module Puppet::ModuleTool
         end
 
         if require_metadata && !Puppet::ModuleTool.is_module_root?(@path)
-          raise ArgumentError, "Unable to find metadata.json in module root at #{@path} See https://docs.puppetlabs.com/puppet/latest/reference/modules_publishing.html for required file format."
+          raise ArgumentError, "Unable to find metadata.json in module root at #{@path} See https://docs.puppet.com/puppet/latest/reference/modules_publishing.html for required file format."
         end
 
         metadata_path   = File.join(@path, 'metadata.json')

data/lib/puppet/module_tool/applications/builder.rb

@@ -65,9 +65,9 @@ module Puppet::ModuleTool
           gitignore = File.join(@path, '.gitignore')
 
           if File.file? pmtignore
-            @ignored_files = PathSpec.new File.read(pmtignore)
+            @ignored_files = PathSpec.new Puppet::FileSystem.read(pmtignore, :encoding => 'utf-8')
           elsif File.file? gitignore
-            @ignored_files = PathSpec.new File.read(gitignore)
+            @ignored_files = PathSpec.new Puppet::FileSystem.read(gitignore, :encoding => 'utf-8')
           else
             @ignored_files = PathSpec.new
           end

data/lib/puppet/module_tool/applications/unpacker.rb

@@ -93,7 +93,7 @@ module Puppet::ModuleTool
       # @api private
       # @return [String] path to temporary unpacking location
       def tmpdir
-        @dir ||= Dir.mktmpdir('tmp-unpacker', Puppet::Forge::Cache.base_path)
+        @dir ||= Dir.mktmpdir('tmp', Puppet::Forge::Cache.base_path)
       end
     end
   end

data/lib/puppet/module_tool/metadata.rb

@@ -15,15 +15,16 @@ module Puppet::ModuleTool
     attr_accessor :module_name
 
     DEFAULTS = {
-      'name'         => nil,
-      'version'      => nil,
-      'author'       => nil,
-      'summary'      => nil,
-      'license'      => 'Apache-2.0',
-      'source'       => '',
-      'project_page' => nil,
-      'issues_url'   => nil,
-      'dependencies' => Set.new.freeze,
+      'name'          => nil,
+      'version'       => nil,
+      'author'        => nil,
+      'summary'       => nil,
+      'license'       => 'Apache-2.0',
+      'source'        => '',
+      'project_page'  => nil,
+      'issues_url'    => nil,
+      'dependencies'  => Set.new.freeze,
+      'data_provider' => nil,
     }
 
     def initialize
@@ -52,6 +53,7 @@ module Puppet::ModuleTool
       process_name(data) if data['name']
       process_version(data) if data['version']
       process_source(data) if data['source']
+      process_data_provider(data) if data['data_provider']
       merge_dependencies(data) if data['dependencies']
 
       @data.merge!(data)
@@ -128,6 +130,10 @@ module Puppet::ModuleTool
       validate_version(data['version'])
     end
 
+    def process_data_provider(data)
+      validate_data_provider(data['data_provider'])
+    end
+
     # Do basic parsing of the source parameter.  If the source is hosted on
     # GitHub, we can predict sensible defaults for both project_page and
     # issues_url.
@@ -188,6 +194,23 @@ module Puppet::ModuleTool
       raise ArgumentError, "Invalid 'version' field in metadata.json: #{err}"
     end
 
+    # Validates that the given _value_ is a symbolic name that starts with a letter
+    # and then contains only letters, digits, or underscore. Will raise an ArgumentError
+    # if that's not the case.
+    #
+    # @param value [Object] The value to be tested
+    def validate_data_provider(value)
+      err = nil
+      if value.is_a?(String)
+        unless value =~ /^[a-zA-Z][a-zA-Z0-9_]*$/
+          err = value =~ /^[a-zA-Z]/ ? 'contains non-alphanumeric characters' : 'must begin with a letter'
+        end
+      else
+        err = 'must be a string'
+      end
+      raise ArgumentError, "field 'data_provider' #{err}" if err
+    end
+
     # Validates that the version range can be parsed by Semantic.
     def validate_version_range(version_range)
       Semantic::VersionRange.parse(version_range)

data/lib/puppet/module_tool/skeleton/templates/generator/Gemfile

@@ -1,7 +1,18 @@
-source 'https://rubygems.org'
+source ENV['GEM_SOURCE'] || 'https://rubygems.org'
 
-puppetversion = ENV.key?('PUPPET_VERSION') ? "#{ENV['PUPPET_VERSION']}" : ['>= 3.3']
+puppetversion = ENV.key?('PUPPET_VERSION') ? ENV['PUPPET_VERSION'] : ['>= 3.3']
+gem 'metadata-json-lint'
 gem 'puppet', puppetversion
-gem 'puppetlabs_spec_helper', '>= 0.8.2'
+gem 'puppetlabs_spec_helper', '>= 1.0.0'
 gem 'puppet-lint', '>= 1.0.0'
 gem 'facter', '>= 1.7.0'
+gem 'rspec-puppet'
+
+# rspec must be v2 for ruby 1.8.7
+if RUBY_VERSION >= '1.8.7' && RUBY_VERSION < '1.9'
+  gem 'rspec', '~> 2.0'
+  gem 'rake', '~> 10.0'
+else
+  # rubocop requires ruby >= 1.9
+  gem 'rubocop'
+end

data/lib/puppet/module_tool/skeleton/templates/generator/README.md.erb

@@ -2,78 +2,82 @@
 
 #### Table of Contents
 
-1. [Overview](#overview)
-2. [Module Description - What the module does and why it is useful](#module-description)
-3. [Setup - The basics of getting started with <%= metadata.name %>](#setup)
+1. [Description](#description)
+1. [Setup - The basics of getting started with <%= metadata.name %>](#setup)
     * [What <%= metadata.name %> affects](#what-<%= metadata.name %>-affects)
     * [Setup requirements](#setup-requirements)
     * [Beginning with <%= metadata.name %>](#beginning-with-<%= metadata.name %>)
-4. [Usage - Configuration options and additional functionality](#usage)
-5. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
-5. [Limitations - OS compatibility, etc.](#limitations)
-6. [Development - Guide for contributing to the module](#development)
+1. [Usage - Configuration options and additional functionality](#usage)
+1. [Reference - An under-the-hood peek at what the module is doing and how](#reference)
+1. [Limitations - OS compatibility, etc.](#limitations)
+1. [Development - Guide for contributing to the module](#development)
 
-## Overview
+## Description
 
-A one-maybe-two sentence summary of what the module does/what problem it solves.
-This is your 30 second elevator pitch for your module. Consider including
-OS/Puppet version it works with.
+Start with a one- or two-sentence summary of what the module does and/or what
+problem it solves. This is your 30-second elevator pitch for your module.
+Consider including OS/Puppet version it works with.
 
-## Module Description
-
-If applicable, this section should have a brief description of the technology
-the module integrates with and what that integration enables. This section
+You can give more descriptive information in a second paragraph. This paragraph
 should answer the questions: "What does this module *do*?" and "Why would I use
-it?"
-
-If your module has a range of functionality (installation, configuration,
-management, etc.) this is the time to mention it.
+it?" If your module has a range of functionality (installation, configuration,
+management, etc.), this is the time to mention it.
 
 ## Setup
 
-### What <%= metadata.name %> affects
+### What <%= metadata.name %> affects **OPTIONAL**
+
+If it's obvious what your module touches, you can skip this section. For
+example, folks can probably figure out that your mysql_instance module affects
+their MySQL instances.
+
+If there's more that they should know about, though, this is the place to mention:
 
 * A list of files, packages, services, or operations that the module will alter,
-  impact, or execute on the system it's installed on.
-* This is a great place to stick any warnings.
-* Can be in list or paragraph form.
+  impact, or execute.
+* Dependencies that your module automatically installs.
+* Warnings or other important notices.
 
 ### Setup Requirements **OPTIONAL**
 
 If your module requires anything extra before setting up (pluginsync enabled,
 etc.), mention it here.
 
-### Beginning with <%= metadata.name %>
+If your most recent release breaks compatibility or requires particular steps
+for upgrading, you might want to include an additional "Upgrading" section
+here.
 
-The very basic steps needed for a user to get the module up and running.
+### Beginning with <%= metadata.name %>
 
-If your most recent release breaks compatibility or requires particular steps
-for upgrading, you may wish to include an additional section here: Upgrading
-(For an example, see http://forge.puppetlabs.com/puppetlabs/firewall).
+The very basic steps needed for a user to get the module up and running. This
+can include setup steps, if necessary, or it can be an example of the most
+basic use of the module.
 
 ## Usage
 
-Put the classes, types, and resources for customizing, configuring, and doing
-the fancy stuff with your module here.
+This section is where you describe how to customize, configure, and do the
+fancy stuff with your module here. It's especially helpful if you include usage
+examples and code samples for doing things with your module.
 
 ## Reference
 
-Here, list the classes, types, providers, facts, etc contained in your module.
-This section should include all of the under-the-hood workings of your module so
-people know what the module is touching on their system but don't need to mess
-with things. (We are working on automating this section!)
+Here, include a complete list of your module's classes, types, providers,
+facts, along with the parameters for each. Users refer to this section (thus
+the name "Reference") to find specific details; most users don't read it per
+se.
 
 ## Limitations
 
-This is where you list OS compatibility, version compatibility, etc.
+This is where you list OS compatibility, version compatibility, etc. If there
+are Known Issues, you might want to include them under their own heading here.
 
 ## Development
 
 Since your module is awesome, other users will want to play with it. Let them
 know what the ground rules for contributing are.
 
-## Release Notes/Contributors/Etc **Optional**
+## Release Notes/Contributors/Etc. **Optional**
 
 If you aren't using changelog, put your release notes here (though you should
-consider using changelog). You may also add any additional sections you feel are
-necessary or important to include here. Please use the `## ` header.
+consider using changelog). You can also add any additional sections you feel
+are necessary or important to include here. Please use the `## ` header.

data/lib/puppet/module_tool/skeleton/templates/generator/Rakefile

@@ -1,17 +1,32 @@
 require 'puppetlabs_spec_helper/rake_tasks'
 require 'puppet-lint/tasks/puppet-lint'
+require 'metadata-json-lint/rake_task'
+
+if RUBY_VERSION >= '1.9'
+  require 'rubocop/rake_task'
+  RuboCop::RakeTask.new
+end
+
 PuppetLint.configuration.send('disable_80chars')
-PuppetLint.configuration.ignore_paths = ["spec/**/*.pp", "pkg/**/*.pp"]
+PuppetLint.configuration.relative = true
+PuppetLint.configuration.ignore_paths = ['spec/**/*.pp', 'pkg/**/*.pp']
 
-desc "Validate manifests, templates, and ruby files"
+desc 'Validate manifests, templates, and ruby files'
 task :validate do
   Dir['manifests/**/*.pp'].each do |manifest|
     sh "puppet parser validate --noop #{manifest}"
   end
-  Dir['spec/**/*.rb','lib/**/*.rb'].each do |ruby_file|
-    sh "ruby -c #{ruby_file}" unless ruby_file =~ /spec\/fixtures/
+  Dir['spec/**/*.rb', 'lib/**/*.rb'].each do |ruby_file|
+    sh "ruby -c #{ruby_file}" unless ruby_file =~ %r{spec/fixtures}
   end
   Dir['templates/**/*.erb'].each do |template|
     sh "erb -P -x -T '-' #{template} | ruby -c"
   end
 end
+
+desc 'Run metadata_lint, lint, validate, and spec tests.'
+task :test do
+  [:metadata_lint, :lint, :validate, :spec].each do |test|
+    Rake::Task[test].invoke
+  end
+end

data/lib/puppet/module_tool/skeleton/templates/generator/{tests → examples}/init.pp.erb

@@ -7,6 +7,6 @@
 # environment (to compare the resulting system state to the desired state).
 #
 # Learn more about module testing here:
-# http://docs.puppetlabs.com/guides/tests_smoke.html
+# https://docs.puppet.com/guides/tests_smoke.html
 #
-include <%= metadata.name %>
+include ::<%= metadata.name %>

data/lib/puppet/module_tool/skeleton/templates/generator/manifests/init.pp.erb

@@ -1,37 +1,44 @@
-# == Class: <%= metadata.name %>
+# Class: <%= metadata.name %>
+# ===========================
 #
 # Full description of class <%= metadata.name %> here.
 #
-# === Parameters
+# Parameters
+# ----------
 #
 # Document parameters here.
 #
-# [*sample_parameter*]
-#   Explanation of what this parameter affects and what it defaults to.
-#   e.g. "Specify one or more upstream ntp servers as an array."
+# * `sample parameter`
+# Explanation of what this parameter affects and what it defaults to.
+# e.g. "Specify one or more upstream ntp servers as an array."
 #
-# === Variables
+# Variables
+# ----------
 #
 # Here you should define a list of variables that this module would require.
 #
-# [*sample_variable*]
-#   Explanation of how this variable affects the function of this class and if
-#   it has a default. e.g. "The parameter enc_ntp_servers must be set by the
-#   External Node Classifier as a comma separated list of hostnames." (Note,
-#   global variables should be avoided in favor of class parameters as
-#   of Puppet 2.6.)
+# * `sample variable`
+#  Explanation of how this variable affects the function of this class and if
+#  it has a default. e.g. "The parameter enc_ntp_servers must be set by the
+#  External Node Classifier as a comma separated list of hostnames." (Note,
+#  global variables should be avoided in favor of class parameters as
+#  of Puppet 2.6.)
 #
-# === Examples
+# Examples
+# --------
 #
-#  class { '<%= metadata.name %>':
-#    servers => [ 'pool.ntp.org', 'ntp.local.company.com' ],
-#  }
+# @example
+#    class { '<%= metadata.name %>':
+#      servers => [ 'pool.ntp.org', 'ntp.local.company.com' ],
+#    }
 #
-# === Authors
+# Authors
+# -------
 #
 # Author Name <author@domain.com>
 #
-# === Copyright
+# Copyright
+# ---------
 #
 # Copyright <%= Time.now.year %> Your name here, unless otherwise noted.
 #

data/lib/puppet/module_tool/skeleton/templates/generator/spec/classes/init_spec.rb.erb

@@ -1,7 +1,6 @@
 require 'spec_helper'
 describe '<%= metadata.name %>' do
-
-  context 'with defaults for all parameters' do
+  context 'with default values for all parameters' do
     it { should contain_class('<%= metadata.name %>') }
   end
 end