puppet 5.4.0 → 5.5.0

data/Gemfile

@@ -49,9 +49,7 @@ group(:development, :test) do
   gem "yarjuf", "~> 2.0"
 
   # json-schema does not support windows, so omit it from the platforms list
-  # json-schema uses multi_json, but chokes with multi_json 1.7.9, so prefer 1.7.7
-  gem "multi_json", "1.7.7", :require => false, :platforms => [:ruby, :jruby]
-  gem "json-schema", "2.1.1", :require => false, :platforms => [:ruby, :jruby]
+  gem "json-schema", "~> 2.0", :require => false, :platforms => [:ruby, :jruby]
 
   if RUBY_VERSION >= '2.0'
     # pin rubocop as 0.50 requires a higher version of the rainbow gem (see below)
@@ -74,6 +72,8 @@ group(:development, :test) do
   gem 'webmock', '~> 1.24'
   gem 'vcr', '~> 2.9'
   gem "hiera-eyaml", :require => false
+
+  gem 'memory_profiler', :platforms => [:mri_21, :mri_22, :mri_23, :mri_24, :mri_25]
 end
 
 group(:development) do

data/MAINTAINERS

@@ -33,34 +33,15 @@
       "github": "Iristyle",
       "name": "Ethan J. Brown"
     },
-    {
-      "github": "HAIL9000",
-      "email": "hailee@puppet.com",
-      "name": "Hailee Kenney"
-    },
     {
       "github": "er0ck",
       "email": "eric.thompson@puppet.com",
       "name": "Eric Thompson"
     },
-    {
-      "github": "johnduarte",
-      "email": "john.duarte@puppet.com",
-      "name": "John Duarte"
-    },
-    {
-      "github": "adrienthebo",
-      "name": "Adrien Thebo"
-    },
     {
       "github": "jtappa",
       "email": "jorie@puppet.com",
       "name": "Jorie Tappa"
-    },
-    {
-      "github": "MosesMendoza",
-      "email": "moses@puppet.com",
-      "name": "Moses Mendoza"
     }
   ]
 }

data/README.md

@@ -23,26 +23,26 @@ HTTP API
 Installation
 ------------
 
-The best way to run Puppet is with [Puppet Enterprise](https://puppet.com/puppet/puppet-enterprise),
+The best way to run Puppet is with [Puppet Enterprise (PE)](https://puppet.com/puppet/puppet-enterprise),
 which also includes orchestration features, a web console, and professional support.
-[The PE documentation is available here.](https://puppet.com/docs/pe/latest)
+The PE documentation is [available here.](https://puppet.com/docs/pe/latest)
 
 To install an open source release of Puppet,
 [see the installation guide on the docs site.](http://puppet.com/docs/puppet/5.4/install_pre.html)
 
 If you need to run Puppet from source as a tester or developer,
-[see the running from source guide on the docs site.](https://docs.puppet.com/puppet/3.8/from_source.html)
+see the [Running Puppet from Source](https://docs.puppet.com/puppet/3.8/from_source.html) guide on the docs site.
 
 Developing and Contributing
 ------
 
 We'd love to get contributions from you! For a quick guide to getting your
-system setup for developing take a look at our [Quickstart
+system setup for developing, take a look at our [Quickstart
 Guide](https://github.com/puppetlabs/puppet/blob/master/docs/quickstart.md). Once you are up and running, take a look at the
 [Contribution Documents](https://github.com/puppetlabs/puppet/blob/master/CONTRIBUTING.md) to see how to get your changes merged
 in.
 
-For more complete docs on developing with puppet you can take a look at the
+For more complete docs on developing with Puppet, take a look at the
 rest of the [developer documents](https://github.com/puppetlabs/puppet/blob/master/docs/index.md).
 
 License

data/Rakefile

@@ -19,6 +19,7 @@ rescue LoadError
 end
 
 require 'rake'
+require 'open3'
 
 Dir['tasks/**/*.rake'].each { |t| load t }
 
@@ -110,6 +111,31 @@ task(:commits) do
   end
 end
 
+desc "verify that changed files are clean of Ruby warnings"
+task(:warnings) do
+  # This rake task looks at all files modified in this branch. This is
+  # accomplished by using the TRAVIS_COMMIT_RANGE environment variable, which
+  # is present in travis CI and populated with the range of commits the PR
+  # contains. If not available, this falls back to `master..HEAD` as a next
+  # best bet as `master` is unlikely to ever be absent.
+  commit_range = ENV['TRAVIS_COMMIT_RANGE'].nil? ? 'master...HEAD' : ENV['TRAVIS_COMMIT_RANGE']
+  ruby_files_ok = true
+  puts "Checking modified files #{commit_range}"
+  %x{git diff --name-only #{commit_range}}.each_line do |modified_file|
+    modified_file.chomp!
+    next unless File.extname(modified_file) == '.rb'
+    puts modified_file
+
+    stdout, stderr, _ = Open3.capture3("ruby -wc \"#{modified_file}\"")
+    unless stderr.empty?
+      ruby_files_ok = false
+      puts stderr
+    end
+    puts stdout
+  end
+  raise "One or more ruby files contain warnings." unless ruby_files_ok
+end
+
 if Rake.application.top_level_tasks.grep(/^gettext:/).any?
   begin
     spec = Gem::Specification.find_by_name 'gettext-setup'

data/conf/auth.conf

@@ -100,6 +100,11 @@ path ~ ^/puppet/v3/report/([^/]+)$
 method save
 allow $1
 
+# allow all nodes to update their own facts
+path ~ ^/puppet/v3/facts/([^/]+)$
+method save
+allow $1
+
 # Allow all nodes to access all file services; this is necessary for
 # pluginsync, file serving from modules, and file serving from custom
 # mount points (see fileserver.conf). Note that the `/file` prefix matches

data/ext/project_data.yaml

@@ -23,6 +23,7 @@ gem_runtime_dependencies:
   # semantic_puppet: ['>= 0.1.3', '< 2']
   fast_gettext: '~> 1.1.2'
   locale: '~> 2.1'
+  multi_json: '~> 1.10'
 gem_rdoc_options:
   - --title
   - "Puppet - Configuration Management"

data/ext/regexp_nodes/regexp_nodes.rb

@@ -89,7 +89,6 @@ class ExternalNode
   # parameters
   def initialize(hostname, classdir = 'classes/', parameterdir = 'parameters/', environmentdir = 'environment/')
     # instance variables that contain the lists of classes and parameters
-    @hostname
     @classes = Set.new
     @parameters = Hash.new("unknown")  # sets a default value of "unknown"
     @environment = "production"
@@ -134,13 +133,13 @@ class ExternalNode
     patternlist = []
 
     begin
-      open(filepath).each { |l|
+      open(filepath).each do |l|
         l.chomp!
 
         next if l =~ /^$/
         next if l =~ /^#/
 
-		if l =~ /^\s*(\S+)/
+        if l =~ /^\s*(\S+)/
           m = Regexp.last_match
           log("found a non-comment line, transforming [#{l}] into [#{m[1]}]")
           l.gsub!(l,m[1])
@@ -151,7 +150,7 @@ class ExternalNode
         pattern = %r{#{l}}
         patternlist <<  pattern
         log("appending [#{pattern}] to patternlist for [#{filepath}]")
-      }
+      end
     rescue StandardError
       log("Problem reading #{filepath}: #{$!}",:err)
       exit(1)

data/lib/puppet/application/apply.rb

@@ -245,7 +245,7 @@ Copyright (c) 2011 Puppet Inc., LLC Licensed under the Apache 2.0 License
             $stderr.puts _("%{file} is not readable") % { file: file }
             exit(63)
           end
-          node.classes = Puppet::FileSystem.read(file, :encoding => 'utf-8').split(/[\s\n]+/)
+          node.classes = Puppet::FileSystem.read(file, :encoding => 'utf-8').split(/[\s]+/)
         end
       end
 

data/lib/puppet/application/cert.rb

@@ -269,8 +269,10 @@ Copyright (c) 2011 Puppet Inc., LLC Licensed under the Apache 2.0 License
     end
     begin
       if subcommand == :destroy
+        raise _("Refusing to destroy all certs, provide an explicit list of certs to destroy") if hosts == :all
+
         signed_hosts = hosts - @ca.waiting?
-        apply(@ca, :revoke, options.merge(:to => signed_hosts))
+        apply(@ca, :revoke, options.merge(:to => signed_hosts)) unless signed_hosts.empty?
       end
       apply(@ca, subcommand, options.merge(:to => hosts, :digest => @digest))
     rescue => detail

data/lib/puppet/application/device.rb

@@ -24,13 +24,16 @@ class Puppet::Application::Device < Puppet::Application
     end
 
     {
+      :apply => nil,
       :waitforcert => nil,
       :detailed_exitcodes => false,
       :verbose => false,
       :debug => false,
       :centrallogs => false,
       :setdest => false,
+      :resource => false,
       :target => nil,
+      :to_yaml => false,
     }.each do |opt,val|
       options[opt] = val
     end
@@ -40,12 +43,18 @@ class Puppet::Application::Device < Puppet::Application
 
   option("--centrallogging")
   option("--debug","-d")
+  option("--resource","-r")
+  option("--to_yaml","-y")
   option("--verbose","-v")
 
   option("--detailed-exitcodes") do |arg|
     options[:detailed_exitcodes] = true
   end
 
+  option("--apply MANIFEST") do |arg|
+    options[:apply] = arg.to_s
+  end
+
   option("--logdest DEST", "-l DEST") do |arg|
     handle_logdest_arg(arg)
   end
@@ -85,6 +94,7 @@ USAGE
   puppet device [-d|--debug] [--detailed-exitcodes] [--deviceconfig <file>]
                 [-h|--help] [-l|--logdest syslog|<file>|console]
                 [-v|--verbose] [-w|--waitforcert <seconds>]
+                [-a|--apply <file>] [-r|--resource <type> [name]]
                 [-t|--target <device>] [--user=<user>] [-V|--version]
 
 
@@ -150,10 +160,21 @@ you can specify '--server <servername>' as an argument.
   appending nature of logging. It must be appended manually to make the content
   valid JSON.
 
+* --apply:
+  Apply a manifest against a remote target. Target must be specified.
+
+* --resource:
+  Displays a resource state as Puppet code, roughly equivalent to
+  `puppet resource`.  Can be filterd by title. Requires --target be specified.
+
 * --target:
   Target a specific device/certificate in the device.conf. Doing so will perform a
   device run against only that device/certificate.
 
+* --to_yaml:
+  Output found resources in yaml format, suitable to use with Hiera and
+  create_resources.
+
 * --user:
   The user to run as.
 
@@ -170,7 +191,7 @@ you can specify '--server <servername>' as an argument.
 
 EXAMPLE
 -------
-      $ puppet device --server puppet.domain.com
+      $ puppet device --target remotehost --verbose
 
 AUTHOR
 ------
@@ -182,10 +203,24 @@ COPYRIGHT
 Copyright (c) 2011 Puppet Inc., LLC
 Licensed under the Apache 2.0 License
       HELP
-    end
+  end
 
 
   def main
+    if options[:resource] and !options[:target]
+      Puppet.err _("resource command requires target")
+      exit(1)
+    end
+    unless options[:apply].nil?
+      if options[:target].nil?
+        Puppet.err _("missing argument: --target is required when using --apply")
+        exit(1)
+      end
+      unless File.file?(options[:apply])
+        Puppet.err _("%{file} does not exist, cannot apply") % { file: options[:apply] }
+        exit(1)
+      end
+    end
     vardir = Puppet[:vardir]
     confdir = Puppet[:confdir]
     certname = Puppet[:certname]
@@ -212,27 +247,61 @@ Licensed under the Apache 2.0 License
           # Handle nil scheme & port
           scheme = "#{device_url.scheme}://" if device_url.scheme
           port = ":#{device_url.port}" if device_url.port
-          Puppet.info _("starting applying configuration to %{target} at %{scheme}%{url_host}%{port}%{url_path}") % { target: device.name, scheme: scheme, url_host: device_url.host, port: port, url_path: device_url.path }
 
           # override local $vardir and $certname
           Puppet[:confdir] = ::File.join(Puppet[:devicedir], device.name)
           Puppet[:vardir] = ::File.join(Puppet[:devicedir], device.name)
           Puppet[:certname] = device.name
 
-          # this will reload and recompute default settings and create the devices sub vardir, or we hope so :-)
-          Puppet.settings.use :main, :agent, :ssl
-
           # this init the device singleton, so that the facts terminus
           # and the various network_device provider can use it
           Puppet::Util::NetworkDevice.init(device)
 
-          # ask for a ssl cert if needed, but at least
-          # setup the ssl system for this device.
-          setup_host
-
-          require 'puppet/configurer'
-          configurer = Puppet::Configurer.new
-          configurer.run(:network_device => true, :pluginsync => Puppet::Configurer.should_pluginsync?)
+          if options[:resource]
+            type, name = parse_args(command_line.args)
+            Puppet.info _("retrieving resource: %{resource} from %{target} at %{scheme}%{url_host}%{port}%{url_path}") % { resource: type, target: device.name, scheme: scheme, url_host: device_url.host, port: port, url_path: device_url.path }
+
+            resources = find_resources(type, name)
+
+            if options[:to_yaml]
+              text = resources.map do |resource|
+                resource.prune_parameters(:parameters_to_include => @extra_params).to_hierayaml.force_encoding(Encoding.default_external)
+              end.join("\n")
+              text.prepend("#{type.downcase}:\n")
+            else
+              text = resources.map do |resource|
+                resource.prune_parameters(:parameters_to_include => @extra_params).to_manifest.force_encoding(Encoding.default_external)
+              end.join("\n")
+            end
+            (puts text)
+          elsif options[:apply]
+            # avoid reporting to server
+            Puppet::Transaction::Report.indirection.terminus_class = :yaml
+            Puppet::Resource::Catalog.indirection.cache_class = nil
+
+            require 'puppet/application/apply'
+            begin
+
+              Puppet[:node_terminus] = :plain
+              Puppet[:catalog_terminus] = :compiler
+              Puppet[:catalog_cache_terminus] = nil
+              Puppet[:facts_terminus] = :network_device
+              Puppet.override(:network_device => true) do
+                Puppet::Application::Apply.new(Puppet::Util::CommandLine.new('puppet', ["apply", options[:apply]])).run_command
+              end
+            end
+          else
+            Puppet.info _("starting applying configuration to %{target} at %{scheme}%{url_host}%{port}%{url_path}") % { target: device.name, scheme: scheme, url_host: device_url.host, port: port, url_path: device_url.path }
+            # this will reload and recompute default settings and create the devices sub vardir
+            Puppet.settings.use :main, :agent, :ssl
+            # ask for a ssl cert if needed, but at least
+            # setup the ssl system for this device.
+            setup_host
+
+            require 'puppet/configurer'
+            configurer = Puppet::Configurer.new
+            configurer.run(:network_device => true, :pluginsync => Puppet::Configurer.should_pluginsync?)
+          end
         rescue => detail
           Puppet.log_exception(detail)
           # If we rescued an error, then we return 1 as the exit code
@@ -258,6 +327,24 @@ Licensed under the Apache 2.0 License
     end
   end
 
+  def parse_args(args)
+    type = args.shift or raise _("You must specify the type to display")
+    Puppet::Type.type(type) or raise _("Could not find type %{type}") % { type: type }
+    name = args.shift
+
+    [type, name]
+  end
+
+  def find_resources(type, name)
+    key = [type, name].join('/')
+
+    if name
+      [ Puppet::Resource.indirection.find( key ) ]
+    else
+      Puppet::Resource.indirection.search( key, {} )
+    end
+  end
+
   def setup_host
     @host = Puppet::SSL::Host.new
     waitforcert = options[:waitforcert] || (Puppet[:onetime] ? 0 : Puppet[:waitforcert])

data/lib/puppet/application/facts.rb

@@ -1,4 +1,9 @@
 require 'puppet/application/indirection_base'
 
 class Puppet::Application::Facts < Puppet::Application::IndirectionBase
+  # Allows `puppet facts` actions to be run against environments that
+  # don't exist locally, such as using the `--environment` flag to make a REST
+  # request to a specific environment on a master. There is no way to set this
+  # behavior per-action, so it must be set for the face as a whole.
+  environment_mode :not_required
 end

data/lib/puppet/application/lookup.rb

@@ -335,13 +335,23 @@ Copyright (c) 2015 Puppet Inc., LLC Licensed under the Apache 2.0 License
       Puppet.settings[:facts_terminus] = 'facter'
     end
 
-    node = Puppet::Node.indirection.find(node) unless node.is_a?(Puppet::Node) # to allow unit tests to pass a node instance
+    unless node.is_a?(Puppet::Node) # to allow unit tests to pass a node instance
+      ni = Puppet::Node.indirection
+      tc = ni.terminus_class
+      if tc == :plain || options[:compile]
+        node = ni.find(node)
+      else
+        ni.terminus_class = :plain
+        node = ni.find(node)
+        ni.terminus_class = tc
+      end
+    end
 
     fact_file = options[:fact_file]
 
     if fact_file
       if fact_file.end_with?("json")
-        given_facts = JSON.parse(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
+        given_facts = Puppet::Util::Json.load(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
       else
         given_facts = YAML.load(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
       end

data/lib/puppet/configurer.rb

@@ -102,22 +102,31 @@ class Puppet::Configurer
   end
 
   # Convert a plain resource catalog into our full host catalog.
-  def convert_catalog(result, duration)
-    catalog = result.to_ral
-    catalog.finalize
-    catalog.retrieval_duration = duration
-    catalog.write_class_file
-    catalog.write_resource_file
+  def convert_catalog(result, duration, options = {})
+    catalog = nil
+
+    catalog_conversion_time = thinmark do
+      catalog = result.to_ral
+      catalog.finalize
+      catalog.retrieval_duration = duration
+      catalog.write_class_file
+      catalog.write_resource_file
+    end
+    options[:report].add_times(:convert_catalog, catalog_conversion_time) if options[:report]
+
     catalog
   end
 
   def get_facts(options)
     if options[:pluginsync]
-      remote_environment_for_plugins = Puppet::Node::Environment.remote(@environment)
-      download_plugins(remote_environment_for_plugins)
+      plugin_sync_time = thinmark do
+        remote_environment_for_plugins = Puppet::Node::Environment.remote(@environment)
+        download_plugins(remote_environment_for_plugins)
 
-      Puppet::GettextConfig.reset_text_domain('agent')
-      Puppet::ModuleTranslations.load_from_vardir(Puppet[:vardir])
+        Puppet::GettextConfig.reset_text_domain('agent')
+        Puppet::ModuleTranslations.load_from_vardir(Puppet[:vardir])
+      end
+      options[:report].add_times(:plugin_sync, plugin_sync_time) if options[:report]
     end
 
     facts_hash = {}
@@ -127,7 +136,10 @@ class Puppet::Configurer
       # get a hash with both of these pieces of information.
       #
       # facts_for_uploading may set Puppet[:node_name_value] as a side effect
-      facts_hash = facts_for_uploading
+      facter_time = thinmark do
+        facts_hash = facts_for_uploading
+      end
+      options[:report].add_times(:fact_generation, facter_time) if options[:report]
     end
     facts_hash
   end
@@ -149,17 +161,17 @@ class Puppet::Configurer
 
     # retrieve_catalog returns json catalog
     catalog = retrieve_catalog(query_options)
-    return convert_catalog(catalog, @duration) if catalog
+    return convert_catalog(catalog, @duration, options) if catalog
 
     Puppet.err _("Could not retrieve catalog; skipping run")
     nil
   end
 
-  def prepare_and_retrieve_catalog_from_cache
+  def prepare_and_retrieve_catalog_from_cache(options = {})
     result = retrieve_catalog_from_cache({:transaction_uuid => @transaction_uuid, :static_catalog => @static_catalog})
     if result
       Puppet.info _("Using cached catalog from environment '%{catalog_env}'") % { catalog_env: result.environment }
-      return convert_catalog(result, @duration)
+      return convert_catalog(result, @duration, options)
     end
     nil
   end
@@ -171,7 +183,10 @@ class Puppet::Configurer
       report.configuration_version = catalog.version
 
       benchmark(:notice, _("Applied catalog in %{seconds} seconds")) do
-        catalog.apply(options)
+        apply_catalog_time = thinmark do
+          catalog.apply(options)
+        end
+        options[:report].add_times(:catalog_application, apply_catalog_time)
       end
     ensure
       report.finalize_report
@@ -238,7 +253,7 @@ class Puppet::Configurer
       Puppet::GettextConfig.reset_text_domain('agent')
       Puppet::ModuleTranslations.load_from_vardir(Puppet[:vardir])
 
-      if catalog = prepare_and_retrieve_catalog_from_cache
+      if catalog = prepare_and_retrieve_catalog_from_cache(options)
         options[:catalog] = catalog
         @cached_catalog_status = 'explicitly_requested'
 
@@ -267,13 +282,18 @@ class Puppet::Configurer
       # We only need to find out the environment to run in if we don't already have a catalog
       unless (options[:catalog] || Puppet[:strict_environment_mode])
         begin
-          if node = options[:node] || Puppet::Node.indirection.find(Puppet[:node_name_value],
+          node = nil
+          node_retr_time = thinmark do
+            node = options[:node] || Puppet::Node.indirection.find(Puppet[:node_name_value],
               :environment => Puppet::Node::Environment.remote(@environment),
               :configured_environment => configured_environment,
               :ignore_cache => true,
               :transaction_uuid => @transaction_uuid,
               :fail_on_404 => true)
+          end
+          options[:report].add_times(:node_retrieval, node_retr_time)
 
+          if node
             # If we have deserialized a node from a rest call, we want to set
             # an environment instance as a simple 'remote' environment reference.
             if !node.has_environment_instance? && node.environment_name