puppet 4.1.0 → 5.0.0

data/lib/puppet/application/lookup.rb

@@ -0,0 +1,357 @@
+require 'puppet/application'
+require 'puppet/pops'
+require 'puppet/node'
+require 'puppet/parser/compiler'
+
+class Puppet::Application::Lookup < Puppet::Application
+
+  RUN_HELP = _("Run 'puppet lookup --help' for more details").freeze
+  DEEP_MERGE_OPTIONS = '--knock-out-prefix, --sort-merged-arrays, and --merge-hash-arrays'.freeze
+
+  run_mode :master
+
+  # Options for lookup
+  option('--merge TYPE') do |arg|
+    options[:merge] = arg
+  end
+
+  option('--debug', '-d')
+
+  option('--verbose', '-v')
+
+  option('--render-as FORMAT') do |format|
+    options[:render_as] = format.downcase.to_sym
+  end
+
+  option('--type TYPE_STRING') do |arg|
+    options[:type] = arg
+  end
+
+  option('--compile', '-c')
+
+  option('--knock-out-prefix PREFIX_STRING') do |arg|
+    options[:prefix] = arg
+  end
+
+  option('--sort-merge-arrays')
+
+  option('--merge-hash-arrays')
+
+  option('--explain')
+
+  option('--explain-options')
+
+  option('--default VALUE') do |arg|
+    options[:default_value] = arg
+  end
+
+  # not yet supported
+  option('--trusted')
+
+  # Options for facts/scope
+  option('--node NODE_NAME') do |arg|
+    options[:node] = arg
+  end
+
+  option('--facts FACT_FILE') do |arg|
+    if %w{.yaml .yml .json}.include?(arg.match(/\.[^.]*$/)[0])
+      options[:fact_file] = arg
+    else
+      raise _("The --fact file only accepts yaml and json files.\n%{run_help}") % { run_help: RUN_HELP }
+    end
+  end
+
+  def app_defaults
+    super.merge({
+      :facts_terminus => 'yaml'
+    })
+  end
+
+  def setup_logs
+    # This sets up logging based on --debug or --verbose if they are set in `options`
+    set_log_level
+
+    # This uses console for everything that is not a compilation
+    Puppet::Util::Log.newdestination(:console)
+  end
+
+  def setup_terminuses
+    require 'puppet/file_serving/content'
+    require 'puppet/file_serving/metadata'
+
+    Puppet::FileServing::Content.indirection.terminus_class = :file_server
+    Puppet::FileServing::Metadata.indirection.terminus_class = :file_server
+
+    Puppet::FileBucket::File.indirection.terminus_class = :file
+  end
+
+  def setup
+    setup_logs
+
+    exit(Puppet.settings.print_configs ? 0 : 1) if Puppet.settings.print_configs?
+
+    Puppet.settings.use :main, :master, :ssl, :metrics
+
+    setup_terminuses
+  end
+
+  def help
+    <<-'HELP'
+
+puppet-lookup(8) -- Interactive Hiera lookup
+========
+
+SYNOPSIS
+--------
+Does Hiera lookups from the command line.
+
+Since this command needs access to your Hiera data, make sure to run it on a
+node that has a copy of that data. This usually means logging into a Puppet
+Server node and running 'puppet lookup' with sudo.
+
+The most common version of this command is:
+
+'puppet lookup <KEY> --node <NAME> --environment <ENV> --explain'
+
+USAGE
+-----
+puppet lookup [--help] [--type <TYPESTRING>] [--merge first|unique|hash|deep]
+  [--knock-out-prefix <PREFIX-STRING>] [--sort-merged-arrays]
+  [--merge-hash-arrays] [--explain] [--environment <ENV>]
+  [--default <VALUE>] [--node <NODE-NAME>] [--facts <FILE>]
+  [--compile]
+  [--render-as s|json|yaml|binary|msgpack] <keys>
+
+DESCRIPTION
+-----------
+The lookup command is a CLI for Puppet's 'lookup()' function. It searches your
+Hiera data and returns a value for the requested lookup key, so you can test and
+explore your data. It is a modern replacement for the 'hiera' command.
+
+Hiera usually relies on a node's facts to locate the relevant data sources. By
+default, 'puppet lookup' uses facts from the node you run the command on, but
+you can get data for any other node with the '--node <NAME>' option. If
+possible, the lookup command will use the requested node's real stored facts
+from PuppetDB; if PuppetDB isn't configured or you want to provide arbitrary
+fact values, you can pass alternate facts as a JSON or YAML file with '--facts
+<FILE>'.
+
+If you're debugging your Hiera data and want to see where values are coming
+from, use the '--explain' option.
+
+If '--explain' isn't specified, lookup exits with 0 if a value was found and 1
+otherwise. With '--explain', lookup always exits with 0 unless there is a major
+error.
+
+You can provide multiple lookup keys to this command, but it only returns a
+value for the first found key, omitting the rest.
+
+For more details about how Hiera works, see the Hiera documentation:
+https://docs.puppet.com/puppet/latest/hiera_intro.html
+
+OPTIONS
+-------
+
+* --help:
+  Print this help message.
+
+* --explain
+  Explain the details of how the lookup was performed and where the final value
+  came from (or the reason no value was found).
+
+* --node <NODE-NAME>
+  Specify which node to look up data for; defaults to the node where the command
+  is run. Since Hiera's purpose is to provide different values for different
+  nodes (usually based on their facts), you'll usually want to use some specific
+  node's facts to explore your data. If the node where you're running this
+  command is configured to talk to PuppetDB, the command will use the requested
+  node's most recent facts. Otherwise, you can override facts with the '--facts'
+  option.
+
+* --facts <FILE>
+  Specify a .json or .yaml file of key => value mappings to override the facts
+  for this lookup. Any facts not specified in this file maintain their
+  original value.
+
+* --environment <ENV>
+  Like with most Puppet commands, you can specify an environment on the command
+  line. This is important for lookup because different environments can have
+  different Hiera data.
+
+* --merge first|unique|hash|deep:
+  Specify the merge behavior, overriding any merge behavior from the data's
+  lookup_options. 'first' returns the first value found. 'unique' appends
+  everything to a merged, deduplicated array. 'hash' performs a simple hash
+  merge by overwriting keys of lower lookup priority. 'deep' performs a deep
+  merge on values of Array and Hash type. There are additional options that can
+  be used with 'deep'.
+
+* --knock-out-prefix <PREFIX-STRING>
+  Can be used with the 'deep' merge strategy. Specifies a prefix to indicate a
+  value should be removed from the final result.
+
+* --sort-merged-arrays
+  Can be used with the 'deep' merge strategy. When this flag is used, all
+  merged arrays are sorted.
+
+* --merge-hash-arrays
+  Can be used with the 'deep' merge strategy. When this flag is used, hashes
+  WITHIN arrays are deep-merged with their counterparts by position.
+
+* --explain-options
+  Explain whether a lookup_options hash affects this lookup, and how that hash
+  was assembled. (lookup_options is how Hiera configures merge behavior in data.)
+
+* --default <VALUE>
+  A value to return if Hiera can't find a value in data. For emulating calls to
+  the 'lookup()' function that include a default.
+
+* --type <TYPESTRING>:
+  Assert that the value has the specified type. For emulating calls to the
+  'lookup()' function that include a data type.
+
+* --compile
+  Perform a full catalog compilation prior to the lookup. If your hierarchy and
+  data only use the $facts, $trusted, and $server_facts variables, you don't
+  need this option; however, if your Hiera configuration uses arbitrary
+  variables set by a Puppet manifest, you might need this option to get accurate
+  data. No catalog compilation takes place unless this flag is given.
+
+* --render-as s|json|yaml|binary|msgpack
+  Specify the output format of the results; "s" means plain text. The default
+  when producing a value is yaml and the default when producing an explanation
+  is s.
+
+EXAMPLE
+-------
+  To look up 'key_name' using the Puppet Server node's facts:
+  $ puppet lookup key_name
+
+  To look up 'key_name' with agent.local's facts:
+  $ puppet lookup --node agent.local key_name
+
+  To get the first value found for 'key_name_one' and 'key_name_two'
+  with agent.local's facts while merging values and knocking out
+  the prefix 'foo' while merging:
+  $ puppet lookup --node agent.local --merge deep --knock-out-prefix foo key_name_one key_name_two
+
+  To lookup 'key_name' with agent.local's facts, and return a default value of
+  'bar' if nothing was found:
+  $ puppet lookup --node agent.local --default bar key_name
+
+  To see an explanation of how the value for 'key_name' would be found, using
+  agent.local's facts:
+  $ puppet lookup --node agent.local --explain key_name
+
+COPYRIGHT
+---------
+Copyright (c) 2015 Puppet Inc., LLC Licensed under the Apache 2.0 License
+
+
+    HELP
+  end
+
+  def main
+    keys = command_line.args
+
+    #unless options[:node]
+    #  raise "No node was given via the '--node' flag for the scope of the lookup.\n#{RUN_HELP}"
+    #end
+
+    if (options[:sort_merge_arrays] || options[:merge_hash_arrays] || options[:prefix]) && options[:merge] != 'deep'
+      raise _("The options %{deep_merge_opts} are only available with '--merge deep'\n%{run_help}") % { deep_merge_opts: DEEP_MERGE_OPTIONS, run_help: RUN_HELP }
+    end
+
+    use_default_value = !options[:default_value].nil?
+    merge_options = nil
+
+    merge = options[:merge]
+    unless merge.nil?
+      strategies = Puppet::Pops::MergeStrategy.strategy_keys
+      unless strategies.include?(merge.to_sym)
+        strategies = strategies.map {|k| "'#{k}'"}
+        raise _("The --merge option only accepts %{strategies}, or %{last_strategy}\n%{run_help}") % { strategies: strategies[0...-1].join(', '), last_strategy: strategies.last, run_help: RUN_HELP }
+      end
+
+      if merge == 'deep'
+        merge_options = {'strategy' => 'deep',
+          'sort_merge_arrays' => !options[:sort_merge_arrays].nil?,
+          'merge_hash_arrays' => !options[:merge_hash_arrays].nil?}
+
+        if options[:prefix]
+          merge_options.merge!({'knockout_prefix' => options[:prefix]})
+        end
+
+      else
+        merge_options = {'strategy' => merge}
+      end
+    end
+
+    explain_data = !!options[:explain]
+    explain_options = !!options[:explain_options]
+    only_explain_options = explain_options && !explain_data
+    if keys.empty?
+      if only_explain_options
+        # Explain lookup_options for lookup of an unqualified value.
+        keys = Puppet::Pops::Lookup::GLOBAL
+      else
+        raise _('No keys were given to lookup.')
+      end
+    end
+    explain = explain_data || explain_options
+
+    # Format defaults to text (:s) when producing an explanation and :yaml when producing the value
+    format = options[:render_as] || (explain ? :s : :yaml)
+    renderer = Puppet::Network::FormatHandler.format(format)
+    raise _("Unknown rendering format '%{format}'") % { format: format } if renderer.nil?
+
+    generate_scope do |scope|
+      lookup_invocation = Puppet::Pops::Lookup::Invocation.new(scope, {}, {}, explain ? Puppet::Pops::Lookup::Explainer.new(explain_options, only_explain_options) : nil)
+      begin
+        type = options.include?(:type) ? Puppet::Pops::Types::TypeParser.singleton.parse(options[:type], scope) : nil
+        result = Puppet::Pops::Lookup.lookup(keys, type, options[:default_value], use_default_value, merge_options, lookup_invocation)
+        puts renderer.render(result) unless explain
+      rescue Puppet::DataBinding::LookupError => e
+        lookup_invocation.report_text { e.message }
+        exit(1) unless explain
+      end
+      puts format == :s ? lookup_invocation.explainer.explain : renderer.render(lookup_invocation.explainer.to_hash) if explain
+    end
+    exit(0)
+  end
+
+  def generate_scope
+    if options[:node]
+      node = options[:node]
+    else
+      node = Puppet[:node_name_value]
+
+      # If we want to lookup the node we are currently on
+      # we must returning these settings to their default values
+      Puppet.settings[:facts_terminus] = 'facter'
+    end
+
+    node = Puppet::Node.indirection.find(node) unless node.is_a?(Puppet::Node) # to allow unit tests to pass a node instance
+
+    fact_file = options[:fact_file]
+
+    if fact_file
+      original_facts = node.parameters
+      if fact_file.end_with?("json")
+        given_facts = JSON.parse(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
+      else
+        given_facts = YAML.load(Puppet::FileSystem.read(fact_file, :encoding => 'utf-8'))
+      end
+
+      unless given_facts.instance_of?(Hash)
+        raise _("Incorrect formatted data in %{fact_file} given via the --facts flag") % { fact_file: fact_file }
+      end
+
+      node.parameters = original_facts.merge(given_facts)
+    end
+
+    Puppet[:code] = 'undef' unless options[:compile]
+    compiler = Puppet::Parser::Compiler.new(node)
+    compiler.compile { |catalog| yield(compiler.topscope); catalog }
+  end
+end

data/lib/puppet/application/master.rb

@@ -61,7 +61,7 @@ specify '--server <servername>' as an argument. Boolean settings translate into
 '--setting' and '--no-setting' pairs.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/puppet/latest/reference/configuration.html for the
 full list of acceptable settings. A commented list of all settings can also be
 generated by running puppet master with '--genconfig'.
 
@@ -92,7 +92,7 @@ generated by running puppet master with '--genconfig'.
   valid JSON.
 
 * --masterport:
-  The port on which to listen for traffic.
+  The port on which to listen for traffic. The default port is 8140.
   (This is a Puppet setting, and can go in puppet.conf.)
 
 * --verbose:
@@ -130,27 +130,20 @@ Luke Kanies
 
 COPYRIGHT
 ---------
-Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
+Copyright (c) 2012 Puppet Inc., LLC Licensed under the Apache 2.0 License
 
     HELP
   end
 
-  # Sets up the 'node_cache_terminus' default to use the Write Only Yaml terminus :write_only_yaml.
-  # If this is not wanted, the setting ´node_cache_terminus´ should be set to nil.
-  # @see Puppet::Node::WriteOnlyYaml
-  # @see #setup_node_cache
-  # @see puppet issue 16753
-  #
   def app_defaults
     super.merge({
-      :node_cache_terminus => :write_only_yaml,
       :facts_terminus => 'yaml'
     })
   end
 
   def preinit
     Signal.trap(:INT) do
-      $stderr.puts "Canceling startup"
+      $stderr.puts _("Canceling startup")
       exit(0)
     end
 
@@ -169,12 +162,12 @@ Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
   def compile
     begin
       unless catalog = Puppet::Resource::Catalog.indirection.find(options[:node])
-        raise "Could not compile catalog for #{options[:node]}"
+        raise _("Could not compile catalog for %{node}") % { node: options[:node] }
       end
 
-      puts PSON::pretty_generate(catalog.to_resource, :allow_nan => true, :max_nesting => false)
+      puts JSON::pretty_generate(catalog.to_resource, :allow_nan => true, :max_nesting => false)
     rescue => detail
-      Puppet.log_exception(detail, "Failed to compile catalog for node #{options[:node]}: #{detail}")
+      Puppet.log_exception(detail, _("Failed to compile catalog for node %{node}: %{detail}") % { node: options[:node], detail: detail })
       exit(30)
     end
     exit(0)
@@ -194,19 +187,20 @@ Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
         begin
           Puppet::Util.chuser
         rescue => detail
-          Puppet.log_exception(detail, "Could not change user to #{Puppet[:user]}: #{detail}")
+          Puppet.log_exception(detail, _("Could not change user to %{user}: %{detail}") % { user: Puppet[:user], detail: detail })
           exit(39)
         end
       else
-        raise Puppet::Error.new("Could not change user to #{Puppet[:user]}. User does not exist and is required to continue.")
+        Puppet.err(_("Could not change user to %{user}. User does not exist and is required to continue.") % { user: Puppet[:user] })
+        exit(74)
       end
     end
 
     if options[:rack]
-      Puppet.deprecation_warning("The Rack Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppetlabs.com/deprecate-rack-webrick-servers for more information.")
+      Puppet.deprecation_warning(_("The Rack Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppet.com/deprecate-rack-webrick-servers for more information."))
       start_rack_master
     else
-      Puppet.deprecation_warning("The WEBrick Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppetlabs.com/deprecate-rack-webrick-servers for more information.")
+      Puppet.deprecation_warning(_("The WEBrick Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppet.com/deprecate-rack-webrick-servers for more information."))
       start_webrick_master
     end
   end
@@ -255,18 +249,17 @@ Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
     Puppet::SSL::Oids.load_custom_oid_file(Puppet[:trusted_oid_mapping_file])
   end
 
-  # Sets up a special node cache "write only yaml" that collects and stores node data in yaml
-  # but never finds or reads anything (this since a real cache causes stale data to be served
-  # in circumstances when the cache can not be cleared).
-  # @see puppet issue 16753
-  # @see Puppet::Node::WriteOnlyYaml
+  # Honor the :node_cache_terminus setting if users have specified it directly.
+  # We normally want this nil as use-cases for querying nodes should be going to
+  # PuppetDB.
+  # @see PUP-6060
   # @return [void]
   def setup_node_cache
     Puppet::Node.indirection.cache_class = Puppet[:node_cache_terminus]
   end
 
   def setup
-    raise Puppet::Error.new("Puppet master is not supported on Microsoft Windows") if Puppet.features.microsoft_windows?
+    raise Puppet::Error.new(_("Puppet master is not supported on Microsoft Windows")) if Puppet.features.microsoft_windows?
 
     setup_logs
 
@@ -294,6 +287,9 @@ Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
     daemon.server = Puppet::Network::Server.new(Puppet[:bindaddress], Puppet[:masterport])
     daemon.daemonize if Puppet[:daemonize]
 
+    # Setup signal traps immediately after daemonization so we clean up the daemon
+    daemon.set_signal_traps
+
     announce_start_of_master
 
     daemon.start
@@ -312,6 +308,6 @@ Copyright (c) 2012 Puppet Labs, LLC Licensed under the Apache 2.0 License
   end
 
   def announce_start_of_master
-    Puppet.notice "Starting Puppet master version #{Puppet.version}"
+    Puppet.notice _("Starting Puppet master version %{version}") % { version: Puppet.version }
   end
 end

data/lib/puppet/application/resource.rb

@@ -72,7 +72,7 @@ setting, so you can specify '--ssldir <directory>' as an
 argument.
 
 See the configuration file documentation at
-http://docs.puppetlabs.com/references/stable/configuration.html for the
+https://docs.puppetlabs.com/puppet/latest/reference/configuration.html for the
 full list of acceptable parameters. A commented list of all
 configuration options can also be generated by running puppet with
 '--genconfig'.
@@ -84,10 +84,6 @@ configuration options can also be generated by running puppet with
   Write the results of the query to a file, open the file in an editor,
   and read the file back in as an executable Puppet manifest.
 
-* --host:
-  When specified, connect to the resource server on the named host
-  and retrieve the list of resouces of the type specified.
-
 * --help:
   Print this help message.
 
@@ -127,32 +123,35 @@ Luke Kanies
 
 COPYRIGHT
 ---------
-Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
+Copyright (c) 2011 Puppet Inc., LLC Licensed under the Apache 2.0 License
 
     HELP
   end
 
   def main
-    type, name, params = parse_args(command_line.args)
+    env = Puppet.lookup(:environments).get(Puppet[:environment])
+    Puppet.override(:current_environment => env, :loaders => Puppet::Pops::Loaders.new(env)) do
+      type, name, params = parse_args(command_line.args)
 
-    raise "Editing with Yaml output is not supported" if options[:edit] and options[:to_yaml]
+      raise _("Editing with Yaml output is not supported") if options[:edit] and options[:to_yaml]
 
-    resources = find_or_save_resources(type, name, params)
+      resources = find_or_save_resources(type, name, params)
 
     if options[:to_yaml]
-      text = resources.
-        map { |resource| resource.prune_parameters(:parameters_to_include => @extra_params).to_hierayaml }.
-        join("\n")
+      text = resources.map do |resource|
+        resource.prune_parameters(:parameters_to_include => @extra_params).to_hierayaml.force_encoding(Encoding.default_external)
+      end.join("\n")
       text.prepend("#{type.downcase}:\n")
     else
-      text = resources.
-        map { |resource| resource.prune_parameters(:parameters_to_include => @extra_params).to_manifest }.
-        join("\n")
+      text = resources.map do |resource|
+        resource.prune_parameters(:parameters_to_include => @extra_params).to_manifest.force_encoding(Encoding.default_external)
+      end.join("\n")
     end
 
-    options[:edit] ?
-      handle_editing(text) :
-      (puts text)
+      options[:edit] ?
+        handle_editing(text) :
+        (puts text)
+    end
   end
 
   def setup
@@ -170,7 +169,7 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
     require 'tempfile'
     # Prefer the current directory, which is more likely to be secure
     # and, in the case of interactive use, accessible to the user.
-    tmpfile = Tempfile.new('x2puppet', Dir.pwd)
+    tmpfile = Tempfile.new('x2puppet', Dir.pwd, :encoding => Encoding::UTF_8)
     begin
       # sync write, so nothing buffers before we invoke the editor.
       tmpfile.sync = true
@@ -190,15 +189,15 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
   end
 
   def parse_args(args)
-    type = args.shift or raise "You must specify the type to display"
-    Puppet::Type.type(type) or raise "Could not find type #{type}"
+    type = args.shift or raise _("You must specify the type to display")
+    Puppet::Type.type(type) or raise _("Could not find type %{type}") % { type: type }
     name = args.shift
     params = {}
     args.each do |setting|
       if setting =~ /^(\w+)=(.+)$/
         params[$1] = $2
       else
-        raise "Invalid parameter setting #{setting}"
+        raise _("Invalid parameter setting %{setting}") % { setting: setting }
       end
     end
 
@@ -220,7 +219,7 @@ Copyright (c) 2011 Puppet Labs, LLC Licensed under the Apache 2.0 License
       end
     else
       if type == "file"
-        raise "Listing all file instances is not supported.  Please specify a file or directory, e.g. puppet resource file /etc"
+        raise _("Listing all file instances is not supported.  Please specify a file or directory, e.g. puppet resource file /etc")
       end
       Puppet::Resource.indirection.search( key, {} )
     end