bolt 1.44.0 → 1.45.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e7033e924c3721ad51ea3ba221527831174bf0a962a09be2d01af56eea9d29a
-  data.tar.gz: 400125a4441d5df92ac45fae11e222be1b0d8925ea501144e61f82f4a60fc62a
+  metadata.gz: 28b4c5859fab6ec57cac9d5ad791434aabfea666711f773f307d296df25274a9
+  data.tar.gz: 8dceb70532864e93b15c9237fff7ef803dba83650519f54dc60bbbce7e85a991
 SHA512:
-  metadata.gz: b25b113a4b89a14c84aec41fe2e8e1fad3a41a82ac4b6e58e8ff3901e46ac2be2a71eafee0cc80545f9d03cdb6d93affd1e44dc90f426fdb415bbfd26e17d16d
-  data.tar.gz: 6b08cf2a56959c286622c1279b2583ef312c673cb08c719727ba4c29142f8e9ea2d449f12a811eba671a7658b1619a735203905d98bae6dfc66ecb7c00b958a0
+  metadata.gz: 88716d23b4acaf2bf97fa73dbd96d802c185591bc1ffa1115268c1c27362be35e6a3e94e8f9a75c3d65e437c5aaf176776a2789ce2c9d7cbeeddd3db88496659
+  data.tar.gz: 8875843ed41a1cba383997f5fbc7e865ae0f8e0a401e84196d9339fd9fdde58652ca99128ed1527a61afa46fbb526abbf13dafd67f22729b14db5739a176f53c

data/bolt-modules/boltlib/lib/puppet/functions/puppetdb_query.rb

@@ -5,13 +5,13 @@ require 'bolt/error'
 # Makes a query to {https://puppet.com/docs/puppetdb/latest/index.html puppetdb}
 # using Bolt's PuppetDB client.
 Puppet::Functions.create_function(:puppetdb_query) do
-  # rubocop:disable Metrics/LineLength
+  # rubocop:disable Layout/LineLength
   # @param query A PQL query.
   #   {https://puppet.com/docs/puppetdb/latest/api/query/tutorial-pql.html Learn more about Puppet's query language, PQL}
   # @return Results of the PuppetDB query.
   # @example Request certnames for all nodes
   #   puppetdb_query('nodes[certname] {}')
-  # rubocop:enable Metrics/LineLength
+  # rubocop:enable Layout/LineLength
   dispatch :make_query do
     param 'Variant[String, Array[Data]]', :query
     return_type 'Array[Data]'

data/lib/bolt/applicator.rb

@@ -82,7 +82,11 @@ module Bolt
     end
 
     def compile(target, ast, plan_vars)
-      trusted = Puppet::Context::TrustedInformation.new('local', target.name, {})
+      # This simplified Puppet node object is what .local uses to determine the
+      # certname of the target
+      node = Puppet::Node.from_data_hash('name' => target.name,
+                                         'parameters' => { 'clientcert' => target.name })
+      trusted = Puppet::Context::TrustedInformation.local(node)
       facts = @inventory.facts(target).merge('bolt' => true)
 
       catalog_input = {
@@ -131,7 +135,11 @@ module Bolt
     end
 
     def future_compile(target, catalog_input)
-      trusted = Puppet::Context::TrustedInformation.new('local', target.name, {})
+      # This simplified Puppet node object is what .local uses to determine the
+      # certname of the target
+      node = Puppet::Node.from_data_hash('name' => target.name,
+                                         'parameters' => { 'clientcert' => target.name })
+      trusted = Puppet::Context::TrustedInformation.local(node)
       catalog_input[:target] = {
         name: target.name,
         facts: @inventory.facts(target).merge('bolt' => true),

data/lib/bolt/apply_result.rb

@@ -53,9 +53,9 @@ module Bolt
 
       unless missing_keys.empty?
         if result['_output']
-          # rubocop:disable Metrics/LineLength
+          # rubocop:disable Layout/LineLength
           msg = "Report result contains an '_output' key. Catalog application may have printed extraneous output to stdout: #{result['_output']}"
-          # rubocop:enable Metrics/LineLength
+          # rubocop:enable Layout/LineLength
         else
           msg = "Report did not contain all expected keys missing: #{missing_keys.join(' ,')}"
         end

data/lib/bolt/catalog.rb

@@ -96,7 +96,26 @@ module Bolt
               Puppet[:strict] = options['strict'] || :warning
               Puppet[:strict_variables] = options['strict_variables'] || false
               ast = Puppet::Pops::Serialization::FromDataConverter.convert(pal_main)
+              # This will be a Program when running via `bolt apply`, but will
+              # only be a subset of the AST when compiling an apply block in a
+              # plan. In that case, we need to discover the definitions (which
+              # would ordinarily be stored on the Program) and construct a Program object.
+              unless ast.is_a?(Puppet::Pops::Model::Program)
+                # Node definitions must be at the top level of the apply block.
+                # That means the apply body either a) consists of just a
+                # NodeDefinition, b) consists of a BlockExpression which may
+                # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
+                definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
+                                ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
+                              elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
+                                [ast]
+                              else
+                                []
+                              end
+                ast = Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
+              end
               compiler.evaluate(ast)
+              compiler.instance_variable_get(:@internal_compiler).send(:evaluate_ast_node)
               compiler.compile_additions
               compiler.with_json_encoding(&:encode)
             end

data/lib/bolt/cli.rb

@@ -664,7 +664,8 @@ module Bolt
       @pal ||= Bolt::PAL.new(config.modulepath,
                              config.hiera_config,
                              config.boltdir.resource_types,
-                             config.compile_concurrency)
+                             config.compile_concurrency,
+                             config.trusted_external)
     end
 
     def convert_plan(plan)

data/lib/bolt/config.rb

@@ -33,7 +33,7 @@ module Bolt
   class Config
     attr_accessor :concurrency, :format, :trace, :log, :puppetdb, :color, :save_rerun,
                   :transport, :transports, :inventoryfile, :compile_concurrency, :boltdir,
-                  :puppetfile_config, :plugins, :plugin_hooks, :future
+                  :puppetfile_config, :plugins, :plugin_hooks, :future, :trusted_external
     attr_writer :modulepath
 
     TRANSPORT_OPTIONS = %i[password run-as sudo-password extensions sudo-executable
@@ -162,6 +162,9 @@ module Bolt
       end
 
       @hiera_config = File.expand_path(data['hiera-config'], @boltdir.path) if data.key?('hiera-config')
+      @trusted_external = if data.key?('trusted-external-command')
+                            File.expand_path(data['trusted-external-command'], @boltdir.path)
+                          end
       @compile_concurrency = data['compile-concurrency'] if data.key?('compile-concurrency')
 
       @save_rerun = data['save-rerun'] if data.key?('save-rerun')
@@ -291,6 +294,7 @@ module Bolt
       end
 
       Bolt::Util.validate_file('hiera-config', @hiera_config) if @hiera_config
+      Bolt::Util.validate_file('trusted-external-command', @trusted_external) if @trusted_external
 
       unless @transport.nil? || Bolt::TRANSPORTS.include?(@transport.to_sym)
         raise UnknownTransportError, @transport

data/lib/bolt/inventory.rb

@@ -90,6 +90,8 @@ module Bolt
 
       @groups.resolve_aliases(@groups.node_aliases)
       collect_groups
+
+      deprecation unless @data.empty?
     end
 
     def validate
@@ -100,6 +102,17 @@ module Bolt
       1
     end
 
+    def deprecation
+      msg = <<~MSG
+        Deprecation Warning: Starting with Bolt 2.0, inventory file v1 will no longer
+        be supported. v1 inventory files can be automatically migrated to v2 using
+        'bolt project migrate'. Inventory files are modified in place and will not
+        preserve formatting or comments.
+      MSG
+      @logger.warn(msg)
+    end
+    private :deprecation
+
     def collect_groups
       # Provide a lookup map for finding a group by name
       @group_lookup = @groups.collect_groups

data/lib/bolt/inventory/group2.rb

@@ -235,7 +235,7 @@ module Bolt
 
         # DEPRECATION : remove this before finalization
         if input.key?('target-lookups')
-          msg = "'target-lookups' are no longer a separate key. Merge 'target-lookups' and 'targets' lists and replace 'plugin' with '_plugin'" # rubocop:disable Metrics/LineLength
+          msg = "'target-lookups' are no longer a separate key. Merge 'target-lookups' and 'targets' lists and replace 'plugin' with '_plugin'" # rubocop:disable Layout/LineLength
           raise ValidationError.new(msg, @name)
         end
 

data/lib/bolt/pal.rb

@@ -39,7 +39,7 @@ module Bolt
 
     attr_reader :modulepath
 
-    def initialize(modulepath, hiera_config, resource_types, max_compiles = Etc.nprocessors)
+    def initialize(modulepath, hiera_config, resource_types, max_compiles = Etc.nprocessors, trusted_external = nil)
       # Nothing works without initialized this global state. Reinitializing
       # is safe and in practice only happens in tests
       self.class.load_puppet
@@ -47,6 +47,7 @@ module Bolt
       @original_modulepath = modulepath
       @modulepath = [BOLTLIB_PATH, *modulepath, MODULES_PATH]
       @hiera_config = hiera_config
+      @trusted_external = trusted_external
       @max_compiles = max_compiles
       @resource_types = resource_types
 
@@ -211,6 +212,7 @@ module Bolt
         Puppet.settings.send(:clear_everything_for_tests)
         Puppet.initialize_settings(cli)
         Puppet::GettextConfig.create_default_text_domain
+        Puppet[:trusted_external_command] = @trusted_external
         self.class.configure_logging
         yield
       end

data/lib/bolt/transport/docker/connection.rb

@@ -157,14 +157,14 @@ module Bolt
                        .map { |str| JSON.parse(str) }
         end
 
-        # rubocop:disable Metrics/LineLength
+        # rubocop:disable Layout/LineLength
         # Executes a Docker CLI command
         #
         # @param subcommand [String] The docker subcommand to run e.g. 'inspect' for `docker inspect`
         # @param command_options [Array] Additional command options e.g. ['--size'] for `docker inspect --size`
         # @param redir_stdin [IO] IO object which will be use to as STDIN in the docker command. Default is nil, which does not perform redirection
         # @return [String, String, Process::Status] The output of the command:  STDOUT, STDERR, Process Status
-        # rubocop:enable Metrics/LineLength
+        # rubocop:enable Layout/LineLength
         def execute_local_docker_command(subcommand, command_options = [], redir_stdin = nil)
           env_hash = {}
           # Set the DOCKER_HOST if we are using a non-default service-url

data/lib/bolt/transport/orch.rb

@@ -81,7 +81,7 @@ module Bolt
               target,
               value: { '_error' => {
                 'kind' => 'puppetlabs.tasks/skipped-node',
-                'msg' => "Node #{target.safe_name} was skipped",
+                'msg' => "Target #{target.safe_name} was skipped",
                 'details' => {}
               } },
               action: 'task', object: task_name

data/lib/bolt/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bolt
-  VERSION = '1.44.0'
+  VERSION = '1.45.0'
 end

data/lib/bolt_ext/puppetdb_inventory.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'bolt/puppetdb'
+require 'bolt/logger'
 require 'json'
 require 'optparse'
 require 'yaml'
@@ -10,6 +11,8 @@ module Bolt
   class PuppetDBInventory
     class CLI
       def initialize(args)
+        Bolt::Logger.initialize_logging
+        @logger = Logging.logger[self]
         @args = args
         @cli_opts = {}
         @parser = build_parser
@@ -66,6 +69,17 @@ query results.
       end
 
       def run
+        Bolt::Logger.configure({ "console" => {} }, true)
+        msg = <<~MSG
+          Deprecation Warning: Starting with Bolt 2.0, inventory file v1 and the
+          bolt-inventory-pdb command will no longer be supported. Use a v2 inventory
+          file instead with the PuppetDB plugin to lookup targets from PuppetDB.
+
+          v1 inventory files can be automatically migrated to v2 using 'bolt project migrate'.
+          Inventory files are modified in place and will not preserve formatting or comments.
+        MSG
+        @logger.warn(msg)
+
         positional_args = @parser.permute(@args)
 
         if @show_help

data/lib/bolt_server/pe/pal.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require 'bolt/pal'
+require 'bolt/util'
+
+module BoltServer
+  module PE
+    class PAL < Bolt::PAL
+      # PE_BOLTLIB_PATH is intended to function exactly like the BOLTLIB_PATH used
+      # in Bolt::PAL. Paths and variable names are similar to what exists in
+      # Bolt::PAL, but with a 'PE' prefix.
+      PE_BOLTLIB_PATH = '/opt/puppetlabs/server/apps/bolt-server/pe-bolt-modules'
+
+      # For now at least, we maintain an entirely separate codedir from
+      # puppetserver by default, so that filesync can work properly. If filesync
+      # is not used, this can instead match the usual puppetserver codedir.
+      # See the `orchestrator.bolt.codedir` tk config setting.
+      DEFAULT_BOLT_CODEDIR = '/opt/puppetlabs/server/data/orchestration-services/code'
+
+      # This function is nearly identical to Bolt::Pal's `with_puppet_settings` with the
+      # one difference that we set the codedir to point to actual code, rather than the
+      # tmpdir. We only use this funtion inside the PEBolt::PAL initializer so that Puppet
+      # is correctly configured to pull environment configuration correctly. If we don't
+      # set codedir in this way: when we try to load and interpolate the modulepath it
+      # won't correctly load.
+      def with_pe_pal_init_settings(codedir, environmentpath, basemodulepath)
+        Dir.mktmpdir('pe-bolt') do |dir|
+          cli = []
+          Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
+            dir = setting == :codedir ? codedir : dir
+            cli << "--#{setting}" << dir
+          end
+          cli << "--environmentpath" << environmentpath
+          cli << "--basemodulepath" << basemodulepath
+          Puppet.settings.send(:clear_everything_for_tests)
+          Puppet.initialize_settings(cli)
+          yield
+          # Ensure the puppet settings go back to what bolt expects after
+          # we finish with the settings we need for PEBolt::PAL init.
+          with_puppet_settings { |_| nil }
+        end
+      end
+
+      def initialize(plan_executor_config, environment_name, hiera_config = nil, max_compiles = nil)
+        # Bolt::PAL#initialize takes the modulepath as its first argument, but we
+        # want to customize it later, so we pass an empty value:
+        super([], hiera_config, max_compiles)
+
+        codedir = plan_executor_config['codedir'] || DEFAULT_BOLT_CODEDIR
+        environmentpath = plan_executor_config['environmentpath'] || "#{codedir}/environments"
+        basemodulepath = plan_executor_config['basemodulepath'] || "#{codedir}/modules:/opt/puppetlabs/puppet/modules"
+
+        with_pe_pal_init_settings(codedir, environmentpath, basemodulepath) do
+          environment = Puppet.lookup(:environments).get!(environment_name)
+
+          # In the instance where the environment is "production" but no production dir
+          # exists, the lookup will succeed, but the configuration will be mostly empty.
+          # For other environments the lookup will fail, but for production we don't
+          # want cryptic messages sent to the user about combining `nil` with a string.
+          # Thus if we do get here and `path_to_env` is empty, just assume it's the
+          # default production environment and continue.
+          #
+          # This should hopefully match puppet's behavior for the default 'production'
+          # environment: _technically_ that environment always exists, but if the dir
+          # isn't there it won't find the module and fail with "plan not found" rather
+          # than "environment doesn't exist"
+          if environment.configuration.path_to_env
+            bolt_config = File.join(environment.configuration.path_to_env, 'bolt.yaml')
+          end
+          # If we find a bolt.yaml config in the environment root, we load that instead of
+          # environment.conf to find the modulepath. modulepath will be the _only_ setting
+          # that will work from bolt.yaml in plans in PE.
+          modulepath_dirs = if bolt_config && File.exist?(bolt_config)
+                              Bolt::Util.read_config_file(bolt_config)['modulepath'].split(File::PATH_SEPARATOR)
+                            else
+                              environment.modulepath
+                            end
+
+          # A new modulepath is created from scratch (rather than using super's @modulepath)
+          # so that we can have full control over all the entries in modulepath. In the future
+          # it's likely we will need to preceed _both_ Bolt::PAL::BOLTLIB_PATH _and_
+          # Bolt::PAL::MODULES_PATH which would be more complex if we tried to use @modulepath since
+          # we need to append our modulepaths and exclude modules shiped in bolt gem code
+          @original_modulepath = modulepath_dirs
+          @modulepath = [PE_BOLTLIB_PATH, Bolt::PAL::BOLTLIB_PATH, *modulepath_dirs]
+        end
+      end
+    end
+  end
+end

data/lib/bolt_server/transport_app.rb

@@ -10,6 +10,10 @@ require 'bolt/task/puppet_server'
 require 'json'
 require 'json-schema'
 
+# These are only needed for the `/plans` endpoint.
+require 'puppet'
+require 'bolt_server/pe/pal'
+
 module BoltServer
   class TransportApp < Sinatra::Base
     # This disables Sinatra's error page generation
@@ -45,6 +49,9 @@ module BoltServer
 
       @file_cache = BoltServer::FileCache.new(@config).setup
 
+      # This is needed until the PAL is threadsafe.
+      @pal_mutex = Mutex.new
+
       super(nil)
     end
 
@@ -173,6 +180,26 @@ module BoltServer
       [@executor.run_script(target, file_location, body['arguments'])]
     end
 
+    def in_pe_pal_env(environment)
+      if environment.nil?
+        [400, '`environment` is a required argument']
+      else
+        @pal_mutex.synchronize do
+          begin
+            pal = BoltServer::PE::PAL.new({}, environment)
+            yield pal
+          rescue Puppet::Environments::EnvironmentNotFound
+            [400, {
+              "class" => 'bolt/unknown-environment',
+              "message" => "Environment #{environment} not found"
+            }.to_json]
+          rescue Bolt::Error => e
+            [400, e.to_json]
+          end
+        end
+      end
+    end
+
     get '/' do
       200
     end
@@ -273,6 +300,36 @@ module BoltServer
       [200, result_set_to_status_hash(result_set, aggregate: aggregate).to_json]
     end
 
+    # Fetches the metadata for a single plan
+    #
+    # @param environment [String] the environment to fetch the plan from
+    get '/plans/:module_name/:plan_name' do
+      in_pe_pal_env(params['environment']) do |pal|
+        plan_info = pal.get_plan_info("#{params[:module_name]}::#{params[:plan_name]}")
+        [200, plan_info.to_json]
+      end
+    end
+
+    # Fetches the list of plans for an environment, optionally fetching all metadata for each plan
+    #
+    # @param environment [String] the environment to fetch the list of plans from
+    # @param metadata [Boolean] Set to true to fetch all metadata for each plan. Defaults to false
+    get '/plans' do
+      in_pe_pal_env(params['environment']) do |pal|
+        plans = pal.list_plans.flatten
+        if params['metadata']
+          plan_info = plans.each_with_object({}) { |plan_name, acc| acc[plan_name] = pal.get_plan_info(plan_name) }
+          [200, plan_info.to_json]
+        else
+          # We structure this array of plans to be an array of hashes so that it matches the structure
+          # returned by the puppetserver API that serves data like this. Structuring the output this way
+          # makes switching between puppetserver and bolt-server easier, which makes changes to switch
+          # to bolt-server smaller/simpler.
+          [200, plans.map { |plan| { 'name' => plan } }.to_json]
+        end
+      end
+    end
+
     error 404 do
       err = Bolt::Error.new("Could not find route #{request.path}",
                             'boltserver/not-found')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bolt
 version: !ruby/object:Gem::Version
-  version: 1.44.0
+  version: 1.45.0
 platform: ruby
 authors:
 - Puppet
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-09 00:00:00.000000000 Z
+date: 2020-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -464,6 +464,7 @@ files:
 - lib/bolt_server/base_config.rb
 - lib/bolt_server/config.rb
 - lib/bolt_server/file_cache.rb
+- lib/bolt_server/pe/pal.rb
 - lib/bolt_server/schemas/action-check_node_connections.json
 - lib/bolt_server/schemas/action-run_command.json
 - lib/bolt_server/schemas/action-run_script.json