bolt 2.10.0 → 2.14.0

data/lib/bolt/applicator.rb

@@ -14,14 +14,16 @@ require 'open3'
 
 module Bolt
   class Applicator
-    def initialize(inventory, executor, modulepath, plugin_dirs, pdb_client, hiera_config, max_compiles, apply_settings)
+    def initialize(inventory, executor, modulepath, plugin_dirs, project,
+                   pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
 
       @inventory = inventory
       @executor = executor
-      @modulepath = modulepath
+      @modulepath = modulepath || []
       @plugin_dirs = plugin_dirs
+      @project = project
       @pdb_client = pdb_client
       @hiera_config = hiera_config ? validate_hiera_config(hiera_config) : nil
       @apply_settings = apply_settings || {}
@@ -34,6 +36,8 @@ module Bolt
           search_dirs << mod.plugins if mod.plugins?
           search_dirs << mod.pluginfacts if mod.pluginfacts?
           search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
           search_dirs
         end
       end
@@ -142,6 +146,7 @@ module Bolt
 
     def apply(args, apply_body, scope)
       raise(ArgumentError, 'apply requires a TargetSpec') if args.empty?
+      raise(ArgumentError, 'apply requires at least one statement in the apply block') if apply_body.nil?
       type0 = Puppet.lookup(:pal_script_compiler).type('TargetSpec')
       Puppet::Pal.assert_type(type0, args[0], 'apply targets')
 
@@ -186,6 +191,7 @@ module Bolt
       scope = {
         code_ast: ast,
         modulepath: @modulepath,
+        project: @project.to_h,
         pdb_config: @pdb_client.config.to_hash,
         hiera_config: @hiera_config,
         plan_vars: plan_vars,

data/lib/bolt/apply_inventory.rb

@@ -70,6 +70,10 @@ module Bolt
       @targets[target.name].features
     end
 
+    def resource(target, type, title)
+      @targets[target.name].resource(type, title)
+    end
+
     def add_to_group(*_params)
       raise InvalidFunctionCall, 'add_to_group'
     end

data/lib/bolt/apply_result.rb

@@ -57,7 +57,7 @@ module Bolt
           msg = "Report result contains an '_output' key. Catalog application may have printed extraneous output to stdout: #{result['_output']}"
           # rubocop:enable Layout/LineLength
         else
-          msg = "Report did not contain all expected keys missing: #{missing_keys.join(' ,')}"
+          msg = "Report did not contain all expected keys missing: #{missing_keys.join(', ')}"
         end
 
         { 'msg' => msg,

data/lib/bolt/apply_target.rb

@@ -62,6 +62,10 @@ module Bolt
       @safe_name
     end
 
+    def resource(type, title)
+      resources[Bolt::ResourceInstance.format_reference(type, title)]
+    end
+
     def parse_uri(string)
       require 'addressable/uri'
       if string.nil?

data/lib/bolt/bolt_option_parser.rb

@@ -20,7 +20,7 @@ module Bolt
     def get_help_text(subcommand, action = nil)
       case subcommand
       when 'apply'
-        { flags: ACTION_OPTS + %w[noop execute compile-concurrency],
+        { flags: ACTION_OPTS + %w[noop execute compile-concurrency hiera-config],
           banner: APPLY_HELP }
       when 'command'
         case action
@@ -172,13 +172,14 @@ module Bolt
           apply
 
       USAGE
-          bolt apply <manifest.pp> [options]
+          bolt apply [manifest.pp] [options]
 
       DESCRIPTION
           Apply Puppet manifest code on the specified targets.
 
       EXAMPLES
-          bolt apply manifest.pp --targets target1,target2
+          bolt apply manifest.pp -t target
+          bolt apply -e "file { '/etc/puppetlabs': ensure => present }" -t target
     HELP
 
     COMMAND_HELP = <<~HELP
@@ -689,7 +690,7 @@ module Bolt
 
       separator "\nRUN CONTEXT OPTIONS"
       define('-c', '--concurrency CONCURRENCY', Integer,
-             'Maximum number of simultaneous connections (default: 100)') do |concurrency|
+             'Maximum number of simultaneous connections') do |concurrency|
         @options[:concurrency] = concurrency
       end
       define('--compile-concurrency CONCURRENCY', Integer,

data/lib/bolt/catalog.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
+require 'bolt/apply_inventory'
 require 'bolt/apply_target'
 require 'bolt/config'
 require 'bolt/error'
 require 'bolt/inventory'
-require 'bolt/apply_inventory'
 require 'bolt/pal'
 require 'bolt/puppetdb'
 require 'bolt/util'
@@ -19,7 +19,7 @@ module Bolt
       @log_level = log_level
     end
 
-    def with_puppet_settings(hiera_config = {})
+    def with_puppet_settings(overrides = {})
       Dir.mktmpdir('bolt') do |dir|
         cli = []
         Puppet::Settings::REQUIRED_APP_SETTINGS.each do |setting|
@@ -31,7 +31,9 @@ module Bolt
         Puppet.settings.override_default(:vendormoduledir, '')
 
         Puppet.initialize_settings(cli)
-        Puppet.settings[:hiera_config] = hiera_config
+        overrides.each do |setting, value|
+          Puppet.settings[setting] = value
+        end
 
         # Use a special logdest that serializes all log messages and their level to stderr.
         Puppet::Util::Log.newdestination(:stderr)
@@ -54,80 +56,51 @@ module Bolt
     end
 
     def compile_catalog(request)
-      pal_main = request['code_ast'] || request['code_string']
-      target = request['target']
       pdb_client = Bolt::PuppetDB::Client.new(Bolt::PuppetDB::Config.new(request['pdb_config']))
-      options = request['puppet_config'] || {}
-      with_puppet_settings(request['hiera_config']) do
-        Puppet[:rich_data] = true
-        Puppet[:node_name_value] = target['name']
-        env_conf = { modulepath: request['modulepath'] || [],
-                     facts: target['facts'] || {} }
-        env_conf[:variables] = {}
+      project = request['project'] || {}
+      bolt_project = Struct.new(:name, :path).new(project['name'], project['path']) unless project.empty?
+      inv = Bolt::ApplyInventory.new(request['config'])
+      puppet_overrides = {
+        bolt_pdb_client: pdb_client,
+        bolt_inventory: inv,
+        bolt_project: bolt_project
+      }
+
+      # Facts will be set by the catalog compiler, so we need to ensure
+      # that any plan or target variables with the same name are not
+      # passed into the apply block to avoid a redefinition error.
+      # Filter out plan and target vars separately and raise a Puppet
+      # warning if there are any collisions for either. Puppet warning
+      # is the only way to log a message that will make it back to Bolt
+      # to be printed.
+      target = request['target']
+      plan_vars = shadow_vars('plan', request['plan_vars'], target['facts'])
+      target_vars = shadow_vars('target', target['variables'], target['facts'])
+      topscope_vars = target_vars.merge(plan_vars)
+      env_conf = { modulepath: request['modulepath'],
+                   facts: target['facts'],
+                   variables: topscope_vars }
+
+      puppet_settings = {
+        node_name_value: target['name'],
+        hiera_config: request['hiera_config']
+      }
+
+      with_puppet_settings(puppet_settings) do
         Puppet::Pal.in_tmp_environment('bolt_catalog', env_conf) do |pal|
-          inv = Bolt::ApplyInventory.new(request['config'])
-          Puppet.override(bolt_pdb_client: pdb_client,
-                          bolt_inventory: inv) do
+          Puppet.override(puppet_overrides) do
             Puppet.lookup(:pal_current_node).trusted_data = target['trusted']
             pal.with_catalog_compiler do |compiler|
-              # Deserializing needs to happen inside the catalog compiler so
-              # loaders are initialized for loading
-              plan_vars = Puppet::Pops::Serialization::FromDataConverter.convert(request['plan_vars'])
-
-              # Facts will be set by the catalog compiler, so we need to ensure
-              # that any plan or target variables with the same name are not
-              # passed into the apply block to avoid a redefinition error.
-              # Filter out plan and target vars separately and raise a Puppet
-              # warning if there are any collisions for either. Puppet warning
-              # is the only way to log a message that will make it back to Bolt
-              # to be printed.
-              pv_collisions, pv_filtered = plan_vars.partition do |k, _|
-                target['facts'].keys.include?(k)
-              end.map(&:to_h)
-              unless pv_collisions.empty?
-                print_pv = pv_collisions.keys.map { |k| "$#{k}" }.join(', ')
-                plural = pv_collisions.keys.length == 1 ? '' : 's'
-                Puppet.warning("Plan variable#{plural} #{print_pv} will be overridden by fact#{plural} " \
-                               "of the same name in the apply block")
-              end
-
-              tv_collisions, tv_filtered = target['variables'].partition do |k, _|
-                target['facts'].keys.include?(k)
-              end.map(&:to_h)
-              unless tv_collisions.empty?
-                print_tv = tv_collisions.keys.map { |k| "$#{k}" }.join(', ')
-                plural = tv_collisions.keys.length == 1 ? '' : 's'
-                Puppet.warning("Target variable#{plural} #{print_tv} " \
-                               "will be overridden by fact#{plural} of the same name in the apply block")
-              end
-
-              pal.send(:add_variables, compiler.send(:topscope), tv_filtered.merge(pv_filtered))
-
+              options = request['puppet_config'] || {}
               # Configure language strictness in the CatalogCompiler. We want Bolt to be able
               # to compile most Puppet 4+ manifests, so we default to allowing deprecated functions.
               Puppet[:strict] = options['strict'] || :warning
               Puppet[:strict_variables] = options['strict_variables'] || false
-              ast = Puppet::Pops::Serialization::FromDataConverter.convert(pal_main)
-              # This will be a Program when running via `bolt apply`, but will
-              # only be a subset of the AST when compiling an apply block in a
-              # plan. In that case, we need to discover the definitions (which
-              # would ordinarily be stored on the Program) and construct a Program object.
-              unless ast.is_a?(Puppet::Pops::Model::Program)
-                # Node definitions must be at the top level of the apply block.
-                # That means the apply body either a) consists of just a
-                # NodeDefinition, b) consists of a BlockExpression which may
-                # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
-                definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
-                                ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
-                              elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
-                                [ast]
-                              else
-                                []
-                              end
-                ast = Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
-              end
+
+              pal_main = request['code_ast'] || request['code_string']
+              ast = build_program(pal_main)
               compiler.evaluate(ast)
-              compiler.instance_variable_get(:@internal_compiler).send(:evaluate_ast_node)
+              compiler.evaluate_ast_node
               compiler.compile_additions
               compiler.with_json_encoding(&:encode)
             end
@@ -135,5 +108,45 @@ module Bolt
         end
       end
     end
+
+    # Warn and remove variables that will be shadowed by facts of the same
+    # name, which are set in scope earlier.
+    def shadow_vars(type, vars, facts)
+      collisions, valid = vars.partition do |k, _|
+        facts.include?(k)
+      end
+      if collisions.any?
+        names = collisions.map { |k, _| "$#{k}" }.join(', ')
+        plural = collisions.length == 1 ? '' : 's'
+        Puppet.warning("#{type.capitalize} variable#{plural} #{names} will be overridden by fact#{plural} " \
+                       "of the same name in the apply block")
+      end
+      valid.to_h
+    end
+
+    def build_program(code)
+      ast = Puppet::Pops::Serialization::FromDataConverter.convert(code)
+
+      # This will be a Program when running via `bolt apply`, but will
+      # only be a subset of the AST when compiling an apply block in a
+      # plan. In that case, we need to discover the definitions (which
+      # would ordinarily be stored on the Program) and construct a Program object.
+      if ast.is_a?(Puppet::Pops::Model::Program)
+        ast
+      else
+        # Node definitions must be at the top level of the apply block.
+        # That means the apply body either a) consists of just a
+        # NodeDefinition, b) consists of a BlockExpression which may
+        # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
+        definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
+                        ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
+                      elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
+                        [ast]
+                      else
+                        []
+                      end
+        Puppet::Pops::Model::Factory.PROGRAM(ast, definitions, ast.locator).model
+      end
+    end
   end
 end

data/lib/bolt/cli.rb

@@ -115,7 +115,7 @@ module Bolt
                   Bolt::Config.from_file(options[:configfile], options)
                 else
                   project = if options[:boltdir]
-                              Bolt::Project.new(options[:boltdir])
+                              Bolt::Project.create_project(options[:boltdir])
                             else
                               Bolt::Project.find_boltdir(Dir.pwd)
                             end
@@ -392,7 +392,7 @@ module Bolt
         end
         code = apply_manifest(options[:code], options[:targets], options[:object], options[:noop])
       else
-        executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop])
+        executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop], config.modified_concurrency)
         targets = options[:targets]
 
         results = nil
@@ -512,7 +512,7 @@ module Bolt
                        params: plan_arguments }
       plan_context[:description] = options[:description] if options[:description]
 
-      executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop])
+      executor = Bolt::Executor.new(config.concurrency, analytics, options[:noop], config.modified_concurrency)
       if options.fetch(:format, 'human') == 'human'
         executor.subscribe(outputter)
       else
@@ -537,7 +537,18 @@ module Bolt
       Puppet[:tasks] = false
       ast = pal.parse_manifest(code, filename)
 
-      executor = Bolt::Executor.new(config.concurrency, analytics, noop)
+      if defined?(ast.body) &&
+         (ast.body.is_a?(Puppet::Pops::Model::HostClassDefinition) ||
+         ast.body.is_a?(Puppet::Pops::Model::ResourceTypeDefinition))
+        message = "Manifest only contains definitions and will result in no changes on the targets. "\
+                  "Definitions must be declared for their resources to be applied. You can read more "\
+                  "about defining and declaring classes and types in the Puppet documentation at "\
+                  "https://puppet.com/docs/puppet/latest/lang_classes.html and "\
+                  "https://puppet.com/docs/puppet/latest/lang_defined_types.html"
+        @logger.warn(message)
+      end
+
+      executor = Bolt::Executor.new(config.concurrency, analytics, noop, config.modified_concurrency)
       executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
       executor.subscribe(log_outputter)
       # apply logging looks like plan logging, so tell the outputter we're in a
@@ -760,7 +771,7 @@ module Bolt
     end
 
     def pal
-      project = config.project.load_as_module? ? config.project : nil
+      project = config.project.project_file? ? config.project : nil
       @pal ||= Bolt::PAL.new(config.modulepath,
                              config.hiera_config,
                              config.project.resource_types,

data/lib/bolt/config.rb

@@ -23,7 +23,7 @@ module Bolt
   end
 
   class Config
-    attr_reader :config_files, :warnings, :data, :transports, :project
+    attr_reader :config_files, :warnings, :data, :transports, :project, :modified_concurrency
 
     TRANSPORT_CONFIG = {
       'ssh'    => Bolt::Config::Transport::SSH,
@@ -34,6 +34,13 @@ module Bolt
       'remote' => Bolt::Config::Transport::Remote
     }.freeze
 
+    TRANSPORT_OPTION = { 'transport' => 'The default transport to use when the '\
+                         'transport for a target is not specified in the URL.' }.freeze
+
+    DEFAULT_TRANSPORT_OPTION = { 'transport' => 'ssh' }.freeze
+
+    CONFIG_IN_INVENTORY = TRANSPORT_CONFIG.merge(TRANSPORT_OPTION)
+
     # NOTE: All configuration options should have a corresponding schema property
     #       in schemas/bolt-config.schema.json
     OPTIONS = {
@@ -47,9 +54,7 @@ module Bolt
                                     "targets on the command line and from plans.",
       "log"                      => "The configuration of the logfile output. Configuration can be set for "\
                                     "`console` and the path to a log file, such as `~/.puppetlabs/bolt/debug.log`.",
-      "modulepath"               => "The module path for loading tasks and plan code. This is either an array "\
-                                    "of directories or a string containing a list of directories separated by the "\
-                                    "OS-specific PATH separator.",
+      "modulepath"               => "An array of directories that Bolt loads content (e.g. plans and tasks) from.",
       "plugin_hooks"             => "Which plugins a specific hook should use.",
       "plugins"                  => "A map of plugins and their configuration data.",
       "puppetdb"                 => "A map containing options for configuring the Bolt PuppetDB client.",
@@ -57,8 +62,8 @@ module Bolt
       "save-rerun"               => "Whether to update `.rerun.json` in the Bolt project directory. If "\
                                     "your target names include passwords, set this value to `false` to avoid "\
                                     "writing passwords to disk.",
-      "transport"                => "The default transport to use when the transport for a target is not "\
-                                    "specified in the URL or inventory.",
+      "transport"                => "The default transport to use when the transport for a target is not specified "\
+                                    "in the URL or inventory.",
       "trusted-external-command" => "The path to an executable on the Bolt controller that can produce "\
                                     "external trusted facts. **External trusted facts are experimental in both "\
                                     "Puppet and Bolt and this API may change or be removed.**"
@@ -66,8 +71,8 @@ module Bolt
 
     DEFAULT_OPTIONS = {
       "color" => true,
-      "concurrency" => 100,
       "compile-concurrency" => "Number of cores",
+      "concurrency" => "100 or one-third of the ulimit, whichever is lower",
       "format" => "human",
       "hiera-config" => "Boltdir/hiera.yaml",
       "inventoryfile" => "Boltdir/inventory.yaml",
@@ -103,49 +108,57 @@ module Bolt
       "show_diff" => false
     }.freeze
 
+    DEFAULT_DEFAULT_CONCURRENCY = 100
+
     def self.default
-      new(Bolt::Project.new('.'), {})
+      new(Bolt::Project.create_project('.'), {})
     end
 
     def self.from_project(project, overrides = {})
-      data = {
-        filepath: project.config_file,
-        data: Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
-      }
+      conf = if project.project_file == project.config_file
+               project.data
+             else
+               Bolt::Util.read_optional_yaml_hash(project.config_file, 'config')
+             end
+
+      data = { filepath: project.config_file, data: conf }
 
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
 
     def self.from_file(configfile, overrides = {})
-      project = Bolt::Project.new(Pathname.new(configfile).expand_path.dirname)
-
-      data = {
-        filepath: project.config_file,
-        data: Bolt::Util.read_yaml_hash(configfile, 'config')
-      }
-      data = load_defaults.push(data).select { |config| config[:data]&.any? }
+      project = Bolt::Project.create_project(Pathname.new(configfile).expand_path.dirname)
+      conf = if project.project_file == project.config_file
+               project.data
+             else
+               Bolt::Util.read_yaml_hash(configfile, 'config')
+             end
+      data = { filepath: project.config_file, data: conf }
+      data = load_defaults(project).push(data).select { |config| config[:data]&.any? }
 
       new(project, data, overrides)
     end
 
-    def self.load_defaults
+    def self.load_defaults(project)
       # Lazy-load expensive gem code
       require 'win32/dir' if Bolt::Util.windows?
 
-      system_path = if Bolt::Util.windows?
-                      Pathname.new(File.join(Dir::COMMON_APPDATA, 'PuppetLabs', 'bolt', 'etc', 'bolt.yaml'))
-                    else
-                      Pathname.new(File.join('/etc', 'puppetlabs', 'bolt', 'bolt.yaml'))
-                    end
+      # Don't load /etc/puppetlabs/bolt/bolt.yaml twice
+      confs = if project.path == Bolt::Project.system_path
+                []
+              else
+                system_path = Pathname.new(File.join(Bolt::Project.system_path, 'bolt.yaml'))
+                [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
+              end
+
       user_path = begin
                     Pathname.new(File.expand_path(File.join('~', '.puppetlabs', 'etc', 'bolt', 'bolt.yaml')))
                   rescue ArgumentError
                     nil
                   end
 
-      confs = [{ filepath: system_path, data: Bolt::Util.read_optional_yaml_hash(system_path, 'config') }]
       confs << { filepath: user_path, data: Bolt::Util.read_optional_yaml_hash(user_path, 'config') } if user_path
       confs
     end
@@ -156,8 +169,8 @@ module Bolt
       end
 
       @logger = Logging.logger[self]
-      @warnings = []
       @project = project
+      @warnings = @project.warnings.dup
       @transports = {}
       @config_files = []
 
@@ -165,7 +178,7 @@ module Bolt
         'apply_settings'      => {},
         'color'               => true,
         'compile-concurrency' => Etc.nprocessors,
-        'concurrency'         => 100,
+        'concurrency'         => default_concurrency,
         'format'              => 'human',
         'log'                 => { 'console' => {} },
         'plugin_hooks'        => {},
@@ -183,6 +196,12 @@ module Bolt
 
       override_data = normalize_overrides(overrides)
 
+      # If we need to lower concurrency and concurrency is not configured
+      ld_concurrency = loaded_data.map(&:keys).flatten.include?('concurrency')
+      @modified_concurrency = default_concurrency != DEFAULT_DEFAULT_CONCURRENCY &&
+                              !ld_concurrency &&
+                              !override_data.key?('concurrency')
+
       @data = merge_config_layers(default_data, *loaded_data, override_data)
 
       TRANSPORT_CONFIG.each do |transport, config|
@@ -458,5 +477,19 @@ module Bolt
         l =~ /[A-Za-z]/ ? "[#{l.upcase}#{l.downcase}]" : l
       end.join
     end
+
+    # Etc::SC_OPEN_MAX is meaningless on windows, not defined in PE Jruby and not available
+    # on some platforms. This method holds the logic to decide whether or not to even consider it.
+    def sc_open_max_available?
+      !Bolt::Util.windows? && defined?(Etc::SC_OPEN_MAX) && Etc.sysconf(Etc::SC_OPEN_MAX)
+    end
+
+    def default_concurrency
+      @default_concurrency ||= if !sc_open_max_available? || Etc.sysconf(Etc::SC_OPEN_MAX) >= 300
+                                 DEFAULT_DEFAULT_CONCURRENCY
+                               else
+                                 Etc.sysconf(Etc::SC_OPEN_MAX) / 7
+                               end
+    end
   end
 end