bolt 2.18.0 → 2.23.0

data/lib/bolt/catalog.rb

@@ -76,7 +76,15 @@ module Bolt
       target = request['target']
       plan_vars = shadow_vars('plan', request['plan_vars'], target['facts'])
       target_vars = shadow_vars('target', target['variables'], target['facts'])
-      topscope_vars = target_vars.merge(plan_vars)
+
+      # Merge plan vars with target vars, while maintaining the order of the plan
+      # vars. It's critical that the order of plan vars is not changed, as Puppet
+      # will deserialize the variables in the order they appear. Variables may
+      # contain local references to variables that appear earlier in a plan. If
+      # these variables are moved before the variable they reference, Puppet will
+      # be unable to deserialize the data and raise an error.
+      topscope_vars = target_vars.reject { |k, _v| plan_vars.key?(k) }.merge(plan_vars)
+
       env_conf = { modulepath: request['modulepath'],
                    facts: target['facts'],
                    variables: topscope_vars }
@@ -138,9 +146,10 @@ module Bolt
         # That means the apply body either a) consists of just a
         # NodeDefinition, b) consists of a BlockExpression which may
         # contain NodeDefinitions, or c) doesn't contain NodeDefinitions.
-        definitions = if ast.is_a?(Puppet::Pops::Model::BlockExpression)
+        definitions = case ast
+                      when Puppet::Pops::Model::BlockExpression
                         ast.statements.select { |st| st.is_a?(Puppet::Pops::Model::NodeDefinition) }
-                      elsif ast.is_a?(Puppet::Pops::Model::NodeDefinition)
+                      when Puppet::Pops::Model::NodeDefinition
                         [ast]
                       else
                         []

data/lib/bolt/cli.rb

@@ -31,8 +31,8 @@ module Bolt
     COMMANDS = { 'command' => %w[run],
                  'script' => %w[run],
                  'task' => %w[show run],
-                 'plan' => %w[show run convert],
-                 'file' => %w[upload],
+                 'plan' => %w[show run convert new],
+                 'file' => %w[download upload],
                  'puppetfile' => %w[install show-modules generate-types],
                  'secret' => %w[encrypt decrypt createkeys],
                  'inventory' => %w[show],
@@ -75,71 +75,100 @@ module Bolt
     end
     private :help?
 
+    # Wrapper method that is called by the Bolt executable. Parses the command and
+    # then loads the project and config. Once config is loaded, it completes the
+    # setup process by configuring Bolt and issuing warnings.
+    #
+    # This separation is needed since the Bolt::Outputter class that normally handles
+    # printing errors relies on config being loaded. All setup that happens before
+    # config is loaded will have errors printed directly to stdout, while all errors
+    # raised after config is loaded are handled by the outputter.
     def parse
-      begin
-        parser = BoltOptionParser.new(options)
-        # This part aims to handle both `bolt <mode> --help` and `bolt help <mode>`.
-        remaining = handle_parser_errors { parser.permute(@argv) } unless @argv.empty?
-        if @argv.empty? || help?(remaining)
-          # Update the parser for the subcommand (or lack thereof)
-          parser.update
-          puts parser.help
-          raise Bolt::CLIExit
-        end
+      parse_command
+      load_config
+      finalize_setup
+    end
 
-        options[:object] = remaining.shift
+    # Parses the command and validates options. All errors that are raised here
+    # are not handled by the outputter, as it relies on config being loaded.
+    def parse_command
+      parser = BoltOptionParser.new(options)
+      # This part aims to handle both `bolt <mode> --help` and `bolt help <mode>`.
+      remaining = handle_parser_errors { parser.permute(@argv) } unless @argv.empty?
+      if @argv.empty? || help?(remaining)
+        # Update the parser for the subcommand (or lack thereof)
+        parser.update
+        puts parser.help
+        raise Bolt::CLIExit
+      end
 
-        # Only parse task_options for task or plan
-        if %w[task plan].include?(options[:subcommand])
-          task_options, remaining = remaining.partition { |s| s =~ /.+=/ }
-          if options[:task_options]
-            unless task_options.empty?
-              raise Bolt::CLIError,
-                    "Parameters must be specified through either the --params " \
-                    "option or param=value pairs, not both"
-            end
-            options[:params_parsed] = true
-          elsif task_options.any?
-            options[:params_parsed] = false
-            options[:task_options] = Hash[task_options.map { |a| a.split('=', 2) }]
-          else
-            options[:params_parsed] = true
-            options[:task_options] = {}
+      options[:object] = remaining.shift
+
+      # Only parse task_options for task or plan
+      if %w[task plan].include?(options[:subcommand])
+        task_options, remaining = remaining.partition { |s| s =~ /.+=/ }
+        if options[:task_options]
+          unless task_options.empty?
+            raise Bolt::CLIError,
+                  "Parameters must be specified through either the --params " \
+                  "option or param=value pairs, not both"
           end
-        end
-        options[:leftovers] = remaining
-
-        validate(options)
-
-        @config = if ENV['BOLT_PROJECT']
-                    project = Bolt::Project.create_project(ENV['BOLT_PROJECT'], 'environment')
-                    Bolt::Config.from_project(project, options)
-                  elsif options[:configfile]
-                    Bolt::Config.from_file(options[:configfile], options)
-                  else
-                    project = if options[:boltdir]
-                                dir = Pathname.new(options[:boltdir])
-                                if (dir + Bolt::Project::BOLTDIR_NAME).directory?
-                                  Bolt::Project.create_project(dir + Bolt::Project::BOLTDIR_NAME)
-                                else
-                                  Bolt::Project.create_project(dir)
-                                end
-                              else
-                                Bolt::Project.find_boltdir(Dir.pwd)
-                              end
-                    Bolt::Config.from_project(project, options)
-                  end
-
-        Bolt::Logger.configure(config.log, config.color)
-      rescue Bolt::Error => e
-        if $stdout.isatty
-          # Print the error message in red, mimicking outputter.fatal_error
-          $stdout.puts("\033[31m#{e.message}\033[0m")
+          options[:params_parsed] = true
+        elsif task_options.any?
+          options[:params_parsed] = false
+          options[:task_options] = Hash[task_options.map { |a| a.split('=', 2) }]
         else
-          $stdout.puts(e.message)
+          options[:params_parsed] = true
+          options[:task_options] = {}
         end
-        raise e
       end
+      options[:leftovers] = remaining
+
+      # Default to verbose for everything except plans
+      unless options.key?(:verbose)
+        options[:verbose] = options[:subcommand] != 'plan'
+      end
+
+      validate(options)
+
+      # Deprecation warnings can't be issued until after config is loaded, so
+      # store them for later.
+      @parser_deprecations = parser.deprecations
+    rescue Bolt::Error => e
+      fatal_error(e)
+      raise e
+    end
+
+    # Loads the project and configuration. All errors that are raised here are not
+    # handled by the outputter, as it relies on config being loaded.
+    def load_config
+      @config = if ENV['BOLT_PROJECT']
+                  project = Bolt::Project.create_project(ENV['BOLT_PROJECT'], 'environment')
+                  Bolt::Config.from_project(project, options)
+                elsif options[:configfile]
+                  Bolt::Config.from_file(options[:configfile], options)
+                else
+                  project = if options[:boltdir]
+                              dir = Pathname.new(options[:boltdir])
+                              if (dir + Bolt::Project::BOLTDIR_NAME).directory?
+                                Bolt::Project.create_project(dir + Bolt::Project::BOLTDIR_NAME)
+                              else
+                                Bolt::Project.create_project(dir)
+                              end
+                            else
+                              Bolt::Project.find_boltdir(Dir.pwd)
+                            end
+                  Bolt::Config.from_project(project, options)
+                end
+    rescue Bolt::Error => e
+      fatal_error(e)
+      raise e
+    end
+
+    # Completes the setup process by configuring Bolt and issuing warnings
+    def finalize_setup
+      Bolt::Logger.configure(config.log, config.color)
+      Bolt::Logger.analytics = analytics
 
       # Logger must be configured before checking path case and project file, otherwise warnings will not display
       config.check_path_case('modulepath', config.modulepath)
@@ -149,28 +178,12 @@ module Bolt
       config_loaded
 
       # Display warnings created during parser and config initialization
-      parser.warnings.each { |warning| @logger.warn(warning[:msg]) }
       config.warnings.each { |warning| @logger.warn(warning[:msg]) }
-
-      # After validation, initialize inventory and targets. Errors here are better to catch early.
-      # After this step
-      # options[:target_args] will contain a string/array version of the targetting options this is passed to plans
-      # options[:targets] will contain a resolved set of Target objects
-      unless options[:subcommand] == 'puppetfile' ||
-             options[:subcommand] == 'secret' ||
-             options[:subcommand] == 'project' ||
-             options[:action] == 'show' ||
-             options[:action] == 'convert'
-
-        update_targets(options)
-      end
-
-      unless options.key?(:verbose)
-        # Default to verbose for everything except plans
-        options[:verbose] = options[:subcommand] != 'plan'
-      end
+      @parser_deprecations.each { |dep| Bolt::Logger.deprecation_warning(dep[:type], dep[:msg]) }
+      config.deprecations.each { |dep| Bolt::Logger.deprecation_warning(dep[:type], dep[:msg]) }
 
       warn_inventory_overrides_cli(options)
+
       options
     rescue Bolt::Error => e
       outputter.fatal_error(e)
@@ -245,6 +258,13 @@ module Bolt
               "Option '--noop' may only be specified when running a task or applying manifest code"
       end
 
+      if options[:env_vars]
+        unless %w[command script].include?(options[:subcommand]) && options[:action] == 'run'
+          raise Bolt::CLIError,
+                "Option '--env-var' may only be specified when running a command or script"
+        end
+      end
+
       if options[:subcommand] == 'apply' && (options[:object] && options[:code])
         raise Bolt::CLIError, "--execute is unsupported when specifying a manifest file"
       end
@@ -263,6 +283,10 @@ module Bolt
         raise Bolt::CLIError, "Must specify a value to #{options[:action]}"
       end
 
+      if options[:subcommand] == 'plan' && options[:action] == 'new' && !options[:object]
+        raise Bolt::CLIError, "Must specify a plan name."
+      end
+
       if options.key?(:debug) && options.key?(:log)
         raise Bolt::CLIError, "Only one of '--debug' or '--log-level' may be specified"
       end
@@ -327,9 +351,12 @@ module Bolt
         exit!
       end
 
-      if options[:action] == 'convert'
-        convert_plan(options[:object])
-        return 0
+      # Initialize inventory and targets. Errors here are better to catch early.
+      # options[:target_args] will contain a string/array version of the targetting options this is passed to plans
+      # options[:targets] will contain a resolved set of Target objects
+      unless %w[project puppetfile secret].include?(options[:subcommand]) ||
+             %w[convert new show].include?(options[:action])
+        update_targets(options)
       end
 
       screen = "#{options[:subcommand]}_#{options[:action]}"
@@ -355,32 +382,37 @@ module Bolt
 
       analytics.screen_view(screen, screen_view_fields)
 
-      if options[:action] == 'show'
-        if options[:subcommand] == 'task'
+      case options[:action]
+      when 'show'
+        case options[:subcommand]
+        when 'task'
           if options[:object]
             show_task(options[:object])
           else
             list_tasks
           end
-        elsif options[:subcommand] == 'plan'
+        when 'plan'
           if options[:object]
             show_plan(options[:object])
           else
             list_plans
           end
-        elsif options[:subcommand] == 'inventory'
+        when 'inventory'
           if options[:detail]
             show_targets
           else
             list_targets
           end
-        elsif options[:subcommand] == 'group'
+        when 'group'
           list_groups
         end
         return 0
-      elsif options[:action] == 'show-modules'
+      when 'show-modules'
         list_modules
         return 0
+      when 'convert'
+        convert_plan(options[:object])
+        return 0
       end
 
       message = 'There may be processes left executing on some nodes.'
@@ -391,17 +423,24 @@ module Bolt
 
       case options[:subcommand]
       when 'project'
-        if options[:action] == 'init'
+        case options[:action]
+        when 'init'
           code = initialize_project
-        elsif options[:action] == 'migrate'
+        when 'migrate'
           code = migrate_project
         end
       when 'plan'
-        code = run_plan(options[:object], options[:task_options], options[:target_args], options)
+        case options[:action]
+        when 'new'
+          code = new_plan(options[:object])
+        when 'run'
+          code = run_plan(options[:object], options[:task_options], options[:target_args], options)
+        end
       when 'puppetfile'
-        if options[:action] == 'generate-types'
+        case options[:action]
+        when 'generate-types'
           code = generate_types
-        elsif options[:action] == 'install'
+        when 'install'
           code = install_puppetfile(config.puppetfile_config, config.puppetfile, config.modulepath)
         end
       when 'secret'
@@ -422,6 +461,7 @@ module Bolt
         elapsed_time = Benchmark.realtime do
           executor_opts = {}
           executor_opts[:description] = options[:description] if options.key?(:description)
+          executor_opts[:env_vars] = options[:env_vars] if options.key?(:env_vars)
           executor.subscribe(outputter)
           executor.subscribe(log_outputter)
           results =
@@ -443,11 +483,22 @@ module Bolt
               src = options[:object]
               dest = options[:leftovers].first
 
+              if src.nil?
+                raise Bolt::CLIError, "A source path must be specified"
+              end
+
               if dest.nil?
                 raise Bolt::CLIError, "A destination path must be specified"
               end
-              validate_file('source file', src, true)
-              executor.upload_file(targets, src, dest, executor_opts)
+
+              case options[:action]
+              when 'download'
+                dest = File.expand_path(dest, Dir.pwd)
+                executor.download_file(targets, src, dest, executor_opts)
+              when 'upload'
+                validate_file('source file', src, true)
+                executor.upload_file(targets, src, dest, executor_opts)
+              end
             end
         end
 
@@ -475,7 +526,7 @@ module Bolt
       tasks = pal.list_tasks
       tasks.select! { |task| task.first.include?(options[:filter]) } if options[:filter]
       tasks.select! { |task| config.project.tasks.include?(task.first) } unless config.project.tasks.nil?
-      outputter.print_tasks(tasks, pal.list_modulepath)
+      outputter.print_tasks(tasks, pal.user_modulepath)
     end
 
     def show_plan(plan_name)
@@ -486,7 +537,7 @@ module Bolt
       plans = pal.list_plans
       plans.select! { |plan| plan.first.include?(options[:filter]) } if options[:filter]
       plans.select! { |plan| config.project.plans.include?(plan.first) } unless config.project.plans.nil?
-      outputter.print_plans(plans, pal.list_modulepath)
+      outputter.print_plans(plans, pal.user_modulepath)
     end
 
     def list_targets
@@ -504,6 +555,118 @@ module Bolt
       outputter.print_groups(groups)
     end
 
+    def new_plan(plan_name)
+      @logger.warn("Command 'bolt plan new' is experimental and subject to changes.")
+
+      if config.project.name.nil?
+        raise Bolt::Error.new(
+          "Project directory '#{config.project.path}' is not a named project. Unable to create "\
+          "a project-level plan. To name a project, set the 'name' key in the 'bolt-project.yaml' "\
+          "configuration file.",
+          "bolt/unnamed-project-error"
+        )
+      end
+
+      if plan_name !~ Bolt::Module::CONTENT_NAME_REGEX
+        message = <<~MESSAGE.chomp
+          Invalid plan name '#{plan_name}'. Plan names are composed of one or more name segments
+          separated by double colons '::'.
+          
+          Each name segment must begin with a lowercase letter, and may only include lowercase
+          letters, digits, and underscores.
+          
+          Examples of valid plan names:
+              - #{config.project.name}
+              - #{config.project.name}::my_plan
+        MESSAGE
+
+        raise Bolt::ValidationError, message
+      end
+
+      prefix, *name_segments, basename = plan_name.split('::')
+
+      # If the plan name is just the project name, then create an 'init' plan.
+      # Otherwise, use the last name segment for the plan's filename.
+      basename ||= 'init'
+
+      unless prefix == config.project.name
+        message = "First segment of plan name '#{plan_name}' must match project name '#{config.project.name}'. "\
+                  "Did you mean '#{config.project.name}::#{plan_name}'?"
+
+        raise Bolt::ValidationError, message
+      end
+
+      dir_path = config.project.plans_path.join(*name_segments)
+
+      %w[pp yaml].each do |ext|
+        next unless (path = config.project.plans_path + "#{basename}.#{ext}").exist?
+        raise Bolt::Error.new(
+          "A plan with the name '#{plan_name}' already exists at '#{path}', nothing to do.",
+          'bolt/existing-plan-error'
+        )
+      end
+
+      begin
+        FileUtils.mkdir_p(dir_path)
+      rescue Errno::EEXIST => e
+        raise Bolt::Error.new(
+          "#{e.message}; unable to create plan directory '#{dir_path}'",
+          'bolt/existing-file-error'
+        )
+      end
+
+      plan_path = dir_path + "#{basename}.yaml"
+
+      plan_template = <<~PLAN
+        # This is the structure of a simple plan. To learn more about writing
+        # YAML plans, see the documentation: http://pup.pt/bolt-yaml-plans
+
+        # The description sets the description of the plan that will appear
+        # in 'bolt plan show' output.
+        description: A plan created with bolt plan new
+
+        # The parameters key defines the parameters that can be passed to
+        # the plan.
+        parameters:
+          targets:
+            type: TargetSpec
+            description: A list of targets to run actions on
+            default: localhost
+
+        # The steps key defines the actions the plan will take in order.
+        steps:
+          - message: Hello from #{plan_name}
+          - name: command_step
+            command: whoami
+            targets: $targets
+
+        # The return key sets the return value of the plan.
+        return: $command_step
+      PLAN
+
+      begin
+        File.write(plan_path, plan_template)
+      rescue Errno::EACCES => e
+        raise Bolt::FileError.new(
+          "#{e.message}; unable to create plan",
+          plan_path
+        )
+      end
+
+      output = <<~OUTPUT
+        Created plan '#{plan_name}' at '#{plan_path}'
+
+        Show this plan with:
+            bolt plan show #{plan_name}
+        Run this plan with:
+            bolt plan run #{plan_name}
+      OUTPUT
+
+      outputter.print_message(output)
+
+      0
+    end
+
     def run_plan(plan_name, plan_arguments, nodes, options)
       unless nodes.empty?
         if plan_arguments['nodes'] || plan_arguments['targets']
@@ -607,8 +770,26 @@ module Bolt
     # Initializes a specified directory as a Bolt project and installs any modules
     # specified by the user, along with their dependencies
     def initialize_project
-      project    = Pathname.new(File.expand_path(options[:object] || Dir.pwd))
-      config     = project + 'bolt.yaml'
+      # Dir.pwd will return backslashes on Windows, but Pathname always uses
+      # forward slashes to concatenate paths. This results in paths like
+      # C:\User\Administrator/modules, which fail module install. This ensure
+      # forward slashes in the cwd path.
+      dir = File.expand_path(Dir.pwd)
+      name = options[:object] || File.basename(dir)
+      if name !~ Bolt::Module::MODULE_NAME_REGEX
+        if options[:object]
+          raise Bolt::ValidationError, "The provided project name '#{name}' is invalid; "\
+            "project name must begin with a lowercase letter and can include lowercase "\
+            "letters, numbers, and underscores."
+        else
+          raise Bolt::ValidationError, "The current directory name '#{name}' is an invalid "\
+            "project name. Please specify a name using 'bolt project init <name>'."
+        end
+      end
+
+      project    = Pathname.new(dir)
+      old_config = project + 'bolt.yaml'
+      config     = project + 'bolt-project.yaml'
       puppetfile = project + 'Puppetfile'
       modulepath = [project + 'modules']
 
@@ -629,18 +810,24 @@ module Bolt
 
       # Warn the user if the project directory already exists. We don't error here since users
       # might not have installed any modules yet.
+      # If both bolt.yaml and bolt-project.yaml exist, this will just warn
+      # about bolt-project.yaml and subsequent Bolt actions will warn about
+      # both files existing
       if config.exist?
-        @logger.warn "Found existing project directory at #{project}"
-      end
-
-      # Create the project directory
-      FileUtils.mkdir_p(project)
-
+        @logger.warn "Found existing project directory at #{project}. Skipping file creation."
+      # This won't get called if bolt-project.yaml exists
+      elsif old_config.exist?
+        @logger.warn "Found existing #{old_config.basename} at #{project}. "\
+          "#{old_config.basename} is deprecated, please rename to #{config.basename}."
       # Bless the project directory as a...wait for it...project
-      if FileUtils.touch(config)
-        outputter.print_message "Successfully created Bolt project at #{project}"
       else
-        raise Bolt::FileError.new("Could not create Bolt project directory at #{project}", nil)
+        begin
+          content = { 'name' => name }
+          File.write(config.to_path, content.to_yaml)
+          outputter.print_message "Successfully created Bolt project at #{project}"
+        rescue StandardError => e
+          raise Bolt::FileError.new("Could not create bolt-project.yaml at #{project}: #{e.message}", nil)
+        end
       end
 
       # Write the generated Puppetfile to the fancy new project
@@ -877,5 +1064,15 @@ module Bolt
     def incomplete_install?
       (Dir.children(Bolt::PAL::MODULES_PATH) - %w[aggregate canary puppetdb_fact]).empty?
     end
+
+    # Mimicks the output from Outputter::Human#fatal_error. This should be used to print
+    # errors prior to config being loaded, as the outputter relies on config being loaded.
+    def fatal_error(error)
+      if $stdout.isatty
+        $stdout.puts("\033[31m#{error.message}\033[0m")
+      else
+        $stdout.puts(error.message)
+      end
+    end
   end
 end