bolt 1.19.0 → 1.20.0

data/lib/bolt/apply_result.rb

@@ -65,7 +65,7 @@ module Bolt
     def initialize(target, error: nil, report: nil)
       @target = target
       @value = {}
-      @type = 'apply'
+      @action = 'apply'
       value['report'] = report if report
       value['_error'] = error if error
       value['_output'] = metrics_message if metrics_message
@@ -77,26 +77,13 @@ module Bolt
       end
     end
 
-    # TODO: We've gotten requests for this type of logging but I'm not sure
-    # what we shold do with it exactly.
-    def log_events
-      logger = Logging.logger[target.name]
-      if (logs = value.dig('report', 'logs'))
-        logs.each do |log|
-          case log["level"]
-          when 'err'
-            logger.error(log['message'])
-          when 'warn'
-            logger.info(log['message'])
-          when 'notice'
-            logger.notice(log['message'])
-          when 'info'
-            logger.info(log['message'])
-          else
-            logger.debug(log["message"])
-          end
-        end
-      end
+    def logs
+      value.dig('report', 'logs') || []
+    end
+
+    # Return only log messages associated with resources
+    def resource_logs
+      logs.reject { |log| log['source'] == 'Puppet' }
     end
 
     def metrics_message

data/lib/bolt/bolt_option_parser.rb

@@ -281,8 +281,8 @@ Usage: bolt apply <manifest.pp> [options]
       define('-h', '--help', 'Display help') do |_|
         @options[:help] = true
       end
-      define('-v', '--verbose', 'Display verbose logging') do |_|
-        @options[:verbose] = true
+      define('-v', '--[no-]verbose', 'Display verbose logging') do |value|
+        @options[:verbose] = value
       end
       define('--debug', 'Display debug logging') do |_|
         @options[:debug] = true

data/lib/bolt/catalog.rb

@@ -65,7 +65,6 @@ module Bolt
       target = request['target']
       pdb_client = Bolt::PuppetDB::Client.new(Bolt::PuppetDB::Config.new(request['pdb_config']))
       options = request['puppet_config'] || {}
-
       with_puppet_settings(request['hiera_config']) do
         Puppet[:rich_data] = true
         Puppet[:node_name_value] = target['name']

data/lib/bolt/cli.rb

@@ -126,6 +126,11 @@ module Bolt
         update_targets(options)
       end
 
+      unless options.key?(:verbose)
+        # Default to verbose for everything except plans
+        options[:verbose] = options[:subcommand] != 'plan'
+      end
+
       options
     rescue Bolt::Error => e
       warn e.message
@@ -261,11 +266,19 @@ module Bolt
         screen += '_object'
       end
 
-      @analytics.screen_view(screen,
-                             output_format: config.format,
-                             target_nodes: options.fetch(:targets, []).count,
-                             inventory_nodes: inventory.node_names.count,
-                             inventory_groups: inventory.group_names.count)
+      screen_view_fields = {
+        output_format: config.format
+      }
+
+      # Only include target and inventory info for commands that take a targets
+      # list. This avoids loading inventory for commands that don't need it.
+      if options.key?(:targets)
+        screen_view_fields.merge!(target_nodes: options[:targets].count,
+                                  inventory_nodes: inventory.node_names.count,
+                                  inventory_groups: inventory.group_names.count)
+      end
+
+      @analytics.screen_view(screen, screen_view_fields)
 
       if options[:action] == 'show'
         if options[:subcommand] == 'task'
@@ -314,29 +327,23 @@ module Bolt
         elapsed_time = Benchmark.realtime do
           executor_opts = {}
           executor_opts['_description'] = options[:description] if options.key?(:description)
+          executor.subscribe(outputter)
+          executor.subscribe(log_outputter)
           results =
             case options[:subcommand]
             when 'command'
-              executor.run_command(targets, options[:object], executor_opts) do |event|
-                outputter.print_event(event)
-              end
+              executor.run_command(targets, options[:object], executor_opts)
             when 'script'
               script = options[:object]
               validate_file('script', script)
-              executor.run_script(
-                targets, script, options[:leftovers], executor_opts
-              ) do |event|
-                outputter.print_event(event)
-              end
+              executor.run_script(targets, script, options[:leftovers], executor_opts)
             when 'task'
               pal.run_task(options[:object],
                            targets,
                            options[:task_options],
                            executor,
                            inventory,
-                           options[:description]) do |event|
-                outputter.print_event(event)
-              end
+                           options[:description])
             when 'file'
               src = options[:object]
               dest = options[:leftovers].first
@@ -345,13 +352,13 @@ module Bolt
                 raise Bolt::CLIError, "A destination path must be specified"
               end
               validate_file('source file', src, true)
-              executor.upload_file(targets, src, dest, executor_opts) do |event|
-                outputter.print_event(event)
-              end
+              executor.upload_file(targets, src, dest, executor_opts)
             end
         end
 
+        executor.shutdown
         rerun.update(results)
+
         outputter.print_summary(results, elapsed_time)
         code = results.ok ? 0 : 2
       end
@@ -398,12 +405,16 @@ module Bolt
       plan_context[:description] = options[:description] if options[:description]
 
       executor = Bolt::Executor.new(config.concurrency, @analytics, options[:noop])
+      executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
+      executor.subscribe(log_outputter)
       executor.start_plan(plan_context)
       result = pal.run_plan(plan_name, plan_arguments, executor, inventory, puppetdb_client)
 
       # If a non-bolt exception bubbles up the plan won't get finished
       executor.finish_plan(result)
+      executor.shutdown
       rerun.update(result)
+
       outputter.print_plan_result(result)
       result.ok? ? 0 : 1
     end
@@ -412,18 +423,25 @@ module Bolt
       ast = pal.parse_manifest(code, filename)
 
       executor = Bolt::Executor.new(config.concurrency, @analytics, noop)
-      # Call start_plan just to enable plan_logging
-      executor.start_plan(nil)
-
-      pal.in_plan_compiler(executor, inventory, puppetdb_client) do |compiler|
-        compiler.call_function('apply_prep', targets)
-      end
+      executor.subscribe(outputter) if options.fetch(:format, 'human') == 'human'
+      executor.subscribe(log_outputter)
+      # apply logging looks like plan logging, so tell the outputter we're in a
+      # plan even though we're not
+      executor.publish_event(type: :plan_start, plan: nil)
+
+      results = nil
+      elapsed_time = Benchmark.realtime do
+        pal.in_plan_compiler(executor, inventory, puppetdb_client) do |compiler|
+          compiler.call_function('apply_prep', targets)
+        end
 
-      results = pal.with_bolt_executor(executor, inventory, puppetdb_client) do
-        Puppet.lookup(:apply_executor).apply_ast(ast, targets, '_catch_errors' => true, '_noop' => noop)
+        results = pal.with_bolt_executor(executor, inventory, puppetdb_client) do
+          Puppet.lookup(:apply_executor).apply_ast(ast, targets, '_catch_errors' => true, '_noop' => noop)
+        end
       end
 
-      outputter.print_apply_result(results)
+      executor.shutdown
+      outputter.print_apply_result(results, elapsed_time)
       rerun.update(results)
 
       results.ok ? 0 : 1
@@ -499,7 +517,11 @@ module Bolt
     end
 
     def outputter
-      @outputter ||= Bolt::Outputter.for_format(config.format, config.color, config.trace)
+      @outputter ||= Bolt::Outputter.for_format(config.format, config.color, options[:verbose], config.trace)
+    end
+
+    def log_outputter
+      @log_outputter ||= Bolt::Outputter::Logger.new(options[:verbose], config.trace)
     end
 
     def bundled_content

data/lib/bolt/config.rb

@@ -175,8 +175,6 @@ module Bolt
 
       if options[:debug]
         @log['console'][:level] = :debug
-      elsif options[:verbose]
-        @log['console'][:level] = :info
       end
 
       @compile_concurrency = options[:'compile-concurrency'] if options[:'compile-concurrency']

data/lib/bolt/executor.rb

@@ -8,7 +8,6 @@ require 'set'
 require 'bolt/analytics'
 require 'bolt/result'
 require 'bolt/config'
-require 'bolt/notifier'
 require 'bolt/result_set'
 require 'bolt/puppetdb'
 
@@ -41,6 +40,8 @@ module Bolt
                          end
       end
       @reported_transports = Set.new
+      @subscribers = Set.new
+      @publisher = Concurrent::SingleThreadExecutor.new
 
       @noop = noop
       @run_as = nil
@@ -50,7 +51,6 @@ module Bolt
                 Concurrent.global_immediate_executor
               end
       @logger.debug { "Started with #{concurrency} max thread(s)" }
-      @notifier = Bolt::Notifier.new
     end
 
     def transport(transport)
@@ -61,6 +61,24 @@ module Bolt
       impl.value
     end
 
+    def subscribe(subscriber)
+      @subscribers << subscriber
+      self
+    end
+
+    def publish_event(event)
+      @subscribers.each do |subscriber|
+        @publisher.post(subscriber) do |sub|
+          sub.handle_event(event)
+        end
+      end
+    end
+
+    def shutdown
+      @publisher.shutdown
+      @publisher.wait_for_termination
+    end
+
     # Starts executing the given block on a list of nodes in parallel, one thread per "batch".
     #
     # This is the main driver of execution on a list of targets. It first
@@ -129,31 +147,19 @@ module Bolt
     end
 
     def log_action(description, targets)
-      # When running a plan, info messages like starting a task are promoted to notice.
-      log_method = @plan_logging ? :notice : :info
-      target_str = if targets.length > 5
-                     "#{targets.count} targets"
-                   else
-                     targets.map(&:uri).join(', ')
-                   end
-
-      @logger.send(log_method, "Starting: #{description} on #{target_str}")
+      publish_event(type: :step_start, description: description, targets: targets)
 
       start_time = Time.now
       results = yield
       duration = Time.now - start_time
 
-      failures = results.error_set.length
-      plural = failures == 1 ? '' : 's'
-
-      @logger.send(log_method, "Finished: #{description} with #{failures} failure#{plural} in #{duration.round(2)} sec")
+      publish_event(type: :step_finish, description: description, result: results, duration: duration)
 
       results
     end
 
     def log_plan(plan_name)
-      log_method = @plan_logging ? :notice : :info
-      @logger.send(log_method, "Starting: plan #{plan_name}")
+      publish_event(type: :plan_start, plan: plan_name)
       start_time = Time.now
 
       results = nil
@@ -161,7 +167,7 @@ module Bolt
         results = yield
       ensure
         duration = Time.now - start_time
-        @logger.send(log_method, "Finished: plan #{plan_name} in #{duration.round(2)} sec")
+        publish_event(type: :plan_finish, plan: plan_name, duration: duration)
       end
 
       results
@@ -220,72 +226,56 @@ module Bolt
       result
     end
 
-    def run_command(targets, command, options = {}, &callback)
+    def run_command(targets, command, options = {})
       description = options.fetch('_description', "command '#{command}'")
       log_action(description, targets) do
-        notify = proc { |event| @notifier.notify(callback, event) if callback }
         options = { '_run_as' => run_as }.merge(options) if run_as
 
-        results = batch_execute(targets) do |transport, batch|
+        batch_execute(targets) do |transport, batch|
           with_node_logging("Running command '#{command}'", batch) do
-            transport.batch_command(batch, command, options, &notify)
+            transport.batch_command(batch, command, options, &method(:publish_event))
           end
         end
-
-        @notifier.shutdown
-        results
       end
     end
 
-    def run_script(targets, script, arguments, options = {}, &callback)
+    def run_script(targets, script, arguments, options = {})
       description = options.fetch('_description', "script #{script}")
       log_action(description, targets) do
-        notify = proc { |event| @notifier.notify(callback, event) if callback }
         options = { '_run_as' => run_as }.merge(options) if run_as
 
-        results = batch_execute(targets) do |transport, batch|
+        batch_execute(targets) do |transport, batch|
           with_node_logging("Running script #{script} with '#{arguments}'", batch) do
-            transport.batch_script(batch, script, arguments, options, &notify)
+            transport.batch_script(batch, script, arguments, options, &method(:publish_event))
           end
         end
-
-        @notifier.shutdown
-        results
       end
     end
 
-    def run_task(targets, task, arguments, options = {}, &callback)
+    def run_task(targets, task, arguments, options = {})
       description = options.fetch('_description', "task #{task.name}")
       log_action(description, targets) do
-        notify = proc { |event| @notifier.notify(callback, event) if callback }
         options = { '_run_as' => run_as }.merge(options) if run_as
         arguments['_task'] = task.name
 
-        results = batch_execute(targets) do |transport, batch|
+        batch_execute(targets) do |transport, batch|
           with_node_logging("Running task #{task.name} with '#{arguments}'", batch) do
-            transport.batch_task(batch, task, arguments, options, &notify)
+            transport.batch_task(batch, task, arguments, options, &method(:publish_event))
           end
         end
-
-        @notifier.shutdown
-        results
       end
     end
 
-    def upload_file(targets, source, destination, options = {}, &callback)
+    def upload_file(targets, source, destination, options = {})
       description = options.fetch('_description', "file upload from #{source} to #{destination}")
       log_action(description, targets) do
-        notify = proc { |event| @notifier.notify(callback, event) if callback }
         options = { '_run_as' => run_as }.merge(options) if run_as
 
-        results = batch_execute(targets) do |transport, batch|
+        batch_execute(targets) do |transport, batch|
           with_node_logging("Uploading file #{source} to #{destination}", batch) do
-            transport.batch_upload(batch, source, destination, options, &notify)
+            transport.batch_upload(batch, source, destination, options, &method(:publish_event))
           end
         end
-
-        @notifier.shutdown
-        results
       end
     end
 
@@ -334,7 +324,6 @@ module Bolt
     # we'll need to refactor.
     def start_plan(plan_context)
       transport('pcp').plan_context = plan_context
-      @plan_logging = true
     end
 
     def finish_plan(plan_result)
@@ -342,11 +331,10 @@ module Bolt
     end
 
     def without_default_logging
-      old_log = @plan_logging
-      @plan_logging = false
+      publish_event(type: :disable_default_output)
       yield
     ensure
-      @plan_logging = old_log
+      publish_event(type: :enable_default_output)
     end
   end
 end

data/lib/bolt/inventory/group.rb

@@ -62,6 +62,11 @@ module Bolt
             msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in node #{node['name']}"
             @logger.warn(msg)
           end
+
+          unless node['config'].nil? || node['config'].is_a?(Hash)
+            raise ValidationError.new("Invalid configuration for node: #{node['name']}", @name)
+          end
+
           config_keys = node['config']&.keys || []
           unless (unexpected_keys = config_keys - CONFIG_KEYS).empty?
             msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in config for node #{node['name']}"

data/lib/bolt/inventory/group2.rb

@@ -102,6 +102,11 @@ module Bolt
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in target #{target['name']}"
           @logger.warn(msg)
         end
+
+        unless target['config'].nil? || target['config'].is_a?(Hash)
+          raise ValidationError.new("Invalid configuration for target: #{target['name']}", @name)
+        end
+
         config_keys = target['config']&.keys || []
         unless (unexpected_keys = config_keys - CONFIG_KEYS).empty?
           msg = "Found unexpected key(s) #{unexpected_keys.join(', ')} in config for target #{target['name']}"

data/lib/bolt/logger.rb

@@ -33,7 +33,7 @@ module Bolt
       root_logger.add_appenders Logging.appenders.stderr(
         'console',
         layout: console_layout(color),
-        level: default_level
+        level: default_console_level
       )
 
       # We set the root logger's level so that it logs everything but we do
@@ -53,7 +53,7 @@ module Bolt
               filename: name[5..-1], # strip the "file:" prefix
               truncate: (params[:append] == false),
               layout: default_layout,
-              level: default_level
+              level: default_file_level
             )
           rescue ArgumentError => e
             raise Bolt::Error.new("Failed to open log #{name}: #{e.message}", 'bolt/log-error')
@@ -81,7 +81,11 @@ module Bolt
       )
     end
 
-    def self.default_level
+    def self.default_console_level
+      :warn
+    end
+
+    def self.default_file_level
       :notice
     end