test-kitchen 4.0.0 → 4.1.0

data/lib/kitchen/logger.rb

@@ -16,7 +16,9 @@
 # limitations under the License.
 
 require "fileutils" unless defined?(FileUtils)
-require "logger"
+require "json" unless defined?(JSON)
+require "logger" unless defined?(Logger)
+require "time" unless defined?(Time)
 
 module Kitchen
   # Logging implementation for Kitchen. By default the console/stdout output
@@ -31,6 +33,15 @@ module Kitchen
     # @return [IO] the log device
     attr_reader :logdev
 
+    # @return [IO] the structured log device
+    attr_reader :structured_logdev
+
+    # @return [String,nil] the expanded text log path, if configured
+    attr_reader :logdev_path
+
+    # @return [String,nil] the expanded structured log path, if configured
+    attr_reader :structured_logdev_path
+
     # @return [Boolean] whether logger is configured for
     #   overwriting
     attr_reader :log_overwrite
@@ -55,13 +66,23 @@ module Kitchen
     #   when Test Kitchen runs.
     #   (default: `$stdout.tty?`)
     def initialize(options = {})
+      @base_metadata = options[:metadata] || {}
       @log_overwrite = if options[:log_overwrite].nil?
                          default_log_overwrite
                        else
                          options[:log_overwrite]
                        end
 
-      @logdev = logdev_logger(options[:logdev], log_overwrite) if options[:logdev]
+      @logdev = device_factory.logdev_logger(options[:logdev], log_overwrite) if options[:logdev]
+      if options[:structured_logdev]
+        @structured_logdev = device_factory.structured_logdev_logger(
+          options[:structured_logdev],
+          log_overwrite,
+          -> { metadata }
+        )
+      end
+      @logdev_path = expanded_log_path(options[:logdev])
+      @structured_logdev_path = expanded_log_path(options[:structured_logdev])
 
       populate_loggers(options)
 
@@ -77,10 +98,12 @@ module Kitchen
     def populate_loggers(options)
       @loggers = []
       @loggers << logdev unless logdev.nil?
-      @loggers << stdout_logger(options[:stdout], options[:color], options[:colorize]) if
+      @loggers << structured_logdev unless structured_logdev.nil?
+      @loggers << device_factory.stdout_logger(options[:stdout], options[:color], options[:colorize]) if
         options[:stdout]
-      @loggers << stdout_logger($stdout, options[:color], options[:colorize]) if
+      @loggers << device_factory.stdout_logger($stdout, options[:color], options[:colorize]) if
         @loggers.empty?
+      @sink_set = SinkSet.new(@loggers)
     end
     private :populate_loggers
 
@@ -91,18 +114,14 @@ module Kitchen
       # @!macro delegate_to_first_logger
       #   @method $1()
       def delegate_to_first_logger(meth)
-        define_method(meth) { |*args| @loggers.first.public_send(meth, *args) }
+        define_method(meth) { |*args, &block| @sink_set.first(meth, *args, &block) }
       end
 
       # @api private
       # @!macro delegate_to_all_loggers
       #   @method $1()
       def delegate_to_all_loggers(meth)
-        define_method(meth) do |*args|
-          result = nil
-          @loggers.each { |l| result = l.public_send(meth, *args) }
-          result
-        end
+        define_method(meth) { |*args, &block| @sink_set.all(meth, *args, &block) }
       end
     end
 
@@ -268,6 +287,23 @@ module Kitchen
     # @see http://is.gd/b13cVn
     delegate_to_all_loggers :close
 
+    # @return [Hash] metadata added to structured log events
+    def metadata
+      metadata_stack.inject(@base_metadata.dup) do |result, values|
+        result.merge(values)
+      end
+    end
+
+    # Temporarily adds metadata to structured log events.
+    #
+    # @param values [Hash] metadata fields to add for the block duration
+    def with_metadata(values)
+      metadata_stack.push(values.compact)
+      yield
+    ensure
+      metadata_stack.pop
+    end
+
     private
 
     # @return [Integer] the default logger level
@@ -282,54 +318,188 @@ module Kitchen
       Kitchen::DEFAULT_LOG_OVERWRITE
     end
 
-    # Construct a new standard out logger.
-    #
-    # @param stdout [IO] the IO object that represents stdout (or similar)
-    # @param color [Symbol] color to use when outputting messages
-    # @param colorize [Boolean] whether to enable color
-    # @return [StdoutLogger] a new logger
-    # @api private
-    def stdout_logger(stdout, color, colorize)
-      logger = StdoutLogger.new(stdout)
-      if colorize
-        logger.formatter = proc do |_severity, _datetime, _progname, msg|
-          Color.colorize(msg.dup.to_s, color).concat("\n")
+    def device_factory
+      @device_factory ||= DeviceFactory.new
+    end
+
+    def expanded_log_path(logdev)
+      File.expand_path(logdev) if logdev.is_a?(String)
+    end
+
+    def metadata_stack
+      Thread.current[metadata_stack_key] ||= []
+    end
+
+    def metadata_stack_key
+      @metadata_stack_key ||= :"kitchen_logger_metadata_#{object_id}"
+    end
+
+    # Internal composite for forwarding stdlib Logger-compatible calls to all
+    # configured logging sinks.
+    class SinkSet
+      def initialize(loggers)
+        @loggers = loggers
+      end
+
+      def first(meth, *args, &block)
+        @loggers.first.public_send(meth, *args, &block)
+      end
+
+      def all(meth, *args, &block)
+        result = nil
+        block = memoized_block(block) if block
+        @loggers.each { |logger| result = logger.public_send(meth, *args, &block) }
+        result
+      end
+
+      private
+
+      def memoized_block(block)
+        evaluated = false
+        value = nil
+        proc do
+          unless evaluated
+            value = block.call
+            evaluated = true
+          end
+          value
         end
-      else
-        logger.formatter = proc do |_severity, _datetime, _progname, msg|
-          msg.dup.concat("\n")
+      end
+    end
+
+    # Internal factory for building the concrete logger devices.
+    class DeviceFactory
+      # Construct a new standard out logger.
+      #
+      # @param stdout [IO] the IO object that represents stdout (or similar)
+      # @param color [Symbol] color to use when outputting messages
+      # @param colorize [Boolean] whether to enable color
+      # @return [StdoutLogger] a new logger
+      # @api private
+      def stdout_logger(stdout, color, colorize)
+        logger = StdoutLogger.new(stdout)
+        logger.formatter = stdout_formatter(color, colorize)
+        logger
+      end
+
+      # Construct a new logdev logger.
+      #
+      # @param filepath_or_logdev [String,IO] a filepath String or IO object
+      # @param log_overwrite [Boolean] apply log overwriting
+      #   if filepath_or_logdev is a file path
+      # @return [LogdevLogger] a new logger
+      # @api private
+      def logdev_logger(filepath_or_logdev, log_overwrite)
+        LogdevLogger.new(resolve_logdev(filepath_or_logdev, log_overwrite))
+      end
+
+      # Construct a new structured logdev logger.
+      #
+      # @param filepath_or_logdev [String,IO] a filepath String or IO object
+      # @param log_overwrite [Boolean] apply log overwriting
+      #   if filepath_or_logdev is a file path
+      # @param metadata_provider [Proc] callable returning structured metadata
+      # @return [StructuredLogdevLogger] a new logger
+      # @api private
+      def structured_logdev_logger(filepath_or_logdev, log_overwrite, metadata_provider)
+        StructuredLogdevLogger.new(
+          resolve_logdev(filepath_or_logdev, log_overwrite),
+          metadata_provider
+        )
+      end
+
+      private
+
+      def stdout_formatter(color, colorize)
+        if colorize
+          proc do |_severity, _datetime, _progname, msg|
+            Color.colorize(msg.dup.to_s, color).concat("\n")
+          end
+        else
+          proc do |_severity, _datetime, _progname, msg|
+            msg.dup.concat("\n")
+          end
+        end
+      end
+
+      # Return an IO object from a filepath String or the IO object itself.
+      #
+      # @param filepath_or_logdev [String,IO] a filepath String or IO object
+      # @param log_overwrite [Boolean] apply log overwriting
+      #   if filepath_or_logdev is a file path
+      # @return [IO] an IO object
+      # @api private
+      def resolve_logdev(filepath_or_logdev, log_overwrite)
+        if filepath_or_logdev.is_a? String
+          mode = log_overwrite ? "wb" : "ab"
+          FileUtils.mkdir_p(File.dirname(filepath_or_logdev))
+          file = File.open(File.expand_path(filepath_or_logdev), mode)
+          file.sync = true
+          file
+        else
+          filepath_or_logdev
         end
       end
-      logger
     end
 
-    # Construct a new logdev logger.
-    #
-    # @param filepath_or_logdev [String,IO] a filepath String or IO object
-    # @param log_overwrite [Boolean] apply log overwriting
-    #   if filepath_or_logdev is a file path
-    # @return [LogdevLogger] a new logger
-    # @api private
-    def logdev_logger(filepath_or_logdev, log_overwrite)
-      LogdevLogger.new(resolve_logdev(filepath_or_logdev, log_overwrite))
+    # Buffers streamed output until a complete line is available for logging.
+    class LineBuffer
+      def initialize(&line_handler)
+        @buffer = +""
+        @line_handler = line_handler
+      end
+
+      def <<(msg)
+        @buffer += msg
+        flush_lines
+      end
+
+      private
+
+      def flush_lines
+        while (i = @buffer.index("\n"))
+          @line_handler.call(@buffer[0, i].chomp)
+          @buffer[0, i + 1] = ""
+        end
+      end
     end
 
-    # Return an IO object from a filepath String or the IO object itself.
-    #
-    # @param filepath_or_logdev [String,IO] a filepath String or IO object
-    # @param log_overwrite [Boolean] apply log overwriting
-    #   if filepath_or_logdev is a file path
-    # @return [IO] an IO object
-    # @api private
-    def resolve_logdev(filepath_or_logdev, log_overwrite)
-      if filepath_or_logdev.is_a? String
-        mode = log_overwrite ? "wb" : "ab"
-        FileUtils.mkdir_p(File.dirname(filepath_or_logdev))
-        file = File.open(File.expand_path(filepath_or_logdev), mode)
-        file.sync = true
-        file
-      else
-        filepath_or_logdev
+    # Rewrites already-prefixed stream lines into the matching logger call.
+    class StreamLineFormatter
+      def initialize(logger)
+        @logger = logger
+      end
+
+      def format(line)
+        case line
+        when /^-----> / then log_line(:banner, line.gsub(/^[ >-]{6} /, ""))
+        when /^D      / then structured_line(:debug, line.gsub(/^D {6}/, ""), line)
+        when /^\$\$\$\$\$\$ / then structured_line(:warn, line.gsub(/^\${6} /, ""), line)
+        when /^>>>>>> / then log_line(:error, line.gsub(/^[ >-]{6} /, ""))
+        when /^!!!!!! / then structured_line(:fatal, line.gsub(/^!{6} /, ""), line)
+        when /^       / then log_line(:info, line.gsub(/^[ >-]{6} /, ""))
+        else log_line(:info, line)
+        end
+      end
+
+      private
+
+      def log_line(severity, message)
+        if severity == :banner
+          @logger.banner(message)
+        elsif @logger.respond_to?(:stream)
+          @logger.stream(severity, message)
+        else
+          @logger.public_send(severity, message)
+        end
+      end
+
+      def structured_line(severity, message, original)
+        if @logger.respond_to?(:stream)
+          @logger.stream(severity, message)
+        else
+          @logger.info(original)
+        end
       end
     end
 
@@ -342,35 +512,174 @@ module Kitchen
       #
       # @param msg [String] a message
       def <<(msg)
-        @buffer ||= ""
-        @buffer += msg
-        while (i = @buffer.index("\n"))
-          format_line(@buffer[0, i].chomp)
-          @buffer[0, i + 1] = ""
-        end
+        line_buffer << msg
       end
 
       # Log a banner message.
       #
       # @param msg [String] a message
       def banner(msg = nil, &block)
-        super_info("-----> #{msg}", &block)
+        if block
+          super_info(nil) { "-----> #{block.call}" }
+        else
+          super_info("-----> #{msg}")
+        end
       end
 
       private
 
-      # Reformat a line if it already contains log formatting.
-      #
-      # @param line [String] a message line
-      # @api private
-      def format_line(line)
-        case line
-        when /^-----> / then banner(line.gsub(/^[ >-]{6} /, ""))
-        when /^>>>>>> / then error(line.gsub(/^[ >-]{6} /, ""))
-        when /^       / then info(line.gsub(/^[ >-]{6} /, ""))
-        else info(line)
+      def line_buffer
+        formatter = StreamLineFormatter.new(self)
+        @line_buffer ||= LineBuffer.new do |line|
+          formatter.format(line)
+        end
+      end
+    end
+
+    # Internal class that emits one JSON object per log event.
+    class StructuredLogdevLogger
+      include ::Logger::Severity
+
+      SEVERITY_NAMES = {
+        DEBUG => "debug",
+        INFO => "info",
+        WARN => "warn",
+        ERROR => "error",
+        FATAL => "fatal",
+        UNKNOWN => "unknown",
+      }.freeze
+
+      attr_accessor :level
+      attr_accessor :progname
+      attr_accessor :datetime_format
+
+      def initialize(logdev, metadata_provider)
+        @logdev = logdev
+        @metadata_provider = metadata_provider
+        @level = INFO
+        @progname = "Kitchen"
+        @sequence = 0
+        @mutex = Mutex.new
+      end
+
+      def add(severity, message = nil, progname = nil)
+        severity ||= UNKNOWN
+        return true if severity < level
+
+        message = if message.nil?
+                    block_given? ? yield : progname
+                  else
+                    message
+                  end
+        write_event(severity, message, "log")
+      end
+
+      def <<(msg)
+        line_buffer << msg
+      end
+
+      def banner(msg = nil)
+        message = block_given? ? yield : msg
+        write_event(INFO, message, "banner") unless INFO < level
+      end
+
+      def stream(severity, msg)
+        severity = severity_const(severity)
+        write_event(severity, msg, "stream") unless severity < level
+      end
+
+      def debug(msg = nil, &block)
+        add(DEBUG, msg, nil, &block)
+      end
+
+      def info(msg = nil, &block)
+        add(INFO, msg, nil, &block)
+      end
+
+      def warn(msg = nil, &block)
+        add(WARN, msg, nil, &block)
+      end
+
+      def error(msg = nil, &block)
+        add(ERROR, msg, nil, &block)
+      end
+
+      def fatal(msg = nil, &block)
+        add(FATAL, msg, nil, &block)
+      end
+
+      def unknown(msg = nil, &block)
+        add(UNKNOWN, msg, nil, &block)
+      end
+
+      def debug?
+        level <= DEBUG
+      end
+
+      def info?
+        level <= INFO
+      end
+
+      def warn?
+        level <= WARN
+      end
+
+      def error?
+        level <= ERROR
+      end
+
+      def fatal?
+        level <= FATAL
+      end
+
+      def close
+        return unless @logdev.respond_to?(:close)
+
+        if @logdev.respond_to?(:closed?)
+          @logdev.close unless @logdev.closed?
+        else
+          @logdev.close
+        end
+      end
+
+      private
+
+      def line_buffer
+        formatter = StreamLineFormatter.new(self)
+        @line_buffer ||= LineBuffer.new do |line|
+          formatter.format(line)
         end
       end
+
+      def next_sequence
+        @sequence += 1
+      end
+
+      def severity_const(severity)
+        return severity if severity.is_a?(Integer)
+
+        ::Logger.const_get(severity.to_s.upcase)
+      end
+
+      def severity_name(severity)
+        SEVERITY_NAMES.fetch(severity, "unknown")
+      end
+
+      def write_event(severity, message, event_type)
+        @mutex.synchronize do
+          event = @metadata_provider.call.merge(
+            timestamp: Time.now.utc.iso8601(6),
+            level: severity_name(severity),
+            event_type:,
+            message: message.to_s,
+            progname:,
+            sequence: next_sequence
+          ).compact
+          @logdev.write(JSON.generate(event))
+          @logdev.write("\n")
+        end
+        true
+      end
     end
 
     # Internal class which reformats logging methods for display as console
@@ -380,42 +689,66 @@ module Kitchen
       #
       # @param msg [String] a message
       def debug(msg = nil, &block)
-        super("D      #{msg}", &block)
+        if block
+          super(nil) { "D      #{block.call}" }
+        else
+          super("D      #{msg}")
+        end
       end
 
       # Log an info message
       #
       # @param msg [String] a message
       def info(msg = nil, &block)
-        super("       #{msg}", &block)
+        if block
+          super(nil) { "       #{block.call}" }
+        else
+          super("       #{msg}")
+        end
       end
 
       # Log a warn message
       #
       # @param msg [String] a message
       def warn(msg = nil, &block)
-        super("$$$$$$ #{msg}", &block)
+        if block
+          super(nil) { "$$$$$$ #{block.call}" }
+        else
+          super("$$$$$$ #{msg}")
+        end
       end
 
       # Log an error message
       #
       # @param msg [String] a message
       def error(msg = nil, &block)
-        super(">>>>>> #{msg}", &block)
+        if block
+          super(nil) { ">>>>>> #{block.call}" }
+        else
+          super(">>>>>> #{msg}")
+        end
       end
 
       # Log a fatal message
       #
       # @param msg [String] a message
       def fatal(msg = nil, &block)
-        super("!!!!!! #{msg}", &block)
+        if block
+          super(nil) { "!!!!!! #{block.call}" }
+        else
+          super("!!!!!! #{msg}")
+        end
       end
 
       # Log an unknown message
       #
       # @param msg [String] a message
       def unknown(msg = nil, &block)
-        super("?????? #{msg}", &block)
+        if block
+          super(nil) { "?????? #{block.call}" }
+        else
+          super("?????? #{msg}")
+        end
       end
     end
   end

data/lib/kitchen/logging.rb

@@ -36,8 +36,8 @@ module Kitchen
       #   @return [nil,true] when the given severity is not high enough (for
       #     this particular logger), log no message, and return true
       def logger_method(meth)
-        define_method(meth) do |*args|
-          logger.public_send(meth, *args)
+        define_method(meth) do |*args, &block|
+          logger.public_send(meth, *args, &block)
         end
       end
     end

data/lib/kitchen/provisioner/base.rb

@@ -105,18 +105,33 @@ module Kitchen
           debug("Executing prepare command: #{prepare_command}")
           conn.execute(prepare_command)
           debug("Executing run command: #{run_command}")
-          conn.execute_with_retry(
-            encode_for_powershell(run_command),
-            config[:retry_on_exit_code],
-            config[:max_retries],
-            config[:wait_for_retry]
-          )
-          info("Downloading files from #{instance.to_str}")
-          config[:downloads].to_h.each do |remotes, local|
-            debug("Downloading #{Array(remotes).join(", ")} to #{local}")
-            conn.download(remotes, local)
+          begin
+            conn.execute_with_retry(
+              encode_for_powershell(run_command),
+              config[:retry_on_exit_code],
+              config[:max_retries],
+              config[:wait_for_retry]
+            )
+          ensure
+            # Retrieval is best-effort: a file we could not fetch must not
+            # fail an otherwise successful converge, and must never replace
+            # an error already in flight from the run command. A transport
+            # that cannot download at all is a bug, not a missing file, so
+            # it still raises when there is no converge error to protect.
+            run_error = $!
+            begin
+              info("Downloading files from #{instance.to_str}")
+              config[:downloads].to_h.each do |remotes, local|
+                debug("Downloading #{Array(remotes).join(", ")} to #{local}")
+                conn.download(remotes, local)
+              end
+              debug("Download complete")
+            rescue => ex
+              raise unless run_error || ex.is_a?(Kitchen::Transport::TransportFailed)
+
+              warn("Failed to download files from #{instance.to_str}: #{ex.message}")
+            end
           end
-          debug("Download complete")
         end
       rescue Kitchen::Transport::TransportFailed => ex
         raise ActionFailed, ex.message
@@ -339,13 +354,8 @@ module Kitchen
 
         # Replace $env:TEMP with the actual Windows temp directory based on the transport username
         if resolved_path.include?("$env:TEMP")
-          # Try to get username from transport configuration
           # For Windows systems, fallback to "Administrator" if not found
-          username = begin
-            instance.transport[:username]
-                     rescue
-                       nil
-          end
+          username = instance.transport[:username] if instance.transport.respond_to?(:[])
           username ||= "Administrator"
 
           temp_path = "C:/Users/#{username}/AppData/Local/Temp"