legion-logging 1.4.2 → 1.5.0

data/lib/legion/logging/methods.rb

@@ -5,6 +5,13 @@ require 'securerandom'
 module Legion
   module Logging
     module Methods
+      COMPONENT_REGEX = %r{
+        /(runners|actors|actor|helpers|hooks|absorbers|matchers|transport|
+        exchanges|queues|messages|data|builders|tools|adapters|engines|
+        formatters|parsers|middleware)/
+      }x
+      EXCEPTION_PRIORITY = { warn: 0, error: 5, fatal: 9 }.freeze
+
       def trace(raw_message = nil, size: @trace_size, log_caller: true)
         return unless @trace_enabled
 
@@ -23,62 +30,36 @@ module Legion
         return unless log.level < 1
 
         message = yield if message.nil? && block_given?
-        message = maybe_redact(message)
-        message = Rainbow(message).blue if @color
-        writer = @async_writer
-        if writer&.alive?
-          writer.push(AsyncWriter::LogEntry.new(level: :debug, message: message, writer_context: nil))
-        else
-          log.debug(message)
-        end
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(:debug, raw)
+        write_async_or_sync(:debug, formatted, raw)
       end
 
       def info(message = nil)
         return unless log.level < 2
 
         message = yield if message.nil? && block_given?
-        message = maybe_redact(message)
-        message = Rainbow(message).green if @color
-        writer = @async_writer
-        if writer&.alive?
-          writer.push(AsyncWriter::LogEntry.new(level: :info, message: message, writer_context: nil))
-        else
-          log.info(message)
-        end
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(:info, raw)
+        write_async_or_sync(:info, formatted, raw)
       end
 
       def warn(message = nil)
         return unless log.level < 3
 
         message = yield if message.nil? && block_given?
-        message = maybe_redact(message)
-        raw = message
-        message = Rainbow(message).yellow if @color
-        writer = @async_writer
-        if writer&.alive?
-          ctx = build_writer_context(:warn, raw)
-          writer.push(AsyncWriter::LogEntry.new(level: :warn, message: message, writer_context: ctx))
-        else
-          log.warn(message)
-          fire_log_writer(:warn, raw)
-        end
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(:warn, raw)
+        write_async_or_sync(:warn, formatted, raw, writer_context: build_writer_context(:warn, raw))
       end
 
       def error(message = nil)
         return unless log.level < 4
 
         message = yield if message.nil? && block_given?
-        message = maybe_redact(message)
-        raw = message
-        message = Rainbow(message).red if @color
-        writer = @async_writer
-        if writer&.alive?
-          ctx = build_writer_context(:error, raw)
-          writer.push(AsyncWriter::LogEntry.new(level: :error, message: message, writer_context: ctx))
-        else
-          log.error(message)
-          fire_log_writer(:error, raw)
-        end
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(:error, raw)
+        write_async_or_sync(:error, formatted, raw, writer_context: build_writer_context(:error, raw))
       end
 
       def fatal(message = nil)
@@ -94,13 +75,22 @@ module Legion
 
       def unknown(message = nil)
         message = yield if message.nil? && block_given?
-        message = maybe_redact(message)
-        message = Rainbow(message).purple if @color
-        writer = @async_writer
-        if writer&.alive?
-          writer.push(AsyncWriter::LogEntry.new(level: :unknown, message: message, writer_context: nil))
-        else
-          log.unknown(message)
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(:unknown, raw)
+        write_async_or_sync(:unknown, formatted, raw)
+      end
+
+      def emit_tagged(level, message = nil, segments: nil, method_ctx: nil)
+        level = level.to_sym
+        message = yield if message.nil? && block_given?
+        return if message.nil?
+
+        raw = maybe_redact(message)
+        formatted = format_message_for_level(level, raw)
+
+        with_tagged_context(segments, method_ctx) do
+          write_forced(level, formatted)
+          fire_log_writer(level, raw) if %i[warn error fatal].include?(level)
         end
       end
 
@@ -116,6 +106,7 @@ module Legion
         level = level.to_sym if level.respond_to?(:to_sym)
         # 1. Log human-readable line to stdout/file (bypass writer callbacks)
         msg = exception.respond_to?(:message) ? exception.message : exception.to_s
+        msg = maybe_redact(msg)
         log.public_send(level, msg) if respond_to?(:log) && log.respond_to?(level)
 
         # 2. Build rich exception event
@@ -168,6 +159,81 @@ module Legion
         message
       end
 
+      def format_message_for_level(level, message)
+        return Rainbow(message).blue if level == :debug && @color
+        return Rainbow(message).green if level == :info && @color
+        return Rainbow(message).yellow if level == :warn && @color
+        return Rainbow(message).red if level == :error && @color
+        return Rainbow(message).darkred if level == :fatal && @color
+        return Rainbow(message).purple if level == :unknown && @color
+
+        message
+      end
+
+      def with_tagged_context(segments, method_ctx)
+        prev_segments = Thread.current[:legion_log_segments]
+        prev_method_ctx = Thread.current[:legion_log_method]
+
+        Thread.current[:legion_log_segments] = segments unless segments.nil?
+        Thread.current[:legion_log_method] = method_ctx unless method_ctx.nil?
+        yield
+      ensure
+        Thread.current[:legion_log_segments] = prev_segments
+        Thread.current[:legion_log_method] = prev_method_ctx
+      end
+
+      def write_forced(level, message)
+        logger = log
+        formatter = logger.formatter || ::Logger::Formatter.new
+        rendered = formatter.call(severity_label_for(level), Time.now, nil, message)
+
+        log_device = logger.instance_variable_get(:@logdev)
+        if log_device.respond_to?(:write)
+          log_device.write(rendered)
+        else
+          $stdout.write(rendered)
+        end
+      end
+
+      def severity_label_for(level)
+        return 'ANY' if level == :unknown
+
+        level.to_s.upcase
+      end
+
+      def write_async_or_sync(level, formatted_message, raw_message, writer_context: nil)
+        writer = @async_writer
+        caller_trace = capture_runner_trace_for_async
+        if writer&.alive?
+          queued = writer.push(AsyncWriter::LogEntry.new(
+                                 level:          level,
+                                 message:        formatted_message,
+                                 writer_context: writer_context,
+                                 segments:       Thread.current[:legion_log_segments],
+                                 method_ctx:     Thread.current[:legion_log_method],
+                                 caller_trace:   caller_trace
+                               ))
+          return if queued
+        end
+
+        with_caller_trace(caller_trace) do
+          log.public_send(level, formatted_message)
+          fire_log_writer(level, raw_message) if writer_context
+        end
+      end
+
+      def capture_runner_trace_for_async
+        build_runner_trace(caller_locations(4, 1)&.first)
+      end
+
+      def with_caller_trace(caller_trace)
+        prev_caller_trace = Thread.current[:legion_log_caller]
+        Thread.current[:legion_log_caller] = caller_trace
+        yield
+      ensure
+        Thread.current[:legion_log_caller] = prev_caller_trace
+      end
+
       def redaction_enabled?
         return false unless defined?(Legion::Settings)
 
@@ -219,16 +285,18 @@ module Legion
           timestamp:      Time.now.to_i,
           app_id:         'legionio',
           type:           'exception_event',
-          priority:       { warn: 0, error: 5, fatal: 9 }[level] || 5,
+          priority:       EXCEPTION_PRIORITY[level] || 5,
           delivery_mode:  2
         }
       end
 
       def build_writer_context(level, message)
-        return nil if Legion::Logging.instance_variable_get(:@log_writer).nil?
+        has_writer = !Legion::Logging.instance_variable_get(:@log_writer).nil?
+        has_hooks = defined?(Legion::Logging::Hooks) && Legion::Logging::Hooks.enabled?
+        return nil unless has_writer || has_hooks
 
         lex_val  = instance_variable_defined?(:@lex) ? @lex : nil
-        lex_segs = instance_variable_defined?(:@lex_segments) ? @lex_segments : nil
+        lex_segs = Thread.current[:legion_log_segments] || (instance_variable_defined?(:@lex_segments) ? @lex_segments : nil)
 
         event = Legion::Logging::EventBuilder.build(
           level:         level,
@@ -242,7 +310,7 @@ module Legion
 
       def fire_log_writer(level, message)
         lex_val  = instance_variable_defined?(:@lex) ? @lex : nil
-        lex_segs = instance_variable_defined?(:@lex_segments) ? @lex_segments : nil
+        lex_segs = Thread.current[:legion_log_segments] || (instance_variable_defined?(:@lex_segments) ? @lex_segments : nil)
 
         event = Legion::Logging::EventBuilder.build(
           level:         level,
@@ -252,13 +320,13 @@ module Legion
           caller_offset: 4
         )
         lex_name = event[:lex] || 'core'
-        component = event.dig(:caller, :file).to_s[%r{/(runners|actors|transport|helpers|builders)/}, 1] || 'unknown'
+        component = event.dig(:caller, :file).to_s[COMPONENT_REGEX, 1] || 'unknown'
         routing_key = "legion.logging.log.#{level}.#{lex_name}.#{component}"
         Legion::Logging.log_writer.call(event, routing_key: routing_key)
+        Legion::Logging::Hooks.fire(level, message, event) if defined?(Legion::Logging::Hooks)
       rescue StandardError => e
-        if respond_to?(:log) && log.respond_to?(:warn)
-          log.warn("fire_log_writer failed for level=#{level}, routing_key=#{routing_key}: #{e.class}: #{e.message}")
-        end
+        rk = defined?(routing_key) ? routing_key : 'unknown'
+        log.warn("fire_log_writer failed for level=#{level}, routing_key=#{rk}: #{e.class}: #{e.message}") if respond_to?(:log) && log.respond_to?(:warn)
       end
     end
   end

data/lib/legion/logging/multi_io.rb

@@ -10,7 +10,6 @@ module Legion
       def write(message)
         @targets.each do |t|
           t.write(message)
-          t.flush if t.respond_to?(:flush)
         end
       end
 

data/lib/legion/logging/redactor.rb

@@ -18,7 +18,18 @@ module Legion
         bearer_token:   %r{Bearer\s+[A-Za-z0-9._~+/=-]{20,}}i
       }.freeze
 
-      SENSITIVE_FIELDS = %w[password secret token api_key authorization].freeze
+      SENSITIVE_FIELDS = %w[
+        password
+        secret
+        token
+        api_key
+        access_key
+        private_key
+        public_key
+        authorization
+      ].freeze
+      SENSITIVE_SUFFIXES = %w[token secret password passphrase credential credentials].freeze
+      SAFE_KEY_FIELDS = %w[primary_key foreign_key sort_key partition_key routing_key].freeze
 
       REDACTED = '[REDACTED]'
 
@@ -49,7 +60,17 @@ module Legion
         private
 
         def sensitive_field?(key)
-          SENSITIVE_FIELDS.include?(key.to_s.downcase)
+          normalized = normalize_key(key)
+          return false if SAFE_KEY_FIELDS.include?(normalized)
+          return true if SENSITIVE_FIELDS.include?(normalized)
+          return true if normalized.include?('authorization')
+          return true if normalized.start_with?('auth_') || normalized.end_with?('_auth')
+          return true if normalized.start_with?('bearer_') || normalized.end_with?('_bearer')
+          return true if SENSITIVE_SUFFIXES.any? { |suffix| normalized.end_with?("_#{suffix}") }
+
+          %w[api access client private public auth secret signing session].any? do |prefix|
+            normalized == "#{prefix}_key"
+          end
         end
 
         def all_patterns
@@ -79,6 +100,16 @@ module Legion
         def reset_pattern_cache!
           @all_patterns = nil
         end
+
+        def refresh_patterns!
+          reset_pattern_cache!
+        end
+
+        public :refresh_patterns!
+
+        def normalize_key(key)
+          key.to_s.downcase.gsub(/[^a-z0-9]+/, '_').gsub(/\A_+|_+\z/, '')
+        end
       end
     end
   end

data/lib/legion/logging/settings.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Legion
+  module Logging
+    module Settings
+      def self.default
+        {
+          level:      :info,
+          trace:      true,
+          trace_size: 4,
+          extended:   true
+        }
+      end
+    end
+  end
+end

data/lib/legion/logging/shipper/file_transport.rb

@@ -11,10 +11,18 @@ module Legion
 
         class << self
           def ship(event)
+            ship_batch([event])
+          end
+
+          def ship_batch(events)
+            batch = Array(events)
+            return true if batch.empty?
+
             path = resolve_path
             FileUtils.mkdir_p(File.dirname(path))
             File.open(path, 'a') do |f|
-              f.puts(::JSON.generate(event))
+              f.write(batch.map { |event| ::JSON.generate(event) }.join("\n"))
+              f.write("\n")
             end
             true
           rescue StandardError => e

data/lib/legion/logging/shipper/http_transport.rb

@@ -10,6 +10,10 @@ module Legion
       module HttpTransport
         class << self
           def ship(events)
+            ship_batch(events)
+          end
+
+          def ship_batch(events)
             endpoint = resolve_endpoint
             return false unless endpoint
 
@@ -29,7 +33,7 @@ module Legion
           def post(uri, body)
             req = Net::HTTP::Post.new(uri)
             req['Content-Type'] = 'application/json'
-            apply_auth(req)
+            apply_auth(req, uri)
 
             Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
                                                      open_timeout: 5, read_timeout: 10) do |http|
@@ -50,11 +54,11 @@ module Legion
             uri.path.include?('/services/collector')
           end
 
-          def apply_auth(req)
+          def apply_auth(req, uri)
             token = auth_token
             return unless token
 
-            req['Authorization'] = if splunk_hec?(URI(req.path.empty? ? '/' : req.uri&.to_s || '/'))
+            req['Authorization'] = if splunk_hec?(uri)
                                      "Splunk #{token}"
                                    else
                                      "Bearer #{token}"

data/lib/legion/logging/shipper.rb

@@ -30,21 +30,25 @@ module Legion
           transport = TRANSPORTS[transport_type]
           return unless transport
 
-          batch = nil
-          @mutex.synchronize do
-            batch = @buffer.dup
-            @buffer.clear
+          @flush_mutex ||= Mutex.new
+          @flush_mutex.synchronize do
+            batch = nil
+            @mutex.synchronize { batch = @buffer.dup }
+            return if batch.empty?
+
+            delivered = deliver(transport, batch)
+            @mutex.synchronize { @buffer.shift(batch.size) if delivered }
+            delivered
           end
-
-          deliver(transport, batch)
         end
 
         def start
           return unless enabled?
           return if @flush_thread&.alive?
 
-          @buffer = []
-          @mutex  = Mutex.new
+          @buffer ||= []
+          @mutex  ||= Mutex.new
+          @flush_mutex ||= Mutex.new
           interval = flush_interval
           @flush_thread = Thread.new do
             loop do
@@ -83,14 +87,14 @@ module Legion
         end
 
         def deliver(transport, batch)
-          if transport.method(:ship).arity == 1
-            # HttpTransport accepts a batch array
-            transport.ship(batch)
+          if transport.respond_to?(:ship_batch)
+            transport.ship_batch(batch)
           else
-            batch.each { |e| transport.ship(e) }
+            batch.all? { |event| transport.ship(event) }
           end
         rescue StandardError => e
           Legion::Logging.error("Shipper deliver failed: #{e.message}") if defined?(Legion::Logging)
+          false
         end
 
         def shippable_level?(level)

data/lib/legion/logging/tagged_logger.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Legion
+  module Logging
+    class TaggedLogger
+      LEVELS = { debug: 0, info: 1, warn: 2, error: 3, fatal: 4, unknown: 5 }.freeze
+
+      attr_reader :segments, :trace_enabled, :extended
+
+      def initialize(
+        segments:,
+        level: Legion::Logging::Settings.default[:level],
+        trace: Legion::Logging::Settings.default[:trace],
+        trace_size: Legion::Logging::Settings.default[:trace_size],
+        extended: Legion::Logging::Settings.default[:extended],
+        **_opts
+      )
+        @segments = segments
+        @level_value =
+          if level.is_a?(Integer)
+            level
+          else
+            default_level = Legion::Logging::Settings.default[:level].to_s.downcase.to_sym
+            LEVELS.fetch(level.to_s.downcase.to_sym, LEVELS.fetch(default_level, LEVELS[:info]))
+          end
+        @trace_enabled = trace
+        @trace_size = trace_size
+        @extended = extended
+      end
+
+      def level
+        @level_value
+      end
+
+      def debug(message = nil)
+        return unless @level_value < 1
+
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:debug, message) }
+      end
+
+      def info(message = nil)
+        return unless @level_value < 2
+
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:info, message) }
+      end
+
+      def warn(message = nil)
+        return unless @level_value < 3
+
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:warn, message) }
+      end
+
+      def error(message = nil)
+        return unless @level_value < 4
+
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:error, message) }
+      end
+
+      def fatal(message = nil)
+        return unless @level_value < 5
+
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:fatal, message) }
+      end
+
+      def unknown(message = nil)
+        message = yield if message.nil? && block_given?
+        with_segments { dispatch(:unknown, message) }
+      end
+
+      def trace(raw_message = nil, size: @trace_size, log_caller: true)
+        return unless @trace_enabled
+
+        raw_message = yield if raw_message.nil? && block_given?
+        message = "Tracing: #{raw_message} "
+        if log_caller
+          frames = size ? caller_locations(2, size) : caller_locations(2)
+          message.concat(frames&.join(', ').to_s)
+        end
+        with_segments { Legion::Logging.unknown(message) }
+      end
+
+      def thread(kvl: false, **_opts)
+        if kvl
+          "thread=#{Thread.current.object_id}"
+        else
+          Thread.current.object_id.to_s
+        end
+      end
+
+      private
+
+      def dispatch(level, message)
+        return unless defined?(Legion::Logging)
+
+        if Legion::Logging.respond_to?(:emit_tagged)
+          Legion::Logging.emit_tagged(level, message, segments: @segments)
+          return
+        end
+
+        if Legion::Logging.respond_to?(level)
+          Legion::Logging.public_send(level, message)
+          return
+        end
+
+        fallback = fallback_level(level)
+        Legion::Logging.public_send(fallback, message) if fallback && Legion::Logging.respond_to?(fallback)
+      end
+
+      def fallback_level(level)
+        return :debug if level == :unknown
+
+        nil
+      end
+
+      def with_segments
+        prev = Thread.current[:legion_log_segments]
+        Thread.current[:legion_log_segments] = @segments
+        yield
+      ensure
+        Thread.current[:legion_log_segments] = prev
+      end
+    end
+  end
+end

data/lib/legion/logging/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Logging
-    VERSION = '1.4.2'
+    VERSION = '1.5.0'
   end
 end

data/lib/legion/logging.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
 require 'legion/logging/version'
+require 'legion/logging/settings'
 require 'legion/logging/logger'
 require 'legion/logging/methods'
 require 'legion/logging/builder'
 require 'legion/logging/event_builder'
 require 'legion/logging/async_writer'
+require 'legion/logging/tagged_logger'
 require 'legion/logging/helper'
 require 'legion/logging/category_registry'
+require 'legion/logging/hooks'
 
 require 'json'
 require 'logger'
@@ -22,7 +25,7 @@ module Legion
       attr_reader :color
       attr_writer :log_writer, :exception_writer
 
-      DEFAULT_LOG_WRITER       = ->(_event, routing_key:) {}
+      DEFAULT_LOG_WRITER = ->(_event, routing_key:) {}
       DEFAULT_EXCEPTION_WRITER = ->(_event, routing_key:, headers:, properties:) {}
 
       def log_writer
@@ -33,6 +36,14 @@ module Legion
         @exception_writer || DEFAULT_EXCEPTION_WRITER
       end
 
+      def current_settings
+        (@current_settings || {}).dup
+      end
+
+      def configuration_generation
+        @configuration_generation || 0
+      end
+
       def register_category(name, description: nil, expected_fields: [])
         CategoryRegistry.register_category(name, description: description, expected_fields: expected_fields)
       end
@@ -41,18 +52,57 @@ module Legion
         CategoryRegistry.registered_categories
       end
 
-      def setup(level: 'info', format: :text, async: true, **options)
+      def on_fatal(&)
+        Hooks.on_fatal(&)
+      end
+
+      def on_error(&)
+        Hooks.on_error(&)
+      end
+
+      def on_warn(&)
+        Hooks.on_warn(&)
+      end
+
+      def enable_hooks!
+        Hooks.enable_hooks!
+      end
+
+      def disable_hooks!
+        Hooks.disable_hooks!
+      end
+
+      def clear_hooks!
+        Hooks.clear_hooks!
+      end
+
+      def setup(level: 'debug', format: :text, async: true, **options)
         output(**options)
         log_level(level)
         log_format(format: format, **options)
         @color = options[:color]
         @color = format != :json && (options[:color] || (options[:color].nil? && options[:log_file].nil?))
+        @current_settings = {
+          level:       level,
+          format:      format.to_sym,
+          async:       async,
+          trace:       options.fetch(:trace, true),
+          trace_size:  options.fetch(:trace_size, 4),
+          extended:    options.fetch(:extended, true),
+          log_file:    options[:log_file],
+          log_stdout:  options[:log_stdout],
+          include_pid: options.fetch(:include_pid, false),
+          color:       @color
+        }.freeze
+        @configuration_generation = configuration_generation + 1
+        Legion::Logging::Redactor.refresh_patterns! if defined?(Legion::Logging::Redactor)
         if async
-          buffer = if defined?(Legion::Settings)
-                     Legion::Settings.dig(:logging, :async, :buffer_size) || 10_000
-                   else
-                     10_000
+          buffer = if defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
+                     logging_settings = Legion::Settings[:logging]
+                     async_settings = logging_settings[:async] if logging_settings.is_a?(Hash)
+                     async_settings[:buffer_size] if async_settings.is_a?(Hash)
                    end
+          buffer ||= 10_000
           start_async_writer(buffer_size: buffer)
         else
           stop_async_writer