logstruct 0.1.11 → 0.2.0

data/lib/log_struct/semantic_logger/concerns/log_methods.rb

@@ -10,89 +10,97 @@ module LogStruct
         requires_ancestor { LogStruct::SemanticLogger::Logger }
 
         # Override log methods to handle LogStruct types and broadcast
-        sig { params(message: T.untyped, payload: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
-        def debug(message = nil, payload = nil, &block)
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
+        def debug(message = nil, payload = nil, exception = nil, &block)
+          result = super(*semantic_logger_args(message, payload, exception), &block)
           instrument_log(message, :debug)
-          result = if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) || message.is_a?(Hash)
-            super(nil, payload: message, &block)
-          else
-            super
-          end
-          broadcasts.each do |logger|
-            next unless logger.respond_to?(:debug)
-            message.is_a?(String) ? logger.debug(message) : (logger.debug(&block) if block)
-          end
+          broadcast_message(:debug, message, block)
           result
         end
 
-        sig { params(message: T.untyped, payload: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
-        def info(message = nil, payload = nil, &block)
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
+        def info(message = nil, payload = nil, exception = nil, &block)
+          result = super(*semantic_logger_args(message, payload, exception), &block)
           instrument_log(message, :info)
-          result = if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) || message.is_a?(Hash)
-            super(nil, payload: message, &block)
-          else
-            super
-          end
-          broadcasts.each do |logger|
-            next unless logger.respond_to?(:info)
-            message.is_a?(String) ? logger.info(message) : (logger.info(&block) if block)
-          end
+          broadcast_message(:info, message, block)
           result
         end
 
-        sig { params(message: T.untyped, payload: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
-        def warn(message = nil, payload = nil, &block)
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
+        def warn(message = nil, payload = nil, exception = nil, &block)
+          result = super(*semantic_logger_args(message, payload, exception), &block)
           instrument_log(message, :warn)
-          result = if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) || message.is_a?(Hash)
-            super(nil, payload: message, &block)
-          else
-            super
-          end
-          broadcasts.each do |logger|
-            next unless logger.respond_to?(:warn)
-            message.is_a?(String) ? logger.warn(message) : (logger.warn(&block) if block)
-          end
+          broadcast_message(:warn, message, block)
           result
         end
 
-        sig { params(message: T.untyped, payload: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
-        def error(message = nil, payload = nil, &block)
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
+        def error(message = nil, payload = nil, exception = nil, &block)
+          result = super(*semantic_logger_args(message, payload, exception), &block)
           instrument_log(message, :error)
-          result = if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) || message.is_a?(Hash)
-            super(nil, payload: message, &block)
-          else
-            super
-          end
-          broadcasts.each do |logger|
-            next unless logger.respond_to?(:error)
-            message.is_a?(String) ? logger.error(message) : (logger.error(&block) if block)
-          end
+          broadcast_message(:error, message, block)
           result
         end
 
-        sig { params(message: T.untyped, payload: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
-        def fatal(message = nil, payload = nil, &block)
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped, block: T.nilable(T.proc.returns(String))).returns(T::Boolean) }
+        def fatal(message = nil, payload = nil, exception = nil, &block)
+          result = super(*semantic_logger_args(message, payload, exception), &block)
           instrument_log(message, :fatal)
-          result = if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct) || message.is_a?(Hash)
-            super(nil, payload: message, &block)
-          else
-            super
-          end
-          broadcasts.each do |logger|
-            next unless logger.respond_to?(:fatal)
-            message.is_a?(String) ? logger.fatal(message) : (logger.fatal(&block) if block)
-          end
+          broadcast_message(:fatal, message, block)
           result
         end
 
         private
 
-        # Instrument log events for subscribers
+        # SemanticLogger's arguments for a line: a LogStruct entry or struct goes in the payload, and a Hash is the
+        # payload itself, so its keys are written at the top level of the line. An exception can come second
+        # (logger.error({id: 1}, error)) or third.
+        sig { params(message: T.untyped, payload: T.untyped, exception: T.untyped).returns([T.untyped, T.untyped, T.untyped]) }
+        def semantic_logger_args(message, payload, exception)
+          if message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct)
+            [nil, {payload: message}, nil]
+          elsif message.is_a?(Hash)
+            [nil, message, payload.is_a?(Exception) ? payload : exception]
+          else
+            [message, payload, exception]
+          end
+        end
+
+        # Sends a LogStruct entry to the log.logstruct subscribers, after its line is queued to be written, so a
+        # subscriber's error can't lose it. The error is handled like the app's other errors (standard_errors, which
+        # raises it from the log call by default), and the lines written while handling it aren't sent to the
+        # subscribers again.
         sig { params(message: T.untyped, level: Symbol).void }
         def instrument_log(message, level)
           return unless message.is_a?(LogStruct::Log::Interfaces::CommonFields) || message.is_a?(T::Struct)
+          return if Thread.current[:logstruct_instrumenting_log]
+
+          Thread.current[:logstruct_instrumenting_log] = true
+          begin
+            ::ActiveSupport::Notifications.instrument("log.logstruct", log: message, level: level)
+          rescue => e
+            LogStruct.handle_exception(e, source: Source::App, context: {notification: "log.logstruct"})
+          ensure
+            Thread.current[:logstruct_instrumenting_log] = false
+          end
+        end
 
-          ::ActiveSupport::Notifications.instrument("log.logstruct", log: message, level: level)
+        # Other loggers get String messages, and what blocks return, scrubbed like a log line's strings
+        sig { params(level: Symbol, message: T.untyped, block: T.nilable(T.proc.returns(String))).void }
+        def broadcast_message(level, message, block)
+          broadcasts.each do |logger|
+            next unless logger.respond_to?(level)
+
+            if message.is_a?(String)
+              logger.public_send(level, StringScrubber.scrub(message))
+            elsif block
+              logger.public_send(level) do
+                # The block type says String, but nothing checks what a block returns
+                text = T.let(block.call, T.untyped)
+                text.is_a?(String) ? StringScrubber.scrub(text) : text
+              end
+            end
+          end
         end
       end
     end

data/lib/log_struct/semantic_logger/formatter.rb

@@ -3,53 +3,13 @@
 
 require "semantic_logger"
 require_relative "../formatter"
+require_relative "../params_errors"
 
 module LogStruct
   module SemanticLogger
-    # High-Performance JSON Formatter with LogStruct Integration
-    #
-    # This formatter extends SemanticLogger's JSON formatter to provide optimal
-    # JSON serialization performance while preserving all LogStruct features
-    # including data filtering, sensitive data scrubbing, and type-safe structures.
-    #
-    # ## Performance Advantages Over Rails Logger:
-    #
-    # ### Serialization Performance
-    # - **Direct JSON generation**: Bypasses intermediate object creation
-    # - **Streaming serialization**: Memory-efficient processing of large objects
-    # - **Type-optimized paths**: Fast serialization for common data types
-    # - **Zero-copy operations**: Minimal memory allocation during serialization
-    #
-    # ### Memory Efficiency
-    # - **Object reuse**: Formatter instances are reused across log calls
-    # - **Lazy evaluation**: Only processes data that will be included in output
-    # - **Efficient buffering**: Optimal buffer sizes for JSON generation
-    # - **Garbage collection friendly**: Minimal object allocation reduces GC pressure
-    #
-    # ### Integration Benefits
-    # - **LogStruct compatibility**: Native support for typed log structures
-    # - **Filter preservation**: Maintains all LogStruct filtering capabilities
-    # - **Scrubbing integration**: Seamless sensitive data scrubbing
-    # - **Error handling**: Robust handling of serialization errors
-    #
-    # ## Feature Preservation:
-    # This formatter maintains full compatibility with LogStruct's features:
-    # - Sensitive data filtering (passwords, tokens, etc.)
-    # - Recursive object scrubbing and processing
-    # - Type-safe log structure handling
-    # - Custom field transformations
-    # - Metadata preservation and enrichment
-    #
-    # ## JSON Output Structure:
-    # The formatter produces consistent, parseable JSON that includes:
-    # - Standard log fields (timestamp, level, message, logger name)
-    # - LogStruct-specific fields (source, event, context)
-    # - SemanticLogger metadata (process ID, thread ID, tags)
-    # - Application-specific payload data
-    #
-    # This combination provides the performance benefits of SemanticLogger with
-    # the structured data benefits of LogStruct, resulting in faster, more
-    # reliable logging for high-traffic applications.
+    # Writes each SemanticLogger line as LogStruct's JSON: LogStruct entries, Hashes, messages and exceptions, with
+    # SemanticLogger's tags and named tags (the request ID and the running Sidekiq job's context among them). LogStruct::Formatter
+    # filters and scrubs every value, within the size limits in config.limits.
     class Formatter < ::SemanticLogger::Formatters::Json
       extend T::Sig
 
@@ -64,12 +24,30 @@ module LogStruct
         # Extract LogStruct from various locations where it might be stored
         logstruct = extract_logstruct(log)
 
+        # The request ID, the running Sidekiq job's context and the other named tags (logger.tagged(user_id: 1))
+        # go in req_id and ctx, and tags (logger.tagged("billing"), config.log_tags, ActiveJob's) in tags
+        named_tags = log.named_tags
+        request_id = named_tags[:request_id]
+        job_context = named_tags[:sidekiq_context]
+        context = named_tags.except(:request_id, :sidekiq_context)
+        context = context.merge(job_context) if job_context.is_a?(Hash)
+        metadata = {
+          request_id: request_id.is_a?(String) ? request_id : nil,
+          context: context.empty? ? nil : context,
+          tags: log.tags.empty? ? nil : log.tags
+        }
+
         json = if logstruct
           # Use our formatter to process LogStruct types directly
-          @logstruct_formatter.call(log.level, log.time, log.name, logstruct)
-        elsif log.payload.is_a?(Hash) || log.payload.is_a?(T::Struct)
-          # Process hashes and T::Structs through our formatter
-          @logstruct_formatter.call(log.level, log.time, log.name, log.payload)
+          @logstruct_formatter.call(log.level, log.time, log.name, logstruct, **metadata)
+        elsif log.exception
+          @logstruct_formatter.call(log.level, log.time, log.name, exception_log(log), **metadata)
+        elsif log.payload.is_a?(Hash)
+          # Process hashes through our formatter
+          payload = with_message(without_reserved_keys(log.payload), log.message)
+          @logstruct_formatter.call(log.level, log.time, log.name, payload, **metadata)
+        elsif log.payload.is_a?(T::Struct)
+          @logstruct_formatter.call(log.level, log.time, log.name, log.payload, **metadata)
         else
           # For plain messages, create a Plain log entry
           message_data = log.payload || log.message
@@ -77,15 +55,7 @@ module LogStruct
             message: message_data,
             timestamp: log.time
           )
-          @logstruct_formatter.call(log.level, log.time, log.name, plain_log)
-        end
-
-        # Add request_id from named_tags if present
-        request_id = log.named_tags[:request_id]
-        if request_id
-          data = JSON.parse(json)
-          data["req_id"] = request_id
-          json = data.to_json
+          @logstruct_formatter.call(log.level, log.time, log.name, plain_log, **metadata)
         end
 
         # SemanticLogger appenders typically add their own newline. Avoid double newlines by stripping ours.
@@ -94,6 +64,58 @@ module LogStruct
 
       private
 
+      # logger.error(exception) and logger.error("Message", exception): an error line with the
+      # exception's class, message and backtrace. A logged message goes in msg, and the exception's
+      # message in error_message. A Hash logged with them adds its keys to the line.
+      sig { params(log: ::SemanticLogger::Log).returns(LogStruct::Log::Error) }
+      def exception_log(log)
+        exception = T.let(log.exception, Exception)
+        message = log.message
+        logged_message = (message.is_a?(String) && !message.empty?) ? message : nil
+        payload = log.payload
+        # Without the input a params parser quoted
+        exception_message = ParamsErrors.messages(exception).first
+        LogStruct::Log::Error.new(
+          source: Source::App,
+          error_class: exception.class,
+          message: logged_message || exception_message,
+          error_message: logged_message ? exception_message : nil,
+          backtrace: exception.backtrace,
+          additional_data: payload.is_a?(Hash) ? without_reserved_keys(payload) : nil,
+          timestamp: log.time
+        )
+      end
+
+      # A message logged beside a Hash (logger.info("Saved", id: 1)) goes in msg, unless the Hash
+      # has its own
+      sig { params(payload: T::Hash[T.untyped, T.untyped], message: T.untyped).returns(T::Hash[T.untyped, T.untyped]) }
+      def with_message(payload, message)
+        return payload unless message.is_a?(String) && !message.empty?
+        return payload if payload.key?(:msg) || payload.key?("msg")
+
+        {msg: message}.merge(payload)
+      end
+
+      # A logged Hash may set its own src and evt, but the line's timestamp and level come from the logger
+      HASH_RESERVED_KEYS = T.let(%w[ts lvl].freeze, T::Array[String])
+
+      sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T::Hash[T.untyped, T.untyped]) }
+      def without_reserved_keys(hash)
+        reserved = hash.keys.select { |key| HASH_RESERVED_KEYS.include?(key.to_s) }
+        return hash if reserved.empty?
+
+        # A copy, since other appenders format the same Hash
+        copy = hash.dup
+        reserved.each do |key|
+          LogStruct.handle_exception(
+            ArgumentError.new("A logged Hash attempted to set reserved key: #{key}"),
+            source: Source::Internal
+          )
+          copy.delete(key)
+        end
+        copy
+      end
+
       # Extract a LogStruct from the various places it might be stored in a SemanticLogger::Log
       sig do
         params(log: ::SemanticLogger::Log).returns(

data/lib/log_struct/semantic_logger/logger.rb

@@ -1,68 +1,28 @@
 # typed: strict
 # frozen_string_literal: true
 
+require "active_support/logger"
 require "semantic_logger"
 require_relative "concerns/log_methods"
 
 module LogStruct
   module SemanticLogger
-    # High-Performance Logger with LogStruct Integration
-    #
-    # This logger extends SemanticLogger::Logger to provide optimal logging performance
-    # while seamlessly integrating with LogStruct's typed logging system.
-    #
-    # ## Key Benefits Over Rails.logger:
-    #
-    # ### Performance
-    # - **10-100x faster** than Rails' default logger for high-volume applications
-    # - **Non-blocking I/O**: Uses background threads for actual log writes
-    # - **Minimal memory allocation**: Efficient object reuse and zero-copy operations
-    # - **Batched writes**: Reduces system calls by batching multiple log entries
-    #
-    # ### Reliability
-    # - **Thread-safe operations**: Safe for use in multi-threaded environments
-    # - **Error resilience**: Logger failures don't crash your application
-    # - **Graceful fallbacks**: Continues operating even if appenders fail
-    #
-    # ### Features
-    # - **Structured logging**: Native support for LogStruct types and hashes
-    # - **Rich metadata**: Automatic inclusion of process ID, thread ID, timestamps
-    # - **Tagged context**: Hierarchical tagging for request/job tracking
-    # - **Multiple destinations**: Simultaneously log to files, STDOUT, cloud services
-    #
-    # ### Development Experience
-    # - **Colorized output**: Beautiful ANSI-colored logs in development
-    # - **Detailed timing**: Built-in measurement of log processing time
-    # - **Context preservation**: Maintains Rails.logger compatibility
-    #
-    # ## Usage Examples
-    #
-    # The logger automatically handles LogStruct types, hashes, and plain messages:
+    # Rails.logger's replacement: a SemanticLogger logger that also takes LogStruct's typed entries and Hashes, and
+    # passes each line on to the loggers added with broadcast_to.
     #
     # ```ruby
     # logger = LogStruct::SemanticLogger::Logger.new("MyApp")
     #
-    # # LogStruct typed logging (optimal performance)
-    # log_entry = LogStruct::Log::Plain.new(
-    #   message: "User authenticated",
-    #   source: LogStruct::Source::App,
-    #   event: LogStruct::Event::Security
-    # )
-    # logger.info(log_entry)
+    # # A typed entry
+    # logger.info(LogStruct::Log::Plain.new(message: "User authenticated", additional_data: {user_id: 123}))
     #
-    # # Hash logging (automatically structured)
-    # logger.info({
-    #   action: "user_login",
-    #   user_id: 123,
-    #   ip_address: "192.168.1.1"
-    # })
+    # # A Hash: its keys are written at the top level of the line
+    # logger.info(action: "user_login", user_id: 123)
     #
-    # # Plain string logging (backward compatibility)
-    # logger.info("User logged in successfully")
+    # # A message, with a Hash or an exception beside it
+    # logger.info("User logged in", user_id: 123)
+    # logger.error("Import failed", error)
     # ```
-    #
-    # The logger is a drop-in replacement for Rails.logger and maintains full
-    # API compatibility while providing significantly enhanced performance.
     class Logger < ::SemanticLogger::Logger
       extend T::Sig
 
@@ -90,6 +50,14 @@ module LogStruct
       # ::Logger, ActiveSupport::Logger, test doubles, etc.
       sig { params(logger: T.untyped).returns(T.untyped) }
       def broadcast_to(logger)
+        # Every SemanticLogger logger writes to the same appenders, so broadcasting to one
+        # would write each message twice. Sidekiq's Rails integration broadcasts Rails.logger
+        # to Sidekiq's logger, for example.
+        return logger if logger.is_a?(::SemanticLogger::Logger)
+        # `rails server` in development broadcasts to a STDOUT logger, which would print each
+        # message again as text beside LogStruct's line
+        return logger if console_logger_with_console_appender?(logger)
+
         @broadcasts << logger
         logger
       end
@@ -104,11 +72,11 @@ module LogStruct
       # Support for tagged logging
       sig { params(tags: T.untyped, block: T.proc.returns(T.untyped)).returns(T.untyped) }
       def tagged(*tags, &block)
-        # Convert tags to array and pass individually to avoid splat issues
         tag_array = tags.flatten
         if tag_array.empty?
           super(&block)
         else
+          # Sorbet can't check a splat of an Array whose size it doesn't know
           super(*T.unsafe(tag_array), &block)
         end
       end
@@ -138,13 +106,21 @@ module LogStruct
         ::SemanticLogger.pop_tags(count)
       end
 
-      # Support for << operator (used by RailsLogSplitter)
+      # Support for << operator (used by RailsLogSplitter). info passes the message on to the broadcast loggers.
       sig { params(msg: String).returns(T.self_type) }
       def <<(msg)
         info(msg)
-        @broadcasts.each { |logger| logger << msg if logger.respond_to?(:<<) }
         self
       end
+
+      private
+
+      sig { params(logger: T.untyped).returns(T::Boolean) }
+      def console_logger_with_console_appender?(logger)
+        # rails server's logger writes to the STDOUT constant, which differs from $stdout when $stdout was replaced
+        ::SemanticLogger.appenders.console_output? &&
+          ::ActiveSupport::Logger.logger_outputs_to?(logger, $stdout, $stderr, STDOUT, STDERR) # rubocop:disable Style/GlobalStdStream
+      end
     end
 
     # Proxy object to provide ActiveJob-compatible formatter interface

data/lib/log_struct/semantic_logger/reopen_after_fork.rb

@@ -0,0 +1,40 @@
+# typed: strict
+# frozen_string_literal: true
+
+module LogStruct
+  module SemanticLogger
+    # SemanticLogger writes lines from a background thread, and a forked child doesn't get its parent's threads, so
+    # lines logged in Puma, Unicorn, Passenger or Pitchfork workers, Resque jobs and other forked children were never
+    # written. Ruby calls Process._fork for every fork, so each child reopens the appenders (a new writer thread).
+    module ReopenAfterFork
+      extend T::Sig
+
+      @installed = T.let(false, T::Boolean)
+
+      sig { void }
+      def self.install!
+        return if @installed
+        @installed = true
+
+        Process.singleton_class.prepend(ForkHook)
+      end
+
+      module ForkHook
+        extend T::Sig
+
+        sig { returns(Integer) }
+        def _fork
+          pid = T.let(super, Integer)
+          if pid.zero?
+            begin
+              ::SemanticLogger.reopen
+            rescue => e
+              LogStruct.handle_exception(e, source: LogStruct::Source::Internal, context: {integration: :semantic_logger})
+            end
+          end
+          pid
+        end
+      end
+    end
+  end
+end