legion-logging 1.5.6 → 1.5.19

data/lib/legion/logging/output_logger.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Legion
+  module Logging
+    # Keeps the standard Logger interface used by Sinatra/APM on the async queue.
+    class OutputLogger < ::Logger
+      attr_accessor :async_writer
+
+      def add(severity, message = nil, progname = nil)
+        writer = @async_writer
+        return super unless writer && !writer.current_thread?
+
+        severity ||= UNKNOWN
+        return true if @logdev.nil? || severity < level
+
+        progname ||= @progname
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            message = progname
+            progname = @progname
+          end
+        end
+        enqueue_output(severity: severity, message: message, progname: progname)
+      end
+      alias log add
+
+      def <<(message)
+        writer = @async_writer
+        return super unless writer && !writer.current_thread?
+
+        enqueue_output(severity: nil, message: message, progname: nil)
+      end
+
+      private
+
+      def enqueue_output(severity:, message:, progname:) # rubocop:disable Naming/PredicateMethod
+        entry = Legion::Logging::AsyncWriter::OutputEntry.new(
+          severity:     severity,
+          message:      Legion::Logging::AsyncWriter.snapshot_value(value: message),
+          progname:     Legion::Logging::AsyncWriter.snapshot_value(value: progname),
+          segments:     Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_segments]),
+          method_ctx:   Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_method]),
+          caller_trace: Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_caller]),
+          conv_id:      Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_conv_id]),
+          request_id:   Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_request_id]),
+          exchange_id:  Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_exchange_id]),
+          chain_id:     Legion::Logging::AsyncWriter.snapshot_value(value: Thread.current[:legion_log_chain_id]),
+          origin:       { timestamp: Time.now.utc, thread: Thread.current.object_id }
+        )
+        raise ClosedQueueError, 'async logging writer is unavailable' unless @async_writer.alive? && @async_writer.push(entry)
+
+        true
+      end
+    end
+  end
+end

data/lib/legion/logging/redactor.rb

@@ -74,6 +74,9 @@ module Legion
         end
 
         def all_patterns
+          return PATTERNS unless defined?(Legion::Settings)
+          return PATTERNS unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
+
           @all_patterns ||= build_patterns
         end
 
@@ -85,8 +88,6 @@ module Legion
         end
 
         def custom_patterns
-          return {} unless defined?(Legion::Settings)
-
           raw = Legion::Settings.dig(:logging, :redactor, :custom_patterns)
           return {} unless raw.is_a?(Hash)
 

data/lib/legion/logging/settings.rb

@@ -3,15 +3,31 @@
 module Legion
   module Logging
     module Settings
+      # Log levels forwarded to the transport bus (and in-process Hooks) by
+      # default. Preserves historic behavior: only warn/error/fatal logs build a
+      # structured event. Override at runtime via logging.transport.log_levels
+      # (e.g. add :debug/:info) — see Methods#forwardable_log_levels.
+      DEFAULT_FORWARD_LOG_LEVELS = %i[warn error fatal].freeze
+
       def self.default
         {
-          level:           :info,
-          trace:           true,
-          trace_size:      4,
-          extended:        true,
-          backtrace_limit: nil
+          level:              :info,
+          trace:              true,
+          trace_size:         4,
+          extended:           true,
+          backtrace_limit:    nil,
+          forward_log_levels: DEFAULT_FORWARD_LOG_LEVELS
         }
       end
+
+      def self.async_buffer_size
+        return 10_000 unless defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
+        return 10_000 unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
+
+        logging = Legion::Settings[:logging]
+        async = logging[:async] if logging.is_a?(Hash)
+        async.is_a?(Hash) ? (async[:buffer_size] || 10_000) : 10_000
+      end
     end
   end
 end

data/lib/legion/logging/shipper/file_transport.rb

@@ -26,7 +26,7 @@ module Legion
             end
             true
           rescue StandardError => e
-            Legion::Logging.error("FileTransport ship failed: #{e.message}") if defined?(Legion::Logging)
+            Kernel.warn("FileTransport ship failed: #{e.message}")
             false
           end
 
@@ -40,6 +40,7 @@ module Legion
 
           def settings_path
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :file, :path)
           end

data/lib/legion/logging/shipper/http_transport.rb

@@ -24,7 +24,7 @@ module Legion
             response = post(uri, body)
             response.is_a?(Net::HTTPSuccess)
           rescue StandardError => e
-            Legion::Logging.error("HttpTransport ship failed: #{e.message}") if defined?(Legion::Logging)
+            Kernel.warn("HttpTransport ship failed: #{e.message}")
             false
           end
 
@@ -36,7 +36,7 @@ module Legion
             apply_auth(req, uri)
 
             Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
-                                                     open_timeout: 5, read_timeout: 10) do |http|
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
               http.request(req, body)
             end
           end
@@ -67,12 +67,14 @@ module Legion
 
           def auth_token
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :auth_token)
           end
 
           def resolve_endpoint
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :endpoint)
           end

data/lib/legion/logging/shipper.rb

@@ -8,134 +8,183 @@ module Legion
   module Logging
     module Shipper
       LEVEL_ORDER = %w[debug info warn error fatal].freeze
-
-      TRANSPORTS = {
-        file: FileTransport,
-        http: HttpTransport
-      }.freeze
+      TRANSPORTS = { file: FileTransport, http: HttpTransport }.freeze
+      MUTEX = Mutex.new
+      WAKE = ConditionVariable.new
+      CHANGED = ConditionVariable.new
+      private_constant :MUTEX, :WAKE, :CHANGED
+
+      @buffer = []
+      @accepted = 0
+      @delivered = 0
+      @attempt = 0
+      @flush_target = 0
 
       class << self
         def ship(event)
           return unless enabled?
           return unless shippable_level?(event[:level] || event['level'])
+          return unless TRANSPORTS.key?(transport_type)
 
-          redacted  = Redactor.redact(event)
-          transport = TRANSPORTS[transport_type]
-          buffer_event(redacted) if transport
-        end
+          redacted = Redactor.redact(event)
+          raise ClosedQueueError, 'log shipper is stopping' unless start
 
-        def flush
-          @mutex ||= Mutex.new
-          batch = @mutex.synchronize do
-            return true if @buffer.nil? || @buffer.empty?
+          MUTEX.synchronize do
+            raise ThreadError, 'log shipper cannot enqueue from its delivery thread' if Thread.current.equal?(@flush_thread)
 
-            flushed = @buffer.dup
-            @buffer.clear
-            flushed
+            CHANGED.wait(MUTEX) while !@stopping && @accepted - @delivered >= @capacity
+            raise ClosedQueueError, 'log shipper is stopping' if @stopping
+
+            @buffer << redacted
+            @accepted += 1
+            @flush_requested = true if @buffer.size >= @batch_size
+            WAKE.signal if @flush_requested
           end
+          true
+        end
+
+        def flush(timeout: 5)
+          return true if MUTEX.synchronize { @accepted == @delivered }
+          return false unless start
+
+          deadline = timeout && (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout)
+          MUTEX.synchronize do
+            return false if Thread.current.equal?(@flush_thread)
+
+            target = @accepted
+            attempt = @attempt
+            @flush_target = [@flush_target, target].max
+            @flush_requested = true
+            WAKE.signal
+            while @delivered < target
+              return false if @attempt > attempt && !@last_success
+              return false unless @flush_thread&.alive?
 
-          transport = TRANSPORTS[transport_type]
-          return true unless transport
+              remaining = deadline && (deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC))
+              return false if remaining && remaining <= 0
 
-          delivered = deliver(transport, batch)
-          @mutex.synchronize { @buffer.prepend(*batch) } unless delivered
-          delivered
+              CHANGED.wait(MUTEX, remaining)
+            end
+            true
+          end
         end
 
         def start
-          @start_mutex ||= Mutex.new
-          @start_mutex.synchronize do
-            return unless enabled?
-            return if @flush_thread&.alive?
-
-            @buffer ||= []
-            @mutex  ||= Mutex.new
-            @running = true
-            interval = flush_interval
-            @flush_thread = Thread.new do
-              while @running
-                sleep interval
-                flush
-              end
+          return false unless enabled?
+
+          MUTEX.synchronize do
+            return !@stopping if @flush_thread&.alive?
+
+            @capacity = Legion::Settings.dig(:logging, :shipper, :buffer_size) || 10_000
+            @batch_size = [batch_size, @capacity].min
+            @interval = flush_interval
+            unless @capacity.is_a?(Integer) && @capacity.positive? && @batch_size.is_a?(Integer) && @batch_size.positive? &&
+                   @interval.is_a?(Numeric) && @interval.positive?
+              raise ArgumentError, 'shipper buffer_size, batch_size, and flush_interval must be positive'
             end
+
+            @transport = TRANSPORTS[transport_type]
+            return false unless @transport
+
+            @stopping = false
+            @flush_requested = @buffer.size >= @batch_size
+            @flush_thread = Thread.new { consume }
             @flush_thread.name = 'legion-log-shipper'
             @flush_thread.abort_on_exception = false
+            true
           end
         end
 
-        def stop
-          @running = false
-          thread = @flush_thread
-          @flush_thread = nil
-          thread&.join(5)
-          flush
+        def stop(timeout: 5)
+          deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout if timeout
+          thread = MUTEX.synchronize do
+            @stopping = true
+            WAKE.broadcast
+            CHANGED.broadcast
+            if !@flush_thread&.alive? && @accepted > @delivered
+              @flush_thread = Thread.new { consume }
+              @flush_thread.name = 'legion-log-shipper'
+              @flush_thread.abort_on_exception = false
+            end
+            @flush_thread
+          end
+          return MUTEX.synchronize { @accepted == @delivered } unless thread
+          return false if Thread.current.equal?(thread)
+
+          remaining = [deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC), 0].max if deadline
+          thread.join(remaining)
+          MUTEX.synchronize { !thread.alive? && @accepted == @delivered }
         end
 
         def enabled?
           return false unless defined?(Legion::Settings)
+          return false unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
           Legion::Settings.dig(:logging, :shipper, :enabled) == true
         end
 
         private
 
-        def buffer_event(event)
-          @buffer ||= []
-          @mutex  ||= Mutex.new
-
-          full = false
-          @mutex.synchronize do
-            @buffer << event
-            full = @buffer.size >= batch_size
+        def consume
+          loop do
+            batch = MUTEX.synchronize do
+              WAKE.wait(MUTEX, @interval) unless @flush_requested || @stopping
+              @flush_requested = false
+              @buffer.shift(@batch_size)
+            end
+            delivered = batch.empty? || deliver(@transport, batch)
+            stopping = MUTEX.synchronize do
+              @attempt += 1
+              @last_success = delivered
+              if delivered
+                @delivered += batch.size
+                @flush_requested = @buffer.size >= @batch_size || @delivered < @flush_target
+              else
+                @buffer.prepend(*batch)
+              end
+              CHANGED.broadcast
+              @stopping && (!delivered || @buffer.empty?)
+            end
+            break if stopping
+          end
+        ensure
+          MUTEX.synchronize do
+            @stopping = true
+            CHANGED.broadcast
           end
-
-          flush if full
         end
 
         def deliver(transport, batch)
-          if transport.respond_to?(:ship_batch)
-            transport.ship_batch(batch)
-          else
-            batch.all? { |event| transport.ship(event) }
-          end
+          transport.ship_batch(batch)
         rescue StandardError => e
-          Legion::Logging.error("Shipper deliver failed: #{e.message}") if defined?(Legion::Logging)
+          Kernel.warn("Shipper deliver failed: #{e.message}")
           false
         end
 
         def shippable_level?(level)
           return true if level.nil?
 
-          min = minimum_level
-          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(min).to_i
+          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(minimum_level).to_i
         end
 
         def transport_type
-          return :file unless defined?(Legion::Settings)
-
-          key = Legion::Settings.dig(:logging, :shipper, :transport)
+          key = Legion::Settings.dig(:logging, :shipper, :transport) if defined?(Legion::Settings)
           key ? key.to_sym : :file
         end
 
         def batch_size
-          return 100 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :batch_size) || 100
         end
 
         def flush_interval
-          return 5 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :flush_interval) || 5
         end
 
         def minimum_level
-          return 'warn' unless defined?(Legion::Settings)
-
           levels = Legion::Settings.dig(:logging, :shipper, :levels)
           return 'warn' unless levels.is_a?(Array) && !levels.empty?
 
-          levels.min_by { |l| LEVEL_ORDER.index(l.to_s) || 99 }.to_s
+          levels.min_by { |level| LEVEL_ORDER.index(level.to_s) || 99 }.to_s
         end
       end
     end

data/lib/legion/logging/siem_exporter.rb

@@ -29,7 +29,7 @@ module Legion
           req.body = ::JSON.dump({ event: redact_phi(event.to_s), time: Time.now.to_f })
 
           Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
-                                                     open_timeout: 5, read_timeout: 10) do |http|
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
             http.request(req)
           end
         rescue StandardError => e

data/lib/legion/logging/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Logging
-    VERSION = '1.5.6'
+    VERSION = '1.5.19'
   end
 end

data/lib/legion/logging.rb

@@ -76,36 +76,37 @@ module Legion
         Hooks.clear_hooks!
       end
 
-      def setup(level: 'debug', format: :text, async: true, **options)
-        output(**options)
-        log_level(level)
-        log_format(format: format, **options)
-        @color = options[:color]
-        @color = format != :json && (options[:color] || (options[:color].nil? && options[:log_file].nil?))
-        @current_settings = {
-          level:       level,
-          format:      format.to_sym,
-          async:       async,
-          trace:       options.fetch(:trace, true),
-          trace_size:  options.fetch(:trace_size, 4),
-          extended:    options.fetch(:extended, true),
-          log_file:    options[:log_file],
-          log_stdout:  options[:log_stdout],
-          include_pid: options.fetch(:include_pid, false),
-          color:       @color
-        }.freeze
-        @configuration_generation = configuration_generation + 1
-        Legion::Logging::Redactor.refresh_patterns! if defined?(Legion::Logging::Redactor)
-        if async
-          buffer = if defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
-                     logging_settings = Legion::Settings[:logging]
-                     async_settings = logging_settings[:async] if logging_settings.is_a?(Hash)
-                     async_settings[:buffer_size] if async_settings.is_a?(Hash)
-                   end
-          buffer ||= 10_000
-          start_async_writer(buffer_size: buffer)
-        else
-          stop_async_writer
+      def setup(level: 'debug', format: :text, async: true, shutdown_timeout: 2, **options)
+        with_lifecycle_lock do
+          return false if @async_writer && stop_async_writer(timeout: shutdown_timeout) == false
+          return false if output(**options) == false
+
+          log_level(level)
+          log_format(format: format, **options)
+          @color = options[:color]
+          @color = format != :json && (options[:color] || (options[:color].nil? && options[:log_file].nil?))
+          @current_settings = {
+            level:       level,
+            format:      format.to_sym,
+            async:       async,
+            trace:       options.fetch(:trace, true),
+            trace_size:  options.fetch(:trace_size, 4),
+            extended:    options.fetch(:extended, true),
+            log_file:    options[:log_file],
+            log_stdout:  options[:log_stdout],
+            include_pid: options.fetch(:include_pid, false),
+            color:       @color
+          }.freeze
+          @configuration_generation = configuration_generation + 1
+          Legion::Logging::Redactor.refresh_patterns! if defined?(Legion::Logging::Redactor)
+          if async
+            start_async_writer(buffer_size: Legion::Logging::Settings.async_buffer_size)
+          else
+            @async_writer = nil
+            @async = false
+            log.async_writer = nil
+            true
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-logging
 version: !ruby/object:Gem::Version
-  version: 1.5.6
+  version: 1.5.19
 platform: ruby
 authors:
 - Esity
@@ -70,6 +70,7 @@ files:
 - lib/legion/logging/method_tracer.rb
 - lib/legion/logging/methods.rb
 - lib/legion/logging/multi_io.rb
+- lib/legion/logging/output_logger.rb
 - lib/legion/logging/redactor.rb
 - lib/legion/logging/settings.rb
 - lib/legion/logging/shipper.rb