legion-logging 1.5.6 → 1.5.21

data/lib/legion/logging/rotation.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Legion
+  module Logging
+    # Settings and descriptor detection shared by the byte-capped file and stdio sinks.
+    module Rotation
+      DEFAULTS = {
+        enabled:     true,
+        max_size_mb: 100,
+        max_files:   5,
+        stdio:       true
+      }.freeze
+
+      F_GETPATH = 50 # macOS fcntl(2) command; not exposed by the Fcntl module
+      MAXPATHLEN = 1024
+
+      class << self
+        # Normalizes rotation settings. Accepts nil, false (disabled), or a Hash
+        # with symbol or string keys. Missing keys fall back to DEFAULTS.
+        def resolve(raw:)
+          overrides = if raw == false
+                        { enabled: false }
+                      elsif raw.is_a?(Hash)
+                        raw.transform_keys(&:to_sym)
+                      else
+                        {}
+                      end
+          merged = DEFAULTS.merge(overrides.slice(*DEFAULTS.keys))
+          merged[:enabled] = merged[:enabled] != false
+          merged[:stdio] = merged[:stdio] != false
+          merged[:max_size_mb] = positive_number(merged[:max_size_mb], DEFAULTS[:max_size_mb])
+          merged[:max_files] = positive_number(merged[:max_files], DEFAULTS[:max_files]).to_i
+          merged[:max_files] = DEFAULTS[:max_files] unless merged[:max_files].positive?
+          bytes = merged[:max_size_mb] * 1024 * 1024
+          unless bytes.finite?
+            merged[:max_size_mb] = DEFAULTS[:max_size_mb]
+            bytes = merged[:max_size_mb] * 1024 * 1024
+          end
+          merged[:max_bytes] = [bytes.to_i, 1].max
+          merged
+        end
+
+        # Reads rotation settings from Legion::Settings once it has loaded.
+        # Guarded against re-entry because Settings may log while loading.
+        def from_settings
+          return nil unless defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
+          return nil if Thread.current[:legion_logging_rotation_lookup]
+
+          lookup_started = true
+          Thread.current[:legion_logging_rotation_lookup] = true
+          return nil if Legion::Settings.respond_to?(:loaded?) && !Legion::Settings.loaded?
+
+          logging = Legion::Settings[:logging]
+          logging[:rotation] if logging.is_a?(Hash)
+        rescue StandardError => e
+          warn("Legion::Logging::Rotation.from_settings failed: #{e.class}: #{e.message}")
+          nil
+        ensure
+          Thread.current[:legion_logging_rotation_lookup] = nil if lookup_started
+        end
+
+        # Shifts path.(n-1) -> path.n ... path -> path.1, dropping the oldest.
+        def shift_files(path:, max_files:)
+          oldest = "#{path}.#{max_files}"
+          FileUtils.rm(oldest, force: false) if File.exist?(oldest)
+          (max_files - 1).downto(1) do |i|
+            src = "#{path}.#{i}"
+            File.rename(src, "#{path}.#{i + 1}") if File.exist?(src)
+          end
+          File.rename(path, "#{path}.1") if File.exist?(path)
+        end
+
+        # Returns the filesystem path behind an IO if it is a regular file opened
+        # in append mode (launchd StandardOutPath, systemd append:, shell >>), or nil.
+        # Non-append redirects (shell >) are left alone.
+        def regular_file_path(io:)
+          return nil unless io.respond_to?(:stat) && io.stat.file?
+          return nil unless append_mode?(io: io)
+
+          path = fd_path(io)
+          return nil unless path && File.file?(path)
+          return nil unless same_file?(io: io, path: path)
+
+          path
+        rescue StandardError => e
+          warn("Legion::Logging::Rotation.regular_file_path failed: #{e.class}: #{e.message}")
+          nil
+        end
+
+        def same_file?(io:, path:)
+          return false unless File.exist?(path)
+
+          io_stat = io.stat
+          path_stat = File.stat(path)
+          io_stat.ino == path_stat.ino && io_stat.dev == path_stat.dev
+        rescue SystemCallError, IOError => e
+          warn("Legion::Logging::Rotation.same_file? path=#{path} failed: #{e.class}: #{e.message}")
+          false
+        end
+
+        def append_mode?(io:)
+          require 'fcntl'
+          io.fcntl(Fcntl::F_GETFL).anybits?(File::APPEND)
+        rescue NotImplementedError, SystemCallError, IOError => e
+          warn("Legion::Logging::Rotation.append_mode? failed: #{e.class}: #{e.message}")
+          false
+        end
+
+        private
+
+        def fd_path(io)
+          if io.respond_to?(:path)
+            path = File.expand_path(io.path)
+            return path if same_file?(io: io, path: path)
+          end
+
+          if RUBY_PLATFORM.include?('darwin')
+            require 'fcntl'
+            buf = "\0" * MAXPATHLEN
+            io.fcntl(F_GETPATH, buf)
+            buf.unpack1('Z*')
+          else
+            proc_link = "/proc/self/fd/#{io.fileno}"
+            File.readlink(proc_link) if File.symlink?(proc_link)
+          end
+        end
+
+        def positive_number(value, default)
+          number = Float(value)
+          number.positive? && number.finite? ? number : default
+        rescue ArgumentError, TypeError => e
+          warn("Legion::Logging::Rotation invalid numeric limit: #{e.class}: #{e.message}")
+          default
+        end
+      end
+    end
+  end
+end
+
+require_relative 'rotating_file'
+require_relative 'stdio_rotator'

data/lib/legion/logging/settings.rb

@@ -3,15 +3,37 @@
 module Legion
   module Logging
     module Settings
+      # Log levels forwarded to the transport bus (and in-process Hooks) by
+      # default. Preserves historic behavior: only warn/error/fatal logs build a
+      # structured event. Override at runtime via logging.transport.log_levels
+      # (e.g. add :debug/:info) — see Methods#forwardable_log_levels.
+      DEFAULT_FORWARD_LOG_LEVELS = %i[warn error fatal].freeze
+
       def self.default
         {
-          level:           :info,
-          trace:           true,
-          trace_size:      4,
-          extended:        true,
-          backtrace_limit: nil
+          level:              :info,
+          trace:              true,
+          trace_size:         4,
+          extended:           true,
+          backtrace_limit:    nil,
+          forward_log_levels: DEFAULT_FORWARD_LOG_LEVELS,
+          rotation:           {
+            enabled:     true,
+            max_size_mb: 100,
+            max_files:   5,
+            stdio:       true
+          }
         }
       end
+
+      def self.async_buffer_size
+        return 10_000 unless defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
+        return 10_000 unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
+
+        logging = Legion::Settings[:logging]
+        async = logging[:async] if logging.is_a?(Hash)
+        async.is_a?(Hash) ? (async[:buffer_size] || 10_000) : 10_000
+      end
     end
   end
 end

data/lib/legion/logging/shipper/file_transport.rb

@@ -26,7 +26,7 @@ module Legion
             end
             true
           rescue StandardError => e
-            Legion::Logging.error("FileTransport ship failed: #{e.message}") if defined?(Legion::Logging)
+            Kernel.warn("FileTransport ship failed: #{e.message}")
             false
           end
 
@@ -40,6 +40,7 @@ module Legion
 
           def settings_path
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :file, :path)
           end

data/lib/legion/logging/shipper/http_transport.rb

@@ -24,7 +24,7 @@ module Legion
             response = post(uri, body)
             response.is_a?(Net::HTTPSuccess)
           rescue StandardError => e
-            Legion::Logging.error("HttpTransport ship failed: #{e.message}") if defined?(Legion::Logging)
+            Kernel.warn("HttpTransport ship failed: #{e.message}")
             false
           end
 
@@ -36,7 +36,7 @@ module Legion
             apply_auth(req, uri)
 
             Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
-                                                     open_timeout: 5, read_timeout: 10) do |http|
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
               http.request(req, body)
             end
           end
@@ -67,12 +67,14 @@ module Legion
 
           def auth_token
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :auth_token)
           end
 
           def resolve_endpoint
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :endpoint)
           end

data/lib/legion/logging/shipper.rb

@@ -8,134 +8,183 @@ module Legion
   module Logging
     module Shipper
       LEVEL_ORDER = %w[debug info warn error fatal].freeze
-
-      TRANSPORTS = {
-        file: FileTransport,
-        http: HttpTransport
-      }.freeze
+      TRANSPORTS = { file: FileTransport, http: HttpTransport }.freeze
+      MUTEX = Mutex.new
+      WAKE = ConditionVariable.new
+      CHANGED = ConditionVariable.new
+      private_constant :MUTEX, :WAKE, :CHANGED
+
+      @buffer = []
+      @accepted = 0
+      @delivered = 0
+      @attempt = 0
+      @flush_target = 0
 
       class << self
         def ship(event)
           return unless enabled?
           return unless shippable_level?(event[:level] || event['level'])
+          return unless TRANSPORTS.key?(transport_type)
 
-          redacted  = Redactor.redact(event)
-          transport = TRANSPORTS[transport_type]
-          buffer_event(redacted) if transport
-        end
+          redacted = Redactor.redact(event)
+          raise ClosedQueueError, 'log shipper is stopping' unless start
 
-        def flush
-          @mutex ||= Mutex.new
-          batch = @mutex.synchronize do
-            return true if @buffer.nil? || @buffer.empty?
+          MUTEX.synchronize do
+            raise ThreadError, 'log shipper cannot enqueue from its delivery thread' if Thread.current.equal?(@flush_thread)
 
-            flushed = @buffer.dup
-            @buffer.clear
-            flushed
+            CHANGED.wait(MUTEX) while !@stopping && @accepted - @delivered >= @capacity
+            raise ClosedQueueError, 'log shipper is stopping' if @stopping
+
+            @buffer << redacted
+            @accepted += 1
+            @flush_requested = true if @buffer.size >= @batch_size
+            WAKE.signal if @flush_requested
           end
+          true
+        end
+
+        def flush(timeout: 5)
+          return true if MUTEX.synchronize { @accepted == @delivered }
+          return false unless start
+
+          deadline = timeout && (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout)
+          MUTEX.synchronize do
+            return false if Thread.current.equal?(@flush_thread)
+
+            target = @accepted
+            attempt = @attempt
+            @flush_target = [@flush_target, target].max
+            @flush_requested = true
+            WAKE.signal
+            while @delivered < target
+              return false if @attempt > attempt && !@last_success
+              return false unless @flush_thread&.alive?
 
-          transport = TRANSPORTS[transport_type]
-          return true unless transport
+              remaining = deadline && (deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC))
+              return false if remaining && remaining <= 0
 
-          delivered = deliver(transport, batch)
-          @mutex.synchronize { @buffer.prepend(*batch) } unless delivered
-          delivered
+              CHANGED.wait(MUTEX, remaining)
+            end
+            true
+          end
         end
 
         def start
-          @start_mutex ||= Mutex.new
-          @start_mutex.synchronize do
-            return unless enabled?
-            return if @flush_thread&.alive?
-
-            @buffer ||= []
-            @mutex  ||= Mutex.new
-            @running = true
-            interval = flush_interval
-            @flush_thread = Thread.new do
-              while @running
-                sleep interval
-                flush
-              end
+          return false unless enabled?
+
+          MUTEX.synchronize do
+            return !@stopping if @flush_thread&.alive?
+
+            @capacity = Legion::Settings.dig(:logging, :shipper, :buffer_size) || 10_000
+            @batch_size = [batch_size, @capacity].min
+            @interval = flush_interval
+            unless @capacity.is_a?(Integer) && @capacity.positive? && @batch_size.is_a?(Integer) && @batch_size.positive? &&
+                   @interval.is_a?(Numeric) && @interval.positive?
+              raise ArgumentError, 'shipper buffer_size, batch_size, and flush_interval must be positive'
             end
+
+            @transport = TRANSPORTS[transport_type]
+            return false unless @transport
+
+            @stopping = false
+            @flush_requested = @buffer.size >= @batch_size
+            @flush_thread = Thread.new { consume }
             @flush_thread.name = 'legion-log-shipper'
             @flush_thread.abort_on_exception = false
+            true
           end
         end
 
-        def stop
-          @running = false
-          thread = @flush_thread
-          @flush_thread = nil
-          thread&.join(5)
-          flush
+        def stop(timeout: 5)
+          deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout if timeout
+          thread = MUTEX.synchronize do
+            @stopping = true
+            WAKE.broadcast
+            CHANGED.broadcast
+            if !@flush_thread&.alive? && @accepted > @delivered
+              @flush_thread = Thread.new { consume }
+              @flush_thread.name = 'legion-log-shipper'
+              @flush_thread.abort_on_exception = false
+            end
+            @flush_thread
+          end
+          return MUTEX.synchronize { @accepted == @delivered } unless thread
+          return false if Thread.current.equal?(thread)
+
+          remaining = [deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC), 0].max if deadline
+          thread.join(remaining)
+          MUTEX.synchronize { !thread.alive? && @accepted == @delivered }
         end
 
         def enabled?
           return false unless defined?(Legion::Settings)
+          return false unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
           Legion::Settings.dig(:logging, :shipper, :enabled) == true
         end
 
         private
 
-        def buffer_event(event)
-          @buffer ||= []
-          @mutex  ||= Mutex.new
-
-          full = false
-          @mutex.synchronize do
-            @buffer << event
-            full = @buffer.size >= batch_size
+        def consume
+          loop do
+            batch = MUTEX.synchronize do
+              WAKE.wait(MUTEX, @interval) unless @flush_requested || @stopping
+              @flush_requested = false
+              @buffer.shift(@batch_size)
+            end
+            delivered = batch.empty? || deliver(@transport, batch)
+            stopping = MUTEX.synchronize do
+              @attempt += 1
+              @last_success = delivered
+              if delivered
+                @delivered += batch.size
+                @flush_requested = @buffer.size >= @batch_size || @delivered < @flush_target
+              else
+                @buffer.prepend(*batch)
+              end
+              CHANGED.broadcast
+              @stopping && (!delivered || @buffer.empty?)
+            end
+            break if stopping
+          end
+        ensure
+          MUTEX.synchronize do
+            @stopping = true
+            CHANGED.broadcast
           end
-
-          flush if full
         end
 
         def deliver(transport, batch)
-          if transport.respond_to?(:ship_batch)
-            transport.ship_batch(batch)
-          else
-            batch.all? { |event| transport.ship(event) }
-          end
+          transport.ship_batch(batch)
         rescue StandardError => e
-          Legion::Logging.error("Shipper deliver failed: #{e.message}") if defined?(Legion::Logging)
+          Kernel.warn("Shipper deliver failed: #{e.message}")
           false
         end
 
         def shippable_level?(level)
           return true if level.nil?
 
-          min = minimum_level
-          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(min).to_i
+          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(minimum_level).to_i
         end
 
         def transport_type
-          return :file unless defined?(Legion::Settings)
-
-          key = Legion::Settings.dig(:logging, :shipper, :transport)
+          key = Legion::Settings.dig(:logging, :shipper, :transport) if defined?(Legion::Settings)
           key ? key.to_sym : :file
         end
 
         def batch_size
-          return 100 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :batch_size) || 100
         end
 
         def flush_interval
-          return 5 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :flush_interval) || 5
         end
 
         def minimum_level
-          return 'warn' unless defined?(Legion::Settings)
-
           levels = Legion::Settings.dig(:logging, :shipper, :levels)
           return 'warn' unless levels.is_a?(Array) && !levels.empty?
 
-          levels.min_by { |l| LEVEL_ORDER.index(l.to_s) || 99 }.to_s
+          levels.min_by { |level| LEVEL_ORDER.index(level.to_s) || 99 }.to_s
         end
       end
     end

data/lib/legion/logging/siem_exporter.rb

@@ -29,7 +29,7 @@ module Legion
           req.body = ::JSON.dump({ event: redact_phi(event.to_s), time: Time.now.to_f })
 
           Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
-                                                     open_timeout: 5, read_timeout: 10) do |http|
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
             http.request(req)
           end
         rescue StandardError => e