legion-logging 1.5.5 → 1.5.19

data/lib/legion/logging/shipper/http_transport.rb

@@ -24,7 +24,7 @@ module Legion
             response = post(uri, body)
             response.is_a?(Net::HTTPSuccess)
           rescue StandardError => e
-            Legion::Logging.error("HttpTransport ship failed: #{e.message}") if defined?(Legion::Logging)
+            Kernel.warn("HttpTransport ship failed: #{e.message}")
             false
           end
 
@@ -36,7 +36,7 @@ module Legion
             apply_auth(req, uri)
 
             Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
-                                                     open_timeout: 5, read_timeout: 10) do |http|
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
               http.request(req, body)
             end
           end
@@ -67,12 +67,14 @@ module Legion
 
           def auth_token
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :auth_token)
           end
 
           def resolve_endpoint
             return nil unless defined?(Legion::Settings)
+            return nil unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
             Legion::Settings.dig(:logging, :shipper, :endpoint)
           end

data/lib/legion/logging/shipper.rb

@@ -8,128 +8,183 @@ module Legion
   module Logging
     module Shipper
       LEVEL_ORDER = %w[debug info warn error fatal].freeze
-
-      TRANSPORTS = {
-        file: FileTransport,
-        http: HttpTransport
-      }.freeze
+      TRANSPORTS = { file: FileTransport, http: HttpTransport }.freeze
+      MUTEX = Mutex.new
+      WAKE = ConditionVariable.new
+      CHANGED = ConditionVariable.new
+      private_constant :MUTEX, :WAKE, :CHANGED
+
+      @buffer = []
+      @accepted = 0
+      @delivered = 0
+      @attempt = 0
+      @flush_target = 0
 
       class << self
         def ship(event)
           return unless enabled?
           return unless shippable_level?(event[:level] || event['level'])
+          return unless TRANSPORTS.key?(transport_type)
+
+          redacted = Redactor.redact(event)
+          raise ClosedQueueError, 'log shipper is stopping' unless start
+
+          MUTEX.synchronize do
+            raise ThreadError, 'log shipper cannot enqueue from its delivery thread' if Thread.current.equal?(@flush_thread)
+
+            CHANGED.wait(MUTEX) while !@stopping && @accepted - @delivered >= @capacity
+            raise ClosedQueueError, 'log shipper is stopping' if @stopping
 
-          redacted  = Redactor.redact(event)
-          transport = TRANSPORTS[transport_type]
-          buffer_event(redacted) if transport
+            @buffer << redacted
+            @accepted += 1
+            @flush_requested = true if @buffer.size >= @batch_size
+            WAKE.signal if @flush_requested
+          end
+          true
         end
 
-        def flush
-          return if @buffer.nil? || @buffer.empty?
+        def flush(timeout: 5)
+          return true if MUTEX.synchronize { @accepted == @delivered }
+          return false unless start
+
+          deadline = timeout && (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout)
+          MUTEX.synchronize do
+            return false if Thread.current.equal?(@flush_thread)
 
-          transport = TRANSPORTS[transport_type]
-          return unless transport
+            target = @accepted
+            attempt = @attempt
+            @flush_target = [@flush_target, target].max
+            @flush_requested = true
+            WAKE.signal
+            while @delivered < target
+              return false if @attempt > attempt && !@last_success
+              return false unless @flush_thread&.alive?
 
-          @flush_mutex ||= Mutex.new
-          @flush_mutex.synchronize do
-            batch = nil
-            @mutex.synchronize { batch = @buffer.dup }
-            return if batch.empty?
+              remaining = deadline && (deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC))
+              return false if remaining && remaining <= 0
 
-            delivered = deliver(transport, batch)
-            @mutex.synchronize { @buffer.shift(batch.size) if delivered }
-            delivered
+              CHANGED.wait(MUTEX, remaining)
+            end
+            true
           end
         end
 
         def start
-          return unless enabled?
-          return if @flush_thread&.alive?
-
-          @buffer ||= []
-          @mutex  ||= Mutex.new
-          @flush_mutex ||= Mutex.new
-          interval = flush_interval
-          @flush_thread = Thread.new do
-            loop do
-              sleep interval
-              flush
+          return false unless enabled?
+
+          MUTEX.synchronize do
+            return !@stopping if @flush_thread&.alive?
+
+            @capacity = Legion::Settings.dig(:logging, :shipper, :buffer_size) || 10_000
+            @batch_size = [batch_size, @capacity].min
+            @interval = flush_interval
+            unless @capacity.is_a?(Integer) && @capacity.positive? && @batch_size.is_a?(Integer) && @batch_size.positive? &&
+                   @interval.is_a?(Numeric) && @interval.positive?
+              raise ArgumentError, 'shipper buffer_size, batch_size, and flush_interval must be positive'
             end
+
+            @transport = TRANSPORTS[transport_type]
+            return false unless @transport
+
+            @stopping = false
+            @flush_requested = @buffer.size >= @batch_size
+            @flush_thread = Thread.new { consume }
+            @flush_thread.name = 'legion-log-shipper'
+            @flush_thread.abort_on_exception = false
+            true
           end
-          @flush_thread.abort_on_exception = false
         end
 
-        def stop
-          @flush_thread&.kill
-          @flush_thread = nil
-          flush
+        def stop(timeout: 5)
+          deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout if timeout
+          thread = MUTEX.synchronize do
+            @stopping = true
+            WAKE.broadcast
+            CHANGED.broadcast
+            if !@flush_thread&.alive? && @accepted > @delivered
+              @flush_thread = Thread.new { consume }
+              @flush_thread.name = 'legion-log-shipper'
+              @flush_thread.abort_on_exception = false
+            end
+            @flush_thread
+          end
+          return MUTEX.synchronize { @accepted == @delivered } unless thread
+          return false if Thread.current.equal?(thread)
+
+          remaining = [deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC), 0].max if deadline
+          thread.join(remaining)
+          MUTEX.synchronize { !thread.alive? && @accepted == @delivered }
         end
 
         def enabled?
           return false unless defined?(Legion::Settings)
+          return false unless Legion::Settings.respond_to?(:loaded?) ? Legion::Settings.loaded? : true
 
           Legion::Settings.dig(:logging, :shipper, :enabled) == true
         end
 
         private
 
-        def buffer_event(event)
-          @buffer  ||= []
-          @mutex   ||= Mutex.new
-
-          full = false
-          @mutex.synchronize do
-            @buffer << event
-            full = @buffer.size >= batch_size
+        def consume
+          loop do
+            batch = MUTEX.synchronize do
+              WAKE.wait(MUTEX, @interval) unless @flush_requested || @stopping
+              @flush_requested = false
+              @buffer.shift(@batch_size)
+            end
+            delivered = batch.empty? || deliver(@transport, batch)
+            stopping = MUTEX.synchronize do
+              @attempt += 1
+              @last_success = delivered
+              if delivered
+                @delivered += batch.size
+                @flush_requested = @buffer.size >= @batch_size || @delivered < @flush_target
+              else
+                @buffer.prepend(*batch)
+              end
+              CHANGED.broadcast
+              @stopping && (!delivered || @buffer.empty?)
+            end
+            break if stopping
+          end
+        ensure
+          MUTEX.synchronize do
+            @stopping = true
+            CHANGED.broadcast
           end
-
-          flush if full
         end
 
         def deliver(transport, batch)
-          if transport.respond_to?(:ship_batch)
-            transport.ship_batch(batch)
-          else
-            batch.all? { |event| transport.ship(event) }
-          end
+          transport.ship_batch(batch)
         rescue StandardError => e
-          Legion::Logging.error("Shipper deliver failed: #{e.message}") if defined?(Legion::Logging)
+          Kernel.warn("Shipper deliver failed: #{e.message}")
           false
         end
 
         def shippable_level?(level)
           return true if level.nil?
 
-          min = minimum_level
-          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(min).to_i
+          LEVEL_ORDER.index(level.to_s.downcase).to_i >= LEVEL_ORDER.index(minimum_level).to_i
         end
 
         def transport_type
-          return :file unless defined?(Legion::Settings)
-
-          key = Legion::Settings.dig(:logging, :shipper, :transport)
+          key = Legion::Settings.dig(:logging, :shipper, :transport) if defined?(Legion::Settings)
           key ? key.to_sym : :file
         end
 
         def batch_size
-          return 100 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :batch_size) || 100
         end
 
         def flush_interval
-          return 5 unless defined?(Legion::Settings)
-
           Legion::Settings.dig(:logging, :shipper, :flush_interval) || 5
         end
 
         def minimum_level
-          return 'warn' unless defined?(Legion::Settings)
-
           levels = Legion::Settings.dig(:logging, :shipper, :levels)
           return 'warn' unless levels.is_a?(Array) && !levels.empty?
 
-          levels.min_by { |l| LEVEL_ORDER.index(l.to_s) || 99 }.to_s
+          levels.min_by { |level| LEVEL_ORDER.index(level.to_s) || 99 }.to_s
         end
       end
     end

data/lib/legion/logging/siem_exporter.rb

@@ -28,7 +28,8 @@ module Legion
           req['Content-Type'] = 'application/json'
           req.body = ::JSON.dump({ event: redact_phi(event.to_s), time: Time.now.to_f })
 
-          Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https',
+                                                     open_timeout: 5, read_timeout: 10, write_timeout: 10) do |http|
             http.request(req)
           end
         rescue StandardError => e

data/lib/legion/logging/tagged_logger.rb

@@ -32,44 +32,44 @@ module Legion
         @level_value
       end
 
-      def debug(message = nil)
+      def debug(message = nil, **ctx)
         return unless @level_value < 1
 
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:debug, message) }
+        with_segments { dispatch(:debug, message, **ctx) }
       end
 
-      def info(message = nil)
+      def info(message = nil, **ctx)
         return unless @level_value < 2
 
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:info, message) }
+        with_segments { dispatch(:info, message, **ctx) }
       end
 
-      def warn(message = nil)
+      def warn(message = nil, **ctx)
         return unless @level_value < 3
 
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:warn, message) }
+        with_segments { dispatch(:warn, message, **ctx) }
       end
 
-      def error(message = nil)
+      def error(message = nil, **ctx)
         return unless @level_value < 4
 
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:error, message) }
+        with_segments { dispatch(:error, message, **ctx) }
       end
 
-      def fatal(message = nil)
+      def fatal(message = nil, **ctx)
         return unless @level_value < 5
 
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:fatal, message) }
+        with_segments { dispatch(:fatal, message, **ctx) }
       end
 
-      def unknown(message = nil)
+      def unknown(message = nil, **ctx)
         message = yield if message.nil? && block_given?
-        with_segments { dispatch(:unknown, message) }
+        with_segments { dispatch(:unknown, message, **ctx) }
       end
 
       def trace(raw_message = nil, size: @trace_size, log_caller: true)
@@ -94,21 +94,23 @@ module Legion
 
       private
 
-      def dispatch(level, message)
+      def dispatch(level, message, **ctx)
         return unless defined?(Legion::Logging)
 
         if Legion::Logging.respond_to?(:emit_tagged)
-          Legion::Logging.emit_tagged(level, message, segments: @segments)
+          Legion::Logging.emit_tagged(level, message, segments: @segments, **ctx)
           return
         end
 
         if Legion::Logging.respond_to?(level)
-          Legion::Logging.public_send(level, message)
+          Legion::Logging.public_send(level, message, **ctx)
           return
         end
 
         fallback = fallback_level(level)
-        Legion::Logging.public_send(fallback, message) if fallback && Legion::Logging.respond_to?(fallback)
+        return unless fallback && Legion::Logging.respond_to?(fallback)
+
+        Legion::Logging.public_send(fallback, message, **ctx)
       end
 
       def fallback_level(level)

data/lib/legion/logging/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Logging
-    VERSION = '1.5.5'
+    VERSION = '1.5.19'
   end
 end

data/lib/legion/logging.rb

@@ -76,36 +76,37 @@ module Legion
         Hooks.clear_hooks!
       end
 
-      def setup(level: 'debug', format: :text, async: true, **options)
-        output(**options)
-        log_level(level)
-        log_format(format: format, **options)
-        @color = options[:color]
-        @color = format != :json && (options[:color] || (options[:color].nil? && options[:log_file].nil?))
-        @current_settings = {
-          level:       level,
-          format:      format.to_sym,
-          async:       async,
-          trace:       options.fetch(:trace, true),
-          trace_size:  options.fetch(:trace_size, 4),
-          extended:    options.fetch(:extended, true),
-          log_file:    options[:log_file],
-          log_stdout:  options[:log_stdout],
-          include_pid: options.fetch(:include_pid, false),
-          color:       @color
-        }.freeze
-        @configuration_generation = configuration_generation + 1
-        Legion::Logging::Redactor.refresh_patterns! if defined?(Legion::Logging::Redactor)
-        if async
-          buffer = if defined?(Legion::Settings) && Legion::Settings.respond_to?(:[])
-                     logging_settings = Legion::Settings[:logging]
-                     async_settings = logging_settings[:async] if logging_settings.is_a?(Hash)
-                     async_settings[:buffer_size] if async_settings.is_a?(Hash)
-                   end
-          buffer ||= 10_000
-          start_async_writer(buffer_size: buffer)
-        else
-          stop_async_writer
+      def setup(level: 'debug', format: :text, async: true, shutdown_timeout: 2, **options)
+        with_lifecycle_lock do
+          return false if @async_writer && stop_async_writer(timeout: shutdown_timeout) == false
+          return false if output(**options) == false
+
+          log_level(level)
+          log_format(format: format, **options)
+          @color = options[:color]
+          @color = format != :json && (options[:color] || (options[:color].nil? && options[:log_file].nil?))
+          @current_settings = {
+            level:       level,
+            format:      format.to_sym,
+            async:       async,
+            trace:       options.fetch(:trace, true),
+            trace_size:  options.fetch(:trace_size, 4),
+            extended:    options.fetch(:extended, true),
+            log_file:    options[:log_file],
+            log_stdout:  options[:log_stdout],
+            include_pid: options.fetch(:include_pid, false),
+            color:       @color
+          }.freeze
+          @configuration_generation = configuration_generation + 1
+          Legion::Logging::Redactor.refresh_patterns! if defined?(Legion::Logging::Redactor)
+          if async
+            start_async_writer(buffer_size: Legion::Logging::Settings.async_buffer_size)
+          else
+            @async_writer = nil
+            @async = false
+            log.async_writer = nil
+            true
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-logging
 version: !ruby/object:Gem::Version
-  version: 1.5.5
+  version: 1.5.19
 platform: ruby
 authors:
 - Esity
@@ -63,12 +63,14 @@ files:
 - lib/legion/logging/builder.rb
 - lib/legion/logging/category_registry.rb
 - lib/legion/logging/event_builder.rb
+- lib/legion/logging/header_builder.rb
 - lib/legion/logging/helper.rb
 - lib/legion/logging/hooks.rb
 - lib/legion/logging/logger.rb
 - lib/legion/logging/method_tracer.rb
 - lib/legion/logging/methods.rb
 - lib/legion/logging/multi_io.rb
+- lib/legion/logging/output_logger.rb
 - lib/legion/logging/redactor.rb
 - lib/legion/logging/settings.rb
 - lib/legion/logging/shipper.rb