onlylogs 0.9.0 → 0.10.0

data/lib/onlylogs/http_device.rb

@@ -16,12 +16,46 @@ require_relative "spool"
 # By default an on-disk Spool buffers any batch we could not deliver and replays it once the
 # drain recovers, so a transient outage or a restart does not lose logs. It is on by default
 # (set ONLYLOGS_SPOOL_DIR empty to disable) and bounded by bytes; see Onlylogs::Spool.
+#
+# Every write checks that a sender thread is alive in the current process and starts one if not:
+# * The device is usually built in the Puma master (production.rb runs before the workers are
+#   forked with preload_app!) and inherited by every worker. Threads do not survive a fork, so
+#   the child would have a queue nobody drains, a keep-alive socket shared with its siblings and a
+#   spool token that makes siblings overwrite each other's batches. The first write in a new
+#   process rebuilds all of that for the child.
+# * The sender must never die, so its error path never raises (see #safe_warn) and every loop
+#   iteration is rescued; should it die anyway, the next write restarts it.
+#
+# Not every failure is worth retrying. The drain's answer decides what happens to a batch:
+# * 2xx: delivered.
+# * 429: the drain is overloaded and asks us to slow down. Pause for Retry-After (or a cooldown)
+#   and keep the batch on disk; nothing is lost, it is just late.
+# * other 4xx: the drain will never accept this batch (unknown token, paused project, body too
+#   big). Retrying cannot help, so the batch is dropped and a warning says why.
+# * 5xx, timeouts, connection errors: retryable. Count towards opening the circuit and spool.
 module Onlylogs
   class HttpDevice
+    # The drain answered with a 4xx other than 429: the batch itself is the problem, not the drain.
+    class Rejected < StandardError; end
+
+    # The drain answered 429: it is up but wants us to back off.
+    class Throttled < StandardError
+      attr_reader :retry_after
+
+      def initialize(message, retry_after: nil)
+        super(message)
+        @retry_after = retry_after
+      end
+    end
+
     DEFAULT_BATCH_SIZE = 100
     DEFAULT_FLUSH_INTERVAL = 0.5
     DEFAULT_MAX_QUEUE_SIZE = 10_000
 
+    # A batch body never exceeds this many bytes, and neither does a single line: a drain cannot
+    # answer 413 to a request we never make. Lines over the cap are cut and marked.
+    DEFAULT_MAX_BATCH_BYTES = 1024 * 1024
+
     # Keep timeouts short: a single slow/dead drain must never stall the app for long.
     DEFAULT_OPEN_TIMEOUT = 0.5
     DEFAULT_READ_TIMEOUT = 0.5
@@ -32,45 +66,51 @@ module Onlylogs
 
     # Open the circuit after this many consecutive failed sends
     CIRCUIT_FAILURE_THRESHOLD = 3
-    # ...and keep it open for this long once it is open.
+    # ...and keep it open for about this long once it is open. The actual pause is jittered
+    # between 0.5x and 1.5x so that every client of a drain that just came back does not retry in
+    # the same second.
     CIRCUIT_COOLDOWN = 30
 
+    # Every setting is validated up front: a typo in an env var must never leave the sender without
+    # timeouts (Net::HTTP treats 0 as "no timeout") or unable to truncate a line. Invalid numbers
+    # fall back to the default with a warning; a drain URL that is not http(s) or has no host falls
+    # back to local-only logging. A misconfiguration is never a boot failure.
     def initialize(
       drain_url: ENV["ONLYLOGS_DRAIN_URL"],
-      batch_size: ENV.fetch("ONLYLOGS_BATCH_SIZE", DEFAULT_BATCH_SIZE).to_i,
-      flush_interval: ENV.fetch("ONLYLOGS_FLUSH_INTERVAL", DEFAULT_FLUSH_INTERVAL).to_f,
-      max_queue_size: ENV.fetch("ONLYLOGS_MAX_QUEUE_SIZE", DEFAULT_MAX_QUEUE_SIZE).to_i,
-      open_timeout: ENV.fetch("ONLYLOGS_OPEN_TIMEOUT", DEFAULT_OPEN_TIMEOUT).to_f,
-      read_timeout: ENV.fetch("ONLYLOGS_READ_TIMEOUT", DEFAULT_READ_TIMEOUT).to_f,
-      circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN).to_f,
-      keep_alive_timeout: ENV.fetch("ONLYLOGS_KEEP_ALIVE_TIMEOUT", DEFAULT_KEEP_ALIVE_TIMEOUT).to_f,
+      batch_size: ENV.fetch("ONLYLOGS_BATCH_SIZE", DEFAULT_BATCH_SIZE),
+      flush_interval: ENV.fetch("ONLYLOGS_FLUSH_INTERVAL", DEFAULT_FLUSH_INTERVAL),
+      max_queue_size: ENV.fetch("ONLYLOGS_MAX_QUEUE_SIZE", DEFAULT_MAX_QUEUE_SIZE),
+      max_batch_bytes: ENV.fetch("ONLYLOGS_MAX_BATCH_BYTES", DEFAULT_MAX_BATCH_BYTES),
+      open_timeout: ENV.fetch("ONLYLOGS_OPEN_TIMEOUT", DEFAULT_OPEN_TIMEOUT),
+      read_timeout: ENV.fetch("ONLYLOGS_READ_TIMEOUT", DEFAULT_READ_TIMEOUT),
+      circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN),
+      keep_alive_timeout: ENV.fetch("ONLYLOGS_KEEP_ALIVE_TIMEOUT", DEFAULT_KEEP_ALIVE_TIMEOUT),
       spool_dir: ENV.fetch("ONLYLOGS_SPOOL_DIR", default_spool_dir),
-      spool_max_bytes: ENV.fetch("ONLYLOGS_SPOOL_MAX_BYTES", Spool::DEFAULT_MAX_BYTES).to_i
+      spool_max_bytes: ENV.fetch("ONLYLOGS_SPOOL_MAX_BYTES", Spool::DEFAULT_MAX_BYTES)
     )
-      @drain_url = drain_url
-      @uri = URI.parse(drain_url) if drain_url
-      @batch_size = batch_size
-      @flush_interval = flush_interval
-      @max_queue_size = max_queue_size
-      @open_timeout = open_timeout
-      @read_timeout = read_timeout
-      @circuit_cooldown = circuit_cooldown
-      @keep_alive_timeout = keep_alive_timeout
-      @queue = Queue.new
-      @mutex = Mutex.new
-      @http_mutex = Mutex.new
-      @http = nil
-      @spool = nil
-
-      @consecutive_failures = 0
-      @circuit_open_until = nil
-      @dropped = 0
+      @uri = parse_drain_url(drain_url)
+      @drain_url = drain_url if @uri
+      @batch_size = integer_setting("ONLYLOGS_BATCH_SIZE", batch_size, DEFAULT_BATCH_SIZE)
+      @flush_interval = float_setting("ONLYLOGS_FLUSH_INTERVAL", flush_interval, DEFAULT_FLUSH_INTERVAL)
+      @max_queue_size = integer_setting("ONLYLOGS_MAX_QUEUE_SIZE", max_queue_size, DEFAULT_MAX_QUEUE_SIZE)
+      @max_batch_bytes = integer_setting("ONLYLOGS_MAX_BATCH_BYTES", max_batch_bytes, DEFAULT_MAX_BATCH_BYTES,
+        min: MIN_BATCH_BYTES)
+      @open_timeout = float_setting("ONLYLOGS_OPEN_TIMEOUT", open_timeout, DEFAULT_OPEN_TIMEOUT)
+      @read_timeout = float_setting("ONLYLOGS_READ_TIMEOUT", read_timeout, DEFAULT_READ_TIMEOUT)
+      @circuit_cooldown = float_setting("ONLYLOGS_CIRCUIT_COOLDOWN", circuit_cooldown, CIRCUIT_COOLDOWN)
+      @keep_alive_timeout = float_setting("ONLYLOGS_KEEP_ALIVE_TIMEOUT", keep_alive_timeout, DEFAULT_KEEP_ALIVE_TIMEOUT)
+      @spool_dir = spool_dir
+      @spool_max_bytes = integer_setting("ONLYLOGS_SPOOL_MAX_BYTES", spool_max_bytes, Spool::DEFAULT_MAX_BYTES)
+      @supervisor_mutex = Mutex.new
+      reset_process_state
 
       if @drain_url
-        @spool = build_spool(spool_dir, spool_max_bytes)
         start_sender
-      else
-        $stderr.puts "Onlylogs::HttpDevice: ONLYLOGS_DRAIN_URL is not set; logging locally only." # rubocop:disable Style/StderrPuts
+        # at_exit procs are inherited by forked children, so this is registered exactly once: a
+        # child that rebuilt its state after the fork closes through the same block.
+        at_exit { close }
+      elsif blank?(drain_url)
+        safe_warn "Onlylogs::HttpDevice: ONLYLOGS_DRAIN_URL is not set; logging locally only."
       end
     end
 
@@ -80,21 +120,69 @@ module Onlylogs
       # No drain configured: nothing to ship. The local fallback (see MultiDevice) still logs it.
       return unless @drain_url
 
-      enqueue(message.chomp)
+      ensure_sender
+      enqueue(truncate(message.chomp))
     end
 
+    # Ships everything still queued, then stops the sender. This is the only synchronous path: there
+    # is deliberately no #flush, when to ship is the sender's decision (batch size or interval).
     def close
-      flush
-      @running = false
+      # A forked child that never logged owns nothing here: the queued lines and the connection
+      # belong to the parent, and finishing an inherited TLS socket would send close_notify on it.
+      return if forked?
+
+      @queue.close
       @sender_thread&.join(2)
       close_connection
     end
 
-    def flush
-      send_batch(drain_queue)
+    private
+
+    TRUNCATION_MARKER = "...[truncated by onlylogs]"
+
+    # A cap that cannot hold the marker would make every truncation raise.
+    MIN_BATCH_BYTES = TRUNCATION_MARKER.bytesize + 1
+
+    def parse_drain_url(url)
+      return if blank?(url)
+
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "not an http(s) URL with a host" unless uri.is_a?(URI::HTTP) && !blank?(uri.host)
+
+      uri
+    rescue URI::InvalidURIError => e
+      safe_warn "Onlylogs::HttpDevice: ONLYLOGS_DRAIN_URL #{url.inspect} is invalid (#{e.message}); logging locally only."
+      nil
+    end
+
+    def blank?(value)
+      value.nil? || value.to_s.strip.empty?
     end
 
-    private
+    def integer_setting(name, value, default, min: 1)
+      number = Integer(value, exception: false)
+      return number if number && number >= min
+
+      fallback_setting(name, value, default, "an integer of at least #{min}")
+    end
+
+    def float_setting(name, value, default)
+      number = Float(value, exception: false)
+      return number if number&.positive?
+
+      fallback_setting(name, value, default, "a positive number")
+    end
+
+    def fallback_setting(name, value, default, expected)
+      safe_warn "Onlylogs::HttpDevice: #{name} is #{value.inspect}, expected #{expected}; using #{default}"
+      default
+    end
+
+    def truncate(line)
+      return line if line.bytesize <= @max_batch_bytes
+
+      line.byteslice(0, @max_batch_bytes - TRUNCATION_MARKER.bytesize).scrub("") + TRUNCATION_MARKER
+    end
 
     # Push a line onto the queue unless it is full. Dropping is intentional: blocking the
     # caller (a request thread) or growing without bound (OOM) are both worse than losing
@@ -106,47 +194,130 @@ module Onlylogs
       end
 
       @queue << line
+    rescue ClosedQueueError
+      nil
     end
 
-    def start_sender
-      @running = true
+    # Cheap on the hot path (a getpid and a thread status check); only the first write after a fork
+    # or after the sender died pays for the rebuild. Several request threads can race here in a
+    # fresh worker, hence the double check under the lock.
+    def ensure_sender
+      return if sender_healthy?
+
+      @supervisor_mutex.synchronize do
+        next if sender_healthy?
 
-      @sender_thread = Thread.new do
-        # Replay anything left in the spool by a previous run or a crashed/redeployed sibling.
-        drain_spool
+        # Deliberately no close_connection here: after a fork the inherited socket is still in use
+        # by the parent, and lines left in the inherited queue are the parent's to ship.
+        reset_process_state if forked?
+        start_sender
+      end
+    end
 
-        batch = []
-        last_flush = Time.now
+    # A closed queue means #close ran: there is nothing left to supervise.
+    def sender_healthy?
+      !forked? && (@queue.closed? || @sender_thread&.alive?)
+    end
 
-        while @running || !@queue.empty?
-          begin
-            line = @queue.pop(true)
-            batch << line if line
-          rescue ThreadError
-            # queue empty
-          end
+    def forked?
+      Process.pid != @pid
+    end
 
-          if batch.any? && (batch.size >= @batch_size || (Time.now - last_flush) >= @flush_interval)
-            send_batch(batch)
+    def reset_process_state
+      @pid = Process.pid
+      @queue = Queue.new
+      @mutex = Mutex.new
+      @http_mutex = Mutex.new
+      @http = nil
+      @sender_thread = nil
+      @spool = build_spool(@spool_dir, @spool_max_bytes) if @drain_url
+      @consecutive_failures = 0
+      @circuit_open_until = nil
+      @dropped = 0
+      @rejected = 0
+      @rejection_warned_at = nil
+    end
+
+    def start_sender
+      @sender_thread = Thread.new { sender_loop }
+    end
+
+    # Blocks on the queue instead of polling it: a partial batch waits for the rest of the flush
+    # interval inside Queue#pop, so the thread costs nothing while idle. A full batch sends early;
+    # a closed queue (see #close) ends the loop once it has been emptied.
+    #
+    # The spool (batches left by an outage, or by a previous run) is replayed one file at a time
+    # between live batches, never all at once: the live queue must not overflow while we catch up,
+    # and a drain that just recovered must not be hit with every client's whole backlog at full
+    # speed. While the queue is idle the loop keeps replaying, one file per turn.
+    def sender_loop
+      batch = []
+      bytes = 0
+      deadline = nil
+
+      loop do
+        line = @queue.pop(timeout: pop_timeout(deadline))
+        break if line.nil? && @queue.closed?
+
+        if line
+          if batch.any? && bytes + line.bytesize + 1 > @max_batch_bytes
+            guard { send_batch(batch) }
             batch = []
-            last_flush = Time.now
+            bytes = 0
+            deadline = nil
           end
-
-          sleep 0.01 if batch.empty?
+          batch << line
+          bytes += line.bytesize + 1
+          deadline ||= monotonic_now + @flush_interval
         end
 
-        send_batch(batch) if batch.any?
+        if batch.any? && (batch.size >= @batch_size || monotonic_now >= deadline)
+          guard { send_batch(batch) }
+          batch = []
+          bytes = 0
+          deadline = nil
+          guard { replay_one }
+        elsif line.nil?
+          guard { replay_one }
+        end
       end
 
-      at_exit { close }
+      guard { send_batch(batch) } if batch.any?
+    end
+
+    # How long the sender may block waiting for the next line: until the partial batch is due,
+    # until the circuit closes if there is a backlog to replay, not at all if we can replay right
+    # now, or indefinitely when there is nothing to do.
+    def pop_timeout(deadline)
+      return [deadline - monotonic_now, 0].max if deadline
+      return nil unless spool_pending?
+
+      circuit_remaining
+    end
+
+    def spool_pending?
+      !@spool.nil? && !@spool.empty?
+    end
+
+    # Last line of defence for the sender thread: whatever escapes the per-batch handling is
+    # reported and the batch given up, never the thread.
+    def guard
+      yield
+    rescue => e
+      safe_warn "Onlylogs::HttpDevice sender error: #{e.class}: #{e.message}"
+    end
+
+    # All of the device's own diagnostics go through here. Kernel.warn itself raises when $stderr is
+    # a closed pipe or a detached tty (EPIPE, EIO, IOError), and an exception inside an error path
+    # would take the sender thread down with it.
+    def safe_warn(message)
+      Kernel.warn(message)
+    rescue
+      nil
     end
 
-    def drain_queue
-      lines = []
-      lines << @queue.pop(true) until @queue.empty?
-      lines
-    rescue ThreadError
-      lines
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
 
     def send_batch(lines)
@@ -164,31 +335,40 @@ module Onlylogs
 
       deliver(body)
       record_success
-      # The drain just answered: replay anything we had buffered while it was unavailable.
-      drain_spool
+    rescue Rejected => e
+      record_rejection(lines.size, e)
+    rescue Throttled => e
+      record_throttle(e)
+      spool_write(body)
     rescue => e
       record_failure
       spool_write(body)
-      Kernel.warn "Onlylogs::HttpDevice error: #{e.class}: #{e.message}"
+      safe_warn "Onlylogs::HttpDevice error: #{e.class}: #{e.message}"
     end
 
     def spool_write(body)
       @spool&.write(body)
     end
 
-    # Replay buffered batches now that the drain is responding. Oldest first; stop at the first
-    # failure (record it and leave the rest on disk) so a drain that just went down again does not
-    # burn the whole backlog into the void.
-    def drain_spool
-      return unless @spool
+    # Replay the oldest buffered batch, if the drain is believed to be up. A batch the drain rejects
+    # for good is deleted too, otherwise it would sit at the head of the spool forever and block
+    # everything behind it.
+    def replay_one
+      return if @spool.nil? || circuit_open?
 
-      @spool.replay do |body|
+      @spool.replay(limit: 1) do |body|
         deliver(body)
         record_success
         true
+      rescue Rejected => e
+        record_rejection(body.count("\n") + 1, e)
+        true
+      rescue Throttled => e
+        record_throttle(e)
+        false
       rescue => e
         record_failure
-        Kernel.warn "Onlylogs::HttpDevice replay error: #{e.class}: #{e.message}"
+        safe_warn "Onlylogs::HttpDevice replay error: #{e.class}: #{e.message}"
         false
       end
     end
@@ -198,7 +378,7 @@ module Onlylogs
 
       Spool.new(dir: dir, max_bytes: max_bytes)
     rescue => e
-      Kernel.warn "Onlylogs::HttpDevice: spool disabled (#{e.class}: #{e.message})"
+      safe_warn "Onlylogs::HttpDevice: spool disabled (#{e.class}: #{e.message})"
       nil
     end
 
@@ -236,13 +416,30 @@ module Onlylogs
       end
     end
 
-    # Net::HTTP does not raise on 4xx/5xx; it returns the response. Treat any non-2xx as a
-    # failed delivery so send_batch records it and the circuit can open. Without this a drain
-    # that is up but answering 500/413 would look like success and we'd silently drop every batch.
+    # Net::HTTP does not raise on 4xx/5xx; it returns the response. Every non-2xx raises so the
+    # caller can tell a drain that is down (retry) from one that refuses the batch (drop) or asks
+    # us to slow down (pause). The drain's body is included: onlylogs.io says why in one line.
     def ensure_success!(response)
       return if response.is_a?(Net::HTTPSuccess)
 
-      raise "drain responded #{response.code} #{response.message}"
+      message = "drain responded #{response.code} #{response.message}"
+      detail = response.body.to_s.lines.first.to_s.strip
+      message += " (#{detail[0, 80]})" unless detail.empty?
+
+      case response
+      when Net::HTTPTooManyRequests
+        raise Throttled.new(message, retry_after: parse_retry_after(response["Retry-After"]))
+      when Net::HTTPClientError
+        raise Rejected, message
+      else
+        raise message
+      end
+    end
+
+    # Only the delay-seconds form; an HTTP-date is rare and the cooldown is a fine fallback.
+    def parse_retry_after(value)
+      seconds = Integer(value.to_s, 10, exception: false)
+      seconds if seconds&.positive?
     end
 
     def build_request(body)
@@ -279,7 +476,12 @@ module Onlylogs
     end
 
     def circuit_open?
-      @mutex.synchronize { !@circuit_open_until.nil? && Time.now < @circuit_open_until }
+      circuit_remaining.positive?
+    end
+
+    # Seconds until the circuit closes again; 0 when it is closed.
+    def circuit_remaining
+      @mutex.synchronize { @circuit_open_until ? [@circuit_open_until - Time.now, 0].max : 0 }
     end
 
     def record_success
@@ -290,7 +492,7 @@ module Onlylogs
     end
 
     def record_failure
-      opened = false
+      pause = nil
       dropped = 0
 
       @mutex.synchronize do
@@ -300,18 +502,51 @@ module Onlylogs
         # (Re)open the circuit. record_failure only runs on a real send attempt — send_batch
         # short-circuits while the circuit is open — so reaching here always means the drain
         # is still down and we should pause again (this is how recovery retries every cooldown).
-        @circuit_open_until = Time.now + @circuit_cooldown
-        opened = true
+        pause = jittered_cooldown
+        @circuit_open_until = Time.now + pause
         dropped = @dropped
         @dropped = 0
       end
 
       # Warn outside the mutex:
       # doing it inside the lock would re-enter @mutex through record_failure and raise a recursive-lock error.
-      return unless opened
+      return unless pause
 
       suffix = dropped.positive? ? " (#{dropped} log lines dropped)" : ""
-      Kernel.warn "Onlylogs::HttpDevice: drain unavailable, pausing for #{@circuit_cooldown}s#{suffix}"
+      safe_warn "Onlylogs::HttpDevice: drain unavailable, pausing for #{pause.round}s#{suffix}"
+    end
+
+    # A 429 is not an outage: the drain is up and told us how long to wait. Open the circuit for
+    # that long without counting a failure, so the batch goes to the spool and is replayed later.
+    def record_throttle(error)
+      pause = error.retry_after || jittered_cooldown
+      @mutex.synchronize { @circuit_open_until = Time.now + pause }
+      safe_warn "Onlylogs::HttpDevice: #{error.message}, pausing for #{pause.round}s"
+    end
+
+    # The drain is up (so the circuit stays closed) but refuses this batch for good. One warning
+    # per cooldown period, with the running count, rather than one per batch: an unknown token
+    # rejects every single batch and would otherwise flood stderr.
+    def record_rejection(line_count, error)
+      rejected = nil
+
+      @mutex.synchronize do
+        @consecutive_failures = 0
+        @rejected += line_count
+        next unless @rejection_warned_at.nil? || monotonic_now - @rejection_warned_at >= @circuit_cooldown
+
+        rejected = @rejected
+        @rejected = 0
+        @rejection_warned_at = monotonic_now
+      end
+
+      return unless rejected
+
+      safe_warn "Onlylogs::HttpDevice: #{error.message}, dropped #{rejected} log lines the drain will not accept"
+    end
+
+    def jittered_cooldown
+      @circuit_cooldown * (0.5 + rand)
     end
   end
 end

data/lib/onlylogs/http_logger.rb

@@ -9,8 +9,10 @@ require_relative "multi_device"
 #
 # This is a plain Onlylogs::Logger whose log device is an Onlylogs::HttpDevice teed with the local
 # fallback. It deliberately does NOT override #add: the stock Logger#add applies the level and
-# formats each line before writing, so a below-level line reaches neither sink. All the batching,
-# circuit breaking and disk spooling lives in HttpDevice.
+# formats each line before writing, so a below-level line reaches neither sink. Nor #flush: Rails
+# calls it after every request, on the request thread, and it must stay the tag reset it inherits.
+# When to ship is the sender's decision (batch size or interval); all the batching, circuit
+# breaking and disk spooling lives in HttpDevice, and shutdown goes through #close.
 module Onlylogs
   class HttpLogger < Onlylogs::Logger
     attr_reader :device
@@ -20,11 +22,6 @@ module Onlylogs
       super(MultiDevice.new(local_fallback, @device))
     end
 
-    # Drain the device's in-memory queue to the drain now (tests and graceful shutdown rely on it).
-    def flush
-      @device.flush
-    end
-
     # Only the remote device is ours to close; the local fallback ($stdout) belongs to the app, so
     # we deliberately do not call super (which would close the whole log device, fallback included).
     def close