onlylogs 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43e963f2acccbac4061747abd2606eb298b7ea138404f6e726e03c7a5ffacb4e
-  data.tar.gz: c9ee6fb1e854dffddb634be9153d285c8db88836b8a2f7046b63f0a8f0734cfa
+  metadata.gz: f27d7205791730383562092985355eb068e2568ff617ef7d050966347ebc54df
+  data.tar.gz: 73b44d11e108d5a030134eeba81fe2f40b49442ef7ceba388d16c3348828d7da
 SHA512:
-  metadata.gz: 0cf479d9e32a82adb41090a4e4525c4f237e75f7ab52485ac29ce8db5cfaf77fada2434dc675078afabf660ee4d3bf1470f916c13976a3cc11ca5823eb041bbe
-  data.tar.gz: 955aa49574feaa833e135a853145858961514ebe4c4a093a3758410b1ad91c5aaa6218f09c4944b29360c0419f9bddd738d4c25b48e99e70b882e95b56b2be60
+  metadata.gz: 7e9f5954ab6144be0f8886503b6a1bb6ece1926fa01ea3275e2bf6523b68ac9334025c41d5bf6ef5aca79ced1cab5d33d548c3cd1a79ac248b0f8dcb7048ed33
+  data.tar.gz: 986ba53d82a32a5c96ec65a029c543bb0b6123a8e94eb83654bfa6ffbae93327fc8bf295604b0f14faf81042fa358faaa112bac69d54c98b6c254fa258aa193f

data/lib/onlylogs/http_logger.rb

@@ -2,6 +2,7 @@
 
 require "net/http"
 require "uri"
+require_relative "spool"
 
 # This logger sends messages to onlylogs.io (or any Vector-compatible sink) directly via HTTP.
 # Unlike SocketLogger, it does not require a sidecar process or Puma plugin,
@@ -13,6 +14,10 @@ require "uri"
 # * cooldown: once the drain is known to be failing we stop attempting
 #   requests for a cooldown period instead of blocking on every send for the full
 #   read timeout (a down host accepts the TCP/TLS connection but never answers).
+#
+# By default an on-disk Spool buffers any batch we could not deliver and replays it once the
+# drain recovers, so a transient outage or a restart does not lose logs. It is on by default
+# (set ONLYLOGS_SPOOL_DIR empty to disable) and bounded by bytes; see Onlylogs::Spool.
 module Onlylogs
   class HttpLogger < Onlylogs::Logger
     DEFAULT_BATCH_SIZE = 100
@@ -23,6 +28,10 @@ module Onlylogs
     DEFAULT_OPEN_TIMEOUT = 0.5
     DEFAULT_READ_TIMEOUT = 0.5
 
+    # How long Net::HTTP may keep an idle connection around for reuse. Comfortably longer than
+    # the default flush interval so normal traffic reuses one connection across many batches.
+    DEFAULT_KEEP_ALIVE_TIMEOUT = 30
+
     # Open the circuit after this many consecutive failed sends
     CIRCUIT_FAILURE_THRESHOLD = 3
     # ...and keep it open for this long once it is open.
@@ -36,32 +45,42 @@ module Onlylogs
       max_queue_size: ENV.fetch("ONLYLOGS_MAX_QUEUE_SIZE", DEFAULT_MAX_QUEUE_SIZE).to_i,
       open_timeout: ENV.fetch("ONLYLOGS_OPEN_TIMEOUT", DEFAULT_OPEN_TIMEOUT).to_f,
       read_timeout: ENV.fetch("ONLYLOGS_READ_TIMEOUT", DEFAULT_READ_TIMEOUT).to_f,
-      circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN).to_f
+      circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN).to_f,
+      keep_alive_timeout: ENV.fetch("ONLYLOGS_KEEP_ALIVE_TIMEOUT", DEFAULT_KEEP_ALIVE_TIMEOUT).to_f,
+      spool_dir: ENV.fetch("ONLYLOGS_SPOOL_DIR", default_spool_dir),
+      spool_max_bytes: ENV.fetch("ONLYLOGS_SPOOL_MAX_BYTES", Spool::DEFAULT_MAX_BYTES).to_i
     )
       super(local_fallback)
       @drain_url = drain_url
+      @uri = URI.parse(drain_url) if drain_url
       @batch_size = batch_size
       @flush_interval = flush_interval
       @max_queue_size = max_queue_size
       @open_timeout = open_timeout
       @read_timeout = read_timeout
       @circuit_cooldown = circuit_cooldown
+      @keep_alive_timeout = keep_alive_timeout
       @queue = Queue.new
       @mutex = Mutex.new
+      @http_mutex = Mutex.new
+      @http = nil
+      @spool = nil
 
       @consecutive_failures = 0
       @circuit_open_until = nil
       @dropped = 0
 
       if @drain_url
+        @spool = build_spool(spool_dir, spool_max_bytes)
         start_sender
       else
-        $stderr.puts "Onlylogs::HttpLogger error: ONLYLOGS_DRAIN_URL is not set; logger is disabled." # rubocop:disable Style/StderrPuts
+        $stderr.puts "Onlylogs::HttpLogger: ONLYLOGS_DRAIN_URL is not set; logging locally only." # rubocop:disable Style/StderrPuts
       end
     end
 
     def add(severity, message = nil, progname = nil, &block)
-      return true unless @drain_url
+      # No drain configured: behave as a plain local logger instead of dropping everything.
+      return super unless @drain_url
 
       if message.nil?
         if block_given?
@@ -81,6 +100,7 @@ module Onlylogs
       flush
       @running = false
       @sender_thread&.join(2)
+      close_connection
     end
 
     def flush
@@ -106,6 +126,9 @@ module Onlylogs
       @running = true
 
       @sender_thread = Thread.new do
+        # Replay anything left in the spool by a previous run or a crashed/redeployed sibling.
+        drain_spool
+
         batch = []
         last_flush = Time.now
 
@@ -142,27 +165,133 @@ module Onlylogs
 
     def send_batch(lines)
       return if lines.empty?
-      # Drain is known to be down: skip the request entirely so we don't block for the
-      # full read timeout on every batch. The lines are dropped (best-effort logging).
-      return if circuit_open?
 
-      uri = URI.parse(@drain_url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = (uri.scheme == "https")
-      http.read_timeout = @read_timeout
-      http.open_timeout = @open_timeout
+      body = lines.join("\n")
 
-      request = Net::HTTP::Post.new(uri.path)
-      request.body = lines.join("\n")
-      request.content_type = "text/plain"
+      # Drain is known to be down: skip the request entirely so we don't block for the full read
+      # timeout on every batch. Buffer the batch so the cooldown does not cost us data (without a
+      # spool configured, spool_write is a no-op and the batch is dropped — best-effort logging).
+      if circuit_open?
+        spool_write(body)
+        return
+      end
 
-      http.start { |h| h.request(request) }
+      deliver(body)
       record_success
+      # The drain just answered: replay anything we had buffered while it was unavailable.
+      drain_spool
     rescue => e
       record_failure
+      spool_write(body)
       Kernel.warn "Onlylogs::HttpLogger error: #{e.class}: #{e.message}"
     end
 
+    def spool_write(body)
+      @spool&.write(body)
+    end
+
+    # Replay buffered batches now that the drain is responding. Oldest first; stop at the first
+    # failure (record it and leave the rest on disk) so a drain that just went down again does not
+    # burn the whole backlog into the void.
+    def drain_spool
+      return unless @spool
+
+      @spool.replay do |body|
+        deliver(body)
+        record_success
+        true
+      rescue => e
+        record_failure
+        Kernel.warn "Onlylogs::HttpLogger replay error: #{e.class}: #{e.message}"
+        false
+      end
+    end
+
+    def build_spool(dir, max_bytes)
+      return if dir.nil? || dir.to_s.strip.empty?
+
+      Spool.new(dir: dir, max_bytes: max_bytes)
+    rescue => e
+      Kernel.warn "Onlylogs::HttpLogger: spool disabled (#{e.class}: #{e.message})"
+      nil
+    end
+
+    # The spool is on by default. It lives under the app's tmp dir, which survives a drain outage
+    # while the app keeps running; point ONLYLOGS_SPOOL_DIR at a persistent volume to also survive
+    # redeploys, or set it empty to disable.
+    def default_spool_dir
+      base = if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
+        Rails.root.to_s
+      else
+        ::Dir.pwd
+      end
+
+      ::File.join(base, "tmp", "onlylogs", "spool")
+    end
+
+    # POST the body over a persistent (kept-alive) connection.
+    def deliver(body)
+      @http_mutex.synchronize do
+        attempts = 0
+        response = begin
+          attempts += 1
+          reused = !@http.nil?
+          connection.request(build_request(body))
+        rescue
+          close_connection
+          retry if reused && attempts < 2
+          raise
+        end
+
+        # Checked outside the rescue on purpose: a non-2xx is an application-level error on a
+        # healthy connection, so it must NOT trigger the reconnect-retry above (that would hammer
+        # an erroring drain on a perfectly good socket). Raising here records a failure instead.
+        ensure_success!(response)
+      end
+    end
+
+    # Net::HTTP does not raise on 4xx/5xx; it returns the response. Treat any non-2xx as a
+    # failed delivery so send_batch records it and the circuit can open. Without this a drain
+    # that is up but answering 500/413 would look like success and we'd silently drop every batch.
+    def ensure_success!(response)
+      return if response.is_a?(Net::HTTPSuccess)
+
+      raise "drain responded #{response.code} #{response.message}"
+    end
+
+    def build_request(body)
+      # request_uri (not path): it defaults to "/" when the drain URL has no path — Net::HTTP::Post.new("")
+      # raises "HTTP request path is empty" — and it carries any query string (e.g. ?token=...) along.
+      request = Net::HTTP::Post.new(@uri.request_uri)
+      request.body = body
+      request.content_type = "text/plain"
+      request
+    end
+
+    # Lazily opens and memoizes the connection. Only assigns @http once #start succeeds, so a
+    # failed connect leaves @http nil and the next send starts clean. Caller holds @http_mutex.
+    def connection
+      return @http if @http
+
+      http = Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = (@uri.scheme == "https")
+      http.read_timeout = @read_timeout
+      http.open_timeout = @open_timeout
+      http.keep_alive_timeout = @keep_alive_timeout
+      http.start
+      @http = http
+    end
+
+    # Caller holds @http_mutex, or no other thread can touch @http (shutdown after the sender
+    # thread has joined).
+    def close_connection
+      @http&.finish
+    rescue IOError
+      # already closed
+    ensure
+      @http = nil
+    end
+
     def circuit_open?
       @mutex.synchronize { !@circuit_open_until.nil? && Time.now < @circuit_open_until }
     end

data/lib/onlylogs/spool.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "securerandom"
+
+module Onlylogs
+  # A bounded, on-disk overflow buffer for log batches that could not be delivered.
+  #
+  # HttpLogger keeps the happy path in memory: only when a send fails or the circuit is open does
+  # a batch get written here, to be replayed once the drain recovers (and on the next boot).
+  # This turns transient-failure / restart data loss into at-least-once delivery: a batch that was in
+  # fact received but whose response was lost will be replayed and show up as a duplicate
+  # downstream. Duplicates are an accepted trade for not losing data.
+  class Spool
+    DEFAULT_MAX_BYTES = 128 * 1024 * 1024 # 128 MB
+
+    def initialize(dir:, max_bytes: DEFAULT_MAX_BYTES)
+      @dir = dir
+      @max_bytes = max_bytes
+      # Unique per instance so two runs (even with a reused pid) never collide on a filename.
+      @token = SecureRandom.hex(4)
+      @seq = 0
+      @mutex = Mutex.new
+      ::FileUtils.mkdir_p(@dir)
+    end
+
+    # Persist a batch body. Rolls the oldest batches off first if the byte cap would be exceeded.
+    def write(body)
+      return if body.nil? || body.empty?
+
+      @mutex.synchronize do
+        evict(body.bytesize)
+        seq = (@seq += 1)
+        final = ::File.join(@dir, "#{@token}-#{format("%09d", seq)}.batch")
+        tmp = "#{final}.tmp"
+        # Write to a temp name then rename: rename is atomic, so replay never reads a
+        # half-written file (it only globs *.batch).
+        ::File.binwrite(tmp, body)
+        ::File.rename(tmp, final)
+      end
+    rescue => e
+      Kernel.warn "Onlylogs::Spool write error: #{e.class}: #{e.message}"
+    end
+
+    # Replay pending batches oldest-first. Yields each body; if the block returns truthy the file
+    # is deleted (delivered), otherwise replay stops and the remaining files are kept for later.
+    def replay
+      pending_files.each do |path|
+        body = read(path)
+        next if body.nil? # already claimed/deleted by another process
+
+        break unless yield(body)
+
+        delete(path)
+      end
+    end
+
+    def empty?
+      pending_files.empty?
+    end
+
+    private
+
+    # Oldest-first. mtime is the primary key; the zero-padded sequence in the filename breaks
+    # ties (and preserves per-process write order when mtimes collide at coarse FS resolution).
+    def pending_files
+      ::Dir.glob(::File.join(@dir, "*.batch")).sort_by { |path| [mtime(path), path] }
+    end
+
+    def mtime(path)
+      ::File.mtime(path)
+    rescue Errno::ENOENT
+      Time.at(0)
+    end
+
+    def read(path)
+      ::File.binread(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    def delete(path)
+      ::File.delete(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    # Delete oldest batches until `incoming` more bytes fit under the cap.
+    def evict(incoming)
+      files = pending_files
+      total = files.sum { |path| size(path) }
+
+      while total + incoming > @max_bytes && (oldest = files.shift)
+        total -= size(oldest)
+        delete(oldest)
+      end
+    end
+
+    def size(path)
+      ::File.size(path)
+    rescue Errno::ENOENT
+      0
+    end
+  end
+end

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/onlylogs.rb

@@ -3,6 +3,7 @@ require "onlylogs/configuration"
 require "onlylogs/engine"
 require "onlylogs/formatter"
 require "onlylogs/logger"
+require "onlylogs/spool"
 require "onlylogs/socket_logger"
 require "onlylogs/http_logger"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Alessandro Rodi
@@ -82,6 +82,7 @@ files:
 - lib/onlylogs/http_logger.rb
 - lib/onlylogs/logger.rb
 - lib/onlylogs/socket_logger.rb
+- lib/onlylogs/spool.rb
 - lib/onlylogs/version.rb
 - lib/puma/plugin/onlylogs_sidecar.rb
 - lib/tasks/onlylogs_tasks.rake