onlylogs 0.5.3 → 0.8.0

data/lib/onlylogs/http_logger.rb

@@ -1,331 +1,34 @@
 # frozen_string_literal: true
 
-require "net/http"
-require "uri"
-require_relative "spool"
+require_relative "http_device"
+require_relative "multi_device"
 
-# This logger sends messages to onlylogs.io (or any Vector-compatible sink) directly via HTTP.
-# Unlike SocketLogger, it does not require a sidecar process or Puma plugin,
-# so it works from any process: Puma, GoodJob, Sidekiq, rake tasks, migrations, etc.
-
-# When the drain is unreachable or unresponsive, we do two things to protect the app:
-# * an upper bound to the in-memory queue: log lines can never accumulate without limit and
-#   exhaust memory
-# * cooldown: once the drain is known to be failing we stop attempting
-#   requests for a cooldown period instead of blocking on every send for the full
-#   read timeout (a down host accepts the TCP/TLS connection but never answers).
+# Logs to $stdout (local fallback) and to onlylogs.io (or any Vector-compatible sink) directly via
+# HTTP. Unlike SocketLogger it does not require a sidecar process or Puma plugin, so it works from
+# any process: Puma, GoodJob, Sidekiq, rake tasks, migrations, etc.
 #
-# By default an on-disk Spool buffers any batch we could not deliver and replays it once the
-# drain recovers, so a transient outage or a restart does not lose logs. It is on by default
-# (set ONLYLOGS_SPOOL_DIR empty to disable) and bounded by bytes; see Onlylogs::Spool.
+# This is a plain Onlylogs::Logger whose log device is an Onlylogs::HttpDevice teed with the local
+# fallback. It deliberately does NOT override #add: the stock Logger#add applies the level and
+# formats each line before writing, so a below-level line reaches neither sink. All the batching,
+# circuit breaking and disk spooling lives in HttpDevice.
 module Onlylogs
   class HttpLogger < Onlylogs::Logger
-    DEFAULT_BATCH_SIZE = 100
-    DEFAULT_FLUSH_INTERVAL = 0.5
-    DEFAULT_MAX_QUEUE_SIZE = 10_000
-
-    # Keep timeouts short: a single slow/dead drain must never stall the app for long.
-    DEFAULT_OPEN_TIMEOUT = 0.5
-    DEFAULT_READ_TIMEOUT = 0.5
-
-    # How long Net::HTTP may keep an idle connection around for reuse. Comfortably longer than
-    # the default flush interval so normal traffic reuses one connection across many batches.
-    DEFAULT_KEEP_ALIVE_TIMEOUT = 30
-
-    # Open the circuit after this many consecutive failed sends
-    CIRCUIT_FAILURE_THRESHOLD = 3
-    # ...and keep it open for this long once it is open.
-    CIRCUIT_COOLDOWN = 30
-
-    def initialize(
-      local_fallback: $stdout,
-      drain_url: ENV["ONLYLOGS_DRAIN_URL"],
-      batch_size: ENV.fetch("ONLYLOGS_BATCH_SIZE", DEFAULT_BATCH_SIZE).to_i,
-      flush_interval: ENV.fetch("ONLYLOGS_FLUSH_INTERVAL", DEFAULT_FLUSH_INTERVAL).to_f,
-      max_queue_size: ENV.fetch("ONLYLOGS_MAX_QUEUE_SIZE", DEFAULT_MAX_QUEUE_SIZE).to_i,
-      open_timeout: ENV.fetch("ONLYLOGS_OPEN_TIMEOUT", DEFAULT_OPEN_TIMEOUT).to_f,
-      read_timeout: ENV.fetch("ONLYLOGS_READ_TIMEOUT", DEFAULT_READ_TIMEOUT).to_f,
-      circuit_cooldown: ENV.fetch("ONLYLOGS_CIRCUIT_COOLDOWN", CIRCUIT_COOLDOWN).to_f,
-      keep_alive_timeout: ENV.fetch("ONLYLOGS_KEEP_ALIVE_TIMEOUT", DEFAULT_KEEP_ALIVE_TIMEOUT).to_f,
-      spool_dir: ENV.fetch("ONLYLOGS_SPOOL_DIR", default_spool_dir),
-      spool_max_bytes: ENV.fetch("ONLYLOGS_SPOOL_MAX_BYTES", Spool::DEFAULT_MAX_BYTES).to_i
-    )
-      super(local_fallback)
-      @drain_url = drain_url
-      @uri = URI.parse(drain_url) if drain_url
-      @batch_size = batch_size
-      @flush_interval = flush_interval
-      @max_queue_size = max_queue_size
-      @open_timeout = open_timeout
-      @read_timeout = read_timeout
-      @circuit_cooldown = circuit_cooldown
-      @keep_alive_timeout = keep_alive_timeout
-      @queue = Queue.new
-      @mutex = Mutex.new
-      @http_mutex = Mutex.new
-      @http = nil
-      @spool = nil
+    attr_reader :device
 
-      @consecutive_failures = 0
-      @circuit_open_until = nil
-      @dropped = 0
-
-      if @drain_url
-        @spool = build_spool(spool_dir, spool_max_bytes)
-        start_sender
-      else
-        $stderr.puts "Onlylogs::HttpLogger: ONLYLOGS_DRAIN_URL is not set; logging locally only." # rubocop:disable Style/StderrPuts
-      end
-    end
-
-    def add(severity, message = nil, progname = nil, &block)
-      # No drain configured: behave as a plain local logger instead of dropping everything.
-      return super unless @drain_url
-
-      if message.nil?
-        if block_given?
-          message = block.call
-        else
-          message = progname
-          progname = nil
-        end
-      end
-
-      formatted = format_message(format_severity(severity), Time.now, progname, message.to_s)
-      enqueue(formatted.chomp) if formatted
-      super
-    end
-
-    def close
-      flush
-      @running = false
-      @sender_thread&.join(2)
-      close_connection
+    def initialize(local_fallback: $stdout, **device_options)
+      @device = HttpDevice.new(**device_options)
+      super(MultiDevice.new(local_fallback, @device))
     end
 
+    # Drain the device's in-memory queue to the drain now (tests and graceful shutdown rely on it).
     def flush
-      send_batch(drain_queue)
-      super
-    end
-
-    private
-
-    # Push a line onto the queue unless it is full. Dropping is intentional: blocking the
-    # caller (a request thread) or growing without bound (OOM) are both worse than losing
-    # logs while the drain is unavailable.
-    def enqueue(line)
-      if @queue.size >= @max_queue_size
-        @mutex.synchronize { @dropped += 1 }
-        return
-      end
-
-      @queue << line
-    end
-
-    def start_sender
-      @running = true
-
-      @sender_thread = Thread.new do
-        # Replay anything left in the spool by a previous run or a crashed/redeployed sibling.
-        drain_spool
-
-        batch = []
-        last_flush = Time.now
-
-        while @running || !@queue.empty?
-          begin
-            line = @queue.pop(true)
-            batch << line if line
-          rescue ThreadError
-            # queue empty
-          end
-
-          if batch.any? && (batch.size >= @batch_size || (Time.now - last_flush) >= @flush_interval)
-            send_batch(batch)
-            batch = []
-            last_flush = Time.now
-          end
-
-          sleep 0.01 if batch.empty?
-        end
-
-        send_batch(batch) if batch.any?
-      end
-
-      at_exit { close }
-    end
-
-    def drain_queue
-      lines = []
-      lines << @queue.pop(true) until @queue.empty?
-      lines
-    rescue ThreadError
-      lines
-    end
-
-    def send_batch(lines)
-      return if lines.empty?
-
-      body = lines.join("\n")
-
-      # Drain is known to be down: skip the request entirely so we don't block for the full read
-      # timeout on every batch. Buffer the batch so the cooldown does not cost us data (without a
-      # spool configured, spool_write is a no-op and the batch is dropped — best-effort logging).
-      if circuit_open?
-        spool_write(body)
-        return
-      end
-
-      deliver(body)
-      record_success
-      # The drain just answered: replay anything we had buffered while it was unavailable.
-      drain_spool
-    rescue => e
-      record_failure
-      spool_write(body)
-      Kernel.warn "Onlylogs::HttpLogger error: #{e.class}: #{e.message}"
+      @device.flush
     end
 
-    def spool_write(body)
-      @spool&.write(body)
-    end
-
-    # Replay buffered batches now that the drain is responding. Oldest first; stop at the first
-    # failure (record it and leave the rest on disk) so a drain that just went down again does not
-    # burn the whole backlog into the void.
-    def drain_spool
-      return unless @spool
-
-      @spool.replay do |body|
-        deliver(body)
-        record_success
-        true
-      rescue => e
-        record_failure
-        Kernel.warn "Onlylogs::HttpLogger replay error: #{e.class}: #{e.message}"
-        false
-      end
-    end
-
-    def build_spool(dir, max_bytes)
-      return if dir.nil? || dir.to_s.strip.empty?
-
-      Spool.new(dir: dir, max_bytes: max_bytes)
-    rescue => e
-      Kernel.warn "Onlylogs::HttpLogger: spool disabled (#{e.class}: #{e.message})"
-      nil
-    end
-
-    # The spool is on by default. It lives under the app's tmp dir, which survives a drain outage
-    # while the app keeps running; point ONLYLOGS_SPOOL_DIR at a persistent volume to also survive
-    # redeploys, or set it empty to disable.
-    def default_spool_dir
-      base = if defined?(Rails) && Rails.respond_to?(:root) && Rails.root
-        Rails.root.to_s
-      else
-        ::Dir.pwd
-      end
-
-      ::File.join(base, "tmp", "onlylogs", "spool")
-    end
-
-    # POST the body over a persistent (kept-alive) connection.
-    def deliver(body)
-      @http_mutex.synchronize do
-        attempts = 0
-        response = begin
-          attempts += 1
-          reused = !@http.nil?
-          connection.request(build_request(body))
-        rescue
-          close_connection
-          retry if reused && attempts < 2
-          raise
-        end
-
-        # Checked outside the rescue on purpose: a non-2xx is an application-level error on a
-        # healthy connection, so it must NOT trigger the reconnect-retry above (that would hammer
-        # an erroring drain on a perfectly good socket). Raising here records a failure instead.
-        ensure_success!(response)
-      end
-    end
-
-    # Net::HTTP does not raise on 4xx/5xx; it returns the response. Treat any non-2xx as a
-    # failed delivery so send_batch records it and the circuit can open. Without this a drain
-    # that is up but answering 500/413 would look like success and we'd silently drop every batch.
-    def ensure_success!(response)
-      return if response.is_a?(Net::HTTPSuccess)
-
-      raise "drain responded #{response.code} #{response.message}"
-    end
-
-    def build_request(body)
-      # request_uri (not path): it defaults to "/" when the drain URL has no path — Net::HTTP::Post.new("")
-      # raises "HTTP request path is empty" — and it carries any query string (e.g. ?token=...) along.
-      request = Net::HTTP::Post.new(@uri.request_uri)
-      request.body = body
-      request.content_type = "text/plain"
-      request
-    end
-
-    # Lazily opens and memoizes the connection. Only assigns @http once #start succeeds, so a
-    # failed connect leaves @http nil and the next send starts clean. Caller holds @http_mutex.
-    def connection
-      return @http if @http
-
-      http = Net::HTTP.new(@uri.host, @uri.port)
-      http.use_ssl = (@uri.scheme == "https")
-      http.read_timeout = @read_timeout
-      http.open_timeout = @open_timeout
-      http.keep_alive_timeout = @keep_alive_timeout
-      http.start
-      @http = http
-    end
-
-    # Caller holds @http_mutex, or no other thread can touch @http (shutdown after the sender
-    # thread has joined).
-    def close_connection
-      @http&.finish
-    rescue IOError
-      # already closed
-    ensure
-      @http = nil
-    end
-
-    def circuit_open?
-      @mutex.synchronize { !@circuit_open_until.nil? && Time.now < @circuit_open_until }
-    end
-
-    def record_success
-      @mutex.synchronize do
-        @consecutive_failures = 0
-        @circuit_open_until = nil
-      end
-    end
-
-    def record_failure
-      opened = false
-      dropped = 0
-
-      @mutex.synchronize do
-        @consecutive_failures += 1
-        next if @consecutive_failures < CIRCUIT_FAILURE_THRESHOLD
-
-        # (Re)open the circuit. record_failure only runs on a real send attempt — send_batch
-        # short-circuits while the circuit is open — so reaching here always means the drain
-        # is still down and we should pause again (this is how recovery retries every cooldown).
-        @circuit_open_until = Time.now + @circuit_cooldown
-        opened = true
-        dropped = @dropped
-        @dropped = 0
-      end
-
-      # Warn outside the mutex:
-      # doing it inside the lock would re-enter @mutex through add -> enqueue and raise a recursive-lock error.
-      return unless opened
-
-      suffix = dropped.positive? ? " (#{dropped} log lines dropped)" : ""
-      Kernel.warn "Onlylogs::HttpLogger: drain unavailable, pausing for #{@circuit_cooldown}s#{suffix}"
+    # Only the remote device is ours to close; the local fallback ($stdout) belongs to the app, so
+    # we deliberately do not call super (which would close the whole log device, fallback included).
+    def close
+      @device.close
     end
   end
 end

data/lib/onlylogs/multi_device.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Onlylogs
+  # A Logger log device that fans each line out to several underlying devices, e.g. the local
+  # $stdout fallback plus a remote sink (Onlylogs::SocketDevice / Onlylogs::HttpDevice).
+  class MultiDevice
+    def initialize(*devices)
+      @devices = devices.compact
+    end
+
+    def write(message)
+      return if message.nil? || message.empty?
+
+      @devices.each { |device| device.write(message) }
+    end
+
+    def close
+      @devices.each do |device|
+        # Never close the process' standard streams — the app owns them, not us.
+        next if device.equal?($stdout) || device.equal?($stderr)
+
+        device.close if device.respond_to?(:close)
+      end
+    end
+
+    def flush
+      @devices.each { |device| device.flush if device.respond_to?(:flush) }
+    end
+  end
+end

data/lib/onlylogs/socket_device.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module Onlylogs
+  # Send failures are reported to $stderr — never through a logger — so a failing socket can never
+  # re-enter logging and deadlock or loop.
+  class SocketDevice
+    DEFAULT_SOCKET = "tmp/sockets/onlylogs-sidecar.sock"
+
+    def initialize(socket_path: ENV.fetch("ONLYLOGS_SIDECAR_SOCKET", DEFAULT_SOCKET))
+      @socket_path = socket_path
+      @socket_mutex = Mutex.new
+      @socket = nil
+    end
+
+    def write(message)
+      return if message.nil? || message.empty?
+
+      socket = ensure_socket
+      socket&.puts(message)
+    rescue Errno::EPIPE, Errno::ECONNREFUSED, Errno::ENOENT => e
+      $stderr.puts "Onlylogs::SocketDevice error: #{e.message}" # rubocop:disable Style/StderrPuts
+      reconnect_socket
+    rescue => e
+      $stderr.puts "Onlylogs::SocketDevice unexpected error: #{e.class}: #{e.message}" # rubocop:disable Style/StderrPuts
+      reconnect_socket
+    end
+
+    def close
+      reconnect_socket
+    end
+
+    private
+
+    def ensure_socket
+      return @socket if @socket
+
+      @socket_mutex.synchronize do
+        @socket ||= UNIXSocket.new(@socket_path)
+      rescue => e
+        $stderr.puts "Unable to connect to Onlylogs sidecar (#{@socket_path}): #{e.message}" # rubocop:disable Style/StderrPuts
+        @socket = nil
+      end
+
+      @socket
+    end
+
+    def reconnect_socket
+      @socket_mutex.synchronize do
+        begin
+          @socket&.close
+        rescue
+          nil
+        end
+        @socket = nil
+      end
+    end
+  end
+end

data/lib/onlylogs/socket_logger.rb

@@ -1,73 +1,27 @@
 # frozen_string_literal: true
 
-require "socket"
-
-# This logger sends messages to onlylogs.io via a UNIX socket connected to the onlylogs sidecar process.
-# You need to have the onlylogs sidecar running for this to work.
-
+require_relative "socket_device"
+require_relative "multi_device"
+
+# Logs to $stdout (local fallback) and to onlylogs.io via a UNIX socket connected to the onlylogs
+# sidecar process. You need to have the onlylogs sidecar running for the socket sink to work.
+#
+# This is a plain Onlylogs::Logger whose log device is the sidecar socket teed with the local
+# fallback. It deliberately does NOT override #add: the stock Logger#add applies the level and
+# formats each line before writing, so a below-level line reaches neither sink.
 module Onlylogs
   class SocketLogger < Onlylogs::Logger
-    DEFAULT_SOCKET = "tmp/sockets/onlylogs-sidecar.sock"
-
-    def initialize(local_fallback: $stdout, socket_path: ENV.fetch("ONLYLOGS_SIDECAR_SOCKET", DEFAULT_SOCKET))
-      super(local_fallback)
-      @socket_path = socket_path
-      @socket_mutex = Mutex.new
-      @socket = nil
-    end
-
-    def add(severity, message = nil, progname = nil, &block)
-      if message.nil?
-        if block_given?
-          message = block.call
-        else
-          message = progname
-          progname = nil
-        end
-      end
-
-      formatted = format_message(format_severity(severity), Time.now, progname, message.to_s)
-      send_to_socket(formatted)
-      super
-    end
-
-    private
-
-    def send_to_socket(payload)
-      return if payload.nil? || payload.empty?
-
-      socket = ensure_socket
-      socket&.puts(payload)
-    rescue Errno::EPIPE, Errno::ECONNREFUSED, Errno::ENOENT => e
-      $stderr.puts "Onlylogs::SocketLogger error: #{e.message}" # rubocop:disable Style/StderrPuts
-      reconnect_socket
-    rescue => e
-      $stderr.puts "Onlylogs::SocketLogger unexpected error: #{e.class}: #{e.message}" # rubocop:disable Style/StderrPuts
-      reconnect_socket
-    end
-
-    def ensure_socket
-      return @socket if @socket
-
-      @socket_mutex.synchronize do
-        @socket ||= UNIXSocket.new(@socket_path)
-      rescue => e
-        $stderr.puts "Unable to connect to Onlylogs sidecar (#{@socket_path}): #{e.message}" # rubocop:disable Style/StderrPuts
-        @socket = nil
-      end
+    attr_reader :device
 
-      @socket
+    def initialize(local_fallback: $stdout, socket_path: ENV.fetch("ONLYLOGS_SIDECAR_SOCKET", SocketDevice::DEFAULT_SOCKET))
+      @device = SocketDevice.new(socket_path: socket_path)
+      super(MultiDevice.new(local_fallback, @device))
     end
 
-    def reconnect_socket
-      @socket_mutex.synchronize do
-        begin
-          @socket&.close
-        rescue
-          nil
-        end
-        @socket = nil
-      end
+    # Only the remote socket is ours to close; the local fallback ($stdout) belongs to the app, so
+    # we deliberately do not call super (which would close the whole log device, fallback included).
+    def close
+      @device.close
     end
   end
 end

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.5.3"
+  VERSION = "0.8.0"
 end

data/lib/onlylogs.rb

@@ -4,6 +4,9 @@ require "onlylogs/engine"
 require "onlylogs/formatter"
 require "onlylogs/logger"
 require "onlylogs/spool"
+require "onlylogs/multi_device"
+require "onlylogs/socket_device"
+require "onlylogs/http_device"
 require "onlylogs/socket_logger"
 require "onlylogs/http_logger"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.8.0
 platform: ruby
 authors:
 - Alessandro Rodi
@@ -45,6 +45,8 @@ files:
 - app/assets/images/onlylogs/favicon/web-app-manifest-192x192.png
 - app/assets/images/onlylogs/favicon/web-app-manifest-512x512.png
 - app/assets/images/onlylogs/logo.png
+- app/assets/javascripts/onlylogs/clusterize.js
+- app/assets/stylesheets/onlylogs/clusterize.css
 - app/channels/onlylogs/application_cable/channel.rb
 - app/channels/onlylogs/logs_channel.rb
 - app/controllers/onlylogs/application_controller.rb
@@ -69,6 +71,7 @@ files:
 - app/views/onlylogs/logs/index.html.erb
 - app/views/onlylogs/shared/_log_container.html.erb
 - app/views/onlylogs/shared/_log_container_styles.html.erb
+- app/views/onlylogs/shared/_range_slider.html.erb
 - bin/onlylogs_sidecar
 - bin/super_grep
 - bin/super_ripgrep
@@ -79,8 +82,11 @@ files:
 - lib/onlylogs/configuration.rb
 - lib/onlylogs/engine.rb
 - lib/onlylogs/formatter.rb
+- lib/onlylogs/http_device.rb
 - lib/onlylogs/http_logger.rb
 - lib/onlylogs/logger.rb
+- lib/onlylogs/multi_device.rb
+- lib/onlylogs/socket_device.rb
 - lib/onlylogs/socket_logger.rb
 - lib/onlylogs/spool.rb
 - lib/onlylogs/version.rb