onlylogs 0.9.0 → 0.10.0

data/lib/onlylogs/spool.rb

@@ -11,8 +11,13 @@ module Onlylogs
   # This turns transient-failure / restart data loss into at-least-once delivery: a batch that was in
   # fact received but whose response was lost will be replayed and show up as a duplicate
   # downstream. Duplicates are an accepted trade for not losing data.
+  #
+  # The byte cap is enforced from an in-memory ledger of the files in the directory, refreshed by
+  # listing the directory at most every LEDGER_TTL seconds (sibling processes such as other Puma
+  # workers write to the same directory), so a write costs one file and not a stat of every file.
   class Spool
     DEFAULT_MAX_BYTES = 128 * 1024 * 1024 # 128 MB
+    LEDGER_TTL = 5
 
     def initialize(dir:, max_bytes: DEFAULT_MAX_BYTES)
       @dir = dir
@@ -21,6 +26,9 @@ module Onlylogs
       @token = SecureRandom.hex(4)
       @seq = 0
       @mutex = Mutex.new
+      @ledger = nil
+      @ledger_bytes = 0
+      @ledger_at = nil
       ::FileUtils.mkdir_p(@dir)
     end
 
@@ -29,6 +37,7 @@ module Onlylogs
       return if body.nil? || body.empty?
 
       @mutex.synchronize do
+        refresh_ledger if ledger_stale?
         evict(body.bytesize)
         seq = (@seq += 1)
         final = ::File.join(@dir, "#{@token}-#{format("%09d", seq)}.batch")
@@ -37,30 +46,59 @@ module Onlylogs
         # half-written file (it only globs *.batch).
         ::File.binwrite(tmp, body)
         ::File.rename(tmp, final)
+        @ledger << [final, body.bytesize]
+        @ledger_bytes += body.bytesize
       end
     rescue => e
       Kernel.warn "Onlylogs::Spool write error: #{e.class}: #{e.message}"
     end
 
-    # Replay pending batches oldest-first. Yields each body; if the block returns truthy the file
-    # is deleted (delivered), otherwise replay stops and the remaining files are kept for later.
-    def replay
-      pending_files.each do |path|
+    # Replay pending batches oldest-first, at most `limit` of them. Yields each body; if the block
+    # returns truthy the file is deleted (delivered, or given up on), otherwise replay stops and the
+    # remaining files are kept for later.
+    #
+    # Works off the ledger rather than listing the directory: with a large backlog replayed one file
+    # at a time, a glob per call would cost more than the delivery itself.
+    def replay(limit: nil)
+      paths = @mutex.synchronize do
+        refresh_ledger if ledger_stale?
+        @ledger.first(limit || @ledger.size).map(&:first)
+      end
+
+      paths.each do |path|
         body = read(path)
-        next if body.nil? # already claimed/deleted by another process
+        if body.nil? # already claimed/deleted by another process
+          forget(path)
+          next
+        end
 
         break unless yield(body)
 
         delete(path)
+        forget(path)
       end
     end
 
     def empty?
-      pending_files.empty?
+      @mutex.synchronize do
+        refresh_ledger if ledger_stale?
+        @ledger.empty?
+      end
     end
 
     private
 
+    def ledger_stale?
+      @ledger.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - @ledger_at > LEDGER_TTL
+    end
+
+    # Caller holds @mutex.
+    def refresh_ledger
+      @ledger = pending_files.map { |path| [path, size(path)] }
+      @ledger_bytes = @ledger.sum { |_, bytes| bytes }
+      @ledger_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+
     # Oldest-first. mtime is the primary key; the zero-padded sequence in the filename breaks
     # ties (and preserves per-process write order when mtimes collide at coarse FS resolution).
     def pending_files
@@ -73,6 +111,18 @@ module Onlylogs
       Time.at(0)
     end
 
+    # Replay is oldest-first and so is the ledger, so the path is nearly always the head.
+    def forget(path)
+      @mutex.synchronize do
+        next if @ledger.nil?
+
+        index = (@ledger.first&.first == path) ? 0 : @ledger.index { |candidate, _| candidate == path }
+        next unless index
+
+        @ledger_bytes -= @ledger.delete_at(index).last
+      end
+    end
+
     def read(path)
       ::File.binread(path)
     rescue Errno::ENOENT
@@ -85,14 +135,12 @@ module Onlylogs
       nil
     end
 
-    # Delete oldest batches until `incoming` more bytes fit under the cap.
+    # Delete oldest batches until `incoming` more bytes fit under the cap. Caller holds @mutex.
     def evict(incoming)
-      files = pending_files
-      total = files.sum { |path| size(path) }
-
-      while total + incoming > @max_bytes && (oldest = files.shift)
-        total -= size(oldest)
-        delete(oldest)
+      while @ledger_bytes + incoming > @max_bytes && (oldest = @ledger.shift)
+        path, bytes = oldest
+        delete(path)
+        @ledger_bytes -= bytes
       end
     end
 

data/lib/onlylogs/version.rb

@@ -1,3 +1,3 @@
 module Onlylogs
-  VERSION = "0.9.0"
+  VERSION = "0.10.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: onlylogs
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
 platform: ruby
 authors:
 - Alessandro Rodi
@@ -58,6 +58,7 @@ files:
 - app/javascript/onlylogs/controllers/index.js
 - app/javascript/onlylogs/controllers/keyboard_shortcuts_controller.js
 - app/javascript/onlylogs/controllers/log_streamer_controller.js
+- app/javascript/onlylogs/controllers/range_slider_controller.js
 - app/javascript/onlylogs/controllers/text_selection_controller.js
 - app/jobs/onlylogs/application_job.rb
 - app/models/onlylogs/ansi_color_parser.rb
@@ -80,6 +81,7 @@ files:
 - db/migrate/20250902112548_create_books.rb
 - lib/onlylogs.rb
 - lib/onlylogs/configuration.rb
+- lib/onlylogs/continuous_log_writer.rb
 - lib/onlylogs/engine.rb
 - lib/onlylogs/formatter.rb
 - lib/onlylogs/http_device.rb