railwatch 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d212beaade3dc390933ea8a41bd49eb1bbf2c896056071516aac8242b39eb07
-  data.tar.gz: e647976f9634c1e0ba390f6dec52866d190401728363e3dcd1fa93aa2976af3d
+  metadata.gz: 8904a8f775771bd894a52f80b7d5ecf4d8a40d0dee09aaf16697e68685c5e106
+  data.tar.gz: de27dad2204510d9e8f3d1bb3ada89901ccd467486c94fde9f0b85ff83ab50ca
 SHA512:
-  metadata.gz: c669c17f558dec72be16f175b06abc5769680591349c2169f6598fc26575191558bd00b903ead0f1f4287bab718c5097b0550ff60e7cfe5fe82fa53940c7a953
-  data.tar.gz: fc9a32a877b22e12e22b7738b4b479c2aa8b9d1dcc5c97f7ed003f5b421ea3e1a95f917221c195c9b01a487ac124d413dc561a22f96e7483e4891bb5db95b685
+  metadata.gz: 5fe5b808c60a3aac92302f8669b82c539634cb655d7317e98279143c5e920f9178d15920a1e75145c9c153ad5d3bfc41ade5004c1df9f1e7c8de29ab2fc20b76
+  data.tar.gz: 251237b9c8ee2a3d252c61eef492d89a532e3c7fef856bde91e19d4d3baa5c831ebc34a0bbbd24b2ced390f76f0ef8106c43cee2d68d6fbf25725f01b074298f

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 0.3.5 (2026-09-20)
+
+- Make the embedded telemetry database give disk back. `PruneTelemetryJob`
+  deleted rows past `retention_days` and nothing ever vacuumed, so the freed
+  pages went on SQLite's freelist to be reused and never returned to the
+  filesystem: the file only ever grew. Measured on a copy of a real production
+  database of the same shape (bulk deletes, continuous inserts), deleting
+  40,148 rows left the file at 21M with 5,164 pages on the freelist; a single
+  `PRAGMA incremental_vacuum` took it to 44K.
+
+  Two halves, and neither is worth anything alone. A new telemetry migration,
+  `EnableIncrementalVacuum`, puts the database into `auto_vacuum=incremental`;
+  and the nightly prune now runs `PRAGMA incremental_vacuum` after its deletes,
+  bounded to 2,000 pages a slice and 25 slices (~200MB at SQLite's 4K default)
+  so a backlog drains over successive nights instead of stalling one.
+
+  The mode cannot be declared in `config/database.yml`: Rails applies its own
+  `DEFAULT_PRAGMAS` before any you declare, and `journal_mode = wal` writes the
+  file header, so by the time `auto_vacuum` runs the database is no longer new
+  and SQLite accepts the statement and ignores it. The migration is numbered
+  below `CreateTelemetry` for the same reason -- it has to run while the file
+  still holds nothing.
+
+- Add `bin/rails railwatch:vacuum:status` and `bin/rails railwatch:vacuum`.
+  An **existing** install cannot change mode without a full `VACUUM`, which
+  rewrites the whole file with the write lock held, so nothing does that on its
+  own: the migration leaves an existing database exactly as it found it.
+  `railwatch:vacuum:status` reports the file, its size, its mode and its
+  freelist and changes nothing; `railwatch:vacuum` says what the conversion
+  will cost and then does it. Installs created from this version on need
+  neither.
+
 ## 0.3.4 (2026-09-20)
 
 - Fix the embedded dashboard offering its install steps to an install that is

data/app/jobs/railwatch/prune_telemetry_job.rb

@@ -20,6 +20,21 @@ module Railwatch
     # writer that long hold would look like a wedge and end the process.
     MAX_BATCHES_PER_TABLE = 40
 
+    # Reclaim, bounded the same way the deletes are. Deleting rows only moves
+    # their pages to the freelist; PRAGMA incremental_vacuum is what hands
+    # them back to the filesystem, and it is only possible at all on a
+    # database in auto_vacuum=incremental (EnableIncrementalVacuum, or
+    # `bin/rails railwatch:vacuum` for a database that predates it).
+    # Measured at 80k-190k pages/s warm, so a full run is well under a second
+    # there; the slicing is for the cold, large file, where each slice takes
+    # and releases the write lock instead of holding it throughout.
+    VACUUM_PAGES_PER_SLICE = 2_000
+    # 50k pages, ~200MB at SQLite's 4KB default. A few thousand pages a night
+    # would never keep up with a night's deletes on a busy app, and the file
+    # would go on growing with the freelist; a backlog past this one still
+    # drains over successive nightly runs.
+    VACUUM_SLICES = 25
+
     RAW = [ Telemetry::Execution, Telemetry::Query, Telemetry::Exception, Telemetry::CacheEvent, Telemetry::Mail,
             Telemetry::Broadcast, Telemetry::Notification, Telemetry::OutgoingRequest, Telemetry::StorageOp,
             Telemetry::ViewRender, Telemetry::Log, Telemetry::EnqueuedJob, Telemetry::Transaction,
@@ -50,7 +65,11 @@ module Railwatch
         Telemetry::IngestBatch.where(received_at: ...cutoff).delete_all
         Telemetry::Process.where(booted_at: ...cutoff).delete_all
         Telemetry::HealthSample.where(sampled_at: ...cutoff).delete_all
-        TelemetryRecord.connection.execute("PRAGMA wal_checkpoint(#{checkpoint})") if TelemetryRecord.connection.adapter_name =~ /sqlite/i
+        # Before the checkpoint, not after: incremental_vacuum truncates the
+        # database file, and in WAL mode that truncation only reaches the file
+        # on disk once it is checkpointed.
+        TelemetryRecord.reclaim_freelist!(slice: VACUUM_PAGES_PER_SLICE, slices: VACUUM_SLICES)
+        TelemetryRecord.connection.execute("PRAGMA wal_checkpoint(#{checkpoint})") if TelemetryRecord.sqlite?
       end
     end
 

data/app/models/railwatch/telemetry_record.rb

@@ -31,6 +31,54 @@ module Railwatch
       end
     end
 
+    # SQLite's auto_vacuum modes. Deleted pages only leave the file in
+    # :incremental, and only when PRAGMA incremental_vacuum asks for them;
+    # :none (the default, and what every database created before
+    # EnableIncrementalVacuum is in) keeps them on the freelist forever.
+    AUTO_VACUUM_MODES = { 0 => :none, 1 => :full, 2 => :incremental }.freeze
+
+    # Everything below reads and writes THIS database, never the host's. The
+    # pragmas are per-database and there is no Active Record wrapper for them,
+    # so they go through this class's own connection on purpose: through
+    # ActiveRecord::Base they would report on, and vacuum, the application's
+    # primary database instead.
+    def self.sqlite? = connection.adapter_name.match?(/sqlite/i)
+
+    def self.auto_vacuum_mode = sqlite? ? AUTO_VACUUM_MODES.fetch(connection.select_value("PRAGMA auto_vacuum").to_i, :unknown) : nil
+
+    def self.freelist_pages = sqlite? ? connection.select_value("PRAGMA freelist_count").to_i : 0
+
+    def self.page_size = sqlite? ? connection.select_value("PRAGMA page_size").to_i : 0
+
+    # Hands freelist pages back to the filesystem, a slice at a time. Each
+    # PRAGMA is its own implicit transaction, so the write lock is taken and
+    # released once per slice rather than held for the whole reclaim -- the
+    # same reason PruneTelemetryJob deletes in bounded batches instead of one
+    # statement. Returns the pages actually reclaimed, which is 0 on a
+    # database that is not in incremental mode: there the pragma is accepted
+    # and does nothing, and calling it would look like work that happened.
+    def self.reclaim_freelist!(slice:, slices:)
+      return 0 unless sqlite? && auto_vacuum_mode == :incremental
+
+      before = freelist_pages
+      slices.times do
+        break if freelist_pages.zero?
+
+        incremental_vacuum(slice.to_i)
+      end
+      before - freelist_pages
+    end
+
+    # Through the raw connection on purpose. incremental_vacuum frees one page
+    # per sqlite3_step, and Active Record's execute steps a statement with no
+    # result columns exactly once -- so through it this pragma frees a single
+    # page whatever count it is handed, which is the kind of no-op that reads
+    # as work done. The sqlite3 gem's own execute steps to completion. Same
+    # raw_connection route Ingest::Writer already takes for its inserts.
+    def self.incremental_vacuum(pages = nil)
+      connection.raw_connection.execute("PRAGMA incremental_vacuum#{"(#{pages})" if pages}")
+    end
+
     # Records arrive as the gem's wire hashes; this is the shared envelope.
     def self.envelope_columns(t)
       t.datetime :occurred_at, null: false, precision: 6

data/db/railwatch_telemetry_migrate/20260903000000_enable_incremental_vacuum.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# Puts the telemetry database into incremental auto-vacuum mode, which is what
+# lets PruneTelemetryJob hand the pages it frees back to the filesystem
+# instead of parking them on the freelist forever. Without it the file only
+# ever grows: deleting a day of telemetry frees pages for SQLite to reuse and
+# returns not one byte to the disk.
+#
+# Why a migration, and why this version number. SQLite only lets a database
+# leave auto_vacuum=none for free while it still holds no pages; after that
+# the only way in is a full VACUUM, which rewrites the entire file with the
+# write lock held. Declaring the pragma in config/database.yml cannot do it
+# either: Rails applies its own DEFAULT_PRAGMAS before any declared ones, and
+# `journal_mode = wal` writes the file header, so by the time
+# `auto_vacuum = incremental` runs the database is no longer new and the
+# statement is a silent no-op (measured: the mode comes out `none`).
+#
+# A migration is the one hook that runs both on a new install's db:prepare and
+# on an existing install's upgrade, and this one is numbered below
+# CreateTelemetry so that on a new database it runs while the file holds
+# nothing but schema_migrations and ar_internal_metadata -- a few kilobytes,
+# where the VACUUM that commits the mode is instant. Rails runs a pending
+# migration whatever its version, so an existing install gets it too; there it
+# does nothing at all, because converting a file that may be tens of gigabytes
+# is not a decision a deploy gets to make. That one is `bin/rails
+# railwatch:vacuum`, run deliberately, which says what it will cost first.
+class EnableIncrementalVacuum < ActiveRecord::Migration[8.1]
+  # VACUUM cannot run inside a transaction.
+  disable_ddl_transaction!
+
+  # Rails' own bookkeeping, which is present before the first migration runs
+  # and so does not make a database "existing".
+  BOOKKEEPING = %w[schema_migrations ar_internal_metadata].freeze
+
+  def up
+    return say("not SQLite (#{connection.adapter_name}); auto_vacuum does not apply") unless sqlite?
+    return say("auto_vacuum is already incremental") if mode == "incremental"
+
+    populated = connection.tables - BOOKKEEPING
+    if populated.any?
+      return say("#{populated.size} tables already here: left in auto_vacuum=#{mode}. Run " \
+                 "`bin/rails railwatch:vacuum` to convert this database when you can spare the lock.")
+    end
+
+    connection.execute("PRAGMA auto_vacuum = incremental")
+    connection.execute("VACUUM")
+    say("auto_vacuum = #{mode}")
+  end
+
+  # Irreversible in the useful sense: going back to `none` is another whole
+  # VACUUM, and nothing about the schema depends on the mode. Rolling back
+  # leaves it where it is rather than spending that on an undo nobody asked
+  # for.
+  def down
+    say("auto_vacuum left as #{sqlite? ? mode : connection.adapter_name}; use `PRAGMA auto_vacuum = none; VACUUM;` to undo it by hand")
+  end
+
+  private
+    def sqlite? = connection.adapter_name.match?(/sqlite/i)
+
+    def mode = { 0 => "none", 1 => "full", 2 => "incremental" }[connection.select_value("PRAGMA auto_vacuum").to_i]
+end

data/docs/embedded.md

@@ -322,7 +322,7 @@ The work that keeps the dashboard current happens in two places:
 | anomaly scan | every 5 minutes | Anomaly rules, when any are enabled |
 | scheduled tasks | every 10 minutes | Missed and late scheduled tasks |
 | auto-resolve | daily | Resolves issues quiet for 14 days |
-| prune | daily | Deletes telemetry older than `retention_days`, then `ANALYZE` |
+| prune | daily | Deletes telemetry older than `retention_days`, returns the freed pages to the filesystem when incremental auto-vacuum is on (see below), then `ANALYZE` |
 
 Because the clock lives in the web process, it keeps running when the
 job worker is down, which is exactly when "scheduled task X missed its
@@ -375,6 +375,52 @@ telemetry database grows with traffic and sampling and is pruned to
 SQLite files in `storage/`, so Litestream or a volume snapshot covers
 them.
 
+### Giving the disk back
+
+Deleting rows does not shrink a SQLite file on its own. The pages go on
+the database's freelist, where later inserts reuse them, and the file
+stays whatever size it reached. So pruning alone keeps the *contents*
+bounded and lets the *file* grow forever.
+
+Railwatch creates its telemetry database in SQLite's
+`auto_vacuum=incremental` mode, and the nightly prune runs `PRAGMA
+incremental_vacuum` after its deletes, which hands those pages back to
+the filesystem. It is bounded -- 2,000 pages a slice, 25 slices, about
+200MB a night at SQLite's 4K default page size -- so a large backlog
+drains over successive nights rather than holding the write lock through
+one enormous reclaim. Nothing to configure.
+
+```
+$ bin/rails railwatch:vacuum:status
+Railwatch telemetry database
+  file    /rails/storage/production_railwatch_telemetry.sqlite3
+  size    1.42 GB
+  mode    auto_vacuum=incremental
+  free    312 pages (1.22 MB) on the freelist
+
+The nightly prune returns up to 195 MB a night on its own.
+`bin/rails railwatch:vacuum` returns all 1.22 MB now.
+```
+
+**Installs created before 0.3.5 report `auto_vacuum=none`, and pruning
+cannot return their space.** SQLite can only set the mode on a database
+that is still empty; on one that has data the only route is a full
+`VACUUM`, which rewrites the entire file with the write lock held.
+Railwatch will not do that to a running application behind your back, so
+upgrading leaves the mode alone and the conversion is a task you run
+when you can spare the lock:
+
+```
+$ bin/rails railwatch:vacuum
+```
+
+It prints the file, its size and its freelist first, then says how long
+the `VACUUM` should take and that it needs about the file's own size in
+free disk for the temporary copy. Telemetry written while it runs waits
+for it and the dashboard is frozen for the duration, so pick a quiet
+moment. Once converted, the nightly prune keeps up on its own and you
+never need to run it again.
+
 ## Switching to the cloud later
 
 Set a token and drop `c.transport = :local` (or set

data/lib/railwatch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Railwatch
-  VERSION = "0.3.4"
+  VERSION = "0.3.5"
 end

data/lib/tasks/railwatch_tasks.rake

@@ -28,6 +28,130 @@ module Railwatch
     end
   end
 
+  # What the embedded telemetry database is costing on disk, and the one-time
+  # conversion that lets pruning give that cost back.
+  #
+  # Deleting rows does not shrink a SQLite file. The pages go on the freelist
+  # and are reused by later inserts, and the only way back to the filesystem
+  # is auto_vacuum=incremental plus PRAGMA incremental_vacuum -- which
+  # PruneTelemetryJob now runs after every prune. A database can only enter
+  # that mode while it is still empty (EnableIncrementalVacuum does that for
+  # every telemetry database created since it shipped) or through a full
+  # VACUUM, which is what this offers to an older one.
+  class TelemetryDisk
+    # Deliberately pessimistic, and said out loud as an estimate: VACUUM
+    # rewrites the whole file, and what that costs is the operator's disk,
+    # not ours to know.
+    VACUUM_BYTES_PER_SECOND = 50 * 1024 * 1024
+
+    # nil, having said why, for an install with no telemetry database to
+    # talk about.
+    def self.open
+      unless Railwatch.config.local?
+        puts "Railwatch reports over HTTP here (transport = :http), so this app has no telemetry database; nothing to vacuum."
+        return nil
+      end
+
+      environment = Railwatch::Environment.current
+      return new(environment) if environment.with_telemetry { Railwatch::TelemetryRecord.sqlite? }
+
+      puts "The railwatch_telemetry database is not SQLite; auto_vacuum does not apply."
+      nil
+    end
+
+    def initialize(environment)
+      @environment = environment
+    end
+
+    def report
+      with_telemetry do
+        <<~TEXT.chomp
+          Railwatch telemetry database
+            file    #{path}
+            size    #{human(file_bytes)}#{" + #{human(wal_bytes)} WAL" if wal_bytes.positive?}
+            mode    auto_vacuum=#{mode}
+            free    #{freelist} pages (#{human(freelist * page_size)}) on the freelist
+        TEXT
+      end
+    end
+
+    def advice
+      with_telemetry do
+        if mode != :incremental
+          "\nPruning cannot return space to the filesystem in this mode: every page it frees stays in this file.\n" \
+            "`bin/rails railwatch:vacuum` converts the database to incremental auto-vacuum with a full VACUUM. " \
+            "That rewrites all #{human(file_bytes)}, needs about that much free disk for the temporary copy, and " \
+            "holds the write lock for roughly #{estimate}. Telemetry written while it runs waits for it."
+        elsif freelist.positive?
+          "\nThe nightly prune returns up to #{human(Railwatch::PruneTelemetryJob::VACUUM_PAGES_PER_SLICE * Railwatch::PruneTelemetryJob::VACUUM_SLICES * page_size)} " \
+            "a night on its own. `bin/rails railwatch:vacuum` returns all #{human(freelist * page_size)} now."
+        else
+          "\nNothing on the freelist: pruning is already returning this database's space as it goes."
+        end
+      end
+    end
+
+    # The whole point of asking explicitly, so this one is not bounded the way
+    # the nightly prune is.
+    def reclaim!
+      with_telemetry do
+        before = disk_bytes
+        if mode == :incremental
+          Railwatch::TelemetryRecord.incremental_vacuum
+        else
+          puts "\nConverting to incremental auto-vacuum (full VACUUM, roughly #{estimate})..."
+          connection.execute("PRAGMA auto_vacuum = incremental")
+          connection.execute("VACUUM")
+        end
+        # A checkpoint that cannot finish says so in its result rather than
+        # raising: a reader still on an older snapshot holds the WAL open.
+        # Counting only the main file would then report the WAL's bytes as
+        # returned while they are still on the disk.
+        busy = checkpoint_busy?
+        [
+          "\nauto_vacuum=#{mode}, #{human(disk_bytes)} on disk including the WAL " \
+            "(#{human(before - disk_bytes)} returned), #{freelist} pages left on the freelist.",
+          busy ? "The WAL could not be truncated yet -- something is still reading it. " \
+                 "Its bytes come back at the next checkpoint; run this again if you want to watch it." : nil
+        ].compact.join("\n")
+      end
+    end
+
+    private
+      def with_telemetry(&) = @environment.with_telemetry(&)
+
+      def connection = Railwatch::TelemetryRecord.connection
+
+      def mode = Railwatch::TelemetryRecord.auto_vacuum_mode
+
+      def freelist = Railwatch::TelemetryRecord.freelist_pages
+
+      def page_size = Railwatch::TelemetryRecord.page_size
+
+      def path = Rails.root.join(Railwatch::TelemetryRecord.connection_db_config.database.to_s)
+
+      def file_bytes = File.exist?(path) ? File.size(path) : 0
+
+      # What this database actually occupies. In WAL mode the pages a VACUUM
+      # frees are not off the disk until the WAL is checkpointed, so the main
+      # file alone understates it -- and, right after a vacuum, flatters it.
+      def disk_bytes = file_bytes + wal_bytes
+
+      # `PRAGMA wal_checkpoint` answers with [busy, log_pages, checkpointed];
+      # a non-zero first column means it gave up rather than failed.
+      def checkpoint_busy?
+        connection.select_rows("PRAGMA wal_checkpoint(TRUNCATE)").dig(0, 0).to_i != 0
+      rescue StandardError
+        false
+      end
+
+      def wal_bytes = File.exist?("#{path}-wal") ? File.size("#{path}-wal") : 0
+
+      def estimate = ActiveSupport::Duration.build([ (file_bytes / VACUUM_BYTES_PER_SECOND.to_f).ceil, 1 ].max).inspect
+
+      def human(bytes) = ActiveSupport::NumberHelper.number_to_human_size(bytes)
+  end
+
   # Where this install's platform lives, derived from config.ingest_url:
   # railwatch:token and railwatch:mcp point at the same host the gem already
   # ships to, so a self-hosted app never gets told to visit railwatch.rebulk.com.
@@ -127,6 +251,25 @@ namespace :railwatch do
                    else "cannot check: #{pending}"
                    end, fatal: true)
       end
+      # An install created before 0.3.5 is in auto_vacuum=none, where the
+      # nightly prune's reclaim is a silent no-op and the file only grows.
+      # Nothing converts it on its own, because that needs a full VACUUM with
+      # the write lock held -- so the check people actually run is where it
+      # has to be said, rather than leaving them to notice the disk.
+      if Railwatch.config.local? && Railwatch::TelemetryRecord.sqlite?
+        vacuum_mode = begin
+          Railwatch::Environment.current.with_telemetry { Railwatch::TelemetryRecord.auto_vacuum_mode }
+        rescue StandardError => e
+          e.message
+        end
+        check.call(vacuum_mode == :incremental, "telemetry disk",
+                   case vacuum_mode
+                   when :incremental then "auto_vacuum=incremental; the nightly prune returns freed pages"
+                   when Symbol then "auto_vacuum=#{vacuum_mode}: pruned pages stay in the file " \
+                                    "(bin/rails railwatch:vacuum:status)"
+                   else "cannot check: #{vacuum_mode}"
+                   end)
+      end
       # The maintenance clock runs in web and worker processes, not in this
       # rake process, so what can be checked here is whether one has ticked.
       last_tick = begin
@@ -297,6 +440,24 @@ namespace :railwatch do
     puts "\nRailwatch is wired up."
   end
 
+  # Disk. `railwatch:vacuum:status` only reads; `railwatch:vacuum` is the
+  # one-time conversion a database created before EnableIncrementalVacuum
+  # needs, and it is a rake task rather than anything automatic because it
+  # rewrites the whole file with the write lock held.
+  desc "Report the telemetry database's size, free pages, and whether pruning can return them to the filesystem"
+  task "vacuum:status" => :environment do
+    disk = Railwatch::TelemetryDisk.open or next
+    puts disk.report
+    puts disk.advice
+  end
+
+  desc "Reclaim disk from the telemetry database. Converts it to incremental auto-vacuum if needed -- a full VACUUM, which locks the file"
+  task vacuum: :environment do
+    disk = Railwatch::TelemetryDisk.open or next
+    puts disk.report
+    puts disk.reclaim!
+  end
+
   namespace :export do
     desc "Show what the export queue is holding and whether it can send"
     task status: :environment do