@origintrail-official/dkg-node-ui 10.0.13 → 10.0.15

package/dist/db.js

@@ -1,8 +1,9 @@
 import Database from 'better-sqlite3';
 import { join } from 'node:path';
-import { RESPONSE_CACHE_BYTES, parseContextGraphJoinPolicyRecord, } from '@origintrail-official/dkg-core';
+import { RESPONSE_CACHE_BYTES, parseContextGraphJoinPolicyRecord, DEFAULT_SYNC_CHECKPOINT_TTL_MS, isValidSyncCheckpointEntry, transitionSyncCheckpointManifestOffset, transitionSyncCheckpointOffset, transitionSyncCheckpointResponderSession, withoutSyncCheckpointResponderSession, } from '@origintrail-official/dkg-core';
+import { RoutineLogRetention, installRoutineLogRetentionSchema, } from './routine-log-retention.js';
 export { SqliteChainEventCursorStore, SqliteContextGraphRegistryScanCursorStore, } from './chain-cursor-stores.js';
-const SCHEMA_VERSION = 31;
+export const SCHEMA_VERSION = 35;
 // Default operator retention. Lowered from 90 → 14 days on V15 (2026-05) after
 // a production incident in which the `logs` table + its FTS5 shadow tables
 // grew to ~9 GB on a 12-day-old node and corrupted the SQLite page (header
@@ -12,9 +13,9 @@ const SCHEMA_VERSION = 31;
 // operator-driven post-mortem, but the mainnet sync storm proved that time
 // retention alone cannot bound worst-case growth. Operators who want longer
 // retention can override via `setRetentionDays()`; the setting is persisted
-// in the `settings` table and re-read on next boot. Time alone is not a hard
-// size bound during a log storm, so routine info/debug rows also have a count
-// ceiling. Warning/error rows keep the full operator-selected time window.
+// in the `settings` table and re-read on next boot. The daemon no longer writes
+// routine logs to SQLite; the established count cap still bounds compatibility
+// writers while warning/error rows keep the operator-selected time window.
 const DEFAULT_RETENTION_DAYS = 14;
 const LEGACY_IMPLICIT_RETENTION_DAYS = 90;
 const DEFAULT_ROUTINE_LOG_ROW_CAP = 1_000_000;
@@ -104,17 +105,17 @@ export class DashboardDB {
     dataDir;
     retentionDays;
     explicitRetentionDays;
-    routineLogRowCap;
-    logVolumePruneBatchRows;
+    routineLogRetention;
     constructor(opts) {
         this.dataDir = opts.dataDir;
         this.explicitRetentionDays = opts.retentionDays !== undefined;
         this.retentionDays = opts.retentionDays ?? DEFAULT_RETENTION_DAYS;
-        this.routineLogRowCap = Math.max(0, Math.floor(opts.routineLogRowCap ?? DEFAULT_ROUTINE_LOG_ROW_CAP));
-        this.logVolumePruneBatchRows = Math.max(1, Math.floor(opts.logVolumePruneBatchRows ?? DEFAULT_LOG_VOLUME_PRUNE_BATCH_ROWS));
+        const routineLogRowCap = Math.max(0, Math.floor(opts.routineLogRowCap ?? DEFAULT_ROUTINE_LOG_ROW_CAP));
+        const logVolumePruneBatchRows = Math.max(1, Math.floor(opts.logVolumePruneBatchRows ?? DEFAULT_LOG_VOLUME_PRUNE_BATCH_ROWS));
         this._memoTtlMs = resolveCacheTtlMs(opts.cacheTtlMs);
         const dbPath = join(opts.dataDir, 'node-ui.db');
         this.db = new Database(dbPath);
+        this.routineLogRetention = new RoutineLogRetention(this.db, routineLogRowCap, logVolumePruneBatchRows);
         this.db.pragma('journal_mode = WAL');
         this.db.pragma('synchronous = NORMAL');
         // Cap the persisted WAL file. A PASSIVE autocheckpoint resets WAL
@@ -172,7 +173,7 @@ export class DashboardDB {
           );
       `);
         };
-        const ensureSyncCheckpointResponderSessionColumns = () => {
+        const ensureSyncCheckpointResumeColumns = () => {
             const table = this.db.prepare(`
         SELECT 1 AS found FROM sqlite_master
         WHERE type = 'table' AND name = 'sync_checkpoints'
@@ -187,6 +188,18 @@ export class DashboardDB {
             if (!columns.has('responder_session_expires_at')) {
                 this.db.exec(`ALTER TABLE sync_checkpoints ADD COLUMN responder_session_expires_at INTEGER;`);
             }
+            if (!columns.has('responder_session_offset')) {
+                this.db.exec(`ALTER TABLE sync_checkpoints ADD COLUMN responder_session_offset INTEGER;`);
+            }
+            if (!columns.has('manifest_digest')) {
+                this.db.exec(`ALTER TABLE sync_checkpoints ADD COLUMN manifest_digest TEXT;`);
+            }
+            if (!columns.has('manifest_prefix_digest')) {
+                this.db.exec(`ALTER TABLE sync_checkpoints ADD COLUMN manifest_prefix_digest TEXT;`);
+            }
+            if (!columns.has('terminal')) {
+                this.db.exec(`ALTER TABLE sync_checkpoints ADD COLUMN terminal INTEGER NOT NULL DEFAULT 0 CHECK (terminal IN (0, 1));`);
+            }
         };
         if (version > SCHEMA_VERSION)
             return;
@@ -194,8 +207,9 @@ export class DashboardDB {
             // Repair restored/development databases that carry the current version
             // but lost an idempotent schema adjunct.
             ensureJoinApprovalRepairMarker();
-            ensureSyncCheckpointResponderSessionColumns();
+            ensureSyncCheckpointResumeColumns();
             ensureJoinPolicyAuditCapTrigger();
+            installRoutineLogRetentionSchema(this.db);
             return;
         }
         if (version < 1) {
@@ -789,7 +803,13 @@ export class DashboardDB {
           key TEXT PRIMARY KEY,
           offset INTEGER NOT NULL CHECK (offset >= 0),
           updated_at INTEGER NOT NULL,
-          expires_at INTEGER NOT NULL
+          expires_at INTEGER NOT NULL,
+          responder_session_id TEXT,
+          responder_session_expires_at INTEGER,
+          responder_session_offset INTEGER CHECK (responder_session_offset >= 0),
+          manifest_digest TEXT,
+          manifest_prefix_digest TEXT,
+          terminal INTEGER NOT NULL DEFAULT 0 CHECK (terminal IN (0, 1))
         );
         CREATE INDEX IF NOT EXISTS idx_sync_checkpoints_expires_at
           ON sync_checkpoints(expires_at);
@@ -1023,7 +1043,7 @@ export class DashboardDB {
             // OFFSET pagination is scoped to an immutable responder row list. Keep
             // the opaque responder token beside the verified offset so a requester
             // restart can resume that same list instead of discarding durable work.
-            ensureSyncCheckpointResponderSessionColumns();
+            ensureSyncCheckpointResumeColumns();
         }
         if (version < 31) {
             this.db.exec(`
@@ -1043,6 +1063,49 @@ export class DashboardDB {
         );
       `);
         }
+        if (version < 32) {
+            this.db.exec(`
+        CREATE TABLE IF NOT EXISTS selected_vm_reconcile_cursors (
+          deployment_id TEXT NOT NULL CHECK (length(trim(deployment_id)) > 0),
+          context_graph_id TEXT NOT NULL,
+          on_chain_context_graph_id TEXT NOT NULL,
+          name_hash TEXT NOT NULL,
+          watermark INTEGER NOT NULL CHECK (watermark >= 0),
+          updated_at INTEGER NOT NULL,
+          PRIMARY KEY (deployment_id, context_graph_id, on_chain_context_graph_id)
+        );
+        CREATE INDEX IF NOT EXISTS idx_selected_vm_reconcile_cursor_cg
+          ON selected_vm_reconcile_cursors(deployment_id, context_graph_id);
+      `);
+        }
+        if (version < 33) {
+            // A durable DATA offset is reusable only with cryptographic proof of the
+            // canonical META generation and verified descriptor prefix that gave it
+            // meaning. Persist both digests beside the offset/session so production
+            // SQLite checkpoints have the same safety contract as the in-memory
+            // requester store and survive daemon restarts.
+            ensureSyncCheckpointResumeColumns();
+        }
+        if (version < 34) {
+            // The V33 manifest-bound continuation must not share a key with V32,
+            // which interpreted OFFSET as an unverified raw responder coordinate.
+            // Drop every pre-versioned durable DATA row before new code starts using
+            // the `|checkpoint:v2` namespace, so a later rollback also fails closed.
+            ensureSyncCheckpointResumeColumns();
+            this.db.prepare(`
+        DELETE FROM sync_checkpoints
+         WHERE key LIKE '%|durable|data%'
+           AND key NOT LIKE '%|durable|data|checkpoint:v2%'
+      `).run();
+        }
+        if (version < 35) {
+            // Retention probes used to find the (cap + 1)th newest routine row via
+            // OFFSET, synchronously scanning the million-row cap on every cleanup
+            // tick. Maintain the exact count transactionally and index only routine
+            // row ids, so overflow checks are O(1) and each prune touches at most one
+            // configured batch.
+            installRoutineLogRetentionSchema(this.db);
+        }
         this.db.pragma(`user_version = ${SCHEMA_VERSION}`);
         if (upgradedExistingDb && !this.explicitRetentionDays) {
             this.retentionDays = LEGACY_IMPLICIT_RETENTION_DAYS;
@@ -1059,14 +1122,7 @@ export class DashboardDB {
     }
     prune() {
         const cutoff = Date.now() - this.retentionDays * 86_400_000;
-        // Count total rows actually deleted across all DELETE statements.
-        // SQLite's per-statement change count is exposed via better-sqlite3's
-        // `Database.run().changes`, but `exec()` returns nothing — for a
-        // proper accounting we'd switch each statement to `prepare/run`. For
-        // the VACUUM gating decision below we only need to know whether
-        // *something* substantial was deleted, so we sample the only table
-        // that actually grows fast in practice: `logs`.
-        const logsDeleted = this.db.prepare(`DELETE FROM logs WHERE ts < ?`).run(cutoff).changes;
+        this.db.prepare(`DELETE FROM logs WHERE ts < ?`).run(cutoff);
         this.db.exec(`DELETE FROM metric_snapshots WHERE ts < ${cutoff}`);
         this.db.exec(`DELETE FROM operation_phases WHERE started_at < ${cutoff}`);
         this.db.exec(`DELETE FROM operations WHERE started_at < ${cutoff}`);
@@ -1098,32 +1154,32 @@ export class DashboardDB {
         const messengerCutoff = Date.now() - 24 * 60 * 60 * 1000;
         this.db.exec(`DELETE FROM message_idempotency WHERE ts < ${messengerCutoff}`);
         // Avoid rebuilding an oversized DB at startup while millions of routine
-        // rows are still live. The daemon trims that backlog in bounded batches;
-        // the final batch performs one compacting VACUUM. Databases already under
-        // the cap retain the established time-prune reclamation behaviour.
+        // rows are still live. The independent volume pruner trims only rows above
+        // the published count cap, in bounded batches, and performs the final
+        // compaction. Ambiguous rows below the cap remain untouched.
         if (!this.hasRoutineLogOverflow()) {
-            this.reclaimFreePagesIfNeeded(logsDeleted > LOGS_VACUUM_DELETE_THRESHOLD);
+            this.reclaimFreePagesIfNeeded(false);
         }
         // Return the WAL file itself to the OS. journal_size_limit bounds it
         // in steady state, but a TRUNCATE checkpoint here shrinks it promptly
-        // on the prune cadence (~6h) and immediately after the VACUUM above,
-        // which rewrites the whole DB through the WAL and momentarily grows
-        // it. Runs unconditionally — independent of the VACUUM gate — because
-        // an idle node still wants its -wal reclaimed.
+        // on the prune cadence (~6h) and after a cleanup VACUUM, which rewrites
+        // the whole DB through the WAL and momentarily grows it. Runs
+        // unconditionally because an idle node still wants its -wal reclaimed.
         this.truncateWal('prune');
     }
     /**
-     * Remove one bounded batch of the oldest routine (non-warning/error) logs.
-     * A count cap complements time retention: a high-rate sync storm can create
-     * millions of rows inside a single day, long before a 14-day cutoff applies.
+     * Remove one bounded batch of the oldest routine (non-warning/error) logs
+     * above the public count cap. The cap is the only ownership-neutral rule we
+     * can safely apply to pre-upgrade rows: historical daemon sink records and
+     * published compatibility API records have identical schemas.
      *
      * Deletion is deliberately incremental so an upgrade does not block node
      * startup on a multi-GB transaction. Once the backlog reaches the cap, one
      * VACUUM returns the accumulated free pages to the OS and the file shrinks.
      */
     pruneLogVolumeBatch() {
-        const overflowCutoff = this.routineLogOverflowCutoff();
-        if (overflowCutoff === null) {
+        const batch = this.routineLogRetention.pruneOverflowBatch();
+        if (!batch.hadOverflow) {
             const reclaim = this.reclaimFreePagesIfNeeded(false);
             if (!reclaim.reclaimPending)
                 this.truncateWal('log-volume prune');
@@ -1136,24 +1192,7 @@ export class DashboardDB {
                         : 'done',
             };
         }
-        const deleted = this.db.prepare(`
-      DELETE FROM logs
-      WHERE id IN (
-        SELECT id
-        FROM logs
-        WHERE id <= @cutoff
-          AND level NOT IN ('warn', 'error')
-        ORDER BY id ASC
-        LIMIT @batchRows
-      )
-    `).run({
-            cutoff: overflowCutoff,
-            batchRows: this.logVolumePruneBatchRows,
-        }).changes;
-        // If the batch filled, conservatively schedule another tick. An exact-size
-        // final batch costs one extra cheap probe before compaction, which is safer
-        // than running a second million-row count after every deletion.
-        const hasMore = deleted === this.logVolumePruneBatchRows;
+        const { deleted, hasMore } = batch;
         const reclaim = hasMore
             ? { compacted: false, reclaimPending: false }
             : this.reclaimFreePagesIfNeeded(deleted > LOGS_VACUUM_DELETE_THRESHOLD);
@@ -1170,18 +1209,8 @@ export class DashboardDB {
                         : 'done',
         };
     }
-    routineLogOverflowCutoff() {
-        const row = this.db.prepare(`
-      SELECT id
-      FROM logs
-      WHERE level NOT IN ('warn', 'error')
-      ORDER BY id DESC
-      LIMIT 1 OFFSET ?
-    `).get(this.routineLogRowCap);
-        return row?.id ?? null;
-    }
     hasRoutineLogOverflow() {
-        return this.routineLogOverflowCutoff() !== null;
+        return this.routineLogRetention.hasOverflow();
     }
     reclaimFreePagesIfNeeded(force) {
         // DELETE / DROP only marks pages reusable; VACUUM is what returns them to
@@ -1640,6 +1669,29 @@ export class DashboardDB {
     deleteVmReconcileNegativesForContextGraph(contextGraphId) {
         this.stmt('deleteVmReconcileNegativesForContextGraph', 'DELETE FROM vm_reconcile_negative_cache WHERE context_graph_id = ?').run(contextGraphId);
     }
+    upsertSelectedVmReconcileCursor(record) {
+        this.stmt('upsertSelectedVmReconcileCursor', `
+      INSERT INTO selected_vm_reconcile_cursors (
+        deployment_id, context_graph_id, on_chain_context_graph_id,
+        name_hash, watermark, updated_at
+      ) VALUES (
+        @deployment_id, @context_graph_id, @on_chain_context_graph_id,
+        @name_hash, @watermark, @updated_at
+      )
+      ON CONFLICT(deployment_id, context_graph_id, on_chain_context_graph_id) DO UPDATE SET
+        name_hash = excluded.name_hash,
+        watermark = excluded.watermark,
+        updated_at = excluded.updated_at
+    `).run(record);
+    }
+    getSelectedVmReconcileCursor(deploymentId, contextGraphId, onChainContextGraphId) {
+        return this.stmt('getSelectedVmReconcileCursor', `
+      SELECT * FROM selected_vm_reconcile_cursors
+      WHERE deployment_id = ?
+        AND context_graph_id = ?
+        AND on_chain_context_graph_id = ?
+    `).get(deploymentId, contextGraphId, onChainContextGraphId);
+    }
     // --- Phase F: chain-driven VM reconciliation telemetry ---
     /**
      * Persist one {@link ReplicationEvent} (best-effort sink target). `ordinal`,
@@ -2464,6 +2516,7 @@ export class DashboardDB {
             module: entry.module,
             message: entry.message,
         });
+        this.routineLogRetention.noteCommittedInsert(entry.level);
     }
     /**
      * Backwards-compatible DB-backed log search. V15 deliberately removed
@@ -2687,11 +2740,12 @@ export class DashboardDB {
         return rows.map((r) => r.id);
     }
     close() {
+        if (!this.db.open)
+            return;
         this.db.close();
     }
 }
 // --- Sync requester checkpoints (issue #1138 A3) ---
-const DEFAULT_SYNC_CHECKPOINT_TTL_MS = 24 * 60 * 60 * 1000;
 export class SqliteSyncCheckpointStore {
     db;
     clock;
@@ -2701,73 +2755,128 @@ export class SqliteSyncCheckpointStore {
         this.clock = options.clock ?? (() => Date.now());
         this.ttlMs = options.ttlMs ?? DEFAULT_SYNC_CHECKPOINT_TTL_MS;
     }
-    get(key) {
-        const now = this.clock();
+    readRow(key) {
         const row = this.db.prepare(`SELECT offset, updated_at, expires_at,
-              responder_session_id, responder_session_expires_at
+              responder_session_id, responder_session_expires_at, responder_session_offset,
+              manifest_digest, manifest_prefix_digest, terminal
          FROM sync_checkpoints WHERE key = ?`).get(key);
         if (!row)
             return undefined;
-        if (row.expires_at < now) {
-            this.delete(key);
-            return undefined;
-        }
-        const hasFreshResponderSession = Boolean(row.responder_session_id)
-            && Number.isSafeInteger(row.responder_session_expires_at)
-            && (row.responder_session_expires_at ?? 0) > now;
-        if (row.responder_session_id && !hasFreshResponderSession) {
-            this.clearResponderSession(key);
-        }
         return {
             offset: row.offset,
             updatedAtMs: row.updated_at,
             expiresAtMs: row.expires_at,
-            ...(hasFreshResponderSession
-                ? {
-                    responderSessionId: row.responder_session_id,
-                    responderSessionExpiresAtMs: row.responder_session_expires_at,
-                }
+            ...(row.terminal === 1 ? { terminal: true } : {}),
+            ...(row.manifest_digest ? { manifestDigest: row.manifest_digest } : {}),
+            ...(row.manifest_prefix_digest
+                ? { manifestPrefixDigest: row.manifest_prefix_digest }
+                : {}),
+            // Preserve each nullable session column independently so the shared
+            // validator can distinguish an absent session from a torn/malformed
+            // persisted session. Collapsing a partial row to no session fields would
+            // turn corrupt durable state into an apparently valid ordinary offset.
+            ...(row.responder_session_id !== null
+                ? { responderSessionId: row.responder_session_id }
+                : {}),
+            ...(row.responder_session_expires_at !== null
+                ? { responderSessionExpiresAtMs: row.responder_session_expires_at }
+                : {}),
+            ...(row.responder_session_offset !== null
+                ? { responderSessionOffset: row.responder_session_offset }
                 : {}),
         };
     }
-    set(key, value, nowMs = this.clock()) {
-        if (!Number.isSafeInteger(value) || value < 0) {
-            throw new Error(`Invalid sync checkpoint offset for ${key}: ${value}`);
+    get(key, now = this.clock()) {
+        const entry = this.readRow(key);
+        if (!entry)
+            return undefined;
+        if (!isValidSyncCheckpointEntry(entry) || entry.expiresAtMs < now) {
+            this.delete(key);
+            return undefined;
+        }
+        if (entry.responderSessionId
+            && (entry.responderSessionExpiresAtMs ?? 0) <= now) {
+            const withoutExpiredSession = withoutSyncCheckpointResponderSession(entry);
+            this.writeEntry(key, withoutExpiredSession);
+            return withoutExpiredSession;
         }
+        return entry;
+    }
+    writeEntry(key, entry) {
         this.db.prepare(`
-      INSERT INTO sync_checkpoints (key, offset, updated_at, expires_at)
-      VALUES (?, ?, ?, ?)
+      INSERT INTO sync_checkpoints (
+        key, offset, updated_at, expires_at,
+        responder_session_id, responder_session_expires_at, responder_session_offset,
+        manifest_digest, manifest_prefix_digest, terminal
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(key) DO UPDATE SET
         offset = excluded.offset,
         updated_at = excluded.updated_at,
-        expires_at = excluded.expires_at
-    `).run(key, value, nowMs, nowMs + this.ttlMs);
+        expires_at = excluded.expires_at,
+        responder_session_id = excluded.responder_session_id,
+        responder_session_expires_at = excluded.responder_session_expires_at,
+        responder_session_offset = excluded.responder_session_offset,
+        manifest_digest = excluded.manifest_digest,
+        manifest_prefix_digest = excluded.manifest_prefix_digest,
+        terminal = excluded.terminal
+    `).run(key, entry.offset, entry.updatedAtMs, entry.expiresAtMs, entry.responderSessionId ?? null, entry.responderSessionExpiresAtMs ?? null, entry.responderSessionOffset ?? null, entry.manifestDigest ?? null, entry.manifestPrefixDigest ?? null, entry.terminal ? 1 : 0);
+    }
+    set(key, value, nowMs = this.clock(), responderSessionOffset) {
+        const transition = this.db.transaction(() => {
+            this.writeEntry(key, transitionSyncCheckpointOffset({
+                key,
+                existing: this.get(key, nowMs),
+                value,
+                nowMs,
+                ttlMs: this.ttlMs,
+                responderSessionOffset,
+            }));
+        });
+        transition();
     }
-    setResponderSession(key, sessionId, expiresAtMs, nowMs = this.clock()) {
-        if (!sessionId || !Number.isSafeInteger(expiresAtMs)) {
-            throw new Error(`Invalid sync responder session for ${key}`);
-        }
+    setManifestBoundOffset(key, value, manifestDigest, nowMs = this.clock(), manifestPrefixDigest, terminal = false, responderSessionOffset) {
+        const transition = this.db.transaction(() => {
+            this.writeEntry(key, transitionSyncCheckpointManifestOffset({
+                key,
+                existing: this.get(key, nowMs),
+                value,
+                manifestDigest,
+                nowMs,
+                ttlMs: this.ttlMs,
+                manifestPrefixDigest,
+                terminal,
+                responderSessionOffset,
+            }));
+        });
+        transition();
+    }
+    setResponderSession(key, sessionId, expiresAtMs, nowMs = this.clock(), manifestDigest, manifestPrefixDigest, responderSessionOffset) {
         if (expiresAtMs <= nowMs) {
             this.clearResponderSession(key);
             return;
         }
-        this.db.prepare(`
-      INSERT INTO sync_checkpoints (
-        key, offset, updated_at, expires_at,
-        responder_session_id, responder_session_expires_at
-      ) VALUES (?, 0, ?, ?, ?, ?)
-      ON CONFLICT(key) DO UPDATE SET
-        responder_session_id = excluded.responder_session_id,
-        responder_session_expires_at = excluded.responder_session_expires_at
-    `).run(key, nowMs, nowMs + this.ttlMs, sessionId, expiresAtMs);
+        const transition = this.db.transaction(() => {
+            this.writeEntry(key, transitionSyncCheckpointResponderSession({
+                key,
+                existing: this.get(key, nowMs),
+                sessionId,
+                expiresAtMs,
+                nowMs,
+                ttlMs: this.ttlMs,
+                manifestDigest,
+                manifestPrefixDigest,
+                responderSessionOffset,
+            }));
+        });
+        transition();
     }
     clearResponderSession(key) {
-        this.db.prepare(`
-      UPDATE sync_checkpoints
-         SET responder_session_id = NULL,
-             responder_session_expires_at = NULL
-       WHERE key = ?
-    `).run(key);
+        const transition = this.db.transaction(() => {
+            const existing = this.readRow(key);
+            if (existing)
+                this.writeEntry(key, withoutSyncCheckpointResponderSession(existing));
+        });
+        transition();
     }
     delete(key) {
         this.db.prepare(`DELETE FROM sync_checkpoints WHERE key = ?`).run(key);