@shawnstack/quickforge 1.7.10 → 1.7.12

package/server/sqlite/database.mjs

@@ -1,3 +1,4 @@
+import { readFileSync, renameSync, statSync, writeFileSync } from 'node:fs'
 import { mkdir } from 'node:fs/promises'
 import os from 'node:os'
 import path from 'node:path'
@@ -7,10 +8,28 @@ import { applySqliteMigrations, inspectSqliteMigrationState, SQLITE_MIGRATIONS }
 
 export const SQLITE_BUSY_TIMEOUT_MS = 5_000
 export const SQLITE_JOURNAL_MODE = 'wal'
-export const SQLITE_SYNCHRONOUS = 1
+export const SQLITE_SYNCHRONOUS = 2
+
+// Startup quick_check tax optimization. All four storage domains share one
+// quickforge.sqlite3 file, and `PRAGMA quick_check` scans the whole file, so
+// four per-domain startup checks used to pay the full-scan cost four times
+// (~30s on a 3GB production library). One gated scan per database per process
+// (sharedQuickCheckCache) plus a cross-restart marker file reduce it to at
+// most one real scan per database per 7 days.
+//
+// Safety argument: WAL + synchronous=FULL make committed transactions immune
+// to torn writes (a crash either replays or rolls back the WAL). quick_check
+// therefore only guards against bit-rot / filesystem-level corruption, which
+// evolves on disk timescales, not per startup; the 7-day cadence plus the
+// manual force entry points (maintenance verify-session-integrity endpoint,
+// QUICKFORGE_SQLITE_QUICK_CHECK=force) keep the blind window bounded. When a
+// quick_check is skipped, all other SQL-level count/join checks still run.
+export const SQLITE_QUICK_CHECK_MARKER_FILENAME = 'quickforge-quick-check.marker.json'
+export const SQLITE_QUICK_CHECK_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000
 
 const sqliteLogger = logger.child({ component: 'sqlite' })
 const TRANSACTION_MODES = new Set(['deferred', 'immediate', 'exclusive'])
+const sharedQuickCheckCache = new Map()
 let initializationPromise = null
 let openState = null
 let savepointCounter = 0
@@ -65,22 +84,138 @@ function verifyPragmas(database) {
 
 function configurePragmas(database) {
   database.exec(`PRAGMA busy_timeout = ${SQLITE_BUSY_TIMEOUT_MS}`)
+  // Session-domain deletes reclaim disk pages through incremental_vacuum (see
+  // the session-state repository), which requires auto_vacuum = INCREMENTAL.
+  // The flag only takes effect on a freshly created (empty) database; an
+  // existing library keeps its current layout until a full VACUUM rebuilds
+  // it, so applying it here is best-effort for old databases. PRAGMA cannot
+  // run inside a transaction — this executes before applySqliteMigrations
+  // opens its BEGIN IMMEDIATE — and it must come AFTER busy_timeout: during
+  // a multi-process first initialization another worker can already hold the
+  // fresh database inside its migration transaction, and without the busy
+  // timeout this pragma would fail immediately with SQLITE_BUSY.
+  database.exec('PRAGMA auto_vacuum = INCREMENTAL')
   database.exec('PRAGMA foreign_keys = ON')
   database.exec('PRAGMA journal_mode = WAL')
-  database.exec('PRAGMA synchronous = NORMAL')
+  // FULL (not NORMAL): decision finalized in docs/architecture/sqlite-storage-foundation.zh-CN.md §3.1
+  // — measured cost is ~0.46 ms/op on the save hot path, in exchange for zero committed-transaction
+  // loss on OS crash/power loss.
+  database.exec('PRAGMA synchronous = FULL')
   return verifyPragmas(database)
 }
 
+function quickCheckMarkerPath(databasePath) {
+  return path.join(path.dirname(databasePath), SQLITE_QUICK_CHECK_MARKER_FILENAME)
+}
+
+// Resolve the main database file path from any storage surface (raw
+// DatabaseSync, the controlled handle, or test duck-typed handles). Returns
+// null for in-memory databases, where no marker is possible and every call
+// runs a real scan.
+function resolveQuickCheckDatabasePath(database) {
+  try {
+    const rows = database.prepare('PRAGMA database_list').all()
+    const main = Array.isArray(rows) ? rows.find((row) => row && row.name === 'main') : null
+    const file = typeof main?.file === 'string' ? main.file.trim() : ''
+    return file ? path.resolve(file) : null
+  } catch {
+    return null
+  }
+}
+
+// Marker read failures (missing, unreadable, malformed) are treated as "no
+// marker": the caller falls through to a real scan.
+function readQuickCheckMarker(databasePath) {
+  try {
+    const marker = JSON.parse(readFileSync(quickCheckMarkerPath(databasePath), 'utf8'))
+    const lastOkAt = Number(marker?.lastOkAt)
+    return Number.isFinite(lastOkAt) ? { ...marker, lastOkAt } : null
+  } catch {
+    return null
+  }
+}
+
+// Best-effort atomic marker write: failures only warn, never affect the
+// quick_check verdict.
+function writeQuickCheckMarker(databasePath, okAt, database) {
+  try {
+    const markerPath = quickCheckMarkerPath(databasePath)
+    const temporaryPath = `${markerPath}.${process.pid}.tmp`
+    let sqliteVersion = null
+    try {
+      sqliteVersion = String(database.prepare('SELECT sqlite_version() AS version').get().version)
+    } catch { /* Optional diagnostic field. */ }
+    let databaseBytes = null
+    try {
+      databaseBytes = statSync(databasePath).size
+    } catch { /* Optional diagnostic field. */ }
+    writeFileSync(temporaryPath, `${JSON.stringify({ app: 'quickforge', lastOkAt: okAt, sqliteVersion, databaseBytes })}\n`, 'utf8')
+    renameSync(temporaryPath, markerPath)
+  } catch (error) {
+    sqliteLogger.warn('SQLite quick_check marker write failed', {
+      error: error instanceof Error ? error.message : String(error),
+    })
+  }
+}
+
+/**
+ * Shared startup `PRAGMA quick_check` gate (see the block comment above for
+ * the rationale and safety argument). Semantics:
+ * - In-process: one successful scan per database file path is reused for the
+ *   process lifetime (unless `maxAgeMs` elapses or `resetSharedQuickCheckCache`
+ *   is called); `force` skips this reuse but still refreshes it after success.
+ * - Cross-restart: a marker file next to the database (`lastOkAt`) skips the
+ *   real scan while younger than `maxAgeMs` (default 7 days). Missing or
+ *   unreadable markers are treated as absent.
+ * - On a real scan success the marker is rewritten (atomic, best-effort).
+ * - On failure the quick_check error is re-thrown as-is: no marker is written
+ *   and the process cache never caches failures, so the next call scans again.
+ * - QUICKFORGE_SQLITE_QUICK_CHECK=force forces a real scan every call.
+ * Returns `{ ok: true, skipped: null | 'process' | 'marker', lastOkAt? }`.
+ */
+export function runSharedSqliteQuickCheck(database, {
+  force = false, maxAgeMs = SQLITE_QUICK_CHECK_MAX_AGE_MS, now = Date.now, writeMarker = true, databasePath = null,
+} = {}) {
+  const forced = force === true || process.env.QUICKFORGE_SQLITE_QUICK_CHECK === 'force'
+  const resolvedPath = databasePath ? path.resolve(databasePath) : resolveQuickCheckDatabasePath(database)
+  if (!forced && resolvedPath) {
+    const cached = sharedQuickCheckCache.get(resolvedPath)
+    if (cached && now() - cached.okAt < maxAgeMs) {
+      return { ok: true, skipped: 'process', lastOkAt: cached.okAt }
+    }
+    const marker = readQuickCheckMarker(resolvedPath)
+    if (marker && now() - marker.lastOkAt < maxAgeMs) {
+      sharedQuickCheckCache.set(resolvedPath, { okAt: marker.lastOkAt })
+      return { ok: true, skipped: 'marker', lastOkAt: marker.lastOkAt }
+    }
+  }
+  const result = database.prepare('PRAGMA quick_check').all().map((row) => row.quick_check)
+  if (result.length !== 1 || result[0] !== 'ok') {
+    throw new Error(`SQLite quick_check failed: ${result.join(', ')}`)
+  }
+  const okAt = now()
+  if (resolvedPath) sharedQuickCheckCache.set(resolvedPath, { okAt })
+  if (writeMarker !== false && resolvedPath) {
+    writeQuickCheckMarker(resolvedPath, okAt, database)
+  }
+  return { ok: true, skipped: null, lastOkAt: okAt }
+}
+
+/** Test helper: clears the in-process quick_check cache, marker files stay. */
+export function resetSharedQuickCheckCache() {
+  sharedQuickCheckCache.clear()
+}
+
 function publicHealth(state, { quickCheck = false } = {}) {
   const alive = Number(state.database.prepare('SELECT 1 AS ok').get().ok) === 1
   const pragmas = verifyPragmas(state.database)
   const migration = inspectSqliteMigrationState(state.database, { migrations: state.migrations })
   let quickCheckResult = null
   if (quickCheck) {
-    quickCheckResult = state.database.prepare('PRAGMA quick_check').all().map((row) => row.quick_check)
-    if (quickCheckResult.length !== 1 || quickCheckResult[0] !== 'ok') {
-      throw new Error(`SQLite quick_check failed: ${quickCheckResult.join(', ')}`)
-    }
+    // Shared gate: process-level dedupe + marker-based cadence; throws on
+    // failure exactly like the previous inline PRAGMA quick_check did.
+    runSharedSqliteQuickCheck(state.database, { databasePath: state.databasePath })
+    quickCheckResult = ['ok']
   }
   return {
     ok: alive && migration.consistent,
@@ -91,7 +226,7 @@ function publicHealth(state, { quickCheck = false } = {}) {
     journalMode: pragmas.journalMode,
     busyTimeout: pragmas.busyTimeout,
     foreignKeys: pragmas.foreignKeys,
-    synchronous: 'normal',
+    synchronous: 'full',
     ...(quickCheck ? { quickCheck: quickCheckResult[0] } : {}),
   }
 }

package/server/sqlite/lan-access-repository.mjs

@@ -1,5 +1,5 @@
 import { createHash } from 'node:crypto'
-import { getSqliteStorage } from './database.mjs'
+import { getSqliteStorage, runSharedSqliteQuickCheck } from './database.mjs'
 import { createRandomToken, safeHashEqual, sha256Base64Url } from '../utils/password-auth.mjs'
 
 // F11 LAN access storage: `lan_access_state` (single config row with a fixed
@@ -544,10 +544,12 @@ export function createLanAccessRepository(storageHandle, { now = () => new Date(
     return exportSnapshot().digest
   }
 
-  function verifyIntegrity({ quickCheck = false } = {}) {
+  function verifyIntegrity({ quickCheck = false, forceQuickCheck = false } = {}) {
     if (quickCheck) {
-      const result = storage.prepare('PRAGMA quick_check').all().map((row) => row.quick_check)
-      if (result.length !== 1 || result[0] !== 'ok') throw new Error(`SQLite quick_check failed: ${result.join(', ')}`)
+      // Shared quick_check gate (process cache + marker cadence) — see
+      // runSharedSqliteQuickCheck in database.mjs; `forceQuickCheck` is the
+      // manual-maintenance escape hatch that always runs a real scan.
+      runSharedSqliteQuickCheck(storage, { force: forceQuickCheck === true })
     }
     const row = currentConfigRow(storage)
     let invalidRecords = 0

package/server/sqlite/migrations.mjs

@@ -460,6 +460,115 @@ export const SQLITE_MIGRATIONS = Object.freeze([
       `)
     },
   }),
+  Object.freeze({
+    // Root cause: session_states is WITHOUT ROWID, so the PK b-tree IS the
+    // table and stores the multi-MB state_json payload inline. metadata_json /
+    // metadata_digest sit AFTER state_json in the record, so every metadata
+    // read must walk the preceding state_json overflow-page chain — an
+    // implicit full-database read (2.93GB cold DB: ~45s for a metadata-only
+    // projection). This covering index carries the metadata columns next to
+    // the PK prefix, making metadata reads index-only (KB-sized entries); the
+    // planner picks it up automatically with zero query changes.
+    // Note: CREATE INDEX scans the table once — on first upgrade of a large
+    // existing database this migration is a one-time cost paid inside
+    // initializeSqliteStorage, before the server starts listening.
+    version: 10,
+    name: 'session_states_metadata_covering_index',
+    up(database) {
+      database.exec(`
+        CREATE INDEX session_states_metadata_cover_idx
+          ON session_states (scope, project_id, session_id, metadata_json, metadata_digest);
+      `)
+    },
+  }),
+  Object.freeze({
+    // v11 session-domain rebuild (session storage v2): one small `sessions`
+    // row per session (no giant state_json column — the multi-GB bloat root
+    // cause), messages always stored row-per-message in `session_messages`
+    // (append-only, no inline/split dual mode), deletes tombstone into
+    // `session_tombstones` and space is reclaimed through incremental_vacuum
+    // (auto_vacuum=INCREMENTAL, set in database.mjs). The JSON mirror outbox
+    // chain, the session_index derived table and the session_storage_state
+    // phase machine are retired with this schema. Old session-domain tables
+    // are RENAMED (not dropped) as a rollback safety net; the maintenance
+    // lock and the share/lan-access/scheduled-runs domains are untouched.
+    // A fresh database runs v1..v11 sequentially, so every renamed table
+    // necessarily exists by the time v11 executes. Renamed tables keep their
+    // old index names (index names are schema-global), so the new tables use
+    // the idx_* naming convention to avoid collisions.
+    version: 11,
+    name: 'session_state_v2_storage',
+    up(database) {
+      database.exec(`
+        ALTER TABLE session_states RENAME TO session_states_v10_backup;
+        ALTER TABLE session_messages RENAME TO session_messages_v10_backup;
+        ALTER TABLE session_index RENAME TO session_index_v10_backup;
+        ALTER TABLE session_state_tombstones RENAME TO session_state_tombstones_v10_backup;
+        ALTER TABLE session_json_mirror_queue RENAME TO session_json_mirror_queue_v10_backup;
+        ALTER TABLE session_storage_state RENAME TO session_storage_state_v10_backup;
+
+        CREATE TABLE sessions (
+          scope TEXT NOT NULL CHECK (scope IN ('global', 'project')),
+          project_id TEXT NOT NULL,
+          session_id TEXT NOT NULL CHECK (length(trim(session_id)) > 0),
+          title TEXT NOT NULL DEFAULT '',
+          title_source TEXT,
+          created_at TEXT NOT NULL CHECK (length(trim(created_at)) > 0),
+          updated_at TEXT NOT NULL CHECK (length(trim(updated_at)) > 0),
+          message_count INTEGER NOT NULL DEFAULT 0 CHECK (typeof(message_count) = 'integer' AND message_count >= 0),
+          state_version INTEGER NOT NULL DEFAULT 0 CHECK (typeof(state_version) = 'integer' AND state_version >= 0),
+          harness TEXT,
+          task_status TEXT,
+          archived_at TEXT,
+          pinned_at TEXT,
+          body_json TEXT NOT NULL CHECK (json_valid(body_json) AND json_type(body_json) = 'object'),
+          meta_json TEXT NOT NULL CHECK (json_valid(meta_json) AND json_type(meta_json) = 'object'),
+          revision INTEGER NOT NULL CHECK (typeof(revision) = 'integer' AND revision > 0),
+          updated_at_ms INTEGER NOT NULL CHECK (typeof(updated_at_ms) = 'integer' AND updated_at_ms >= 0),
+          PRIMARY KEY (scope, project_id, session_id),
+          CHECK (
+            (scope = 'global' AND project_id = '') OR
+            (scope = 'project' AND length(trim(project_id)) > 0)
+          )
+        );
+
+        CREATE INDEX idx_sessions_list ON sessions (scope, project_id, archived_at, updated_at DESC);
+        CREATE INDEX idx_sessions_pinned ON sessions (scope, project_id, updated_at DESC) WHERE pinned_at IS NOT NULL;
+        CREATE INDEX idx_sessions_session_id ON sessions (session_id);
+
+        CREATE TABLE session_messages (
+          scope TEXT NOT NULL CHECK (scope IN ('global', 'project')),
+          project_id TEXT NOT NULL,
+          session_id TEXT NOT NULL CHECK (length(trim(session_id)) > 0),
+          seq INTEGER NOT NULL CHECK (typeof(seq) = 'integer' AND seq >= 0),
+          message_id TEXT CHECK (message_id IS NULL OR length(trim(message_id)) > 0),
+          message_json TEXT NOT NULL CHECK (json_valid(message_json) AND json_type(message_json) = 'object'),
+          message_digest TEXT NOT NULL CHECK (length(message_digest) = 64 AND message_digest NOT GLOB '*[^0-9a-f]*'),
+          PRIMARY KEY (scope, project_id, session_id, seq),
+          UNIQUE (scope, project_id, session_id, message_id),
+          FOREIGN KEY (scope, project_id, session_id) REFERENCES sessions (scope, project_id, session_id) ON DELETE CASCADE,
+          CHECK (
+            (scope = 'global' AND project_id = '') OR
+            (scope = 'project' AND length(trim(project_id)) > 0)
+          )
+        );
+
+        CREATE INDEX idx_session_messages_session_id ON session_messages (session_id);
+
+        CREATE TABLE session_tombstones (
+          scope TEXT NOT NULL CHECK (scope IN ('global', 'project')),
+          project_id TEXT NOT NULL,
+          session_id TEXT NOT NULL CHECK (length(trim(session_id)) > 0),
+          deleted_at INTEGER NOT NULL CHECK (typeof(deleted_at) = 'integer' AND deleted_at >= 0),
+          PRIMARY KEY (scope, project_id, session_id),
+          CHECK (
+            (scope = 'global' AND project_id = '') OR
+            (scope = 'project' AND length(trim(project_id)) > 0)
+          )
+        ) WITHOUT ROWID;
+      `)
+    },
+  }),
 ])
 
 function readUserVersion(database) {

package/server/sqlite/scheduled-task-runs-repository.mjs

@@ -314,8 +314,10 @@ export function createScheduledTaskRunsRepository(storageHandle, options = {}) {
     return database.prepare('SELECT * FROM scheduled_task_runs WHERE task_id = ? AND id = ?').get(taskId, runId)
   }
 
-  function insert(database, value) {
-    database.prepare(`INSERT INTO scheduled_task_runs (${INSERT_COLUMNS}) VALUES (${INSERT_PLACEHOLDERS})`).run(...insertParameters(value))
+  // The optional statement lets the cutover replaceAll reuse one precompiled insert.
+  function insert(database, value, statement = null) {
+    const insertStatement = statement ?? database.prepare(`INSERT INTO scheduled_task_runs (${INSERT_COLUMNS}) VALUES (${INSERT_PLACEHOLDERS})`)
+    insertStatement.run(...insertParameters(value))
   }
 
   function upsert(database, value) {
@@ -392,7 +394,8 @@ export function createScheduledTaskRunsRepository(storageHandle, options = {}) {
       }
       return storage.transaction((database) => {
         database.prepare('DELETE FROM scheduled_task_runs').run()
-        for (const value of values) insert(database, value)
+        const insertStatement = database.prepare(`INSERT INTO scheduled_task_runs (${INSERT_COLUMNS}) VALUES (${INSERT_PLACEHOLDERS})`)
+        for (const value of values) insert(database, value, insertStatement)
         return values.length
       })
     },