@moretea-labs/forge 1.7.2 → 1.8.1

package/src/runtime/control-plane/persistence/sqlite-store.ts

@@ -1,5 +1,6 @@
+import { createHash } from 'node:crypto';
 import { createRequire } from 'node:module';
-import { copyFileSync, existsSync, mkdirSync, renameSync, unlinkSync } from 'fs';
+import { closeSync, copyFileSync, existsSync, mkdirSync, openSync, readSync, renameSync, statSync, unlinkSync } from 'fs';
 import { dirname, join } from 'path';
 import { durableControllerHome } from '../../../cli/repositories/controller-home';
 
@@ -15,6 +16,8 @@ interface SqliteStatement {
   get(...parameters: unknown[]): unknown;
   all(...parameters: unknown[]): unknown[];
   run(...parameters: unknown[]): unknown;
+  /** Bun statements retain native resources until finalized; Node's StatementSync does not expose this method. */
+  finalize?(): void;
 }
 
 export interface SqliteDatabase {
@@ -59,10 +62,114 @@ export interface ControlPlaneDatabaseInspection {
   orphanRecordCount: number;
 }
 
+/**
+ * Ephemeral database-wide change fence. SQLite data_version changes whenever a
+ * different connection commits to the same database, including domain tables
+ * that do not participate in control_plane_audit.
+ */
+export interface ControlPlaneDatabaseChangeObserver {
+  path: string;
+  dataVersion(): number;
+  close(): void;
+}
+
+const CONTROL_PLANE_DATABASE_HASH_CHUNK_BYTES = 1024 * 1024;
+
+/** Full snapshot identity without allocating the whole SQLite file. */
+export function sha256ControlPlaneDatabaseFile(path: string): string {
+  const descriptor = openSync(path, 'r');
+  const hash = createHash('sha256');
+  const buffer = Buffer.allocUnsafe(CONTROL_PLANE_DATABASE_HASH_CHUNK_BYTES);
+  try {
+    for (;;) {
+      const bytesRead = readSync(descriptor, buffer, 0, buffer.length, null);
+      if (bytesRead === 0) break;
+      hash.update(buffer.subarray(0, bytesRead));
+    }
+    return hash.digest('hex');
+  } finally {
+    closeSync(descriptor);
+  }
+}
+
+export const CONTROL_PLANE_SQLITE_MAINTENANCE_POLICY_VERSION = 'control-plane-sqlite-maintenance-v1' as const;
+export const DEFAULT_CONTROL_PLANE_SQLITE_VACUUM_MIN_RECLAIMABLE_BYTES = 64 * 1024 * 1024;
+export const DEFAULT_CONTROL_PLANE_SQLITE_VACUUM_MIN_RECLAIMABLE_RATIO = 0.20;
+
+export interface ControlPlaneDatabaseMaintenanceOptions {
+  minimumReclaimableBytes?: number;
+  minimumReclaimableRatio?: number;
+  allowVacuum?: boolean;
+}
+
+export interface ControlPlaneDatabaseMaintenanceReport {
+  policyVersion: typeof CONTROL_PLANE_SQLITE_MAINTENANCE_POLICY_VERSION;
+  path: string;
+  checkpointMode: 'passive';
+  checkpointed: boolean;
+  pageSize: number;
+  pageCountBefore: number;
+  freePageCountBefore: number;
+  reclaimableBytesBefore: number;
+  reclaimableRatioBefore: number;
+  vacuumEligible: boolean;
+  vacuumAttempted: boolean;
+  vacuumed: boolean;
+  pageCountAfter: number;
+  freePageCountAfter: number;
+  reclaimableBytesAfter: number;
+  reclaimableRatioAfter: number;
+  reclaimedBytes: number;
+  skippedReason?: 'database_missing' | 'below_threshold' | 'vacuum_disabled' | 'database_busy';
+}
+
 export const CONTROL_PLANE_SCHEMA_VERSION = 1;
 const DATABASE_FILE = 'control-plane.sqlite';
 const require = createRequire(import.meta.url);
 
+interface ReusableReadDatabase {
+  database: SqliteDatabase;
+  fileIdentity: string;
+}
+
+const reusableReadDatabasePaths = new Set<string>();
+const reusableReadDatabases = new Map<string, ReusableReadDatabase>();
+
+function databaseFileIdentity(path: string): string | undefined {
+  try {
+    const stat = statSync(path);
+    // Content changes must not invalidate a WAL reader. Identity changes only
+    // when the live database file itself is replaced.
+    return stat.ino
+      ? `${stat.dev}:${stat.ino}`
+      : `${stat.dev}:birth:${stat.birthtimeMs}`;
+  } catch {
+    return undefined;
+  }
+}
+
+function closeReusableReadDatabase(path: string): void {
+  const cached = reusableReadDatabases.get(path);
+  if (!cached) return;
+  reusableReadDatabases.delete(path);
+  cached.database.close();
+}
+
+/**
+ * Opt in only for the long-lived Canonical Runtime. CLI, tests, and short-lived
+ * workers retain one-open-per-operation semantics so process exit and temporary
+ * Controller Home cleanup never depend on hidden SQLite handles.
+ */
+export function enableControlPlaneReadConnectionReuse(controllerHome: string): void {
+  reusableReadDatabasePaths.add(controlPlaneDatabasePath(controllerHome));
+}
+
+export function disableControlPlaneReadConnectionReuse(controllerHome: string): void {
+  const path = controlPlaneDatabasePath(controllerHome);
+  reusableReadDatabasePaths.delete(path);
+  closeReusableReadDatabase(path);
+}
+
 function now(): string {
   return new Date().toISOString();
 }
@@ -98,9 +205,28 @@ function scalar(row: unknown): unknown {
   return Object.values(row as Record<string, unknown>)[0];
 }
 
+/**
+ * Keep native statement lifetime inside the SQLite adapter. Bun retains a
+ * Windows file handle for each unfinalized statement even after db.close(),
+ * while Node's StatementSync has no explicit finalizer. The optional call
+ * gives both runtimes identical Controller Home release semantics.
+ */
+function withSqliteStatement<T>(
+  database: SqliteDatabase,
+  sql: string,
+  operation: (statement: SqliteStatement) => T,
+): T {
+  const statement = database.prepare(sql);
+  try {
+    return operation(statement);
+  } finally {
+    statement.finalize?.();
+  }
+}
+
 function assertDatabaseIntegrity(database: SqliteDatabase, path: string): void {
   try {
-    const result = scalar(database.prepare('PRAGMA quick_check').get());
+    const result = withSqliteStatement(database, 'PRAGMA quick_check', (statement) => scalar(statement.get()));
     if (result !== 'ok') throw new Error(String(result ?? 'missing quick_check result'));
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
@@ -110,13 +236,21 @@ function assertDatabaseIntegrity(database: SqliteDatabase, path: string): void {
 }
 
 function tableExists(database: SqliteDatabase, table: string): boolean {
-  const row = database.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?").get(table) as { name?: string } | undefined;
+  const row = withSqliteStatement(
+    database,
+    "SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
+    (statement) => statement.get(table) as { name?: string } | undefined,
+  );
   return row?.name === table;
 }
 
 function currentSchemaVersion(database: SqliteDatabase): number | undefined {
   if (!tableExists(database, 'control_plane_schema')) return undefined;
-  const value = scalar(database.prepare('SELECT MAX(version) AS version FROM control_plane_schema').get());
+  const value = withSqliteStatement(
+    database,
+    'SELECT MAX(version) AS version FROM control_plane_schema',
+    (statement) => scalar(statement.get()),
+  );
   return typeof value === 'number' && Number.isInteger(value) ? value : undefined;
 }
 
@@ -163,7 +297,11 @@ function initializeSchema(database: SqliteDatabase): void {
     CREATE INDEX IF NOT EXISTS control_plane_audit_lookup
       ON control_plane_audit (namespace, scope, record_key, revision);
   `);
-  database.prepare('INSERT OR IGNORE INTO control_plane_schema (version, applied_at) VALUES (?, ?)').run(CONTROL_PLANE_SCHEMA_VERSION, now());
+  withSqliteStatement(
+    database,
+    'INSERT OR IGNORE INTO control_plane_schema (version, applied_at) VALUES (?, ?)',
+    (statement) => statement.run(CONTROL_PLANE_SCHEMA_VERSION, now()),
+  );
 }
 
 export function controlPlaneDatabasePath(controllerHome: string): string {
@@ -186,9 +324,10 @@ function openDatabase(controllerHome: string): SqliteDatabase {
     // inspectOpenDatabase(), which is the authoritative integrity boundary.
     // Keep schema fail-closed on every open, but reserve the full integrity scan
     // for those lifecycle/inspection boundaries.
+    let schemaVersion: number | undefined;
     if (existed) {
       try {
-        assertSupportedSchema(database, path);
+        schemaVersion = assertSupportedSchema(database, path);
       } catch (error) {
         const busy = sqliteBusyError(path, error);
         if (busy) throw busy;
@@ -198,8 +337,14 @@ function openDatabase(controllerHome: string): SqliteDatabase {
       }
     }
     database.exec('PRAGMA journal_mode = WAL;');
-    initializeSchema(database);
-    assertSupportedSchema(database, path, true);
+    // Schema DDL is bootstrap work, not transaction work. Existing supported
+    // databases have already crossed the fail-closed schema boundary above;
+    // rerunning CREATE INDEX/TABLE + INSERT OR IGNORE on every write only burns
+    // allocations/SQLite preparation and then validates the same schema twice.
+    if (schemaVersion === undefined) {
+      initializeSchema(database);
+      assertSupportedSchema(database, path, true);
+    }
     return database;
   } catch (error) {
     database.close();
@@ -238,6 +383,68 @@ function openDatabaseForRead(controllerHome: string): SqliteDatabase {
   return openDatabase(controllerHome);
 }
 
+export function openControlPlaneDatabaseChangeObserver(controllerHome: string): ControlPlaneDatabaseChangeObserver {
+  const path = controlPlaneDatabasePath(controllerHome);
+  const database = openDatabaseForRead(controllerHome);
+  let closed = false;
+  const dataVersion = () => {
+    if (closed) throw new Error('CONTROL_PLANE_DATABASE_CHANGE_OBSERVER_CLOSED');
+    return integerPragma(database, 'PRAGMA data_version', 'data_version');
+  };
+  // Prime the connection before returning it so the first caller observation
+  // is relative to an established SQLite connection, not connection startup.
+  dataVersion();
+  return {
+    path,
+    dataVersion,
+    close() {
+      if (closed) return;
+      closed = true;
+      database.close();
+    },
+  };
+}
+
+function withDatabaseForRead<T>(controllerHome: string, operation: (database: SqliteDatabase) => T): T {
+  const path = controlPlaneDatabasePath(controllerHome);
+  if (!reusableReadDatabasePaths.has(path)) {
+    const database = openDatabaseForRead(controllerHome);
+    try {
+      return operation(database);
+    } finally {
+      database.close();
+    }
+  }
+
+  const liveIdentity = databaseFileIdentity(path);
+  const cached = reusableReadDatabases.get(path);
+  if (cached && liveIdentity && cached.fileIdentity === liveIdentity) {
+    return operation(cached.database);
+  }
+  if (cached) closeReusableReadDatabase(path);
+
+  const database = openDatabaseForRead(controllerHome);
+  const openedIdentity = databaseFileIdentity(path);
+  if (!openedIdentity) {
+    database.close();
+    throw new Error(`CONTROL_PLANE_SQLITE_MISSING: ${path}`);
+  }
+  reusableReadDatabases.set(path, { database, fileIdentity: openedIdentity });
+  try {
+    return operation(database);
+  } catch (error) {
+    // Do not retain a reader that failed while evaluating a query/schema. The
+    // next operation gets a clean connection and repeats the fail-closed checks.
+    closeReusableReadDatabase(path);
+    throw error;
+  }
+}
+
+/** Read-only snapshot boundary for domain stores that need several indexed lookups in one hot-path operation. */
+export function withControlPlaneReadDatabase<T>(controllerHome: string, operation: (database: SqliteDatabase) => T): T {
+  return withDatabaseForRead(controllerHome, operation);
+}
+
 function rowToRecord<T>(row: StoredRecordRow): ControlPlaneRecord<T> {
   let value: T;
   try {
@@ -258,11 +465,11 @@ function rowToRecord<T>(row: StoredRecordRow): ControlPlaneRecord<T> {
 }
 
 function selectRecord<T>(database: SqliteDatabase, namespace: string, scope: string, key: string): ControlPlaneRecord<T> | undefined {
-  const row = database.prepare(`
-    SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
-    FROM control_plane_records
-    WHERE namespace = ? AND scope = ? AND record_key = ?
-  `).get(namespace, scope, key) as StoredRecordRow | undefined;
+  const row = withSqliteStatement(database, `
+      SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+      FROM control_plane_records
+      WHERE namespace = ? AND scope = ? AND record_key = ?
+    `, (statement) => statement.get(namespace, scope, key) as StoredRecordRow | undefined);
   return row ? rowToRecord<T>(row) : undefined;
 }
 
@@ -272,22 +479,30 @@ function writeRecord<T>(
   existing?: ControlPlaneRecord<T>,
 ): ControlPlaneRecord<T> {
   const at = now();
-  const revision = (existing?.revision ?? 0) + 1;
+  const previousAuditRevision = Number(withSqliteStatement(database, `
+      SELECT MAX(revision) AS revision FROM control_plane_audit
+      WHERE namespace = ? AND scope = ? AND record_key = ?
+    `, (statement) => scalar(statement.get(input.namespace, input.scope, input.key))) ?? 0);
+  const revision = existing
+    ? existing.revision + 1
+    : Math.max(0, Number.isSafeInteger(previousAuditRevision) ? previousAuditRevision : 0) + 1;
   const createdAt = existing?.createdAt ?? at;
-  database.prepare(`
-    INSERT INTO control_plane_records (
-      namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(namespace, scope, record_key) DO UPDATE SET
-      schema_version = excluded.schema_version,
-      revision = excluded.revision,
-      payload = excluded.payload,
-      updated_at = excluded.updated_at
-  `).run(input.namespace, input.scope, input.key, input.schemaVersion, revision, JSON.stringify(input.value), createdAt, at);
-  database.prepare(`
-    INSERT INTO control_plane_audit (namespace, scope, record_key, action, revision, occurred_at)
-    VALUES (?, ?, ?, ?, ?, ?)
-  `).run(input.namespace, input.scope, input.key, input.action, revision, at);
+  withSqliteStatement(database, `
+      INSERT INTO control_plane_records (
+        namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(namespace, scope, record_key) DO UPDATE SET
+        schema_version = excluded.schema_version,
+        revision = excluded.revision,
+        payload = excluded.payload,
+        updated_at = excluded.updated_at
+    `, (statement) => statement.run(
+    input.namespace, input.scope, input.key, input.schemaVersion, revision, JSON.stringify(input.value), createdAt, at,
+  ));
+  withSqliteStatement(database, `
+      INSERT INTO control_plane_audit (namespace, scope, record_key, action, revision, occurred_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `, (statement) => statement.run(input.namespace, input.scope, input.key, input.action, revision, at));
   return {
     namespace: input.namespace,
     scope: input.scope,
@@ -352,12 +567,24 @@ export function withControlPlaneTransaction<T>(controllerHome: string, operation
 }
 
 export function readControlPlaneRecord<T>(controllerHome: string, namespace: string, scope: string, key: string): ControlPlaneRecord<T> | undefined {
-  const database = openDatabaseForRead(controllerHome);
-  try {
-    return selectRecord<T>(database, namespace, scope, key);
-  } finally {
-    database.close();
-  }
+  return withDatabaseForRead(controllerHome, (database) =>
+    selectRecord<T>(database, namespace, scope, key));
+}
+
+export function listControlPlaneRecordKeys(
+  controllerHome: string,
+  input: { namespace: string; scope: string; limit?: number },
+): string[] {
+  const limit = Math.max(1, Math.min(Math.trunc(input.limit ?? 5_000), 5_000));
+  return withDatabaseForRead(controllerHome, (database) =>
+    withSqliteStatement(database, `
+      SELECT record_key
+      FROM control_plane_records
+      WHERE namespace = ? AND scope = ?
+      ORDER BY record_key ASC
+      LIMIT ?
+    `, (statement) => (statement.all(input.namespace, input.scope, limit) as Array<{ record_key: string }>)
+      .map((row) => row.record_key)));
 }
 
 export function writeControlPlaneRecord<T>(
@@ -376,6 +603,72 @@ export function writeControlPlaneRecord<T>(
     writeControlPlaneRecordWithinTransaction(database, input));
 }
 
+export function deleteControlPlaneRecordWithinTransaction(
+  database: SqliteDatabase,
+  input: { namespace: string; scope: string; key: string; action?: string; expectedRevision?: number },
+): boolean {
+  const existing = selectRecord(database, input.namespace, input.scope, input.key);
+  if (input.expectedRevision !== undefined && existing?.revision !== input.expectedRevision) {
+    throw new ControlPlaneConflictError(input.namespace, input.scope, input.key, input.expectedRevision, existing?.revision);
+  }
+  if (!existing) return false;
+  const revision = existing.revision + 1;
+  const at = now();
+  withSqliteStatement(database, `
+      DELETE FROM control_plane_records
+      WHERE namespace = ? AND scope = ? AND record_key = ?
+    `, (statement) => statement.run(input.namespace, input.scope, input.key));
+  withSqliteStatement(database, `
+      INSERT INTO control_plane_audit (namespace, scope, record_key, action, revision, occurred_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `, (statement) => statement.run(input.namespace, input.scope, input.key, input.action ?? 'delete', revision, at));
+  return true;
+}
+
+export function deleteControlPlaneRecord(
+  controllerHome: string,
+  input: { namespace: string; scope: string; key: string; action?: string; expectedRevision?: number },
+): boolean {
+  return withControlPlaneTransaction(controllerHome, (database) =>
+    deleteControlPlaneRecordWithinTransaction(database, input));
+}
+
+export function deleteControlPlaneRecordsWithinTransaction(
+  database: SqliteDatabase,
+  input: { namespace: string; scope: string; action?: string },
+): number {
+  const rows = withSqliteStatement(database, `
+      SELECT record_key, revision
+      FROM control_plane_records
+      WHERE namespace = ? AND scope = ?
+      ORDER BY record_key ASC
+    `, (statement) => statement.all(input.namespace, input.scope) as Array<{ record_key: string; revision: number }>);
+  if (rows.length === 0) return 0;
+
+  const at = now();
+  withSqliteStatement(database, `
+      DELETE FROM control_plane_records
+      WHERE namespace = ? AND scope = ? AND record_key = ?
+    `, (remove) => withSqliteStatement(database, `
+      INSERT INTO control_plane_audit (namespace, scope, record_key, action, revision, occurred_at)
+      VALUES (?, ?, ?, ?, ?, ?)
+    `, (audit) => {
+      for (const row of rows) {
+        remove.run(input.namespace, input.scope, row.record_key);
+        audit.run(input.namespace, input.scope, row.record_key, input.action ?? 'delete_scope', row.revision + 1, at);
+      }
+    }));
+  return rows.length;
+}
+
+export function deleteControlPlaneRecords(
+  controllerHome: string,
+  input: { namespace: string; scope: string; action?: string },
+): number {
+  return withControlPlaneTransaction(controllerHome, (database) =>
+    deleteControlPlaneRecordsWithinTransaction(database, input));
+}
+
 export class ControlPlaneConflictError extends Error {
   readonly expectedRevision: number | null;
   readonly actualRevision: number | undefined;
@@ -388,32 +681,162 @@ export class ControlPlaneConflictError extends Error {
   }
 }
 
+export function listControlPlaneRecordsWithinTransaction<T>(
+  database: SqliteDatabase,
+  input: { namespace: string; scope?: string; limit?: number },
+): ControlPlaneRecord<T>[] {
+  const limit = Math.max(1, Math.min(Math.trunc(input.limit ?? 1000), 5000));
+  const rows = input.scope === undefined
+    ? withSqliteStatement(database, `
+        SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+        FROM control_plane_records
+        WHERE namespace = ?
+        ORDER BY updated_at ASC, record_key ASC
+        LIMIT ?
+      `, (statement) => statement.all(input.namespace, limit))
+    : withSqliteStatement(database, `
+        SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+        FROM control_plane_records
+        WHERE namespace = ? AND scope = ?
+        ORDER BY updated_at ASC, record_key ASC
+        LIMIT ?
+      `, (statement) => statement.all(input.namespace, input.scope, limit));
+  return (rows as StoredRecordRow[]).map((row) => rowToRecord<T>(row));
+}
+
 export function listControlPlaneRecords<T>(
   controllerHome: string,
   input: { namespace: string; scope?: string; limit?: number },
 ): ControlPlaneRecord<T>[] {
-  const database = openDatabaseForRead(controllerHome);
-  try {
-    const limit = Math.max(1, Math.min(Math.trunc(input.limit ?? 1000), 5000));
-    const rows = input.scope === undefined
-      ? database.prepare(`
-          SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
-          FROM control_plane_records
-          WHERE namespace = ?
-          ORDER BY updated_at ASC, record_key ASC
-          LIMIT ?
-        `).all(input.namespace, limit)
-      : database.prepare(`
-          SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
-          FROM control_plane_records
-          WHERE namespace = ? AND scope = ?
-          ORDER BY updated_at ASC, record_key ASC
-          LIMIT ?
-        `).all(input.namespace, input.scope, limit);
+  return withDatabaseForRead(controllerHome, (database) =>
+    listControlPlaneRecordsWithinTransaction<T>(database, input));
+}
+
+/** Exact stable-key lookup across scopes. This is a read of the existing Control Plane authority, not a second id registry. */
+export function findControlPlaneRecordsByKey<T>(
+  controllerHome: string,
+  input: { namespace: string; key: string; limit?: number },
+): ControlPlaneRecord<T>[] {
+  const limit = Math.max(1, Math.min(Math.trunc(input.limit ?? 2), 100));
+  return withDatabaseForRead(controllerHome, (database) => {
+    const rows = withSqliteStatement(database, `
+      SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+      FROM control_plane_records
+      WHERE namespace = ? AND record_key = ?
+      ORDER BY scope ASC
+      LIMIT ?
+    `, (statement) => statement.all(input.namespace, input.key, limit));
     return (rows as StoredRecordRow[]).map((row) => rowToRecord<T>(row));
-  } finally {
-    database.close();
+  });
+}
+
+/**
+ * Candidate-only read optimization for top-level JSON text fields. This never
+ * establishes domain authority: missing/null/non-text values remain candidates,
+ * and callers must still run their canonical semantic validation.
+ */
+export interface PayloadTextExclusion {
+  namespace: string;
+  field: string;
+  excludedValues: readonly string[];
+}
+
+function payloadTextExclusionPredicate(input: PayloadTextExclusion): string {
+  const field = input.field.trim();
+  if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(field)) {
+    throw new Error(`CONTROL_PLANE_JSON_FIELD_INVALID: ${input.field}`);
   }
+  const values = [...new Set(input.excludedValues.map((value) => value.trim()).filter(Boolean))].sort();
+  if (values.length > 32) throw new Error('CONTROL_PLANE_JSON_EXCLUSION_LIMIT_EXCEEDED');
+  // These literals are escaped here, never supplied as SQL by a domain caller.
+  // SQLite must see identical literals in the query and partial-index predicate.
+  const literal = (value: string) => `'${value.replaceAll("'", "''")}'`;
+  const namespace = `namespace = ${literal(input.namespace)}`;
+  if (values.length === 0) return namespace;
+  const extraction = `json_extract(payload, ${literal(`$.${field}`)})`;
+  return `${namespace} AND (${extraction} IS NULL OR ${extraction} NOT IN (${values.map(literal).join(', ')}))`;
+}
+
+function payloadTextExclusionIndexName(input: PayloadTextExclusion, predicate = payloadTextExclusionPredicate(input)): string {
+  const prefix = `control_plane_candidate_${createHash('sha256').update(`${input.namespace}/${input.field}`).digest('hex').slice(0, 16)}_`;
+  return `${prefix}${createHash('sha256').update(predicate).digest('hex').slice(0, 16)}`;
+}
+
+/** Explicit startup DDL. JSON remains authority; SQLite maintains the derived index in every write transaction. */
+export function initializeControlPlanePayloadTextExclusionIndex(
+  controllerHome: string,
+  input: PayloadTextExclusion,
+): void {
+  const predicate = payloadTextExclusionPredicate(input);
+  const prefix = `control_plane_candidate_${createHash('sha256').update(`${input.namespace}/${input.field}`).digest('hex').slice(0, 16)}_`;
+  const name = payloadTextExclusionIndexName(input, predicate);
+  withControlPlaneTransaction(controllerHome, (database) => {
+    database.exec(`CREATE INDEX IF NOT EXISTS ${name} ON control_plane_records (scope, updated_at, record_key) WHERE ${predicate}`);
+    // Retire obsolete predicates for this domain field in the same transaction.
+    const old = withSqliteStatement(database, "SELECT name FROM sqlite_master WHERE type = 'index' AND substr(name, 1, ?) = ?",
+      (statement) => statement.all(prefix.length, prefix)) as Array<{ name: string }>;
+    for (const index of old) {
+      if (index.name !== name && /^control_plane_candidate_[a-f0-9]{16}_[a-f0-9]{16}$/.test(index.name)) {
+        database.exec(`DROP INDEX ${index.name}`);
+      }
+    }
+  });
+}
+
+export function listControlPlaneRecordsExcludingPayloadTextValues<T>(
+  controllerHome: string,
+  input: PayloadTextExclusion & { scope: string; limit?: number },
+): ControlPlaneRecord<T>[] {
+  const predicate = payloadTextExclusionPredicate(input);
+  const candidateIndex = payloadTextExclusionIndexName(input, predicate);
+  const limit = Math.max(1, Math.min(Math.trunc(input.limit ?? 1_000), 5_000));
+  return withDatabaseForRead(controllerHome, (database) => {
+    const candidateIndexExists = withSqliteStatement(database,
+      "SELECT 1 AS present FROM sqlite_master WHERE type = 'index' AND name = ? LIMIT 1",
+      (statement) => statement.get(candidateIndex) != null);
+    const indexHint = candidateIndexExists ? ` INDEXED BY ${candidateIndex}` : '';
+    const rows = withSqliteStatement(database, `
+      SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+      FROM control_plane_records${indexHint}
+      WHERE ${predicate} AND scope = ?
+      ORDER BY updated_at ASC, record_key ASC
+      LIMIT ?
+    `, (statement) => statement.all(input.scope, limit));
+    return (rows as StoredRecordRow[]).map((row) => rowToRecord<T>(row));
+  });
+}
+
+/**
+ * Internal correctness surface for authorities that must observe every durable
+ * fact in a namespace/scope. User-facing and diagnostic list APIs should keep
+ * using listControlPlaneRecords so their output remains bounded.
+ */
+export function listAllControlPlaneRecordsWithinTransaction<T>(
+  database: SqliteDatabase,
+  input: { namespace: string; scope?: string },
+): ControlPlaneRecord<T>[] {
+  const rows = input.scope === undefined
+    ? withSqliteStatement(database, `
+        SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+        FROM control_plane_records
+        WHERE namespace = ?
+        ORDER BY updated_at ASC, record_key ASC
+      `, (statement) => statement.all(input.namespace))
+    : withSqliteStatement(database, `
+        SELECT namespace, scope, record_key, schema_version, revision, payload, created_at, updated_at
+        FROM control_plane_records
+        WHERE namespace = ? AND scope = ?
+        ORDER BY updated_at ASC, record_key ASC
+      `, (statement) => statement.all(input.namespace, input.scope));
+  return (rows as StoredRecordRow[]).map((row) => rowToRecord<T>(row));
+}
+
+export function listAllControlPlaneRecords<T>(
+  controllerHome: string,
+  input: { namespace: string; scope?: string },
+): ControlPlaneRecord<T>[] {
+  return withDatabaseForRead(controllerHome, (database) =>
+    listAllControlPlaneRecordsWithinTransaction<T>(database, input));
 }
 
 function inspectOpenDatabase(database: SqliteDatabase, path: string): ControlPlaneDatabaseInspection {
@@ -422,34 +845,42 @@ function inspectOpenDatabase(database: SqliteDatabase, path: string): ControlPla
   if (!tableExists(database, 'control_plane_records') || !tableExists(database, 'control_plane_audit')) {
     throw new Error(`CONTROL_PLANE_BACKUP_INVALID: ${path}: required tables are missing`);
   }
-  const recordCount = Number(scalar(database.prepare('SELECT COUNT(*) AS count FROM control_plane_records').get()) ?? 0);
-  const auditEventCount = Number(scalar(database.prepare('SELECT COUNT(*) AS count FROM control_plane_audit').get()) ?? 0);
-  const orphanRecordCount = Number(scalar(database.prepare(`
-    SELECT COUNT(*) AS count
-    FROM control_plane_records record
-    WHERE (
-      SELECT COUNT(DISTINCT audit.revision)
-      FROM control_plane_audit audit
-      WHERE audit.namespace = record.namespace
-        AND audit.scope = record.scope
-        AND audit.record_key = record.record_key
-        AND audit.revision BETWEEN 1 AND record.revision
-    ) != record.revision
-      OR COALESCE((
-        SELECT MIN(audit.revision)
-        FROM control_plane_audit audit
-        WHERE audit.namespace = record.namespace
-          AND audit.scope = record.scope
-          AND audit.record_key = record.record_key
-      ), 0) != 1
-      OR COALESCE((
-        SELECT MAX(audit.revision)
+  const recordCount = Number(withSqliteStatement(
+    database,
+    'SELECT COUNT(*) AS count FROM control_plane_records',
+    (statement) => scalar(statement.get()),
+  ) ?? 0);
+  const auditEventCount = Number(withSqliteStatement(
+    database,
+    'SELECT COUNT(*) AS count FROM control_plane_audit',
+    (statement) => scalar(statement.get()),
+  ) ?? 0);
+  const orphanRecordCount = Number(withSqliteStatement(database, `
+      SELECT COUNT(*) AS count
+      FROM control_plane_records record
+      WHERE (
+        SELECT COUNT(DISTINCT audit.revision)
         FROM control_plane_audit audit
         WHERE audit.namespace = record.namespace
           AND audit.scope = record.scope
           AND audit.record_key = record.record_key
-      ), 0) != record.revision
-  `).get()) ?? 0);
+          AND audit.revision BETWEEN 1 AND record.revision
+      ) != record.revision
+        OR COALESCE((
+          SELECT MIN(audit.revision)
+          FROM control_plane_audit audit
+          WHERE audit.namespace = record.namespace
+            AND audit.scope = record.scope
+            AND audit.record_key = record.record_key
+        ), 0) != 1
+        OR COALESCE((
+          SELECT MAX(audit.revision)
+          FROM control_plane_audit audit
+          WHERE audit.namespace = record.namespace
+            AND audit.scope = record.scope
+            AND audit.record_key = record.record_key
+        ), 0) != record.revision
+    `, (statement) => scalar(statement.get())) ?? 0);
   if (orphanRecordCount > 0) {
     throw new Error(`CONTROL_PLANE_AUDIT_CONTINUITY_INVALID: ${path}: discontinuous_records=${orphanRecordCount}`);
   }
@@ -476,6 +907,147 @@ export function inspectControlPlaneDatabaseFile(path: string): ControlPlaneDatab
   }
 }
 
+interface ControlPlaneDatabasePageSnapshot {
+  pageSize: number;
+  pageCount: number;
+  freePageCount: number;
+  reclaimableBytes: number;
+  reclaimableRatio: number;
+}
+
+function integerPragma(database: SqliteDatabase, sql: string, label: string): number {
+  const value = Number(withSqliteStatement(database, sql, (statement) => scalar(statement.get())));
+  if (!Number.isFinite(value) || value < 0) throw new Error(`CONTROL_PLANE_SQLITE_PRAGMA_INVALID: ${label}`);
+  return Math.floor(value);
+}
+
+function controlPlaneDatabasePageSnapshot(database: SqliteDatabase): ControlPlaneDatabasePageSnapshot {
+  const pageSize = integerPragma(database, 'PRAGMA page_size', 'page_size');
+  const pageCount = integerPragma(database, 'PRAGMA page_count', 'page_count');
+  const freePageCount = integerPragma(database, 'PRAGMA freelist_count', 'freelist_count');
+  const reclaimableBytes = freePageCount * pageSize;
+  return {
+    pageSize,
+    pageCount,
+    freePageCount,
+    reclaimableBytes,
+    reclaimableRatio: pageCount > 0 ? freePageCount / pageCount : 0,
+  };
+}
+
+/**
+ * Physical SQLite maintenance only. Domain lifecycles own row deletion; this
+ * boundary never decides semantic terminality or removes canonical records.
+ * PASSIVE checkpointing is non-blocking, and VACUUM is attempted only when
+ * already-reclaimable free pages exceed both configured thresholds. A live
+ * writer causes a fail-closed busy skip instead of lock stealing or corruption
+ * recovery.
+ */
+export function maintainControlPlaneDatabase(
+  controllerHome: string,
+  options: ControlPlaneDatabaseMaintenanceOptions = {},
+): ControlPlaneDatabaseMaintenanceReport {
+  const path = controlPlaneDatabasePath(controllerHome);
+  const report: ControlPlaneDatabaseMaintenanceReport = {
+    policyVersion: CONTROL_PLANE_SQLITE_MAINTENANCE_POLICY_VERSION,
+    path,
+    checkpointMode: 'passive',
+    checkpointed: false,
+    pageSize: 0,
+    pageCountBefore: 0,
+    freePageCountBefore: 0,
+    reclaimableBytesBefore: 0,
+    reclaimableRatioBefore: 0,
+    vacuumEligible: false,
+    vacuumAttempted: false,
+    vacuumed: false,
+    pageCountAfter: 0,
+    freePageCountAfter: 0,
+    reclaimableBytesAfter: 0,
+    reclaimableRatioAfter: 0,
+    reclaimedBytes: 0,
+  };
+  if (!existsSync(path)) {
+    report.skippedReason = 'database_missing';
+    return report;
+  }
+
+  let database: SqliteDatabase;
+  try {
+    database = openRawDatabase(path);
+  } catch (error) {
+    if ((error instanceof Error ? error.message : String(error)).startsWith('CONTROL_PLANE_SQLITE_BUSY:')) {
+      report.skippedReason = 'database_busy';
+      return report;
+    }
+    throw error;
+  }
+  try {
+    database.exec('PRAGMA busy_timeout = 0; PRAGMA foreign_keys = ON;');
+    assertSupportedSchema(database, path, true);
+    try {
+      database.exec('PRAGMA wal_checkpoint(PASSIVE);');
+      report.checkpointed = true;
+    } catch (error) {
+      const busy = sqliteBusyError(path, error);
+      if (busy) {
+        report.skippedReason = 'database_busy';
+        return report;
+      }
+      throw error;
+    }
+
+    const before = controlPlaneDatabasePageSnapshot(database);
+    report.pageSize = before.pageSize;
+    report.pageCountBefore = before.pageCount;
+    report.freePageCountBefore = before.freePageCount;
+    report.reclaimableBytesBefore = before.reclaimableBytes;
+    report.reclaimableRatioBefore = before.reclaimableRatio;
+    report.pageCountAfter = before.pageCount;
+    report.freePageCountAfter = before.freePageCount;
+    report.reclaimableBytesAfter = before.reclaimableBytes;
+    report.reclaimableRatioAfter = before.reclaimableRatio;
+
+    const minimumReclaimableBytes = Math.max(0, Math.floor(options.minimumReclaimableBytes ?? DEFAULT_CONTROL_PLANE_SQLITE_VACUUM_MIN_RECLAIMABLE_BYTES));
+    const configuredRatio = Number(options.minimumReclaimableRatio ?? DEFAULT_CONTROL_PLANE_SQLITE_VACUUM_MIN_RECLAIMABLE_RATIO);
+    const minimumReclaimableRatio = Number.isFinite(configuredRatio) ? Math.min(1, Math.max(0, configuredRatio)) : DEFAULT_CONTROL_PLANE_SQLITE_VACUUM_MIN_RECLAIMABLE_RATIO;
+    report.vacuumEligible = before.freePageCount > 0
+      && before.reclaimableBytes >= minimumReclaimableBytes
+      && before.reclaimableRatio >= minimumReclaimableRatio;
+    if (!report.vacuumEligible) {
+      report.skippedReason = 'below_threshold';
+      return report;
+    }
+    if (options.allowVacuum === false) {
+      report.skippedReason = 'vacuum_disabled';
+      return report;
+    }
+
+    report.vacuumAttempted = true;
+    try {
+      database.exec('VACUUM;');
+    } catch (error) {
+      const busy = sqliteBusyError(path, error);
+      if (busy) {
+        report.skippedReason = 'database_busy';
+        return report;
+      }
+      throw error;
+    }
+    assertDatabaseIntegrity(database, path);
+    const after = controlPlaneDatabasePageSnapshot(database);
+    report.vacuumed = true;
+    report.pageCountAfter = after.pageCount;
+    report.freePageCountAfter = after.freePageCount;
+    report.reclaimableBytesAfter = after.reclaimableBytes;
+    report.reclaimableRatioAfter = after.reclaimableRatio;
+    report.reclaimedBytes = Math.max(0, (before.pageCount - after.pageCount) * before.pageSize);
+    return report;
+  } finally {
+    database.close();
+  }
+}
+
 export function backupControlPlaneDatabase(controllerHome: string, destinationPath: string): ControlPlaneDatabaseInspection {
   const destination = join(destinationPath);
   if (existsSync(destination)) throw new Error(`CONTROL_PLANE_BACKUP_EXISTS: ${destination}`);
@@ -497,6 +1069,10 @@ export function backupControlPlaneDatabase(controllerHome: string, destinationPa
 export function restoreControlPlaneDatabase(controllerHome: string, backupPath: string): ControlPlaneDatabaseInspection {
   if (!existsSync(backupPath)) throw new Error(`CONTROL_PLANE_BACKUP_MISSING: ${backupPath}`);
   const target = controlPlaneDatabasePath(controllerHome);
+  // A reusable Runtime reader refers to the current inode, not the pathname.
+  // Close it before replacing the live authority; reuse remains enabled and the
+  // next read opens the verified replacement.
+  closeReusableReadDatabase(target);
   const staging = `${target}.restore-${process.pid}-${Date.now()}`;
   copyFileSync(backupPath, staging);
   try {