@equationalapplications/core-llm-wiki 7.12.0 → 7.13.1

package/dist/testing.js

@@ -38,6 +38,8 @@ var SQLITE_RESULT_CODE_NAMES = {
   // prefix and we normalize it back to the symbolic name so callers can branch on
   // the same string the rest of the ecosystem uses.
   2067: "SQLITE_CONSTRAINT_UNIQUE",
+  // 267 = SQLITE_CORRUPT_VTAB — a corrupt FTS5 shadow table (ftsCorruptionClass).
+  267: "SQLITE_CORRUPT_VTAB",
   100: "SQLITE_ROW",
   101: "SQLITE_DONE"
 };
@@ -1407,7 +1409,7 @@ var ImportExportService = class {
     if (merge) {
       await this.searchService.syncEntries(entityId, [...upsertedFactIds]);
     } else {
-      await this.searchService.sync(entityId);
+      await this.searchService.sync(entityId, { ids: [.../* @__PURE__ */ new Set([...upsertedFactIds, ...softDeletedFactIds])] });
     }
     for (const fact of bundle.facts) {
       if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
@@ -2340,7 +2342,8 @@ var IngestionService = class {
    *
    * @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
    *   (manifest-valid edges persisted), superseded (prior facts soft-deleted
-   *   plus prior source-ref edges hard-deleted).
+   *   plus prior source-ref edges hard-deleted), and supersededFactIds (the
+   *   ids of those soft-deleted facts, for the caller's post-commit index sync).
    *
    * Node-ownership pre-flight (spec §3, REQ-GRAPH-01): the first statement
    * is `assertGraphNodeOwnership(entityId, params.nodes, tx)`, which runs
@@ -2468,7 +2471,8 @@ var IngestionService = class {
     return {
       nodesWritten: wikiFacts.length,
       edgesWritten: edgesInsertedCount,
-      superseded: deletedFactIds.length + deletedEdgeCount
+      superseded: deletedFactIds.length + deletedEdgeCount,
+      supersededFactIds: deletedFactIds
     };
   }
   /**
@@ -2882,6 +2886,17 @@ async function createTemporalIndexesIfColumnsExist(db, prefix) {
   }
 }
 
+// src/services/search/fts5Sql.ts
+var FTS5_STATE_KEY = "fts5_index_state";
+var FTS5_REBUILD_WM_KEY = "fts5_rebuild_wm";
+var FTS5_REBUILD_SNAPSHOT_KEY = "fts5_rebuild_snapshot";
+var FTS5_SWEEP_CURSOR_KEY = "fts5_orphan_sweep_cursor";
+function fts5PendingIndexesDdl(p) {
+  return `
+CREATE INDEX IF NOT EXISTS ${p}fts_pending_id ON ${p}fts_pending(id);
+CREATE INDEX IF NOT EXISTS ${p}fts_pending_entity ON ${p}fts_pending(entity_id);`;
+}
+
 // src/db/migrations.ts
 var MIGRATIONS = [
   {
@@ -3162,6 +3177,17 @@ var MIGRATIONS = [
     run: async (db, prefix) => {
       await createLegacySourceTypeIndex(db, prefix);
     }
+  },
+  {
+    version: 15,
+    description: "fts_pending(id) and fts_pending(entity_id) indexes for the ids-first and entity-targeted drains",
+    run: async (db, prefix) => {
+      const row = await db.getFirstAsync(
+        `SELECT count(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
+        [`${prefix}fts_pending`]
+      );
+      if (row?.n) await db.execAsync(fts5PendingIndexesDdl(prefix));
+    }
   }
 ];
 async function createLegacySourceTypeIndex(db, prefix) {
@@ -3186,6 +3212,21 @@ async function resetNormalizeWatermarks(meta) {
   await meta.deleteMeta(SRI_WM_KEY);
 }
 
+// src/services/search/fts5Progress.ts
+async function writeRebuildStart(tx, prefix, meta) {
+  const e = await tx.getFirstAsync(`SELECT coalesce(max(rowid), 0) AS hi FROM ${prefix}entries`);
+  const m = await tx.getFirstAsync(`SELECT coalesce(max(fts_rowid), 0) AS hi FROM ${prefix}fts_map`);
+  await meta.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify({ rowid: 0, id: null }), tx);
+  await meta.setMeta(FTS5_REBUILD_SNAPSHOT_KEY, JSON.stringify({ entries: e?.hi ?? 0, map: m?.hi ?? 0 }), tx);
+  await meta.setMeta(FTS5_SWEEP_CURSOR_KEY, "0", tx);
+}
+async function restartRebuildAfterVacuum(db, prefix, meta) {
+  await db.withTransactionAsync(async (tx) => {
+    if (await meta.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
+    await writeRebuildStart(tx, prefix, meta);
+  });
+}
+
 // src/services/librarian/constants.ts
 var FUZZY_THRESHOLD = 0.5;
 var MIN_TOKENS_TO_QUALIFY = 3;
@@ -4121,6 +4162,7 @@ var MaintenanceService = class {
       if (vacuum) {
         await resetNormalizeWatermarks(this.metadataRepo);
         await this.metadataRepo.vacuum();
+        await restartRebuildAfterVacuum(this.db, this.prefix, this.metadataRepo);
       }
       await this.searchService.syncEntries(entityId, syncedIds);
       return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
@@ -5814,7 +5856,7 @@ var UnavailableIndexStrategy = class {
   hasIndexedEntity(_entityId) {
     return false;
   }
-  async drain() {
+  async drain(_opts) {
   }
 };
 
@@ -5860,24 +5902,31 @@ var _SearchService = class _SearchService {
   isKeywordAvailable() {
     return !(this.indexStrategy instanceof UnavailableIndexStrategy);
   }
+  /** True when the installed strategy drains a durable ledger (FTS5, or the degraded stand-in). */
+  hasLedger() {
+    return this.indexStrategy.drain !== void 0;
+  }
   /**
    * One serialized drain turn for strategies with a durable ledger. Like
    * sync(), it never rejects: the keyword index is rebuildable, and a failed
-   * drain leaves the ledger rows in place for the next turn.
+   * drain leaves the ledger rows in place for the next turn. Resolves false
+   * when the drain failed.
    */
-  drainTurn(entityId) {
+  drainTurn(entityId, opts) {
     const work = this.syncChain.then(async () => {
       try {
         try {
-          await this.indexStrategy.drain();
+          await this.indexStrategy.drain(opts);
         } finally {
           this.evictCache(entityId);
         }
+        return true;
       } catch (err) {
         console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
+        return false;
       }
     });
-    this.syncChain = work;
+    this.syncChain = work.then(() => void 0);
     return work;
   }
   /**
@@ -5887,9 +5936,18 @@ var _SearchService = class _SearchService {
    * Rebuilds are serialized per instance and never reject: the keyword index
    * is a rebuildable cache over SQLite, so degraded keyword search is the
    * correct failure mode and killing the host process is not.
+   *
+   * With a ledger strategy, `sync()` runs one full quantum bounded by
+   * `opts.deadline`; `sync(entityId)` is a write turn (ledger only).
+   * `opts.ids` are the caller's touched ids: a ledger strategy drains them
+   * first, to completion; MiniSearch ignores them and rebuilds the entity in
+   * one scan.
    */
-  async sync(entityId) {
-    if (this.indexStrategy.drain) return this.drainTurn(entityId);
+  async sync(entityId, opts) {
+    if (this.indexStrategy.drain) {
+      await this.drainTurn(entityId, entityId === void 0 ? { mode: "full", deadline: opts?.deadline, ids: opts?.ids } : { mode: "ledger", entityId, ids: opts?.ids });
+      return;
+    }
     const work = this.syncChain.then(async () => {
       try {
         try {
@@ -5930,13 +5988,17 @@ var _SearchService = class _SearchService {
    * — including on an empty `ids` list, which is a no-op only for an entity
    * the index already tracks, and even then one that waits for rebuilds
    * already queued on the chain.
+   *
+   * Resolves false only when a ledger drain failed: the caller's ids were not
+   * applied (their ledger rows remain), so a caller holding them retries.
    */
   async syncEntries(entityId, ids) {
-    if (this.indexStrategy.drain) return this.drainTurn(entityId);
+    if (this.indexStrategy.drain) return this.drainTurn(entityId, { mode: "ledger", entityId, ids: [...new Set(ids)] });
     const uniqueIds = [...new Set(ids)];
     const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
     if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
-      return this.syncChain;
+      await this.syncChain;
+      return true;
     }
     const work = this.syncChain.then(async () => {
       try {
@@ -5965,7 +6027,8 @@ var _SearchService = class _SearchService {
       }
     });
     this.syncChain = work;
-    return work;
+    await work;
+    return true;
   }
   /**
    * Forces the entity's next syncEntries to rebuild it in full. For writes core
@@ -5981,11 +6044,13 @@ var _SearchService = class _SearchService {
    * so only those entities are rebuilt. Each rebuild still takes its own turn
    * on the sync chain, and like syncEntries this never rejects. With nothing
    * stale it resolves at once, without waiting on work already on the chain.
+   * Resolves false only when a ledger drain failed (see syncEntries).
    */
-  async syncStale() {
-    if (this.indexStrategy.drain) return this.drainTurn();
+  async syncStale(opts) {
+    if (this.indexStrategy.drain) return this.drainTurn(void 0, { mode: "full", ids: opts?.ids });
     const ids = [...this.staleEntities];
     await Promise.all(ids.map((id) => this.syncEntries(id, [])));
+    return true;
   }
   /**
    * Clears the parsed vector cache. Useful for mid-loop flush guarantees