@equationalapplications/core-llm-wiki 7.9.1 → 7.10.0

package/dist/index.mjs

@@ -1,5 +1,5 @@
-import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, WikiTransactionError, extractSqliteCode } from './chunk-7FRLQ7ZO.mjs';
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-7FRLQ7ZO.mjs';
+import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, WikiTransactionError, extractSqliteCode } from './chunk-LOVBNRCV.mjs';
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-LOVBNRCV.mjs';
 import { deriveTrustTier, isStaleAfter, appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, latestVerified, formatSourcesJson, appendEventIdComment, parseVerifiedFlexible, parseCitationsList } from '@equationalapplications/core-okf';
 import MiniSearch from 'minisearch';
 
@@ -2994,6 +2994,280 @@ function assertOwnedBy(entityId, documents) {
   }
 }
 
+// src/services/search/fts5Sql.ts
+var FTS5_STATE_KEY = "fts5_index_state";
+var FTS5_DRAIN_CHUNK = 500;
+function fts5TriggerNames(prefix) {
+  return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
+}
+function tagsExpr(col) {
+  return `CASE WHEN json_valid(${col}) THEN
+      CASE WHEN json_type(${col}) = 'array' THEN
+        (SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
+                                       WHEN 'null' THEN '' ELSE value END, ' ')
+           FROM json_each(${col}))
+      ELSE ${col} END
+    ELSE ${col} END`;
+}
+function fts5TablesDdl(p) {
+  return `
+CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
+  id UNINDEXED, entity_id UNINDEXED, title, body, tags,
+  tokenize = 'porter unicode61'
+);
+CREATE TABLE IF NOT EXISTS ${p}fts_map (
+  fts_rowid INTEGER PRIMARY KEY,
+  id TEXT NOT NULL UNIQUE,
+  entity_id TEXT NOT NULL
+);
+CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
+CREATE TABLE IF NOT EXISTS ${p}fts_pending (
+  seq INTEGER PRIMARY KEY,
+  entity_id TEXT NOT NULL,
+  id TEXT NOT NULL
+);`;
+}
+function fts5TriggersDdl(p) {
+  const [ai, au, ad] = fts5TriggerNames(p);
+  return `
+CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
+  INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
+END;
+CREATE TRIGGER IF NOT EXISTS ${au}
+AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
+  INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
+  INSERT INTO ${p}fts_pending (entity_id, id)
+    SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
+END;
+CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
+  INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
+END;`;
+}
+function drainHiSql(p) {
+  return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
+}
+function drainChunkSql(p) {
+  const chunkIds = `SELECT id FROM ${p}fts_pending WHERE seq <= ?`;
+  return [
+    `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
+    `DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
+    `INSERT INTO ${p}fts_map (id, entity_id)
+       SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
+    `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
+       SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
+       FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
+       WHERE m.id IN (${chunkIds})`,
+    `DELETE FROM ${p}fts_pending WHERE seq <= ?`
+  ];
+}
+function rebuildSql(p) {
+  return [
+    `INSERT INTO ${p}fts_map (id, entity_id) SELECT id, entity_id FROM ${p}entries WHERE deleted_at IS NULL`,
+    `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
+       SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
+       FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id`
+  ];
+}
+function fts5LedgerDropSql(p) {
+  return [
+    ...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
+    `DROP TABLE IF EXISTS ${p}fts_pending;`,
+    `DROP TABLE IF EXISTS ${p}fts_map;`
+  ].join("\n");
+}
+
+// src/services/search/ftsQuery.ts
+var TOKEN = /[\p{L}\p{N}]+/gu;
+function buildFtsMatchQuery(query) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const m of query.toLowerCase().matchAll(TOKEN)) seen.add(m[0]);
+  if (seen.size === 0) return null;
+  return [...seen].map((t) => `"${t}"*`).join(" OR ");
+}
+
+// src/services/search/Fts5IndexStrategy.ts
+var Fts5IndexStrategy = class {
+  constructor(db, prefix, metadataRepo) {
+    this.db = db;
+    this.prefix = prefix;
+    this.metadataRepo = metadataRepo;
+  }
+  async init() {
+    const state = await this.metadataRepo.getMeta(FTS5_STATE_KEY);
+    if (state === "live" && await this.triggersPresent()) {
+      await this.db.execAsync(fts5TablesDdl(this.prefix));
+      return;
+    }
+    await this.db.withTransactionAsync(async (tx) => {
+      await tx.execAsync(`DROP TABLE IF EXISTS ${this.prefix}entries_fts;`);
+      await tx.execAsync(fts5TablesDdl(this.prefix));
+      await this.rebuildInTx(tx);
+      await tx.execAsync(fts5TriggersDdl(this.prefix));
+      await this.metadataRepo.setMeta(FTS5_STATE_KEY, "live", tx);
+    });
+  }
+  async rebuildFromSource() {
+    await this.db.withTransactionAsync((tx) => this.rebuildInTx(tx));
+  }
+  async drain() {
+    const hiSql = drainHiSql(this.prefix);
+    const chunk = drainChunkSql(this.prefix);
+    for (; ; ) {
+      const done = await this.db.withTransactionAsync(async (tx) => {
+        const row = await tx.getFirstAsync(hiSql);
+        if (row?.hi == null) return true;
+        for (const sql of chunk) await tx.runAsync(sql, [row.hi]);
+        return false;
+      });
+      if (done) return;
+    }
+  }
+  async replace(entityId, ids, documents) {
+    assertEntity(entityId, documents);
+    const docIds = documents.map((d) => d.id);
+    await this.db.withTransactionAsync(async (tx) => {
+      await this.deleteByIds(tx, entityId, ids);
+      await this.deleteByIds(tx, null, docIds);
+      await this.insertDocs(tx, documents);
+    });
+  }
+  async replaceEntity(entityId, documents) {
+    assertEntity(entityId, documents);
+    const p = this.prefix;
+    await this.db.withTransactionAsync(async (tx) => {
+      await tx.runAsync(
+        `DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT fts_rowid FROM ${p}fts_map WHERE entity_id = ?)`,
+        [entityId]
+      );
+      await tx.runAsync(`DELETE FROM ${p}fts_map WHERE entity_id = ?`, [entityId]);
+      await this.insertDocs(tx, documents);
+    });
+  }
+  async replaceAll(documents) {
+    const p = this.prefix;
+    await this.db.withTransactionAsync(async (tx) => {
+      await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
+      await this.insertDocs(tx, documents);
+    });
+  }
+  async search(query, options) {
+    if (options.entityIds.length === 0) return [];
+    const match = buildFtsMatchQuery(query);
+    if (match === null) return [];
+    const cap = options.preFilterLimit ?? options.limit;
+    const limit = cap >= Number.MAX_SAFE_INTEGER ? -1 : cap;
+    const t = `${this.prefix}entries_fts`;
+    const bm25 = `bm25(${t}, 0.0, 0.0, 2.0, 1.0, 1.0)`;
+    return this.db.getAllAsync(
+      `SELECT id, entity_id, max(0.0, -${bm25}) AS score
+         FROM ${t}
+        WHERE ${t} MATCH ? AND entity_id IN (SELECT value FROM json_each(?))
+        ORDER BY ${bm25}, id
+        LIMIT ?`,
+      [match, JSON.stringify(options.entityIds), limit]
+    );
+  }
+  async deleteByIds(tx, entityId, ids) {
+    if (ids.length === 0) return;
+    const p = this.prefix;
+    const json = JSON.stringify(ids);
+    const whereEntity = entityId === null ? "" : " AND entity_id = ?";
+    const args = [json];
+    if (entityId !== null) args.push(entityId);
+    await tx.runAsync(
+      `DELETE FROM ${p}entries_fts WHERE rowid IN (
+         SELECT fts_rowid FROM ${p}fts_map WHERE id IN (SELECT value FROM json_each(?))${whereEntity})`,
+      args
+    );
+    await tx.runAsync(
+      `DELETE FROM ${p}fts_map WHERE id IN (SELECT value FROM json_each(?))${whereEntity}`,
+      args
+    );
+  }
+  /** One INSERT pair per document: a throw part-way rolls back the whole transaction. */
+  async insertDocs(tx, documents) {
+    const p = this.prefix;
+    for (const d of documents) {
+      const { lastInsertRowId } = await tx.runAsync(
+        `INSERT INTO ${p}fts_map (id, entity_id) VALUES (?, ?)`,
+        [d.id, d.entity_id]
+      );
+      await tx.runAsync(
+        `INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags) VALUES (?, ?, ?, ?, ?, ?)`,
+        [lastInsertRowId, d.id, d.entity_id, d.title, d.body, d.tags]
+      );
+    }
+  }
+  async rebuildInTx(tx) {
+    const p = this.prefix;
+    await tx.execAsync(`DELETE FROM ${p}entries_fts; DELETE FROM ${p}fts_map; DELETE FROM ${p}fts_pending;`);
+    for (const sql of rebuildSql(p)) await tx.runAsync(sql);
+  }
+  async triggersPresent() {
+    const names = fts5TriggerNames(this.prefix);
+    const rows = await this.db.getAllAsync(
+      `SELECT name FROM sqlite_master WHERE type = 'trigger' AND name IN (?, ?, ?)`,
+      [...names]
+    );
+    return rows.length === names.length;
+  }
+};
+function assertEntity(entityId, documents) {
+  for (const d of documents) {
+    if (d.entity_id !== entityId) {
+      throw new TypeError(`IndexDocument ${d.id} has entity_id ${d.entity_id}, expected ${entityId}`);
+    }
+  }
+}
+
+// src/services/search/createIndexStrategy.ts
+async function probeFts5(db, prefix) {
+  const name = `temp.${prefix}fts5_probe`;
+  try {
+    await db.execAsync(`CREATE VIRTUAL TABLE IF NOT EXISTS ${name} USING fts5(x)`);
+    const row = await db.getFirstAsync(`SELECT json_valid('[]') AS ok`);
+    return row?.ok === 1;
+  } catch {
+    return false;
+  } finally {
+    try {
+      await db.execAsync(`DROP TABLE IF EXISTS ${name}`);
+    } catch {
+    }
+  }
+}
+async function detachFts5(db, prefix, metadataRepo) {
+  const hasLedger = await metadataRepo.tableExists(`${prefix}fts_pending`);
+  const state = await metadataRepo.getMeta(FTS5_STATE_KEY);
+  if (!hasLedger && state !== "live") return;
+  await db.withTransactionAsync(async (tx) => {
+    await tx.execAsync(fts5LedgerDropSql(prefix));
+    await metadataRepo.setMeta(FTS5_STATE_KEY, "detached", tx);
+  });
+  try {
+    await db.execAsync(`DROP TABLE IF EXISTS ${prefix}entries_fts`);
+  } catch {
+  }
+}
+async function createIndexStrategy(db, prefix, metadataRepo, preferred) {
+  if (preferred !== "minisearch") {
+    if (await probeFts5(db, prefix)) {
+      const strategy = new Fts5IndexStrategy(db, prefix, metadataRepo);
+      try {
+        await strategy.init();
+        return strategy;
+      } catch (err) {
+        if (preferred === "fts5") throw err;
+        console.warn("[WikiMemory] FTS5 init failed; falling back to MiniSearch:", err);
+      }
+    } else if (preferred === "fts5") {
+      throw new Error(`indexStrategy 'fts5' was requested, but this SQLite build lacks FTS5 or json1.`);
+    }
+  }
+  await detachFts5(db, prefix, metadataRepo);
+  return new MiniSearchIndexStrategy();
+}
+
 // src/services/MaintenanceScheduler.ts
 var MaintenanceScheduler = class {
   constructor(db, options, maintenanceService, metadataRepo, eventRepo, entryRepo, jobManager) {
@@ -3850,6 +4124,14 @@ var WikiMemory = class {
         }
       }
     });
+    this.searchService.setIndexStrategy(
+      await createIndexStrategy(
+        this.db,
+        this.prefix,
+        this.metadataRepo,
+        this.options.config?.indexStrategy ?? "minisearch"
+      )
+    );
     await this.searchService.sync();
   }
   async hasChanged(entityId, sourceRefOrEntries, sourceHashArg) {
@@ -4114,6 +4396,11 @@ var WikiMemory = class {
    * entity stays stale and the next call retries it. Does not compute
    * embeddings; that remains the maintenance sweep's job.
    *
+   * With the FTS5 index strategy this drains the index ledger in its own
+   * transactions, so it MUST be called after the host transaction commits,
+   * never from inside a `withTransactionAsync` callback. Calling it inside
+   * one waits on the transaction mutex that the callback holds.
+   *
    * @throws {TypeError} when `entityId` is given but is not a non-empty string.
    */
   async syncSearchIndex(entityId) {