@equationalapplications/core-llm-wiki 7.8.0 → 7.9.1

package/README.md

@@ -1254,6 +1254,15 @@ const { nodesWritten, edgesWritten, superseded } = await wikiMemory.upsertGraph(
 
 `upsertGraph` is "the tail of `ingestDocument` with the middle (LLM extraction) step removed" — it accepts caller-supplied nodes (`{ id, type, title, body? }`) and edges (`{ type, sourceId, targetId, id? }`) and writes them under the same `(sourceRef, sourceHash)` semantics. If a *different* live `sourceRef` already holds the same `sourceHash`, it throws `WikiSourceRefHashCollision`; re-writing the identical `(sourceRef, sourceHash)` is a no-op returning zero counts. The adapter parameter is required so writes participate in the caller's transaction.
 
+Core never sees your transaction commit, so it can't index the new nodes itself. Until you tell it, they won't show up in keyword or hybrid `read()` results on the same instance. Call `syncSearchIndex` **after** the commit, not inside the transaction:
+
+```typescript
+await db.withTransactionAsync((tx) => wikiMemory.upsertGraph('entity-123', params, tx));
+await wikiMemory.syncSearchIndex('entity-123'); // or syncSearchIndex() for every entity written
+```
+
+It is cheap on an entity that is already current, runs in order with core's own index syncs, and never rejects for a valid `entityId` (an empty or non-string id throws `TypeError`). It does not compute embeddings; those still come from the maintenance sweep.
+
 Entry IDs occupy one globally shared namespace within a database. An ID belongs
 to its existing entity while its row exists, including after soft deletion.
 Same-entity updates and resurrection retain their existing behavior; physically

package/dist/{chunk-M76CF2Q6.mjs → chunk-7FRLQ7ZO.mjs}

@@ -1,4 +1,3 @@
-import MiniSearch from 'minisearch';
 import { latestVerified } from '@equationalapplications/core-okf';
 
 var __typeError = (msg) => {
@@ -426,11 +425,28 @@ function cosineSimilarity(a, b) {
   return denom === 0 ? 0 : dot / denom;
 }
 
+// src/utils/indexDoc.ts
+function toIndexDoc(row) {
+  return {
+    id: row.id,
+    entity_id: row.entity_id,
+    title: row.title,
+    body: row.body,
+    tags: (() => {
+      try {
+        const parsed = JSON.parse(row.tags);
+        return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
+      } catch {
+        return row.tags;
+      }
+    })()
+  };
+}
+
 // src/services/SearchService.ts
 var _SearchService = class _SearchService {
-  constructor(entryRepo) {
+  constructor(entryRepo, indexStrategy) {
     this.entryRepo = entryRepo;
-    this.miniSearchEntryIdsByEntity = /* @__PURE__ */ new Map();
     this.vectorCache = /* @__PURE__ */ new Map();
     /**
      * Serializes rebuilds. `rebuildIndex` awaits a repository read between
@@ -456,35 +472,13 @@ var _SearchService = class _SearchService {
      * its flag must outlive the turn (#233 review).
      */
     this.staleEpochs = /* @__PURE__ */ new Map();
-    this.miniSearch = this.createMiniSearch();
-  }
-  /**
-   * A fresh index with the production options. clearAll() swaps one in because
-   * MiniSearch.removeAll() empties the index but leaves dirtCount (and its
-   * vacuum bookkeeping) at its old value, which would trip syncEntries'
-   * conditional vacuum early after a clear.
-   */
-  createMiniSearch() {
-    return new MiniSearch({
-      fields: ["title", "body", "tags"],
-      storeFields: ["entity_id"],
-      // Vacuuming is driven explicitly at the end of each serialized rebuild
-      // (see sync). Auto-vacuum fires on its own schedule, asynchronously with
-      // respect to the caller, and traversing the tree mid-rebuild is what
-      // threw the uncaught TypeError in MiniSearch.performVacuuming (#64).
-      autoVacuum: false,
-      searchOptions: {
-        boost: { title: 2 },
-        fuzzy: 0.2,
-        prefix: true
-      }
-    });
+    this.indexStrategy = indexStrategy;
   }
   /**
    * Rebuilds the search index and clears the vector cache for a given entity.
    * A direct replacement for manually syncing state after a DB transaction.
    *
-   * Rebuilds are serialized per instance and never reject: the MiniSearch index
+   * Rebuilds are serialized per instance and never reject: the keyword index
    * is a rebuildable cache over SQLite, so degraded keyword search is the
    * correct failure mode and killing the host process is not.
    */
@@ -493,19 +487,21 @@ var _SearchService = class _SearchService {
       try {
         try {
           const epochsBefore = new Map(this.staleEpochs);
-          await this.rebuildIndex(entityId);
-          if (entityId) {
-            if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
-              this.staleEntities.delete(entityId);
-            }
-          } else {
+          if (entityId === void 0) {
+            const rows = await this.entryRepo.findMiniSearchRows();
+            await this.indexStrategy.replaceAll(rows.map(toIndexDoc));
             for (const id of [...this.staleEntities]) {
               if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
                 this.staleEntities.delete(id);
               }
             }
+          } else {
+            await this.rebuildIndex(entityId);
+            if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
+              this.staleEntities.delete(entityId);
+            }
           }
-          await this.miniSearch.vacuum();
+          await this.indexStrategy.vacuum?.();
         } finally {
           this.evictCache(entityId);
         }
@@ -530,27 +526,20 @@ var _SearchService = class _SearchService {
    */
   async syncEntries(entityId, ids) {
     const uniqueIds = [...new Set(ids)];
-    if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && this.miniSearchEntryIdsByEntity.has(entityId)) {
+    const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
+    if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
       return this.syncChain;
     }
     const work = this.syncChain.then(async () => {
       try {
         try {
           const epochsBefore = new Map(this.staleEpochs);
-          const needsRebuild = () => this.staleEntities.has(entityId) || !this.miniSearchEntryIdsByEntity.has(entityId);
+          const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
           if (!needsRebuild()) {
             const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
-            const tracked = this.miniSearchEntryIdsByEntity.get(entityId);
-            if (tracked && !this.staleEntities.has(entityId)) {
-              for (const id of uniqueIds) {
-                if (tracked.delete(id) && this.miniSearch.has(id)) this.miniSearch.discard(id);
-              }
-              const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
-              if (documents.length > 0) this.miniSearch.addAll(documents);
-              for (const document of documents) tracked.add(document.id);
-              if (this.miniSearch.dirtCount > 0) {
-                await this.miniSearch.vacuum();
-              }
+            if (!this.staleEntities.has(entityId)) {
+              await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
+              await this.indexStrategy.vacuum?.();
               return;
             }
           }
@@ -558,7 +547,7 @@ var _SearchService = class _SearchService {
           if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
             this.staleEntities.delete(entityId);
           }
-          await this.miniSearch.vacuum();
+          await this.indexStrategy.vacuum?.();
         } finally {
           this.evictCache(entityId);
         }
@@ -578,6 +567,16 @@ var _SearchService = class _SearchService {
     this.staleEntities.add(entityId);
     this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
   }
+  /**
+   * Runs syncEntries(id, []) for each entity marked stale when this is called,
+   * so only those entities are rebuilt. Each rebuild still takes its own turn
+   * on the sync chain, and like syncEntries this never rejects. With nothing
+   * stale it resolves at once, without waiting on work already on the chain.
+   */
+  async syncStale() {
+    const ids = [...this.staleEntities];
+    await Promise.all(ids.map((id) => this.syncEntries(id, [])));
+  }
   /**
    * Clears the parsed vector cache. Useful for mid-loop flush guarantees
    * or memory pressure evictions.
@@ -592,39 +591,28 @@ var _SearchService = class _SearchService {
   /**
    * Fully resets the search service.
    */
-  clearAll() {
+  async clearAll() {
     this.vectorCache.clear();
-    this.miniSearch = this.createMiniSearch();
-    this.miniSearchEntryIdsByEntity.clear();
+    await this.indexStrategy.replaceAll([]);
     this.staleEntities.clear();
     this.staleEpochs.clear();
   }
   /**
-   * Executes a keyword search against the active MiniSearch index.
+   * Executes a keyword search against the active index strategy.
    */
-  searchKeyword(query, entityIds, limit) {
-    const entityIdSet = new Set(entityIds);
-    const results = this.miniSearch.search(query, {
-      filter: (r) => entityIdSet.has(r.entity_id),
-      combineWith: "OR"
-    });
+  async searchKeyword(query, entityIds, limit) {
+    const results = finiteResults(await this.indexStrategy.search(query, { entityIds, limit }));
     return results.sort((a, b) => this._compareSearchResults(a, b)).slice(0, limit);
   }
   /**
-   * Pre-fetches MiniSearch scores for candidate hydration, used during hybrid weighting.
+   * Pre-fetches keyword scores for candidate hydration, used during hybrid weighting.
    */
-  getMiniSearchScores(query, entityIds, preFilterLimit) {
-    const entityIdSet = new Set(entityIds);
-    let results = this.miniSearch.search(query, {
-      filter: (r) => entityIdSet.has(r.entity_id),
-      combineWith: "OR"
-    }).sort((a, b) => this._compareSearchResults(a, b));
-    if (preFilterLimit !== void 0) {
-      results = results.slice(0, preFilterLimit);
-    }
+  async getKeywordScores(query, entityIds, preFilterLimit) {
+    const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
+    const results = finiteResults(await this.indexStrategy.search(query, opts));
     if (results.length === 0) return /* @__PURE__ */ new Map();
-    const maxMsScore = Math.max(1, results[0]?.score ?? 1);
-    return new Map(results.map((r) => [r.id, r.score / maxMsScore]));
+    const maxScore = Math.max(1, results[0]?.score ?? 1);
+    return new Map(results.map((r) => [r.id, r.score / maxScore]));
   }
   /**
    * Score candidate rows using in-process JS cosine similarity.
@@ -686,53 +674,16 @@ var _SearchService = class _SearchService {
     return scored.slice(0, limit);
   }
   // --- Internal Index Management ---
+  /**
+   * Rebuild a single entity's index from scratch. Reads every row for
+   * `entityId` from the repository (already filtered by `deleted_at IS NULL`)
+   * and hands the lot to `indexStrategy.replaceEntity`, which owns the
+   * drop-then-add bookkeeping so callers don't need to know the strategy's
+   * id-tracking shape.
+   */
   async rebuildIndex(entityId) {
-    if (entityId) {
-      const rows2 = await this.entryRepo.findMiniSearchRows(entityId);
-      const previousIds = this.miniSearchEntryIdsByEntity.get(entityId);
-      if (previousIds) {
-        for (const id of previousIds) {
-          this.miniSearch.discard(id);
-        }
-      }
-      const documents2 = rows2.map((row) => this.normalizeMiniSearchRow(row));
-      if (documents2.length > 0) {
-        this.miniSearch.addAll(documents2);
-      }
-      this.miniSearchEntryIdsByEntity.set(
-        entityId,
-        new Set(documents2.map((document) => document.id))
-      );
-      return;
-    }
-    const rows = await this.entryRepo.findMiniSearchRows();
-    this.miniSearch.removeAll();
-    this.miniSearchEntryIdsByEntity.clear();
-    const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
-    if (documents.length > 0) {
-      this.miniSearch.addAll(documents);
-    }
-    for (const document of documents) {
-      const ids = this.miniSearchEntryIdsByEntity.get(document.entity_id) ?? /* @__PURE__ */ new Set();
-      ids.add(document.id);
-      this.miniSearchEntryIdsByEntity.set(document.entity_id, ids);
-    }
-  }
-  normalizeMiniSearchRow(row) {
-    return {
-      id: row.id,
-      entity_id: row.entity_id,
-      title: row.title,
-      body: row.body,
-      tags: (() => {
-        try {
-          const parsed = JSON.parse(row.tags);
-          return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
-        } catch {
-          return row.tags;
-        }
-      })()
-    };
+    const rows = await this.entryRepo.findMiniSearchRows(entityId);
+    await this.indexStrategy.replaceEntity(entityId, rows.map(toIndexDoc));
   }
   _tieBreakSort(items) {
     items.sort((a, b) => this._compareScoredRows(a, b));
@@ -747,11 +698,11 @@ var _SearchService = class _SearchService {
     return a.id.localeCompare(b.id);
   }
   /**
-   * MiniSearch breaks equal-score ties by internal insertion order, which
-   * syncEntries changes: a discarded-and-re-added id moves to the end. Every
-   * caller truncates these results to a limit, so which ids survive a tie at
-   * the boundary must not depend on write history. Re-sort exact score ties
-   * by id — the same final tie-break _compareScoredRows applies.
+   * The keyword index breaks equal-score ties by internal insertion order,
+   * which syncEntries changes: a discarded-and-re-added id moves to the end.
+   * Every caller truncates these results to a limit, so which ids survive a
+   * tie at the boundary must not depend on write history. Re-sort exact
+   * score ties by id — the same final tie-break _compareScoredRows applies.
    */
   _compareSearchResults(a, b) {
     const scoreDiff = b.score - a.score;
@@ -772,6 +723,9 @@ _SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
  */
 _SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
 var SearchService = _SearchService;
+function finiteResults(results) {
+  return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
+}
 
 // src/services/JobManager.ts
 var JobManager = class {
@@ -3652,7 +3606,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
       }
     }
     if (!ranked) {
-      ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
+      ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
     }
     const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
     const byId = new Map(facts.map((f) => [f.id, f]));
@@ -5113,7 +5067,7 @@ var MaintenanceService = class {
     if (!query) return [];
     const cached = cache?.get(query);
     if (cached) return cached;
-    const hits = this.searchService.searchKeyword(
+    const hits = await this.searchService.searchKeyword(
       query,
       [entityId],
       cap * HEAL_ANCHOR_SEARCH_OVERFETCH
@@ -6105,7 +6059,7 @@ var RetrievalService = class {
           let miniSearchScores;
           if (effectivePreFilterLimit !== void 0) {
             populateCache = false;
-            const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
+            const preResults = (await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER)).filter((r) => !excludedIds.has(r.id));
             if (preResults.length === 0) {
               candidateRows = null;
             } else {
@@ -6132,7 +6086,7 @@ var RetrievalService = class {
               candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
             }
             if (weight !== void 0 && weight < 1) {
-              miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
+              miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
             }
           }
           if (candidateRows === null) {
@@ -6375,7 +6329,7 @@ var RetrievalService = class {
                   const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
                   const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
                   const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
-                  const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
+                  const keywordResults = await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
                   const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
                   const topResultIds = new Set(topResults.map((r) => r.id));
                   const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
@@ -6478,7 +6432,7 @@ var RetrievalService = class {
       if (!usedEmbed && scoredEntityIds.length > 0) {
         const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
         const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
-        const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
+        const results = (await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit)).filter((r) => !excludedIds.has(r.id));
         const candidates = results.map((r) => ({
           id: r.id,
           entity_id: r.entity_id,
@@ -6786,5 +6740,5 @@ var WriteService = class {
 };
 
 export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
-//# sourceMappingURL=chunk-M76CF2Q6.mjs.map
-//# sourceMappingURL=chunk-M76CF2Q6.mjs.map
+//# sourceMappingURL=chunk-7FRLQ7ZO.mjs.map
+//# sourceMappingURL=chunk-7FRLQ7ZO.mjs.map