@equationalapplications/core-llm-wiki 7.8.0 → 7.9.1

package/dist/testing.js

@@ -1,11 +1,6 @@
 'use strict';
 
 var coreOkf = require('@equationalapplications/core-okf');
-var MiniSearch = require('minisearch');
-
-function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
-
-var MiniSearch__default = /*#__PURE__*/_interopDefault(MiniSearch);
 
 // src/db/sqliteCodes.ts
 var SQLITE_RESULT_CODE_NAMES = {
@@ -3183,7 +3178,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
       }
     }
     if (!ranked) {
-      ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
+      ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
     }
     const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
     const byId = new Map(facts.map((f) => [f.id, f]));
@@ -4634,7 +4629,7 @@ var MaintenanceService = class {
     if (!query) return [];
     const cached = cache?.get(query);
     if (cached) return cached;
-    const hits = this.searchService.searchKeyword(
+    const hits = await this.searchService.searchKeyword(
       query,
       [entityId],
       cap * HEAL_ANCHOR_SEARCH_OVERFETCH
@@ -4897,7 +4892,7 @@ var RetrievalService = class {
           let miniSearchScores;
           if (effectivePreFilterLimit !== void 0) {
             populateCache = false;
-            const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
+            const preResults = (await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER)).filter((r) => !excludedIds.has(r.id));
             if (preResults.length === 0) {
               candidateRows = null;
             } else {
@@ -4924,7 +4919,7 @@ var RetrievalService = class {
               candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
             }
             if (weight !== void 0 && weight < 1) {
-              miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
+              miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
             }
           }
           if (candidateRows === null) {
@@ -5167,7 +5162,7 @@ var RetrievalService = class {
                   const hasActiveFloorsKeywordFallback = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
                   const keywordOversampledLimit = hasActiveFloorsKeywordFallback ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
                   const preFilteredIds = effectivePreFilterLimit !== void 0 ? new Set(candidateRows.map((r) => r.id)) : void 0;
-                  const keywordResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
+                  const keywordResults = await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, keywordOversampledLimit);
                   const topResults = preFilteredIds === void 0 ? keywordResults : keywordResults.filter((r) => preFilteredIds.has(r.id));
                   const topResultIds = new Set(topResults.map((r) => r.id));
                   const candidateMap = new Map(candidateRows.filter((r) => topResultIds.has(r.id)).map((row) => [row.id, row]));
@@ -5270,7 +5265,7 @@ var RetrievalService = class {
       if (!usedEmbed && scoredEntityIds.length > 0) {
         const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
         const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
-        const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
+        const results = (await this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit)).filter((r) => !excludedIds.has(r.id));
         const candidates = results.map((r) => ({
           id: r.id,
           entity_id: r.entity_id,
@@ -5407,10 +5402,29 @@ var RetrievalService = class {
     return scored;
   }
 };
+
+// src/utils/indexDoc.ts
+function toIndexDoc(row) {
+  return {
+    id: row.id,
+    entity_id: row.entity_id,
+    title: row.title,
+    body: row.body,
+    tags: (() => {
+      try {
+        const parsed = JSON.parse(row.tags);
+        return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
+      } catch {
+        return row.tags;
+      }
+    })()
+  };
+}
+
+// src/services/SearchService.ts
 var _SearchService = class _SearchService {
-  constructor(entryRepo) {
+  constructor(entryRepo, indexStrategy) {
     this.entryRepo = entryRepo;
-    this.miniSearchEntryIdsByEntity = /* @__PURE__ */ new Map();
     this.vectorCache = /* @__PURE__ */ new Map();
     /**
      * Serializes rebuilds. `rebuildIndex` awaits a repository read between
@@ -5436,35 +5450,13 @@ var _SearchService = class _SearchService {
      * its flag must outlive the turn (#233 review).
      */
     this.staleEpochs = /* @__PURE__ */ new Map();
-    this.miniSearch = this.createMiniSearch();
-  }
-  /**
-   * A fresh index with the production options. clearAll() swaps one in because
-   * MiniSearch.removeAll() empties the index but leaves dirtCount (and its
-   * vacuum bookkeeping) at its old value, which would trip syncEntries'
-   * conditional vacuum early after a clear.
-   */
-  createMiniSearch() {
-    return new MiniSearch__default.default({
-      fields: ["title", "body", "tags"],
-      storeFields: ["entity_id"],
-      // Vacuuming is driven explicitly at the end of each serialized rebuild
-      // (see sync). Auto-vacuum fires on its own schedule, asynchronously with
-      // respect to the caller, and traversing the tree mid-rebuild is what
-      // threw the uncaught TypeError in MiniSearch.performVacuuming (#64).
-      autoVacuum: false,
-      searchOptions: {
-        boost: { title: 2 },
-        fuzzy: 0.2,
-        prefix: true
-      }
-    });
+    this.indexStrategy = indexStrategy;
   }
   /**
    * Rebuilds the search index and clears the vector cache for a given entity.
    * A direct replacement for manually syncing state after a DB transaction.
    *
-   * Rebuilds are serialized per instance and never reject: the MiniSearch index
+   * Rebuilds are serialized per instance and never reject: the keyword index
    * is a rebuildable cache over SQLite, so degraded keyword search is the
    * correct failure mode and killing the host process is not.
    */
@@ -5473,19 +5465,21 @@ var _SearchService = class _SearchService {
       try {
         try {
           const epochsBefore = new Map(this.staleEpochs);
-          await this.rebuildIndex(entityId);
-          if (entityId) {
-            if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
-              this.staleEntities.delete(entityId);
-            }
-          } else {
+          if (entityId === void 0) {
+            const rows = await this.entryRepo.findMiniSearchRows();
+            await this.indexStrategy.replaceAll(rows.map(toIndexDoc));
             for (const id of [...this.staleEntities]) {
               if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
                 this.staleEntities.delete(id);
               }
             }
+          } else {
+            await this.rebuildIndex(entityId);
+            if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
+              this.staleEntities.delete(entityId);
+            }
           }
-          await this.miniSearch.vacuum();
+          await this.indexStrategy.vacuum?.();
         } finally {
           this.evictCache(entityId);
         }
@@ -5510,27 +5504,20 @@ var _SearchService = class _SearchService {
    */
   async syncEntries(entityId, ids) {
     const uniqueIds = [...new Set(ids)];
-    if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && this.miniSearchEntryIdsByEntity.has(entityId)) {
+    const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
+    if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
       return this.syncChain;
     }
     const work = this.syncChain.then(async () => {
       try {
         try {
           const epochsBefore = new Map(this.staleEpochs);
-          const needsRebuild = () => this.staleEntities.has(entityId) || !this.miniSearchEntryIdsByEntity.has(entityId);
+          const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
           if (!needsRebuild()) {
             const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
-            const tracked = this.miniSearchEntryIdsByEntity.get(entityId);
-            if (tracked && !this.staleEntities.has(entityId)) {
-              for (const id of uniqueIds) {
-                if (tracked.delete(id) && this.miniSearch.has(id)) this.miniSearch.discard(id);
-              }
-              const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
-              if (documents.length > 0) this.miniSearch.addAll(documents);
-              for (const document of documents) tracked.add(document.id);
-              if (this.miniSearch.dirtCount > 0) {
-                await this.miniSearch.vacuum();
-              }
+            if (!this.staleEntities.has(entityId)) {
+              await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
+              await this.indexStrategy.vacuum?.();
               return;
             }
           }
@@ -5538,7 +5525,7 @@ var _SearchService = class _SearchService {
           if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
             this.staleEntities.delete(entityId);
           }
-          await this.miniSearch.vacuum();
+          await this.indexStrategy.vacuum?.();
         } finally {
           this.evictCache(entityId);
         }
@@ -5558,6 +5545,16 @@ var _SearchService = class _SearchService {
     this.staleEntities.add(entityId);
     this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
   }
+  /**
+   * Runs syncEntries(id, []) for each entity marked stale when this is called,
+   * so only those entities are rebuilt. Each rebuild still takes its own turn
+   * on the sync chain, and like syncEntries this never rejects. With nothing
+   * stale it resolves at once, without waiting on work already on the chain.
+   */
+  async syncStale() {
+    const ids = [...this.staleEntities];
+    await Promise.all(ids.map((id) => this.syncEntries(id, [])));
+  }
   /**
    * Clears the parsed vector cache. Useful for mid-loop flush guarantees
    * or memory pressure evictions.
@@ -5572,39 +5569,28 @@ var _SearchService = class _SearchService {
   /**
    * Fully resets the search service.
    */
-  clearAll() {
+  async clearAll() {
     this.vectorCache.clear();
-    this.miniSearch = this.createMiniSearch();
-    this.miniSearchEntryIdsByEntity.clear();
+    await this.indexStrategy.replaceAll([]);
     this.staleEntities.clear();
     this.staleEpochs.clear();
   }
   /**
-   * Executes a keyword search against the active MiniSearch index.
+   * Executes a keyword search against the active index strategy.
    */
-  searchKeyword(query, entityIds, limit) {
-    const entityIdSet = new Set(entityIds);
-    const results = this.miniSearch.search(query, {
-      filter: (r) => entityIdSet.has(r.entity_id),
-      combineWith: "OR"
-    });
+  async searchKeyword(query, entityIds, limit) {
+    const results = finiteResults(await this.indexStrategy.search(query, { entityIds, limit }));
     return results.sort((a, b) => this._compareSearchResults(a, b)).slice(0, limit);
   }
   /**
-   * Pre-fetches MiniSearch scores for candidate hydration, used during hybrid weighting.
+   * Pre-fetches keyword scores for candidate hydration, used during hybrid weighting.
    */
-  getMiniSearchScores(query, entityIds, preFilterLimit) {
-    const entityIdSet = new Set(entityIds);
-    let results = this.miniSearch.search(query, {
-      filter: (r) => entityIdSet.has(r.entity_id),
-      combineWith: "OR"
-    }).sort((a, b) => this._compareSearchResults(a, b));
-    if (preFilterLimit !== void 0) {
-      results = results.slice(0, preFilterLimit);
-    }
+  async getKeywordScores(query, entityIds, preFilterLimit) {
+    const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
+    const results = finiteResults(await this.indexStrategy.search(query, opts));
     if (results.length === 0) return /* @__PURE__ */ new Map();
-    const maxMsScore = Math.max(1, results[0]?.score ?? 1);
-    return new Map(results.map((r) => [r.id, r.score / maxMsScore]));
+    const maxScore = Math.max(1, results[0]?.score ?? 1);
+    return new Map(results.map((r) => [r.id, r.score / maxScore]));
   }
   /**
    * Score candidate rows using in-process JS cosine similarity.
@@ -5666,53 +5652,16 @@ var _SearchService = class _SearchService {
     return scored.slice(0, limit);
   }
   // --- Internal Index Management ---
+  /**
+   * Rebuild a single entity's index from scratch. Reads every row for
+   * `entityId` from the repository (already filtered by `deleted_at IS NULL`)
+   * and hands the lot to `indexStrategy.replaceEntity`, which owns the
+   * drop-then-add bookkeeping so callers don't need to know the strategy's
+   * id-tracking shape.
+   */
   async rebuildIndex(entityId) {
-    if (entityId) {
-      const rows2 = await this.entryRepo.findMiniSearchRows(entityId);
-      const previousIds = this.miniSearchEntryIdsByEntity.get(entityId);
-      if (previousIds) {
-        for (const id of previousIds) {
-          this.miniSearch.discard(id);
-        }
-      }
-      const documents2 = rows2.map((row) => this.normalizeMiniSearchRow(row));
-      if (documents2.length > 0) {
-        this.miniSearch.addAll(documents2);
-      }
-      this.miniSearchEntryIdsByEntity.set(
-        entityId,
-        new Set(documents2.map((document) => document.id))
-      );
-      return;
-    }
-    const rows = await this.entryRepo.findMiniSearchRows();
-    this.miniSearch.removeAll();
-    this.miniSearchEntryIdsByEntity.clear();
-    const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
-    if (documents.length > 0) {
-      this.miniSearch.addAll(documents);
-    }
-    for (const document of documents) {
-      const ids = this.miniSearchEntryIdsByEntity.get(document.entity_id) ?? /* @__PURE__ */ new Set();
-      ids.add(document.id);
-      this.miniSearchEntryIdsByEntity.set(document.entity_id, ids);
-    }
-  }
-  normalizeMiniSearchRow(row) {
-    return {
-      id: row.id,
-      entity_id: row.entity_id,
-      title: row.title,
-      body: row.body,
-      tags: (() => {
-        try {
-          const parsed = JSON.parse(row.tags);
-          return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
-        } catch {
-          return row.tags;
-        }
-      })()
-    };
+    const rows = await this.entryRepo.findMiniSearchRows(entityId);
+    await this.indexStrategy.replaceEntity(entityId, rows.map(toIndexDoc));
   }
   _tieBreakSort(items) {
     items.sort((a, b) => this._compareScoredRows(a, b));
@@ -5727,11 +5676,11 @@ var _SearchService = class _SearchService {
     return a.id.localeCompare(b.id);
   }
   /**
-   * MiniSearch breaks equal-score ties by internal insertion order, which
-   * syncEntries changes: a discarded-and-re-added id moves to the end. Every
-   * caller truncates these results to a limit, so which ids survive a tie at
-   * the boundary must not depend on write history. Re-sort exact score ties
-   * by id — the same final tie-break _compareScoredRows applies.
+   * The keyword index breaks equal-score ties by internal insertion order,
+   * which syncEntries changes: a discarded-and-re-added id moves to the end.
+   * Every caller truncates these results to a limit, so which ids survive a
+   * tie at the boundary must not depend on write history. Re-sort exact
+   * score ties by id — the same final tie-break _compareScoredRows applies.
    */
   _compareSearchResults(a, b) {
     const scoreDiff = b.score - a.score;
@@ -5752,6 +5701,9 @@ _SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
  */
 _SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
 var SearchService = _SearchService;
+function finiteResults(results) {
+  return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
+}
 
 // src/services/JobManager.ts
 var JobManager = class {