@equationalapplications/core-llm-wiki 7.8.0 → 7.9.1

package/dist/index.mjs

@@ -1,6 +1,7 @@
-import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, WikiTransactionError, extractSqliteCode } from './chunk-M76CF2Q6.mjs';
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-M76CF2Q6.mjs';
+import { BaseRepository, WikiGraphNodeOwnershipConflict, liveAtSql, __privateAdd, MetadataRepository, EmbeddingService, SearchService, TemporalService, JobManager, PromptService, resolveGrounding, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, DiagnosticBuffer, validateManifest, WikiDraftNotFound, generateId, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, typeSatisfies, normalizeTitleKey, edgeTripleAllowed, MAX_EMBED_ATTEMPTS, UsageMeter, WikiBusyError, WikiInvalidReadOptions, isLiveAt, packFactsByBudget, factTokenCost, WikiTransactionError, extractSqliteCode } from './chunk-7FRLQ7ZO.mjs';
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, EMBED_CHARS_CEILING, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-7FRLQ7ZO.mjs';
 import { deriveTrustTier, isStaleAfter, appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, latestVerified, formatSourcesJson, appendEventIdComment, parseVerifiedFlexible, parseCitationsList } from '@equationalapplications/core-okf';
+import MiniSearch from 'minisearch';
 
 // src/db/schema.ts
 async function setupDatabase(db, prefix) {
@@ -2803,6 +2804,195 @@ function mapRowToEdge(row) {
     created_at: Number(row.created_at)
   };
 }
+var MiniSearchIndexStrategy = class {
+  constructor() {
+    /**
+     * Per-entity id set. `replaceEntity()` and `hasIndexedEntity()` read it;
+     * `addDocuments()` and `discardIds()` keep it in sync. `SearchService` does
+     * not see this map; it only sees `hasIndexedEntity`. The previous
+     * `miniSearchEntryIdsByEntity` lived on `SearchService`.
+     */
+    this.entryIdsByEntity = /* @__PURE__ */ new Map();
+    /**
+     * Reverse of `entryIdsByEntity`. `replace(entityId, ids, …)` uses it to
+     * decide which ids are owned by `entityId` (the only safe membership test
+     * — a rebuild that failed between its discard pass and the set replacement
+     * can leave the forward set claiming ids the MiniSearch instance does not
+     * hold). The pre-extraction `syncEntries` deleted from one set per id, not
+     * from every entity's set.
+     */
+    this.entityIdById = /* @__PURE__ */ new Map();
+    this.index = this.createIndex();
+  }
+  /**
+   * Build a fresh MiniSearch instance with the project's index options.
+   * Called from the constructor and from `replaceAll`. We do not use
+   * `removeAll()` on an existing instance because that empties the index but
+   * leaves `dirtCount` and the vacuum bookkeeping intact, which would trip
+   * `syncEntries`' conditional vacuum early after a clear.
+   */
+  createIndex() {
+    return new MiniSearch({
+      fields: ["title", "body", "tags"],
+      storeFields: ["entity_id"],
+      // Vacuuming is driven explicitly at the end of each serialized rebuild
+      // (see SearchService.syncEntries). Auto-vacuum fires on its own
+      // schedule, asynchronously with respect to the caller, and traversing
+      // the tree mid-rebuild is what threw the uncaught TypeError in
+      // MiniSearch.performVacuuming (#64).
+      autoVacuum: false,
+      searchOptions: {
+        boost: { title: 2 },
+        fuzzy: 0.2,
+        prefix: true
+      }
+    });
+  }
+  /**
+   * Incremental sync. Removes only ids currently tracked under `entityId`
+   * (other entities' docs are untouched), then upserts `documents` by id.
+   * The same id appearing in both `ids` and `documents` is discarded first
+   * so the upsert does not throw on a duplicate id.
+   *
+   * Every document must carry `entityId` as its `entity_id`; a mismatch is
+   * rejected before the write mutates anything. A tracking-only call
+   * (`replace(entityId, [], [])`) registers the entity as indexed, matching
+   * the interface contract.
+   *
+   * Atomicity: never awaits between the discard pass and `addDocuments`, so
+   * a concurrent `search()` sees the index wholly before or wholly after,
+   * never half-updated. This is the precondition the pre-extraction
+   * `syncEntries` kept with its "no await from here to `addAll`" comment.
+   */
+  async replace(entityId, ids, documents) {
+    assertOwnedBy(entityId, documents);
+    this.discardIds(ids, entityId);
+    if (!this.entryIdsByEntity.has(entityId)) this.entryIdsByEntity.set(entityId, /* @__PURE__ */ new Set());
+    this.addDocuments(documents);
+  }
+  /**
+   * Per-entity rebuild. Removes every id tracked under `entityId` (with the
+   * `index.has(id)` guard so a stale tracking set cannot make `discard()`
+   * throw), then writes `documents`. The entity stays registered as
+   * indexed — even when `documents` is empty — so the next `syncEntries`
+   * call can take the fast path. Every document must belong to `entityId`.
+   */
+  async replaceEntity(entityId, documents) {
+    assertOwnedBy(entityId, documents);
+    const previous = this.entryIdsByEntity.get(entityId);
+    if (previous) this.discardIds([...previous]);
+    this.entryIdsByEntity.set(entityId, /* @__PURE__ */ new Set());
+    this.addDocuments(documents);
+  }
+  /**
+   * Global rebuild / clear. Replaces the MiniSearch instance and clears both
+   * id maps, so `hasIndexedEntity` returns `false` for every entity afterwards.
+   * `SearchService.clearAll()` calls `replaceAll([])`.
+   */
+  async replaceAll(documents) {
+    this.index = this.createIndex();
+    this.entryIdsByEntity.clear();
+    this.entityIdById.clear();
+    this.addDocuments(documents);
+  }
+  /**
+   * Best-match-first search over the documents whose `entity_id` is in
+   * `options.entityIds`. `preFilterLimit` overrides `limit` (hybrid retrieval
+   * uses this to over-fetch candidates before the `limit` cap). Scores are
+   * raw BM25+ — `SearchService.getKeywordScores` and `searchKeyword` apply
+   * any scaling, not the strategy.
+   */
+  async search(query, options) {
+    if (options.entityIds.length === 0) return [];
+    const entityIdSet = new Set(options.entityIds);
+    const limit = options.preFilterLimit ?? options.limit;
+    const raw = this.index.search(query, {
+      filter: (r) => entityIdSet.has(r.entity_id),
+      combineWith: "OR"
+    });
+    raw.sort((a, b) => {
+      const scoreDiff = b.score - a.score;
+      if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
+      return a.id.localeCompare(b.id);
+    });
+    return raw.slice(0, limit).map((r) => ({
+      id: r.id,
+      entity_id: r.entity_id,
+      score: r.score
+    }));
+  }
+  /**
+   * Explicit vacuum. MiniSearch 7.2.0's `dirtCount` rule: search after an
+   * un-vacuumed discard inverts the relative scores of term-sharing
+   * documents at any index size. `SearchService.syncEntries` calls this
+   * after a discard-bearing turn. No-op when `dirtCount === 0`.
+   */
+  async vacuum() {
+    if (this.index.dirtCount > 0) {
+      await this.index.vacuum();
+    }
+  }
+  /**
+   * `true` when `entityId` has been indexed at least once under the current
+   * strategy state. Backed by `entryIdsByEntity.has(entityId)`, which is
+   * kept in sync by `addDocuments`, `replaceEntity`, and `replaceAll`.
+   */
+  hasIndexedEntity(entityId) {
+    return this.entryIdsByEntity.has(entityId);
+  }
+  /**
+   * Add `documents` to the MiniSearch instance and register them in
+   * `entryIdsByEntity` and `entityIdById`. MiniSearch mutates documents
+   * during `add`, so each doc is copied first. `add` throws on an id the
+   * index already holds — `replace` and `replaceEntity` discard first for
+   * that reason.
+   *
+   * Adds and registers one document at a time: if a duplicate id throws
+   * part-way through, every document already in the index is tracked, so
+   * the next rebuild can discard it instead of failing on it again. This is
+   * the precondition the per-turn recovery path relies on (#258 review).
+   */
+  addDocuments(documents) {
+    for (const d of documents) {
+      const doc = { ...d };
+      this.index.add(doc);
+      const set = this.entryIdsByEntity.get(doc.entity_id) ?? /* @__PURE__ */ new Set();
+      set.add(doc.id);
+      this.entryIdsByEntity.set(doc.entity_id, set);
+      this.entityIdById.set(doc.id, doc.entity_id);
+    }
+  }
+  /**
+   * Discard `ids` from the MiniSearch instance and from both id maps, but
+   * only for ids currently owned by `entityId` (the second argument). When
+   * `entityId` is omitted (used by `replaceEntity` for the bulk rebuild
+   * path), every id is discarded regardless of owning entity.
+   *
+   * `index.has(id)` is the membership test of record: a rebuild that failed
+   * between its own discard pass and the set replacement can leave the
+   * tracking set claiming ids the index no longer holds — `discard()` throws
+   * on those, so we guard.
+   */
+  discardIds(ids, entityId) {
+    for (const id of ids) {
+      const owning = this.entityIdById.get(id);
+      if (owning === void 0) continue;
+      if (entityId !== void 0 && owning !== entityId) continue;
+      if (this.index.has(id)) this.index.discard(id);
+      this.entryIdsByEntity.get(owning)?.delete(id);
+      this.entityIdById.delete(id);
+    }
+  }
+};
+function assertOwnedBy(entityId, documents) {
+  for (const doc of documents) {
+    if (doc.entity_id !== entityId) {
+      throw new Error(
+        `IndexStrategy write for entity ${entityId} got document ${doc.id} under entity ${doc.entity_id}`
+      );
+    }
+  }
+}
 
 // src/services/MaintenanceScheduler.ts
 var MaintenanceScheduler = class {
@@ -3509,7 +3699,10 @@ var WikiMemory = class {
     );
     this.lintService = new LintService(this.lintRepo, this.ontologyService);
     this.embeddingService = new EmbeddingService(this.db, this.options, this.entryRepo, this.metadataRepo);
-    this.searchService = new SearchService(this.entryRepo);
+    this.searchService = new SearchService(
+      this.entryRepo,
+      new MiniSearchIndexStrategy()
+    );
     this.temporalService = new TemporalService(this.db, this.options, this.entryRepo, this.searchService, this.embeddingService);
     this.jobManager = new JobManager(this.prefix);
     this.promptService = new PromptService(options.config?.prompts, resolveGrounding(options.config?.grounding));
@@ -3902,6 +4095,32 @@ var WikiMemory = class {
   clearVectorCache() {
     this.searchService.evictCache();
   }
+  /**
+   * Brings the keyword search index up to date with SQLite after a host
+   * transaction that used `upsertGraph` has committed. Core never sees that
+   * commit, so without this the nodes stay out of keyword, hybrid and
+   * fallback retrieval until the entity's next core write.
+   *
+   * With `entityId`, rebuilds that entity if it is stale or has never been
+   * indexed; otherwise it is a no-op that still waits for rebuilds already
+   * queued. Without an argument, rebuilds every entity currently marked
+   * stale; when none is, it resolves immediately and does not wait for rebuilds
+   * already queued.
+   *
+   * Call only after the host's transaction commits: called inside it, the
+   * rebuild may index rows that later roll back, or miss them and clear the
+   * stale flag anyway. Serialized with core's internal index syncs and takes
+   * no maintenance lock. Never rejects for a valid `entityId` — on failure the
+   * entity stays stale and the next call retries it. Does not compute
+   * embeddings; that remains the maintenance sweep's job.
+   *
+   * @throws {TypeError} when `entityId` is given but is not a non-empty string.
+   */
+  async syncSearchIndex(entityId) {
+    if (entityId === void 0) return this.searchService.syncStale();
+    assertEntityId(entityId);
+    return this.searchService.syncEntries(entityId, []);
+  }
   async exportDump(entityIds) {
     return this.importExportService.exportDump(entityIds);
   }
@@ -3948,9 +4167,11 @@ var WikiMemory = class {
    * MUST be called from inside an open `db.withTransactionAsync` callback —
    * the third argument is the caller's `tx`. The method does not open a
    * nested transaction, does not acquire any lock, and does not perform
-   * post-commit work (search sync, embedding, cache eviction). Hosts that
-   * want embeddings synced should drive the existing maintenance sweep
-   * (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
+   * post-commit work (search sync, embedding, cache eviction). After the
+   * transaction commits, call `syncSearchIndex(entityId)` — or
+   * `syncSearchIndex()` after writing several entities in one transaction —
+   * to make the new nodes keyword-searchable. Embeddings still come from the maintenance
+   * sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
    * scoped via `listEntityIds`).
    *
    * Contract (full pre-flight validation lives inside