@equationalapplications/core-llm-wiki 4.22.0 → 4.23.1

package/README.md

@@ -42,6 +42,7 @@ const wikiMemory = new WikiMemory(db, {
       // Your LLM call for extracting facts, tasks
       return 'Model output';
     },
+    maxOutputTokens: 4096, // optional — lets runHeal/runOntologyBackfill size their first LLM call correctly instead of discovering the ceiling via a truncated response and retrying smaller
     embed: async (text: string) => {
       // Your embedding service (e.g., OpenAI, Cohere, local)
       const response = await fetch('https://your-app.example.com/api/embed', {
@@ -550,18 +551,21 @@ have `okf_type = NULL` and no edges, so `traverseGraph` cannot reach them.
 
 ```ts
 const result = await wiki.runOntologyBackfill(entityId);
-// { scanned, typed, failedValidation, edgesAdded, remaining, deferred }
+// { scanned, typed, failedValidation, edgesAdded, remaining, deferred, skipped }
 ```
 
 - **When to call it:** you own the trigger — the library provides the operation,
   not a scheduler (same as `runLibrarian`/`runHeal`). A good default is after
   each sync/import completes. `WikiBusyError` under concurrency is safe to
   swallow; the next trigger retries.
-- **Cost model:** one LLM call only when eligible untyped facts exist; otherwise
-  one SELECT. Each run scans at most 25 facts (override via
-  `options.batchSize`), oldest first, with the full serialized prompt
-  (system + user) capped at 40k chars. One exception: a single fact whose
-  prompt alone exceeds the cap is still sent by itself so it is never starved.
+- **Cost model:** one or more LLM calls only when eligible untyped facts exist;
+  otherwise one SELECT. Each run scans at most 25 facts (override via
+  `options.batchSize`), oldest first. Facts are sent to the LLM in bounded
+  sub-batches — sized from `llmProvider.maxOutputTokens` when supplied,
+  otherwise a conservative default — with the full serialized prompt
+  (system + user) capped at 40k chars. A sub-batch whose response is truncated
+  or fails to parse is halved and retried automatically; a single fact that
+  still fails alone is counted in `skipped` rather than aborting the run.
 - **Convergence:** loop `while (result.remaining > 0)` to drain a backlog.
   `remaining` counts only *eligible* untyped facts, so the loop terminates even
   when unclassifiable facts exist; those are cooldown-stamped and retried after

package/dist/{chunk-LCAU6NYC.mjs → chunk-MYZJLVX4.mjs}

@@ -256,9 +256,23 @@ var _SearchService = class _SearchService {
     this.entryRepo = entryRepo;
     this.miniSearchEntryIdsByEntity = /* @__PURE__ */ new Map();
     this.vectorCache = /* @__PURE__ */ new Map();
+    /**
+     * Serializes rebuilds. `rebuildIndex` awaits a repository read between
+     * snapshotting the previous id set and discarding it, so two concurrent
+     * sync() calls for one entity can interleave: a slow, stale read lands last
+     * and discards documents the fresh read just added. Chaining also keeps
+     * discard()/addAll() out of each other's way, which is what accrued the
+     * auto-vacuum debt behind the TypeError in #64.
+     */
+    this.syncChain = Promise.resolve();
     this.miniSearch = new MiniSearch({
       fields: ["title", "body", "tags"],
       storeFields: ["entity_id"],
+      // Vacuuming is driven explicitly at the end of each serialized rebuild
+      // (see sync). Auto-vacuum fires on its own schedule, asynchronously with
+      // respect to the caller, and traversing the tree mid-rebuild is what
+      // threw the uncaught TypeError in MiniSearch.performVacuuming (#64).
+      autoVacuum: false,
       searchOptions: {
         boost: { title: 2 },
         fuzzy: 0.2,
@@ -269,10 +283,26 @@ var _SearchService = class _SearchService {
   /**
    * Rebuilds the search index and clears the vector cache for a given entity.
    * A direct replacement for manually syncing state after a DB transaction.
+   *
+   * Rebuilds are serialized per instance and never reject: the MiniSearch index
+   * is a rebuildable cache over SQLite, so degraded keyword search is the
+   * correct failure mode and killing the host process is not.
    */
   async sync(entityId) {
-    await this.rebuildIndex(entityId);
-    this.evictCache(entityId);
+    const work = this.syncChain.then(async () => {
+      try {
+        try {
+          await this.rebuildIndex(entityId);
+          await this.miniSearch.vacuum();
+        } finally {
+          this.evictCache(entityId);
+        }
+      } catch (err) {
+        console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
+      }
+    });
+    this.syncChain = work;
+    return work;
   }
   /**
    * Clears the parsed vector cache. Useful for mid-loop flush guarantees
@@ -1423,12 +1453,122 @@ var MetadataRepository = class extends BaseRepository {
   }
 };
 
+// src/services/BoundedLlmCall.ts
+var DEFAULT_BATCH_SIZE = 10;
+var ESTIMATED_OUTPUT_TOKENS_PER_ITEM = 150;
+var OUTPUT_BUDGET_FRACTION = 0.8;
+var TRUNCATION_PATTERNS = [
+  /truncat/i,
+  /token limit/i,
+  /max(imum)?[ _-]?tokens?/i,
+  /output limit/i,
+  /length limit/i,
+  /finish[_ ]?reason/i
+];
+var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
+function isTruncationError(err) {
+  const message = err instanceof Error ? err.message : String(err ?? "");
+  if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
+  return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
+}
+function initialBatchSize(maxOutputTokens) {
+  if (!maxOutputTokens || !Number.isFinite(maxOutputTokens) || maxOutputTokens <= 0) {
+    return DEFAULT_BATCH_SIZE;
+  }
+  const estimate = Math.floor(
+    maxOutputTokens * OUTPUT_BUDGET_FRACTION / ESTIMATED_OUTPUT_TOKENS_PER_ITEM
+  );
+  return Math.max(DEFAULT_BATCH_SIZE, estimate);
+}
+var promptLength = (prompts) => prompts.systemPrompt.length + prompts.userPrompt.length;
+async function runBatched(args) {
+  const { items, buildPrompt, call, parse, maxPromptChars, maxOutputTokens, onSkip } = args;
+  const results = [];
+  const skipped = [];
+  let batches = 0;
+  let batchSize = initialBatchSize(maxOutputTokens);
+  const trim = async (candidate) => {
+    const whole = await buildPrompt(candidate);
+    if (candidate.length <= 1 || promptLength(whole) <= maxPromptChars) {
+      return { batch: candidate, prompts: whole };
+    }
+    let low = 2;
+    let high = candidate.length - 1;
+    let best;
+    let bestPrompts;
+    while (low <= high) {
+      const mid = Math.floor((low + high) / 2);
+      const batch = candidate.slice(0, mid);
+      const prompts = await buildPrompt(batch);
+      if (promptLength(prompts) <= maxPromptChars) {
+        best = batch;
+        bestPrompts = prompts;
+        low = mid + 1;
+      } else {
+        high = mid - 1;
+      }
+    }
+    if (best && bestPrompts) return { batch: best, prompts: bestPrompts };
+    const single = candidate.slice(0, 1);
+    return { batch: single, prompts: await buildPrompt(single) };
+  };
+  const onFailure = async (batch, err) => {
+    if (batch.length <= 1) {
+      if (batch.length === 1) {
+        skipped.push(batch[0]);
+        onSkip?.(batch[0], err);
+      }
+      return;
+    }
+    const mid = Math.ceil(batch.length / 2);
+    if (mid < batchSize) batchSize = mid;
+    let i = 0;
+    while (i < batch.length) {
+      const size = Math.min(batchSize, batch.length - i);
+      const trimmed = await trim(batch.slice(i, i + size));
+      await attempt(trimmed.batch, trimmed.prompts);
+      i += trimmed.batch.length;
+    }
+  };
+  const attempt = async (batch, prebuilt) => {
+    if (batch.length === 0) return;
+    const prompts = prebuilt ?? await buildPrompt(batch);
+    batches++;
+    let responseText;
+    try {
+      responseText = await call(prompts);
+    } catch (err) {
+      if (!isTruncationError(err)) throw err;
+      await onFailure(batch, err);
+      return;
+    }
+    let result;
+    try {
+      result = parse(responseText, batch);
+    } catch (err) {
+      await onFailure(batch, err);
+      return;
+    }
+    results.push(result);
+  };
+  let index = 0;
+  while (index < items.length) {
+    const { batch, prompts } = await trim(items.slice(index, index + batchSize));
+    index += batch.length;
+    await attempt(batch, prompts);
+  }
+  return { results, skipped, batches };
+}
+
 // src/services/MaintenanceService.ts
 var FUZZY_THRESHOLD = 0.5;
 var MIN_TOKENS_TO_QUALIFY = 3;
 var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
 var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
 var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
+var HEAL_MAX_ANCHORS = 50;
+var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
+var HEAL_MAX_PROMPT_CHARS = 4e4;
 var MaintenanceService = class {
   constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
     this.db = db;
@@ -1811,30 +1951,56 @@ var MaintenanceService = class {
         console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during heal orphan pass for ${factId}:`, hookErr);
       }
     }
-    const allFactsRows = await this.entryRepo.findAllByEntityId(entityId);
+    const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId);
     const allTasks = await this.taskRepo.findAllPending([entityId]);
     const recentEvents = await this.eventRepo.getRecent(entityId, 20);
-    const healCandidates = allFactsRows.filter((f) => f.source_type !== "immutable_document");
-    const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
-    const healCandidatesForPrompt = healCandidates.map((f) => {
+    const toPromptShape = (f) => {
       const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
       return { ...rest, tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags };
+    };
+    const anchorCache = /* @__PURE__ */ new Map();
+    const outcome = await runBatched({
+      items: healCandidates,
+      buildPrompt: async (batch) => {
+        const documentAnchors = await this._selectHealAnchors(entityId, batch, anchorCache);
+        return this.promptService.buildHealPrompt(
+          batch.map(toPromptShape),
+          documentAnchors,
+          allTasks,
+          recentEvents,
+          promptOverride
+        );
+      },
+      call: (prompts) => this.options.llmProvider.generateText(prompts),
+      parse: (responseText, batch) => {
+        const result = parseJsonResponse(responseText);
+        return {
+          batch,
+          downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
+          deleted: Array.isArray(result.deleted) ? result.deleted : [],
+          newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
+        };
+      },
+      maxOutputTokens: this.options.llmProvider.maxOutputTokens,
+      maxPromptChars: HEAL_MAX_PROMPT_CHARS,
+      onSkip: (fact, err) => {
+        console.warn(
+          `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
+          err
+        );
+      }
     });
-    const { systemPrompt, userPrompt } = this.promptService.buildHealPrompt(
-      healCandidatesForPrompt,
-      documentAnchors,
-      allTasks,
-      recentEvents,
-      promptOverride
-    );
-    const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
-    const result = parseJsonResponse(responseText);
-    const mutableIds = new Set(healCandidates.map((f) => f.id));
-    const downgraded = Array.isArray(result.downgraded) ? result.downgraded : [];
-    const deleted = Array.isArray(result.deleted) ? result.deleted : [];
-    const newFacts = Array.isArray(result.newFacts) ? result.newFacts : [];
-    const safeDowngraded = Array.from(new Set(downgraded.filter((id) => mutableIds.has(id))));
-    const safeDeleted = Array.from(new Set(deleted.filter((id) => mutableIds.has(id))));
+    const safeDowngradedSet = /* @__PURE__ */ new Set();
+    const safeDeletedSet = /* @__PURE__ */ new Set();
+    const newFacts = [];
+    for (const batchResult of outcome.results) {
+      const mutableIds = new Set(batchResult.batch.map((f) => f.id));
+      for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
+      for (const id of batchResult.deleted) if (mutableIds.has(id)) safeDeletedSet.add(id);
+      newFacts.push(...batchResult.newFacts);
+    }
+    const safeDowngraded = Array.from(safeDowngradedSet);
+    const safeDeleted = Array.from(safeDeletedSet);
     const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
     const insertedFacts = [];
     const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
@@ -1901,7 +2067,7 @@ var MaintenanceService = class {
     }
     const now = Date.now();
     const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
-    const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
+    const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped: 0 };
     const ontologyService = this.ontologyService;
     if (!ontologyService) {
       return { ...zeroed, remaining: 0, deferred: 0 };
@@ -1923,18 +2089,79 @@ var MaintenanceService = class {
       options?.promptOverride,
       ontologyContext
     );
-    const batch = [candidates[0]];
-    let built = buildPrompt(batch);
-    for (let i = 1; i < candidates.length; i++) {
-      const next = buildPrompt([...batch, candidates[i]]);
-      if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
-      batch.push(candidates[i]);
-      built = next;
-    }
-    const { systemPrompt, userPrompt } = built;
-    const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
-    const parsed = parseJsonResponse(responseText);
-    const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
+    const outcome = await runBatched({
+      items: candidates,
+      buildPrompt,
+      call: (prompts) => this.options.llmProvider.generateText(prompts),
+      parse: (responseText, batch) => {
+        const parsed = parseJsonResponse(responseText);
+        return {
+          batch,
+          classifications: Array.isArray(parsed.classifications) ? parsed.classifications : [],
+          ontologyUpdates: parsed.ontology_updates
+        };
+      },
+      maxOutputTokens: this.options.llmProvider.maxOutputTokens,
+      maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
+      onSkip: (fact, err) => {
+        console.warn(
+          `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
+          err
+        );
+      }
+    });
+    let typed = 0;
+    let failedValidation = 0;
+    let edgesAdded = 0;
+    let scanned = 0;
+    let abortedOntologyOff = false;
+    for (const batchResult of outcome.results) {
+      const applied = await this._applyOntologyBackfillBatch(entityId, batchResult, now);
+      if (applied.abortedOntologyOff) {
+        abortedOntologyOff = true;
+        break;
+      }
+      typed += applied.typed;
+      failedValidation += applied.failedValidation;
+      edgesAdded += applied.edgesAdded;
+      scanned += batchResult.batch.length;
+    }
+    if (abortedOntologyOff) {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return {
+        scanned,
+        typed,
+        failedValidation,
+        edgesAdded,
+        skipped: outcome.skipped.length,
+        remaining: 0,
+        deferred: counts2.deferred
+      };
+    }
+    if (outcome.skipped.length > 0) {
+      await this.entryRepo.markOntologyChecked(outcome.skipped.map((f) => f.id), entityId, now, this.db);
+    }
+    this.searchService.evictCache(entityId);
+    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+    return {
+      scanned,
+      typed,
+      failedValidation,
+      edgesAdded,
+      skipped: outcome.skipped.length,
+      remaining: counts.eligible,
+      deferred: counts.deferred
+    };
+  }
+  /**
+   * Applies one parsed backfill batch in its own transaction. Per-batch rather
+   * than one transaction for the pass, so mergeEmergentUpdates semantics and
+   * the mid-flight `mode === 'off'` abort check keep the shape they had when a
+   * pass was a single call.
+   */
+  async _applyOntologyBackfillBatch(entityId, batchResult, now) {
+    const ontologyService = this.ontologyService;
+    const { batch, classifications, ontologyUpdates } = batchResult;
     let typed = 0;
     let failedValidation = 0;
     let edgesAdded = 0;
@@ -1945,8 +2172,8 @@ var MaintenanceService = class {
         abortedOntologyOff = true;
         return;
       }
-      if (txMode === "emergent" && parsed.ontology_updates) {
-        manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
+      if (txMode === "emergent" && ontologyUpdates) {
+        manifest = await ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
       }
       const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
       const titleIndex = /* @__PURE__ */ new Map();
@@ -2000,19 +2227,58 @@ var MaintenanceService = class {
       }
       await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
     });
-    if (abortedOntologyOff) {
-      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
-      return { ...zeroed, remaining: 0, deferred: counts2.deferred };
-    }
-    this.searchService.evictCache(entityId);
-    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
-    return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
+    return { typed, failedValidation, edgesAdded, abortedOntologyOff };
   }
   _validatePruneDuration(value, name) {
     if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
       throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
     }
   }
+  /**
+   * Anchors relevant to one batch of heal candidates.
+   *
+   * Heal used to pass every immutable_document fact for the entity — 2560 rows
+   * against 31 candidates on the corpus behind #63 — which is what blew the
+   * output ceiling. Anchors are now retrieved by keyword relevance to the batch
+   * and capped.
+   *
+   * The MiniSearch index holds all facts, not only anchors, so hits are
+   * overfetched and the source_type restriction is applied after retrieval, in
+   * SQL. Search rank order is preserved through the filter.
+   *
+   * Accepted tradeoff: an anchor that contradicts a candidate while sharing no
+   * vocabulary with it is now missed. Exhaustive-but-broken traded for
+   * relevance-scoped-and-working.
+   *
+   * `cache` is keyed by the derived query rather than by the batch, so two
+   * batches that reduce to the same query share one lookup. Caller-owned and
+   * per-pass — see the call site in doRunHeal.
+   */
+  async _selectHealAnchors(entityId, batch, cache) {
+    const query = batch.map((f) => f.title).join(" ").trim();
+    if (!query) return [];
+    const cached = cache?.get(query);
+    if (cached) return cached;
+    const hits = this.searchService.searchKeyword(
+      query,
+      [entityId],
+      HEAL_MAX_ANCHORS * HEAL_ANCHOR_SEARCH_OVERFETCH
+    );
+    const hitIds = hits.map((h) => h.id);
+    const anchors = [];
+    if (hitIds.length > 0) {
+      const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
+      const byId = new Map(rows.map((r) => [r.id, r]));
+      for (const id of hitIds) {
+        const row = byId.get(id);
+        if (!row) continue;
+        anchors.push(row);
+        if (anchors.length >= HEAL_MAX_ANCHORS) break;
+      }
+    }
+    cache?.set(query, anchors);
+    return anchors;
+  }
   _sanitizeRankerError(err) {
     return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
   }
@@ -3222,5 +3488,5 @@ var WriteService = class {
 };
 
 export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, validateInlineEdges, validateManifest };
-//# sourceMappingURL=chunk-LCAU6NYC.mjs.map
-//# sourceMappingURL=chunk-LCAU6NYC.mjs.map
+//# sourceMappingURL=chunk-MYZJLVX4.mjs.map
+//# sourceMappingURL=chunk-MYZJLVX4.mjs.map