@equationalapplications/core-llm-wiki 7.7.2 → 7.7.4

package/README.md

@@ -361,7 +361,7 @@ new WikiMemory(db, {
   - Instructions, the ontology manifest, existing facts and identifiers never count, so a model cannot ground a claim by quoting them.
 - **The check.** Both sides are normalized with NFKC, whitespace runs collapse to one space, and matching is case-sensitive. A fact with more than 10 quotes, or any quote not found, fails.
 - **Diagnostics.** `grounding_missing` (reasons `no_evidence`, `evidence_too_short`) and `grounding_failed` (reasons `quote_not_found`, `too_many_quotes`), one per fact, with the new fact's `factId`. Quotes are never included.
-- **Duplicate titles in one ingest.** When chunks yield facts with the same title, ingest keeps one: a grounded fact beats one with missing or failed evidence, and on a tie the first one wins. The others are reported as `fact_deduplicated`. A fact already stored for the same `sourceRef` is not replaced by a later partial ingest.
+- **Duplicate titles in one ingest.** When chunks yield facts with the same title, ingest keeps one: a grounded fact beats one with missing or failed evidence, and on a tie the first one wins. The others are reported as `fact_deduplicated`. A fact already stored for the same `sourceRef` is not replaced by a later partial ingest, and that skip is reported the same way. Either way the diagnostic carries `{ sourceRef, chunkIndex, itemIndex }` for the skipped fact, whether or not grounding is enabled.
 - **`upsertGraph`** nodes are host-supplied and never grounded.
 - **Librarian and heal** synthesize across events, so their pass rates are unknown. Measure them on your own event log before opting them in.
 - Evidence quotes are not stored.
@@ -1217,6 +1217,8 @@ await wikiMemory.ingestDocument(
 - `'throw'`: Pre-check before any LLM call; throw `WikiDuplicateHashError` (carries the canonical `sourceRef`).
 - The guard only considers **live** references — soft-deleted refs do not trigger it in any mode.
 
+When the pre-check passes but a concurrent writer claims the hash before the write commits, the transaction rolls back after the LLM extraction has already run. You still receive the diagnostics that describe that extraction — `ingest_chunk_failed`, `fact_rejected` and `fact_deduplicated` — in every mode, and also in the rarer case where the winning writer has itself rolled back by the time core looks, so the call surfaces the raw database error instead of `WikiDuplicateHashError`. Diagnostics tied to the rolled-back write (`grounding_*`, `edge_dropped`) are dropped, because the `factId`s they carry name rows that were never committed.
+
 ## Batch Change Detection
 
 Check multiple documents for changes in one call:

package/dist/{chunk-NWODQRSM.mjs → chunk-JV2OQ5CM.mjs}

@@ -2018,6 +2018,23 @@ var DiagnosticBuffer = class {
     this.items = [];
     for (const input of pending) emitDiagnostic(options, input);
   }
+  /**
+   * Emit only the buffered diagnostics whose code is in `codes` and discard
+   * the rest. The buffer is empty afterwards either way.
+   *
+   * For an operation that did real work before its write was rolled back:
+   * some buffered diagnostics describe that work and are still true, while
+   * others describe rows that no longer exist (anything carrying a `factId`
+   * minted inside the aborted transaction). `flush` would emit both and
+   * `discard` would drop both; this emits the first set only.
+   */
+  flushOnly(options, codes) {
+    const pending = this.items;
+    this.items = [];
+    for (const input of pending) {
+      if (codes.includes(input.code)) emitDiagnostic(options, input);
+    }
+  }
   discard() {
     this.items = [];
   }
@@ -2034,6 +2051,16 @@ function edgeDropDiagnostic(drop, ctx) {
 }
 
 // src/services/IngestionService.ts
+var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
+function dedupDiagnostic(entityId, sourceRef, locator) {
+  return {
+    entityId,
+    operation: "ingest",
+    trigger: "call",
+    code: "fact_deduplicated",
+    detail: locator ? { sourceRef, chunkIndex: locator.chunkIndex, itemIndex: locator.itemIndex, reason: "exact_title" } : { sourceRef, reason: "exact_title" }
+  };
+}
 function zeroChunkResult(duplicateOf) {
   const result = {
     truncated: false,
@@ -2170,11 +2197,7 @@ var IngestionService = class {
       const diagBuffer = new DiagnosticBuffer();
       const diagBase = { entityId, operation: "ingest", trigger: "call" };
       const pushDeduplicated = (chunkIndex, itemIndex) => {
-        diagBuffer.push({
-          ...diagBase,
-          code: "fact_deduplicated",
-          detail: { sourceRef, chunkIndex, itemIndex, reason: "exact_title" }
-        });
+        diagBuffer.push(dedupDiagnostic(entityId, sourceRef, { chunkIndex, itemIndex }));
       };
       for (const [chunkIndex, slot] of chunkResults.entries()) {
         if (slot.status === "failed") {
@@ -2201,7 +2224,7 @@ var IngestionService = class {
         });
       }
       const orderedChunkFacts = [];
-      const groundingLedger = /* @__PURE__ */ new Map();
+      const factLedger = /* @__PURE__ */ new Map();
       for (const [chunkIndex, slot] of chunkResults.entries()) {
         if (slot.status === "failed") continue;
         const keptFacts = [];
@@ -2209,7 +2232,11 @@ var IngestionService = class {
           const winner = winners.get(normalizeTitleKey(fact.title));
           if (winner?.chunkIndex !== chunkIndex || winner.k !== k) return;
           keptFacts.push(fact);
-          if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: winner.itemIndex });
+          factLedger.set(fact, {
+            verdict: ingestGrounding ? slot.verdicts[k] : null,
+            chunkIndex,
+            itemIndex: winner.itemIndex
+          });
         });
         orderedChunkFacts.push({ facts: keptFacts, ontology_updates: slot.ontology_updates });
       }
@@ -2239,7 +2266,7 @@ var IngestionService = class {
       try {
         if (failedChunks === 0) {
           const fullResult = await this.db.withTransactionAsync(async (tx) => {
-            return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger);
+            return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, factLedger);
           });
           deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
           insertedFacts.push(...fullResult.insertedFacts);
@@ -2247,13 +2274,14 @@ var IngestionService = class {
           const partialResult = await this.db.withTransactionAsync(async (tx) => {
             const flat = [];
             for (const slot of orderedChunkFacts) flat.push(...slot.facts);
-            return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, groundingLedger);
+            return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, factLedger);
           });
           insertedFacts.push(...partialResult.insertedDescriptors);
         }
       } catch (err) {
         const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
         if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
+        diagBuffer.flushOnly(this.options, LLM_PASS_DIAGNOSTIC_CODES);
         const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
         if (canonical === null) throw err;
         if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
@@ -2481,7 +2509,7 @@ var IngestionService = class {
    * On the partial path the caller never invokes this method; the empty
    * arrays stay empty.
    */
-  async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
+  async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, factLedger = /* @__PURE__ */ new Map()) {
     const deletedSourceFactIds = [];
     const insertedFacts = [];
     const nodeTrust = /* @__PURE__ */ new Map();
@@ -2528,9 +2556,9 @@ var IngestionService = class {
           tags: fact.tags,
           confidence: fact.confidence
         });
-        const grounding = groundingLedger.get(fact);
-        if (grounding) {
-          const outcome = groundingOutcome(grounding.verdict, now);
+        const locator = factLedger.get(fact);
+        if (locator?.verdict) {
+          const outcome = groundingOutcome(locator.verdict, now);
           nodeTrust.set(id, outcome.trust);
           if (outcome.diagnostic) {
             diagBuffer.push({
@@ -2538,7 +2566,7 @@ var IngestionService = class {
               operation: "ingest",
               trigger: "call",
               code: outcome.diagnostic.code,
-              detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
+              detail: { factId: id, sourceRef, chunkIndex: locator.chunkIndex, itemIndex: locator.itemIndex, reason: outcome.diagnostic.reason }
             });
           }
         }
@@ -2597,7 +2625,7 @@ var IngestionService = class {
    * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
    * Returns `{ inserted, skippedDuplicate }` for observability.
    */
-  async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
+  async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, factLedger = /* @__PURE__ */ new Map()) {
     const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
     const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
     const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
@@ -2607,28 +2635,22 @@ var IngestionService = class {
     const insertedDescriptors = [];
     for (const fact of dedupedFacts) {
       const normalizedTitle = normalizeTitleKey(fact.title);
+      const locator = factLedger.get(fact);
       if (liveTitles.has(normalizedTitle)) {
-        diagBuffer.push({
-          entityId,
-          operation: "ingest",
-          trigger: "call",
-          code: "fact_deduplicated",
-          detail: { sourceRef, reason: "exact_title" }
-        });
+        diagBuffer.push(dedupDiagnostic(entityId, sourceRef, locator));
         skippedDuplicate++;
         continue;
       }
       liveTitles.add(normalizedTitle);
       const id = generateId("fact_");
-      const grounding = groundingLedger.get(fact);
-      const outcome = grounding ? groundingOutcome(grounding.verdict, now) : null;
-      if (grounding && outcome?.diagnostic) {
+      const outcome = locator?.verdict ? groundingOutcome(locator.verdict, now) : null;
+      if (locator?.verdict && outcome?.diagnostic) {
         diagBuffer.push({
           entityId,
           operation: "ingest",
           trigger: "call",
           code: outcome.diagnostic.code,
-          detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
+          detail: { factId: id, sourceRef, chunkIndex: locator.chunkIndex, itemIndex: locator.itemIndex, reason: outcome.diagnostic.reason }
         });
       }
       const wikiFact = {
@@ -5641,5 +5663,5 @@ var WriteService = class {
 };
 
 export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
-//# sourceMappingURL=chunk-NWODQRSM.mjs.map
-//# sourceMappingURL=chunk-NWODQRSM.mjs.map
+//# sourceMappingURL=chunk-JV2OQ5CM.mjs.map
+//# sourceMappingURL=chunk-JV2OQ5CM.mjs.map