@equationalapplications/core-llm-wiki 7.7.0 → 7.7.2

package/dist/index.js

@@ -107,7 +107,7 @@ async function setupDatabase(db, prefix) {
       UNIQUE(entity_id, source_id, target_id, edge_type)
     );
 
-    CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges(entity_id);
+    CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
 
     CREATE TABLE IF NOT EXISTS ${prefix}events (
       id TEXT PRIMARY KEY,
@@ -620,6 +620,16 @@ var MIGRATIONS = [
         if (!existing.has(name)) await db.execAsync(sql);
       }
     }
+  },
+  {
+    version: 12,
+    description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
+    run: async (db, prefix) => {
+      await db.execAsync(`
+        CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
+        DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
+      `);
+    }
   }
 ];
 for (let i = 1; i < MIGRATIONS.length; i++) {
@@ -4837,11 +4847,16 @@ var IngestionService = class {
       let ingestedChunks = 0;
       let failedChunks = 0;
       const failures = [];
-      const seen = /* @__PURE__ */ new Set();
-      const orderedChunkFacts = [];
-      const groundingLedger = /* @__PURE__ */ new Map();
+      const winners = /* @__PURE__ */ new Map();
       const diagBuffer = new DiagnosticBuffer();
       const diagBase = { entityId, operation: "ingest", trigger: "call" };
+      const pushDeduplicated = (chunkIndex, itemIndex) => {
+        diagBuffer.push({
+          ...diagBase,
+          code: "fact_deduplicated",
+          detail: { sourceRef, chunkIndex, itemIndex, reason: "exact_title" }
+        });
+      };
       for (const [chunkIndex, slot] of chunkResults.entries()) {
         if (slot.status === "failed") {
           failedChunks++;
@@ -4852,22 +4867,32 @@ var IngestionService = class {
         for (const r of slot.rejected) {
           diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { sourceRef, chunkIndex, itemIndex: r.itemIndex, reason: r.reason } });
         }
-        const dedupedFacts = [];
         slot.facts.forEach((fact, k) => {
-          const normalizedTitle = normalizeTitleKey(fact.title);
-          if (!seen.has(normalizedTitle)) {
-            seen.add(normalizedTitle);
-            dedupedFacts.push(fact);
-            if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: slot.itemIndexes[k] });
+          const key = normalizeTitleKey(fact.title);
+          const candidate = { chunkIndex, k, itemIndex: slot.itemIndexes[k], rank: slot.verdicts[k]?.status === "grounded" ? 1 : 0 };
+          const current = winners.get(key);
+          if (!current) {
+            winners.set(key, candidate);
+          } else if (candidate.rank > current.rank) {
+            pushDeduplicated(current.chunkIndex, current.itemIndex);
+            winners.set(key, candidate);
           } else {
-            diagBuffer.push({
-              ...diagBase,
-              code: "fact_deduplicated",
-              detail: { sourceRef, chunkIndex, itemIndex: slot.itemIndexes[k], reason: "exact_title" }
-            });
+            pushDeduplicated(candidate.chunkIndex, candidate.itemIndex);
           }
         });
-        orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
+      }
+      const orderedChunkFacts = [];
+      const groundingLedger = /* @__PURE__ */ new Map();
+      for (const [chunkIndex, slot] of chunkResults.entries()) {
+        if (slot.status === "failed") continue;
+        const keptFacts = [];
+        slot.facts.forEach((fact, k) => {
+          const winner = winners.get(normalizeTitleKey(fact.title));
+          if (winner?.chunkIndex !== chunkIndex || winner.k !== k) return;
+          keptFacts.push(fact);
+          if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: winner.itemIndex });
+        });
+        orderedChunkFacts.push({ facts: keptFacts, ontology_updates: slot.ontology_updates });
       }
       for (const reason of ["parse", "llm"]) {
         const ofReason = failures.filter((f) => f.source === reason);