@equationalapplications/core-llm-wiki 7.5.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/{chunk-3P7FAKJA.mjs → chunk-YBWE26XR.mjs} +224 -41
- package/dist/chunk-YBWE26XR.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +225 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -5
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-ClpOUiLI.d.mts → testing-d1SrpDvg.d.mts} +52 -3
- package/dist/{testing-ClpOUiLI.d.ts → testing-d1SrpDvg.d.ts} +52 -3
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +221 -38
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-3P7FAKJA.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -593,6 +593,19 @@ function validateTags(tags) {
|
|
|
593
593
|
if (!Array.isArray(tags)) return [];
|
|
594
594
|
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
595
595
|
}
|
|
596
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
597
|
+
function normalizeEvidence(raw) {
|
|
598
|
+
if (!Array.isArray(raw)) return void 0;
|
|
599
|
+
const out = [];
|
|
600
|
+
for (const entry of raw) {
|
|
601
|
+
if (typeof entry !== "string") continue;
|
|
602
|
+
const trimmed = entry.trim();
|
|
603
|
+
if (!trimmed) continue;
|
|
604
|
+
out.push(trimmed);
|
|
605
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
606
|
+
}
|
|
607
|
+
return out;
|
|
608
|
+
}
|
|
596
609
|
function validateFact(fact) {
|
|
597
610
|
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
598
611
|
const title = clip(fact.title, 80);
|
|
@@ -600,13 +613,17 @@ function validateFact(fact) {
|
|
|
600
613
|
if (!title || !body) return null;
|
|
601
614
|
let confidence = fact.confidence;
|
|
602
615
|
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
603
|
-
|
|
616
|
+
const valid = {
|
|
604
617
|
...fact,
|
|
605
618
|
title,
|
|
606
619
|
body,
|
|
607
620
|
confidence,
|
|
608
621
|
tags: validateTags(fact.tags)
|
|
609
622
|
};
|
|
623
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
624
|
+
if (evidence) valid.evidence = evidence;
|
|
625
|
+
else delete valid.evidence;
|
|
626
|
+
return valid;
|
|
610
627
|
}
|
|
611
628
|
function validateTask(task) {
|
|
612
629
|
if (typeof task?.description !== "string") return null;
|
|
@@ -1509,17 +1526,94 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
1509
1526
|
}
|
|
1510
1527
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
1511
1528
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
1529
|
+
var GROUNDING_SOURCE = {
|
|
1530
|
+
ingest: { key: "facts", source: "the document chunk" },
|
|
1531
|
+
librarian: { key: "facts", source: 'the "summary" text of the events' },
|
|
1532
|
+
heal: { key: "newFacts", source: 'the "summary" text of the recent events or the "body" text of the document anchors' }
|
|
1533
|
+
};
|
|
1534
|
+
function groundingEvidenceBlock(writer, cfg) {
|
|
1535
|
+
const { key, source } = GROUNDING_SOURCE[writer];
|
|
1536
|
+
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1537
|
+
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1538
|
+
}
|
|
1512
1539
|
|
|
1513
1540
|
// src/utils/healConstants.ts
|
|
1514
1541
|
var HEAL_MAX_ANCHORS = 50;
|
|
1515
1542
|
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
1516
1543
|
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
1517
1544
|
var HEAL_MAX_TASKS = 50;
|
|
1545
|
+
var HEAL_ANCHOR_BODY_CHARS = 800;
|
|
1546
|
+
|
|
1547
|
+
// src/utils/grounding.ts
|
|
1548
|
+
var GROUNDING_VERIFIER = "process:grounding-check";
|
|
1549
|
+
var WRITERS = ["ingest", "librarian", "heal"];
|
|
1550
|
+
function positiveInt(value, fallback) {
|
|
1551
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
1552
|
+
}
|
|
1553
|
+
function resolveGrounding(config) {
|
|
1554
|
+
if (!config || config.mode !== "draft") return null;
|
|
1555
|
+
const writers = Array.isArray(config.writers) ? config.writers.filter((w) => WRITERS.includes(w)) : ["ingest"];
|
|
1556
|
+
return {
|
|
1557
|
+
writers: new Set(writers),
|
|
1558
|
+
minEvidenceChars: positiveInt(config.minEvidenceChars, 20),
|
|
1559
|
+
// Never ask for more quotes than checkGrounding accepts (MAX_EVIDENCE_QUOTES).
|
|
1560
|
+
maxEvidence: Math.min(MAX_EVIDENCE_QUOTES, positiveInt(config.maxEvidence, 3)),
|
|
1561
|
+
maxEvidenceChars: positiveInt(config.maxEvidenceChars, 300)
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
function normalizeForGrounding(text) {
|
|
1565
|
+
return text.normalize("NFKC").replace(/\s+/g, " ").trim();
|
|
1566
|
+
}
|
|
1567
|
+
function buildGroundingCorpus(parts) {
|
|
1568
|
+
return parts.filter((p) => typeof p === "string").map(normalizeForGrounding);
|
|
1569
|
+
}
|
|
1570
|
+
function checkGrounding(evidence, normalizedCorpus, cfg) {
|
|
1571
|
+
const quotes = evidence ?? [];
|
|
1572
|
+
if (quotes.length > MAX_EVIDENCE_QUOTES) return { status: "failed", reason: "too_many_quotes" };
|
|
1573
|
+
if (quotes.length === 0) return { status: "missing", reason: "no_evidence" };
|
|
1574
|
+
const qualifying = quotes.map(normalizeForGrounding).filter((q) => q.length >= cfg.minEvidenceChars);
|
|
1575
|
+
if (qualifying.length === 0) return { status: "missing", reason: "evidence_too_short" };
|
|
1576
|
+
if (qualifying.some((q) => !normalizedCorpus.some((part) => part.includes(q)))) {
|
|
1577
|
+
return { status: "failed", reason: "quote_not_found" };
|
|
1578
|
+
}
|
|
1579
|
+
return {
|
|
1580
|
+
status: "grounded",
|
|
1581
|
+
retained: qualifying.slice(0, cfg.maxEvidence).map((q) => safeSlice(q, 0, cfg.maxEvidenceChars))
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
function groundingOutcome(verdict, now) {
|
|
1585
|
+
if (verdict.status === "grounded") {
|
|
1586
|
+
return {
|
|
1587
|
+
trust: {
|
|
1588
|
+
lifecycle_status: "stable",
|
|
1589
|
+
okf_verified: [{ by: GROUNDING_VERIFIER, at: new Date(now).toISOString() }],
|
|
1590
|
+
last_verified_at: now,
|
|
1591
|
+
last_verified_by: GROUNDING_VERIFIER
|
|
1592
|
+
}
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
return {
|
|
1596
|
+
trust: { lifecycle_status: "draft" },
|
|
1597
|
+
diagnostic: { code: verdict.status === "missing" ? "grounding_missing" : "grounding_failed", reason: verdict.reason }
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1518
1600
|
|
|
1519
1601
|
// src/services/PromptService.ts
|
|
1520
1602
|
var PromptService = class {
|
|
1521
|
-
constructor(globalOverrides) {
|
|
1603
|
+
constructor(globalOverrides, grounding = null) {
|
|
1522
1604
|
this.globalOverrides = globalOverrides;
|
|
1605
|
+
this.grounding = grounding;
|
|
1606
|
+
}
|
|
1607
|
+
/** The resolved grounding config when `writer` is in `grounding.writers`; otherwise null (spec §6.2). */
|
|
1608
|
+
groundingFor(writer) {
|
|
1609
|
+
return this.grounding?.writers.has(writer) ? this.grounding : null;
|
|
1610
|
+
}
|
|
1611
|
+
/** Appended after any override and after ontology context, so it is always last. */
|
|
1612
|
+
appendGrounding(systemPrompt, writer) {
|
|
1613
|
+
const cfg = this.groundingFor(writer);
|
|
1614
|
+
return cfg ? `${systemPrompt}
|
|
1615
|
+
|
|
1616
|
+
${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
1523
1617
|
}
|
|
1524
1618
|
hydrate(template, variables) {
|
|
1525
1619
|
return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
|
|
@@ -1549,13 +1643,13 @@ ${ctx.ontologyModeInstructions}`;
|
|
|
1549
1643
|
const hasDocumentChunk = /\{\{\s*documentChunk\s*\}\}/.test(template);
|
|
1550
1644
|
if (hasDocumentChunk || this.hasOntologyPlaceholders(template)) {
|
|
1551
1645
|
return {
|
|
1552
|
-
systemPrompt: this.buildSystemPrompt(template, { documentChunk }, ontologyContext),
|
|
1646
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { documentChunk }, ontologyContext), "ingest"),
|
|
1553
1647
|
userPrompt: hasDocumentChunk ? "Please extract the facts." : `Document Chunk:
|
|
1554
1648
|
${documentChunk}`
|
|
1555
1649
|
};
|
|
1556
1650
|
}
|
|
1557
1651
|
return {
|
|
1558
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1652
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "ingest"),
|
|
1559
1653
|
userPrompt: `Document Chunk:
|
|
1560
1654
|
${documentChunk}`
|
|
1561
1655
|
};
|
|
@@ -1564,23 +1658,27 @@ ${documentChunk}`
|
|
|
1564
1658
|
const template = runtimeOverride ?? this.globalOverrides?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT;
|
|
1565
1659
|
const hasEvents = /\{\{\s*events\s*\}\}/.test(template);
|
|
1566
1660
|
const hasCurrentFacts = /\{\{\s*currentFacts\s*\}\}/.test(template);
|
|
1661
|
+
const eventsShown = hasEvents || !hasCurrentFacts;
|
|
1662
|
+
const corpusField = this.groundingFor("librarian") ? { groundingCorpus: buildGroundingCorpus(eventsShown ? events.map(summaryOf) : []) } : {};
|
|
1567
1663
|
if (hasEvents || hasCurrentFacts || this.hasOntologyPlaceholders(template)) {
|
|
1568
1664
|
return {
|
|
1569
|
-
systemPrompt: this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext),
|
|
1665
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext), "librarian"),
|
|
1570
1666
|
userPrompt: hasEvents || hasCurrentFacts ? "Please synthesize the context." : `Events:
|
|
1571
1667
|
${JSON.stringify(events, null, 2)}
|
|
1572
1668
|
|
|
1573
1669
|
Current Facts:
|
|
1574
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1670
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1671
|
+
...corpusField
|
|
1575
1672
|
};
|
|
1576
1673
|
}
|
|
1577
1674
|
return {
|
|
1578
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1675
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "librarian"),
|
|
1579
1676
|
userPrompt: `Events:
|
|
1580
1677
|
${JSON.stringify(events, null, 2)}
|
|
1581
1678
|
|
|
1582
1679
|
Current Facts:
|
|
1583
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1680
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1681
|
+
...corpusField
|
|
1584
1682
|
};
|
|
1585
1683
|
}
|
|
1586
1684
|
/**
|
|
@@ -1610,40 +1708,54 @@ ${JSON.stringify(currentFacts, null, 2)}`
|
|
|
1610
1708
|
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
1611
1709
|
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
1612
1710
|
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
1711
|
+
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
1712
|
+
const hasRecentEvents = /\{\{\s*recentEvents\s*\}\}/.test(template);
|
|
1713
|
+
const hasDocumentAnchors = /\{\{\s*documentAnchors\s*\}\}/.test(template);
|
|
1714
|
+
const usesPlaceholders = /\{\{\s*healCandidates\s*\}\}/.test(template) || hasDocumentAnchors || /\{\{\s*allTasks\s*\}\}/.test(template) || hasRecentEvents;
|
|
1715
|
+
const healGrounding = this.groundingFor("heal");
|
|
1716
|
+
const promptAnchors = healGrounding ? effectiveAnchors.map(toGroundingAnchor) : effectiveAnchors;
|
|
1717
|
+
const eventsShown = !usesPlaceholders || hasRecentEvents;
|
|
1718
|
+
const anchorsShown = !usesPlaceholders || hasDocumentAnchors;
|
|
1719
|
+
const groundingCorpus = healGrounding ? buildGroundingCorpus([
|
|
1720
|
+
...eventsShown ? effectiveEvents.map(summaryOf) : [],
|
|
1721
|
+
...anchorsShown ? effectiveAnchors.map((a, i) => ({ status: a?.lifecycle_status, shown: promptAnchors[i] })).filter((x) => x.status !== "draft").map((x) => x.shown?.body) : []
|
|
1722
|
+
]) : null;
|
|
1613
1723
|
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
1614
1724
|
healCandidates,
|
|
1615
1725
|
attemptLevel,
|
|
1616
1726
|
bodyTruncationChars
|
|
1617
1727
|
);
|
|
1618
|
-
const
|
|
1619
|
-
if (
|
|
1728
|
+
const corpusField = groundingCorpus !== null ? { groundingCorpus } : {};
|
|
1729
|
+
if (usesPlaceholders) {
|
|
1620
1730
|
return {
|
|
1621
1731
|
prompts: {
|
|
1622
|
-
systemPrompt: this.hydrate(template, {
|
|
1732
|
+
systemPrompt: this.appendGrounding(this.hydrate(template, {
|
|
1623
1733
|
healCandidates: shapedCandidates,
|
|
1624
|
-
documentAnchors:
|
|
1734
|
+
documentAnchors: promptAnchors,
|
|
1625
1735
|
allTasks: effectiveTasks,
|
|
1626
1736
|
recentEvents: effectiveEvents
|
|
1627
|
-
}),
|
|
1737
|
+
}), "heal"),
|
|
1628
1738
|
userPrompt: "Please heal the memory graph."
|
|
1629
1739
|
},
|
|
1630
|
-
degraded
|
|
1740
|
+
degraded,
|
|
1741
|
+
...corpusField
|
|
1631
1742
|
};
|
|
1632
1743
|
}
|
|
1633
1744
|
return {
|
|
1634
1745
|
prompts: {
|
|
1635
|
-
systemPrompt: template,
|
|
1746
|
+
systemPrompt: this.appendGrounding(template, "heal"),
|
|
1636
1747
|
userPrompt: `Heal Candidates:
|
|
1637
1748
|
${JSON.stringify(shapedCandidates, null, 2)}
|
|
1638
1749
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
1639
|
-
${JSON.stringify(
|
|
1750
|
+
${JSON.stringify(promptAnchors, null, 2)}
|
|
1640
1751
|
All Tasks:
|
|
1641
1752
|
${JSON.stringify(effectiveTasks, null, 2)}
|
|
1642
1753
|
Recent Events:
|
|
1643
1754
|
${JSON.stringify(effectiveEvents, null, 2)}
|
|
1644
1755
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
1645
1756
|
},
|
|
1646
|
-
degraded
|
|
1757
|
+
degraded,
|
|
1758
|
+
...corpusField
|
|
1647
1759
|
};
|
|
1648
1760
|
}
|
|
1649
1761
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -1693,6 +1805,19 @@ function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
|
1693
1805
|
}
|
|
1694
1806
|
return { shapedCandidates, degraded };
|
|
1695
1807
|
}
|
|
1808
|
+
function summaryOf(event) {
|
|
1809
|
+
return event?.summary;
|
|
1810
|
+
}
|
|
1811
|
+
function toGroundingAnchor(anchor) {
|
|
1812
|
+
if (typeof anchor !== "object" || anchor === null) return anchor;
|
|
1813
|
+
const a = anchor;
|
|
1814
|
+
return {
|
|
1815
|
+
id: a.id,
|
|
1816
|
+
title: a.title,
|
|
1817
|
+
source_ref: a.source_ref,
|
|
1818
|
+
body: typeof a.body === "string" ? safeSlice(a.body, 0, HEAL_ANCHOR_BODY_CHARS) : ""
|
|
1819
|
+
};
|
|
1820
|
+
}
|
|
1696
1821
|
|
|
1697
1822
|
// src/utils/chunkingDefaults.ts
|
|
1698
1823
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -1722,7 +1847,7 @@ var IngestionService = class {
|
|
|
1722
1847
|
this.jobManager = jobManager;
|
|
1723
1848
|
this.embeddingService = embeddingService;
|
|
1724
1849
|
this.ontologyService = ontologyService;
|
|
1725
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
1850
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
1726
1851
|
}
|
|
1727
1852
|
async ingestDocument(entityId, params, opts) {
|
|
1728
1853
|
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
@@ -1755,6 +1880,7 @@ var IngestionService = class {
|
|
|
1755
1880
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1756
1881
|
if (chunks.length === 0) return zeroChunkResult();
|
|
1757
1882
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
1883
|
+
const ingestGrounding = this.promptService.groundingFor("ingest");
|
|
1758
1884
|
const chunkResults = await withConcurrency(
|
|
1759
1885
|
chunks.map((chunk, chunkIndex) => async () => {
|
|
1760
1886
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
@@ -1778,11 +1904,16 @@ var IngestionService = class {
|
|
|
1778
1904
|
rejected.push({ itemIndex, reason: factRejectionReason(raw) });
|
|
1779
1905
|
}
|
|
1780
1906
|
});
|
|
1907
|
+
const verdicts = ingestGrounding ? (() => {
|
|
1908
|
+
const corpus = buildGroundingCorpus([chunk]);
|
|
1909
|
+
return facts.map((f) => checkGrounding(f.evidence, corpus, ingestGrounding));
|
|
1910
|
+
})() : [];
|
|
1781
1911
|
return {
|
|
1782
1912
|
status: "ok",
|
|
1783
1913
|
facts,
|
|
1784
1914
|
itemIndexes,
|
|
1785
1915
|
rejected,
|
|
1916
|
+
verdicts,
|
|
1786
1917
|
ontology_updates: result2.ontology_updates
|
|
1787
1918
|
};
|
|
1788
1919
|
} catch (e) {
|
|
@@ -1827,6 +1958,7 @@ var IngestionService = class {
|
|
|
1827
1958
|
const failures = [];
|
|
1828
1959
|
const seen = /* @__PURE__ */ new Set();
|
|
1829
1960
|
const orderedChunkFacts = [];
|
|
1961
|
+
const groundingLedger = /* @__PURE__ */ new Map();
|
|
1830
1962
|
const diagBuffer = new DiagnosticBuffer();
|
|
1831
1963
|
const diagBase = { entityId, operation: "ingest", trigger: "call" };
|
|
1832
1964
|
for (const [chunkIndex, slot] of chunkResults.entries()) {
|
|
@@ -1845,6 +1977,7 @@ var IngestionService = class {
|
|
|
1845
1977
|
if (!seen.has(normalizedTitle)) {
|
|
1846
1978
|
seen.add(normalizedTitle);
|
|
1847
1979
|
dedupedFacts.push(fact);
|
|
1980
|
+
if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: slot.itemIndexes[k] });
|
|
1848
1981
|
} else {
|
|
1849
1982
|
diagBuffer.push({
|
|
1850
1983
|
...diagBase,
|
|
@@ -1881,7 +2014,7 @@ var IngestionService = class {
|
|
|
1881
2014
|
try {
|
|
1882
2015
|
if (failedChunks === 0) {
|
|
1883
2016
|
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1884
|
-
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer);
|
|
2017
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger);
|
|
1885
2018
|
});
|
|
1886
2019
|
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
1887
2020
|
insertedFacts.push(...fullResult.insertedFacts);
|
|
@@ -1889,7 +2022,7 @@ var IngestionService = class {
|
|
|
1889
2022
|
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1890
2023
|
const flat = [];
|
|
1891
2024
|
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
1892
|
-
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer);
|
|
2025
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, groundingLedger);
|
|
1893
2026
|
});
|
|
1894
2027
|
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
1895
2028
|
}
|
|
@@ -2031,7 +2164,8 @@ var IngestionService = class {
|
|
|
2031
2164
|
last_accessed_at: null,
|
|
2032
2165
|
access_count: 0,
|
|
2033
2166
|
deleted_at: null,
|
|
2034
|
-
okf_type: normalized.okf_type
|
|
2167
|
+
okf_type: normalized.okf_type,
|
|
2168
|
+
...opts?.nodeTrust?.get(node.id)
|
|
2035
2169
|
};
|
|
2036
2170
|
wikiFacts.push(wikiFact);
|
|
2037
2171
|
}
|
|
@@ -2122,9 +2256,10 @@ var IngestionService = class {
|
|
|
2122
2256
|
* On the partial path the caller never invokes this method; the empty
|
|
2123
2257
|
* arrays stay empty.
|
|
2124
2258
|
*/
|
|
2125
|
-
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer) {
|
|
2259
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2126
2260
|
const deletedSourceFactIds = [];
|
|
2127
2261
|
const insertedFacts = [];
|
|
2262
|
+
const nodeTrust = /* @__PURE__ */ new Map();
|
|
2128
2263
|
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
2129
2264
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
2130
2265
|
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
@@ -2168,6 +2303,20 @@ var IngestionService = class {
|
|
|
2168
2303
|
tags: fact.tags,
|
|
2169
2304
|
confidence: fact.confidence
|
|
2170
2305
|
});
|
|
2306
|
+
const grounding = groundingLedger.get(fact);
|
|
2307
|
+
if (grounding) {
|
|
2308
|
+
const outcome = groundingOutcome(grounding.verdict, now);
|
|
2309
|
+
nodeTrust.set(id, outcome.trust);
|
|
2310
|
+
if (outcome.diagnostic) {
|
|
2311
|
+
diagBuffer.push({
|
|
2312
|
+
entityId,
|
|
2313
|
+
operation: "ingest",
|
|
2314
|
+
trigger: "call",
|
|
2315
|
+
code: outcome.diagnostic.code,
|
|
2316
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2317
|
+
});
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2171
2320
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2172
2321
|
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
2173
2322
|
if (normalized.edges.length > 0) {
|
|
@@ -2199,7 +2348,7 @@ var IngestionService = class {
|
|
|
2199
2348
|
entityId,
|
|
2200
2349
|
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
2201
2350
|
tx,
|
|
2202
|
-
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" } }
|
|
2351
|
+
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" }, ...nodeTrust.size > 0 ? { nodeTrust } : {} }
|
|
2203
2352
|
);
|
|
2204
2353
|
return { deletedSourceFactIds, insertedFacts };
|
|
2205
2354
|
}
|
|
@@ -2223,7 +2372,7 @@ var IngestionService = class {
|
|
|
2223
2372
|
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
2224
2373
|
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
2225
2374
|
*/
|
|
2226
|
-
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer) {
|
|
2375
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2227
2376
|
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
2228
2377
|
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
2229
2378
|
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
@@ -2246,6 +2395,17 @@ var IngestionService = class {
|
|
|
2246
2395
|
}
|
|
2247
2396
|
liveTitles.add(normalizedTitle);
|
|
2248
2397
|
const id = generateId("fact_");
|
|
2398
|
+
const grounding = groundingLedger.get(fact);
|
|
2399
|
+
const outcome = grounding ? groundingOutcome(grounding.verdict, now) : null;
|
|
2400
|
+
if (grounding && outcome?.diagnostic) {
|
|
2401
|
+
diagBuffer.push({
|
|
2402
|
+
entityId,
|
|
2403
|
+
operation: "ingest",
|
|
2404
|
+
trigger: "call",
|
|
2405
|
+
code: outcome.diagnostic.code,
|
|
2406
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2407
|
+
});
|
|
2408
|
+
}
|
|
2249
2409
|
const wikiFact = {
|
|
2250
2410
|
id,
|
|
2251
2411
|
entity_id: entityId,
|
|
@@ -2264,7 +2424,8 @@ var IngestionService = class {
|
|
|
2264
2424
|
last_accessed_at: null,
|
|
2265
2425
|
access_count: 0,
|
|
2266
2426
|
deleted_at: null,
|
|
2267
|
-
okf_type: null
|
|
2427
|
+
okf_type: null,
|
|
2428
|
+
...outcome?.trust
|
|
2268
2429
|
};
|
|
2269
2430
|
await this.entryRepo.upsert(wikiFact, tx);
|
|
2270
2431
|
insertedDescriptors.push({
|
|
@@ -2460,7 +2621,7 @@ async function runBatched(args) {
|
|
|
2460
2621
|
}
|
|
2461
2622
|
let result;
|
|
2462
2623
|
try {
|
|
2463
|
-
result = parse(responseText, batch);
|
|
2624
|
+
result = parse(responseText, batch, prompts);
|
|
2464
2625
|
} catch (err) {
|
|
2465
2626
|
await onFailure(batch, err, attemptLevel, false);
|
|
2466
2627
|
return;
|
|
@@ -2541,7 +2702,7 @@ var MaintenanceService = class {
|
|
|
2541
2702
|
this.jobManager = jobManager;
|
|
2542
2703
|
this.embeddingService = embeddingService;
|
|
2543
2704
|
this.ontologyService = ontologyService;
|
|
2544
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
2705
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
2545
2706
|
}
|
|
2546
2707
|
async runPrune(entityId, options) {
|
|
2547
2708
|
this.jobManager.acquireLock("prune", entityId);
|
|
@@ -2846,8 +3007,10 @@ var MaintenanceService = class {
|
|
|
2846
3007
|
};
|
|
2847
3008
|
});
|
|
2848
3009
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
2849
|
-
const
|
|
2850
|
-
|
|
3010
|
+
const promptEvents = events.reverse();
|
|
3011
|
+
const librarianGrounding = this.promptService.groundingFor("librarian");
|
|
3012
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = this.promptService.buildLibrarianPrompt(
|
|
3013
|
+
promptEvents,
|
|
2851
3014
|
currentFacts,
|
|
2852
3015
|
promptOverride,
|
|
2853
3016
|
ontologyContext
|
|
@@ -2916,6 +3079,10 @@ var MaintenanceService = class {
|
|
|
2916
3079
|
const validationDrops = [];
|
|
2917
3080
|
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
2918
3081
|
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3082
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3083
|
+
if (grounding?.diagnostic) {
|
|
3084
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3085
|
+
}
|
|
2919
3086
|
const factObj = {
|
|
2920
3087
|
id,
|
|
2921
3088
|
entity_id: entityId,
|
|
@@ -2931,7 +3098,8 @@ var MaintenanceService = class {
|
|
|
2931
3098
|
last_accessed_at: null,
|
|
2932
3099
|
access_count: 0,
|
|
2933
3100
|
deleted_at: null,
|
|
2934
|
-
okf_type: normalized.okf_type
|
|
3101
|
+
okf_type: normalized.okf_type,
|
|
3102
|
+
...grounding?.trust
|
|
2935
3103
|
};
|
|
2936
3104
|
await this.entryRepo.upsert(factObj, tx);
|
|
2937
3105
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3047,11 +3215,14 @@ var MaintenanceService = class {
|
|
|
3047
3215
|
}
|
|
3048
3216
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
3049
3217
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
3218
|
+
const healGrounding = this.promptService.groundingFor("heal");
|
|
3050
3219
|
const toPromptShape = (f) => {
|
|
3051
3220
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
3052
|
-
|
|
3221
|
+
const shown = healGrounding ? (({ lifecycle_status: _l, okf_verified: _v, last_verified_at: _a, last_verified_by: _b, ...r }) => r)(rest) : rest;
|
|
3222
|
+
return { ...shown, tags: typeof shown.tags === "string" ? JSON.parse(shown.tags) : shown.tags };
|
|
3053
3223
|
};
|
|
3054
3224
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
3225
|
+
const corpusByPrompt = /* @__PURE__ */ new WeakMap();
|
|
3055
3226
|
const degraded = [];
|
|
3056
3227
|
const outcome = await runBatched({
|
|
3057
3228
|
items: healCandidates,
|
|
@@ -3064,9 +3235,10 @@ var MaintenanceService = class {
|
|
|
3064
3235
|
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
3065
3236
|
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
3066
3237
|
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
3067
|
-
anchorCache
|
|
3238
|
+
anchorCache,
|
|
3239
|
+
healGrounding !== null
|
|
3068
3240
|
);
|
|
3069
|
-
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
3241
|
+
const { prompts, degraded: batchDegraded, groundingCorpus } = await this.promptService.buildHealPrompt(
|
|
3070
3242
|
batch.map(toPromptShape),
|
|
3071
3243
|
documentAnchors,
|
|
3072
3244
|
allTasks,
|
|
@@ -3076,16 +3248,18 @@ var MaintenanceService = class {
|
|
|
3076
3248
|
bodyTruncationChars
|
|
3077
3249
|
);
|
|
3078
3250
|
degraded.push(...batchDegraded);
|
|
3251
|
+
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3079
3252
|
return prompts;
|
|
3080
3253
|
},
|
|
3081
3254
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
3082
|
-
parse: (responseText, batch) => {
|
|
3255
|
+
parse: (responseText, batch, prompts) => {
|
|
3083
3256
|
const result = parseJsonResponse(responseText);
|
|
3084
3257
|
return {
|
|
3085
3258
|
batch,
|
|
3086
3259
|
downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
|
|
3087
3260
|
deleted: Array.isArray(result.deleted) ? result.deleted : [],
|
|
3088
|
-
newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
|
|
3261
|
+
newFacts: Array.isArray(result.newFacts) ? result.newFacts : [],
|
|
3262
|
+
corpus: corpusByPrompt.get(prompts) ?? null
|
|
3089
3263
|
};
|
|
3090
3264
|
},
|
|
3091
3265
|
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
@@ -3100,6 +3274,7 @@ var MaintenanceService = class {
|
|
|
3100
3274
|
const safeDeletedSet = /* @__PURE__ */ new Set();
|
|
3101
3275
|
const newFacts = [];
|
|
3102
3276
|
const newFactItemIndexes = [];
|
|
3277
|
+
const newFactCorpora = [];
|
|
3103
3278
|
for (const batchResult of outcome.results) {
|
|
3104
3279
|
const mutableIds = new Set(batchResult.batch.map((f) => f.id));
|
|
3105
3280
|
for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
|
|
@@ -3107,6 +3282,7 @@ var MaintenanceService = class {
|
|
|
3107
3282
|
batchResult.newFacts.forEach((raw, itemIndex) => {
|
|
3108
3283
|
newFacts.push(raw);
|
|
3109
3284
|
newFactItemIndexes.push(itemIndex);
|
|
3285
|
+
newFactCorpora.push(batchResult.corpus);
|
|
3110
3286
|
});
|
|
3111
3287
|
}
|
|
3112
3288
|
const safeDowngraded = Array.from(safeDowngradedSet);
|
|
@@ -3114,12 +3290,14 @@ var MaintenanceService = class {
|
|
|
3114
3290
|
const diagBuffer = new DiagnosticBuffer();
|
|
3115
3291
|
const validNewFacts = [];
|
|
3116
3292
|
const validNewFactItemIndexes = [];
|
|
3293
|
+
const validNewFactCorpora = [];
|
|
3117
3294
|
newFacts.forEach((raw, k) => {
|
|
3118
3295
|
const itemIndex = newFactItemIndexes[k];
|
|
3119
3296
|
const valid = validateFact(raw);
|
|
3120
3297
|
if (valid) {
|
|
3121
3298
|
validNewFacts.push(valid);
|
|
3122
3299
|
validNewFactItemIndexes.push(itemIndex);
|
|
3300
|
+
validNewFactCorpora.push(newFactCorpora[k]);
|
|
3123
3301
|
} else {
|
|
3124
3302
|
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3125
3303
|
}
|
|
@@ -3149,6 +3327,10 @@ var MaintenanceService = class {
|
|
|
3149
3327
|
continue;
|
|
3150
3328
|
}
|
|
3151
3329
|
const id = generateId("fact_");
|
|
3330
|
+
const grounding = healGrounding ? groundingOutcome(checkGrounding(fact.evidence, validNewFactCorpora[k] ?? [], healGrounding), now) : null;
|
|
3331
|
+
if (grounding?.diagnostic) {
|
|
3332
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validNewFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3333
|
+
}
|
|
3152
3334
|
const factObj = {
|
|
3153
3335
|
id,
|
|
3154
3336
|
entity_id: entityId,
|
|
@@ -3163,7 +3345,8 @@ var MaintenanceService = class {
|
|
|
3163
3345
|
updated_at: now,
|
|
3164
3346
|
last_accessed_at: null,
|
|
3165
3347
|
access_count: 0,
|
|
3166
|
-
deleted_at: null
|
|
3348
|
+
deleted_at: null,
|
|
3349
|
+
...grounding?.trust
|
|
3167
3350
|
};
|
|
3168
3351
|
await this.entryRepo.upsert(factObj, tx);
|
|
3169
3352
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3527,7 +3710,7 @@ var MaintenanceService = class {
|
|
|
3527
3710
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
3528
3711
|
* per-pass — see the call site in doRunHeal.
|
|
3529
3712
|
*/
|
|
3530
|
-
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
3713
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache, withBodies = false) {
|
|
3531
3714
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
3532
3715
|
if (!query) return [];
|
|
3533
3716
|
const cached = cache?.get(query);
|
|
@@ -3540,7 +3723,7 @@ var MaintenanceService = class {
|
|
|
3540
3723
|
const hitIds = hits.map((h) => h.id);
|
|
3541
3724
|
const anchors = [];
|
|
3542
3725
|
if (hitIds.length > 0) {
|
|
3543
|
-
const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
3726
|
+
const rows = withBodies ? await this.entryRepo.findAnchorRowsByIds(entityId, hitIds, void 0, { withBody: true }) : await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
3544
3727
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
3545
3728
|
for (const id of hitIds) {
|
|
3546
3729
|
const row = byId.get(id);
|