@equationalapplications/core-llm-wiki 7.5.0 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/{chunk-3P7FAKJA.mjs → chunk-VOSMYISQ.mjs} +243 -41
- package/dist/chunk-VOSMYISQ.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +434 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +196 -5
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-ClpOUiLI.d.mts → testing-A4Sofxyo.d.mts} +136 -4
- package/dist/{testing-ClpOUiLI.d.ts → testing-A4Sofxyo.d.ts} +136 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +236 -38
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-3P7FAKJA.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -593,6 +593,19 @@ function validateTags(tags) {
|
|
|
593
593
|
if (!Array.isArray(tags)) return [];
|
|
594
594
|
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
595
595
|
}
|
|
596
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
597
|
+
function normalizeEvidence(raw) {
|
|
598
|
+
if (!Array.isArray(raw)) return void 0;
|
|
599
|
+
const out = [];
|
|
600
|
+
for (const entry of raw) {
|
|
601
|
+
if (typeof entry !== "string") continue;
|
|
602
|
+
const trimmed = entry.trim();
|
|
603
|
+
if (!trimmed) continue;
|
|
604
|
+
out.push(trimmed);
|
|
605
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
606
|
+
}
|
|
607
|
+
return out;
|
|
608
|
+
}
|
|
596
609
|
function validateFact(fact) {
|
|
597
610
|
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
598
611
|
const title = clip(fact.title, 80);
|
|
@@ -600,13 +613,17 @@ function validateFact(fact) {
|
|
|
600
613
|
if (!title || !body) return null;
|
|
601
614
|
let confidence = fact.confidence;
|
|
602
615
|
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
603
|
-
|
|
616
|
+
const valid = {
|
|
604
617
|
...fact,
|
|
605
618
|
title,
|
|
606
619
|
body,
|
|
607
620
|
confidence,
|
|
608
621
|
tags: validateTags(fact.tags)
|
|
609
622
|
};
|
|
623
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
624
|
+
if (evidence) valid.evidence = evidence;
|
|
625
|
+
else delete valid.evidence;
|
|
626
|
+
return valid;
|
|
610
627
|
}
|
|
611
628
|
function validateTask(task) {
|
|
612
629
|
if (typeof task?.description !== "string") return null;
|
|
@@ -1509,17 +1526,94 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
1509
1526
|
}
|
|
1510
1527
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
1511
1528
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
1529
|
+
var GROUNDING_SOURCE = {
|
|
1530
|
+
ingest: { key: "facts", source: "the document chunk" },
|
|
1531
|
+
librarian: { key: "facts", source: 'the "summary" text of the events' },
|
|
1532
|
+
heal: { key: "newFacts", source: 'the "summary" text of the recent events or the "body" text of the document anchors' }
|
|
1533
|
+
};
|
|
1534
|
+
function groundingEvidenceBlock(writer, cfg) {
|
|
1535
|
+
const { key, source } = GROUNDING_SOURCE[writer];
|
|
1536
|
+
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1537
|
+
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1538
|
+
}
|
|
1512
1539
|
|
|
1513
1540
|
// src/utils/healConstants.ts
|
|
1514
1541
|
var HEAL_MAX_ANCHORS = 50;
|
|
1515
1542
|
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
1516
1543
|
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
1517
1544
|
var HEAL_MAX_TASKS = 50;
|
|
1545
|
+
var HEAL_ANCHOR_BODY_CHARS = 800;
|
|
1546
|
+
|
|
1547
|
+
// src/utils/grounding.ts
|
|
1548
|
+
var GROUNDING_VERIFIER = "process:grounding-check";
|
|
1549
|
+
var WRITERS = ["ingest", "librarian", "heal"];
|
|
1550
|
+
function positiveInt(value, fallback) {
|
|
1551
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
1552
|
+
}
|
|
1553
|
+
function resolveGrounding(config) {
|
|
1554
|
+
if (!config || config.mode !== "draft") return null;
|
|
1555
|
+
const writers = Array.isArray(config.writers) ? config.writers.filter((w) => WRITERS.includes(w)) : ["ingest"];
|
|
1556
|
+
return {
|
|
1557
|
+
writers: new Set(writers),
|
|
1558
|
+
minEvidenceChars: positiveInt(config.minEvidenceChars, 20),
|
|
1559
|
+
// Never ask for more quotes than checkGrounding accepts (MAX_EVIDENCE_QUOTES).
|
|
1560
|
+
maxEvidence: Math.min(MAX_EVIDENCE_QUOTES, positiveInt(config.maxEvidence, 3)),
|
|
1561
|
+
maxEvidenceChars: positiveInt(config.maxEvidenceChars, 300)
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
function normalizeForGrounding(text) {
|
|
1565
|
+
return text.normalize("NFKC").replace(/\s+/g, " ").trim();
|
|
1566
|
+
}
|
|
1567
|
+
function buildGroundingCorpus(parts) {
|
|
1568
|
+
return parts.filter((p) => typeof p === "string").map(normalizeForGrounding);
|
|
1569
|
+
}
|
|
1570
|
+
function checkGrounding(evidence, normalizedCorpus, cfg) {
|
|
1571
|
+
const quotes = evidence ?? [];
|
|
1572
|
+
if (quotes.length > MAX_EVIDENCE_QUOTES) return { status: "failed", reason: "too_many_quotes" };
|
|
1573
|
+
if (quotes.length === 0) return { status: "missing", reason: "no_evidence" };
|
|
1574
|
+
const qualifying = quotes.map(normalizeForGrounding).filter((q) => q.length >= cfg.minEvidenceChars);
|
|
1575
|
+
if (qualifying.length === 0) return { status: "missing", reason: "evidence_too_short" };
|
|
1576
|
+
if (qualifying.some((q) => !normalizedCorpus.some((part) => part.includes(q)))) {
|
|
1577
|
+
return { status: "failed", reason: "quote_not_found" };
|
|
1578
|
+
}
|
|
1579
|
+
return {
|
|
1580
|
+
status: "grounded",
|
|
1581
|
+
retained: qualifying.slice(0, cfg.maxEvidence).map((q) => safeSlice(q, 0, cfg.maxEvidenceChars))
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
function groundingOutcome(verdict, now) {
|
|
1585
|
+
if (verdict.status === "grounded") {
|
|
1586
|
+
return {
|
|
1587
|
+
trust: {
|
|
1588
|
+
lifecycle_status: "stable",
|
|
1589
|
+
okf_verified: [{ by: GROUNDING_VERIFIER, at: new Date(now).toISOString() }],
|
|
1590
|
+
last_verified_at: now,
|
|
1591
|
+
last_verified_by: GROUNDING_VERIFIER
|
|
1592
|
+
}
|
|
1593
|
+
};
|
|
1594
|
+
}
|
|
1595
|
+
return {
|
|
1596
|
+
trust: { lifecycle_status: "draft" },
|
|
1597
|
+
diagnostic: { code: verdict.status === "missing" ? "grounding_missing" : "grounding_failed", reason: verdict.reason }
|
|
1598
|
+
};
|
|
1599
|
+
}
|
|
1518
1600
|
|
|
1519
1601
|
// src/services/PromptService.ts
|
|
1520
1602
|
var PromptService = class {
|
|
1521
|
-
constructor(globalOverrides) {
|
|
1603
|
+
constructor(globalOverrides, grounding = null) {
|
|
1522
1604
|
this.globalOverrides = globalOverrides;
|
|
1605
|
+
this.grounding = grounding;
|
|
1606
|
+
}
|
|
1607
|
+
/** The resolved grounding config when `writer` is in `grounding.writers`; otherwise null (spec §6.2). */
|
|
1608
|
+
groundingFor(writer) {
|
|
1609
|
+
return this.grounding?.writers.has(writer) ? this.grounding : null;
|
|
1610
|
+
}
|
|
1611
|
+
/** Appended after any override and after ontology context, so it is always last. */
|
|
1612
|
+
appendGrounding(systemPrompt, writer) {
|
|
1613
|
+
const cfg = this.groundingFor(writer);
|
|
1614
|
+
return cfg ? `${systemPrompt}
|
|
1615
|
+
|
|
1616
|
+
${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
1523
1617
|
}
|
|
1524
1618
|
hydrate(template, variables) {
|
|
1525
1619
|
return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
|
|
@@ -1549,13 +1643,13 @@ ${ctx.ontologyModeInstructions}`;
|
|
|
1549
1643
|
const hasDocumentChunk = /\{\{\s*documentChunk\s*\}\}/.test(template);
|
|
1550
1644
|
if (hasDocumentChunk || this.hasOntologyPlaceholders(template)) {
|
|
1551
1645
|
return {
|
|
1552
|
-
systemPrompt: this.buildSystemPrompt(template, { documentChunk }, ontologyContext),
|
|
1646
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { documentChunk }, ontologyContext), "ingest"),
|
|
1553
1647
|
userPrompt: hasDocumentChunk ? "Please extract the facts." : `Document Chunk:
|
|
1554
1648
|
${documentChunk}`
|
|
1555
1649
|
};
|
|
1556
1650
|
}
|
|
1557
1651
|
return {
|
|
1558
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1652
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "ingest"),
|
|
1559
1653
|
userPrompt: `Document Chunk:
|
|
1560
1654
|
${documentChunk}`
|
|
1561
1655
|
};
|
|
@@ -1564,23 +1658,27 @@ ${documentChunk}`
|
|
|
1564
1658
|
const template = runtimeOverride ?? this.globalOverrides?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT;
|
|
1565
1659
|
const hasEvents = /\{\{\s*events\s*\}\}/.test(template);
|
|
1566
1660
|
const hasCurrentFacts = /\{\{\s*currentFacts\s*\}\}/.test(template);
|
|
1661
|
+
const eventsShown = hasEvents || !hasCurrentFacts;
|
|
1662
|
+
const corpusField = this.groundingFor("librarian") ? { groundingCorpus: buildGroundingCorpus(eventsShown ? events.map(summaryOf) : []) } : {};
|
|
1567
1663
|
if (hasEvents || hasCurrentFacts || this.hasOntologyPlaceholders(template)) {
|
|
1568
1664
|
return {
|
|
1569
|
-
systemPrompt: this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext),
|
|
1665
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext), "librarian"),
|
|
1570
1666
|
userPrompt: hasEvents || hasCurrentFacts ? "Please synthesize the context." : `Events:
|
|
1571
1667
|
${JSON.stringify(events, null, 2)}
|
|
1572
1668
|
|
|
1573
1669
|
Current Facts:
|
|
1574
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1670
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1671
|
+
...corpusField
|
|
1575
1672
|
};
|
|
1576
1673
|
}
|
|
1577
1674
|
return {
|
|
1578
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1675
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "librarian"),
|
|
1579
1676
|
userPrompt: `Events:
|
|
1580
1677
|
${JSON.stringify(events, null, 2)}
|
|
1581
1678
|
|
|
1582
1679
|
Current Facts:
|
|
1583
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1680
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1681
|
+
...corpusField
|
|
1584
1682
|
};
|
|
1585
1683
|
}
|
|
1586
1684
|
/**
|
|
@@ -1610,40 +1708,54 @@ ${JSON.stringify(currentFacts, null, 2)}`
|
|
|
1610
1708
|
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
1611
1709
|
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
1612
1710
|
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
1711
|
+
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
1712
|
+
const hasRecentEvents = /\{\{\s*recentEvents\s*\}\}/.test(template);
|
|
1713
|
+
const hasDocumentAnchors = /\{\{\s*documentAnchors\s*\}\}/.test(template);
|
|
1714
|
+
const usesPlaceholders = /\{\{\s*healCandidates\s*\}\}/.test(template) || hasDocumentAnchors || /\{\{\s*allTasks\s*\}\}/.test(template) || hasRecentEvents;
|
|
1715
|
+
const healGrounding = this.groundingFor("heal");
|
|
1716
|
+
const promptAnchors = healGrounding ? effectiveAnchors.map(toGroundingAnchor) : effectiveAnchors;
|
|
1717
|
+
const eventsShown = !usesPlaceholders || hasRecentEvents;
|
|
1718
|
+
const anchorsShown = !usesPlaceholders || hasDocumentAnchors;
|
|
1719
|
+
const groundingCorpus = healGrounding ? buildGroundingCorpus([
|
|
1720
|
+
...eventsShown ? effectiveEvents.map(summaryOf) : [],
|
|
1721
|
+
...anchorsShown ? effectiveAnchors.map((a, i) => ({ status: a?.lifecycle_status, shown: promptAnchors[i] })).filter((x) => x.status !== "draft").map((x) => x.shown?.body) : []
|
|
1722
|
+
]) : null;
|
|
1613
1723
|
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
1614
1724
|
healCandidates,
|
|
1615
1725
|
attemptLevel,
|
|
1616
1726
|
bodyTruncationChars
|
|
1617
1727
|
);
|
|
1618
|
-
const
|
|
1619
|
-
if (
|
|
1728
|
+
const corpusField = groundingCorpus !== null ? { groundingCorpus } : {};
|
|
1729
|
+
if (usesPlaceholders) {
|
|
1620
1730
|
return {
|
|
1621
1731
|
prompts: {
|
|
1622
|
-
systemPrompt: this.hydrate(template, {
|
|
1732
|
+
systemPrompt: this.appendGrounding(this.hydrate(template, {
|
|
1623
1733
|
healCandidates: shapedCandidates,
|
|
1624
|
-
documentAnchors:
|
|
1734
|
+
documentAnchors: promptAnchors,
|
|
1625
1735
|
allTasks: effectiveTasks,
|
|
1626
1736
|
recentEvents: effectiveEvents
|
|
1627
|
-
}),
|
|
1737
|
+
}), "heal"),
|
|
1628
1738
|
userPrompt: "Please heal the memory graph."
|
|
1629
1739
|
},
|
|
1630
|
-
degraded
|
|
1740
|
+
degraded,
|
|
1741
|
+
...corpusField
|
|
1631
1742
|
};
|
|
1632
1743
|
}
|
|
1633
1744
|
return {
|
|
1634
1745
|
prompts: {
|
|
1635
|
-
systemPrompt: template,
|
|
1746
|
+
systemPrompt: this.appendGrounding(template, "heal"),
|
|
1636
1747
|
userPrompt: `Heal Candidates:
|
|
1637
1748
|
${JSON.stringify(shapedCandidates, null, 2)}
|
|
1638
1749
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
1639
|
-
${JSON.stringify(
|
|
1750
|
+
${JSON.stringify(promptAnchors, null, 2)}
|
|
1640
1751
|
All Tasks:
|
|
1641
1752
|
${JSON.stringify(effectiveTasks, null, 2)}
|
|
1642
1753
|
Recent Events:
|
|
1643
1754
|
${JSON.stringify(effectiveEvents, null, 2)}
|
|
1644
1755
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
1645
1756
|
},
|
|
1646
|
-
degraded
|
|
1757
|
+
degraded,
|
|
1758
|
+
...corpusField
|
|
1647
1759
|
};
|
|
1648
1760
|
}
|
|
1649
1761
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -1662,6 +1774,21 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
1662
1774
|
${JSON.stringify(facts, null, 2)}`
|
|
1663
1775
|
};
|
|
1664
1776
|
}
|
|
1777
|
+
/**
|
|
1778
|
+
* The system prompt each writer would send, without hydrating data (spec
|
|
1779
|
+
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
1780
|
+
* placeholders and leaves data placeholders verbatim; heal never receives
|
|
1781
|
+
* ontology context, matching `buildHealPrompt`.
|
|
1782
|
+
*/
|
|
1783
|
+
buildInstructionTemplates(ontologyContext) {
|
|
1784
|
+
const o = this.globalOverrides;
|
|
1785
|
+
return {
|
|
1786
|
+
ingest: this.appendGrounding(this.buildSystemPrompt(o?.ingestSystemPrompt ?? INGEST_SYSTEM_PROMPT, {}, ontologyContext), "ingest"),
|
|
1787
|
+
librarian: this.appendGrounding(this.buildSystemPrompt(o?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT, {}, ontologyContext), "librarian"),
|
|
1788
|
+
heal: this.appendGrounding(o?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT, "heal"),
|
|
1789
|
+
ontologyBackfill: this.buildSystemPrompt(o?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT, {}, ontologyContext)
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1665
1792
|
};
|
|
1666
1793
|
function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
1667
1794
|
if (attemptLevel < 3) {
|
|
@@ -1693,6 +1820,19 @@ function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
|
1693
1820
|
}
|
|
1694
1821
|
return { shapedCandidates, degraded };
|
|
1695
1822
|
}
|
|
1823
|
+
function summaryOf(event) {
|
|
1824
|
+
return event?.summary;
|
|
1825
|
+
}
|
|
1826
|
+
function toGroundingAnchor(anchor) {
|
|
1827
|
+
if (typeof anchor !== "object" || anchor === null) return anchor;
|
|
1828
|
+
const a = anchor;
|
|
1829
|
+
return {
|
|
1830
|
+
id: a.id,
|
|
1831
|
+
title: a.title,
|
|
1832
|
+
source_ref: a.source_ref,
|
|
1833
|
+
body: typeof a.body === "string" ? safeSlice(a.body, 0, HEAL_ANCHOR_BODY_CHARS) : ""
|
|
1834
|
+
};
|
|
1835
|
+
}
|
|
1696
1836
|
|
|
1697
1837
|
// src/utils/chunkingDefaults.ts
|
|
1698
1838
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -1722,7 +1862,7 @@ var IngestionService = class {
|
|
|
1722
1862
|
this.jobManager = jobManager;
|
|
1723
1863
|
this.embeddingService = embeddingService;
|
|
1724
1864
|
this.ontologyService = ontologyService;
|
|
1725
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
1865
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
1726
1866
|
}
|
|
1727
1867
|
async ingestDocument(entityId, params, opts) {
|
|
1728
1868
|
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
@@ -1755,6 +1895,7 @@ var IngestionService = class {
|
|
|
1755
1895
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1756
1896
|
if (chunks.length === 0) return zeroChunkResult();
|
|
1757
1897
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
1898
|
+
const ingestGrounding = this.promptService.groundingFor("ingest");
|
|
1758
1899
|
const chunkResults = await withConcurrency(
|
|
1759
1900
|
chunks.map((chunk, chunkIndex) => async () => {
|
|
1760
1901
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
@@ -1778,11 +1919,16 @@ var IngestionService = class {
|
|
|
1778
1919
|
rejected.push({ itemIndex, reason: factRejectionReason(raw) });
|
|
1779
1920
|
}
|
|
1780
1921
|
});
|
|
1922
|
+
const verdicts = ingestGrounding ? (() => {
|
|
1923
|
+
const corpus = buildGroundingCorpus([chunk]);
|
|
1924
|
+
return facts.map((f) => checkGrounding(f.evidence, corpus, ingestGrounding));
|
|
1925
|
+
})() : [];
|
|
1781
1926
|
return {
|
|
1782
1927
|
status: "ok",
|
|
1783
1928
|
facts,
|
|
1784
1929
|
itemIndexes,
|
|
1785
1930
|
rejected,
|
|
1931
|
+
verdicts,
|
|
1786
1932
|
ontology_updates: result2.ontology_updates
|
|
1787
1933
|
};
|
|
1788
1934
|
} catch (e) {
|
|
@@ -1827,6 +1973,7 @@ var IngestionService = class {
|
|
|
1827
1973
|
const failures = [];
|
|
1828
1974
|
const seen = /* @__PURE__ */ new Set();
|
|
1829
1975
|
const orderedChunkFacts = [];
|
|
1976
|
+
const groundingLedger = /* @__PURE__ */ new Map();
|
|
1830
1977
|
const diagBuffer = new DiagnosticBuffer();
|
|
1831
1978
|
const diagBase = { entityId, operation: "ingest", trigger: "call" };
|
|
1832
1979
|
for (const [chunkIndex, slot] of chunkResults.entries()) {
|
|
@@ -1845,6 +1992,7 @@ var IngestionService = class {
|
|
|
1845
1992
|
if (!seen.has(normalizedTitle)) {
|
|
1846
1993
|
seen.add(normalizedTitle);
|
|
1847
1994
|
dedupedFacts.push(fact);
|
|
1995
|
+
if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: slot.itemIndexes[k] });
|
|
1848
1996
|
} else {
|
|
1849
1997
|
diagBuffer.push({
|
|
1850
1998
|
...diagBase,
|
|
@@ -1881,7 +2029,7 @@ var IngestionService = class {
|
|
|
1881
2029
|
try {
|
|
1882
2030
|
if (failedChunks === 0) {
|
|
1883
2031
|
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1884
|
-
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer);
|
|
2032
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger);
|
|
1885
2033
|
});
|
|
1886
2034
|
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
1887
2035
|
insertedFacts.push(...fullResult.insertedFacts);
|
|
@@ -1889,7 +2037,7 @@ var IngestionService = class {
|
|
|
1889
2037
|
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1890
2038
|
const flat = [];
|
|
1891
2039
|
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
1892
|
-
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer);
|
|
2040
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, groundingLedger);
|
|
1893
2041
|
});
|
|
1894
2042
|
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
1895
2043
|
}
|
|
@@ -2031,7 +2179,8 @@ var IngestionService = class {
|
|
|
2031
2179
|
last_accessed_at: null,
|
|
2032
2180
|
access_count: 0,
|
|
2033
2181
|
deleted_at: null,
|
|
2034
|
-
okf_type: normalized.okf_type
|
|
2182
|
+
okf_type: normalized.okf_type,
|
|
2183
|
+
...opts?.nodeTrust?.get(node.id)
|
|
2035
2184
|
};
|
|
2036
2185
|
wikiFacts.push(wikiFact);
|
|
2037
2186
|
}
|
|
@@ -2122,9 +2271,10 @@ var IngestionService = class {
|
|
|
2122
2271
|
* On the partial path the caller never invokes this method; the empty
|
|
2123
2272
|
* arrays stay empty.
|
|
2124
2273
|
*/
|
|
2125
|
-
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer) {
|
|
2274
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2126
2275
|
const deletedSourceFactIds = [];
|
|
2127
2276
|
const insertedFacts = [];
|
|
2277
|
+
const nodeTrust = /* @__PURE__ */ new Map();
|
|
2128
2278
|
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
2129
2279
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
2130
2280
|
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
@@ -2168,6 +2318,20 @@ var IngestionService = class {
|
|
|
2168
2318
|
tags: fact.tags,
|
|
2169
2319
|
confidence: fact.confidence
|
|
2170
2320
|
});
|
|
2321
|
+
const grounding = groundingLedger.get(fact);
|
|
2322
|
+
if (grounding) {
|
|
2323
|
+
const outcome = groundingOutcome(grounding.verdict, now);
|
|
2324
|
+
nodeTrust.set(id, outcome.trust);
|
|
2325
|
+
if (outcome.diagnostic) {
|
|
2326
|
+
diagBuffer.push({
|
|
2327
|
+
entityId,
|
|
2328
|
+
operation: "ingest",
|
|
2329
|
+
trigger: "call",
|
|
2330
|
+
code: outcome.diagnostic.code,
|
|
2331
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2332
|
+
});
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2171
2335
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2172
2336
|
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
2173
2337
|
if (normalized.edges.length > 0) {
|
|
@@ -2199,7 +2363,7 @@ var IngestionService = class {
|
|
|
2199
2363
|
entityId,
|
|
2200
2364
|
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
2201
2365
|
tx,
|
|
2202
|
-
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" } }
|
|
2366
|
+
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" }, ...nodeTrust.size > 0 ? { nodeTrust } : {} }
|
|
2203
2367
|
);
|
|
2204
2368
|
return { deletedSourceFactIds, insertedFacts };
|
|
2205
2369
|
}
|
|
@@ -2223,7 +2387,7 @@ var IngestionService = class {
|
|
|
2223
2387
|
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
2224
2388
|
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
2225
2389
|
*/
|
|
2226
|
-
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer) {
|
|
2390
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2227
2391
|
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
2228
2392
|
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
2229
2393
|
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
@@ -2246,6 +2410,17 @@ var IngestionService = class {
|
|
|
2246
2410
|
}
|
|
2247
2411
|
liveTitles.add(normalizedTitle);
|
|
2248
2412
|
const id = generateId("fact_");
|
|
2413
|
+
const grounding = groundingLedger.get(fact);
|
|
2414
|
+
const outcome = grounding ? groundingOutcome(grounding.verdict, now) : null;
|
|
2415
|
+
if (grounding && outcome?.diagnostic) {
|
|
2416
|
+
diagBuffer.push({
|
|
2417
|
+
entityId,
|
|
2418
|
+
operation: "ingest",
|
|
2419
|
+
trigger: "call",
|
|
2420
|
+
code: outcome.diagnostic.code,
|
|
2421
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2422
|
+
});
|
|
2423
|
+
}
|
|
2249
2424
|
const wikiFact = {
|
|
2250
2425
|
id,
|
|
2251
2426
|
entity_id: entityId,
|
|
@@ -2264,7 +2439,8 @@ var IngestionService = class {
|
|
|
2264
2439
|
last_accessed_at: null,
|
|
2265
2440
|
access_count: 0,
|
|
2266
2441
|
deleted_at: null,
|
|
2267
|
-
okf_type: null
|
|
2442
|
+
okf_type: null,
|
|
2443
|
+
...outcome?.trust
|
|
2268
2444
|
};
|
|
2269
2445
|
await this.entryRepo.upsert(wikiFact, tx);
|
|
2270
2446
|
insertedDescriptors.push({
|
|
@@ -2460,7 +2636,7 @@ async function runBatched(args) {
|
|
|
2460
2636
|
}
|
|
2461
2637
|
let result;
|
|
2462
2638
|
try {
|
|
2463
|
-
result = parse(responseText, batch);
|
|
2639
|
+
result = parse(responseText, batch, prompts);
|
|
2464
2640
|
} catch (err) {
|
|
2465
2641
|
await onFailure(batch, err, attemptLevel, false);
|
|
2466
2642
|
return;
|
|
@@ -2541,7 +2717,7 @@ var MaintenanceService = class {
|
|
|
2541
2717
|
this.jobManager = jobManager;
|
|
2542
2718
|
this.embeddingService = embeddingService;
|
|
2543
2719
|
this.ontologyService = ontologyService;
|
|
2544
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
2720
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
2545
2721
|
}
|
|
2546
2722
|
async runPrune(entityId, options) {
|
|
2547
2723
|
this.jobManager.acquireLock("prune", entityId);
|
|
@@ -2846,8 +3022,10 @@ var MaintenanceService = class {
|
|
|
2846
3022
|
};
|
|
2847
3023
|
});
|
|
2848
3024
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
2849
|
-
const
|
|
2850
|
-
|
|
3025
|
+
const promptEvents = events.reverse();
|
|
3026
|
+
const librarianGrounding = this.promptService.groundingFor("librarian");
|
|
3027
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = this.promptService.buildLibrarianPrompt(
|
|
3028
|
+
promptEvents,
|
|
2851
3029
|
currentFacts,
|
|
2852
3030
|
promptOverride,
|
|
2853
3031
|
ontologyContext
|
|
@@ -2916,6 +3094,10 @@ var MaintenanceService = class {
|
|
|
2916
3094
|
const validationDrops = [];
|
|
2917
3095
|
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
2918
3096
|
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3097
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3098
|
+
if (grounding?.diagnostic) {
|
|
3099
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3100
|
+
}
|
|
2919
3101
|
const factObj = {
|
|
2920
3102
|
id,
|
|
2921
3103
|
entity_id: entityId,
|
|
@@ -2931,7 +3113,8 @@ var MaintenanceService = class {
|
|
|
2931
3113
|
last_accessed_at: null,
|
|
2932
3114
|
access_count: 0,
|
|
2933
3115
|
deleted_at: null,
|
|
2934
|
-
okf_type: normalized.okf_type
|
|
3116
|
+
okf_type: normalized.okf_type,
|
|
3117
|
+
...grounding?.trust
|
|
2935
3118
|
};
|
|
2936
3119
|
await this.entryRepo.upsert(factObj, tx);
|
|
2937
3120
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3047,11 +3230,14 @@ var MaintenanceService = class {
|
|
|
3047
3230
|
}
|
|
3048
3231
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
3049
3232
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
3233
|
+
const healGrounding = this.promptService.groundingFor("heal");
|
|
3050
3234
|
const toPromptShape = (f) => {
|
|
3051
3235
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
3052
|
-
|
|
3236
|
+
const shown = healGrounding ? (({ lifecycle_status: _l, okf_verified: _v, last_verified_at: _a, last_verified_by: _b, ...r }) => r)(rest) : rest;
|
|
3237
|
+
return { ...shown, tags: typeof shown.tags === "string" ? JSON.parse(shown.tags) : shown.tags };
|
|
3053
3238
|
};
|
|
3054
3239
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
3240
|
+
const corpusByPrompt = /* @__PURE__ */ new WeakMap();
|
|
3055
3241
|
const degraded = [];
|
|
3056
3242
|
const outcome = await runBatched({
|
|
3057
3243
|
items: healCandidates,
|
|
@@ -3064,9 +3250,10 @@ var MaintenanceService = class {
|
|
|
3064
3250
|
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
3065
3251
|
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
3066
3252
|
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
3067
|
-
anchorCache
|
|
3253
|
+
anchorCache,
|
|
3254
|
+
healGrounding !== null
|
|
3068
3255
|
);
|
|
3069
|
-
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
3256
|
+
const { prompts, degraded: batchDegraded, groundingCorpus } = await this.promptService.buildHealPrompt(
|
|
3070
3257
|
batch.map(toPromptShape),
|
|
3071
3258
|
documentAnchors,
|
|
3072
3259
|
allTasks,
|
|
@@ -3076,16 +3263,18 @@ var MaintenanceService = class {
|
|
|
3076
3263
|
bodyTruncationChars
|
|
3077
3264
|
);
|
|
3078
3265
|
degraded.push(...batchDegraded);
|
|
3266
|
+
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3079
3267
|
return prompts;
|
|
3080
3268
|
},
|
|
3081
3269
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
3082
|
-
parse: (responseText, batch) => {
|
|
3270
|
+
parse: (responseText, batch, prompts) => {
|
|
3083
3271
|
const result = parseJsonResponse(responseText);
|
|
3084
3272
|
return {
|
|
3085
3273
|
batch,
|
|
3086
3274
|
downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
|
|
3087
3275
|
deleted: Array.isArray(result.deleted) ? result.deleted : [],
|
|
3088
|
-
newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
|
|
3276
|
+
newFacts: Array.isArray(result.newFacts) ? result.newFacts : [],
|
|
3277
|
+
corpus: corpusByPrompt.get(prompts) ?? null
|
|
3089
3278
|
};
|
|
3090
3279
|
},
|
|
3091
3280
|
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
@@ -3100,6 +3289,7 @@ var MaintenanceService = class {
|
|
|
3100
3289
|
const safeDeletedSet = /* @__PURE__ */ new Set();
|
|
3101
3290
|
const newFacts = [];
|
|
3102
3291
|
const newFactItemIndexes = [];
|
|
3292
|
+
const newFactCorpora = [];
|
|
3103
3293
|
for (const batchResult of outcome.results) {
|
|
3104
3294
|
const mutableIds = new Set(batchResult.batch.map((f) => f.id));
|
|
3105
3295
|
for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
|
|
@@ -3107,6 +3297,7 @@ var MaintenanceService = class {
|
|
|
3107
3297
|
batchResult.newFacts.forEach((raw, itemIndex) => {
|
|
3108
3298
|
newFacts.push(raw);
|
|
3109
3299
|
newFactItemIndexes.push(itemIndex);
|
|
3300
|
+
newFactCorpora.push(batchResult.corpus);
|
|
3110
3301
|
});
|
|
3111
3302
|
}
|
|
3112
3303
|
const safeDowngraded = Array.from(safeDowngradedSet);
|
|
@@ -3114,12 +3305,14 @@ var MaintenanceService = class {
|
|
|
3114
3305
|
const diagBuffer = new DiagnosticBuffer();
|
|
3115
3306
|
const validNewFacts = [];
|
|
3116
3307
|
const validNewFactItemIndexes = [];
|
|
3308
|
+
const validNewFactCorpora = [];
|
|
3117
3309
|
newFacts.forEach((raw, k) => {
|
|
3118
3310
|
const itemIndex = newFactItemIndexes[k];
|
|
3119
3311
|
const valid = validateFact(raw);
|
|
3120
3312
|
if (valid) {
|
|
3121
3313
|
validNewFacts.push(valid);
|
|
3122
3314
|
validNewFactItemIndexes.push(itemIndex);
|
|
3315
|
+
validNewFactCorpora.push(newFactCorpora[k]);
|
|
3123
3316
|
} else {
|
|
3124
3317
|
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3125
3318
|
}
|
|
@@ -3149,6 +3342,10 @@ var MaintenanceService = class {
|
|
|
3149
3342
|
continue;
|
|
3150
3343
|
}
|
|
3151
3344
|
const id = generateId("fact_");
|
|
3345
|
+
const grounding = healGrounding ? groundingOutcome(checkGrounding(fact.evidence, validNewFactCorpora[k] ?? [], healGrounding), now) : null;
|
|
3346
|
+
if (grounding?.diagnostic) {
|
|
3347
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validNewFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3348
|
+
}
|
|
3152
3349
|
const factObj = {
|
|
3153
3350
|
id,
|
|
3154
3351
|
entity_id: entityId,
|
|
@@ -3163,7 +3360,8 @@ var MaintenanceService = class {
|
|
|
3163
3360
|
updated_at: now,
|
|
3164
3361
|
last_accessed_at: null,
|
|
3165
3362
|
access_count: 0,
|
|
3166
|
-
deleted_at: null
|
|
3363
|
+
deleted_at: null,
|
|
3364
|
+
...grounding?.trust
|
|
3167
3365
|
};
|
|
3168
3366
|
await this.entryRepo.upsert(factObj, tx);
|
|
3169
3367
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3527,7 +3725,7 @@ var MaintenanceService = class {
|
|
|
3527
3725
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
3528
3726
|
* per-pass — see the call site in doRunHeal.
|
|
3529
3727
|
*/
|
|
3530
|
-
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
3728
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache, withBodies = false) {
|
|
3531
3729
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
3532
3730
|
if (!query) return [];
|
|
3533
3731
|
const cached = cache?.get(query);
|
|
@@ -3540,7 +3738,7 @@ var MaintenanceService = class {
|
|
|
3540
3738
|
const hitIds = hits.map((h) => h.id);
|
|
3541
3739
|
const anchors = [];
|
|
3542
3740
|
if (hitIds.length > 0) {
|
|
3543
|
-
const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
3741
|
+
const rows = withBodies ? await this.entryRepo.findAnchorRowsByIds(entityId, hitIds, void 0, { withBody: true }) : await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
3544
3742
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
3545
3743
|
for (const id of hitIds) {
|
|
3546
3744
|
const row = byId.get(id);
|