@equationalapplications/core-llm-wiki 7.4.0 → 7.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -0
- package/dist/{chunk-G5OR2VWD.mjs → chunk-YBWE26XR.mjs} +363 -48
- package/dist/chunk-YBWE26XR.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +364 -48
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -5
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-Dmh1kfkd.d.mts → testing-d1SrpDvg.d.mts} +117 -3
- package/dist/{testing-Dmh1kfkd.d.ts → testing-d1SrpDvg.d.ts} +117 -3
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +360 -45
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-G5OR2VWD.mjs.map +0 -1
|
@@ -1513,6 +1513,19 @@ function validateTags(tags) {
|
|
|
1513
1513
|
if (!Array.isArray(tags)) return [];
|
|
1514
1514
|
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
1515
1515
|
}
|
|
1516
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
1517
|
+
function normalizeEvidence(raw) {
|
|
1518
|
+
if (!Array.isArray(raw)) return void 0;
|
|
1519
|
+
const out = [];
|
|
1520
|
+
for (const entry of raw) {
|
|
1521
|
+
if (typeof entry !== "string") continue;
|
|
1522
|
+
const trimmed = entry.trim();
|
|
1523
|
+
if (!trimmed) continue;
|
|
1524
|
+
out.push(trimmed);
|
|
1525
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
1526
|
+
}
|
|
1527
|
+
return out;
|
|
1528
|
+
}
|
|
1516
1529
|
function validateFact(fact) {
|
|
1517
1530
|
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
1518
1531
|
const title = clip(fact.title, 80);
|
|
@@ -1520,13 +1533,17 @@ function validateFact(fact) {
|
|
|
1520
1533
|
if (!title || !body) return null;
|
|
1521
1534
|
let confidence = fact.confidence;
|
|
1522
1535
|
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
1523
|
-
|
|
1536
|
+
const valid = {
|
|
1524
1537
|
...fact,
|
|
1525
1538
|
title,
|
|
1526
1539
|
body,
|
|
1527
1540
|
confidence,
|
|
1528
1541
|
tags: validateTags(fact.tags)
|
|
1529
1542
|
};
|
|
1543
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
1544
|
+
if (evidence) valid.evidence = evidence;
|
|
1545
|
+
else delete valid.evidence;
|
|
1546
|
+
return valid;
|
|
1530
1547
|
}
|
|
1531
1548
|
function validateTask(task) {
|
|
1532
1549
|
if (typeof task?.description !== "string") return null;
|
|
@@ -1602,17 +1619,94 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
1602
1619
|
}
|
|
1603
1620
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
1604
1621
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
1622
|
+
var GROUNDING_SOURCE = {
|
|
1623
|
+
ingest: { key: "facts", source: "the document chunk" },
|
|
1624
|
+
librarian: { key: "facts", source: 'the "summary" text of the events' },
|
|
1625
|
+
heal: { key: "newFacts", source: 'the "summary" text of the recent events or the "body" text of the document anchors' }
|
|
1626
|
+
};
|
|
1627
|
+
function groundingEvidenceBlock(writer, cfg) {
|
|
1628
|
+
const { key, source } = GROUNDING_SOURCE[writer];
|
|
1629
|
+
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1630
|
+
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1631
|
+
}
|
|
1605
1632
|
|
|
1606
1633
|
// src/utils/healConstants.ts
|
|
1607
1634
|
var HEAL_MAX_ANCHORS = 50;
|
|
1608
1635
|
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
1609
1636
|
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
1610
1637
|
var HEAL_MAX_TASKS = 50;
|
|
1638
|
+
var HEAL_ANCHOR_BODY_CHARS = 800;
|
|
1639
|
+
|
|
1640
|
+
// src/utils/grounding.ts
|
|
1641
|
+
var GROUNDING_VERIFIER = "process:grounding-check";
|
|
1642
|
+
var WRITERS = ["ingest", "librarian", "heal"];
|
|
1643
|
+
function positiveInt(value, fallback) {
|
|
1644
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
1645
|
+
}
|
|
1646
|
+
function resolveGrounding(config) {
|
|
1647
|
+
if (!config || config.mode !== "draft") return null;
|
|
1648
|
+
const writers = Array.isArray(config.writers) ? config.writers.filter((w) => WRITERS.includes(w)) : ["ingest"];
|
|
1649
|
+
return {
|
|
1650
|
+
writers: new Set(writers),
|
|
1651
|
+
minEvidenceChars: positiveInt(config.minEvidenceChars, 20),
|
|
1652
|
+
// Never ask for more quotes than checkGrounding accepts (MAX_EVIDENCE_QUOTES).
|
|
1653
|
+
maxEvidence: Math.min(MAX_EVIDENCE_QUOTES, positiveInt(config.maxEvidence, 3)),
|
|
1654
|
+
maxEvidenceChars: positiveInt(config.maxEvidenceChars, 300)
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
function normalizeForGrounding(text) {
|
|
1658
|
+
return text.normalize("NFKC").replace(/\s+/g, " ").trim();
|
|
1659
|
+
}
|
|
1660
|
+
function buildGroundingCorpus(parts) {
|
|
1661
|
+
return parts.filter((p) => typeof p === "string").map(normalizeForGrounding);
|
|
1662
|
+
}
|
|
1663
|
+
function checkGrounding(evidence, normalizedCorpus, cfg) {
|
|
1664
|
+
const quotes = evidence ?? [];
|
|
1665
|
+
if (quotes.length > MAX_EVIDENCE_QUOTES) return { status: "failed", reason: "too_many_quotes" };
|
|
1666
|
+
if (quotes.length === 0) return { status: "missing", reason: "no_evidence" };
|
|
1667
|
+
const qualifying = quotes.map(normalizeForGrounding).filter((q) => q.length >= cfg.minEvidenceChars);
|
|
1668
|
+
if (qualifying.length === 0) return { status: "missing", reason: "evidence_too_short" };
|
|
1669
|
+
if (qualifying.some((q) => !normalizedCorpus.some((part) => part.includes(q)))) {
|
|
1670
|
+
return { status: "failed", reason: "quote_not_found" };
|
|
1671
|
+
}
|
|
1672
|
+
return {
|
|
1673
|
+
status: "grounded",
|
|
1674
|
+
retained: qualifying.slice(0, cfg.maxEvidence).map((q) => safeSlice(q, 0, cfg.maxEvidenceChars))
|
|
1675
|
+
};
|
|
1676
|
+
}
|
|
1677
|
+
function groundingOutcome(verdict, now) {
|
|
1678
|
+
if (verdict.status === "grounded") {
|
|
1679
|
+
return {
|
|
1680
|
+
trust: {
|
|
1681
|
+
lifecycle_status: "stable",
|
|
1682
|
+
okf_verified: [{ by: GROUNDING_VERIFIER, at: new Date(now).toISOString() }],
|
|
1683
|
+
last_verified_at: now,
|
|
1684
|
+
last_verified_by: GROUNDING_VERIFIER
|
|
1685
|
+
}
|
|
1686
|
+
};
|
|
1687
|
+
}
|
|
1688
|
+
return {
|
|
1689
|
+
trust: { lifecycle_status: "draft" },
|
|
1690
|
+
diagnostic: { code: verdict.status === "missing" ? "grounding_missing" : "grounding_failed", reason: verdict.reason }
|
|
1691
|
+
};
|
|
1692
|
+
}
|
|
1611
1693
|
|
|
1612
1694
|
// src/services/PromptService.ts
|
|
1613
1695
|
var PromptService = class {
|
|
1614
|
-
constructor(globalOverrides) {
|
|
1696
|
+
constructor(globalOverrides, grounding = null) {
|
|
1615
1697
|
this.globalOverrides = globalOverrides;
|
|
1698
|
+
this.grounding = grounding;
|
|
1699
|
+
}
|
|
1700
|
+
/** The resolved grounding config when `writer` is in `grounding.writers`; otherwise null (spec §6.2). */
|
|
1701
|
+
groundingFor(writer) {
|
|
1702
|
+
return this.grounding?.writers.has(writer) ? this.grounding : null;
|
|
1703
|
+
}
|
|
1704
|
+
/** Appended after any override and after ontology context, so it is always last. */
|
|
1705
|
+
appendGrounding(systemPrompt, writer) {
|
|
1706
|
+
const cfg = this.groundingFor(writer);
|
|
1707
|
+
return cfg ? `${systemPrompt}
|
|
1708
|
+
|
|
1709
|
+
${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
1616
1710
|
}
|
|
1617
1711
|
hydrate(template, variables) {
|
|
1618
1712
|
return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
|
|
@@ -1642,13 +1736,13 @@ ${ctx.ontologyModeInstructions}`;
|
|
|
1642
1736
|
const hasDocumentChunk = /\{\{\s*documentChunk\s*\}\}/.test(template);
|
|
1643
1737
|
if (hasDocumentChunk || this.hasOntologyPlaceholders(template)) {
|
|
1644
1738
|
return {
|
|
1645
|
-
systemPrompt: this.buildSystemPrompt(template, { documentChunk }, ontologyContext),
|
|
1739
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { documentChunk }, ontologyContext), "ingest"),
|
|
1646
1740
|
userPrompt: hasDocumentChunk ? "Please extract the facts." : `Document Chunk:
|
|
1647
1741
|
${documentChunk}`
|
|
1648
1742
|
};
|
|
1649
1743
|
}
|
|
1650
1744
|
return {
|
|
1651
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1745
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "ingest"),
|
|
1652
1746
|
userPrompt: `Document Chunk:
|
|
1653
1747
|
${documentChunk}`
|
|
1654
1748
|
};
|
|
@@ -1657,23 +1751,27 @@ ${documentChunk}`
|
|
|
1657
1751
|
const template = runtimeOverride ?? this.globalOverrides?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT;
|
|
1658
1752
|
const hasEvents = /\{\{\s*events\s*\}\}/.test(template);
|
|
1659
1753
|
const hasCurrentFacts = /\{\{\s*currentFacts\s*\}\}/.test(template);
|
|
1754
|
+
const eventsShown = hasEvents || !hasCurrentFacts;
|
|
1755
|
+
const corpusField = this.groundingFor("librarian") ? { groundingCorpus: buildGroundingCorpus(eventsShown ? events.map(summaryOf) : []) } : {};
|
|
1660
1756
|
if (hasEvents || hasCurrentFacts || this.hasOntologyPlaceholders(template)) {
|
|
1661
1757
|
return {
|
|
1662
|
-
systemPrompt: this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext),
|
|
1758
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext), "librarian"),
|
|
1663
1759
|
userPrompt: hasEvents || hasCurrentFacts ? "Please synthesize the context." : `Events:
|
|
1664
1760
|
${JSON.stringify(events, null, 2)}
|
|
1665
1761
|
|
|
1666
1762
|
Current Facts:
|
|
1667
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1763
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1764
|
+
...corpusField
|
|
1668
1765
|
};
|
|
1669
1766
|
}
|
|
1670
1767
|
return {
|
|
1671
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
1768
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "librarian"),
|
|
1672
1769
|
userPrompt: `Events:
|
|
1673
1770
|
${JSON.stringify(events, null, 2)}
|
|
1674
1771
|
|
|
1675
1772
|
Current Facts:
|
|
1676
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
1773
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
1774
|
+
...corpusField
|
|
1677
1775
|
};
|
|
1678
1776
|
}
|
|
1679
1777
|
/**
|
|
@@ -1703,40 +1801,54 @@ ${JSON.stringify(currentFacts, null, 2)}`
|
|
|
1703
1801
|
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
1704
1802
|
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
1705
1803
|
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
1804
|
+
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
1805
|
+
const hasRecentEvents = /\{\{\s*recentEvents\s*\}\}/.test(template);
|
|
1806
|
+
const hasDocumentAnchors = /\{\{\s*documentAnchors\s*\}\}/.test(template);
|
|
1807
|
+
const usesPlaceholders = /\{\{\s*healCandidates\s*\}\}/.test(template) || hasDocumentAnchors || /\{\{\s*allTasks\s*\}\}/.test(template) || hasRecentEvents;
|
|
1808
|
+
const healGrounding = this.groundingFor("heal");
|
|
1809
|
+
const promptAnchors = healGrounding ? effectiveAnchors.map(toGroundingAnchor) : effectiveAnchors;
|
|
1810
|
+
const eventsShown = !usesPlaceholders || hasRecentEvents;
|
|
1811
|
+
const anchorsShown = !usesPlaceholders || hasDocumentAnchors;
|
|
1812
|
+
const groundingCorpus = healGrounding ? buildGroundingCorpus([
|
|
1813
|
+
...eventsShown ? effectiveEvents.map(summaryOf) : [],
|
|
1814
|
+
...anchorsShown ? effectiveAnchors.map((a, i) => ({ status: a?.lifecycle_status, shown: promptAnchors[i] })).filter((x) => x.status !== "draft").map((x) => x.shown?.body) : []
|
|
1815
|
+
]) : null;
|
|
1706
1816
|
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
1707
1817
|
healCandidates,
|
|
1708
1818
|
attemptLevel,
|
|
1709
1819
|
bodyTruncationChars
|
|
1710
1820
|
);
|
|
1711
|
-
const
|
|
1712
|
-
if (
|
|
1821
|
+
const corpusField = groundingCorpus !== null ? { groundingCorpus } : {};
|
|
1822
|
+
if (usesPlaceholders) {
|
|
1713
1823
|
return {
|
|
1714
1824
|
prompts: {
|
|
1715
|
-
systemPrompt: this.hydrate(template, {
|
|
1825
|
+
systemPrompt: this.appendGrounding(this.hydrate(template, {
|
|
1716
1826
|
healCandidates: shapedCandidates,
|
|
1717
|
-
documentAnchors:
|
|
1827
|
+
documentAnchors: promptAnchors,
|
|
1718
1828
|
allTasks: effectiveTasks,
|
|
1719
1829
|
recentEvents: effectiveEvents
|
|
1720
|
-
}),
|
|
1830
|
+
}), "heal"),
|
|
1721
1831
|
userPrompt: "Please heal the memory graph."
|
|
1722
1832
|
},
|
|
1723
|
-
degraded
|
|
1833
|
+
degraded,
|
|
1834
|
+
...corpusField
|
|
1724
1835
|
};
|
|
1725
1836
|
}
|
|
1726
1837
|
return {
|
|
1727
1838
|
prompts: {
|
|
1728
|
-
systemPrompt: template,
|
|
1839
|
+
systemPrompt: this.appendGrounding(template, "heal"),
|
|
1729
1840
|
userPrompt: `Heal Candidates:
|
|
1730
1841
|
${JSON.stringify(shapedCandidates, null, 2)}
|
|
1731
1842
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
1732
|
-
${JSON.stringify(
|
|
1843
|
+
${JSON.stringify(promptAnchors, null, 2)}
|
|
1733
1844
|
All Tasks:
|
|
1734
1845
|
${JSON.stringify(effectiveTasks, null, 2)}
|
|
1735
1846
|
Recent Events:
|
|
1736
1847
|
${JSON.stringify(effectiveEvents, null, 2)}
|
|
1737
1848
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
1738
1849
|
},
|
|
1739
|
-
degraded
|
|
1850
|
+
degraded,
|
|
1851
|
+
...corpusField
|
|
1740
1852
|
};
|
|
1741
1853
|
}
|
|
1742
1854
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -1786,6 +1898,19 @@ function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
|
1786
1898
|
}
|
|
1787
1899
|
return { shapedCandidates, degraded };
|
|
1788
1900
|
}
|
|
1901
|
+
function summaryOf(event) {
|
|
1902
|
+
return event?.summary;
|
|
1903
|
+
}
|
|
1904
|
+
function toGroundingAnchor(anchor) {
|
|
1905
|
+
if (typeof anchor !== "object" || anchor === null) return anchor;
|
|
1906
|
+
const a = anchor;
|
|
1907
|
+
return {
|
|
1908
|
+
id: a.id,
|
|
1909
|
+
title: a.title,
|
|
1910
|
+
source_ref: a.source_ref,
|
|
1911
|
+
body: typeof a.body === "string" ? safeSlice(a.body, 0, HEAL_ANCHOR_BODY_CHARS) : ""
|
|
1912
|
+
};
|
|
1913
|
+
}
|
|
1789
1914
|
|
|
1790
1915
|
// src/utils/chunkingDefaults.ts
|
|
1791
1916
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -1913,7 +2038,7 @@ var IngestionService = class {
|
|
|
1913
2038
|
this.jobManager = jobManager;
|
|
1914
2039
|
this.embeddingService = embeddingService;
|
|
1915
2040
|
this.ontologyService = ontologyService;
|
|
1916
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
2041
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
1917
2042
|
}
|
|
1918
2043
|
async ingestDocument(entityId, params, opts) {
|
|
1919
2044
|
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
@@ -1946,6 +2071,7 @@ var IngestionService = class {
|
|
|
1946
2071
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1947
2072
|
if (chunks.length === 0) return zeroChunkResult();
|
|
1948
2073
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
2074
|
+
const ingestGrounding = this.promptService.groundingFor("ingest");
|
|
1949
2075
|
const chunkResults = await withConcurrency(
|
|
1950
2076
|
chunks.map((chunk, chunkIndex) => async () => {
|
|
1951
2077
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
@@ -1969,11 +2095,16 @@ var IngestionService = class {
|
|
|
1969
2095
|
rejected.push({ itemIndex, reason: factRejectionReason(raw) });
|
|
1970
2096
|
}
|
|
1971
2097
|
});
|
|
2098
|
+
const verdicts = ingestGrounding ? (() => {
|
|
2099
|
+
const corpus = buildGroundingCorpus([chunk]);
|
|
2100
|
+
return facts.map((f) => checkGrounding(f.evidence, corpus, ingestGrounding));
|
|
2101
|
+
})() : [];
|
|
1972
2102
|
return {
|
|
1973
2103
|
status: "ok",
|
|
1974
2104
|
facts,
|
|
1975
2105
|
itemIndexes,
|
|
1976
2106
|
rejected,
|
|
2107
|
+
verdicts,
|
|
1977
2108
|
ontology_updates: result2.ontology_updates
|
|
1978
2109
|
};
|
|
1979
2110
|
} catch (e) {
|
|
@@ -2018,6 +2149,7 @@ var IngestionService = class {
|
|
|
2018
2149
|
const failures = [];
|
|
2019
2150
|
const seen = /* @__PURE__ */ new Set();
|
|
2020
2151
|
const orderedChunkFacts = [];
|
|
2152
|
+
const groundingLedger = /* @__PURE__ */ new Map();
|
|
2021
2153
|
const diagBuffer = new DiagnosticBuffer();
|
|
2022
2154
|
const diagBase = { entityId, operation: "ingest", trigger: "call" };
|
|
2023
2155
|
for (const [chunkIndex, slot] of chunkResults.entries()) {
|
|
@@ -2036,6 +2168,7 @@ var IngestionService = class {
|
|
|
2036
2168
|
if (!seen.has(normalizedTitle)) {
|
|
2037
2169
|
seen.add(normalizedTitle);
|
|
2038
2170
|
dedupedFacts.push(fact);
|
|
2171
|
+
if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: slot.itemIndexes[k] });
|
|
2039
2172
|
} else {
|
|
2040
2173
|
diagBuffer.push({
|
|
2041
2174
|
...diagBase,
|
|
@@ -2072,7 +2205,7 @@ var IngestionService = class {
|
|
|
2072
2205
|
try {
|
|
2073
2206
|
if (failedChunks === 0) {
|
|
2074
2207
|
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
2075
|
-
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer);
|
|
2208
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger);
|
|
2076
2209
|
});
|
|
2077
2210
|
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
2078
2211
|
insertedFacts.push(...fullResult.insertedFacts);
|
|
@@ -2080,7 +2213,7 @@ var IngestionService = class {
|
|
|
2080
2213
|
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
2081
2214
|
const flat = [];
|
|
2082
2215
|
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
2083
|
-
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer);
|
|
2216
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, groundingLedger);
|
|
2084
2217
|
});
|
|
2085
2218
|
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
2086
2219
|
}
|
|
@@ -2222,7 +2355,8 @@ var IngestionService = class {
|
|
|
2222
2355
|
last_accessed_at: null,
|
|
2223
2356
|
access_count: 0,
|
|
2224
2357
|
deleted_at: null,
|
|
2225
|
-
okf_type: normalized.okf_type
|
|
2358
|
+
okf_type: normalized.okf_type,
|
|
2359
|
+
...opts?.nodeTrust?.get(node.id)
|
|
2226
2360
|
};
|
|
2227
2361
|
wikiFacts.push(wikiFact);
|
|
2228
2362
|
}
|
|
@@ -2313,9 +2447,10 @@ var IngestionService = class {
|
|
|
2313
2447
|
* On the partial path the caller never invokes this method; the empty
|
|
2314
2448
|
* arrays stay empty.
|
|
2315
2449
|
*/
|
|
2316
|
-
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer) {
|
|
2450
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2317
2451
|
const deletedSourceFactIds = [];
|
|
2318
2452
|
const insertedFacts = [];
|
|
2453
|
+
const nodeTrust = /* @__PURE__ */ new Map();
|
|
2319
2454
|
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
2320
2455
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
2321
2456
|
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
@@ -2359,6 +2494,20 @@ var IngestionService = class {
|
|
|
2359
2494
|
tags: fact.tags,
|
|
2360
2495
|
confidence: fact.confidence
|
|
2361
2496
|
});
|
|
2497
|
+
const grounding = groundingLedger.get(fact);
|
|
2498
|
+
if (grounding) {
|
|
2499
|
+
const outcome = groundingOutcome(grounding.verdict, now);
|
|
2500
|
+
nodeTrust.set(id, outcome.trust);
|
|
2501
|
+
if (outcome.diagnostic) {
|
|
2502
|
+
diagBuffer.push({
|
|
2503
|
+
entityId,
|
|
2504
|
+
operation: "ingest",
|
|
2505
|
+
trigger: "call",
|
|
2506
|
+
code: outcome.diagnostic.code,
|
|
2507
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2508
|
+
});
|
|
2509
|
+
}
|
|
2510
|
+
}
|
|
2362
2511
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2363
2512
|
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
2364
2513
|
if (normalized.edges.length > 0) {
|
|
@@ -2390,7 +2539,7 @@ var IngestionService = class {
|
|
|
2390
2539
|
entityId,
|
|
2391
2540
|
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
2392
2541
|
tx,
|
|
2393
|
-
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" } }
|
|
2542
|
+
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" }, ...nodeTrust.size > 0 ? { nodeTrust } : {} }
|
|
2394
2543
|
);
|
|
2395
2544
|
return { deletedSourceFactIds, insertedFacts };
|
|
2396
2545
|
}
|
|
@@ -2414,7 +2563,7 @@ var IngestionService = class {
|
|
|
2414
2563
|
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
2415
2564
|
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
2416
2565
|
*/
|
|
2417
|
-
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer) {
|
|
2566
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
2418
2567
|
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
2419
2568
|
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
2420
2569
|
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
@@ -2437,6 +2586,17 @@ var IngestionService = class {
|
|
|
2437
2586
|
}
|
|
2438
2587
|
liveTitles.add(normalizedTitle);
|
|
2439
2588
|
const id = generateId("fact_");
|
|
2589
|
+
const grounding = groundingLedger.get(fact);
|
|
2590
|
+
const outcome = grounding ? groundingOutcome(grounding.verdict, now) : null;
|
|
2591
|
+
if (grounding && outcome?.diagnostic) {
|
|
2592
|
+
diagBuffer.push({
|
|
2593
|
+
entityId,
|
|
2594
|
+
operation: "ingest",
|
|
2595
|
+
trigger: "call",
|
|
2596
|
+
code: outcome.diagnostic.code,
|
|
2597
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
2598
|
+
});
|
|
2599
|
+
}
|
|
2440
2600
|
const wikiFact = {
|
|
2441
2601
|
id,
|
|
2442
2602
|
entity_id: entityId,
|
|
@@ -2455,7 +2615,8 @@ var IngestionService = class {
|
|
|
2455
2615
|
last_accessed_at: null,
|
|
2456
2616
|
access_count: 0,
|
|
2457
2617
|
deleted_at: null,
|
|
2458
|
-
okf_type: null
|
|
2618
|
+
okf_type: null,
|
|
2619
|
+
...outcome?.trust
|
|
2459
2620
|
};
|
|
2460
2621
|
await this.entryRepo.upsert(wikiFact, tx);
|
|
2461
2622
|
insertedDescriptors.push({
|
|
@@ -2471,6 +2632,51 @@ var IngestionService = class {
|
|
|
2471
2632
|
}
|
|
2472
2633
|
};
|
|
2473
2634
|
|
|
2635
|
+
// src/utils/classifier.ts
|
|
2636
|
+
var isUnit = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0 && v <= 1;
|
|
2637
|
+
function validateClassifierAnswer(response, key, question) {
|
|
2638
|
+
try {
|
|
2639
|
+
if (response === null || typeof response !== "object") return { ok: false, reason: "malformed" };
|
|
2640
|
+
const answers = response.answers;
|
|
2641
|
+
if (answers === null || typeof answers !== "object") return { ok: false, reason: "missing_answer" };
|
|
2642
|
+
if (!Object.prototype.hasOwnProperty.call(answers, key)) return { ok: false, reason: "missing_answer" };
|
|
2643
|
+
const raw = answers[key];
|
|
2644
|
+
if (raw === null || typeof raw !== "object") return { ok: false, reason: "missing_answer" };
|
|
2645
|
+
const a = raw;
|
|
2646
|
+
if (a.kind !== question.kind) return { ok: false, reason: "kind_mismatch" };
|
|
2647
|
+
if (question.kind === "choice") {
|
|
2648
|
+
if (typeof a.choice !== "string" || !question.options.includes(a.choice)) return { ok: false, reason: "choice_not_offered" };
|
|
2649
|
+
if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
|
|
2650
|
+
const probs = a.probabilities;
|
|
2651
|
+
if (probs === null || typeof probs !== "object" || Array.isArray(probs)) return { ok: false, reason: "invalid_probability" };
|
|
2652
|
+
const probabilities = {};
|
|
2653
|
+
for (const [k, v] of Object.entries(probs)) {
|
|
2654
|
+
if (!isUnit(v)) return { ok: false, reason: "invalid_probability" };
|
|
2655
|
+
probabilities[k] = v;
|
|
2656
|
+
}
|
|
2657
|
+
return { ok: true, answer: { kind: "choice", choice: a.choice, confidence: a.confidence, probabilities } };
|
|
2658
|
+
}
|
|
2659
|
+
if (question.kind === "binary") {
|
|
2660
|
+
if (!isUnit(a.probability)) return { ok: false, reason: "invalid_probability" };
|
|
2661
|
+
return { ok: true, answer: { kind: "binary", probability: a.probability } };
|
|
2662
|
+
}
|
|
2663
|
+
const maxScore = question.levels.length - 1;
|
|
2664
|
+
if (typeof a.score !== "number" || !Number.isFinite(a.score) || a.score < 0 || a.score > maxScore) {
|
|
2665
|
+
return { ok: false, reason: "score_out_of_range" };
|
|
2666
|
+
}
|
|
2667
|
+
if (!isUnit(a.confidence)) return { ok: false, reason: "invalid_probability" };
|
|
2668
|
+
if (!Array.isArray(a.probabilities) || !a.probabilities.every(isUnit)) return { ok: false, reason: "invalid_probability" };
|
|
2669
|
+
return { ok: true, answer: { kind: "score", score: a.score, confidence: a.confidence, probabilities: a.probabilities.slice() } };
|
|
2670
|
+
} catch {
|
|
2671
|
+
return { ok: false, reason: "malformed" };
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
function classifierStateForFact(fact) {
|
|
2675
|
+
const parts = [fact.title, fact.body];
|
|
2676
|
+
if (Array.isArray(fact.tags) && fact.tags.length > 0) parts.push(`Tags: ${fact.tags.join(", ")}`);
|
|
2677
|
+
return parts.join("\n\n");
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2474
2680
|
// src/repositories/BaseRepository.ts
|
|
2475
2681
|
var BaseRepository = class {
|
|
2476
2682
|
constructor(db, prefix) {
|
|
@@ -2764,7 +2970,7 @@ async function runBatched(args) {
|
|
|
2764
2970
|
}
|
|
2765
2971
|
let result;
|
|
2766
2972
|
try {
|
|
2767
|
-
result = parse(responseText, batch);
|
|
2973
|
+
result = parse(responseText, batch, prompts);
|
|
2768
2974
|
} catch (err) {
|
|
2769
2975
|
await onFailure(batch, err, attemptLevel, false);
|
|
2770
2976
|
return;
|
|
@@ -2845,7 +3051,7 @@ var MaintenanceService = class {
|
|
|
2845
3051
|
this.jobManager = jobManager;
|
|
2846
3052
|
this.embeddingService = embeddingService;
|
|
2847
3053
|
this.ontologyService = ontologyService;
|
|
2848
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
3054
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
2849
3055
|
}
|
|
2850
3056
|
async runPrune(entityId, options) {
|
|
2851
3057
|
this.jobManager.acquireLock("prune", entityId);
|
|
@@ -3150,8 +3356,10 @@ var MaintenanceService = class {
|
|
|
3150
3356
|
};
|
|
3151
3357
|
});
|
|
3152
3358
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3153
|
-
const
|
|
3154
|
-
|
|
3359
|
+
const promptEvents = events.reverse();
|
|
3360
|
+
const librarianGrounding = this.promptService.groundingFor("librarian");
|
|
3361
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = this.promptService.buildLibrarianPrompt(
|
|
3362
|
+
promptEvents,
|
|
3155
3363
|
currentFacts,
|
|
3156
3364
|
promptOverride,
|
|
3157
3365
|
ontologyContext
|
|
@@ -3220,6 +3428,10 @@ var MaintenanceService = class {
|
|
|
3220
3428
|
const validationDrops = [];
|
|
3221
3429
|
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
3222
3430
|
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3431
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3432
|
+
if (grounding?.diagnostic) {
|
|
3433
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3434
|
+
}
|
|
3223
3435
|
const factObj = {
|
|
3224
3436
|
id,
|
|
3225
3437
|
entity_id: entityId,
|
|
@@ -3235,7 +3447,8 @@ var MaintenanceService = class {
|
|
|
3235
3447
|
last_accessed_at: null,
|
|
3236
3448
|
access_count: 0,
|
|
3237
3449
|
deleted_at: null,
|
|
3238
|
-
okf_type: normalized.okf_type
|
|
3450
|
+
okf_type: normalized.okf_type,
|
|
3451
|
+
...grounding?.trust
|
|
3239
3452
|
};
|
|
3240
3453
|
await this.entryRepo.upsert(factObj, tx);
|
|
3241
3454
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3351,11 +3564,14 @@ var MaintenanceService = class {
|
|
|
3351
3564
|
}
|
|
3352
3565
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
3353
3566
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
3567
|
+
const healGrounding = this.promptService.groundingFor("heal");
|
|
3354
3568
|
const toPromptShape = (f) => {
|
|
3355
3569
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
3356
|
-
|
|
3570
|
+
const shown = healGrounding ? (({ lifecycle_status: _l, okf_verified: _v, last_verified_at: _a, last_verified_by: _b, ...r }) => r)(rest) : rest;
|
|
3571
|
+
return { ...shown, tags: typeof shown.tags === "string" ? JSON.parse(shown.tags) : shown.tags };
|
|
3357
3572
|
};
|
|
3358
3573
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
3574
|
+
const corpusByPrompt = /* @__PURE__ */ new WeakMap();
|
|
3359
3575
|
const degraded = [];
|
|
3360
3576
|
const outcome = await runBatched({
|
|
3361
3577
|
items: healCandidates,
|
|
@@ -3368,9 +3584,10 @@ var MaintenanceService = class {
|
|
|
3368
3584
|
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
3369
3585
|
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
3370
3586
|
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
3371
|
-
anchorCache
|
|
3587
|
+
anchorCache,
|
|
3588
|
+
healGrounding !== null
|
|
3372
3589
|
);
|
|
3373
|
-
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
3590
|
+
const { prompts, degraded: batchDegraded, groundingCorpus } = await this.promptService.buildHealPrompt(
|
|
3374
3591
|
batch.map(toPromptShape),
|
|
3375
3592
|
documentAnchors,
|
|
3376
3593
|
allTasks,
|
|
@@ -3380,16 +3597,18 @@ var MaintenanceService = class {
|
|
|
3380
3597
|
bodyTruncationChars
|
|
3381
3598
|
);
|
|
3382
3599
|
degraded.push(...batchDegraded);
|
|
3600
|
+
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3383
3601
|
return prompts;
|
|
3384
3602
|
},
|
|
3385
3603
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
3386
|
-
parse: (responseText, batch) => {
|
|
3604
|
+
parse: (responseText, batch, prompts) => {
|
|
3387
3605
|
const result = parseJsonResponse(responseText);
|
|
3388
3606
|
return {
|
|
3389
3607
|
batch,
|
|
3390
3608
|
downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
|
|
3391
3609
|
deleted: Array.isArray(result.deleted) ? result.deleted : [],
|
|
3392
|
-
newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
|
|
3610
|
+
newFacts: Array.isArray(result.newFacts) ? result.newFacts : [],
|
|
3611
|
+
corpus: corpusByPrompt.get(prompts) ?? null
|
|
3393
3612
|
};
|
|
3394
3613
|
},
|
|
3395
3614
|
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
@@ -3404,6 +3623,7 @@ var MaintenanceService = class {
|
|
|
3404
3623
|
const safeDeletedSet = /* @__PURE__ */ new Set();
|
|
3405
3624
|
const newFacts = [];
|
|
3406
3625
|
const newFactItemIndexes = [];
|
|
3626
|
+
const newFactCorpora = [];
|
|
3407
3627
|
for (const batchResult of outcome.results) {
|
|
3408
3628
|
const mutableIds = new Set(batchResult.batch.map((f) => f.id));
|
|
3409
3629
|
for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
|
|
@@ -3411,6 +3631,7 @@ var MaintenanceService = class {
|
|
|
3411
3631
|
batchResult.newFacts.forEach((raw, itemIndex) => {
|
|
3412
3632
|
newFacts.push(raw);
|
|
3413
3633
|
newFactItemIndexes.push(itemIndex);
|
|
3634
|
+
newFactCorpora.push(batchResult.corpus);
|
|
3414
3635
|
});
|
|
3415
3636
|
}
|
|
3416
3637
|
const safeDowngraded = Array.from(safeDowngradedSet);
|
|
@@ -3418,12 +3639,14 @@ var MaintenanceService = class {
|
|
|
3418
3639
|
const diagBuffer = new DiagnosticBuffer();
|
|
3419
3640
|
const validNewFacts = [];
|
|
3420
3641
|
const validNewFactItemIndexes = [];
|
|
3642
|
+
const validNewFactCorpora = [];
|
|
3421
3643
|
newFacts.forEach((raw, k) => {
|
|
3422
3644
|
const itemIndex = newFactItemIndexes[k];
|
|
3423
3645
|
const valid = validateFact(raw);
|
|
3424
3646
|
if (valid) {
|
|
3425
3647
|
validNewFacts.push(valid);
|
|
3426
3648
|
validNewFactItemIndexes.push(itemIndex);
|
|
3649
|
+
validNewFactCorpora.push(newFactCorpora[k]);
|
|
3427
3650
|
} else {
|
|
3428
3651
|
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3429
3652
|
}
|
|
@@ -3453,6 +3676,10 @@ var MaintenanceService = class {
|
|
|
3453
3676
|
continue;
|
|
3454
3677
|
}
|
|
3455
3678
|
const id = generateId("fact_");
|
|
3679
|
+
const grounding = healGrounding ? groundingOutcome(checkGrounding(fact.evidence, validNewFactCorpora[k] ?? [], healGrounding), now) : null;
|
|
3680
|
+
if (grounding?.diagnostic) {
|
|
3681
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validNewFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3682
|
+
}
|
|
3456
3683
|
const factObj = {
|
|
3457
3684
|
id,
|
|
3458
3685
|
entity_id: entityId,
|
|
@@ -3467,7 +3694,8 @@ var MaintenanceService = class {
|
|
|
3467
3694
|
updated_at: now,
|
|
3468
3695
|
last_accessed_at: null,
|
|
3469
3696
|
access_count: 0,
|
|
3470
|
-
deleted_at: null
|
|
3697
|
+
deleted_at: null,
|
|
3698
|
+
...grounding?.trust
|
|
3471
3699
|
};
|
|
3472
3700
|
await this.entryRepo.upsert(factObj, tx);
|
|
3473
3701
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -3539,7 +3767,7 @@ var MaintenanceService = class {
|
|
|
3539
3767
|
if (!ontologyService) {
|
|
3540
3768
|
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
3541
3769
|
}
|
|
3542
|
-
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
3770
|
+
const { mode, manifest: effectiveManifest } = await ontologyService.getEffectiveState(entityId);
|
|
3543
3771
|
if (mode === "off") {
|
|
3544
3772
|
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3545
3773
|
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
@@ -3549,6 +3777,14 @@ var MaintenanceService = class {
|
|
|
3549
3777
|
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3550
3778
|
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
3551
3779
|
}
|
|
3780
|
+
const classifierMode = options?.classifier ?? this.options.config?.ontology?.backfillClassifier ?? "llm";
|
|
3781
|
+
const classify = this.options.llmProvider.classify;
|
|
3782
|
+
if (classifierMode === "auto" && typeof classify === "function") {
|
|
3783
|
+
const slugs = effectiveManifest.node_types.map((n) => n.type);
|
|
3784
|
+
if (slugs.length > 0 && slugs.length <= 255) {
|
|
3785
|
+
return this._runClassifierBackfill(entityId, candidates, effectiveManifest, now, recheckCutoff);
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3552
3788
|
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
3553
3789
|
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
3554
3790
|
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
@@ -3627,6 +3863,87 @@ var MaintenanceService = class {
|
|
|
3627
3863
|
deferred: counts.deferred
|
|
3628
3864
|
};
|
|
3629
3865
|
}
|
|
3866
|
+
/**
|
|
3867
|
+
* Classifier-mode backfill (spec §7.3): one `choice` question per untyped
|
|
3868
|
+
* fact over the effective manifest's node-type slugs. Accepted answers go
|
|
3869
|
+
* through `_applyOntologyBackfillBatch` exactly like LLM classifications.
|
|
3870
|
+
* No edges are proposed.
|
|
3871
|
+
*/
|
|
3872
|
+
async _runClassifierBackfill(entityId, candidates, manifest, now, recheckCutoff) {
|
|
3873
|
+
const provider = this.options.llmProvider;
|
|
3874
|
+
const rawMin = this.options.config?.ontology?.classifyMinConfidence;
|
|
3875
|
+
const minConfidence = typeof rawMin === "number" && Number.isFinite(rawMin) && rawMin >= 0 && rawMin <= 1 ? rawMin : 0.5;
|
|
3876
|
+
const rawConcurrency = this.options.config?.chunkConcurrency ?? 1;
|
|
3877
|
+
const concurrency = Number.isFinite(rawConcurrency) && rawConcurrency >= 1 ? Math.floor(rawConcurrency) : 1;
|
|
3878
|
+
const question = {
|
|
3879
|
+
kind: "choice",
|
|
3880
|
+
options: manifest.node_types.map((n) => n.type),
|
|
3881
|
+
instructions: "Choose the ontology node type that best describes this fact.\n" + manifest.node_types.map((n) => `- ${n.type}: ${n.description}`).join("\n")
|
|
3882
|
+
};
|
|
3883
|
+
const outcomes = await withConcurrency(
|
|
3884
|
+
candidates.map((fact) => async () => {
|
|
3885
|
+
let response;
|
|
3886
|
+
try {
|
|
3887
|
+
response = await provider.classify({ state: classifierStateForFact(fact), questions: { okf_type: question } });
|
|
3888
|
+
} catch {
|
|
3889
|
+
return { fact, kind: "threw" };
|
|
3890
|
+
}
|
|
3891
|
+
const checked = validateClassifierAnswer(response, "okf_type", question);
|
|
3892
|
+
if (!checked.ok) return { fact, kind: "invalid", reason: checked.reason };
|
|
3893
|
+
if (checked.answer.kind !== "choice") return { fact, kind: "invalid", reason: "kind_mismatch" };
|
|
3894
|
+
if (checked.answer.confidence < minConfidence) return { fact, kind: "low_confidence" };
|
|
3895
|
+
return { fact, kind: "accepted", okfType: checked.answer.choice };
|
|
3896
|
+
}),
|
|
3897
|
+
concurrency
|
|
3898
|
+
);
|
|
3899
|
+
const attempted = outcomes.filter((o) => o.kind !== "threw").map((o) => o.fact);
|
|
3900
|
+
const skipped = outcomes.length - attempted.length;
|
|
3901
|
+
const classifications = outcomes.filter((o) => o.kind === "accepted").map((o) => ({ id: o.fact.id, okf_type: o.okfType }));
|
|
3902
|
+
const invalidCount = outcomes.filter((o) => o.kind === "invalid").length;
|
|
3903
|
+
let typed = 0;
|
|
3904
|
+
let failedValidation = 0;
|
|
3905
|
+
let edgesAdded = 0;
|
|
3906
|
+
let aborted = false;
|
|
3907
|
+
if (attempted.length > 0) {
|
|
3908
|
+
const applied = await this._applyOntologyBackfillBatch(
|
|
3909
|
+
entityId,
|
|
3910
|
+
{ batch: attempted, classifications, ontologyUpdates: void 0 },
|
|
3911
|
+
now
|
|
3912
|
+
);
|
|
3913
|
+
aborted = applied.abortedOntologyOff;
|
|
3914
|
+
if (!aborted) {
|
|
3915
|
+
typed = applied.typed;
|
|
3916
|
+
failedValidation = invalidCount + applied.failedValidation;
|
|
3917
|
+
edgesAdded = applied.edgesAdded;
|
|
3918
|
+
}
|
|
3919
|
+
}
|
|
3920
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
3921
|
+
const diagBase = { entityId, operation: "ontologyBackfill", trigger: "call" };
|
|
3922
|
+
for (const o of outcomes) {
|
|
3923
|
+
if (o.kind === "low_confidence") {
|
|
3924
|
+
diagBuffer.push({ ...diagBase, code: "classification_low_confidence", detail: { factId: o.fact.id, reason: "below_threshold" } });
|
|
3925
|
+
} else if (o.kind === "invalid") {
|
|
3926
|
+
diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: o.reason } });
|
|
3927
|
+
} else if (o.kind === "threw") {
|
|
3928
|
+
diagBuffer.push({ ...diagBase, code: "classification_invalid", detail: { factId: o.fact.id, reason: "classify_threw" } });
|
|
3929
|
+
}
|
|
3930
|
+
}
|
|
3931
|
+
diagBuffer.flush(this.options);
|
|
3932
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
3933
|
+
if (aborted) {
|
|
3934
|
+
return { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0, skipped, remaining: 0, deferred: counts.deferred };
|
|
3935
|
+
}
|
|
3936
|
+
this.searchService.evictCache(entityId);
|
|
3937
|
+
return {
|
|
3938
|
+
scanned: candidates.length,
|
|
3939
|
+
typed,
|
|
3940
|
+
failedValidation,
|
|
3941
|
+
edgesAdded,
|
|
3942
|
+
skipped,
|
|
3943
|
+
remaining: counts.eligible,
|
|
3944
|
+
deferred: counts.deferred
|
|
3945
|
+
};
|
|
3946
|
+
}
|
|
3630
3947
|
/**
|
|
3631
3948
|
* Applies one parsed backfill batch in its own transaction. Per-batch rather
|
|
3632
3949
|
* than one transaction for the pass, so mergeEmergentUpdates semantics and
|
|
@@ -3742,7 +4059,7 @@ var MaintenanceService = class {
|
|
|
3742
4059
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
3743
4060
|
* per-pass — see the call site in doRunHeal.
|
|
3744
4061
|
*/
|
|
3745
|
-
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
4062
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache, withBodies = false) {
|
|
3746
4063
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
3747
4064
|
if (!query) return [];
|
|
3748
4065
|
const cached = cache?.get(query);
|
|
@@ -3755,7 +4072,7 @@ var MaintenanceService = class {
|
|
|
3755
4072
|
const hitIds = hits.map((h) => h.id);
|
|
3756
4073
|
const anchors = [];
|
|
3757
4074
|
if (hitIds.length > 0) {
|
|
3758
|
-
const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
4075
|
+
const rows = withBodies ? await this.entryRepo.findAnchorRowsByIds(entityId, hitIds, void 0, { withBody: true }) : await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
3759
4076
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
3760
4077
|
for (const id of hitIds) {
|
|
3761
4078
|
const row = byId.get(id);
|
|
@@ -4330,8 +4647,7 @@ var EmbeddingService = class {
|
|
|
4330
4647
|
}
|
|
4331
4648
|
}
|
|
4332
4649
|
async tryEmbedFact(fact, ctx) {
|
|
4333
|
-
|
|
4334
|
-
if (typeof embedFn !== "function") return { ok: false, kind: "no_provider" };
|
|
4650
|
+
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
4335
4651
|
let tagsStr;
|
|
4336
4652
|
if (Array.isArray(fact.tags)) {
|
|
4337
4653
|
tagsStr = fact.tags.join(" ");
|
|
@@ -4348,7 +4664,7 @@ var EmbeddingService = class {
|
|
|
4348
4664
|
const text = clip(`${fact.title} ${fact.body} ${tagsStr}`.trim(), maxEmbedChars);
|
|
4349
4665
|
let float32Vector;
|
|
4350
4666
|
try {
|
|
4351
|
-
const vector = await
|
|
4667
|
+
const vector = await this.options.llmProvider.embed(text);
|
|
4352
4668
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
4353
4669
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
4354
4670
|
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
@@ -4607,7 +4923,6 @@ var RetrievalService = class {
|
|
|
4607
4923
|
const hybridWeight = options?.hybridWeight ?? config?.hybridWeight;
|
|
4608
4924
|
const weight = hybridWeight !== void 0 && !Number.isNaN(hybridWeight) ? Math.max(0, Math.min(1, hybridWeight)) : void 0;
|
|
4609
4925
|
const skipEmbed = weight === 0;
|
|
4610
|
-
const embedFn = this.options.llmProvider.embed;
|
|
4611
4926
|
let facts = [];
|
|
4612
4927
|
let scoreByFactId;
|
|
4613
4928
|
if (maxResults === 0) ; else if (trimmedQuery) {
|
|
@@ -4618,11 +4933,11 @@ var RetrievalService = class {
|
|
|
4618
4933
|
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + draftPad;
|
|
4619
4934
|
if (scoredEntityIds.length === 0) {
|
|
4620
4935
|
usedEmbed = true;
|
|
4621
|
-
} else if (!skipEmbed &&
|
|
4936
|
+
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
4622
4937
|
let rankerShouldRethrow = false;
|
|
4623
4938
|
let pendingRankerFallbackError;
|
|
4624
4939
|
try {
|
|
4625
|
-
const queryVec = await
|
|
4940
|
+
const queryVec = await this.options.llmProvider.embed(trimmedQuery);
|
|
4626
4941
|
if (queryVec.length === 0 || !queryVec.every((v) => typeof v === "number" && isFinite(v))) {
|
|
4627
4942
|
throw new Error(
|
|
4628
4943
|
"embed() returned an empty or non-finite vector. Falling back to keyword search."
|
|
@@ -5291,6 +5606,6 @@ var WriteService = class {
|
|
|
5291
5606
|
}
|
|
5292
5607
|
};
|
|
5293
5608
|
|
|
5294
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
5295
|
-
//# sourceMappingURL=chunk-
|
|
5296
|
-
//# sourceMappingURL=chunk-
|
|
5609
|
+
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
5610
|
+
//# sourceMappingURL=chunk-YBWE26XR.mjs.map
|
|
5611
|
+
//# sourceMappingURL=chunk-YBWE26XR.mjs.map
|