@equationalapplications/core-llm-wiki 7.5.0 → 7.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -0
- package/dist/{chunk-3P7FAKJA.mjs → chunk-VOSMYISQ.mjs} +243 -41
- package/dist/chunk-VOSMYISQ.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +434 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +196 -5
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-ClpOUiLI.d.mts → testing-A4Sofxyo.d.mts} +136 -4
- package/dist/{testing-ClpOUiLI.d.ts → testing-A4Sofxyo.d.ts} +136 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +236 -38
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-3P7FAKJA.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1144,15 +1144,16 @@ var EntryRepository = class extends BaseRepository {
|
|
|
1144
1144
|
* fact, not only immutable_document rows, so the source-type restriction has
|
|
1145
1145
|
* to be applied here, after retrieval.
|
|
1146
1146
|
*/
|
|
1147
|
-
async findAnchorRowsByIds(entityId, ids, tx) {
|
|
1147
|
+
async findAnchorRowsByIds(entityId, ids, tx, opts) {
|
|
1148
1148
|
if (ids.length === 0) return [];
|
|
1149
1149
|
const executor = this.getExecutor(tx);
|
|
1150
|
+
const columns = opts?.withBody ? "id, title, source_ref, body, lifecycle_status" : "id, title, source_ref";
|
|
1150
1151
|
const rows = [];
|
|
1151
1152
|
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
1152
1153
|
const chunk = ids.slice(i, i + this.chunkSize);
|
|
1153
1154
|
const placeholders = chunk.map(() => "?").join(", ");
|
|
1154
1155
|
const chunkRows = await executor.getAllAsync(
|
|
1155
|
-
`SELECT
|
|
1156
|
+
`SELECT ${columns} FROM ${this.prefix}entries
|
|
1156
1157
|
WHERE entity_id = ? AND deleted_at IS NULL
|
|
1157
1158
|
AND source_type = 'immutable_document'
|
|
1158
1159
|
AND id IN (${placeholders})`,
|
|
@@ -1591,6 +1592,35 @@ var EntryRepository = class extends BaseRepository {
|
|
|
1591
1592
|
}
|
|
1592
1593
|
return out;
|
|
1593
1594
|
}
|
|
1595
|
+
/**
|
|
1596
|
+
* Per source ref, the hash of the latest live row (same ordering as
|
|
1597
|
+
* {@link findLatestSourceHashes}) and whether ANY live row carries a hash.
|
|
1598
|
+
* Refs with no live row are absent. Chunked under the bind-variable limit.
|
|
1599
|
+
*/
|
|
1600
|
+
async findSourceStates(entityId, sourceRefs, tx) {
|
|
1601
|
+
const out = /* @__PURE__ */ new Map();
|
|
1602
|
+
const deduped = Array.from(new Set(sourceRefs));
|
|
1603
|
+
if (deduped.length === 0) return out;
|
|
1604
|
+
const executor = this.getExecutor(tx);
|
|
1605
|
+
const chunkLimit = Math.max(1, this.chunkSize - 1);
|
|
1606
|
+
for (let i = 0; i < deduped.length; i += chunkLimit) {
|
|
1607
|
+
const chunk = deduped.slice(i, i + chunkLimit);
|
|
1608
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1609
|
+
const rows = await executor.getAllAsync(
|
|
1610
|
+
`WITH ranked AS (
|
|
1611
|
+
SELECT source_ref, source_hash,
|
|
1612
|
+
ROW_NUMBER() OVER (PARTITION BY source_ref ORDER BY updated_at DESC, id ASC) AS rn,
|
|
1613
|
+
MAX(CASE WHEN source_hash IS NOT NULL THEN 1 ELSE 0 END) OVER (PARTITION BY source_ref) AS any_hashed
|
|
1614
|
+
FROM ${this.prefix}entries
|
|
1615
|
+
WHERE entity_id = ? AND source_ref IN (${placeholders}) AND deleted_at IS NULL
|
|
1616
|
+
)
|
|
1617
|
+
SELECT source_ref, source_hash, any_hashed FROM ranked WHERE rn = 1`,
|
|
1618
|
+
[entityId, ...chunk]
|
|
1619
|
+
);
|
|
1620
|
+
for (const r of rows) out.set(r.source_ref, { latestHash: r.source_hash, anyHashed: Number(r.any_hashed) === 1 });
|
|
1621
|
+
}
|
|
1622
|
+
return out;
|
|
1623
|
+
}
|
|
1594
1624
|
/**
|
|
1595
1625
|
* Return the live source_refs for an entity that hold the given source_hash.
|
|
1596
1626
|
* Used by the ingestDocument duplicate-hash guard and by hosts auditing
|
|
@@ -2703,6 +2733,10 @@ function typeSatisfies(declaredType, concreteType, manifest) {
|
|
|
2703
2733
|
const parent = typeof def?.parent_type === "string" ? def.parent_type.trim().toLowerCase() : "";
|
|
2704
2734
|
return parent !== "" && parent === declared;
|
|
2705
2735
|
}
|
|
2736
|
+
function edgeTripleAllowed(manifest, edgeType, sourceType, targetType) {
|
|
2737
|
+
const wanted = edgeType.trim().toLowerCase();
|
|
2738
|
+
return (manifest.edge_types ?? []).some((d) => typeof d?.type === "string" && d.type.trim().toLowerCase() === wanted && typeof d.source_type === "string" && typeSatisfies(d.source_type, sourceType, manifest) && typeof d.target_type === "string" && typeSatisfies(d.target_type, targetType, manifest));
|
|
2739
|
+
}
|
|
2706
2740
|
function validateManifest(manifest) {
|
|
2707
2741
|
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
2708
2742
|
for (const node of manifest.node_types ?? []) {
|
|
@@ -4154,6 +4188,19 @@ function validateTags(tags) {
|
|
|
4154
4188
|
if (!Array.isArray(tags)) return [];
|
|
4155
4189
|
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
4156
4190
|
}
|
|
4191
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
4192
|
+
function normalizeEvidence(raw) {
|
|
4193
|
+
if (!Array.isArray(raw)) return void 0;
|
|
4194
|
+
const out = [];
|
|
4195
|
+
for (const entry of raw) {
|
|
4196
|
+
if (typeof entry !== "string") continue;
|
|
4197
|
+
const trimmed = entry.trim();
|
|
4198
|
+
if (!trimmed) continue;
|
|
4199
|
+
out.push(trimmed);
|
|
4200
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
4201
|
+
}
|
|
4202
|
+
return out;
|
|
4203
|
+
}
|
|
4157
4204
|
function validateFact(fact) {
|
|
4158
4205
|
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
4159
4206
|
const title = clip(fact.title, 80);
|
|
@@ -4161,13 +4208,17 @@ function validateFact(fact) {
|
|
|
4161
4208
|
if (!title || !body) return null;
|
|
4162
4209
|
let confidence = fact.confidence;
|
|
4163
4210
|
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
4164
|
-
|
|
4211
|
+
const valid = {
|
|
4165
4212
|
...fact,
|
|
4166
4213
|
title,
|
|
4167
4214
|
body,
|
|
4168
4215
|
confidence,
|
|
4169
4216
|
tags: validateTags(fact.tags)
|
|
4170
4217
|
};
|
|
4218
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
4219
|
+
if (evidence) valid.evidence = evidence;
|
|
4220
|
+
else delete valid.evidence;
|
|
4221
|
+
return valid;
|
|
4171
4222
|
}
|
|
4172
4223
|
function validateTask(task) {
|
|
4173
4224
|
if (typeof task?.description !== "string") return null;
|
|
@@ -4212,6 +4263,60 @@ function jaccardScore(a, b) {
|
|
|
4212
4263
|
return intersection.size / union.size;
|
|
4213
4264
|
}
|
|
4214
4265
|
|
|
4266
|
+
// src/utils/grounding.ts
|
|
4267
|
+
var GROUNDING_VERIFIER = "process:grounding-check";
|
|
4268
|
+
var WRITERS = ["ingest", "librarian", "heal"];
|
|
4269
|
+
function positiveInt(value, fallback) {
|
|
4270
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
4271
|
+
}
|
|
4272
|
+
function resolveGrounding(config) {
|
|
4273
|
+
if (!config || config.mode !== "draft") return null;
|
|
4274
|
+
const writers = Array.isArray(config.writers) ? config.writers.filter((w) => WRITERS.includes(w)) : ["ingest"];
|
|
4275
|
+
return {
|
|
4276
|
+
writers: new Set(writers),
|
|
4277
|
+
minEvidenceChars: positiveInt(config.minEvidenceChars, 20),
|
|
4278
|
+
// Never ask for more quotes than checkGrounding accepts (MAX_EVIDENCE_QUOTES).
|
|
4279
|
+
maxEvidence: Math.min(MAX_EVIDENCE_QUOTES, positiveInt(config.maxEvidence, 3)),
|
|
4280
|
+
maxEvidenceChars: positiveInt(config.maxEvidenceChars, 300)
|
|
4281
|
+
};
|
|
4282
|
+
}
|
|
4283
|
+
function normalizeForGrounding(text) {
|
|
4284
|
+
return text.normalize("NFKC").replace(/\s+/g, " ").trim();
|
|
4285
|
+
}
|
|
4286
|
+
function buildGroundingCorpus(parts) {
|
|
4287
|
+
return parts.filter((p) => typeof p === "string").map(normalizeForGrounding);
|
|
4288
|
+
}
|
|
4289
|
+
function checkGrounding(evidence, normalizedCorpus, cfg) {
|
|
4290
|
+
const quotes = evidence ?? [];
|
|
4291
|
+
if (quotes.length > MAX_EVIDENCE_QUOTES) return { status: "failed", reason: "too_many_quotes" };
|
|
4292
|
+
if (quotes.length === 0) return { status: "missing", reason: "no_evidence" };
|
|
4293
|
+
const qualifying = quotes.map(normalizeForGrounding).filter((q) => q.length >= cfg.minEvidenceChars);
|
|
4294
|
+
if (qualifying.length === 0) return { status: "missing", reason: "evidence_too_short" };
|
|
4295
|
+
if (qualifying.some((q) => !normalizedCorpus.some((part) => part.includes(q)))) {
|
|
4296
|
+
return { status: "failed", reason: "quote_not_found" };
|
|
4297
|
+
}
|
|
4298
|
+
return {
|
|
4299
|
+
status: "grounded",
|
|
4300
|
+
retained: qualifying.slice(0, cfg.maxEvidence).map((q) => safeSlice(q, 0, cfg.maxEvidenceChars))
|
|
4301
|
+
};
|
|
4302
|
+
}
|
|
4303
|
+
function groundingOutcome(verdict, now) {
|
|
4304
|
+
if (verdict.status === "grounded") {
|
|
4305
|
+
return {
|
|
4306
|
+
trust: {
|
|
4307
|
+
lifecycle_status: "stable",
|
|
4308
|
+
okf_verified: [{ by: GROUNDING_VERIFIER, at: new Date(now).toISOString() }],
|
|
4309
|
+
last_verified_at: now,
|
|
4310
|
+
last_verified_by: GROUNDING_VERIFIER
|
|
4311
|
+
}
|
|
4312
|
+
};
|
|
4313
|
+
}
|
|
4314
|
+
return {
|
|
4315
|
+
trust: { lifecycle_status: "draft" },
|
|
4316
|
+
diagnostic: { code: verdict.status === "missing" ? "grounding_missing" : "grounding_failed", reason: verdict.reason }
|
|
4317
|
+
};
|
|
4318
|
+
}
|
|
4319
|
+
|
|
4215
4320
|
// src/utils/diagnostics.ts
|
|
4216
4321
|
var SEVERITY = {
|
|
4217
4322
|
ingest_chunk_failed: "warn",
|
|
@@ -4341,17 +4446,40 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
4341
4446
|
}
|
|
4342
4447
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
4343
4448
|
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
4449
|
+
var GROUNDING_SOURCE = {
|
|
4450
|
+
ingest: { key: "facts", source: "the document chunk" },
|
|
4451
|
+
librarian: { key: "facts", source: 'the "summary" text of the events' },
|
|
4452
|
+
heal: { key: "newFacts", source: 'the "summary" text of the recent events or the "body" text of the document anchors' }
|
|
4453
|
+
};
|
|
4454
|
+
function groundingEvidenceBlock(writer, cfg) {
|
|
4455
|
+
const { key, source } = GROUNDING_SOURCE[writer];
|
|
4456
|
+
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
4457
|
+
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
4458
|
+
}
|
|
4344
4459
|
|
|
4345
4460
|
// src/utils/healConstants.ts
|
|
4346
4461
|
var HEAL_MAX_ANCHORS = 50;
|
|
4347
4462
|
var HEAL_ANCHORS_PER_CANDIDATE = 4;
|
|
4348
4463
|
var HEAL_MAX_FACT_BODY_CHARS_L3 = 4e3;
|
|
4349
4464
|
var HEAL_MAX_TASKS = 50;
|
|
4465
|
+
var HEAL_ANCHOR_BODY_CHARS = 800;
|
|
4350
4466
|
|
|
4351
4467
|
// src/services/PromptService.ts
|
|
4352
4468
|
var PromptService = class {
|
|
4353
|
-
constructor(globalOverrides) {
|
|
4469
|
+
constructor(globalOverrides, grounding = null) {
|
|
4354
4470
|
this.globalOverrides = globalOverrides;
|
|
4471
|
+
this.grounding = grounding;
|
|
4472
|
+
}
|
|
4473
|
+
/** The resolved grounding config when `writer` is in `grounding.writers`; otherwise null (spec §6.2). */
|
|
4474
|
+
groundingFor(writer) {
|
|
4475
|
+
return this.grounding?.writers.has(writer) ? this.grounding : null;
|
|
4476
|
+
}
|
|
4477
|
+
/** Appended after any override and after ontology context, so it is always last. */
|
|
4478
|
+
appendGrounding(systemPrompt, writer) {
|
|
4479
|
+
const cfg = this.groundingFor(writer);
|
|
4480
|
+
return cfg ? `${systemPrompt}
|
|
4481
|
+
|
|
4482
|
+
${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
4355
4483
|
}
|
|
4356
4484
|
hydrate(template, variables) {
|
|
4357
4485
|
return template.replace(/\{\{\s*(\w+)\s*\}\}/g, (_match, key) => {
|
|
@@ -4381,13 +4509,13 @@ ${ctx.ontologyModeInstructions}`;
|
|
|
4381
4509
|
const hasDocumentChunk = /\{\{\s*documentChunk\s*\}\}/.test(template);
|
|
4382
4510
|
if (hasDocumentChunk || this.hasOntologyPlaceholders(template)) {
|
|
4383
4511
|
return {
|
|
4384
|
-
systemPrompt: this.buildSystemPrompt(template, { documentChunk }, ontologyContext),
|
|
4512
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { documentChunk }, ontologyContext), "ingest"),
|
|
4385
4513
|
userPrompt: hasDocumentChunk ? "Please extract the facts." : `Document Chunk:
|
|
4386
4514
|
${documentChunk}`
|
|
4387
4515
|
};
|
|
4388
4516
|
}
|
|
4389
4517
|
return {
|
|
4390
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
4518
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "ingest"),
|
|
4391
4519
|
userPrompt: `Document Chunk:
|
|
4392
4520
|
${documentChunk}`
|
|
4393
4521
|
};
|
|
@@ -4396,23 +4524,27 @@ ${documentChunk}`
|
|
|
4396
4524
|
const template = runtimeOverride ?? this.globalOverrides?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT;
|
|
4397
4525
|
const hasEvents = /\{\{\s*events\s*\}\}/.test(template);
|
|
4398
4526
|
const hasCurrentFacts = /\{\{\s*currentFacts\s*\}\}/.test(template);
|
|
4527
|
+
const eventsShown = hasEvents || !hasCurrentFacts;
|
|
4528
|
+
const corpusField = this.groundingFor("librarian") ? { groundingCorpus: buildGroundingCorpus(eventsShown ? events.map(summaryOf) : []) } : {};
|
|
4399
4529
|
if (hasEvents || hasCurrentFacts || this.hasOntologyPlaceholders(template)) {
|
|
4400
4530
|
return {
|
|
4401
|
-
systemPrompt: this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext),
|
|
4531
|
+
systemPrompt: this.appendGrounding(this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext), "librarian"),
|
|
4402
4532
|
userPrompt: hasEvents || hasCurrentFacts ? "Please synthesize the context." : `Events:
|
|
4403
4533
|
${JSON.stringify(events, null, 2)}
|
|
4404
4534
|
|
|
4405
4535
|
Current Facts:
|
|
4406
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
4536
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
4537
|
+
...corpusField
|
|
4407
4538
|
};
|
|
4408
4539
|
}
|
|
4409
4540
|
return {
|
|
4410
|
-
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
4541
|
+
systemPrompt: this.appendGrounding(this.appendOntology(template, ontologyContext), "librarian"),
|
|
4411
4542
|
userPrompt: `Events:
|
|
4412
4543
|
${JSON.stringify(events, null, 2)}
|
|
4413
4544
|
|
|
4414
4545
|
Current Facts:
|
|
4415
|
-
${JSON.stringify(currentFacts, null, 2)}
|
|
4546
|
+
${JSON.stringify(currentFacts, null, 2)}`,
|
|
4547
|
+
...corpusField
|
|
4416
4548
|
};
|
|
4417
4549
|
}
|
|
4418
4550
|
/**
|
|
@@ -4442,40 +4574,54 @@ ${JSON.stringify(currentFacts, null, 2)}`
|
|
|
4442
4574
|
const effectiveEvents = attemptLevel >= 2 ? [] : recentEvents;
|
|
4443
4575
|
const maxAnchors = Math.max(1, Math.min(HEAL_MAX_ANCHORS, healCandidates.length * HEAL_ANCHORS_PER_CANDIDATE));
|
|
4444
4576
|
const effectiveAnchors = documentAnchors.slice(0, maxAnchors);
|
|
4577
|
+
const template = runtimeOverride ?? this.globalOverrides?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT;
|
|
4578
|
+
const hasRecentEvents = /\{\{\s*recentEvents\s*\}\}/.test(template);
|
|
4579
|
+
const hasDocumentAnchors = /\{\{\s*documentAnchors\s*\}\}/.test(template);
|
|
4580
|
+
const usesPlaceholders = /\{\{\s*healCandidates\s*\}\}/.test(template) || hasDocumentAnchors || /\{\{\s*allTasks\s*\}\}/.test(template) || hasRecentEvents;
|
|
4581
|
+
const healGrounding = this.groundingFor("heal");
|
|
4582
|
+
const promptAnchors = healGrounding ? effectiveAnchors.map(toGroundingAnchor) : effectiveAnchors;
|
|
4583
|
+
const eventsShown = !usesPlaceholders || hasRecentEvents;
|
|
4584
|
+
const anchorsShown = !usesPlaceholders || hasDocumentAnchors;
|
|
4585
|
+
const groundingCorpus = healGrounding ? buildGroundingCorpus([
|
|
4586
|
+
...eventsShown ? effectiveEvents.map(summaryOf) : [],
|
|
4587
|
+
...anchorsShown ? effectiveAnchors.map((a, i) => ({ status: a?.lifecycle_status, shown: promptAnchors[i] })).filter((x) => x.status !== "draft").map((x) => x.shown?.body) : []
|
|
4588
|
+
]) : null;
|
|
4445
4589
|
const { shapedCandidates, degraded } = applyBodyTruncation(
|
|
4446
4590
|
healCandidates,
|
|
4447
4591
|
attemptLevel,
|
|
4448
4592
|
bodyTruncationChars
|
|
4449
4593
|
);
|
|
4450
|
-
const
|
|
4451
|
-
if (
|
|
4594
|
+
const corpusField = groundingCorpus !== null ? { groundingCorpus } : {};
|
|
4595
|
+
if (usesPlaceholders) {
|
|
4452
4596
|
return {
|
|
4453
4597
|
prompts: {
|
|
4454
|
-
systemPrompt: this.hydrate(template, {
|
|
4598
|
+
systemPrompt: this.appendGrounding(this.hydrate(template, {
|
|
4455
4599
|
healCandidates: shapedCandidates,
|
|
4456
|
-
documentAnchors:
|
|
4600
|
+
documentAnchors: promptAnchors,
|
|
4457
4601
|
allTasks: effectiveTasks,
|
|
4458
4602
|
recentEvents: effectiveEvents
|
|
4459
|
-
}),
|
|
4603
|
+
}), "heal"),
|
|
4460
4604
|
userPrompt: "Please heal the memory graph."
|
|
4461
4605
|
},
|
|
4462
|
-
degraded
|
|
4606
|
+
degraded,
|
|
4607
|
+
...corpusField
|
|
4463
4608
|
};
|
|
4464
4609
|
}
|
|
4465
4610
|
return {
|
|
4466
4611
|
prompts: {
|
|
4467
|
-
systemPrompt: template,
|
|
4612
|
+
systemPrompt: this.appendGrounding(template, "heal"),
|
|
4468
4613
|
userPrompt: `Heal Candidates:
|
|
4469
4614
|
${JSON.stringify(shapedCandidates, null, 2)}
|
|
4470
4615
|
Document Anchors (DO NOT MODIFY OR DELETE):
|
|
4471
|
-
${JSON.stringify(
|
|
4616
|
+
${JSON.stringify(promptAnchors, null, 2)}
|
|
4472
4617
|
All Tasks:
|
|
4473
4618
|
${JSON.stringify(effectiveTasks, null, 2)}
|
|
4474
4619
|
Recent Events:
|
|
4475
4620
|
${JSON.stringify(effectiveEvents, null, 2)}
|
|
4476
4621
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
4477
4622
|
},
|
|
4478
|
-
degraded
|
|
4623
|
+
degraded,
|
|
4624
|
+
...corpusField
|
|
4479
4625
|
};
|
|
4480
4626
|
}
|
|
4481
4627
|
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
@@ -4494,6 +4640,21 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
4494
4640
|
${JSON.stringify(facts, null, 2)}`
|
|
4495
4641
|
};
|
|
4496
4642
|
}
|
|
4643
|
+
/**
|
|
4644
|
+
* The system prompt each writer would send, without hydrating data (spec
|
|
4645
|
+
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
4646
|
+
* placeholders and leaves data placeholders verbatim; heal never receives
|
|
4647
|
+
* ontology context, matching `buildHealPrompt`.
|
|
4648
|
+
*/
|
|
4649
|
+
buildInstructionTemplates(ontologyContext) {
|
|
4650
|
+
const o = this.globalOverrides;
|
|
4651
|
+
return {
|
|
4652
|
+
ingest: this.appendGrounding(this.buildSystemPrompt(o?.ingestSystemPrompt ?? INGEST_SYSTEM_PROMPT, {}, ontologyContext), "ingest"),
|
|
4653
|
+
librarian: this.appendGrounding(this.buildSystemPrompt(o?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT, {}, ontologyContext), "librarian"),
|
|
4654
|
+
heal: this.appendGrounding(o?.healSystemPrompt ?? HEAL_SYSTEM_PROMPT, "heal"),
|
|
4655
|
+
ontologyBackfill: this.buildSystemPrompt(o?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT, {}, ontologyContext)
|
|
4656
|
+
};
|
|
4657
|
+
}
|
|
4497
4658
|
};
|
|
4498
4659
|
function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
4499
4660
|
if (attemptLevel < 3) {
|
|
@@ -4525,6 +4686,19 @@ function applyBodyTruncation(candidates, attemptLevel, bodyTruncationChars) {
|
|
|
4525
4686
|
}
|
|
4526
4687
|
return { shapedCandidates, degraded };
|
|
4527
4688
|
}
|
|
4689
|
+
function summaryOf(event) {
|
|
4690
|
+
return event?.summary;
|
|
4691
|
+
}
|
|
4692
|
+
function toGroundingAnchor(anchor) {
|
|
4693
|
+
if (typeof anchor !== "object" || anchor === null) return anchor;
|
|
4694
|
+
const a = anchor;
|
|
4695
|
+
return {
|
|
4696
|
+
id: a.id,
|
|
4697
|
+
title: a.title,
|
|
4698
|
+
source_ref: a.source_ref,
|
|
4699
|
+
body: typeof a.body === "string" ? safeSlice(a.body, 0, HEAL_ANCHOR_BODY_CHARS) : ""
|
|
4700
|
+
};
|
|
4701
|
+
}
|
|
4528
4702
|
|
|
4529
4703
|
// src/utils/chunkingDefaults.ts
|
|
4530
4704
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
@@ -4554,7 +4728,7 @@ var IngestionService = class {
|
|
|
4554
4728
|
this.jobManager = jobManager;
|
|
4555
4729
|
this.embeddingService = embeddingService;
|
|
4556
4730
|
this.ontologyService = ontologyService;
|
|
4557
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
4731
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
4558
4732
|
}
|
|
4559
4733
|
async ingestDocument(entityId, params, opts) {
|
|
4560
4734
|
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
@@ -4587,6 +4761,7 @@ var IngestionService = class {
|
|
|
4587
4761
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
4588
4762
|
if (chunks.length === 0) return zeroChunkResult();
|
|
4589
4763
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
4764
|
+
const ingestGrounding = this.promptService.groundingFor("ingest");
|
|
4590
4765
|
const chunkResults = await withConcurrency(
|
|
4591
4766
|
chunks.map((chunk, chunkIndex) => async () => {
|
|
4592
4767
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
@@ -4610,11 +4785,16 @@ var IngestionService = class {
|
|
|
4610
4785
|
rejected.push({ itemIndex, reason: factRejectionReason(raw) });
|
|
4611
4786
|
}
|
|
4612
4787
|
});
|
|
4788
|
+
const verdicts = ingestGrounding ? (() => {
|
|
4789
|
+
const corpus = buildGroundingCorpus([chunk]);
|
|
4790
|
+
return facts.map((f) => checkGrounding(f.evidence, corpus, ingestGrounding));
|
|
4791
|
+
})() : [];
|
|
4613
4792
|
return {
|
|
4614
4793
|
status: "ok",
|
|
4615
4794
|
facts,
|
|
4616
4795
|
itemIndexes,
|
|
4617
4796
|
rejected,
|
|
4797
|
+
verdicts,
|
|
4618
4798
|
ontology_updates: result2.ontology_updates
|
|
4619
4799
|
};
|
|
4620
4800
|
} catch (e) {
|
|
@@ -4659,6 +4839,7 @@ var IngestionService = class {
|
|
|
4659
4839
|
const failures = [];
|
|
4660
4840
|
const seen = /* @__PURE__ */ new Set();
|
|
4661
4841
|
const orderedChunkFacts = [];
|
|
4842
|
+
const groundingLedger = /* @__PURE__ */ new Map();
|
|
4662
4843
|
const diagBuffer = new DiagnosticBuffer();
|
|
4663
4844
|
const diagBase = { entityId, operation: "ingest", trigger: "call" };
|
|
4664
4845
|
for (const [chunkIndex, slot] of chunkResults.entries()) {
|
|
@@ -4677,6 +4858,7 @@ var IngestionService = class {
|
|
|
4677
4858
|
if (!seen.has(normalizedTitle)) {
|
|
4678
4859
|
seen.add(normalizedTitle);
|
|
4679
4860
|
dedupedFacts.push(fact);
|
|
4861
|
+
if (ingestGrounding) groundingLedger.set(fact, { verdict: slot.verdicts[k], chunkIndex, itemIndex: slot.itemIndexes[k] });
|
|
4680
4862
|
} else {
|
|
4681
4863
|
diagBuffer.push({
|
|
4682
4864
|
...diagBase,
|
|
@@ -4713,7 +4895,7 @@ var IngestionService = class {
|
|
|
4713
4895
|
try {
|
|
4714
4896
|
if (failedChunks === 0) {
|
|
4715
4897
|
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4716
|
-
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer);
|
|
4898
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger);
|
|
4717
4899
|
});
|
|
4718
4900
|
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
4719
4901
|
insertedFacts.push(...fullResult.insertedFacts);
|
|
@@ -4721,7 +4903,7 @@ var IngestionService = class {
|
|
|
4721
4903
|
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4722
4904
|
const flat = [];
|
|
4723
4905
|
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
4724
|
-
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer);
|
|
4906
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx, diagBuffer, groundingLedger);
|
|
4725
4907
|
});
|
|
4726
4908
|
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
4727
4909
|
}
|
|
@@ -4863,7 +5045,8 @@ var IngestionService = class {
|
|
|
4863
5045
|
last_accessed_at: null,
|
|
4864
5046
|
access_count: 0,
|
|
4865
5047
|
deleted_at: null,
|
|
4866
|
-
okf_type: normalized.okf_type
|
|
5048
|
+
okf_type: normalized.okf_type,
|
|
5049
|
+
...opts?.nodeTrust?.get(node.id)
|
|
4867
5050
|
};
|
|
4868
5051
|
wikiFacts.push(wikiFact);
|
|
4869
5052
|
}
|
|
@@ -4954,9 +5137,10 @@ var IngestionService = class {
|
|
|
4954
5137
|
* On the partial path the caller never invokes this method; the empty
|
|
4955
5138
|
* arrays stay empty.
|
|
4956
5139
|
*/
|
|
4957
|
-
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer) {
|
|
5140
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
4958
5141
|
const deletedSourceFactIds = [];
|
|
4959
5142
|
const insertedFacts = [];
|
|
5143
|
+
const nodeTrust = /* @__PURE__ */ new Map();
|
|
4960
5144
|
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
4961
5145
|
const titleIndex = /* @__PURE__ */ new Map();
|
|
4962
5146
|
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
@@ -5000,6 +5184,20 @@ var IngestionService = class {
|
|
|
5000
5184
|
tags: fact.tags,
|
|
5001
5185
|
confidence: fact.confidence
|
|
5002
5186
|
});
|
|
5187
|
+
const grounding = groundingLedger.get(fact);
|
|
5188
|
+
if (grounding) {
|
|
5189
|
+
const outcome = groundingOutcome(grounding.verdict, now);
|
|
5190
|
+
nodeTrust.set(id, outcome.trust);
|
|
5191
|
+
if (outcome.diagnostic) {
|
|
5192
|
+
diagBuffer.push({
|
|
5193
|
+
entityId,
|
|
5194
|
+
operation: "ingest",
|
|
5195
|
+
trigger: "call",
|
|
5196
|
+
code: outcome.diagnostic.code,
|
|
5197
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
5198
|
+
});
|
|
5199
|
+
}
|
|
5200
|
+
}
|
|
5003
5201
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
5004
5202
|
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
5005
5203
|
if (normalized.edges.length > 0) {
|
|
@@ -5031,7 +5229,7 @@ var IngestionService = class {
|
|
|
5031
5229
|
entityId,
|
|
5032
5230
|
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
5033
5231
|
tx,
|
|
5034
|
-
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" } }
|
|
5232
|
+
{ strict: false, diag: { buffer: diagBuffer, operation: "ingest" }, ...nodeTrust.size > 0 ? { nodeTrust } : {} }
|
|
5035
5233
|
);
|
|
5036
5234
|
return { deletedSourceFactIds, insertedFacts };
|
|
5037
5235
|
}
|
|
@@ -5055,7 +5253,7 @@ var IngestionService = class {
|
|
|
5055
5253
|
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
5056
5254
|
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
5057
5255
|
*/
|
|
5058
|
-
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer) {
|
|
5256
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx, diagBuffer, groundingLedger = /* @__PURE__ */ new Map()) {
|
|
5059
5257
|
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
5060
5258
|
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
5061
5259
|
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
@@ -5078,6 +5276,17 @@ var IngestionService = class {
|
|
|
5078
5276
|
}
|
|
5079
5277
|
liveTitles.add(normalizedTitle);
|
|
5080
5278
|
const id = generateId("fact_");
|
|
5279
|
+
const grounding = groundingLedger.get(fact);
|
|
5280
|
+
const outcome = grounding ? groundingOutcome(grounding.verdict, now) : null;
|
|
5281
|
+
if (grounding && outcome?.diagnostic) {
|
|
5282
|
+
diagBuffer.push({
|
|
5283
|
+
entityId,
|
|
5284
|
+
operation: "ingest",
|
|
5285
|
+
trigger: "call",
|
|
5286
|
+
code: outcome.diagnostic.code,
|
|
5287
|
+
detail: { factId: id, sourceRef, chunkIndex: grounding.chunkIndex, itemIndex: grounding.itemIndex, reason: outcome.diagnostic.reason }
|
|
5288
|
+
});
|
|
5289
|
+
}
|
|
5081
5290
|
const wikiFact = {
|
|
5082
5291
|
id,
|
|
5083
5292
|
entity_id: entityId,
|
|
@@ -5096,7 +5305,8 @@ var IngestionService = class {
|
|
|
5096
5305
|
last_accessed_at: null,
|
|
5097
5306
|
access_count: 0,
|
|
5098
5307
|
deleted_at: null,
|
|
5099
|
-
okf_type: null
|
|
5308
|
+
okf_type: null,
|
|
5309
|
+
...outcome?.trust
|
|
5100
5310
|
};
|
|
5101
5311
|
await this.entryRepo.upsert(wikiFact, tx);
|
|
5102
5312
|
insertedDescriptors.push({
|
|
@@ -5264,7 +5474,7 @@ async function runBatched(args) {
|
|
|
5264
5474
|
}
|
|
5265
5475
|
let result;
|
|
5266
5476
|
try {
|
|
5267
|
-
result = parse(responseText, batch);
|
|
5477
|
+
result = parse(responseText, batch, prompts);
|
|
5268
5478
|
} catch (err) {
|
|
5269
5479
|
await onFailure(batch, err, attemptLevel, false);
|
|
5270
5480
|
return;
|
|
@@ -5345,7 +5555,7 @@ var MaintenanceService = class {
|
|
|
5345
5555
|
this.jobManager = jobManager;
|
|
5346
5556
|
this.embeddingService = embeddingService;
|
|
5347
5557
|
this.ontologyService = ontologyService;
|
|
5348
|
-
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
5558
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
5349
5559
|
}
|
|
5350
5560
|
async runPrune(entityId, options) {
|
|
5351
5561
|
this.jobManager.acquireLock("prune", entityId);
|
|
@@ -5650,8 +5860,10 @@ var MaintenanceService = class {
|
|
|
5650
5860
|
};
|
|
5651
5861
|
});
|
|
5652
5862
|
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
5653
|
-
const
|
|
5654
|
-
|
|
5863
|
+
const promptEvents = events.reverse();
|
|
5864
|
+
const librarianGrounding = this.promptService.groundingFor("librarian");
|
|
5865
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = this.promptService.buildLibrarianPrompt(
|
|
5866
|
+
promptEvents,
|
|
5655
5867
|
currentFacts,
|
|
5656
5868
|
promptOverride,
|
|
5657
5869
|
ontologyContext
|
|
@@ -5720,6 +5932,10 @@ var MaintenanceService = class {
|
|
|
5720
5932
|
const validationDrops = [];
|
|
5721
5933
|
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
5722
5934
|
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
5935
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
5936
|
+
if (grounding?.diagnostic) {
|
|
5937
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
5938
|
+
}
|
|
5723
5939
|
const factObj = {
|
|
5724
5940
|
id,
|
|
5725
5941
|
entity_id: entityId,
|
|
@@ -5735,7 +5951,8 @@ var MaintenanceService = class {
|
|
|
5735
5951
|
last_accessed_at: null,
|
|
5736
5952
|
access_count: 0,
|
|
5737
5953
|
deleted_at: null,
|
|
5738
|
-
okf_type: normalized.okf_type
|
|
5954
|
+
okf_type: normalized.okf_type,
|
|
5955
|
+
...grounding?.trust
|
|
5739
5956
|
};
|
|
5740
5957
|
await this.entryRepo.upsert(factObj, tx);
|
|
5741
5958
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -5851,11 +6068,14 @@ var MaintenanceService = class {
|
|
|
5851
6068
|
}
|
|
5852
6069
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
5853
6070
|
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
6071
|
+
const healGrounding = this.promptService.groundingFor("heal");
|
|
5854
6072
|
const toPromptShape = (f) => {
|
|
5855
6073
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
5856
|
-
|
|
6074
|
+
const shown = healGrounding ? (({ lifecycle_status: _l, okf_verified: _v, last_verified_at: _a, last_verified_by: _b, ...r }) => r)(rest) : rest;
|
|
6075
|
+
return { ...shown, tags: typeof shown.tags === "string" ? JSON.parse(shown.tags) : shown.tags };
|
|
5857
6076
|
};
|
|
5858
6077
|
const anchorCache = /* @__PURE__ */ new Map();
|
|
6078
|
+
const corpusByPrompt = /* @__PURE__ */ new WeakMap();
|
|
5859
6079
|
const degraded = [];
|
|
5860
6080
|
const outcome = await runBatched({
|
|
5861
6081
|
items: healCandidates,
|
|
@@ -5868,9 +6088,10 @@ var MaintenanceService = class {
|
|
|
5868
6088
|
// not overfetch the same 200 keyword hits a 25-fact batch once did.
|
|
5869
6089
|
// buildHealPrompt applies the matching cap on its side — values must match.
|
|
5870
6090
|
Math.min(HEAL_MAX_ANCHORS, batch.length * HEAL_ANCHORS_PER_CANDIDATE),
|
|
5871
|
-
anchorCache
|
|
6091
|
+
anchorCache,
|
|
6092
|
+
healGrounding !== null
|
|
5872
6093
|
);
|
|
5873
|
-
const { prompts, degraded: batchDegraded } = await this.promptService.buildHealPrompt(
|
|
6094
|
+
const { prompts, degraded: batchDegraded, groundingCorpus } = await this.promptService.buildHealPrompt(
|
|
5874
6095
|
batch.map(toPromptShape),
|
|
5875
6096
|
documentAnchors,
|
|
5876
6097
|
allTasks,
|
|
@@ -5880,16 +6101,18 @@ var MaintenanceService = class {
|
|
|
5880
6101
|
bodyTruncationChars
|
|
5881
6102
|
);
|
|
5882
6103
|
degraded.push(...batchDegraded);
|
|
6104
|
+
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
5883
6105
|
return prompts;
|
|
5884
6106
|
},
|
|
5885
6107
|
call: (prompts) => this.options.llmProvider.generateText(prompts),
|
|
5886
|
-
parse: (responseText, batch) => {
|
|
6108
|
+
parse: (responseText, batch, prompts) => {
|
|
5887
6109
|
const result = parseJsonResponse(responseText);
|
|
5888
6110
|
return {
|
|
5889
6111
|
batch,
|
|
5890
6112
|
downgraded: Array.isArray(result.downgraded) ? result.downgraded : [],
|
|
5891
6113
|
deleted: Array.isArray(result.deleted) ? result.deleted : [],
|
|
5892
|
-
newFacts: Array.isArray(result.newFacts) ? result.newFacts : []
|
|
6114
|
+
newFacts: Array.isArray(result.newFacts) ? result.newFacts : [],
|
|
6115
|
+
corpus: corpusByPrompt.get(prompts) ?? null
|
|
5893
6116
|
};
|
|
5894
6117
|
},
|
|
5895
6118
|
maxOutputTokens: this.options.llmProvider.maxOutputTokens,
|
|
@@ -5904,6 +6127,7 @@ var MaintenanceService = class {
|
|
|
5904
6127
|
const safeDeletedSet = /* @__PURE__ */ new Set();
|
|
5905
6128
|
const newFacts = [];
|
|
5906
6129
|
const newFactItemIndexes = [];
|
|
6130
|
+
const newFactCorpora = [];
|
|
5907
6131
|
for (const batchResult of outcome.results) {
|
|
5908
6132
|
const mutableIds = new Set(batchResult.batch.map((f) => f.id));
|
|
5909
6133
|
for (const id of batchResult.downgraded) if (mutableIds.has(id)) safeDowngradedSet.add(id);
|
|
@@ -5911,6 +6135,7 @@ var MaintenanceService = class {
|
|
|
5911
6135
|
batchResult.newFacts.forEach((raw, itemIndex) => {
|
|
5912
6136
|
newFacts.push(raw);
|
|
5913
6137
|
newFactItemIndexes.push(itemIndex);
|
|
6138
|
+
newFactCorpora.push(batchResult.corpus);
|
|
5914
6139
|
});
|
|
5915
6140
|
}
|
|
5916
6141
|
const safeDowngraded = Array.from(safeDowngradedSet);
|
|
@@ -5918,12 +6143,14 @@ var MaintenanceService = class {
|
|
|
5918
6143
|
const diagBuffer = new DiagnosticBuffer();
|
|
5919
6144
|
const validNewFacts = [];
|
|
5920
6145
|
const validNewFactItemIndexes = [];
|
|
6146
|
+
const validNewFactCorpora = [];
|
|
5921
6147
|
newFacts.forEach((raw, k) => {
|
|
5922
6148
|
const itemIndex = newFactItemIndexes[k];
|
|
5923
6149
|
const valid = validateFact(raw);
|
|
5924
6150
|
if (valid) {
|
|
5925
6151
|
validNewFacts.push(valid);
|
|
5926
6152
|
validNewFactItemIndexes.push(itemIndex);
|
|
6153
|
+
validNewFactCorpora.push(newFactCorpora[k]);
|
|
5927
6154
|
} else {
|
|
5928
6155
|
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
5929
6156
|
}
|
|
@@ -5953,6 +6180,10 @@ var MaintenanceService = class {
|
|
|
5953
6180
|
continue;
|
|
5954
6181
|
}
|
|
5955
6182
|
const id = generateId("fact_");
|
|
6183
|
+
const grounding = healGrounding ? groundingOutcome(checkGrounding(fact.evidence, validNewFactCorpora[k] ?? [], healGrounding), now) : null;
|
|
6184
|
+
if (grounding?.diagnostic) {
|
|
6185
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validNewFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
6186
|
+
}
|
|
5956
6187
|
const factObj = {
|
|
5957
6188
|
id,
|
|
5958
6189
|
entity_id: entityId,
|
|
@@ -5967,7 +6198,8 @@ var MaintenanceService = class {
|
|
|
5967
6198
|
updated_at: now,
|
|
5968
6199
|
last_accessed_at: null,
|
|
5969
6200
|
access_count: 0,
|
|
5970
|
-
deleted_at: null
|
|
6201
|
+
deleted_at: null,
|
|
6202
|
+
...grounding?.trust
|
|
5971
6203
|
};
|
|
5972
6204
|
await this.entryRepo.upsert(factObj, tx);
|
|
5973
6205
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
@@ -6331,7 +6563,7 @@ var MaintenanceService = class {
|
|
|
6331
6563
|
* batches that reduce to the same query share one lookup. Caller-owned and
|
|
6332
6564
|
* per-pass — see the call site in doRunHeal.
|
|
6333
6565
|
*/
|
|
6334
|
-
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache) {
|
|
6566
|
+
async _selectHealAnchors(entityId, batch, cap = HEAL_MAX_ANCHORS, cache, withBodies = false) {
|
|
6335
6567
|
const query = batch.map((f) => f.title).join(" ").trim();
|
|
6336
6568
|
if (!query) return [];
|
|
6337
6569
|
const cached = cache?.get(query);
|
|
@@ -6344,7 +6576,7 @@ var MaintenanceService = class {
|
|
|
6344
6576
|
const hitIds = hits.map((h) => h.id);
|
|
6345
6577
|
const anchors = [];
|
|
6346
6578
|
if (hitIds.length > 0) {
|
|
6347
|
-
const rows = await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
6579
|
+
const rows = withBodies ? await this.entryRepo.findAnchorRowsByIds(entityId, hitIds, void 0, { withBody: true }) : await this.entryRepo.findAnchorRowsByIds(entityId, hitIds);
|
|
6348
6580
|
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
6349
6581
|
for (const id of hitIds) {
|
|
6350
6582
|
const row = byId.get(id);
|
|
@@ -8081,6 +8313,105 @@ var GraphTraversalService = class {
|
|
|
8081
8313
|
return { nodes, edges: filteredEdges };
|
|
8082
8314
|
}
|
|
8083
8315
|
};
|
|
8316
|
+
|
|
8317
|
+
// src/repositories/LintRepository.ts
|
|
8318
|
+
var LintRepository = class extends BaseRepository {
|
|
8319
|
+
async countFactHealth(entityId) {
|
|
8320
|
+
const row = await this.db.getFirstAsync(
|
|
8321
|
+
`SELECT
|
|
8322
|
+
SUM(CASE WHEN okf_type IS NULL THEN 1 ELSE 0 END) AS untyped,
|
|
8323
|
+
SUM(CASE WHEN lifecycle_status = 'draft' THEN 1 ELSE 0 END) AS drafts,
|
|
8324
|
+
SUM(CASE WHEN source_type = 'librarian_inferred'
|
|
8325
|
+
AND COALESCE(json_array_length(CASE WHEN json_valid(okf_verified) THEN okf_verified END), 0) = 0
|
|
8326
|
+
THEN 1 ELSE 0 END) AS unverified
|
|
8327
|
+
FROM ${this.prefix}entries
|
|
8328
|
+
WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
8329
|
+
[entityId]
|
|
8330
|
+
);
|
|
8331
|
+
return {
|
|
8332
|
+
untypedFacts: Number(row?.untyped ?? 0),
|
|
8333
|
+
drafts: Number(row?.drafts ?? 0),
|
|
8334
|
+
unverifiedInferred: Number(row?.unverified ?? 0)
|
|
8335
|
+
};
|
|
8336
|
+
}
|
|
8337
|
+
danglingWhere() {
|
|
8338
|
+
const live = (col) => `EXISTS (SELECT 1 FROM ${this.prefix}entries n WHERE n.id = e.${col} AND n.entity_id = e.entity_id AND n.deleted_at IS NULL)`;
|
|
8339
|
+
return `e.entity_id = ? AND (NOT ${live("source_id")} OR NOT ${live("target_id")})`;
|
|
8340
|
+
}
|
|
8341
|
+
async countDanglingEdges(entityId) {
|
|
8342
|
+
const row = await this.db.getFirstAsync(
|
|
8343
|
+
`SELECT COUNT(*) AS n FROM ${this.prefix}edges e WHERE ${this.danglingWhere()}`,
|
|
8344
|
+
[entityId]
|
|
8345
|
+
);
|
|
8346
|
+
return Number(row?.n ?? 0);
|
|
8347
|
+
}
|
|
8348
|
+
async sampleDanglingEdgeIds(entityId, limit) {
|
|
8349
|
+
const rows = await this.db.getAllAsync(
|
|
8350
|
+
`SELECT e.id FROM ${this.prefix}edges e WHERE ${this.danglingWhere()} ORDER BY e.id ASC LIMIT ?`,
|
|
8351
|
+
[entityId, limit]
|
|
8352
|
+
);
|
|
8353
|
+
return rows.map((r) => r.id);
|
|
8354
|
+
}
|
|
8355
|
+
/** Keyset page of edges whose endpoints are both live in this entity, with endpoint types. */
|
|
8356
|
+
async pageLiveEdges(entityId, afterId, limit) {
|
|
8357
|
+
return this.db.getAllAsync(
|
|
8358
|
+
`SELECT e.id, e.edge_type, s.okf_type AS source_type, t.okf_type AS target_type
|
|
8359
|
+
FROM ${this.prefix}edges e
|
|
8360
|
+
JOIN ${this.prefix}entries s ON s.id = e.source_id AND s.entity_id = e.entity_id AND s.deleted_at IS NULL
|
|
8361
|
+
JOIN ${this.prefix}entries t ON t.id = e.target_id AND t.entity_id = e.entity_id AND t.deleted_at IS NULL
|
|
8362
|
+
WHERE e.entity_id = ? AND e.id > ?
|
|
8363
|
+
ORDER BY e.id ASC
|
|
8364
|
+
LIMIT ?`,
|
|
8365
|
+
[entityId, afterId, limit]
|
|
8366
|
+
);
|
|
8367
|
+
}
|
|
8368
|
+
};
|
|
8369
|
+
|
|
8370
|
+
// src/services/LintService.ts
|
|
8371
|
+
var LINT_SAMPLE_SIZE = 20;
|
|
8372
|
+
var LINT_PAGE_SIZE = 500;
|
|
8373
|
+
var LintService = class {
|
|
8374
|
+
constructor(lintRepo, ontologyService) {
|
|
8375
|
+
this.lintRepo = lintRepo;
|
|
8376
|
+
this.ontologyService = ontologyService;
|
|
8377
|
+
}
|
|
8378
|
+
/**
|
|
8379
|
+
* Read-only (spec §8.1). Manifest violations stream the entity's live-endpoint
|
|
8380
|
+
* edges in keyset pages and check each triple in JS against the effective
|
|
8381
|
+
* manifest. `getEffectiveState` is called without a transaction, so a seed
|
|
8382
|
+
* manifest is cached, never persisted. Counts come from separate statements,
|
|
8383
|
+
* not one snapshot.
|
|
8384
|
+
*/
|
|
8385
|
+
async lint(entityId) {
|
|
8386
|
+
const health = await this.lintRepo.countFactHealth(entityId);
|
|
8387
|
+
const danglingEdges = await this.lintRepo.countDanglingEdges(entityId);
|
|
8388
|
+
const danglingEdgeIds = danglingEdges > 0 ? await this.lintRepo.sampleDanglingEdgeIds(entityId, LINT_SAMPLE_SIZE) : [];
|
|
8389
|
+
const { mode, manifest } = await this.ontologyService.getEffectiveState(entityId);
|
|
8390
|
+
const constrained = mode !== "off" && ((manifest.node_types?.length ?? 0) > 0 || (manifest.edge_types?.length ?? 0) > 0);
|
|
8391
|
+
let manifestViolations = 0;
|
|
8392
|
+
const manifestViolationEdgeIds = [];
|
|
8393
|
+
if (constrained) {
|
|
8394
|
+
let after = "";
|
|
8395
|
+
for (; ; ) {
|
|
8396
|
+
const page = await this.lintRepo.pageLiveEdges(entityId, after, LINT_PAGE_SIZE);
|
|
8397
|
+
for (const e of page) {
|
|
8398
|
+
if (!edgeTripleAllowed(manifest, e.edge_type, e.source_type ?? "", e.target_type ?? "")) {
|
|
8399
|
+
manifestViolations++;
|
|
8400
|
+
if (manifestViolationEdgeIds.length < LINT_SAMPLE_SIZE) manifestViolationEdgeIds.push(e.id);
|
|
8401
|
+
}
|
|
8402
|
+
}
|
|
8403
|
+
if (page.length < LINT_PAGE_SIZE) break;
|
|
8404
|
+
after = page[page.length - 1].id;
|
|
8405
|
+
}
|
|
8406
|
+
}
|
|
8407
|
+
return {
|
|
8408
|
+
danglingEdges,
|
|
8409
|
+
manifestViolations,
|
|
8410
|
+
...health,
|
|
8411
|
+
sample: { danglingEdgeIds, manifestViolationEdgeIds }
|
|
8412
|
+
};
|
|
8413
|
+
}
|
|
8414
|
+
};
|
|
8084
8415
|
var OkfTrustWritesRepository = class {
|
|
8085
8416
|
constructor(db, prefix) {
|
|
8086
8417
|
this.db = db;
|
|
@@ -8271,15 +8602,17 @@ var WikiMemory = class {
|
|
|
8271
8602
|
this.eventRepo = new EventRepository(this.db, this.prefix);
|
|
8272
8603
|
this.edgeRepo = new EdgeRepository(this.db, this.prefix);
|
|
8273
8604
|
this.metadataRepo = new MetadataRepository(this.db, this.prefix);
|
|
8605
|
+
this.lintRepo = new LintRepository(this.db, this.prefix);
|
|
8274
8606
|
this.ontologyService = new OntologyService(
|
|
8275
8607
|
this.metadataRepo,
|
|
8276
8608
|
this.edgeRepo,
|
|
8277
8609
|
options.config?.ontology
|
|
8278
8610
|
);
|
|
8611
|
+
this.lintService = new LintService(this.lintRepo, this.ontologyService);
|
|
8279
8612
|
this.embeddingService = new EmbeddingService(this.db, this.options, this.entryRepo, this.metadataRepo);
|
|
8280
8613
|
this.searchService = new SearchService(this.entryRepo);
|
|
8281
8614
|
this.jobManager = new JobManager(this.prefix);
|
|
8282
|
-
this.promptService = new PromptService(options.config?.prompts);
|
|
8615
|
+
this.promptService = new PromptService(options.config?.prompts, resolveGrounding(options.config?.grounding));
|
|
8283
8616
|
this.ingestionService = new IngestionService(
|
|
8284
8617
|
this.db,
|
|
8285
8618
|
this.prefix,
|
|
@@ -8458,6 +8791,61 @@ var WikiMemory = class {
|
|
|
8458
8791
|
return { sourceRef: e.rawSourceRef, changed, duplicateOf: canonical };
|
|
8459
8792
|
});
|
|
8460
8793
|
}
|
|
8794
|
+
/**
|
|
8795
|
+
* Batch pending state per source (spec §8.2). `current` exactly when
|
|
8796
|
+
* `hasChanged` is false; `partial` when live rows exist but none carries a
|
|
8797
|
+
* hash (a first ingest with a failed chunk, or imported rows without a
|
|
8798
|
+
* hash). A failed re-ingest of an already-hashed ref reports `changed`. Validation and raw-ref echo match
|
|
8799
|
+
* batched `hasChanged`. Input order and duplicates are preserved. A
|
|
8800
|
+
* non-array `sources`, or an entry that is not an object (including a hole
|
|
8801
|
+
* in a sparse array), throws a `TypeError` before any SQL runs.
|
|
8802
|
+
*/
|
|
8803
|
+
async pendingSources(entityId, sources) {
|
|
8804
|
+
assertEntityId(entityId);
|
|
8805
|
+
if (!Array.isArray(sources)) {
|
|
8806
|
+
throw new TypeError("Invalid sources: must be an array of { sourceRef, sourceHash } objects.");
|
|
8807
|
+
}
|
|
8808
|
+
if (sources.length === 0) return [];
|
|
8809
|
+
const normalized = Array.from(sources, (s, i) => {
|
|
8810
|
+
if (s === null || typeof s !== "object") {
|
|
8811
|
+
throw new TypeError(`Invalid sources[${i}]: must be a { sourceRef, sourceHash } object.`);
|
|
8812
|
+
}
|
|
8813
|
+
const { sourceRef: rawSourceRef, sourceHash: rawSourceHash } = s;
|
|
8814
|
+
const sourceRef = normalizeSourceRef(rawSourceRef);
|
|
8815
|
+
if (!sourceRef) {
|
|
8816
|
+
const shown = typeof rawSourceRef === "string" ? JSON.stringify(rawSourceRef) : `<${typeof rawSourceRef}>`;
|
|
8817
|
+
throw new Error(`Invalid sourceRef: ${shown}`);
|
|
8818
|
+
}
|
|
8819
|
+
const sourceHash = normalizeSourceHash(rawSourceHash);
|
|
8820
|
+
if (!sourceHash) throw new Error("Invalid sourceHash: must be a 64-character hex string (normalized to lowercase)");
|
|
8821
|
+
return { rawSourceRef, sourceRef, sourceHash };
|
|
8822
|
+
});
|
|
8823
|
+
const states = await this.entryRepo.findSourceStates(entityId, normalized.map((n) => n.sourceRef));
|
|
8824
|
+
return normalized.map((n) => {
|
|
8825
|
+
const state = states.get(n.sourceRef);
|
|
8826
|
+
let status;
|
|
8827
|
+
if (!state) status = "new";
|
|
8828
|
+
else if (!state.anyHashed) status = "partial";
|
|
8829
|
+
else if (state.latestHash !== null && normalizeSourceHash(state.latestHash) === n.sourceHash) status = "current";
|
|
8830
|
+
else status = "changed";
|
|
8831
|
+
return { sourceRef: n.rawSourceRef, status };
|
|
8832
|
+
});
|
|
8833
|
+
}
|
|
8834
|
+
/** Read-only maintenance report for one entity (spec §8.1). Reports, never repairs. */
|
|
8835
|
+
async lint(entityId) {
|
|
8836
|
+
assertEntityId(entityId);
|
|
8837
|
+
return this.lintService.lint(entityId);
|
|
8838
|
+
}
|
|
8839
|
+
/**
|
|
8840
|
+
* Effective system prompts for ingest, librarian, heal and ontology backfill
|
|
8841
|
+
* (spec §8.3). Backs the `wiki_get_instructions` tool. Templates only;
|
|
8842
|
+
* overrides are returned verbatim, so keep secrets out of `WikiConfig.prompts`.
|
|
8843
|
+
*/
|
|
8844
|
+
async getInstructions(entityId) {
|
|
8845
|
+
assertEntityId(entityId);
|
|
8846
|
+
const ontologyContext = await this.ontologyService.buildPromptContext(entityId);
|
|
8847
|
+
return this.promptService.buildInstructionTemplates(ontologyContext);
|
|
8848
|
+
}
|
|
8461
8849
|
/**
|
|
8462
8850
|
* Returns the live source_refs for an entity, one row per ref, with the most
|
|
8463
8851
|
* recently-updated live `source_hash` and a live fact count. Refs are sorted
|
|
@@ -8858,6 +9246,11 @@ var WikiMemory = class {
|
|
|
8858
9246
|
}
|
|
8859
9247
|
};
|
|
8860
9248
|
_testAccessNonTestEnvWarned = new WeakMap();
|
|
9249
|
+
function assertEntityId(entityId) {
|
|
9250
|
+
if (typeof entityId !== "string" || entityId.length === 0) {
|
|
9251
|
+
throw new TypeError("Invalid entityId: must be a non-empty string.");
|
|
9252
|
+
}
|
|
9253
|
+
}
|
|
8861
9254
|
|
|
8862
9255
|
// src/utils/formatContext.ts
|
|
8863
9256
|
function validateMaxOption(value, name) {
|