@equationalapplications/core-llm-wiki 7.7.6 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-PBUZLSOM.mjs → chunk-M76CF2Q6.mjs} +1193 -189
- package/dist/chunk-M76CF2Q6.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1837 -289
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +575 -32
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BIFNLUva.d.mts → testing-CWZaXKLj.d.mts} +560 -4
- package/dist/{testing-BIFNLUva.d.ts → testing-CWZaXKLj.d.ts} +560 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1142 -201
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-PBUZLSOM.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -153,6 +153,15 @@ var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
|
153
153
|
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
154
154
|
}
|
|
155
155
|
};
|
|
156
|
+
var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
157
|
+
constructor(reason) {
|
|
158
|
+
super(`supersede() rejected: ${reason}`);
|
|
159
|
+
this.code = "WIKI_SUPERSEDE_REJECTED";
|
|
160
|
+
this.name = "WikiSupersedeError";
|
|
161
|
+
this.reason = reason;
|
|
162
|
+
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
156
165
|
|
|
157
166
|
// src/utils/pure.ts
|
|
158
167
|
var MAX_REPAIR_CANDIDATES = 5;
|
|
@@ -686,7 +695,14 @@ var SEVERITY = {
|
|
|
686
695
|
grounding_missing: "warn",
|
|
687
696
|
grounding_failed: "warn",
|
|
688
697
|
classification_low_confidence: "info",
|
|
689
|
-
classification_invalid: "warn"
|
|
698
|
+
classification_invalid: "warn",
|
|
699
|
+
llm_usage: "info",
|
|
700
|
+
librarian_gate: "info",
|
|
701
|
+
librarian_op_rejected: "warn",
|
|
702
|
+
contradicts_document: "warn",
|
|
703
|
+
resolve_failed: "warn",
|
|
704
|
+
read_budget: "info",
|
|
705
|
+
event_retention_held: "info"
|
|
690
706
|
};
|
|
691
707
|
var MESSAGE = {
|
|
692
708
|
ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
|
|
@@ -701,7 +717,14 @@ var MESSAGE = {
|
|
|
701
717
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
702
718
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
703
719
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
704
|
-
classification_invalid: "A classifier answer was invalid and was not applied."
|
|
720
|
+
classification_invalid: "A classifier answer was invalid and was not applied.",
|
|
721
|
+
llm_usage: "An LLM call completed; token usage attached.",
|
|
722
|
+
librarian_gate: "Librarian gate outcome counts for one pass.",
|
|
723
|
+
librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
|
|
724
|
+
contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
|
|
725
|
+
resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
|
|
726
|
+
read_budget: "A read was packed to a token budget.",
|
|
727
|
+
event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
|
|
705
728
|
};
|
|
706
729
|
var warnedNonFunction = /* @__PURE__ */ new WeakSet();
|
|
707
730
|
function copyDetail(detail) {
|
|
@@ -851,6 +874,21 @@ var EmbeddingService = class {
|
|
|
851
874
|
}
|
|
852
875
|
}
|
|
853
876
|
async tryEmbedFact(fact, ctx) {
|
|
877
|
+
const r = await this.embedTextForFact(fact, ctx);
|
|
878
|
+
return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
|
|
879
|
+
}
|
|
880
|
+
/**
|
|
881
|
+
* Compute half: build the embed text, call embed(), validate the result,
|
|
882
|
+
* mark any embed-time failure, and return the validated Float32 vector.
|
|
883
|
+
*
|
|
884
|
+
* Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
|
|
885
|
+
* `provider_error`. All embed-time failures are marked via `markFailure`;
|
|
886
|
+
* `no_provider` deliberately does not mark (spec §2.4).
|
|
887
|
+
*
|
|
888
|
+
* Storage errors are NOT possible here — that is `storeFactVector`'s domain.
|
|
889
|
+
*/
|
|
890
|
+
async embedTextForFact(fact, ctx, opts) {
|
|
891
|
+
const markFailures = opts?.markFailures !== false;
|
|
854
892
|
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
855
893
|
let tagsStr;
|
|
856
894
|
if (Array.isArray(fact.tags)) {
|
|
@@ -871,8 +909,8 @@ var EmbeddingService = class {
|
|
|
871
909
|
const vector = await this.options.llmProvider.embed(text);
|
|
872
910
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
873
911
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
874
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
875
|
-
await this.markFailure(fact.id, "invalid_vector");
|
|
912
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
913
|
+
if (markFailures) await this.markFailure(fact.id, "invalid_vector");
|
|
876
914
|
return { ok: false, kind: "invalid_vector" };
|
|
877
915
|
}
|
|
878
916
|
float32Vector = new Float32Array(vector);
|
|
@@ -885,16 +923,24 @@ var EmbeddingService = class {
|
|
|
885
923
|
}
|
|
886
924
|
if (hasNonFinite) {
|
|
887
925
|
console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
|
|
888
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
889
|
-
await this.markFailure(fact.id, "float32_overflow");
|
|
926
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
927
|
+
if (markFailures) await this.markFailure(fact.id, "float32_overflow");
|
|
890
928
|
return { ok: false, kind: "float32_overflow" };
|
|
891
929
|
}
|
|
892
930
|
} catch (err) {
|
|
893
931
|
console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
|
|
894
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
895
|
-
await this.markFailure(fact.id, "provider_error");
|
|
932
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
933
|
+
if (markFailures) await this.markFailure(fact.id, "provider_error");
|
|
896
934
|
return { ok: false, kind: "provider_error" };
|
|
897
935
|
}
|
|
936
|
+
return { ok: true, vector: float32Vector };
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Storage half: persist the computed vector to the entry row, then fire the
|
|
940
|
+
* `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
|
|
941
|
+
* failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
|
|
942
|
+
*/
|
|
943
|
+
async storeFactVector(fact, float32Vector, ctx) {
|
|
898
944
|
try {
|
|
899
945
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
900
946
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
@@ -1246,6 +1292,12 @@ var ImportExportService = class {
|
|
|
1246
1292
|
})()
|
|
1247
1293
|
};
|
|
1248
1294
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
1295
|
+
await this.entryRepo.setTemporal(fact.id, entityId, {
|
|
1296
|
+
valid_from: fact.valid_from ?? null,
|
|
1297
|
+
valid_to: fact.valid_to ?? null,
|
|
1298
|
+
superseded_by: fact.superseded_by ?? null,
|
|
1299
|
+
superseded_at: fact.superseded_at ?? null
|
|
1300
|
+
}, tx, { outbox: false });
|
|
1249
1301
|
if (blobData != null) {
|
|
1250
1302
|
factsWithPreservedBlob.set(fact.id, blobData);
|
|
1251
1303
|
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
@@ -1324,7 +1376,8 @@ var ImportExportService = class {
|
|
|
1324
1376
|
event_type: event.event_type,
|
|
1325
1377
|
summary: event.summary,
|
|
1326
1378
|
related_entry_id: event.related_entry_id ?? null,
|
|
1327
|
-
created_at: event.created_at
|
|
1379
|
+
created_at: event.created_at,
|
|
1380
|
+
occurred_at: event.occurred_at ?? null
|
|
1328
1381
|
},
|
|
1329
1382
|
tx
|
|
1330
1383
|
);
|
|
@@ -1557,6 +1610,21 @@ function groundingEvidenceBlock(writer, cfg) {
|
|
|
1557
1610
|
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1558
1611
|
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1559
1612
|
}
|
|
1613
|
+
var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
|
|
1614
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1615
|
+
{
|
|
1616
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
|
|
1617
|
+
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
1618
|
+
}
|
|
1619
|
+
One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
|
|
1620
|
+
var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
|
|
1621
|
+
- "ADD": the candidate is new information; the existing facts are about something else.
|
|
1622
|
+
- "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
|
|
1623
|
+
- "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
|
|
1624
|
+
- "NOOP": an existing fact already states the candidate.
|
|
1625
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1626
|
+
{ "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
|
|
1627
|
+
"target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
|
|
1560
1628
|
|
|
1561
1629
|
// src/utils/healConstants.ts
|
|
1562
1630
|
var HEAL_MAX_ANCHORS = 50;
|
|
@@ -1795,6 +1863,22 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
1795
1863
|
${JSON.stringify(facts, null, 2)}`
|
|
1796
1864
|
};
|
|
1797
1865
|
}
|
|
1866
|
+
/** Ops extract (spec §5.2): events only — no existing facts. */
|
|
1867
|
+
buildOpsExtractPrompt(events, ontologyContext) {
|
|
1868
|
+
const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
|
|
1869
|
+
const grounding = this.groundingFor("librarian");
|
|
1870
|
+
return {
|
|
1871
|
+
systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
|
|
1872
|
+
userPrompt: `Events:
|
|
1873
|
+
${lines.join("\n")}`,
|
|
1874
|
+
...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
/** Ops resolve (spec §5.4): only each item's own neighbours. */
|
|
1878
|
+
buildOpsResolvePrompt(items) {
|
|
1879
|
+
return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
|
|
1880
|
+
${JSON.stringify({ items })}` };
|
|
1881
|
+
}
|
|
1798
1882
|
/**
|
|
1799
1883
|
* The system prompt each writer would send, without hydrating data (spec
|
|
1800
1884
|
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
@@ -1859,6 +1943,59 @@ function toGroundingAnchor(anchor) {
|
|
|
1859
1943
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
1860
1944
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
1861
1945
|
|
|
1946
|
+
// src/utils/usage.ts
|
|
1947
|
+
function estimateTokens(text) {
|
|
1948
|
+
return Math.ceil(text.length / 4);
|
|
1949
|
+
}
|
|
1950
|
+
var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
|
|
1951
|
+
constructor(requiredEstimate, remaining) {
|
|
1952
|
+
super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
|
|
1953
|
+
this.name = "WikiBudgetExhausted";
|
|
1954
|
+
this.requiredEstimate = requiredEstimate;
|
|
1955
|
+
this.remaining = remaining;
|
|
1956
|
+
Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
|
|
1957
|
+
}
|
|
1958
|
+
};
|
|
1959
|
+
|
|
1960
|
+
// src/utils/llmCall.ts
|
|
1961
|
+
function validUsage(u) {
|
|
1962
|
+
if (!u || typeof u !== "object") return false;
|
|
1963
|
+
const { inputTokens, outputTokens } = u;
|
|
1964
|
+
return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
|
|
1965
|
+
}
|
|
1966
|
+
async function callLlm(options, ctx, prompts) {
|
|
1967
|
+
const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
|
|
1968
|
+
if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
|
|
1969
|
+
throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
|
|
1970
|
+
}
|
|
1971
|
+
const provider = options.llmProvider;
|
|
1972
|
+
const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
|
|
1973
|
+
let text;
|
|
1974
|
+
let reported;
|
|
1975
|
+
if (typeof provider.generateTextWithUsage === "function") {
|
|
1976
|
+
const result = await provider.generateTextWithUsage(params);
|
|
1977
|
+
if (!result || typeof result.text !== "string") {
|
|
1978
|
+
throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
|
|
1979
|
+
}
|
|
1980
|
+
text = result.text;
|
|
1981
|
+
reported = validUsage(result.usage) ? result.usage : void 0;
|
|
1982
|
+
} else {
|
|
1983
|
+
text = await provider.generateText(params);
|
|
1984
|
+
}
|
|
1985
|
+
const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
|
|
1986
|
+
ctx.meter?.record(record);
|
|
1987
|
+
if (options.config?.reportLlmUsage === true) {
|
|
1988
|
+
emitDiagnostic(options, {
|
|
1989
|
+
code: "llm_usage",
|
|
1990
|
+
operation: ctx.operation,
|
|
1991
|
+
trigger: ctx.trigger,
|
|
1992
|
+
entityId: ctx.entityId,
|
|
1993
|
+
detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
return text;
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1862
1999
|
// src/services/IngestionService.ts
|
|
1863
2000
|
var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
|
|
1864
2001
|
function dedupDiagnostic(entityId, sourceRef, locator) {
|
|
@@ -1935,7 +2072,7 @@ var IngestionService = class {
|
|
|
1935
2072
|
ontologyContext
|
|
1936
2073
|
);
|
|
1937
2074
|
try {
|
|
1938
|
-
const responseText = await this.options
|
|
2075
|
+
const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
|
|
1939
2076
|
const result2 = parseJsonResponse(responseText);
|
|
1940
2077
|
const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
|
|
1941
2078
|
const facts = [];
|
|
@@ -2619,6 +2756,8 @@ async function runBatched(args) {
|
|
|
2619
2756
|
const results = [];
|
|
2620
2757
|
const skipped = [];
|
|
2621
2758
|
let batches = 0;
|
|
2759
|
+
const settled = /* @__PURE__ */ new Set();
|
|
2760
|
+
let budgetStop;
|
|
2622
2761
|
let batchSize = initialBatchSize(maxOutputTokens);
|
|
2623
2762
|
const trim = async (candidate) => {
|
|
2624
2763
|
const whole = await buildPrompt(candidate);
|
|
@@ -2652,6 +2791,7 @@ async function runBatched(args) {
|
|
|
2652
2791
|
} else {
|
|
2653
2792
|
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
2654
2793
|
skipped.push({ item: batch[0], reason });
|
|
2794
|
+
settled.add(batch[0]);
|
|
2655
2795
|
onSkip?.(batch[0], err);
|
|
2656
2796
|
}
|
|
2657
2797
|
return;
|
|
@@ -2660,7 +2800,7 @@ async function runBatched(args) {
|
|
|
2660
2800
|
const mid = Math.ceil(batch.length / 2);
|
|
2661
2801
|
if (mid < batchSize) batchSize = mid;
|
|
2662
2802
|
let i = 0;
|
|
2663
|
-
while (i < batch.length) {
|
|
2803
|
+
while (i < batch.length && !budgetStop) {
|
|
2664
2804
|
const size = Math.min(batchSize, batch.length - i);
|
|
2665
2805
|
const trimmed = await trim(batch.slice(i, i + size));
|
|
2666
2806
|
await attempt(trimmed.batch, trimmed.prompts);
|
|
@@ -2668,6 +2808,7 @@ async function runBatched(args) {
|
|
|
2668
2808
|
}
|
|
2669
2809
|
};
|
|
2670
2810
|
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
2811
|
+
if (budgetStop) return;
|
|
2671
2812
|
if (batch.length === 0) return;
|
|
2672
2813
|
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
2673
2814
|
batches++;
|
|
@@ -2675,6 +2816,10 @@ async function runBatched(args) {
|
|
|
2675
2816
|
try {
|
|
2676
2817
|
responseText = await call(prompts);
|
|
2677
2818
|
} catch (err) {
|
|
2819
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
2820
|
+
budgetStop = { requiredEstimate: err.requiredEstimate };
|
|
2821
|
+
return;
|
|
2822
|
+
}
|
|
2678
2823
|
await onFailure(batch, err, attemptLevel, true);
|
|
2679
2824
|
return;
|
|
2680
2825
|
}
|
|
@@ -2686,19 +2831,784 @@ async function runBatched(args) {
|
|
|
2686
2831
|
return;
|
|
2687
2832
|
}
|
|
2688
2833
|
results.push(result);
|
|
2834
|
+
for (const item of batch) settled.add(item);
|
|
2689
2835
|
};
|
|
2690
2836
|
let index = 0;
|
|
2691
|
-
while (index < items.length) {
|
|
2837
|
+
while (index < items.length && !budgetStop) {
|
|
2692
2838
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
2693
2839
|
index += batch.length;
|
|
2694
2840
|
await attempt(batch, prompts, 0);
|
|
2695
2841
|
}
|
|
2696
|
-
|
|
2842
|
+
const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
|
|
2843
|
+
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
2697
2844
|
}
|
|
2698
2845
|
|
|
2699
|
-
// src/services/
|
|
2846
|
+
// src/services/librarian/constants.ts
|
|
2700
2847
|
var FUZZY_THRESHOLD = 0.5;
|
|
2701
2848
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
2849
|
+
|
|
2850
|
+
// src/services/librarian/legacy.ts
|
|
2851
|
+
var runLegacyLibrarianPass = async (deps, ctx) => {
|
|
2852
|
+
const { entityId, trigger } = ctx;
|
|
2853
|
+
const promptOverride = ctx.promptOverride;
|
|
2854
|
+
const events = await deps.eventRepo.getRecent(entityId, 50);
|
|
2855
|
+
const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
|
|
2856
|
+
const currentFacts = currentFactsRows.map((f) => {
|
|
2857
|
+
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
2858
|
+
return {
|
|
2859
|
+
...rest,
|
|
2860
|
+
tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
|
|
2861
|
+
};
|
|
2862
|
+
});
|
|
2863
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
2864
|
+
const promptEvents = events.reverse();
|
|
2865
|
+
const librarianGrounding = deps.promptService.groundingFor("librarian");
|
|
2866
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
|
|
2867
|
+
promptEvents,
|
|
2868
|
+
currentFacts,
|
|
2869
|
+
promptOverride,
|
|
2870
|
+
ontologyContext
|
|
2871
|
+
);
|
|
2872
|
+
let responseText;
|
|
2873
|
+
try {
|
|
2874
|
+
responseText = await callLlm(
|
|
2875
|
+
deps.options,
|
|
2876
|
+
{ operation: "librarian", entityId, trigger, meter: ctx.meter },
|
|
2877
|
+
{ systemPrompt, userPrompt }
|
|
2878
|
+
);
|
|
2879
|
+
} catch (err) {
|
|
2880
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
2881
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
2882
|
+
}
|
|
2883
|
+
throw err;
|
|
2884
|
+
}
|
|
2885
|
+
const result = parseJsonResponse(responseText);
|
|
2886
|
+
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
2887
|
+
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
2888
|
+
const ontologyUpdates = result.ontology_updates;
|
|
2889
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
2890
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
2891
|
+
const validFacts = [];
|
|
2892
|
+
const validFactItemIndexes = [];
|
|
2893
|
+
facts.forEach((raw, itemIndex) => {
|
|
2894
|
+
const valid = validateFact(raw);
|
|
2895
|
+
if (valid) {
|
|
2896
|
+
validFacts.push(valid);
|
|
2897
|
+
validFactItemIndexes.push(itemIndex);
|
|
2898
|
+
} else {
|
|
2899
|
+
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
2900
|
+
}
|
|
2901
|
+
});
|
|
2902
|
+
const validTasks = [];
|
|
2903
|
+
tasks.forEach((raw, itemIndex) => {
|
|
2904
|
+
const valid = validateTask(raw);
|
|
2905
|
+
if (valid) validTasks.push(valid);
|
|
2906
|
+
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
2907
|
+
});
|
|
2908
|
+
const now = Date.now();
|
|
2909
|
+
const insertedFacts = [];
|
|
2910
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
2911
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
2912
|
+
if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
|
|
2913
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
2914
|
+
}
|
|
2915
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
2916
|
+
for (const existing of currentFactsRows) {
|
|
2917
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
2918
|
+
id: existing.id,
|
|
2919
|
+
okf_type: existing.okf_type ?? null
|
|
2920
|
+
});
|
|
2921
|
+
}
|
|
2922
|
+
const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
2923
|
+
const pendingEdges = [];
|
|
2924
|
+
for (const [k, fact] of validFacts.entries()) {
|
|
2925
|
+
const newTokens = titleTokens(fact.title);
|
|
2926
|
+
let skip = false;
|
|
2927
|
+
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
2928
|
+
for (const existing of factsForDedupe) {
|
|
2929
|
+
if (existing.source_type !== "librarian_inferred") continue;
|
|
2930
|
+
const existingTokens = titleTokens(existing.title);
|
|
2931
|
+
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
2932
|
+
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
2933
|
+
skip = true;
|
|
2934
|
+
break;
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
if (skip) {
|
|
2940
|
+
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
2941
|
+
continue;
|
|
2942
|
+
}
|
|
2943
|
+
const ontologyFact = fact;
|
|
2944
|
+
const id = generateId("fact_");
|
|
2945
|
+
const validationDrops = [];
|
|
2946
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
2947
|
+
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
2948
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
2949
|
+
if (grounding?.diagnostic) {
|
|
2950
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
2951
|
+
}
|
|
2952
|
+
const factObj = {
|
|
2953
|
+
id,
|
|
2954
|
+
entity_id: entityId,
|
|
2955
|
+
title: fact.title,
|
|
2956
|
+
body: fact.body,
|
|
2957
|
+
tags: fact.tags,
|
|
2958
|
+
confidence: fact.confidence,
|
|
2959
|
+
source_type: "librarian_inferred",
|
|
2960
|
+
source_hash: null,
|
|
2961
|
+
source_ref: null,
|
|
2962
|
+
created_at: now,
|
|
2963
|
+
updated_at: now,
|
|
2964
|
+
last_accessed_at: null,
|
|
2965
|
+
access_count: 0,
|
|
2966
|
+
deleted_at: null,
|
|
2967
|
+
okf_type: normalized.okf_type,
|
|
2968
|
+
...grounding?.trust
|
|
2969
|
+
};
|
|
2970
|
+
await deps.entryRepo.upsert(factObj, tx);
|
|
2971
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2972
|
+
factsForDedupe.push(factObj);
|
|
2973
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
2974
|
+
if (normalized.edges.length > 0) {
|
|
2975
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
for (const item of pendingEdges) {
|
|
2979
|
+
const resolveDrops = [];
|
|
2980
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
2981
|
+
entityId,
|
|
2982
|
+
item.sourceId,
|
|
2983
|
+
item.sourceType,
|
|
2984
|
+
item.edges ?? [],
|
|
2985
|
+
manifest,
|
|
2986
|
+
titleIndex,
|
|
2987
|
+
tx,
|
|
2988
|
+
now,
|
|
2989
|
+
resolveDrops
|
|
2990
|
+
);
|
|
2991
|
+
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
2992
|
+
}
|
|
2993
|
+
for (const task of validTasks) {
|
|
2994
|
+
const id = generateId("task_");
|
|
2995
|
+
const taskObj = {
|
|
2996
|
+
id,
|
|
2997
|
+
entity_id: entityId,
|
|
2998
|
+
description: task.description,
|
|
2999
|
+
status: "pending",
|
|
3000
|
+
priority: task.priority,
|
|
3001
|
+
created_at: now,
|
|
3002
|
+
updated_at: now,
|
|
3003
|
+
resolved_at: null,
|
|
3004
|
+
deleted_at: null
|
|
3005
|
+
};
|
|
3006
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
3007
|
+
}
|
|
3008
|
+
});
|
|
3009
|
+
diagBuffer.flush(deps.options);
|
|
3010
|
+
await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
3011
|
+
for (const fact of insertedFacts) {
|
|
3012
|
+
await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
3013
|
+
}
|
|
3014
|
+
deps.searchService.evictCache(entityId);
|
|
3015
|
+
const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
|
|
3016
|
+
const out = { processedThrough, factsWritten: insertedFacts.length };
|
|
3017
|
+
return out;
|
|
3018
|
+
};
|
|
3019
|
+
|
|
3020
|
+
// src/services/librarian/ops/text.ts
|
|
3021
|
+
var OPS_NEIGHBOUR_BODY_CHARS = 400;
|
|
3022
|
+
function normalizeFactText(title, body) {
|
|
3023
|
+
return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
3024
|
+
}
|
|
3025
|
+
function labelEvents(events) {
|
|
3026
|
+
return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
|
|
3027
|
+
}
|
|
3028
|
+
function parseValidFrom(v) {
|
|
3029
|
+
if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
|
|
3030
|
+
if (typeof v === "string") {
|
|
3031
|
+
const t = Date.parse(v);
|
|
3032
|
+
return Number.isFinite(t) && t >= 0 ? t : void 0;
|
|
3033
|
+
}
|
|
3034
|
+
return void 0;
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
// src/services/librarian/ops/extract.ts
|
|
3038
|
+
var OPS_MAX_EVENTS_PER_PASS = 50;
|
|
3039
|
+
var OPS_MAX_EXTRACT_CHARS = 24e3;
|
|
3040
|
+
var RESOLVE_RESERVE_FACTOR = 2;
|
|
3041
|
+
var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
|
|
3042
|
+
async function selectBatch(deps, entityId, wm, meter) {
|
|
3043
|
+
const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
|
|
3044
|
+
if (events.length === 0) return { batch: [] };
|
|
3045
|
+
let n = 0;
|
|
3046
|
+
let chars = 0;
|
|
3047
|
+
for (const e of events) {
|
|
3048
|
+
if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
|
|
3049
|
+
chars += e.summary.length;
|
|
3050
|
+
n++;
|
|
3051
|
+
}
|
|
3052
|
+
let batch = labelEvents(events.slice(0, n));
|
|
3053
|
+
if (meter && meter.budget !== void 0) {
|
|
3054
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3055
|
+
for (; ; ) {
|
|
3056
|
+
const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
|
|
3057
|
+
if (meter.fits(need)) break;
|
|
3058
|
+
if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
|
|
3059
|
+
batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
return { batch };
|
|
3063
|
+
}
|
|
3064
|
+
async function runExtract(deps, ctx, batch) {
|
|
3065
|
+
const { entityId, trigger } = ctx;
|
|
3066
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3067
|
+
const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
|
|
3068
|
+
const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
|
|
3069
|
+
const parsed = parseJsonResponse(text);
|
|
3070
|
+
const base = { entityId, operation: "librarian", trigger };
|
|
3071
|
+
const diagnostics = [];
|
|
3072
|
+
const byLabel = new Map(batch.map((l) => [l.label, l]));
|
|
3073
|
+
const newestAt = batch[batch.length - 1].at;
|
|
3074
|
+
const candidates = [];
|
|
3075
|
+
(Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
|
|
3076
|
+
const fact = validateFact(raw);
|
|
3077
|
+
if (!fact) {
|
|
3078
|
+
diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
|
|
3079
|
+
return;
|
|
3080
|
+
}
|
|
3081
|
+
const r = raw;
|
|
3082
|
+
const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
|
|
3083
|
+
const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
|
|
3084
|
+
candidates.push({ index, fact, sourceLabel, validFrom });
|
|
3085
|
+
});
|
|
3086
|
+
const tasks = [];
|
|
3087
|
+
(Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
|
|
3088
|
+
const task = validateTask(raw);
|
|
3089
|
+
if (task) tasks.push(task);
|
|
3090
|
+
else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
|
|
3091
|
+
});
|
|
3092
|
+
return {
|
|
3093
|
+
batch,
|
|
3094
|
+
candidates,
|
|
3095
|
+
tasks,
|
|
3096
|
+
...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
|
|
3097
|
+
groundingCorpus: prompt.groundingCorpus ?? null,
|
|
3098
|
+
diagnostics
|
|
3099
|
+
};
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
// src/utils/cosine.ts
|
|
3103
|
+
function cosineSimilarity(a, b) {
|
|
3104
|
+
let dot = 0, normA = 0, normB = 0;
|
|
3105
|
+
const len = Math.min(a.length, b.length);
|
|
3106
|
+
for (let i = 0; i < len; i++) {
|
|
3107
|
+
dot += a[i] * b[i];
|
|
3108
|
+
normA += a[i] * a[i];
|
|
3109
|
+
normB += b[i] * b[i];
|
|
3110
|
+
}
|
|
3111
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
3112
|
+
return denom === 0 ? 0 : dot / denom;
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
// src/services/librarian/ops/gate.ts
|
|
3116
|
+
var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
|
|
3117
|
+
function resolveGateConfig(cfg) {
|
|
3118
|
+
const kRaw = cfg?.k;
|
|
3119
|
+
const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
|
|
3120
|
+
const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
|
|
3121
|
+
const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
|
|
3122
|
+
if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
|
|
3123
|
+
return { k, dupThreshold, novelThreshold };
|
|
3124
|
+
}
|
|
3125
|
+
function titlesMatch(a, b) {
|
|
3126
|
+
const ta = titleTokens(a);
|
|
3127
|
+
const tb = titleTokens(b);
|
|
3128
|
+
if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
|
|
3129
|
+
return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
|
|
3130
|
+
}
|
|
3131
|
+
function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
3132
|
+
const key = normalizeFactText(candidate.title, candidate.body);
|
|
3133
|
+
const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
|
|
3134
|
+
if (same) return { kind: "noop", target: same.id };
|
|
3135
|
+
if (neighbours.length === 0) return { kind: "add" };
|
|
3136
|
+
if (mode === "keyword") return { kind: "ambiguous" };
|
|
3137
|
+
const best = neighbours[0];
|
|
3138
|
+
const score = best.score ?? 0;
|
|
3139
|
+
if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
|
|
3140
|
+
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3141
|
+
return { kind: "ambiguous" };
|
|
3142
|
+
}
|
|
3143
|
+
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3144
|
+
const { entityId, trigger } = ctx;
|
|
3145
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3146
|
+
const unique = [];
|
|
3147
|
+
const inBatchDuplicates = [];
|
|
3148
|
+
for (const c of candidates) {
|
|
3149
|
+
const key = normalizeFactText(c.fact.title, c.fact.body);
|
|
3150
|
+
if (seen.has(key)) inBatchDuplicates.push(c);
|
|
3151
|
+
else {
|
|
3152
|
+
seen.add(key);
|
|
3153
|
+
unique.push(c);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3157
|
+
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3158
|
+
const stored = [];
|
|
3159
|
+
if (canEmbed && unique.length > 0) {
|
|
3160
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3161
|
+
if (nonLive.has(row.id)) continue;
|
|
3162
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3163
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
const gated = [];
|
|
3167
|
+
for (const c of unique) {
|
|
3168
|
+
let ranked = null;
|
|
3169
|
+
let vector = null;
|
|
3170
|
+
let mode = "keyword";
|
|
3171
|
+
if (canEmbed) {
|
|
3172
|
+
const r = await deps.embeddingService.embedTextForFact(
|
|
3173
|
+
{ id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
|
|
3174
|
+
void 0,
|
|
3175
|
+
{ markFailures: false }
|
|
3176
|
+
);
|
|
3177
|
+
if (r.ok) {
|
|
3178
|
+
vector = r.vector;
|
|
3179
|
+
if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
|
|
3180
|
+
ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
|
|
3181
|
+
mode = "vector";
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
if (!ranked) {
|
|
3186
|
+
ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
3187
|
+
}
|
|
3188
|
+
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3189
|
+
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
3190
|
+
const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
|
|
3191
|
+
const f = byId.get(r.id);
|
|
3192
|
+
return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
|
|
3193
|
+
});
|
|
3194
|
+
gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
|
|
3195
|
+
}
|
|
3196
|
+
const count = (k) => gated.filter((g) => g.decision.kind === k).length;
|
|
3197
|
+
const gateDiagnostic = {
|
|
3198
|
+
code: "librarian_gate",
|
|
3199
|
+
operation: "librarian",
|
|
3200
|
+
trigger,
|
|
3201
|
+
entityId,
|
|
3202
|
+
detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
|
|
3203
|
+
};
|
|
3204
|
+
return { gated, inBatchDuplicates, gateDiagnostic };
|
|
3205
|
+
}
|
|
3206
|
+
|
|
3207
|
+
// src/services/librarian/ops/resolve.ts
|
|
3208
|
+
var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
|
|
3209
|
+
function validateOp(raw, aliases) {
|
|
3210
|
+
if (!raw || typeof raw !== "object") return null;
|
|
3211
|
+
const r = raw;
|
|
3212
|
+
if (r.op === "ADD") return { op: "ADD" };
|
|
3213
|
+
if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
|
|
3214
|
+
const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
|
|
3215
|
+
if (!targetId) return null;
|
|
3216
|
+
if (r.op === "NOOP") return { op: "NOOP", targetId };
|
|
3217
|
+
if (r.op === "SUPERSEDE") {
|
|
3218
|
+
const validFrom = parseValidFrom(r.valid_from);
|
|
3219
|
+
return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
|
|
3220
|
+
}
|
|
3221
|
+
const update = { op: "UPDATE", targetId };
|
|
3222
|
+
if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
|
|
3223
|
+
if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
|
|
3224
|
+
return update;
|
|
3225
|
+
}
|
|
3226
|
+
var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
|
|
3227
|
+
async function runResolve(deps, ctx, ambiguous) {
|
|
3228
|
+
const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
|
|
3229
|
+
if (ambiguous.length === 0) return out;
|
|
3230
|
+
const { entityId, trigger } = ctx;
|
|
3231
|
+
const outcome = await runBatched({
|
|
3232
|
+
items: ambiguous,
|
|
3233
|
+
buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
|
|
3234
|
+
batch.map((g, i) => ({
|
|
3235
|
+
item: i,
|
|
3236
|
+
candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
|
|
3237
|
+
existing: g.neighbours.map((n) => ({
|
|
3238
|
+
ref: n.ref,
|
|
3239
|
+
title: n.title,
|
|
3240
|
+
body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
|
|
3241
|
+
}))
|
|
3242
|
+
}))
|
|
3243
|
+
),
|
|
3244
|
+
call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
|
|
3245
|
+
parse: (text, batch) => {
|
|
3246
|
+
const parsed = parseJsonResponse(text);
|
|
3247
|
+
if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
|
|
3248
|
+
return { batch, ops: parsed.ops };
|
|
3249
|
+
},
|
|
3250
|
+
maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
|
|
3251
|
+
maxOutputTokens: deps.options.llmProvider.maxOutputTokens
|
|
3252
|
+
});
|
|
3253
|
+
if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
|
|
3254
|
+
for (const { batch, ops } of outcome.results) {
|
|
3255
|
+
const byItem = /* @__PURE__ */ new Map();
|
|
3256
|
+
for (const raw of ops) {
|
|
3257
|
+
const item = raw?.item;
|
|
3258
|
+
if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
|
|
3259
|
+
byItem.set(item, raw);
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
batch.forEach((g, i) => {
|
|
3263
|
+
const op = validateOp(byItem.get(i), aliasMapOf(g));
|
|
3264
|
+
if (op) out.ops.set(g, op);
|
|
3265
|
+
else out.rejected.push(g);
|
|
3266
|
+
});
|
|
3267
|
+
}
|
|
3268
|
+
if (outcome.skipped.some((s) => s.reason === "call_error")) {
|
|
3269
|
+
throw new Error("resolve: transient provider error; retry the pass");
|
|
3270
|
+
}
|
|
3271
|
+
for (const s of outcome.skipped) out.failed.push(s.item);
|
|
3272
|
+
return out;
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
// src/utils/temporal.ts
|
|
3276
|
+
function assertEpochMs(name, v) {
|
|
3277
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
3278
|
+
throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
|
|
3279
|
+
}
|
|
3280
|
+
return Math.trunc(v);
|
|
3281
|
+
}
|
|
3282
|
+
|
|
3283
|
+
// src/services/TemporalService.ts
|
|
3284
|
+
var HISTORY_MAX_DEPTH = 100;
|
|
3285
|
+
var TemporalService = class {
|
|
3286
|
+
constructor(db, options, entryRepo, searchService, embeddingService) {
|
|
3287
|
+
this.db = db;
|
|
3288
|
+
this.options = options;
|
|
3289
|
+
this.entryRepo = entryRepo;
|
|
3290
|
+
this.searchService = searchService;
|
|
3291
|
+
this.embeddingService = embeddingService;
|
|
3292
|
+
}
|
|
3293
|
+
/**
|
|
3294
|
+
* Link `oldId` → `newId` inside the caller's transaction. `newId` must
|
|
3295
|
+
* already exist. Throws WikiSupersedeError on any rule violation; the
|
|
3296
|
+
* caller's transaction then rolls back.
|
|
3297
|
+
*/
|
|
3298
|
+
async supersedeInTx(tx, entityId, oldId, newId, t, now) {
|
|
3299
|
+
const old = await this.entryRepo.findTemporalRow(oldId, tx);
|
|
3300
|
+
if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3301
|
+
if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3302
|
+
if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3303
|
+
if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
|
|
3304
|
+
if (newId === oldId) throw new WikiSupersedeError("cycle");
|
|
3305
|
+
const repl = await this.entryRepo.findTemporalRow(newId, tx);
|
|
3306
|
+
if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3307
|
+
if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3308
|
+
if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3309
|
+
const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
|
|
3310
|
+
if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
|
|
3311
|
+
await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
|
|
3312
|
+
if (repl.valid_from == null) {
|
|
3313
|
+
await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
3317
|
+
const now = Date.now();
|
|
3318
|
+
const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
|
|
3319
|
+
let created = null;
|
|
3320
|
+
if (typeof replacement !== "string") {
|
|
3321
|
+
if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
|
|
3322
|
+
throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
|
|
3323
|
+
}
|
|
3324
|
+
created = {
|
|
3325
|
+
id: generateId("fact_"),
|
|
3326
|
+
entity_id: entityId,
|
|
3327
|
+
title: replacement.title,
|
|
3328
|
+
body: replacement.body,
|
|
3329
|
+
tags: Array.isArray(replacement.tags) ? replacement.tags : [],
|
|
3330
|
+
confidence: replacement.confidence ?? "certain",
|
|
3331
|
+
source_type: replacement.source_type ?? "user_stated",
|
|
3332
|
+
source_hash: null,
|
|
3333
|
+
source_ref: null,
|
|
3334
|
+
created_at: now,
|
|
3335
|
+
updated_at: now,
|
|
3336
|
+
last_accessed_at: null,
|
|
3337
|
+
access_count: 0,
|
|
3338
|
+
deleted_at: null
|
|
3339
|
+
};
|
|
3340
|
+
}
|
|
3341
|
+
const newId = typeof replacement === "string" ? replacement : created.id;
|
|
3342
|
+
const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
|
|
3343
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3344
|
+
if (created) {
|
|
3345
|
+
await this.entryRepo.upsert(created, tx);
|
|
3346
|
+
if (newValidFrom !== void 0) {
|
|
3347
|
+
await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
|
|
3348
|
+
}
|
|
3349
|
+
}
|
|
3350
|
+
await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
|
|
3351
|
+
});
|
|
3352
|
+
await this.searchService.syncEntries(entityId, [oldId, newId]);
|
|
3353
|
+
if (created) {
|
|
3354
|
+
await this.embeddingService.embedFact(
|
|
3355
|
+
{ id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
|
|
3356
|
+
{ operation: "supersede", trigger: "call" }
|
|
3357
|
+
);
|
|
3358
|
+
}
|
|
3359
|
+
this.searchService.evictCache(entityId);
|
|
3360
|
+
return { newId };
|
|
3361
|
+
}
|
|
3362
|
+
async history(entityId, factId) {
|
|
3363
|
+
const start = await this.entryRepo.findTemporalRow(factId);
|
|
3364
|
+
if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
|
|
3365
|
+
const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
|
|
3366
|
+
const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
|
|
3367
|
+
return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
|
|
3368
|
+
}
|
|
3369
|
+
};
|
|
3370
|
+
|
|
3371
|
+
// src/services/librarian/ops/apply.ts
|
|
3372
|
+
async function applyOps(deps, ctx, input) {
|
|
3373
|
+
const { entityId, trigger } = ctx;
|
|
3374
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3375
|
+
const grounding = deps.promptService.groundingFor("librarian");
|
|
3376
|
+
const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
|
|
3377
|
+
const rejected = new Set(input.rejected);
|
|
3378
|
+
const failed = new Set(input.failed);
|
|
3379
|
+
const written = [];
|
|
3380
|
+
const updatedIds = [];
|
|
3381
|
+
const now = Date.now();
|
|
3382
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
3383
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3384
|
+
if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
|
|
3385
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
|
|
3386
|
+
}
|
|
3387
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3388
|
+
for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
|
|
3389
|
+
titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
|
|
3390
|
+
}
|
|
3391
|
+
const pendingEdges = [];
|
|
3392
|
+
const insert = async (g, draft, validFrom, overrides) => {
|
|
3393
|
+
const fact = g.candidate.fact;
|
|
3394
|
+
const title = overrides?.title ?? fact.title;
|
|
3395
|
+
const body = overrides?.body ?? fact.body;
|
|
3396
|
+
const id = generateId("fact_");
|
|
3397
|
+
const drops = [];
|
|
3398
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
|
|
3399
|
+
for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
|
|
3400
|
+
const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
|
|
3401
|
+
if (g2?.diagnostic) {
|
|
3402
|
+
input.diagnostics.push({
|
|
3403
|
+
...diagBase,
|
|
3404
|
+
code: g2.diagnostic.code,
|
|
3405
|
+
detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
|
|
3406
|
+
});
|
|
3407
|
+
}
|
|
3408
|
+
const row = {
|
|
3409
|
+
id,
|
|
3410
|
+
entity_id: entityId,
|
|
3411
|
+
title,
|
|
3412
|
+
body,
|
|
3413
|
+
tags: fact.tags,
|
|
3414
|
+
confidence: fact.confidence,
|
|
3415
|
+
source_type: "librarian_inferred",
|
|
3416
|
+
source_hash: null,
|
|
3417
|
+
source_ref: null,
|
|
3418
|
+
created_at: now,
|
|
3419
|
+
updated_at: now,
|
|
3420
|
+
last_accessed_at: null,
|
|
3421
|
+
access_count: 0,
|
|
3422
|
+
deleted_at: null,
|
|
3423
|
+
okf_type: normalized.okf_type,
|
|
3424
|
+
...g2?.trust,
|
|
3425
|
+
...draft ? { lifecycle_status: "draft" } : {}
|
|
3426
|
+
};
|
|
3427
|
+
await deps.entryRepo.upsert(row, tx);
|
|
3428
|
+
await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
|
|
3429
|
+
titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
|
|
3430
|
+
if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3431
|
+
const textChanged = title !== fact.title || body !== fact.body;
|
|
3432
|
+
written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
|
|
3433
|
+
return id;
|
|
3434
|
+
};
|
|
3435
|
+
for (const g of input.gated) {
|
|
3436
|
+
const idx = g.candidate.index;
|
|
3437
|
+
const vf = g.candidate.validFrom;
|
|
3438
|
+
if (g.decision.kind === "noop") continue;
|
|
3439
|
+
if (g.decision.kind === "add") {
|
|
3440
|
+
await insert(g, false, vf);
|
|
3441
|
+
continue;
|
|
3442
|
+
}
|
|
3443
|
+
if (failed.has(g)) {
|
|
3444
|
+
const id = await insert(g, true, vf);
|
|
3445
|
+
input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
|
|
3446
|
+
continue;
|
|
3447
|
+
}
|
|
3448
|
+
const op = input.ops.get(g);
|
|
3449
|
+
if (rejected.has(g) || !op) {
|
|
3450
|
+
await insert(g, false, vf);
|
|
3451
|
+
input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
|
|
3452
|
+
continue;
|
|
3453
|
+
}
|
|
3454
|
+
if (op.op === "NOOP") continue;
|
|
3455
|
+
if (op.op === "ADD") {
|
|
3456
|
+
await insert(g, false, vf);
|
|
3457
|
+
continue;
|
|
3458
|
+
}
|
|
3459
|
+
const target = g.neighbours.find((n) => n.id === op.targetId);
|
|
3460
|
+
if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
|
|
3461
|
+
const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
|
|
3462
|
+
if (existing) {
|
|
3463
|
+
const newTitle = op.title ?? g.candidate.fact.title;
|
|
3464
|
+
await deps.entryRepo.upsert({
|
|
3465
|
+
...existing,
|
|
3466
|
+
title: newTitle,
|
|
3467
|
+
body: op.body ?? g.candidate.fact.body,
|
|
3468
|
+
updated_at: now
|
|
3469
|
+
}, tx);
|
|
3470
|
+
titleIndex.delete(normalizeTitleKey(existing.title));
|
|
3471
|
+
titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
|
|
3472
|
+
updatedIds.push(existing.id);
|
|
3473
|
+
continue;
|
|
3474
|
+
}
|
|
3475
|
+
}
|
|
3476
|
+
const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
|
|
3477
|
+
const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
|
|
3478
|
+
if (target.source_type === "immutable_document") {
|
|
3479
|
+
const id = await insert(g, true, supersedeFrom, overrides);
|
|
3480
|
+
input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
|
|
3481
|
+
continue;
|
|
3482
|
+
}
|
|
3483
|
+
const newId = await insert(g, false, supersedeFrom, overrides);
|
|
3484
|
+
try {
|
|
3485
|
+
await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
|
|
3486
|
+
} catch (err) {
|
|
3487
|
+
if (!(err instanceof WikiSupersedeError)) throw err;
|
|
3488
|
+
input.diagnostics.push({
|
|
3489
|
+
...diagBase,
|
|
3490
|
+
code: "librarian_op_rejected",
|
|
3491
|
+
detail: { factId: newId, itemIndex: idx, reason: err.reason }
|
|
3492
|
+
});
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
for (const item of pendingEdges) {
|
|
3496
|
+
const resolveDrops = [];
|
|
3497
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
3498
|
+
entityId,
|
|
3499
|
+
item.sourceId,
|
|
3500
|
+
item.sourceType,
|
|
3501
|
+
item.edges,
|
|
3502
|
+
manifest,
|
|
3503
|
+
titleIndex,
|
|
3504
|
+
tx,
|
|
3505
|
+
now,
|
|
3506
|
+
resolveDrops
|
|
3507
|
+
);
|
|
3508
|
+
for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
|
|
3509
|
+
}
|
|
3510
|
+
for (const task of input.tasks) {
|
|
3511
|
+
const taskObj = {
|
|
3512
|
+
id: generateId("task_"),
|
|
3513
|
+
entity_id: entityId,
|
|
3514
|
+
description: task.description,
|
|
3515
|
+
status: "pending",
|
|
3516
|
+
priority: task.priority,
|
|
3517
|
+
created_at: now,
|
|
3518
|
+
updated_at: now,
|
|
3519
|
+
resolved_at: null,
|
|
3520
|
+
deleted_at: null
|
|
3521
|
+
};
|
|
3522
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
3523
|
+
}
|
|
3524
|
+
});
|
|
3525
|
+
return { written, updatedIds };
|
|
3526
|
+
}
|
|
3527
|
+
|
|
3528
|
+
// src/services/librarian/ops/index.ts
|
|
3529
|
+
var runOpsLibrarianPass = async (deps, ctx) => {
|
|
3530
|
+
const { entityId, trigger } = ctx;
|
|
3531
|
+
const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
|
|
3532
|
+
await deps.seedWatermark(entityId);
|
|
3533
|
+
const wm = await deps.getWatermark(entityId);
|
|
3534
|
+
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
3535
|
+
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
3536
|
+
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
3537
|
+
let ex;
|
|
3538
|
+
try {
|
|
3539
|
+
ex = await runExtract(deps, ctx, sel.batch);
|
|
3540
|
+
} catch (err) {
|
|
3541
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
3542
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
3543
|
+
}
|
|
3544
|
+
throw err;
|
|
3545
|
+
}
|
|
3546
|
+
const diagnostics = new DiagnosticBuffer();
|
|
3547
|
+
for (const d of ex.diagnostics) diagnostics.push(d);
|
|
3548
|
+
const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
|
|
3549
|
+
for (const dup of inBatchDuplicates) {
|
|
3550
|
+
diagnostics.push({
|
|
3551
|
+
code: "fact_deduplicated",
|
|
3552
|
+
operation: "librarian",
|
|
3553
|
+
trigger,
|
|
3554
|
+
entityId,
|
|
3555
|
+
detail: { itemIndex: dup.index, reason: "in_batch" }
|
|
3556
|
+
});
|
|
3557
|
+
}
|
|
3558
|
+
diagnostics.push(gateDiagnostic);
|
|
3559
|
+
const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
|
|
3560
|
+
if (res.budgetStop) {
|
|
3561
|
+
diagnostics.discard();
|
|
3562
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
|
|
3563
|
+
}
|
|
3564
|
+
const { written, updatedIds } = await applyOps(deps, ctx, {
|
|
3565
|
+
gated,
|
|
3566
|
+
ops: res.ops,
|
|
3567
|
+
rejected: res.rejected,
|
|
3568
|
+
failed: res.failed,
|
|
3569
|
+
tasks: ex.tasks,
|
|
3570
|
+
ontologyUpdates: ex.ontologyUpdates,
|
|
3571
|
+
groundingCorpus: ex.groundingCorpus,
|
|
3572
|
+
diagnostics
|
|
3573
|
+
});
|
|
3574
|
+
diagnostics.flush(deps.options);
|
|
3575
|
+
await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
|
|
3576
|
+
const embedCtx = { operation: "librarian", trigger };
|
|
3577
|
+
for (const w of written) {
|
|
3578
|
+
if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
|
|
3579
|
+
else await deps.embeddingService.embedFact(w.fact, embedCtx);
|
|
3580
|
+
}
|
|
3581
|
+
if (updatedIds.length > 0) {
|
|
3582
|
+
for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
|
|
3583
|
+
await deps.embeddingService.embedFact(
|
|
3584
|
+
{ id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
|
|
3585
|
+
embedCtx
|
|
3586
|
+
);
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3589
|
+
deps.searchService.evictCache(entityId);
|
|
3590
|
+
const last = sel.batch[sel.batch.length - 1].event;
|
|
3591
|
+
return {
|
|
3592
|
+
processedThrough: { at: last.created_at, id: last.id },
|
|
3593
|
+
factsWritten: written.length + updatedIds.length
|
|
3594
|
+
};
|
|
3595
|
+
};
|
|
3596
|
+
|
|
3597
|
+
// src/services/librarian/index.ts
|
|
3598
|
+
var STRATEGIES = /* @__PURE__ */ new Map([
|
|
3599
|
+
["legacy", runLegacyLibrarianPass],
|
|
3600
|
+
["ops", runOpsLibrarianPass]
|
|
3601
|
+
]);
|
|
3602
|
+
async function runLibrarianStrategy(deps, ctx) {
|
|
3603
|
+
const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
|
|
3604
|
+
const fn = STRATEGIES.get(strategy);
|
|
3605
|
+
if (!fn) {
|
|
3606
|
+
throw new Error(`Unknown librarian strategy: ${strategy}`);
|
|
3607
|
+
}
|
|
3608
|
+
return fn(deps, ctx);
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
// src/services/MaintenanceService.ts
|
|
2702
3612
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
2703
3613
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
2704
3614
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
@@ -2771,10 +3681,23 @@ var MaintenanceService = class {
|
|
|
2771
3681
|
const vacuum = options?.vacuum ?? false;
|
|
2772
3682
|
this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
|
|
2773
3683
|
this._validatePruneDuration(retainEventsFor, "retainEventsFor");
|
|
3684
|
+
const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
|
|
3685
|
+
this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
|
|
2774
3686
|
const now = Date.now();
|
|
3687
|
+
let syncedIds = [];
|
|
2775
3688
|
let deletedEntries = 0;
|
|
2776
3689
|
let deletedTasks = 0;
|
|
2777
3690
|
let deletedEvents = 0;
|
|
3691
|
+
if (pruneSupersededAfter !== null) {
|
|
3692
|
+
const expiredCutoff = now - pruneSupersededAfter * 864e5;
|
|
3693
|
+
const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
|
|
3694
|
+
if (expiredIds.length > 0) {
|
|
3695
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3696
|
+
await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
|
|
3697
|
+
});
|
|
3698
|
+
syncedIds = [...expiredIds];
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
2778
3701
|
if (retainSoftDeletedFor !== null) {
|
|
2779
3702
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
2780
3703
|
const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
|
|
@@ -2790,6 +3713,7 @@ var MaintenanceService = class {
|
|
|
2790
3713
|
}
|
|
2791
3714
|
}
|
|
2792
3715
|
const succeededIds = succeeded.map((r) => r.id);
|
|
3716
|
+
syncedIds = [...syncedIds, ...succeededIds];
|
|
2793
3717
|
await this.db.withTransactionAsync(async (tx) => {
|
|
2794
3718
|
if (succeededIds.length > 0) {
|
|
2795
3719
|
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
@@ -2797,7 +3721,7 @@ var MaintenanceService = class {
|
|
|
2797
3721
|
deletedTasks = await this.taskRepo.bulkDeletePruned(entityId, cutoff, tx);
|
|
2798
3722
|
});
|
|
2799
3723
|
if (failure) {
|
|
2800
|
-
await this.searchService.
|
|
3724
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
2801
3725
|
const remaining = entriesToDelete.length - succeeded.length - 1;
|
|
2802
3726
|
const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
|
|
2803
3727
|
if (isTimeout) {
|
|
@@ -2825,13 +3749,23 @@ var MaintenanceService = class {
|
|
|
2825
3749
|
}
|
|
2826
3750
|
if (retainEventsFor !== null) {
|
|
2827
3751
|
const cutoff = now - retainEventsFor * 864e5;
|
|
2828
|
-
const
|
|
2829
|
-
|
|
3752
|
+
const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
|
|
3753
|
+
if (guarded) {
|
|
3754
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
3755
|
+
deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
|
|
3756
|
+
const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
|
|
3757
|
+
if (held > 0) {
|
|
3758
|
+
emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
|
|
3759
|
+
}
|
|
3760
|
+
} else {
|
|
3761
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
3762
|
+
deletedEvents = eventResult.changes;
|
|
3763
|
+
}
|
|
2830
3764
|
}
|
|
2831
3765
|
if (vacuum) {
|
|
2832
3766
|
await this.metadataRepo.vacuum();
|
|
2833
3767
|
}
|
|
2834
|
-
await this.searchService.
|
|
3768
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
2835
3769
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
2836
3770
|
} finally {
|
|
2837
3771
|
this.jobManager.releaseLock("prune", entityId);
|
|
@@ -3001,8 +3935,8 @@ var MaintenanceService = class {
|
|
|
3001
3935
|
deletedEntries += refResult;
|
|
3002
3936
|
}
|
|
3003
3937
|
});
|
|
3004
|
-
await this.searchService.sync(entityId);
|
|
3005
3938
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
3939
|
+
await this.searchService.syncEntries(entityId, uniqueDeletedIds);
|
|
3006
3940
|
for (const factId of uniqueDeletedIds) {
|
|
3007
3941
|
try {
|
|
3008
3942
|
await this.embeddingService.notifyEmbeddingPersistedOrThrow(entityId, factId, null);
|
|
@@ -3056,155 +3990,72 @@ var MaintenanceService = class {
|
|
|
3056
3990
|
}
|
|
3057
3991
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
3058
3992
|
async doRunLibrarian(entityId, promptOverride, trigger = "call") {
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
3078
|
-
const result = parseJsonResponse(responseText);
|
|
3079
|
-
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
3080
|
-
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
3081
|
-
const ontologyUpdates = result.ontology_updates;
|
|
3082
|
-
const diagBuffer = new DiagnosticBuffer();
|
|
3083
|
-
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3084
|
-
const validFacts = [];
|
|
3085
|
-
const validFactItemIndexes = [];
|
|
3086
|
-
facts.forEach((raw, itemIndex) => {
|
|
3087
|
-
const valid = validateFact(raw);
|
|
3088
|
-
if (valid) {
|
|
3089
|
-
validFacts.push(valid);
|
|
3090
|
-
validFactItemIndexes.push(itemIndex);
|
|
3091
|
-
} else {
|
|
3092
|
-
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3093
|
-
}
|
|
3094
|
-
});
|
|
3095
|
-
const validTasks = [];
|
|
3096
|
-
tasks.forEach((raw, itemIndex) => {
|
|
3097
|
-
const valid = validateTask(raw);
|
|
3098
|
-
if (valid) validTasks.push(valid);
|
|
3099
|
-
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
3100
|
-
});
|
|
3101
|
-
const now = Date.now();
|
|
3102
|
-
const insertedFacts = [];
|
|
3103
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
3104
|
-
let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3105
|
-
if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
|
|
3106
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
3107
|
-
}
|
|
3108
|
-
const titleIndex = /* @__PURE__ */ new Map();
|
|
3109
|
-
for (const existing of currentFactsRows) {
|
|
3110
|
-
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3111
|
-
id: existing.id,
|
|
3112
|
-
okf_type: existing.okf_type ?? null
|
|
3113
|
-
});
|
|
3114
|
-
}
|
|
3115
|
-
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
3116
|
-
const pendingEdges = [];
|
|
3117
|
-
for (const [k, fact] of validFacts.entries()) {
|
|
3118
|
-
const newTokens = titleTokens(fact.title);
|
|
3119
|
-
let skip = false;
|
|
3120
|
-
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3121
|
-
for (const existing of factsForDedupe) {
|
|
3122
|
-
if (existing.source_type !== "librarian_inferred") continue;
|
|
3123
|
-
const existingTokens = titleTokens(existing.title);
|
|
3124
|
-
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3125
|
-
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
3126
|
-
skip = true;
|
|
3127
|
-
break;
|
|
3128
|
-
}
|
|
3129
|
-
}
|
|
3130
|
-
}
|
|
3131
|
-
}
|
|
3132
|
-
if (skip) {
|
|
3133
|
-
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
3134
|
-
continue;
|
|
3135
|
-
}
|
|
3136
|
-
const ontologyFact = fact;
|
|
3137
|
-
const id = generateId("fact_");
|
|
3138
|
-
const validationDrops = [];
|
|
3139
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
3140
|
-
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3141
|
-
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3142
|
-
if (grounding?.diagnostic) {
|
|
3143
|
-
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3144
|
-
}
|
|
3145
|
-
const factObj = {
|
|
3146
|
-
id,
|
|
3147
|
-
entity_id: entityId,
|
|
3148
|
-
title: fact.title,
|
|
3149
|
-
body: fact.body,
|
|
3150
|
-
tags: fact.tags,
|
|
3151
|
-
confidence: fact.confidence,
|
|
3152
|
-
source_type: "librarian_inferred",
|
|
3153
|
-
source_hash: null,
|
|
3154
|
-
source_ref: null,
|
|
3155
|
-
created_at: now,
|
|
3156
|
-
updated_at: now,
|
|
3157
|
-
last_accessed_at: null,
|
|
3158
|
-
access_count: 0,
|
|
3159
|
-
deleted_at: null,
|
|
3160
|
-
okf_type: normalized.okf_type,
|
|
3161
|
-
...grounding?.trust
|
|
3162
|
-
};
|
|
3163
|
-
await this.entryRepo.upsert(factObj, tx);
|
|
3164
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3165
|
-
factsForDedupe.push(factObj);
|
|
3166
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3167
|
-
if (normalized.edges.length > 0) {
|
|
3168
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3169
|
-
}
|
|
3170
|
-
}
|
|
3171
|
-
for (const item of pendingEdges) {
|
|
3172
|
-
const resolveDrops = [];
|
|
3173
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
3174
|
-
entityId,
|
|
3175
|
-
item.sourceId,
|
|
3176
|
-
item.sourceType,
|
|
3177
|
-
item.edges ?? [],
|
|
3178
|
-
manifest,
|
|
3179
|
-
titleIndex,
|
|
3180
|
-
tx,
|
|
3181
|
-
now,
|
|
3182
|
-
resolveDrops
|
|
3183
|
-
);
|
|
3184
|
-
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
3185
|
-
}
|
|
3186
|
-
for (const task of validTasks) {
|
|
3187
|
-
const id = generateId("task_");
|
|
3188
|
-
const taskObj = {
|
|
3189
|
-
id,
|
|
3190
|
-
entity_id: entityId,
|
|
3191
|
-
description: task.description,
|
|
3192
|
-
status: "pending",
|
|
3193
|
-
priority: task.priority,
|
|
3194
|
-
created_at: now,
|
|
3195
|
-
updated_at: now,
|
|
3196
|
-
resolved_at: null,
|
|
3197
|
-
deleted_at: null
|
|
3198
|
-
};
|
|
3199
|
-
await this.taskRepo.upsert(taskObj, tx);
|
|
3200
|
-
}
|
|
3201
|
-
});
|
|
3202
|
-
diagBuffer.flush(this.options);
|
|
3203
|
-
await this.searchService.sync(entityId);
|
|
3204
|
-
for (const fact of insertedFacts) {
|
|
3205
|
-
await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
3993
|
+
await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
|
|
3994
|
+
}
|
|
3995
|
+
/**
|
|
3996
|
+
* Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
|
|
3997
|
+
* §5.1). Builds the strategy deps from `this` and dispatches via
|
|
3998
|
+
* `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
|
|
3999
|
+
* class. Returns the strategy's `LibrarianResult` so deferred/budgeted
|
|
4000
|
+
* schedulers (PR-C) can advance watermarks; the existing void-returning
|
|
4001
|
+
* `doRunLibrarian` discards it.
|
|
4002
|
+
*
|
|
4003
|
+
* Spec §10.3: one place advances the watermark, for every strategy, only
|
|
4004
|
+
* after the pass resolved (its transaction committed) and never backwards.
|
|
4005
|
+
* A budget-stopped pass wrote nothing and must not move the watermark.
|
|
4006
|
+
*/
|
|
4007
|
+
async runLibrarianPass(ctx) {
|
|
4008
|
+
const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
|
|
4009
|
+
if (result.processedThrough && !result.budgetStop) {
|
|
4010
|
+
await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
|
|
3206
4011
|
}
|
|
3207
|
-
|
|
4012
|
+
return result;
|
|
4013
|
+
}
|
|
4014
|
+
/**
|
|
4015
|
+
* Strategy deps shared between legacy and ops librarians (spec §5.1). The
|
|
4016
|
+
* ops strategy (PR-B Task 8) reads/anchors the watermark through these two
|
|
4017
|
+
* hooks so the dispatcher itself never reaches into the metadata table —
|
|
4018
|
+
* the watermark invariant (one place advances, never backwards) lives here.
|
|
4019
|
+
*/
|
|
4020
|
+
librarianDeps() {
|
|
4021
|
+
return {
|
|
4022
|
+
db: this.db,
|
|
4023
|
+
options: this.options,
|
|
4024
|
+
entryRepo: this.entryRepo,
|
|
4025
|
+
taskRepo: this.taskRepo,
|
|
4026
|
+
eventRepo: this.eventRepo,
|
|
4027
|
+
searchService: this.searchService,
|
|
4028
|
+
embeddingService: this.embeddingService,
|
|
4029
|
+
promptService: this.promptService,
|
|
4030
|
+
ontologyService: this.ontologyService,
|
|
4031
|
+
seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
|
|
4032
|
+
getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
|
|
4033
|
+
};
|
|
4034
|
+
}
|
|
4035
|
+
/**
|
|
4036
|
+
* Persist `next` as the entity's librarian watermark when it is strictly
|
|
4037
|
+
* newer than the stored value, under the (created_at, id) ordering used by
|
|
4038
|
+
* the op-based librarian (spec §10.3). Never moves backwards.
|
|
4039
|
+
*/
|
|
4040
|
+
async advanceLibrarianWatermark(entityId, next) {
|
|
4041
|
+
const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
4042
|
+
const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
|
|
4043
|
+
if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
|
|
4044
|
+
}
|
|
4045
|
+
/**
|
|
4046
|
+
* Spec §10.3: first ops pass on a legacy DB starts after what the count
|
|
4047
|
+
* checkpoint already covered. No-op when a watermark already exists or the
|
|
4048
|
+
* memory checkpoint is 0; otherwise anchor the watermark to the event at
|
|
4049
|
+
* offset `min(memory_checkpoint, totalEvents) - 1`.
|
|
4050
|
+
*/
|
|
4051
|
+
async seedLibrarianWatermark(entityId) {
|
|
4052
|
+
if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
|
|
4053
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
4054
|
+
const memory = cp.memory ?? 0;
|
|
4055
|
+
if (memory <= 0) return;
|
|
4056
|
+
const total = await this.eventRepo.count(entityId);
|
|
4057
|
+
const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
|
|
4058
|
+
if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
|
|
3208
4059
|
}
|
|
3209
4060
|
/**
|
|
3210
4061
|
* Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
|
|
@@ -3258,7 +4109,7 @@ var MaintenanceService = class {
|
|
|
3258
4109
|
}
|
|
3259
4110
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
3260
4111
|
if (healCandidates.length === 0) {
|
|
3261
|
-
await this.searchService.
|
|
4112
|
+
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
3262
4113
|
this.searchService.evictCache(entityId);
|
|
3263
4114
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
3264
4115
|
return {
|
|
@@ -3310,7 +4161,7 @@ var MaintenanceService = class {
|
|
|
3310
4161
|
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3311
4162
|
return prompts;
|
|
3312
4163
|
},
|
|
3313
|
-
call: (prompts) => this.options
|
|
4164
|
+
call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
|
|
3314
4165
|
parse: (responseText, batch, prompts) => {
|
|
3315
4166
|
const result = parseJsonResponse(responseText);
|
|
3316
4167
|
return {
|
|
@@ -3414,9 +4265,10 @@ var MaintenanceService = class {
|
|
|
3414
4265
|
const callErrorIds = new Set(
|
|
3415
4266
|
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
3416
4267
|
);
|
|
4268
|
+
const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
|
|
3417
4269
|
await this.entryRepo.markHealChecked(
|
|
3418
4270
|
[
|
|
3419
|
-
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
4271
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
|
|
3420
4272
|
...insertedFacts.map((f) => f.id)
|
|
3421
4273
|
],
|
|
3422
4274
|
entityId,
|
|
@@ -3428,7 +4280,14 @@ var MaintenanceService = class {
|
|
|
3428
4280
|
diagBuffer.push({ ...diagBase, code: "heal_skipped", detail: { factId: item.id, reason } });
|
|
3429
4281
|
}
|
|
3430
4282
|
diagBuffer.flush(this.options);
|
|
3431
|
-
|
|
4283
|
+
const healSyncIds = Array.from(/* @__PURE__ */ new Set([
|
|
4284
|
+
...orphanedIds,
|
|
4285
|
+
...staleDowngradedIds,
|
|
4286
|
+
...safeDowngraded,
|
|
4287
|
+
...uniqueDeletedFactIds,
|
|
4288
|
+
...insertedFacts.map((f) => f.id)
|
|
4289
|
+
]));
|
|
4290
|
+
await this.searchService.syncEntries(entityId, healSyncIds);
|
|
3432
4291
|
for (const factId of uniqueDeletedFactIds) {
|
|
3433
4292
|
try {
|
|
3434
4293
|
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
@@ -3461,7 +4320,8 @@ var MaintenanceService = class {
|
|
|
3461
4320
|
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
3462
4321
|
degraded: healedDegraded,
|
|
3463
4322
|
remaining: counts.eligible,
|
|
3464
|
-
deferred: counts.deferred
|
|
4323
|
+
deferred: counts.deferred,
|
|
4324
|
+
...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
|
|
3465
4325
|
};
|
|
3466
4326
|
}
|
|
3467
4327
|
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
@@ -3505,7 +4365,7 @@ var MaintenanceService = class {
|
|
|
3505
4365
|
const outcome = await runBatched({
|
|
3506
4366
|
items: candidates,
|
|
3507
4367
|
buildPrompt,
|
|
3508
|
-
call: (prompts) => this.options
|
|
4368
|
+
call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
|
|
3509
4369
|
parse: (responseText, batch) => {
|
|
3510
4370
|
const parsed = parseJsonResponse(responseText);
|
|
3511
4371
|
return {
|
|
@@ -3890,6 +4750,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
3890
4750
|
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
3891
4751
|
}
|
|
3892
4752
|
|
|
4753
|
+
// src/utils/budget.ts
|
|
4754
|
+
function factTokenCost(f) {
|
|
4755
|
+
return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
|
|
4756
|
+
}
|
|
4757
|
+
function packFactsByBudget(items, budget, scores) {
|
|
4758
|
+
const candidates = items.length;
|
|
4759
|
+
if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
|
|
4760
|
+
let top = items[0];
|
|
4761
|
+
let topCost = factTokenCost(top);
|
|
4762
|
+
if (topCost > budget) {
|
|
4763
|
+
const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
|
|
4764
|
+
top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
|
|
4765
|
+
topCost = factTokenCost(top);
|
|
4766
|
+
}
|
|
4767
|
+
const kept = [{ rank: 0, item: top }];
|
|
4768
|
+
let used = topCost;
|
|
4769
|
+
const rest = items.slice(1).map((item, i) => {
|
|
4770
|
+
const rank = i + 1;
|
|
4771
|
+
const cost = factTokenCost(item);
|
|
4772
|
+
const s = scores?.get(item.id);
|
|
4773
|
+
const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
|
|
4774
|
+
return { rank, item, cost, density: value / Math.max(1, cost) };
|
|
4775
|
+
});
|
|
4776
|
+
rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
|
|
4777
|
+
for (const r of rest) {
|
|
4778
|
+
if (used + r.cost <= budget) {
|
|
4779
|
+
kept.push({ rank: r.rank, item: r.item });
|
|
4780
|
+
used += r.cost;
|
|
4781
|
+
}
|
|
4782
|
+
}
|
|
4783
|
+
kept.sort((a, b) => a.rank - b.rank);
|
|
4784
|
+
return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
|
|
4785
|
+
}
|
|
4786
|
+
|
|
3893
4787
|
// src/services/RetrievalService.ts
|
|
3894
4788
|
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
3895
4789
|
var RetrievalService = class {
|
|
@@ -3925,6 +4819,28 @@ var RetrievalService = class {
|
|
|
3925
4819
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
3926
4820
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
3927
4821
|
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
4822
|
+
const rawAsOf = options?.asOf;
|
|
4823
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
4824
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
4825
|
+
}
|
|
4826
|
+
const tokenBudget = options?.tokenBudget;
|
|
4827
|
+
if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
|
|
4828
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
4829
|
+
}
|
|
4830
|
+
const applyBudget = (input) => {
|
|
4831
|
+
if (tokenBudget === void 0) return input;
|
|
4832
|
+
const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
|
|
4833
|
+
emitDiagnostic(this.options, {
|
|
4834
|
+
code: "read_budget",
|
|
4835
|
+
operation: "read",
|
|
4836
|
+
trigger: "call",
|
|
4837
|
+
entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
|
|
4838
|
+
detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
|
|
4839
|
+
});
|
|
4840
|
+
return packed.items;
|
|
4841
|
+
};
|
|
4842
|
+
const liveMode = rawAsOf === void 0 ? "current" : "asOf";
|
|
4843
|
+
const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
|
|
3928
4844
|
const trimmedQuery = query.trim();
|
|
3929
4845
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
3930
4846
|
entityIds,
|
|
@@ -3944,8 +4860,10 @@ var RetrievalService = class {
|
|
|
3944
4860
|
let usedEmbed = false;
|
|
3945
4861
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
3946
4862
|
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
3947
|
-
const
|
|
3948
|
-
const
|
|
4863
|
+
const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
|
|
4864
|
+
const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
|
|
4865
|
+
const excludedPad = excludedIds.size;
|
|
4866
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
|
|
3949
4867
|
if (scoredEntityIds.length === 0) {
|
|
3950
4868
|
usedEmbed = true;
|
|
3951
4869
|
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
@@ -3979,7 +4897,7 @@ var RetrievalService = class {
|
|
|
3979
4897
|
let miniSearchScores;
|
|
3980
4898
|
if (effectivePreFilterLimit !== void 0) {
|
|
3981
4899
|
populateCache = false;
|
|
3982
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !
|
|
4900
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
|
|
3983
4901
|
if (preResults.length === 0) {
|
|
3984
4902
|
candidateRows = null;
|
|
3985
4903
|
} else {
|
|
@@ -4001,9 +4919,9 @@ var RetrievalService = class {
|
|
|
4001
4919
|
}
|
|
4002
4920
|
} else {
|
|
4003
4921
|
if (useRanker) {
|
|
4004
|
-
candidateRows = this.
|
|
4922
|
+
candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
|
|
4005
4923
|
} else {
|
|
4006
|
-
candidateRows = this.
|
|
4924
|
+
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
4007
4925
|
}
|
|
4008
4926
|
if (weight !== void 0 && weight < 1) {
|
|
4009
4927
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -4288,7 +5206,7 @@ var RetrievalService = class {
|
|
|
4288
5206
|
// read() re-sorts after applying tier weights
|
|
4289
5207
|
});
|
|
4290
5208
|
}
|
|
4291
|
-
if (
|
|
5209
|
+
if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
|
|
4292
5210
|
if (scored.length > 0) {
|
|
4293
5211
|
scored = scored.map((row) => ({
|
|
4294
5212
|
...row,
|
|
@@ -4352,7 +5270,7 @@ var RetrievalService = class {
|
|
|
4352
5270
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
4353
5271
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
4354
5272
|
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
4355
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !
|
|
5273
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
|
|
4356
5274
|
const candidates = results.map((r) => ({
|
|
4357
5275
|
id: r.id,
|
|
4358
5276
|
entity_id: r.entity_id,
|
|
@@ -4370,13 +5288,18 @@ var RetrievalService = class {
|
|
|
4370
5288
|
}
|
|
4371
5289
|
}
|
|
4372
5290
|
}
|
|
5291
|
+
facts = applyBudget(facts);
|
|
4373
5292
|
if (facts.length > 0) {
|
|
4374
5293
|
const ids = facts.map((f) => f.id);
|
|
4375
5294
|
const now = Date.now();
|
|
4376
5295
|
await this.entryRepo.trackAccess(ids, now);
|
|
4377
5296
|
}
|
|
4378
5297
|
} else {
|
|
4379
|
-
facts =
|
|
5298
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
|
|
5299
|
+
...excludeDrafts ? { excludeDrafts: true } : {},
|
|
5300
|
+
live: { mode: liveMode, t: liveAt }
|
|
5301
|
+
});
|
|
5302
|
+
facts = applyBudget(facts);
|
|
4380
5303
|
}
|
|
4381
5304
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
4382
5305
|
const [tasks, events] = await Promise.all([
|
|
@@ -4412,8 +5335,8 @@ var RetrievalService = class {
|
|
|
4412
5335
|
_tieBreakSort(items) {
|
|
4413
5336
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
4414
5337
|
}
|
|
4415
|
-
|
|
4416
|
-
return
|
|
5338
|
+
_withoutIds(rows, excludeIds) {
|
|
5339
|
+
return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
|
|
4417
5340
|
}
|
|
4418
5341
|
/**
|
|
4419
5342
|
* Comparator for score + deterministic tie-break fields.
|
|
@@ -4484,21 +5407,6 @@ var RetrievalService = class {
|
|
|
4484
5407
|
return scored;
|
|
4485
5408
|
}
|
|
4486
5409
|
};
|
|
4487
|
-
|
|
4488
|
-
// src/utils/cosine.ts
|
|
4489
|
-
function cosineSimilarity(a, b) {
|
|
4490
|
-
let dot = 0, normA = 0, normB = 0;
|
|
4491
|
-
const len = Math.min(a.length, b.length);
|
|
4492
|
-
for (let i = 0; i < len; i++) {
|
|
4493
|
-
dot += a[i] * b[i];
|
|
4494
|
-
normA += a[i] * a[i];
|
|
4495
|
-
normB += b[i] * b[i];
|
|
4496
|
-
}
|
|
4497
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
4498
|
-
return denom === 0 ? 0 : dot / denom;
|
|
4499
|
-
}
|
|
4500
|
-
|
|
4501
|
-
// src/services/SearchService.ts
|
|
4502
5410
|
var _SearchService = class _SearchService {
|
|
4503
5411
|
constructor(entryRepo) {
|
|
4504
5412
|
this.entryRepo = entryRepo;
|
|
@@ -5281,6 +6189,18 @@ var WriteService = class {
|
|
|
5281
6189
|
this.metadataRepo = metadataRepo;
|
|
5282
6190
|
this.jobManager = jobManager;
|
|
5283
6191
|
this.maintenanceService = maintenanceService;
|
|
6192
|
+
this.inflight = /* @__PURE__ */ new Set();
|
|
6193
|
+
}
|
|
6194
|
+
track(p) {
|
|
6195
|
+
this.inflight.add(p);
|
|
6196
|
+
p.finally(() => this.inflight.delete(p)).catch(() => {
|
|
6197
|
+
});
|
|
6198
|
+
}
|
|
6199
|
+
/** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
|
|
6200
|
+
async drain() {
|
|
6201
|
+
while (this.inflight.size > 0) {
|
|
6202
|
+
await Promise.allSettled([...this.inflight]);
|
|
6203
|
+
}
|
|
5284
6204
|
}
|
|
5285
6205
|
async write(entityId, event) {
|
|
5286
6206
|
if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
|
|
@@ -5307,6 +6227,7 @@ var WriteService = class {
|
|
|
5307
6227
|
}
|
|
5308
6228
|
const id = generateId("evt_");
|
|
5309
6229
|
const now = Date.now();
|
|
6230
|
+
const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
|
|
5310
6231
|
let eventType = event.event_type;
|
|
5311
6232
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
5312
6233
|
eventType = "observation";
|
|
@@ -5317,8 +6238,14 @@ var WriteService = class {
|
|
|
5317
6238
|
event_type: eventType,
|
|
5318
6239
|
summary,
|
|
5319
6240
|
related_entry_id: relatedEntryId,
|
|
5320
|
-
created_at: now
|
|
6241
|
+
created_at: now,
|
|
6242
|
+
occurred_at: occurredAt
|
|
5321
6243
|
};
|
|
6244
|
+
const deferred = this.options.config?.maintenance === "deferred";
|
|
6245
|
+
if (deferred) {
|
|
6246
|
+
await this.eventRepo.add(newEvent);
|
|
6247
|
+
return;
|
|
6248
|
+
}
|
|
5322
6249
|
let shouldRunLibrarian = false;
|
|
5323
6250
|
let librarianCount = 0;
|
|
5324
6251
|
let prevMemoryCheckpoint = 0;
|
|
@@ -5340,27 +6267,41 @@ var WriteService = class {
|
|
|
5340
6267
|
prevMemoryCheckpoint = memoryCheckpoint;
|
|
5341
6268
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
5342
6269
|
}
|
|
6270
|
+
} else {
|
|
6271
|
+
const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
|
|
6272
|
+
if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
|
|
6273
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
|
|
6274
|
+
const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
|
|
6275
|
+
if (Math.ceil(chars / 4) >= tokenThreshold) {
|
|
6276
|
+
shouldRunLibrarian = true;
|
|
6277
|
+
librarianCount = count;
|
|
6278
|
+
prevMemoryCheckpoint = memoryCheckpoint;
|
|
6279
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
6280
|
+
}
|
|
6281
|
+
}
|
|
5343
6282
|
}
|
|
5344
6283
|
});
|
|
5345
6284
|
if (shouldRunLibrarian) {
|
|
5346
6285
|
try {
|
|
5347
6286
|
this.jobManager.acquireLock("librarian", entityId);
|
|
5348
6287
|
const stage = { job: "librarian" };
|
|
5349
|
-
this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6288
|
+
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
5350
6289
|
console.error(err);
|
|
5351
6290
|
this.reportBackgroundFailure(entityId, stage.job);
|
|
5352
6291
|
}).finally(() => {
|
|
5353
6292
|
this.jobManager.releaseLock("librarian", entityId);
|
|
5354
6293
|
});
|
|
6294
|
+
this.track(job);
|
|
5355
6295
|
} catch (e) {
|
|
5356
6296
|
if (!(e instanceof WikiBusyError)) throw e;
|
|
5357
6297
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
|
|
5358
6298
|
}
|
|
5359
6299
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
5360
|
-
this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6300
|
+
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
5361
6301
|
console.error(err);
|
|
5362
6302
|
this.reportBackgroundFailure(entityId, "heal");
|
|
5363
6303
|
});
|
|
6304
|
+
this.track(healJob);
|
|
5364
6305
|
}
|
|
5365
6306
|
}
|
|
5366
6307
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {
|