@equationalapplications/core-llm-wiki 7.7.7 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-46X3HO77.mjs → chunk-M76CF2Q6.mjs} +1180 -185
- package/dist/chunk-M76CF2Q6.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1824 -285
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +575 -32
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BIFNLUva.d.mts → testing-CWZaXKLj.d.mts} +560 -4
- package/dist/{testing-BIFNLUva.d.ts → testing-CWZaXKLj.d.ts} +560 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1129 -197
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-46X3HO77.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -153,6 +153,15 @@ var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
|
|
|
153
153
|
Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
|
|
154
154
|
}
|
|
155
155
|
};
|
|
156
|
+
var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
157
|
+
constructor(reason) {
|
|
158
|
+
super(`supersede() rejected: ${reason}`);
|
|
159
|
+
this.code = "WIKI_SUPERSEDE_REJECTED";
|
|
160
|
+
this.name = "WikiSupersedeError";
|
|
161
|
+
this.reason = reason;
|
|
162
|
+
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
163
|
+
}
|
|
164
|
+
};
|
|
156
165
|
|
|
157
166
|
// src/utils/pure.ts
|
|
158
167
|
var MAX_REPAIR_CANDIDATES = 5;
|
|
@@ -686,7 +695,14 @@ var SEVERITY = {
|
|
|
686
695
|
grounding_missing: "warn",
|
|
687
696
|
grounding_failed: "warn",
|
|
688
697
|
classification_low_confidence: "info",
|
|
689
|
-
classification_invalid: "warn"
|
|
698
|
+
classification_invalid: "warn",
|
|
699
|
+
llm_usage: "info",
|
|
700
|
+
librarian_gate: "info",
|
|
701
|
+
librarian_op_rejected: "warn",
|
|
702
|
+
contradicts_document: "warn",
|
|
703
|
+
resolve_failed: "warn",
|
|
704
|
+
read_budget: "info",
|
|
705
|
+
event_retention_held: "info"
|
|
690
706
|
};
|
|
691
707
|
var MESSAGE = {
|
|
692
708
|
ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
|
|
@@ -701,7 +717,14 @@ var MESSAGE = {
|
|
|
701
717
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
702
718
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
703
719
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
704
|
-
classification_invalid: "A classifier answer was invalid and was not applied."
|
|
720
|
+
classification_invalid: "A classifier answer was invalid and was not applied.",
|
|
721
|
+
llm_usage: "An LLM call completed; token usage attached.",
|
|
722
|
+
librarian_gate: "Librarian gate outcome counts for one pass.",
|
|
723
|
+
librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
|
|
724
|
+
contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
|
|
725
|
+
resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
|
|
726
|
+
read_budget: "A read was packed to a token budget.",
|
|
727
|
+
event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
|
|
705
728
|
};
|
|
706
729
|
var warnedNonFunction = /* @__PURE__ */ new WeakSet();
|
|
707
730
|
function copyDetail(detail) {
|
|
@@ -851,6 +874,21 @@ var EmbeddingService = class {
|
|
|
851
874
|
}
|
|
852
875
|
}
|
|
853
876
|
async tryEmbedFact(fact, ctx) {
|
|
877
|
+
const r = await this.embedTextForFact(fact, ctx);
|
|
878
|
+
return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
|
|
879
|
+
}
|
|
880
|
+
/**
|
|
881
|
+
* Compute half: build the embed text, call embed(), validate the result,
|
|
882
|
+
* mark any embed-time failure, and return the validated Float32 vector.
|
|
883
|
+
*
|
|
884
|
+
* Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
|
|
885
|
+
* `provider_error`. All embed-time failures are marked via `markFailure`;
|
|
886
|
+
* `no_provider` deliberately does not mark (spec §2.4).
|
|
887
|
+
*
|
|
888
|
+
* Storage errors are NOT possible here — that is `storeFactVector`'s domain.
|
|
889
|
+
*/
|
|
890
|
+
async embedTextForFact(fact, ctx, opts) {
|
|
891
|
+
const markFailures = opts?.markFailures !== false;
|
|
854
892
|
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
855
893
|
let tagsStr;
|
|
856
894
|
if (Array.isArray(fact.tags)) {
|
|
@@ -871,8 +909,8 @@ var EmbeddingService = class {
|
|
|
871
909
|
const vector = await this.options.llmProvider.embed(text);
|
|
872
910
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
873
911
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
874
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
875
|
-
await this.markFailure(fact.id, "invalid_vector");
|
|
912
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
913
|
+
if (markFailures) await this.markFailure(fact.id, "invalid_vector");
|
|
876
914
|
return { ok: false, kind: "invalid_vector" };
|
|
877
915
|
}
|
|
878
916
|
float32Vector = new Float32Array(vector);
|
|
@@ -885,16 +923,24 @@ var EmbeddingService = class {
|
|
|
885
923
|
}
|
|
886
924
|
if (hasNonFinite) {
|
|
887
925
|
console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
|
|
888
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
889
|
-
await this.markFailure(fact.id, "float32_overflow");
|
|
926
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
927
|
+
if (markFailures) await this.markFailure(fact.id, "float32_overflow");
|
|
890
928
|
return { ok: false, kind: "float32_overflow" };
|
|
891
929
|
}
|
|
892
930
|
} catch (err) {
|
|
893
931
|
console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
|
|
894
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
895
|
-
await this.markFailure(fact.id, "provider_error");
|
|
932
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
933
|
+
if (markFailures) await this.markFailure(fact.id, "provider_error");
|
|
896
934
|
return { ok: false, kind: "provider_error" };
|
|
897
935
|
}
|
|
936
|
+
return { ok: true, vector: float32Vector };
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Storage half: persist the computed vector to the entry row, then fire the
|
|
940
|
+
* `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
|
|
941
|
+
* failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
|
|
942
|
+
*/
|
|
943
|
+
async storeFactVector(fact, float32Vector, ctx) {
|
|
898
944
|
try {
|
|
899
945
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
900
946
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
@@ -1246,6 +1292,12 @@ var ImportExportService = class {
|
|
|
1246
1292
|
})()
|
|
1247
1293
|
};
|
|
1248
1294
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
1295
|
+
await this.entryRepo.setTemporal(fact.id, entityId, {
|
|
1296
|
+
valid_from: fact.valid_from ?? null,
|
|
1297
|
+
valid_to: fact.valid_to ?? null,
|
|
1298
|
+
superseded_by: fact.superseded_by ?? null,
|
|
1299
|
+
superseded_at: fact.superseded_at ?? null
|
|
1300
|
+
}, tx, { outbox: false });
|
|
1249
1301
|
if (blobData != null) {
|
|
1250
1302
|
factsWithPreservedBlob.set(fact.id, blobData);
|
|
1251
1303
|
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
@@ -1324,7 +1376,8 @@ var ImportExportService = class {
|
|
|
1324
1376
|
event_type: event.event_type,
|
|
1325
1377
|
summary: event.summary,
|
|
1326
1378
|
related_entry_id: event.related_entry_id ?? null,
|
|
1327
|
-
created_at: event.created_at
|
|
1379
|
+
created_at: event.created_at,
|
|
1380
|
+
occurred_at: event.occurred_at ?? null
|
|
1328
1381
|
},
|
|
1329
1382
|
tx
|
|
1330
1383
|
);
|
|
@@ -1557,6 +1610,21 @@ function groundingEvidenceBlock(writer, cfg) {
|
|
|
1557
1610
|
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1558
1611
|
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1559
1612
|
}
|
|
1613
|
+
var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
|
|
1614
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1615
|
+
{
|
|
1616
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
|
|
1617
|
+
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
1618
|
+
}
|
|
1619
|
+
One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
|
|
1620
|
+
var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
|
|
1621
|
+
- "ADD": the candidate is new information; the existing facts are about something else.
|
|
1622
|
+
- "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
|
|
1623
|
+
- "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
|
|
1624
|
+
- "NOOP": an existing fact already states the candidate.
|
|
1625
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1626
|
+
{ "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
|
|
1627
|
+
"target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
|
|
1560
1628
|
|
|
1561
1629
|
// src/utils/healConstants.ts
|
|
1562
1630
|
var HEAL_MAX_ANCHORS = 50;
|
|
@@ -1795,6 +1863,22 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
1795
1863
|
${JSON.stringify(facts, null, 2)}`
|
|
1796
1864
|
};
|
|
1797
1865
|
}
|
|
1866
|
+
/** Ops extract (spec §5.2): events only — no existing facts. */
|
|
1867
|
+
buildOpsExtractPrompt(events, ontologyContext) {
|
|
1868
|
+
const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
|
|
1869
|
+
const grounding = this.groundingFor("librarian");
|
|
1870
|
+
return {
|
|
1871
|
+
systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
|
|
1872
|
+
userPrompt: `Events:
|
|
1873
|
+
${lines.join("\n")}`,
|
|
1874
|
+
...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
|
|
1875
|
+
};
|
|
1876
|
+
}
|
|
1877
|
+
/** Ops resolve (spec §5.4): only each item's own neighbours. */
|
|
1878
|
+
buildOpsResolvePrompt(items) {
|
|
1879
|
+
return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
|
|
1880
|
+
${JSON.stringify({ items })}` };
|
|
1881
|
+
}
|
|
1798
1882
|
/**
|
|
1799
1883
|
* The system prompt each writer would send, without hydrating data (spec
|
|
1800
1884
|
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
@@ -1859,6 +1943,59 @@ function toGroundingAnchor(anchor) {
|
|
|
1859
1943
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
1860
1944
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
1861
1945
|
|
|
1946
|
+
// src/utils/usage.ts
|
|
1947
|
+
function estimateTokens(text) {
|
|
1948
|
+
return Math.ceil(text.length / 4);
|
|
1949
|
+
}
|
|
1950
|
+
var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
|
|
1951
|
+
constructor(requiredEstimate, remaining) {
|
|
1952
|
+
super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
|
|
1953
|
+
this.name = "WikiBudgetExhausted";
|
|
1954
|
+
this.requiredEstimate = requiredEstimate;
|
|
1955
|
+
this.remaining = remaining;
|
|
1956
|
+
Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
|
|
1957
|
+
}
|
|
1958
|
+
};
|
|
1959
|
+
|
|
1960
|
+
// src/utils/llmCall.ts
|
|
1961
|
+
function validUsage(u) {
|
|
1962
|
+
if (!u || typeof u !== "object") return false;
|
|
1963
|
+
const { inputTokens, outputTokens } = u;
|
|
1964
|
+
return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
|
|
1965
|
+
}
|
|
1966
|
+
async function callLlm(options, ctx, prompts) {
|
|
1967
|
+
const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
|
|
1968
|
+
if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
|
|
1969
|
+
throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
|
|
1970
|
+
}
|
|
1971
|
+
const provider = options.llmProvider;
|
|
1972
|
+
const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
|
|
1973
|
+
let text;
|
|
1974
|
+
let reported;
|
|
1975
|
+
if (typeof provider.generateTextWithUsage === "function") {
|
|
1976
|
+
const result = await provider.generateTextWithUsage(params);
|
|
1977
|
+
if (!result || typeof result.text !== "string") {
|
|
1978
|
+
throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
|
|
1979
|
+
}
|
|
1980
|
+
text = result.text;
|
|
1981
|
+
reported = validUsage(result.usage) ? result.usage : void 0;
|
|
1982
|
+
} else {
|
|
1983
|
+
text = await provider.generateText(params);
|
|
1984
|
+
}
|
|
1985
|
+
const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
|
|
1986
|
+
ctx.meter?.record(record);
|
|
1987
|
+
if (options.config?.reportLlmUsage === true) {
|
|
1988
|
+
emitDiagnostic(options, {
|
|
1989
|
+
code: "llm_usage",
|
|
1990
|
+
operation: ctx.operation,
|
|
1991
|
+
trigger: ctx.trigger,
|
|
1992
|
+
entityId: ctx.entityId,
|
|
1993
|
+
detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
return text;
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1862
1999
|
// src/services/IngestionService.ts
|
|
1863
2000
|
var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
|
|
1864
2001
|
function dedupDiagnostic(entityId, sourceRef, locator) {
|
|
@@ -1935,7 +2072,7 @@ var IngestionService = class {
|
|
|
1935
2072
|
ontologyContext
|
|
1936
2073
|
);
|
|
1937
2074
|
try {
|
|
1938
|
-
const responseText = await this.options
|
|
2075
|
+
const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
|
|
1939
2076
|
const result2 = parseJsonResponse(responseText);
|
|
1940
2077
|
const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
|
|
1941
2078
|
const facts = [];
|
|
@@ -2619,6 +2756,8 @@ async function runBatched(args) {
|
|
|
2619
2756
|
const results = [];
|
|
2620
2757
|
const skipped = [];
|
|
2621
2758
|
let batches = 0;
|
|
2759
|
+
const settled = /* @__PURE__ */ new Set();
|
|
2760
|
+
let budgetStop;
|
|
2622
2761
|
let batchSize = initialBatchSize(maxOutputTokens);
|
|
2623
2762
|
const trim = async (candidate) => {
|
|
2624
2763
|
const whole = await buildPrompt(candidate);
|
|
@@ -2652,6 +2791,7 @@ async function runBatched(args) {
|
|
|
2652
2791
|
} else {
|
|
2653
2792
|
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
2654
2793
|
skipped.push({ item: batch[0], reason });
|
|
2794
|
+
settled.add(batch[0]);
|
|
2655
2795
|
onSkip?.(batch[0], err);
|
|
2656
2796
|
}
|
|
2657
2797
|
return;
|
|
@@ -2660,7 +2800,7 @@ async function runBatched(args) {
|
|
|
2660
2800
|
const mid = Math.ceil(batch.length / 2);
|
|
2661
2801
|
if (mid < batchSize) batchSize = mid;
|
|
2662
2802
|
let i = 0;
|
|
2663
|
-
while (i < batch.length) {
|
|
2803
|
+
while (i < batch.length && !budgetStop) {
|
|
2664
2804
|
const size = Math.min(batchSize, batch.length - i);
|
|
2665
2805
|
const trimmed = await trim(batch.slice(i, i + size));
|
|
2666
2806
|
await attempt(trimmed.batch, trimmed.prompts);
|
|
@@ -2668,6 +2808,7 @@ async function runBatched(args) {
|
|
|
2668
2808
|
}
|
|
2669
2809
|
};
|
|
2670
2810
|
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
2811
|
+
if (budgetStop) return;
|
|
2671
2812
|
if (batch.length === 0) return;
|
|
2672
2813
|
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
2673
2814
|
batches++;
|
|
@@ -2675,6 +2816,10 @@ async function runBatched(args) {
|
|
|
2675
2816
|
try {
|
|
2676
2817
|
responseText = await call(prompts);
|
|
2677
2818
|
} catch (err) {
|
|
2819
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
2820
|
+
budgetStop = { requiredEstimate: err.requiredEstimate };
|
|
2821
|
+
return;
|
|
2822
|
+
}
|
|
2678
2823
|
await onFailure(batch, err, attemptLevel, true);
|
|
2679
2824
|
return;
|
|
2680
2825
|
}
|
|
@@ -2686,19 +2831,784 @@ async function runBatched(args) {
|
|
|
2686
2831
|
return;
|
|
2687
2832
|
}
|
|
2688
2833
|
results.push(result);
|
|
2834
|
+
for (const item of batch) settled.add(item);
|
|
2689
2835
|
};
|
|
2690
2836
|
let index = 0;
|
|
2691
|
-
while (index < items.length) {
|
|
2837
|
+
while (index < items.length && !budgetStop) {
|
|
2692
2838
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
2693
2839
|
index += batch.length;
|
|
2694
2840
|
await attempt(batch, prompts, 0);
|
|
2695
2841
|
}
|
|
2696
|
-
|
|
2842
|
+
const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
|
|
2843
|
+
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
2697
2844
|
}
|
|
2698
2845
|
|
|
2699
|
-
// src/services/
|
|
2846
|
+
// src/services/librarian/constants.ts
|
|
2700
2847
|
var FUZZY_THRESHOLD = 0.5;
|
|
2701
2848
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
2849
|
+
|
|
2850
|
+
// src/services/librarian/legacy.ts
|
|
2851
|
+
var runLegacyLibrarianPass = async (deps, ctx) => {
|
|
2852
|
+
const { entityId, trigger } = ctx;
|
|
2853
|
+
const promptOverride = ctx.promptOverride;
|
|
2854
|
+
const events = await deps.eventRepo.getRecent(entityId, 50);
|
|
2855
|
+
const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
|
|
2856
|
+
const currentFacts = currentFactsRows.map((f) => {
|
|
2857
|
+
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
2858
|
+
return {
|
|
2859
|
+
...rest,
|
|
2860
|
+
tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
|
|
2861
|
+
};
|
|
2862
|
+
});
|
|
2863
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
2864
|
+
const promptEvents = events.reverse();
|
|
2865
|
+
const librarianGrounding = deps.promptService.groundingFor("librarian");
|
|
2866
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
|
|
2867
|
+
promptEvents,
|
|
2868
|
+
currentFacts,
|
|
2869
|
+
promptOverride,
|
|
2870
|
+
ontologyContext
|
|
2871
|
+
);
|
|
2872
|
+
let responseText;
|
|
2873
|
+
try {
|
|
2874
|
+
responseText = await callLlm(
|
|
2875
|
+
deps.options,
|
|
2876
|
+
{ operation: "librarian", entityId, trigger, meter: ctx.meter },
|
|
2877
|
+
{ systemPrompt, userPrompt }
|
|
2878
|
+
);
|
|
2879
|
+
} catch (err) {
|
|
2880
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
2881
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
2882
|
+
}
|
|
2883
|
+
throw err;
|
|
2884
|
+
}
|
|
2885
|
+
const result = parseJsonResponse(responseText);
|
|
2886
|
+
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
2887
|
+
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
2888
|
+
const ontologyUpdates = result.ontology_updates;
|
|
2889
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
2890
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
2891
|
+
const validFacts = [];
|
|
2892
|
+
const validFactItemIndexes = [];
|
|
2893
|
+
facts.forEach((raw, itemIndex) => {
|
|
2894
|
+
const valid = validateFact(raw);
|
|
2895
|
+
if (valid) {
|
|
2896
|
+
validFacts.push(valid);
|
|
2897
|
+
validFactItemIndexes.push(itemIndex);
|
|
2898
|
+
} else {
|
|
2899
|
+
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
2900
|
+
}
|
|
2901
|
+
});
|
|
2902
|
+
const validTasks = [];
|
|
2903
|
+
tasks.forEach((raw, itemIndex) => {
|
|
2904
|
+
const valid = validateTask(raw);
|
|
2905
|
+
if (valid) validTasks.push(valid);
|
|
2906
|
+
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
2907
|
+
});
|
|
2908
|
+
const now = Date.now();
|
|
2909
|
+
const insertedFacts = [];
|
|
2910
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
2911
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
2912
|
+
if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
|
|
2913
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
2914
|
+
}
|
|
2915
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
2916
|
+
for (const existing of currentFactsRows) {
|
|
2917
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
2918
|
+
id: existing.id,
|
|
2919
|
+
okf_type: existing.okf_type ?? null
|
|
2920
|
+
});
|
|
2921
|
+
}
|
|
2922
|
+
const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
2923
|
+
const pendingEdges = [];
|
|
2924
|
+
for (const [k, fact] of validFacts.entries()) {
|
|
2925
|
+
const newTokens = titleTokens(fact.title);
|
|
2926
|
+
let skip = false;
|
|
2927
|
+
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
2928
|
+
for (const existing of factsForDedupe) {
|
|
2929
|
+
if (existing.source_type !== "librarian_inferred") continue;
|
|
2930
|
+
const existingTokens = titleTokens(existing.title);
|
|
2931
|
+
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
2932
|
+
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
2933
|
+
skip = true;
|
|
2934
|
+
break;
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
}
|
|
2938
|
+
}
|
|
2939
|
+
if (skip) {
|
|
2940
|
+
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
2941
|
+
continue;
|
|
2942
|
+
}
|
|
2943
|
+
const ontologyFact = fact;
|
|
2944
|
+
const id = generateId("fact_");
|
|
2945
|
+
const validationDrops = [];
|
|
2946
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
2947
|
+
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
2948
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
2949
|
+
if (grounding?.diagnostic) {
|
|
2950
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
2951
|
+
}
|
|
2952
|
+
const factObj = {
|
|
2953
|
+
id,
|
|
2954
|
+
entity_id: entityId,
|
|
2955
|
+
title: fact.title,
|
|
2956
|
+
body: fact.body,
|
|
2957
|
+
tags: fact.tags,
|
|
2958
|
+
confidence: fact.confidence,
|
|
2959
|
+
source_type: "librarian_inferred",
|
|
2960
|
+
source_hash: null,
|
|
2961
|
+
source_ref: null,
|
|
2962
|
+
created_at: now,
|
|
2963
|
+
updated_at: now,
|
|
2964
|
+
last_accessed_at: null,
|
|
2965
|
+
access_count: 0,
|
|
2966
|
+
deleted_at: null,
|
|
2967
|
+
okf_type: normalized.okf_type,
|
|
2968
|
+
...grounding?.trust
|
|
2969
|
+
};
|
|
2970
|
+
await deps.entryRepo.upsert(factObj, tx);
|
|
2971
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2972
|
+
factsForDedupe.push(factObj);
|
|
2973
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
2974
|
+
if (normalized.edges.length > 0) {
|
|
2975
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
2976
|
+
}
|
|
2977
|
+
}
|
|
2978
|
+
for (const item of pendingEdges) {
|
|
2979
|
+
const resolveDrops = [];
|
|
2980
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
2981
|
+
entityId,
|
|
2982
|
+
item.sourceId,
|
|
2983
|
+
item.sourceType,
|
|
2984
|
+
item.edges ?? [],
|
|
2985
|
+
manifest,
|
|
2986
|
+
titleIndex,
|
|
2987
|
+
tx,
|
|
2988
|
+
now,
|
|
2989
|
+
resolveDrops
|
|
2990
|
+
);
|
|
2991
|
+
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
2992
|
+
}
|
|
2993
|
+
for (const task of validTasks) {
|
|
2994
|
+
const id = generateId("task_");
|
|
2995
|
+
const taskObj = {
|
|
2996
|
+
id,
|
|
2997
|
+
entity_id: entityId,
|
|
2998
|
+
description: task.description,
|
|
2999
|
+
status: "pending",
|
|
3000
|
+
priority: task.priority,
|
|
3001
|
+
created_at: now,
|
|
3002
|
+
updated_at: now,
|
|
3003
|
+
resolved_at: null,
|
|
3004
|
+
deleted_at: null
|
|
3005
|
+
};
|
|
3006
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
3007
|
+
}
|
|
3008
|
+
});
|
|
3009
|
+
diagBuffer.flush(deps.options);
|
|
3010
|
+
await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
3011
|
+
for (const fact of insertedFacts) {
|
|
3012
|
+
await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
3013
|
+
}
|
|
3014
|
+
deps.searchService.evictCache(entityId);
|
|
3015
|
+
const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
|
|
3016
|
+
const out = { processedThrough, factsWritten: insertedFacts.length };
|
|
3017
|
+
return out;
|
|
3018
|
+
};
|
|
3019
|
+
|
|
3020
|
+
// src/services/librarian/ops/text.ts
|
|
3021
|
+
var OPS_NEIGHBOUR_BODY_CHARS = 400;
|
|
3022
|
+
function normalizeFactText(title, body) {
|
|
3023
|
+
return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
3024
|
+
}
|
|
3025
|
+
function labelEvents(events) {
|
|
3026
|
+
return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
|
|
3027
|
+
}
|
|
3028
|
+
function parseValidFrom(v) {
|
|
3029
|
+
if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
|
|
3030
|
+
if (typeof v === "string") {
|
|
3031
|
+
const t = Date.parse(v);
|
|
3032
|
+
return Number.isFinite(t) && t >= 0 ? t : void 0;
|
|
3033
|
+
}
|
|
3034
|
+
return void 0;
|
|
3035
|
+
}
|
|
3036
|
+
|
|
3037
|
+
// src/services/librarian/ops/extract.ts
|
|
3038
|
+
var OPS_MAX_EVENTS_PER_PASS = 50;
|
|
3039
|
+
var OPS_MAX_EXTRACT_CHARS = 24e3;
|
|
3040
|
+
var RESOLVE_RESERVE_FACTOR = 2;
|
|
3041
|
+
var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
|
|
3042
|
+
async function selectBatch(deps, entityId, wm, meter) {
|
|
3043
|
+
const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
|
|
3044
|
+
if (events.length === 0) return { batch: [] };
|
|
3045
|
+
let n = 0;
|
|
3046
|
+
let chars = 0;
|
|
3047
|
+
for (const e of events) {
|
|
3048
|
+
if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
|
|
3049
|
+
chars += e.summary.length;
|
|
3050
|
+
n++;
|
|
3051
|
+
}
|
|
3052
|
+
let batch = labelEvents(events.slice(0, n));
|
|
3053
|
+
if (meter && meter.budget !== void 0) {
|
|
3054
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3055
|
+
for (; ; ) {
|
|
3056
|
+
const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
|
|
3057
|
+
if (meter.fits(need)) break;
|
|
3058
|
+
if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
|
|
3059
|
+
batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
|
|
3060
|
+
}
|
|
3061
|
+
}
|
|
3062
|
+
return { batch };
|
|
3063
|
+
}
|
|
3064
|
+
async function runExtract(deps, ctx, batch) {
|
|
3065
|
+
const { entityId, trigger } = ctx;
|
|
3066
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3067
|
+
const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
|
|
3068
|
+
const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
|
|
3069
|
+
const parsed = parseJsonResponse(text);
|
|
3070
|
+
const base = { entityId, operation: "librarian", trigger };
|
|
3071
|
+
const diagnostics = [];
|
|
3072
|
+
const byLabel = new Map(batch.map((l) => [l.label, l]));
|
|
3073
|
+
const newestAt = batch[batch.length - 1].at;
|
|
3074
|
+
const candidates = [];
|
|
3075
|
+
(Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
|
|
3076
|
+
const fact = validateFact(raw);
|
|
3077
|
+
if (!fact) {
|
|
3078
|
+
diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
|
|
3079
|
+
return;
|
|
3080
|
+
}
|
|
3081
|
+
const r = raw;
|
|
3082
|
+
const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
|
|
3083
|
+
const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
|
|
3084
|
+
candidates.push({ index, fact, sourceLabel, validFrom });
|
|
3085
|
+
});
|
|
3086
|
+
const tasks = [];
|
|
3087
|
+
(Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
|
|
3088
|
+
const task = validateTask(raw);
|
|
3089
|
+
if (task) tasks.push(task);
|
|
3090
|
+
else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
|
|
3091
|
+
});
|
|
3092
|
+
return {
|
|
3093
|
+
batch,
|
|
3094
|
+
candidates,
|
|
3095
|
+
tasks,
|
|
3096
|
+
...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
|
|
3097
|
+
groundingCorpus: prompt.groundingCorpus ?? null,
|
|
3098
|
+
diagnostics
|
|
3099
|
+
};
|
|
3100
|
+
}
|
|
3101
|
+
|
|
3102
|
+
// src/utils/cosine.ts
|
|
3103
|
+
function cosineSimilarity(a, b) {
|
|
3104
|
+
let dot = 0, normA = 0, normB = 0;
|
|
3105
|
+
const len = Math.min(a.length, b.length);
|
|
3106
|
+
for (let i = 0; i < len; i++) {
|
|
3107
|
+
dot += a[i] * b[i];
|
|
3108
|
+
normA += a[i] * a[i];
|
|
3109
|
+
normB += b[i] * b[i];
|
|
3110
|
+
}
|
|
3111
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
3112
|
+
return denom === 0 ? 0 : dot / denom;
|
|
3113
|
+
}
|
|
3114
|
+
|
|
3115
|
+
// src/services/librarian/ops/gate.ts
|
|
3116
|
+
var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
|
|
3117
|
+
function resolveGateConfig(cfg) {
|
|
3118
|
+
const kRaw = cfg?.k;
|
|
3119
|
+
const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
|
|
3120
|
+
const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
|
|
3121
|
+
const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
|
|
3122
|
+
if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
|
|
3123
|
+
return { k, dupThreshold, novelThreshold };
|
|
3124
|
+
}
|
|
3125
|
+
function titlesMatch(a, b) {
|
|
3126
|
+
const ta = titleTokens(a);
|
|
3127
|
+
const tb = titleTokens(b);
|
|
3128
|
+
if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
|
|
3129
|
+
return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
|
|
3130
|
+
}
|
|
3131
|
+
function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
3132
|
+
const key = normalizeFactText(candidate.title, candidate.body);
|
|
3133
|
+
const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
|
|
3134
|
+
if (same) return { kind: "noop", target: same.id };
|
|
3135
|
+
if (neighbours.length === 0) return { kind: "add" };
|
|
3136
|
+
if (mode === "keyword") return { kind: "ambiguous" };
|
|
3137
|
+
const best = neighbours[0];
|
|
3138
|
+
const score = best.score ?? 0;
|
|
3139
|
+
if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
|
|
3140
|
+
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3141
|
+
return { kind: "ambiguous" };
|
|
3142
|
+
}
|
|
3143
|
+
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3144
|
+
const { entityId, trigger } = ctx;
|
|
3145
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3146
|
+
const unique = [];
|
|
3147
|
+
const inBatchDuplicates = [];
|
|
3148
|
+
for (const c of candidates) {
|
|
3149
|
+
const key = normalizeFactText(c.fact.title, c.fact.body);
|
|
3150
|
+
if (seen.has(key)) inBatchDuplicates.push(c);
|
|
3151
|
+
else {
|
|
3152
|
+
seen.add(key);
|
|
3153
|
+
unique.push(c);
|
|
3154
|
+
}
|
|
3155
|
+
}
|
|
3156
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3157
|
+
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3158
|
+
const stored = [];
|
|
3159
|
+
if (canEmbed && unique.length > 0) {
|
|
3160
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3161
|
+
if (nonLive.has(row.id)) continue;
|
|
3162
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3163
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
const gated = [];
|
|
3167
|
+
for (const c of unique) {
|
|
3168
|
+
let ranked = null;
|
|
3169
|
+
let vector = null;
|
|
3170
|
+
let mode = "keyword";
|
|
3171
|
+
if (canEmbed) {
|
|
3172
|
+
const r = await deps.embeddingService.embedTextForFact(
|
|
3173
|
+
{ id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
|
|
3174
|
+
void 0,
|
|
3175
|
+
{ markFailures: false }
|
|
3176
|
+
);
|
|
3177
|
+
if (r.ok) {
|
|
3178
|
+
vector = r.vector;
|
|
3179
|
+
if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
|
|
3180
|
+
ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
|
|
3181
|
+
mode = "vector";
|
|
3182
|
+
}
|
|
3183
|
+
}
|
|
3184
|
+
}
|
|
3185
|
+
if (!ranked) {
|
|
3186
|
+
ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
3187
|
+
}
|
|
3188
|
+
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3189
|
+
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
3190
|
+
const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
|
|
3191
|
+
const f = byId.get(r.id);
|
|
3192
|
+
return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
|
|
3193
|
+
});
|
|
3194
|
+
gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
|
|
3195
|
+
}
|
|
3196
|
+
const count = (k) => gated.filter((g) => g.decision.kind === k).length;
|
|
3197
|
+
const gateDiagnostic = {
|
|
3198
|
+
code: "librarian_gate",
|
|
3199
|
+
operation: "librarian",
|
|
3200
|
+
trigger,
|
|
3201
|
+
entityId,
|
|
3202
|
+
detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
|
|
3203
|
+
};
|
|
3204
|
+
return { gated, inBatchDuplicates, gateDiagnostic };
|
|
3205
|
+
}
|
|
3206
|
+
|
|
3207
|
+
// src/services/librarian/ops/resolve.ts
|
|
3208
|
+
var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
|
|
3209
|
+
function validateOp(raw, aliases) {
|
|
3210
|
+
if (!raw || typeof raw !== "object") return null;
|
|
3211
|
+
const r = raw;
|
|
3212
|
+
if (r.op === "ADD") return { op: "ADD" };
|
|
3213
|
+
if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
|
|
3214
|
+
const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
|
|
3215
|
+
if (!targetId) return null;
|
|
3216
|
+
if (r.op === "NOOP") return { op: "NOOP", targetId };
|
|
3217
|
+
if (r.op === "SUPERSEDE") {
|
|
3218
|
+
const validFrom = parseValidFrom(r.valid_from);
|
|
3219
|
+
return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
|
|
3220
|
+
}
|
|
3221
|
+
const update = { op: "UPDATE", targetId };
|
|
3222
|
+
if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
|
|
3223
|
+
if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
|
|
3224
|
+
return update;
|
|
3225
|
+
}
|
|
3226
|
+
var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
|
|
3227
|
+
async function runResolve(deps, ctx, ambiguous) {
|
|
3228
|
+
const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
|
|
3229
|
+
if (ambiguous.length === 0) return out;
|
|
3230
|
+
const { entityId, trigger } = ctx;
|
|
3231
|
+
const outcome = await runBatched({
|
|
3232
|
+
items: ambiguous,
|
|
3233
|
+
buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
|
|
3234
|
+
batch.map((g, i) => ({
|
|
3235
|
+
item: i,
|
|
3236
|
+
candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
|
|
3237
|
+
existing: g.neighbours.map((n) => ({
|
|
3238
|
+
ref: n.ref,
|
|
3239
|
+
title: n.title,
|
|
3240
|
+
body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
|
|
3241
|
+
}))
|
|
3242
|
+
}))
|
|
3243
|
+
),
|
|
3244
|
+
call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
|
|
3245
|
+
parse: (text, batch) => {
|
|
3246
|
+
const parsed = parseJsonResponse(text);
|
|
3247
|
+
if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
|
|
3248
|
+
return { batch, ops: parsed.ops };
|
|
3249
|
+
},
|
|
3250
|
+
maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
|
|
3251
|
+
maxOutputTokens: deps.options.llmProvider.maxOutputTokens
|
|
3252
|
+
});
|
|
3253
|
+
if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
|
|
3254
|
+
for (const { batch, ops } of outcome.results) {
|
|
3255
|
+
const byItem = /* @__PURE__ */ new Map();
|
|
3256
|
+
for (const raw of ops) {
|
|
3257
|
+
const item = raw?.item;
|
|
3258
|
+
if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
|
|
3259
|
+
byItem.set(item, raw);
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
batch.forEach((g, i) => {
|
|
3263
|
+
const op = validateOp(byItem.get(i), aliasMapOf(g));
|
|
3264
|
+
if (op) out.ops.set(g, op);
|
|
3265
|
+
else out.rejected.push(g);
|
|
3266
|
+
});
|
|
3267
|
+
}
|
|
3268
|
+
if (outcome.skipped.some((s) => s.reason === "call_error")) {
|
|
3269
|
+
throw new Error("resolve: transient provider error; retry the pass");
|
|
3270
|
+
}
|
|
3271
|
+
for (const s of outcome.skipped) out.failed.push(s.item);
|
|
3272
|
+
return out;
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
// src/utils/temporal.ts
|
|
3276
|
+
function assertEpochMs(name, v) {
|
|
3277
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
3278
|
+
throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
|
|
3279
|
+
}
|
|
3280
|
+
return Math.trunc(v);
|
|
3281
|
+
}
|
|
3282
|
+
|
|
3283
|
+
// src/services/TemporalService.ts
|
|
3284
|
+
var HISTORY_MAX_DEPTH = 100;
|
|
3285
|
+
var TemporalService = class {
|
|
3286
|
+
constructor(db, options, entryRepo, searchService, embeddingService) {
|
|
3287
|
+
this.db = db;
|
|
3288
|
+
this.options = options;
|
|
3289
|
+
this.entryRepo = entryRepo;
|
|
3290
|
+
this.searchService = searchService;
|
|
3291
|
+
this.embeddingService = embeddingService;
|
|
3292
|
+
}
|
|
3293
|
+
/**
|
|
3294
|
+
* Link `oldId` → `newId` inside the caller's transaction. `newId` must
|
|
3295
|
+
* already exist. Throws WikiSupersedeError on any rule violation; the
|
|
3296
|
+
* caller's transaction then rolls back.
|
|
3297
|
+
*/
|
|
3298
|
+
async supersedeInTx(tx, entityId, oldId, newId, t, now) {
|
|
3299
|
+
const old = await this.entryRepo.findTemporalRow(oldId, tx);
|
|
3300
|
+
if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3301
|
+
if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3302
|
+
if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3303
|
+
if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
|
|
3304
|
+
if (newId === oldId) throw new WikiSupersedeError("cycle");
|
|
3305
|
+
const repl = await this.entryRepo.findTemporalRow(newId, tx);
|
|
3306
|
+
if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3307
|
+
if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3308
|
+
if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3309
|
+
const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
|
|
3310
|
+
if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
|
|
3311
|
+
await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
|
|
3312
|
+
if (repl.valid_from == null) {
|
|
3313
|
+
await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
|
|
3314
|
+
}
|
|
3315
|
+
}
|
|
3316
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
3317
|
+
const now = Date.now();
|
|
3318
|
+
const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
|
|
3319
|
+
let created = null;
|
|
3320
|
+
if (typeof replacement !== "string") {
|
|
3321
|
+
if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
|
|
3322
|
+
throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
|
|
3323
|
+
}
|
|
3324
|
+
created = {
|
|
3325
|
+
id: generateId("fact_"),
|
|
3326
|
+
entity_id: entityId,
|
|
3327
|
+
title: replacement.title,
|
|
3328
|
+
body: replacement.body,
|
|
3329
|
+
tags: Array.isArray(replacement.tags) ? replacement.tags : [],
|
|
3330
|
+
confidence: replacement.confidence ?? "certain",
|
|
3331
|
+
source_type: replacement.source_type ?? "user_stated",
|
|
3332
|
+
source_hash: null,
|
|
3333
|
+
source_ref: null,
|
|
3334
|
+
created_at: now,
|
|
3335
|
+
updated_at: now,
|
|
3336
|
+
last_accessed_at: null,
|
|
3337
|
+
access_count: 0,
|
|
3338
|
+
deleted_at: null
|
|
3339
|
+
};
|
|
3340
|
+
}
|
|
3341
|
+
const newId = typeof replacement === "string" ? replacement : created.id;
|
|
3342
|
+
const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
|
|
3343
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3344
|
+
if (created) {
|
|
3345
|
+
await this.entryRepo.upsert(created, tx);
|
|
3346
|
+
if (newValidFrom !== void 0) {
|
|
3347
|
+
await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
|
|
3348
|
+
}
|
|
3349
|
+
}
|
|
3350
|
+
await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
|
|
3351
|
+
});
|
|
3352
|
+
await this.searchService.syncEntries(entityId, [oldId, newId]);
|
|
3353
|
+
if (created) {
|
|
3354
|
+
await this.embeddingService.embedFact(
|
|
3355
|
+
{ id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
|
|
3356
|
+
{ operation: "supersede", trigger: "call" }
|
|
3357
|
+
);
|
|
3358
|
+
}
|
|
3359
|
+
this.searchService.evictCache(entityId);
|
|
3360
|
+
return { newId };
|
|
3361
|
+
}
|
|
3362
|
+
async history(entityId, factId) {
|
|
3363
|
+
const start = await this.entryRepo.findTemporalRow(factId);
|
|
3364
|
+
if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
|
|
3365
|
+
const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
|
|
3366
|
+
const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
|
|
3367
|
+
return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
|
|
3368
|
+
}
|
|
3369
|
+
};
|
|
3370
|
+
|
|
3371
|
+
// src/services/librarian/ops/apply.ts
|
|
3372
|
+
async function applyOps(deps, ctx, input) {
|
|
3373
|
+
const { entityId, trigger } = ctx;
|
|
3374
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3375
|
+
const grounding = deps.promptService.groundingFor("librarian");
|
|
3376
|
+
const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
|
|
3377
|
+
const rejected = new Set(input.rejected);
|
|
3378
|
+
const failed = new Set(input.failed);
|
|
3379
|
+
const written = [];
|
|
3380
|
+
const updatedIds = [];
|
|
3381
|
+
const now = Date.now();
|
|
3382
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
3383
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3384
|
+
if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
|
|
3385
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
|
|
3386
|
+
}
|
|
3387
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3388
|
+
for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
|
|
3389
|
+
titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
|
|
3390
|
+
}
|
|
3391
|
+
const pendingEdges = [];
|
|
3392
|
+
const insert = async (g, draft, validFrom, overrides) => {
|
|
3393
|
+
const fact = g.candidate.fact;
|
|
3394
|
+
const title = overrides?.title ?? fact.title;
|
|
3395
|
+
const body = overrides?.body ?? fact.body;
|
|
3396
|
+
const id = generateId("fact_");
|
|
3397
|
+
const drops = [];
|
|
3398
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
|
|
3399
|
+
for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
|
|
3400
|
+
const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
|
|
3401
|
+
if (g2?.diagnostic) {
|
|
3402
|
+
input.diagnostics.push({
|
|
3403
|
+
...diagBase,
|
|
3404
|
+
code: g2.diagnostic.code,
|
|
3405
|
+
detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
|
|
3406
|
+
});
|
|
3407
|
+
}
|
|
3408
|
+
const row = {
|
|
3409
|
+
id,
|
|
3410
|
+
entity_id: entityId,
|
|
3411
|
+
title,
|
|
3412
|
+
body,
|
|
3413
|
+
tags: fact.tags,
|
|
3414
|
+
confidence: fact.confidence,
|
|
3415
|
+
source_type: "librarian_inferred",
|
|
3416
|
+
source_hash: null,
|
|
3417
|
+
source_ref: null,
|
|
3418
|
+
created_at: now,
|
|
3419
|
+
updated_at: now,
|
|
3420
|
+
last_accessed_at: null,
|
|
3421
|
+
access_count: 0,
|
|
3422
|
+
deleted_at: null,
|
|
3423
|
+
okf_type: normalized.okf_type,
|
|
3424
|
+
...g2?.trust,
|
|
3425
|
+
...draft ? { lifecycle_status: "draft" } : {}
|
|
3426
|
+
};
|
|
3427
|
+
await deps.entryRepo.upsert(row, tx);
|
|
3428
|
+
await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
|
|
3429
|
+
titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
|
|
3430
|
+
if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3431
|
+
const textChanged = title !== fact.title || body !== fact.body;
|
|
3432
|
+
written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
|
|
3433
|
+
return id;
|
|
3434
|
+
};
|
|
3435
|
+
for (const g of input.gated) {
|
|
3436
|
+
const idx = g.candidate.index;
|
|
3437
|
+
const vf = g.candidate.validFrom;
|
|
3438
|
+
if (g.decision.kind === "noop") continue;
|
|
3439
|
+
if (g.decision.kind === "add") {
|
|
3440
|
+
await insert(g, false, vf);
|
|
3441
|
+
continue;
|
|
3442
|
+
}
|
|
3443
|
+
if (failed.has(g)) {
|
|
3444
|
+
const id = await insert(g, true, vf);
|
|
3445
|
+
input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
|
|
3446
|
+
continue;
|
|
3447
|
+
}
|
|
3448
|
+
const op = input.ops.get(g);
|
|
3449
|
+
if (rejected.has(g) || !op) {
|
|
3450
|
+
await insert(g, false, vf);
|
|
3451
|
+
input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
|
|
3452
|
+
continue;
|
|
3453
|
+
}
|
|
3454
|
+
if (op.op === "NOOP") continue;
|
|
3455
|
+
if (op.op === "ADD") {
|
|
3456
|
+
await insert(g, false, vf);
|
|
3457
|
+
continue;
|
|
3458
|
+
}
|
|
3459
|
+
const target = g.neighbours.find((n) => n.id === op.targetId);
|
|
3460
|
+
if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
|
|
3461
|
+
const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
|
|
3462
|
+
if (existing) {
|
|
3463
|
+
const newTitle = op.title ?? g.candidate.fact.title;
|
|
3464
|
+
await deps.entryRepo.upsert({
|
|
3465
|
+
...existing,
|
|
3466
|
+
title: newTitle,
|
|
3467
|
+
body: op.body ?? g.candidate.fact.body,
|
|
3468
|
+
updated_at: now
|
|
3469
|
+
}, tx);
|
|
3470
|
+
titleIndex.delete(normalizeTitleKey(existing.title));
|
|
3471
|
+
titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
|
|
3472
|
+
updatedIds.push(existing.id);
|
|
3473
|
+
continue;
|
|
3474
|
+
}
|
|
3475
|
+
}
|
|
3476
|
+
const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
|
|
3477
|
+
const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
|
|
3478
|
+
if (target.source_type === "immutable_document") {
|
|
3479
|
+
const id = await insert(g, true, supersedeFrom, overrides);
|
|
3480
|
+
input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
|
|
3481
|
+
continue;
|
|
3482
|
+
}
|
|
3483
|
+
const newId = await insert(g, false, supersedeFrom, overrides);
|
|
3484
|
+
try {
|
|
3485
|
+
await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
|
|
3486
|
+
} catch (err) {
|
|
3487
|
+
if (!(err instanceof WikiSupersedeError)) throw err;
|
|
3488
|
+
input.diagnostics.push({
|
|
3489
|
+
...diagBase,
|
|
3490
|
+
code: "librarian_op_rejected",
|
|
3491
|
+
detail: { factId: newId, itemIndex: idx, reason: err.reason }
|
|
3492
|
+
});
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
for (const item of pendingEdges) {
|
|
3496
|
+
const resolveDrops = [];
|
|
3497
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
3498
|
+
entityId,
|
|
3499
|
+
item.sourceId,
|
|
3500
|
+
item.sourceType,
|
|
3501
|
+
item.edges,
|
|
3502
|
+
manifest,
|
|
3503
|
+
titleIndex,
|
|
3504
|
+
tx,
|
|
3505
|
+
now,
|
|
3506
|
+
resolveDrops
|
|
3507
|
+
);
|
|
3508
|
+
for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
|
|
3509
|
+
}
|
|
3510
|
+
for (const task of input.tasks) {
|
|
3511
|
+
const taskObj = {
|
|
3512
|
+
id: generateId("task_"),
|
|
3513
|
+
entity_id: entityId,
|
|
3514
|
+
description: task.description,
|
|
3515
|
+
status: "pending",
|
|
3516
|
+
priority: task.priority,
|
|
3517
|
+
created_at: now,
|
|
3518
|
+
updated_at: now,
|
|
3519
|
+
resolved_at: null,
|
|
3520
|
+
deleted_at: null
|
|
3521
|
+
};
|
|
3522
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
3523
|
+
}
|
|
3524
|
+
});
|
|
3525
|
+
return { written, updatedIds };
|
|
3526
|
+
}
|
|
3527
|
+
|
|
3528
|
+
// src/services/librarian/ops/index.ts
|
|
3529
|
+
var runOpsLibrarianPass = async (deps, ctx) => {
|
|
3530
|
+
const { entityId, trigger } = ctx;
|
|
3531
|
+
const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
|
|
3532
|
+
await deps.seedWatermark(entityId);
|
|
3533
|
+
const wm = await deps.getWatermark(entityId);
|
|
3534
|
+
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
3535
|
+
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
3536
|
+
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
3537
|
+
let ex;
|
|
3538
|
+
try {
|
|
3539
|
+
ex = await runExtract(deps, ctx, sel.batch);
|
|
3540
|
+
} catch (err) {
|
|
3541
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
3542
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
3543
|
+
}
|
|
3544
|
+
throw err;
|
|
3545
|
+
}
|
|
3546
|
+
const diagnostics = new DiagnosticBuffer();
|
|
3547
|
+
for (const d of ex.diagnostics) diagnostics.push(d);
|
|
3548
|
+
const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
|
|
3549
|
+
for (const dup of inBatchDuplicates) {
|
|
3550
|
+
diagnostics.push({
|
|
3551
|
+
code: "fact_deduplicated",
|
|
3552
|
+
operation: "librarian",
|
|
3553
|
+
trigger,
|
|
3554
|
+
entityId,
|
|
3555
|
+
detail: { itemIndex: dup.index, reason: "in_batch" }
|
|
3556
|
+
});
|
|
3557
|
+
}
|
|
3558
|
+
diagnostics.push(gateDiagnostic);
|
|
3559
|
+
const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
|
|
3560
|
+
if (res.budgetStop) {
|
|
3561
|
+
diagnostics.discard();
|
|
3562
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
|
|
3563
|
+
}
|
|
3564
|
+
const { written, updatedIds } = await applyOps(deps, ctx, {
|
|
3565
|
+
gated,
|
|
3566
|
+
ops: res.ops,
|
|
3567
|
+
rejected: res.rejected,
|
|
3568
|
+
failed: res.failed,
|
|
3569
|
+
tasks: ex.tasks,
|
|
3570
|
+
ontologyUpdates: ex.ontologyUpdates,
|
|
3571
|
+
groundingCorpus: ex.groundingCorpus,
|
|
3572
|
+
diagnostics
|
|
3573
|
+
});
|
|
3574
|
+
diagnostics.flush(deps.options);
|
|
3575
|
+
await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
|
|
3576
|
+
const embedCtx = { operation: "librarian", trigger };
|
|
3577
|
+
for (const w of written) {
|
|
3578
|
+
if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
|
|
3579
|
+
else await deps.embeddingService.embedFact(w.fact, embedCtx);
|
|
3580
|
+
}
|
|
3581
|
+
if (updatedIds.length > 0) {
|
|
3582
|
+
for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
|
|
3583
|
+
await deps.embeddingService.embedFact(
|
|
3584
|
+
{ id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
|
|
3585
|
+
embedCtx
|
|
3586
|
+
);
|
|
3587
|
+
}
|
|
3588
|
+
}
|
|
3589
|
+
deps.searchService.evictCache(entityId);
|
|
3590
|
+
const last = sel.batch[sel.batch.length - 1].event;
|
|
3591
|
+
return {
|
|
3592
|
+
processedThrough: { at: last.created_at, id: last.id },
|
|
3593
|
+
factsWritten: written.length + updatedIds.length
|
|
3594
|
+
};
|
|
3595
|
+
};
|
|
3596
|
+
|
|
3597
|
+
// src/services/librarian/index.ts
|
|
3598
|
+
var STRATEGIES = /* @__PURE__ */ new Map([
|
|
3599
|
+
["legacy", runLegacyLibrarianPass],
|
|
3600
|
+
["ops", runOpsLibrarianPass]
|
|
3601
|
+
]);
|
|
3602
|
+
async function runLibrarianStrategy(deps, ctx) {
|
|
3603
|
+
const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
|
|
3604
|
+
const fn = STRATEGIES.get(strategy);
|
|
3605
|
+
if (!fn) {
|
|
3606
|
+
throw new Error(`Unknown librarian strategy: ${strategy}`);
|
|
3607
|
+
}
|
|
3608
|
+
return fn(deps, ctx);
|
|
3609
|
+
}
|
|
3610
|
+
|
|
3611
|
+
// src/services/MaintenanceService.ts
|
|
2702
3612
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
2703
3613
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
2704
3614
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
@@ -2771,11 +3681,23 @@ var MaintenanceService = class {
|
|
|
2771
3681
|
const vacuum = options?.vacuum ?? false;
|
|
2772
3682
|
this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
|
|
2773
3683
|
this._validatePruneDuration(retainEventsFor, "retainEventsFor");
|
|
3684
|
+
const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
|
|
3685
|
+
this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
|
|
2774
3686
|
const now = Date.now();
|
|
2775
3687
|
let syncedIds = [];
|
|
2776
3688
|
let deletedEntries = 0;
|
|
2777
3689
|
let deletedTasks = 0;
|
|
2778
3690
|
let deletedEvents = 0;
|
|
3691
|
+
if (pruneSupersededAfter !== null) {
|
|
3692
|
+
const expiredCutoff = now - pruneSupersededAfter * 864e5;
|
|
3693
|
+
const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
|
|
3694
|
+
if (expiredIds.length > 0) {
|
|
3695
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3696
|
+
await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
|
|
3697
|
+
});
|
|
3698
|
+
syncedIds = [...expiredIds];
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
2779
3701
|
if (retainSoftDeletedFor !== null) {
|
|
2780
3702
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
2781
3703
|
const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
|
|
@@ -2791,7 +3713,7 @@ var MaintenanceService = class {
|
|
|
2791
3713
|
}
|
|
2792
3714
|
}
|
|
2793
3715
|
const succeededIds = succeeded.map((r) => r.id);
|
|
2794
|
-
syncedIds = succeededIds;
|
|
3716
|
+
syncedIds = [...syncedIds, ...succeededIds];
|
|
2795
3717
|
await this.db.withTransactionAsync(async (tx) => {
|
|
2796
3718
|
if (succeededIds.length > 0) {
|
|
2797
3719
|
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
@@ -2827,8 +3749,18 @@ var MaintenanceService = class {
|
|
|
2827
3749
|
}
|
|
2828
3750
|
if (retainEventsFor !== null) {
|
|
2829
3751
|
const cutoff = now - retainEventsFor * 864e5;
|
|
2830
|
-
const
|
|
2831
|
-
|
|
3752
|
+
const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
|
|
3753
|
+
if (guarded) {
|
|
3754
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
3755
|
+
deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
|
|
3756
|
+
const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
|
|
3757
|
+
if (held > 0) {
|
|
3758
|
+
emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
|
|
3759
|
+
}
|
|
3760
|
+
} else {
|
|
3761
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
3762
|
+
deletedEvents = eventResult.changes;
|
|
3763
|
+
}
|
|
2832
3764
|
}
|
|
2833
3765
|
if (vacuum) {
|
|
2834
3766
|
await this.metadataRepo.vacuum();
|
|
@@ -3058,155 +3990,72 @@ var MaintenanceService = class {
|
|
|
3058
3990
|
}
|
|
3059
3991
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
3060
3992
|
async doRunLibrarian(entityId, promptOverride, trigger = "call") {
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
3080
|
-
const result = parseJsonResponse(responseText);
|
|
3081
|
-
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
3082
|
-
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
3083
|
-
const ontologyUpdates = result.ontology_updates;
|
|
3084
|
-
const diagBuffer = new DiagnosticBuffer();
|
|
3085
|
-
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3086
|
-
const validFacts = [];
|
|
3087
|
-
const validFactItemIndexes = [];
|
|
3088
|
-
facts.forEach((raw, itemIndex) => {
|
|
3089
|
-
const valid = validateFact(raw);
|
|
3090
|
-
if (valid) {
|
|
3091
|
-
validFacts.push(valid);
|
|
3092
|
-
validFactItemIndexes.push(itemIndex);
|
|
3093
|
-
} else {
|
|
3094
|
-
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3095
|
-
}
|
|
3096
|
-
});
|
|
3097
|
-
const validTasks = [];
|
|
3098
|
-
tasks.forEach((raw, itemIndex) => {
|
|
3099
|
-
const valid = validateTask(raw);
|
|
3100
|
-
if (valid) validTasks.push(valid);
|
|
3101
|
-
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
3102
|
-
});
|
|
3103
|
-
const now = Date.now();
|
|
3104
|
-
const insertedFacts = [];
|
|
3105
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
3106
|
-
let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3107
|
-
if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
|
|
3108
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
3109
|
-
}
|
|
3110
|
-
const titleIndex = /* @__PURE__ */ new Map();
|
|
3111
|
-
for (const existing of currentFactsRows) {
|
|
3112
|
-
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3113
|
-
id: existing.id,
|
|
3114
|
-
okf_type: existing.okf_type ?? null
|
|
3115
|
-
});
|
|
3116
|
-
}
|
|
3117
|
-
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
3118
|
-
const pendingEdges = [];
|
|
3119
|
-
for (const [k, fact] of validFacts.entries()) {
|
|
3120
|
-
const newTokens = titleTokens(fact.title);
|
|
3121
|
-
let skip = false;
|
|
3122
|
-
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3123
|
-
for (const existing of factsForDedupe) {
|
|
3124
|
-
if (existing.source_type !== "librarian_inferred") continue;
|
|
3125
|
-
const existingTokens = titleTokens(existing.title);
|
|
3126
|
-
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3127
|
-
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
3128
|
-
skip = true;
|
|
3129
|
-
break;
|
|
3130
|
-
}
|
|
3131
|
-
}
|
|
3132
|
-
}
|
|
3133
|
-
}
|
|
3134
|
-
if (skip) {
|
|
3135
|
-
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
3136
|
-
continue;
|
|
3137
|
-
}
|
|
3138
|
-
const ontologyFact = fact;
|
|
3139
|
-
const id = generateId("fact_");
|
|
3140
|
-
const validationDrops = [];
|
|
3141
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
3142
|
-
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3143
|
-
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3144
|
-
if (grounding?.diagnostic) {
|
|
3145
|
-
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3146
|
-
}
|
|
3147
|
-
const factObj = {
|
|
3148
|
-
id,
|
|
3149
|
-
entity_id: entityId,
|
|
3150
|
-
title: fact.title,
|
|
3151
|
-
body: fact.body,
|
|
3152
|
-
tags: fact.tags,
|
|
3153
|
-
confidence: fact.confidence,
|
|
3154
|
-
source_type: "librarian_inferred",
|
|
3155
|
-
source_hash: null,
|
|
3156
|
-
source_ref: null,
|
|
3157
|
-
created_at: now,
|
|
3158
|
-
updated_at: now,
|
|
3159
|
-
last_accessed_at: null,
|
|
3160
|
-
access_count: 0,
|
|
3161
|
-
deleted_at: null,
|
|
3162
|
-
okf_type: normalized.okf_type,
|
|
3163
|
-
...grounding?.trust
|
|
3164
|
-
};
|
|
3165
|
-
await this.entryRepo.upsert(factObj, tx);
|
|
3166
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3167
|
-
factsForDedupe.push(factObj);
|
|
3168
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3169
|
-
if (normalized.edges.length > 0) {
|
|
3170
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
for (const item of pendingEdges) {
|
|
3174
|
-
const resolveDrops = [];
|
|
3175
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
3176
|
-
entityId,
|
|
3177
|
-
item.sourceId,
|
|
3178
|
-
item.sourceType,
|
|
3179
|
-
item.edges ?? [],
|
|
3180
|
-
manifest,
|
|
3181
|
-
titleIndex,
|
|
3182
|
-
tx,
|
|
3183
|
-
now,
|
|
3184
|
-
resolveDrops
|
|
3185
|
-
);
|
|
3186
|
-
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
3187
|
-
}
|
|
3188
|
-
for (const task of validTasks) {
|
|
3189
|
-
const id = generateId("task_");
|
|
3190
|
-
const taskObj = {
|
|
3191
|
-
id,
|
|
3192
|
-
entity_id: entityId,
|
|
3193
|
-
description: task.description,
|
|
3194
|
-
status: "pending",
|
|
3195
|
-
priority: task.priority,
|
|
3196
|
-
created_at: now,
|
|
3197
|
-
updated_at: now,
|
|
3198
|
-
resolved_at: null,
|
|
3199
|
-
deleted_at: null
|
|
3200
|
-
};
|
|
3201
|
-
await this.taskRepo.upsert(taskObj, tx);
|
|
3202
|
-
}
|
|
3203
|
-
});
|
|
3204
|
-
diagBuffer.flush(this.options);
|
|
3205
|
-
await this.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
3206
|
-
for (const fact of insertedFacts) {
|
|
3207
|
-
await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
3993
|
+
await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
|
|
3994
|
+
}
|
|
3995
|
+
/**
|
|
3996
|
+
* Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
|
|
3997
|
+
* §5.1). Builds the strategy deps from `this` and dispatches via
|
|
3998
|
+
* `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
|
|
3999
|
+
* class. Returns the strategy's `LibrarianResult` so deferred/budgeted
|
|
4000
|
+
* schedulers (PR-C) can advance watermarks; the existing void-returning
|
|
4001
|
+
* `doRunLibrarian` discards it.
|
|
4002
|
+
*
|
|
4003
|
+
* Spec §10.3: one place advances the watermark, for every strategy, only
|
|
4004
|
+
* after the pass resolved (its transaction committed) and never backwards.
|
|
4005
|
+
* A budget-stopped pass wrote nothing and must not move the watermark.
|
|
4006
|
+
*/
|
|
4007
|
+
async runLibrarianPass(ctx) {
|
|
4008
|
+
const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
|
|
4009
|
+
if (result.processedThrough && !result.budgetStop) {
|
|
4010
|
+
await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
|
|
3208
4011
|
}
|
|
3209
|
-
|
|
4012
|
+
return result;
|
|
4013
|
+
}
|
|
4014
|
+
/**
|
|
4015
|
+
* Strategy deps shared between legacy and ops librarians (spec §5.1). The
|
|
4016
|
+
* ops strategy (PR-B Task 8) reads/anchors the watermark through these two
|
|
4017
|
+
* hooks so the dispatcher itself never reaches into the metadata table —
|
|
4018
|
+
* the watermark invariant (one place advances, never backwards) lives here.
|
|
4019
|
+
*/
|
|
4020
|
+
librarianDeps() {
|
|
4021
|
+
return {
|
|
4022
|
+
db: this.db,
|
|
4023
|
+
options: this.options,
|
|
4024
|
+
entryRepo: this.entryRepo,
|
|
4025
|
+
taskRepo: this.taskRepo,
|
|
4026
|
+
eventRepo: this.eventRepo,
|
|
4027
|
+
searchService: this.searchService,
|
|
4028
|
+
embeddingService: this.embeddingService,
|
|
4029
|
+
promptService: this.promptService,
|
|
4030
|
+
ontologyService: this.ontologyService,
|
|
4031
|
+
seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
|
|
4032
|
+
getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
|
|
4033
|
+
};
|
|
4034
|
+
}
|
|
4035
|
+
/**
|
|
4036
|
+
* Persist `next` as the entity's librarian watermark when it is strictly
|
|
4037
|
+
* newer than the stored value, under the (created_at, id) ordering used by
|
|
4038
|
+
* the op-based librarian (spec §10.3). Never moves backwards.
|
|
4039
|
+
*/
|
|
4040
|
+
async advanceLibrarianWatermark(entityId, next) {
|
|
4041
|
+
const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
4042
|
+
const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
|
|
4043
|
+
if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
|
|
4044
|
+
}
|
|
4045
|
+
/**
|
|
4046
|
+
* Spec §10.3: first ops pass on a legacy DB starts after what the count
|
|
4047
|
+
* checkpoint already covered. No-op when a watermark already exists or the
|
|
4048
|
+
* memory checkpoint is 0; otherwise anchor the watermark to the event at
|
|
4049
|
+
* offset `min(memory_checkpoint, totalEvents) - 1`.
|
|
4050
|
+
*/
|
|
4051
|
+
async seedLibrarianWatermark(entityId) {
|
|
4052
|
+
if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
|
|
4053
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
4054
|
+
const memory = cp.memory ?? 0;
|
|
4055
|
+
if (memory <= 0) return;
|
|
4056
|
+
const total = await this.eventRepo.count(entityId);
|
|
4057
|
+
const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
|
|
4058
|
+
if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
|
|
3210
4059
|
}
|
|
3211
4060
|
/**
|
|
3212
4061
|
* Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
|
|
@@ -3312,7 +4161,7 @@ var MaintenanceService = class {
|
|
|
3312
4161
|
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3313
4162
|
return prompts;
|
|
3314
4163
|
},
|
|
3315
|
-
call: (prompts) => this.options
|
|
4164
|
+
call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
|
|
3316
4165
|
parse: (responseText, batch, prompts) => {
|
|
3317
4166
|
const result = parseJsonResponse(responseText);
|
|
3318
4167
|
return {
|
|
@@ -3416,9 +4265,10 @@ var MaintenanceService = class {
|
|
|
3416
4265
|
const callErrorIds = new Set(
|
|
3417
4266
|
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
3418
4267
|
);
|
|
4268
|
+
const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
|
|
3419
4269
|
await this.entryRepo.markHealChecked(
|
|
3420
4270
|
[
|
|
3421
|
-
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
4271
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
|
|
3422
4272
|
...insertedFacts.map((f) => f.id)
|
|
3423
4273
|
],
|
|
3424
4274
|
entityId,
|
|
@@ -3470,7 +4320,8 @@ var MaintenanceService = class {
|
|
|
3470
4320
|
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
3471
4321
|
degraded: healedDegraded,
|
|
3472
4322
|
remaining: counts.eligible,
|
|
3473
|
-
deferred: counts.deferred
|
|
4323
|
+
deferred: counts.deferred,
|
|
4324
|
+
...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
|
|
3474
4325
|
};
|
|
3475
4326
|
}
|
|
3476
4327
|
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
@@ -3514,7 +4365,7 @@ var MaintenanceService = class {
|
|
|
3514
4365
|
const outcome = await runBatched({
|
|
3515
4366
|
items: candidates,
|
|
3516
4367
|
buildPrompt,
|
|
3517
|
-
call: (prompts) => this.options
|
|
4368
|
+
call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
|
|
3518
4369
|
parse: (responseText, batch) => {
|
|
3519
4370
|
const parsed = parseJsonResponse(responseText);
|
|
3520
4371
|
return {
|
|
@@ -3899,6 +4750,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
3899
4750
|
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
3900
4751
|
}
|
|
3901
4752
|
|
|
4753
|
+
// src/utils/budget.ts
|
|
4754
|
+
function factTokenCost(f) {
|
|
4755
|
+
return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
|
|
4756
|
+
}
|
|
4757
|
+
function packFactsByBudget(items, budget, scores) {
|
|
4758
|
+
const candidates = items.length;
|
|
4759
|
+
if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
|
|
4760
|
+
let top = items[0];
|
|
4761
|
+
let topCost = factTokenCost(top);
|
|
4762
|
+
if (topCost > budget) {
|
|
4763
|
+
const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
|
|
4764
|
+
top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
|
|
4765
|
+
topCost = factTokenCost(top);
|
|
4766
|
+
}
|
|
4767
|
+
const kept = [{ rank: 0, item: top }];
|
|
4768
|
+
let used = topCost;
|
|
4769
|
+
const rest = items.slice(1).map((item, i) => {
|
|
4770
|
+
const rank = i + 1;
|
|
4771
|
+
const cost = factTokenCost(item);
|
|
4772
|
+
const s = scores?.get(item.id);
|
|
4773
|
+
const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
|
|
4774
|
+
return { rank, item, cost, density: value / Math.max(1, cost) };
|
|
4775
|
+
});
|
|
4776
|
+
rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
|
|
4777
|
+
for (const r of rest) {
|
|
4778
|
+
if (used + r.cost <= budget) {
|
|
4779
|
+
kept.push({ rank: r.rank, item: r.item });
|
|
4780
|
+
used += r.cost;
|
|
4781
|
+
}
|
|
4782
|
+
}
|
|
4783
|
+
kept.sort((a, b) => a.rank - b.rank);
|
|
4784
|
+
return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
|
|
4785
|
+
}
|
|
4786
|
+
|
|
3902
4787
|
// src/services/RetrievalService.ts
|
|
3903
4788
|
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
3904
4789
|
var RetrievalService = class {
|
|
@@ -3934,6 +4819,28 @@ var RetrievalService = class {
|
|
|
3934
4819
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
3935
4820
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
3936
4821
|
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
4822
|
+
const rawAsOf = options?.asOf;
|
|
4823
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
4824
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
4825
|
+
}
|
|
4826
|
+
const tokenBudget = options?.tokenBudget;
|
|
4827
|
+
if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
|
|
4828
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
4829
|
+
}
|
|
4830
|
+
const applyBudget = (input) => {
|
|
4831
|
+
if (tokenBudget === void 0) return input;
|
|
4832
|
+
const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
|
|
4833
|
+
emitDiagnostic(this.options, {
|
|
4834
|
+
code: "read_budget",
|
|
4835
|
+
operation: "read",
|
|
4836
|
+
trigger: "call",
|
|
4837
|
+
entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
|
|
4838
|
+
detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
|
|
4839
|
+
});
|
|
4840
|
+
return packed.items;
|
|
4841
|
+
};
|
|
4842
|
+
const liveMode = rawAsOf === void 0 ? "current" : "asOf";
|
|
4843
|
+
const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
|
|
3937
4844
|
const trimmedQuery = query.trim();
|
|
3938
4845
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
3939
4846
|
entityIds,
|
|
@@ -3953,8 +4860,10 @@ var RetrievalService = class {
|
|
|
3953
4860
|
let usedEmbed = false;
|
|
3954
4861
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
3955
4862
|
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
3956
|
-
const
|
|
3957
|
-
const
|
|
4863
|
+
const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
|
|
4864
|
+
const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
|
|
4865
|
+
const excludedPad = excludedIds.size;
|
|
4866
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
|
|
3958
4867
|
if (scoredEntityIds.length === 0) {
|
|
3959
4868
|
usedEmbed = true;
|
|
3960
4869
|
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
@@ -3988,7 +4897,7 @@ var RetrievalService = class {
|
|
|
3988
4897
|
let miniSearchScores;
|
|
3989
4898
|
if (effectivePreFilterLimit !== void 0) {
|
|
3990
4899
|
populateCache = false;
|
|
3991
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !
|
|
4900
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
|
|
3992
4901
|
if (preResults.length === 0) {
|
|
3993
4902
|
candidateRows = null;
|
|
3994
4903
|
} else {
|
|
@@ -4010,9 +4919,9 @@ var RetrievalService = class {
|
|
|
4010
4919
|
}
|
|
4011
4920
|
} else {
|
|
4012
4921
|
if (useRanker) {
|
|
4013
|
-
candidateRows = this.
|
|
4922
|
+
candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
|
|
4014
4923
|
} else {
|
|
4015
|
-
candidateRows = this.
|
|
4924
|
+
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
4016
4925
|
}
|
|
4017
4926
|
if (weight !== void 0 && weight < 1) {
|
|
4018
4927
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -4297,7 +5206,7 @@ var RetrievalService = class {
|
|
|
4297
5206
|
// read() re-sorts after applying tier weights
|
|
4298
5207
|
});
|
|
4299
5208
|
}
|
|
4300
|
-
if (
|
|
5209
|
+
if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
|
|
4301
5210
|
if (scored.length > 0) {
|
|
4302
5211
|
scored = scored.map((row) => ({
|
|
4303
5212
|
...row,
|
|
@@ -4361,7 +5270,7 @@ var RetrievalService = class {
|
|
|
4361
5270
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
4362
5271
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
4363
5272
|
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
4364
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !
|
|
5273
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
|
|
4365
5274
|
const candidates = results.map((r) => ({
|
|
4366
5275
|
id: r.id,
|
|
4367
5276
|
entity_id: r.entity_id,
|
|
@@ -4379,13 +5288,18 @@ var RetrievalService = class {
|
|
|
4379
5288
|
}
|
|
4380
5289
|
}
|
|
4381
5290
|
}
|
|
5291
|
+
facts = applyBudget(facts);
|
|
4382
5292
|
if (facts.length > 0) {
|
|
4383
5293
|
const ids = facts.map((f) => f.id);
|
|
4384
5294
|
const now = Date.now();
|
|
4385
5295
|
await this.entryRepo.trackAccess(ids, now);
|
|
4386
5296
|
}
|
|
4387
5297
|
} else {
|
|
4388
|
-
facts =
|
|
5298
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
|
|
5299
|
+
...excludeDrafts ? { excludeDrafts: true } : {},
|
|
5300
|
+
live: { mode: liveMode, t: liveAt }
|
|
5301
|
+
});
|
|
5302
|
+
facts = applyBudget(facts);
|
|
4389
5303
|
}
|
|
4390
5304
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
4391
5305
|
const [tasks, events] = await Promise.all([
|
|
@@ -4421,8 +5335,8 @@ var RetrievalService = class {
|
|
|
4421
5335
|
_tieBreakSort(items) {
|
|
4422
5336
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
4423
5337
|
}
|
|
4424
|
-
|
|
4425
|
-
return
|
|
5338
|
+
_withoutIds(rows, excludeIds) {
|
|
5339
|
+
return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
|
|
4426
5340
|
}
|
|
4427
5341
|
/**
|
|
4428
5342
|
* Comparator for score + deterministic tie-break fields.
|
|
@@ -4493,21 +5407,6 @@ var RetrievalService = class {
|
|
|
4493
5407
|
return scored;
|
|
4494
5408
|
}
|
|
4495
5409
|
};
|
|
4496
|
-
|
|
4497
|
-
// src/utils/cosine.ts
|
|
4498
|
-
function cosineSimilarity(a, b) {
|
|
4499
|
-
let dot = 0, normA = 0, normB = 0;
|
|
4500
|
-
const len = Math.min(a.length, b.length);
|
|
4501
|
-
for (let i = 0; i < len; i++) {
|
|
4502
|
-
dot += a[i] * b[i];
|
|
4503
|
-
normA += a[i] * a[i];
|
|
4504
|
-
normB += b[i] * b[i];
|
|
4505
|
-
}
|
|
4506
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
4507
|
-
return denom === 0 ? 0 : dot / denom;
|
|
4508
|
-
}
|
|
4509
|
-
|
|
4510
|
-
// src/services/SearchService.ts
|
|
4511
5410
|
var _SearchService = class _SearchService {
|
|
4512
5411
|
constructor(entryRepo) {
|
|
4513
5412
|
this.entryRepo = entryRepo;
|
|
@@ -5290,6 +6189,18 @@ var WriteService = class {
|
|
|
5290
6189
|
this.metadataRepo = metadataRepo;
|
|
5291
6190
|
this.jobManager = jobManager;
|
|
5292
6191
|
this.maintenanceService = maintenanceService;
|
|
6192
|
+
this.inflight = /* @__PURE__ */ new Set();
|
|
6193
|
+
}
|
|
6194
|
+
track(p) {
|
|
6195
|
+
this.inflight.add(p);
|
|
6196
|
+
p.finally(() => this.inflight.delete(p)).catch(() => {
|
|
6197
|
+
});
|
|
6198
|
+
}
|
|
6199
|
+
/** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
|
|
6200
|
+
async drain() {
|
|
6201
|
+
while (this.inflight.size > 0) {
|
|
6202
|
+
await Promise.allSettled([...this.inflight]);
|
|
6203
|
+
}
|
|
5293
6204
|
}
|
|
5294
6205
|
async write(entityId, event) {
|
|
5295
6206
|
if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
|
|
@@ -5316,6 +6227,7 @@ var WriteService = class {
|
|
|
5316
6227
|
}
|
|
5317
6228
|
const id = generateId("evt_");
|
|
5318
6229
|
const now = Date.now();
|
|
6230
|
+
const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
|
|
5319
6231
|
let eventType = event.event_type;
|
|
5320
6232
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
5321
6233
|
eventType = "observation";
|
|
@@ -5326,8 +6238,14 @@ var WriteService = class {
|
|
|
5326
6238
|
event_type: eventType,
|
|
5327
6239
|
summary,
|
|
5328
6240
|
related_entry_id: relatedEntryId,
|
|
5329
|
-
created_at: now
|
|
6241
|
+
created_at: now,
|
|
6242
|
+
occurred_at: occurredAt
|
|
5330
6243
|
};
|
|
6244
|
+
const deferred = this.options.config?.maintenance === "deferred";
|
|
6245
|
+
if (deferred) {
|
|
6246
|
+
await this.eventRepo.add(newEvent);
|
|
6247
|
+
return;
|
|
6248
|
+
}
|
|
5331
6249
|
let shouldRunLibrarian = false;
|
|
5332
6250
|
let librarianCount = 0;
|
|
5333
6251
|
let prevMemoryCheckpoint = 0;
|
|
@@ -5349,27 +6267,41 @@ var WriteService = class {
|
|
|
5349
6267
|
prevMemoryCheckpoint = memoryCheckpoint;
|
|
5350
6268
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
5351
6269
|
}
|
|
6270
|
+
} else {
|
|
6271
|
+
const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
|
|
6272
|
+
if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
|
|
6273
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
|
|
6274
|
+
const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
|
|
6275
|
+
if (Math.ceil(chars / 4) >= tokenThreshold) {
|
|
6276
|
+
shouldRunLibrarian = true;
|
|
6277
|
+
librarianCount = count;
|
|
6278
|
+
prevMemoryCheckpoint = memoryCheckpoint;
|
|
6279
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
6280
|
+
}
|
|
6281
|
+
}
|
|
5352
6282
|
}
|
|
5353
6283
|
});
|
|
5354
6284
|
if (shouldRunLibrarian) {
|
|
5355
6285
|
try {
|
|
5356
6286
|
this.jobManager.acquireLock("librarian", entityId);
|
|
5357
6287
|
const stage = { job: "librarian" };
|
|
5358
|
-
this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6288
|
+
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
5359
6289
|
console.error(err);
|
|
5360
6290
|
this.reportBackgroundFailure(entityId, stage.job);
|
|
5361
6291
|
}).finally(() => {
|
|
5362
6292
|
this.jobManager.releaseLock("librarian", entityId);
|
|
5363
6293
|
});
|
|
6294
|
+
this.track(job);
|
|
5364
6295
|
} catch (e) {
|
|
5365
6296
|
if (!(e instanceof WikiBusyError)) throw e;
|
|
5366
6297
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
|
|
5367
6298
|
}
|
|
5368
6299
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
5369
|
-
this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6300
|
+
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
5370
6301
|
console.error(err);
|
|
5371
6302
|
this.reportBackgroundFailure(entityId, "heal");
|
|
5372
6303
|
});
|
|
6304
|
+
this.track(healJob);
|
|
5373
6305
|
}
|
|
5374
6306
|
}
|
|
5375
6307
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {
|