@equationalapplications/core-llm-wiki 7.7.6 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-PBUZLSOM.mjs → chunk-M76CF2Q6.mjs} +1193 -189
- package/dist/chunk-M76CF2Q6.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1837 -289
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +575 -32
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BIFNLUva.d.mts → testing-CWZaXKLj.d.mts} +560 -4
- package/dist/{testing-BIFNLUva.d.ts → testing-CWZaXKLj.d.ts} +560 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1142 -201
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-PBUZLSOM.mjs.map +0 -1
|
@@ -176,6 +176,15 @@ var WikiDraftNotFound = class _WikiDraftNotFound extends Error {
|
|
|
176
176
|
Object.setPrototypeOf(this, _WikiDraftNotFound.prototype);
|
|
177
177
|
}
|
|
178
178
|
};
|
|
179
|
+
var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
180
|
+
constructor(reason) {
|
|
181
|
+
super(`supersede() rejected: ${reason}`);
|
|
182
|
+
this.code = "WIKI_SUPERSEDE_REJECTED";
|
|
183
|
+
this.name = "WikiSupersedeError";
|
|
184
|
+
this.reason = reason;
|
|
185
|
+
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
186
|
+
}
|
|
187
|
+
};
|
|
179
188
|
|
|
180
189
|
// src/utils/ids.ts
|
|
181
190
|
var _injectedGetRandomValues = null;
|
|
@@ -1745,6 +1754,21 @@ function groundingEvidenceBlock(writer, cfg) {
|
|
|
1745
1754
|
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
1746
1755
|
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
1747
1756
|
}
|
|
1757
|
+
var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
|
|
1758
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1759
|
+
{
|
|
1760
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
|
|
1761
|
+
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
1762
|
+
}
|
|
1763
|
+
One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
|
|
1764
|
+
var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
|
|
1765
|
+
- "ADD": the candidate is new information; the existing facts are about something else.
|
|
1766
|
+
- "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
|
|
1767
|
+
- "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
|
|
1768
|
+
- "NOOP": an existing fact already states the candidate.
|
|
1769
|
+
Return ONLY a valid JSON object matching this schema:
|
|
1770
|
+
{ "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
|
|
1771
|
+
"target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
|
|
1748
1772
|
|
|
1749
1773
|
// src/utils/healConstants.ts
|
|
1750
1774
|
var HEAL_MAX_ANCHORS = 50;
|
|
@@ -1983,6 +2007,22 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
1983
2007
|
${JSON.stringify(facts, null, 2)}`
|
|
1984
2008
|
};
|
|
1985
2009
|
}
|
|
2010
|
+
/** Ops extract (spec §5.2): events only — no existing facts. */
|
|
2011
|
+
buildOpsExtractPrompt(events, ontologyContext) {
|
|
2012
|
+
const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
|
|
2013
|
+
const grounding = this.groundingFor("librarian");
|
|
2014
|
+
return {
|
|
2015
|
+
systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
|
|
2016
|
+
userPrompt: `Events:
|
|
2017
|
+
${lines.join("\n")}`,
|
|
2018
|
+
...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
/** Ops resolve (spec §5.4): only each item's own neighbours. */
|
|
2022
|
+
buildOpsResolvePrompt(items) {
|
|
2023
|
+
return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
|
|
2024
|
+
${JSON.stringify({ items })}` };
|
|
2025
|
+
}
|
|
1986
2026
|
/**
|
|
1987
2027
|
* The system prompt each writer would send, without hydrating data (spec
|
|
1988
2028
|
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
@@ -2061,7 +2101,14 @@ var SEVERITY = {
|
|
|
2061
2101
|
grounding_missing: "warn",
|
|
2062
2102
|
grounding_failed: "warn",
|
|
2063
2103
|
classification_low_confidence: "info",
|
|
2064
|
-
classification_invalid: "warn"
|
|
2104
|
+
classification_invalid: "warn",
|
|
2105
|
+
llm_usage: "info",
|
|
2106
|
+
librarian_gate: "info",
|
|
2107
|
+
librarian_op_rejected: "warn",
|
|
2108
|
+
contradicts_document: "warn",
|
|
2109
|
+
resolve_failed: "warn",
|
|
2110
|
+
read_budget: "info",
|
|
2111
|
+
event_retention_held: "info"
|
|
2065
2112
|
};
|
|
2066
2113
|
var MESSAGE = {
|
|
2067
2114
|
ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
|
|
@@ -2076,7 +2123,14 @@ var MESSAGE = {
|
|
|
2076
2123
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
2077
2124
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
2078
2125
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
2079
|
-
classification_invalid: "A classifier answer was invalid and was not applied."
|
|
2126
|
+
classification_invalid: "A classifier answer was invalid and was not applied.",
|
|
2127
|
+
llm_usage: "An LLM call completed; token usage attached.",
|
|
2128
|
+
librarian_gate: "Librarian gate outcome counts for one pass.",
|
|
2129
|
+
librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
|
|
2130
|
+
contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
|
|
2131
|
+
resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
|
|
2132
|
+
read_budget: "A read was packed to a token budget.",
|
|
2133
|
+
event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
|
|
2080
2134
|
};
|
|
2081
2135
|
var warnedNonFunction = /* @__PURE__ */ new WeakSet();
|
|
2082
2136
|
function copyDetail(detail) {
|
|
@@ -2162,6 +2216,79 @@ function edgeDropDiagnostic(drop, ctx) {
|
|
|
2162
2216
|
return { code: "edge_dropped", operation: ctx.operation, trigger: ctx.trigger, entityId: ctx.entityId, detail };
|
|
2163
2217
|
}
|
|
2164
2218
|
|
|
2219
|
+
// src/utils/usage.ts
|
|
2220
|
+
function estimateTokens(text) {
|
|
2221
|
+
return Math.ceil(text.length / 4);
|
|
2222
|
+
}
|
|
2223
|
+
var UsageMeter = class {
|
|
2224
|
+
constructor(budget) {
|
|
2225
|
+
this.used = 0;
|
|
2226
|
+
this.estimated = false;
|
|
2227
|
+
if (budget !== void 0 && (typeof budget !== "number" || !Number.isFinite(budget) || budget < 0)) {
|
|
2228
|
+
throw new TypeError("tokenBudget must be a finite number >= 0");
|
|
2229
|
+
}
|
|
2230
|
+
this.budget = budget;
|
|
2231
|
+
}
|
|
2232
|
+
get remaining() {
|
|
2233
|
+
return this.budget === void 0 ? Infinity : Math.max(0, this.budget - this.used);
|
|
2234
|
+
}
|
|
2235
|
+
fits(estimate) {
|
|
2236
|
+
return estimate <= this.remaining;
|
|
2237
|
+
}
|
|
2238
|
+
record(r) {
|
|
2239
|
+
this.used += r.inputTokens + r.outputTokens;
|
|
2240
|
+
if (r.estimated) this.estimated = true;
|
|
2241
|
+
}
|
|
2242
|
+
};
|
|
2243
|
+
var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
|
|
2244
|
+
constructor(requiredEstimate, remaining) {
|
|
2245
|
+
super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
|
|
2246
|
+
this.name = "WikiBudgetExhausted";
|
|
2247
|
+
this.requiredEstimate = requiredEstimate;
|
|
2248
|
+
this.remaining = remaining;
|
|
2249
|
+
Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
|
|
2250
|
+
}
|
|
2251
|
+
};
|
|
2252
|
+
|
|
2253
|
+
// src/utils/llmCall.ts
|
|
2254
|
+
function validUsage(u) {
|
|
2255
|
+
if (!u || typeof u !== "object") return false;
|
|
2256
|
+
const { inputTokens, outputTokens } = u;
|
|
2257
|
+
return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
|
|
2258
|
+
}
|
|
2259
|
+
async function callLlm(options, ctx, prompts) {
|
|
2260
|
+
const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
|
|
2261
|
+
if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
|
|
2262
|
+
throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
|
|
2263
|
+
}
|
|
2264
|
+
const provider = options.llmProvider;
|
|
2265
|
+
const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
|
|
2266
|
+
let text;
|
|
2267
|
+
let reported;
|
|
2268
|
+
if (typeof provider.generateTextWithUsage === "function") {
|
|
2269
|
+
const result = await provider.generateTextWithUsage(params);
|
|
2270
|
+
if (!result || typeof result.text !== "string") {
|
|
2271
|
+
throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
|
|
2272
|
+
}
|
|
2273
|
+
text = result.text;
|
|
2274
|
+
reported = validUsage(result.usage) ? result.usage : void 0;
|
|
2275
|
+
} else {
|
|
2276
|
+
text = await provider.generateText(params);
|
|
2277
|
+
}
|
|
2278
|
+
const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
|
|
2279
|
+
ctx.meter?.record(record);
|
|
2280
|
+
if (options.config?.reportLlmUsage === true) {
|
|
2281
|
+
emitDiagnostic(options, {
|
|
2282
|
+
code: "llm_usage",
|
|
2283
|
+
operation: ctx.operation,
|
|
2284
|
+
trigger: ctx.trigger,
|
|
2285
|
+
entityId: ctx.entityId,
|
|
2286
|
+
detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
|
|
2287
|
+
});
|
|
2288
|
+
}
|
|
2289
|
+
return text;
|
|
2290
|
+
}
|
|
2291
|
+
|
|
2165
2292
|
// src/services/IngestionService.ts
|
|
2166
2293
|
var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
|
|
2167
2294
|
function dedupDiagnostic(entityId, sourceRef, locator) {
|
|
@@ -2238,7 +2365,7 @@ var IngestionService = class {
|
|
|
2238
2365
|
ontologyContext
|
|
2239
2366
|
);
|
|
2240
2367
|
try {
|
|
2241
|
-
const responseText = await this.options
|
|
2368
|
+
const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
|
|
2242
2369
|
const result2 = parseJsonResponse(responseText);
|
|
2243
2370
|
const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
|
|
2244
2371
|
const facts = [];
|
|
@@ -2908,6 +3035,37 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2908
3035
|
[entityId]
|
|
2909
3036
|
);
|
|
2910
3037
|
}
|
|
3038
|
+
/**
|
|
3039
|
+
* Read the op-based librarian's high-water mark. Returns `null` when the
|
|
3040
|
+
* entity has never had its watermark set.
|
|
3041
|
+
*/
|
|
3042
|
+
async getLibrarianWatermark(entityId, tx) {
|
|
3043
|
+
const executor = this.getExecutor(tx);
|
|
3044
|
+
const row = await executor.getFirstAsync(
|
|
3045
|
+
`SELECT librarian_watermark_at, librarian_watermark_id FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
3046
|
+
[entityId]
|
|
3047
|
+
);
|
|
3048
|
+
if (!row || row.librarian_watermark_at == null || row.librarian_watermark_id == null) {
|
|
3049
|
+
return null;
|
|
3050
|
+
}
|
|
3051
|
+
return { at: row.librarian_watermark_at, id: row.librarian_watermark_id };
|
|
3052
|
+
}
|
|
3053
|
+
/**
|
|
3054
|
+
* Write the op-based librarian's high-water mark. Uses `ON CONFLICT`
|
|
3055
|
+
* `DO UPDATE` so existing `memory_checkpoint` / `heal_checkpoint` rows
|
|
3056
|
+
* are preserved verbatim.
|
|
3057
|
+
*/
|
|
3058
|
+
async setLibrarianWatermark(entityId, wm, tx) {
|
|
3059
|
+
const executor = this.getExecutor(tx);
|
|
3060
|
+
await executor.runAsync(
|
|
3061
|
+
`INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint, heal_checkpoint, librarian_watermark_at, librarian_watermark_id)
|
|
3062
|
+
VALUES (?, 0, 0, ?, ?)
|
|
3063
|
+
ON CONFLICT(entity_id) DO UPDATE SET
|
|
3064
|
+
librarian_watermark_at = excluded.librarian_watermark_at,
|
|
3065
|
+
librarian_watermark_id = excluded.librarian_watermark_id`,
|
|
3066
|
+
[entityId, wm.at, wm.id]
|
|
3067
|
+
);
|
|
3068
|
+
}
|
|
2911
3069
|
// META TABLE METHODS
|
|
2912
3070
|
async getMeta(key, tx) {
|
|
2913
3071
|
const executor = this.getExecutor(tx);
|
|
@@ -3080,6 +3238,8 @@ async function runBatched(args) {
|
|
|
3080
3238
|
const results = [];
|
|
3081
3239
|
const skipped = [];
|
|
3082
3240
|
let batches = 0;
|
|
3241
|
+
const settled = /* @__PURE__ */ new Set();
|
|
3242
|
+
let budgetStop;
|
|
3083
3243
|
let batchSize = initialBatchSize(maxOutputTokens);
|
|
3084
3244
|
const trim = async (candidate) => {
|
|
3085
3245
|
const whole = await buildPrompt(candidate);
|
|
@@ -3113,6 +3273,7 @@ async function runBatched(args) {
|
|
|
3113
3273
|
} else {
|
|
3114
3274
|
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
3115
3275
|
skipped.push({ item: batch[0], reason });
|
|
3276
|
+
settled.add(batch[0]);
|
|
3116
3277
|
onSkip?.(batch[0], err);
|
|
3117
3278
|
}
|
|
3118
3279
|
return;
|
|
@@ -3121,7 +3282,7 @@ async function runBatched(args) {
|
|
|
3121
3282
|
const mid = Math.ceil(batch.length / 2);
|
|
3122
3283
|
if (mid < batchSize) batchSize = mid;
|
|
3123
3284
|
let i = 0;
|
|
3124
|
-
while (i < batch.length) {
|
|
3285
|
+
while (i < batch.length && !budgetStop) {
|
|
3125
3286
|
const size = Math.min(batchSize, batch.length - i);
|
|
3126
3287
|
const trimmed = await trim(batch.slice(i, i + size));
|
|
3127
3288
|
await attempt(trimmed.batch, trimmed.prompts);
|
|
@@ -3129,6 +3290,7 @@ async function runBatched(args) {
|
|
|
3129
3290
|
}
|
|
3130
3291
|
};
|
|
3131
3292
|
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
3293
|
+
if (budgetStop) return;
|
|
3132
3294
|
if (batch.length === 0) return;
|
|
3133
3295
|
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
3134
3296
|
batches++;
|
|
@@ -3136,6 +3298,10 @@ async function runBatched(args) {
|
|
|
3136
3298
|
try {
|
|
3137
3299
|
responseText = await call(prompts);
|
|
3138
3300
|
} catch (err) {
|
|
3301
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
3302
|
+
budgetStop = { requiredEstimate: err.requiredEstimate };
|
|
3303
|
+
return;
|
|
3304
|
+
}
|
|
3139
3305
|
await onFailure(batch, err, attemptLevel, true);
|
|
3140
3306
|
return;
|
|
3141
3307
|
}
|
|
@@ -3147,19 +3313,781 @@ async function runBatched(args) {
|
|
|
3147
3313
|
return;
|
|
3148
3314
|
}
|
|
3149
3315
|
results.push(result);
|
|
3316
|
+
for (const item of batch) settled.add(item);
|
|
3150
3317
|
};
|
|
3151
3318
|
let index = 0;
|
|
3152
|
-
while (index < items.length) {
|
|
3319
|
+
while (index < items.length && !budgetStop) {
|
|
3153
3320
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
3154
3321
|
index += batch.length;
|
|
3155
3322
|
await attempt(batch, prompts, 0);
|
|
3156
3323
|
}
|
|
3157
|
-
|
|
3324
|
+
const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
|
|
3325
|
+
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
3158
3326
|
}
|
|
3159
3327
|
|
|
3160
|
-
// src/services/
|
|
3328
|
+
// src/services/librarian/constants.ts
|
|
3161
3329
|
var FUZZY_THRESHOLD = 0.5;
|
|
3162
3330
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
3331
|
+
|
|
3332
|
+
// src/services/librarian/legacy.ts
|
|
3333
|
+
var runLegacyLibrarianPass = async (deps, ctx) => {
|
|
3334
|
+
const { entityId, trigger } = ctx;
|
|
3335
|
+
const promptOverride = ctx.promptOverride;
|
|
3336
|
+
const events = await deps.eventRepo.getRecent(entityId, 50);
|
|
3337
|
+
const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
|
|
3338
|
+
const currentFacts = currentFactsRows.map((f) => {
|
|
3339
|
+
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
3340
|
+
return {
|
|
3341
|
+
...rest,
|
|
3342
|
+
tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
|
|
3343
|
+
};
|
|
3344
|
+
});
|
|
3345
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3346
|
+
const promptEvents = events.reverse();
|
|
3347
|
+
const librarianGrounding = deps.promptService.groundingFor("librarian");
|
|
3348
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
|
|
3349
|
+
promptEvents,
|
|
3350
|
+
currentFacts,
|
|
3351
|
+
promptOverride,
|
|
3352
|
+
ontologyContext
|
|
3353
|
+
);
|
|
3354
|
+
let responseText;
|
|
3355
|
+
try {
|
|
3356
|
+
responseText = await callLlm(
|
|
3357
|
+
deps.options,
|
|
3358
|
+
{ operation: "librarian", entityId, trigger, meter: ctx.meter },
|
|
3359
|
+
{ systemPrompt, userPrompt }
|
|
3360
|
+
);
|
|
3361
|
+
} catch (err) {
|
|
3362
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
3363
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
3364
|
+
}
|
|
3365
|
+
throw err;
|
|
3366
|
+
}
|
|
3367
|
+
const result = parseJsonResponse(responseText);
|
|
3368
|
+
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
3369
|
+
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
3370
|
+
const ontologyUpdates = result.ontology_updates;
|
|
3371
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
3372
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3373
|
+
const validFacts = [];
|
|
3374
|
+
const validFactItemIndexes = [];
|
|
3375
|
+
facts.forEach((raw, itemIndex) => {
|
|
3376
|
+
const valid = validateFact(raw);
|
|
3377
|
+
if (valid) {
|
|
3378
|
+
validFacts.push(valid);
|
|
3379
|
+
validFactItemIndexes.push(itemIndex);
|
|
3380
|
+
} else {
|
|
3381
|
+
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3382
|
+
}
|
|
3383
|
+
});
|
|
3384
|
+
const validTasks = [];
|
|
3385
|
+
tasks.forEach((raw, itemIndex) => {
|
|
3386
|
+
const valid = validateTask(raw);
|
|
3387
|
+
if (valid) validTasks.push(valid);
|
|
3388
|
+
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
3389
|
+
});
|
|
3390
|
+
const now = Date.now();
|
|
3391
|
+
const insertedFacts = [];
|
|
3392
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
3393
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3394
|
+
if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
|
|
3395
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
3396
|
+
}
|
|
3397
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3398
|
+
for (const existing of currentFactsRows) {
|
|
3399
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3400
|
+
id: existing.id,
|
|
3401
|
+
okf_type: existing.okf_type ?? null
|
|
3402
|
+
});
|
|
3403
|
+
}
|
|
3404
|
+
const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
3405
|
+
const pendingEdges = [];
|
|
3406
|
+
for (const [k, fact] of validFacts.entries()) {
|
|
3407
|
+
const newTokens = titleTokens(fact.title);
|
|
3408
|
+
let skip = false;
|
|
3409
|
+
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3410
|
+
for (const existing of factsForDedupe) {
|
|
3411
|
+
if (existing.source_type !== "librarian_inferred") continue;
|
|
3412
|
+
const existingTokens = titleTokens(existing.title);
|
|
3413
|
+
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3414
|
+
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
3415
|
+
skip = true;
|
|
3416
|
+
break;
|
|
3417
|
+
}
|
|
3418
|
+
}
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
if (skip) {
|
|
3422
|
+
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
3423
|
+
continue;
|
|
3424
|
+
}
|
|
3425
|
+
const ontologyFact = fact;
|
|
3426
|
+
const id = generateId("fact_");
|
|
3427
|
+
const validationDrops = [];
|
|
3428
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
3429
|
+
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3430
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3431
|
+
if (grounding?.diagnostic) {
|
|
3432
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3433
|
+
}
|
|
3434
|
+
const factObj = {
|
|
3435
|
+
id,
|
|
3436
|
+
entity_id: entityId,
|
|
3437
|
+
title: fact.title,
|
|
3438
|
+
body: fact.body,
|
|
3439
|
+
tags: fact.tags,
|
|
3440
|
+
confidence: fact.confidence,
|
|
3441
|
+
source_type: "librarian_inferred",
|
|
3442
|
+
source_hash: null,
|
|
3443
|
+
source_ref: null,
|
|
3444
|
+
created_at: now,
|
|
3445
|
+
updated_at: now,
|
|
3446
|
+
last_accessed_at: null,
|
|
3447
|
+
access_count: 0,
|
|
3448
|
+
deleted_at: null,
|
|
3449
|
+
okf_type: normalized.okf_type,
|
|
3450
|
+
...grounding?.trust
|
|
3451
|
+
};
|
|
3452
|
+
await deps.entryRepo.upsert(factObj, tx);
|
|
3453
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3454
|
+
factsForDedupe.push(factObj);
|
|
3455
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3456
|
+
if (normalized.edges.length > 0) {
|
|
3457
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
for (const item of pendingEdges) {
|
|
3461
|
+
const resolveDrops = [];
|
|
3462
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
3463
|
+
entityId,
|
|
3464
|
+
item.sourceId,
|
|
3465
|
+
item.sourceType,
|
|
3466
|
+
item.edges ?? [],
|
|
3467
|
+
manifest,
|
|
3468
|
+
titleIndex,
|
|
3469
|
+
tx,
|
|
3470
|
+
now,
|
|
3471
|
+
resolveDrops
|
|
3472
|
+
);
|
|
3473
|
+
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
3474
|
+
}
|
|
3475
|
+
for (const task of validTasks) {
|
|
3476
|
+
const id = generateId("task_");
|
|
3477
|
+
const taskObj = {
|
|
3478
|
+
id,
|
|
3479
|
+
entity_id: entityId,
|
|
3480
|
+
description: task.description,
|
|
3481
|
+
status: "pending",
|
|
3482
|
+
priority: task.priority,
|
|
3483
|
+
created_at: now,
|
|
3484
|
+
updated_at: now,
|
|
3485
|
+
resolved_at: null,
|
|
3486
|
+
deleted_at: null
|
|
3487
|
+
};
|
|
3488
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
3489
|
+
}
|
|
3490
|
+
});
|
|
3491
|
+
diagBuffer.flush(deps.options);
|
|
3492
|
+
await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
3493
|
+
for (const fact of insertedFacts) {
|
|
3494
|
+
await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
3495
|
+
}
|
|
3496
|
+
deps.searchService.evictCache(entityId);
|
|
3497
|
+
const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
|
|
3498
|
+
const out = { processedThrough, factsWritten: insertedFacts.length };
|
|
3499
|
+
return out;
|
|
3500
|
+
};
|
|
3501
|
+
|
|
3502
|
+
// src/services/librarian/ops/text.ts
|
|
3503
|
+
var OPS_NEIGHBOUR_BODY_CHARS = 400;
|
|
3504
|
+
function normalizeFactText(title, body) {
|
|
3505
|
+
return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
3506
|
+
}
|
|
3507
|
+
function labelEvents(events) {
|
|
3508
|
+
return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
|
|
3509
|
+
}
|
|
3510
|
+
function parseValidFrom(v) {
|
|
3511
|
+
if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
|
|
3512
|
+
if (typeof v === "string") {
|
|
3513
|
+
const t = Date.parse(v);
|
|
3514
|
+
return Number.isFinite(t) && t >= 0 ? t : void 0;
|
|
3515
|
+
}
|
|
3516
|
+
return void 0;
|
|
3517
|
+
}
|
|
3518
|
+
|
|
3519
|
+
// src/services/librarian/ops/extract.ts
|
|
3520
|
+
var OPS_MAX_EVENTS_PER_PASS = 50;
|
|
3521
|
+
var OPS_MAX_EXTRACT_CHARS = 24e3;
|
|
3522
|
+
var RESOLVE_RESERVE_FACTOR = 2;
|
|
3523
|
+
var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
|
|
3524
|
+
async function selectBatch(deps, entityId, wm, meter) {
|
|
3525
|
+
const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
|
|
3526
|
+
if (events.length === 0) return { batch: [] };
|
|
3527
|
+
let n = 0;
|
|
3528
|
+
let chars = 0;
|
|
3529
|
+
for (const e of events) {
|
|
3530
|
+
if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
|
|
3531
|
+
chars += e.summary.length;
|
|
3532
|
+
n++;
|
|
3533
|
+
}
|
|
3534
|
+
let batch = labelEvents(events.slice(0, n));
|
|
3535
|
+
if (meter && meter.budget !== void 0) {
|
|
3536
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3537
|
+
for (; ; ) {
|
|
3538
|
+
const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
|
|
3539
|
+
if (meter.fits(need)) break;
|
|
3540
|
+
if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
|
|
3541
|
+
batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
|
|
3542
|
+
}
|
|
3543
|
+
}
|
|
3544
|
+
return { batch };
|
|
3545
|
+
}
|
|
3546
|
+
async function runExtract(deps, ctx, batch) {
|
|
3547
|
+
const { entityId, trigger } = ctx;
|
|
3548
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3549
|
+
const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
|
|
3550
|
+
const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
|
|
3551
|
+
const parsed = parseJsonResponse(text);
|
|
3552
|
+
const base = { entityId, operation: "librarian", trigger };
|
|
3553
|
+
const diagnostics = [];
|
|
3554
|
+
const byLabel = new Map(batch.map((l) => [l.label, l]));
|
|
3555
|
+
const newestAt = batch[batch.length - 1].at;
|
|
3556
|
+
const candidates = [];
|
|
3557
|
+
(Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
|
|
3558
|
+
const fact = validateFact(raw);
|
|
3559
|
+
if (!fact) {
|
|
3560
|
+
diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
|
|
3561
|
+
return;
|
|
3562
|
+
}
|
|
3563
|
+
const r = raw;
|
|
3564
|
+
const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
|
|
3565
|
+
const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
|
|
3566
|
+
candidates.push({ index, fact, sourceLabel, validFrom });
|
|
3567
|
+
});
|
|
3568
|
+
const tasks = [];
|
|
3569
|
+
(Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
|
|
3570
|
+
const task = validateTask(raw);
|
|
3571
|
+
if (task) tasks.push(task);
|
|
3572
|
+
else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
|
|
3573
|
+
});
|
|
3574
|
+
return {
|
|
3575
|
+
batch,
|
|
3576
|
+
candidates,
|
|
3577
|
+
tasks,
|
|
3578
|
+
...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
|
|
3579
|
+
groundingCorpus: prompt.groundingCorpus ?? null,
|
|
3580
|
+
diagnostics
|
|
3581
|
+
};
|
|
3582
|
+
}
|
|
3583
|
+
|
|
3584
|
+
// src/services/librarian/ops/gate.ts
|
|
3585
|
+
var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
|
|
3586
|
+
function resolveGateConfig(cfg) {
|
|
3587
|
+
const kRaw = cfg?.k;
|
|
3588
|
+
const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
|
|
3589
|
+
const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
|
|
3590
|
+
const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
|
|
3591
|
+
if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
|
|
3592
|
+
return { k, dupThreshold, novelThreshold };
|
|
3593
|
+
}
|
|
3594
|
+
function titlesMatch(a, b) {
|
|
3595
|
+
const ta = titleTokens(a);
|
|
3596
|
+
const tb = titleTokens(b);
|
|
3597
|
+
if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
|
|
3598
|
+
return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
|
|
3599
|
+
}
|
|
3600
|
+
function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
3601
|
+
const key = normalizeFactText(candidate.title, candidate.body);
|
|
3602
|
+
const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
|
|
3603
|
+
if (same) return { kind: "noop", target: same.id };
|
|
3604
|
+
if (neighbours.length === 0) return { kind: "add" };
|
|
3605
|
+
if (mode === "keyword") return { kind: "ambiguous" };
|
|
3606
|
+
const best = neighbours[0];
|
|
3607
|
+
const score = best.score ?? 0;
|
|
3608
|
+
if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
|
|
3609
|
+
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3610
|
+
return { kind: "ambiguous" };
|
|
3611
|
+
}
|
|
3612
|
+
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3613
|
+
const { entityId, trigger } = ctx;
|
|
3614
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3615
|
+
const unique = [];
|
|
3616
|
+
const inBatchDuplicates = [];
|
|
3617
|
+
for (const c of candidates) {
|
|
3618
|
+
const key = normalizeFactText(c.fact.title, c.fact.body);
|
|
3619
|
+
if (seen.has(key)) inBatchDuplicates.push(c);
|
|
3620
|
+
else {
|
|
3621
|
+
seen.add(key);
|
|
3622
|
+
unique.push(c);
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3626
|
+
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3627
|
+
const stored = [];
|
|
3628
|
+
if (canEmbed && unique.length > 0) {
|
|
3629
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3630
|
+
if (nonLive.has(row.id)) continue;
|
|
3631
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3632
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
3633
|
+
}
|
|
3634
|
+
}
|
|
3635
|
+
const gated = [];
|
|
3636
|
+
for (const c of unique) {
|
|
3637
|
+
let ranked = null;
|
|
3638
|
+
let vector = null;
|
|
3639
|
+
let mode = "keyword";
|
|
3640
|
+
if (canEmbed) {
|
|
3641
|
+
const r = await deps.embeddingService.embedTextForFact(
|
|
3642
|
+
{ id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
|
|
3643
|
+
void 0,
|
|
3644
|
+
{ markFailures: false }
|
|
3645
|
+
);
|
|
3646
|
+
if (r.ok) {
|
|
3647
|
+
vector = r.vector;
|
|
3648
|
+
if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
|
|
3649
|
+
ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
|
|
3650
|
+
mode = "vector";
|
|
3651
|
+
}
|
|
3652
|
+
}
|
|
3653
|
+
}
|
|
3654
|
+
if (!ranked) {
|
|
3655
|
+
ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
3656
|
+
}
|
|
3657
|
+
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3658
|
+
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
3659
|
+
const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
|
|
3660
|
+
const f = byId.get(r.id);
|
|
3661
|
+
return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
|
|
3662
|
+
});
|
|
3663
|
+
gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
|
|
3664
|
+
}
|
|
3665
|
+
const count = (k) => gated.filter((g) => g.decision.kind === k).length;
|
|
3666
|
+
const gateDiagnostic = {
|
|
3667
|
+
code: "librarian_gate",
|
|
3668
|
+
operation: "librarian",
|
|
3669
|
+
trigger,
|
|
3670
|
+
entityId,
|
|
3671
|
+
detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
|
|
3672
|
+
};
|
|
3673
|
+
return { gated, inBatchDuplicates, gateDiagnostic };
|
|
3674
|
+
}
|
|
3675
|
+
|
|
3676
|
+
// src/services/librarian/ops/resolve.ts
|
|
3677
|
+
var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
|
|
3678
|
+
function validateOp(raw, aliases) {
|
|
3679
|
+
if (!raw || typeof raw !== "object") return null;
|
|
3680
|
+
const r = raw;
|
|
3681
|
+
if (r.op === "ADD") return { op: "ADD" };
|
|
3682
|
+
if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
|
|
3683
|
+
const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
|
|
3684
|
+
if (!targetId) return null;
|
|
3685
|
+
if (r.op === "NOOP") return { op: "NOOP", targetId };
|
|
3686
|
+
if (r.op === "SUPERSEDE") {
|
|
3687
|
+
const validFrom = parseValidFrom(r.valid_from);
|
|
3688
|
+
return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
|
|
3689
|
+
}
|
|
3690
|
+
const update = { op: "UPDATE", targetId };
|
|
3691
|
+
if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
|
|
3692
|
+
if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
|
|
3693
|
+
return update;
|
|
3694
|
+
}
|
|
3695
|
+
var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
|
|
3696
|
+
async function runResolve(deps, ctx, ambiguous) {
|
|
3697
|
+
const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
|
|
3698
|
+
if (ambiguous.length === 0) return out;
|
|
3699
|
+
const { entityId, trigger } = ctx;
|
|
3700
|
+
const outcome = await runBatched({
|
|
3701
|
+
items: ambiguous,
|
|
3702
|
+
buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
|
|
3703
|
+
batch.map((g, i) => ({
|
|
3704
|
+
item: i,
|
|
3705
|
+
candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
|
|
3706
|
+
existing: g.neighbours.map((n) => ({
|
|
3707
|
+
ref: n.ref,
|
|
3708
|
+
title: n.title,
|
|
3709
|
+
body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
|
|
3710
|
+
}))
|
|
3711
|
+
}))
|
|
3712
|
+
),
|
|
3713
|
+
call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
|
|
3714
|
+
parse: (text, batch) => {
|
|
3715
|
+
const parsed = parseJsonResponse(text);
|
|
3716
|
+
if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
|
|
3717
|
+
return { batch, ops: parsed.ops };
|
|
3718
|
+
},
|
|
3719
|
+
maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
|
|
3720
|
+
maxOutputTokens: deps.options.llmProvider.maxOutputTokens
|
|
3721
|
+
});
|
|
3722
|
+
if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
|
|
3723
|
+
for (const { batch, ops } of outcome.results) {
|
|
3724
|
+
const byItem = /* @__PURE__ */ new Map();
|
|
3725
|
+
for (const raw of ops) {
|
|
3726
|
+
const item = raw?.item;
|
|
3727
|
+
if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
|
|
3728
|
+
byItem.set(item, raw);
|
|
3729
|
+
}
|
|
3730
|
+
}
|
|
3731
|
+
batch.forEach((g, i) => {
|
|
3732
|
+
const op = validateOp(byItem.get(i), aliasMapOf(g));
|
|
3733
|
+
if (op) out.ops.set(g, op);
|
|
3734
|
+
else out.rejected.push(g);
|
|
3735
|
+
});
|
|
3736
|
+
}
|
|
3737
|
+
if (outcome.skipped.some((s) => s.reason === "call_error")) {
|
|
3738
|
+
throw new Error("resolve: transient provider error; retry the pass");
|
|
3739
|
+
}
|
|
3740
|
+
for (const s of outcome.skipped) out.failed.push(s.item);
|
|
3741
|
+
return out;
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
// src/utils/temporal.ts
|
|
3745
|
+
function liveAtSql(mode, alias = "") {
|
|
3746
|
+
const from = mode === "asOf" ? `COALESCE(${alias}valid_from, ${alias}created_at) <= ?` : `(${alias}valid_from IS NULL OR ${alias}valid_from <= ?)`;
|
|
3747
|
+
return `(${from} AND (${alias}valid_to IS NULL OR ${alias}valid_to > ?))`;
|
|
3748
|
+
}
|
|
3749
|
+
function isLiveAt(f, mode, t) {
|
|
3750
|
+
if (f.deleted_at != null) return false;
|
|
3751
|
+
const from = mode === "asOf" ? f.valid_from ?? f.created_at : f.valid_from;
|
|
3752
|
+
if (from != null && from > t) return false;
|
|
3753
|
+
return f.valid_to == null || f.valid_to > t;
|
|
3754
|
+
}
|
|
3755
|
+
function assertEpochMs(name, v) {
|
|
3756
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
3757
|
+
throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
|
|
3758
|
+
}
|
|
3759
|
+
return Math.trunc(v);
|
|
3760
|
+
}
|
|
3761
|
+
|
|
3762
|
+
// src/services/TemporalService.ts
|
|
3763
|
+
var HISTORY_MAX_DEPTH = 100;
|
|
3764
|
+
var TemporalService = class {
|
|
3765
|
+
constructor(db, options, entryRepo, searchService, embeddingService) {
|
|
3766
|
+
this.db = db;
|
|
3767
|
+
this.options = options;
|
|
3768
|
+
this.entryRepo = entryRepo;
|
|
3769
|
+
this.searchService = searchService;
|
|
3770
|
+
this.embeddingService = embeddingService;
|
|
3771
|
+
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Link `oldId` → `newId` inside the caller's transaction. `newId` must
|
|
3774
|
+
* already exist. Throws WikiSupersedeError on any rule violation; the
|
|
3775
|
+
* caller's transaction then rolls back.
|
|
3776
|
+
*/
|
|
3777
|
+
async supersedeInTx(tx, entityId, oldId, newId, t, now) {
|
|
3778
|
+
const old = await this.entryRepo.findTemporalRow(oldId, tx);
|
|
3779
|
+
if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3780
|
+
if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3781
|
+
if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3782
|
+
if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
|
|
3783
|
+
if (newId === oldId) throw new WikiSupersedeError("cycle");
|
|
3784
|
+
const repl = await this.entryRepo.findTemporalRow(newId, tx);
|
|
3785
|
+
if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
3786
|
+
if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
3787
|
+
if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
3788
|
+
const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
|
|
3789
|
+
if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
|
|
3790
|
+
await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
|
|
3791
|
+
if (repl.valid_from == null) {
|
|
3792
|
+
await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
|
|
3793
|
+
}
|
|
3794
|
+
}
|
|
3795
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
3796
|
+
const now = Date.now();
|
|
3797
|
+
const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
|
|
3798
|
+
let created = null;
|
|
3799
|
+
if (typeof replacement !== "string") {
|
|
3800
|
+
if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
|
|
3801
|
+
throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
|
|
3802
|
+
}
|
|
3803
|
+
created = {
|
|
3804
|
+
id: generateId("fact_"),
|
|
3805
|
+
entity_id: entityId,
|
|
3806
|
+
title: replacement.title,
|
|
3807
|
+
body: replacement.body,
|
|
3808
|
+
tags: Array.isArray(replacement.tags) ? replacement.tags : [],
|
|
3809
|
+
confidence: replacement.confidence ?? "certain",
|
|
3810
|
+
source_type: replacement.source_type ?? "user_stated",
|
|
3811
|
+
source_hash: null,
|
|
3812
|
+
source_ref: null,
|
|
3813
|
+
created_at: now,
|
|
3814
|
+
updated_at: now,
|
|
3815
|
+
last_accessed_at: null,
|
|
3816
|
+
access_count: 0,
|
|
3817
|
+
deleted_at: null
|
|
3818
|
+
};
|
|
3819
|
+
}
|
|
3820
|
+
const newId = typeof replacement === "string" ? replacement : created.id;
|
|
3821
|
+
const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
|
|
3822
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3823
|
+
if (created) {
|
|
3824
|
+
await this.entryRepo.upsert(created, tx);
|
|
3825
|
+
if (newValidFrom !== void 0) {
|
|
3826
|
+
await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
|
|
3827
|
+
}
|
|
3828
|
+
}
|
|
3829
|
+
await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
|
|
3830
|
+
});
|
|
3831
|
+
await this.searchService.syncEntries(entityId, [oldId, newId]);
|
|
3832
|
+
if (created) {
|
|
3833
|
+
await this.embeddingService.embedFact(
|
|
3834
|
+
{ id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
|
|
3835
|
+
{ operation: "supersede", trigger: "call" }
|
|
3836
|
+
);
|
|
3837
|
+
}
|
|
3838
|
+
this.searchService.evictCache(entityId);
|
|
3839
|
+
return { newId };
|
|
3840
|
+
}
|
|
3841
|
+
async history(entityId, factId) {
|
|
3842
|
+
const start = await this.entryRepo.findTemporalRow(factId);
|
|
3843
|
+
if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
|
|
3844
|
+
const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
|
|
3845
|
+
const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
|
|
3846
|
+
return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
|
|
3847
|
+
}
|
|
3848
|
+
};
|
|
3849
|
+
|
|
3850
|
+
// src/services/librarian/ops/apply.ts
|
|
3851
|
+
async function applyOps(deps, ctx, input) {
|
|
3852
|
+
const { entityId, trigger } = ctx;
|
|
3853
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3854
|
+
const grounding = deps.promptService.groundingFor("librarian");
|
|
3855
|
+
const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
|
|
3856
|
+
const rejected = new Set(input.rejected);
|
|
3857
|
+
const failed = new Set(input.failed);
|
|
3858
|
+
const written = [];
|
|
3859
|
+
const updatedIds = [];
|
|
3860
|
+
const now = Date.now();
|
|
3861
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
3862
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3863
|
+
if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
|
|
3864
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
|
|
3865
|
+
}
|
|
3866
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
3867
|
+
for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
|
|
3868
|
+
titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
|
|
3869
|
+
}
|
|
3870
|
+
const pendingEdges = [];
|
|
3871
|
+
const insert = async (g, draft, validFrom, overrides) => {
|
|
3872
|
+
const fact = g.candidate.fact;
|
|
3873
|
+
const title = overrides?.title ?? fact.title;
|
|
3874
|
+
const body = overrides?.body ?? fact.body;
|
|
3875
|
+
const id = generateId("fact_");
|
|
3876
|
+
const drops = [];
|
|
3877
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
|
|
3878
|
+
for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
|
|
3879
|
+
const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
|
|
3880
|
+
if (g2?.diagnostic) {
|
|
3881
|
+
input.diagnostics.push({
|
|
3882
|
+
...diagBase,
|
|
3883
|
+
code: g2.diagnostic.code,
|
|
3884
|
+
detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
|
|
3885
|
+
});
|
|
3886
|
+
}
|
|
3887
|
+
const row = {
|
|
3888
|
+
id,
|
|
3889
|
+
entity_id: entityId,
|
|
3890
|
+
title,
|
|
3891
|
+
body,
|
|
3892
|
+
tags: fact.tags,
|
|
3893
|
+
confidence: fact.confidence,
|
|
3894
|
+
source_type: "librarian_inferred",
|
|
3895
|
+
source_hash: null,
|
|
3896
|
+
source_ref: null,
|
|
3897
|
+
created_at: now,
|
|
3898
|
+
updated_at: now,
|
|
3899
|
+
last_accessed_at: null,
|
|
3900
|
+
access_count: 0,
|
|
3901
|
+
deleted_at: null,
|
|
3902
|
+
okf_type: normalized.okf_type,
|
|
3903
|
+
...g2?.trust,
|
|
3904
|
+
...draft ? { lifecycle_status: "draft" } : {}
|
|
3905
|
+
};
|
|
3906
|
+
await deps.entryRepo.upsert(row, tx);
|
|
3907
|
+
await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
|
|
3908
|
+
titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
|
|
3909
|
+
if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3910
|
+
const textChanged = title !== fact.title || body !== fact.body;
|
|
3911
|
+
written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
|
|
3912
|
+
return id;
|
|
3913
|
+
};
|
|
3914
|
+
for (const g of input.gated) {
|
|
3915
|
+
const idx = g.candidate.index;
|
|
3916
|
+
const vf = g.candidate.validFrom;
|
|
3917
|
+
if (g.decision.kind === "noop") continue;
|
|
3918
|
+
if (g.decision.kind === "add") {
|
|
3919
|
+
await insert(g, false, vf);
|
|
3920
|
+
continue;
|
|
3921
|
+
}
|
|
3922
|
+
if (failed.has(g)) {
|
|
3923
|
+
const id = await insert(g, true, vf);
|
|
3924
|
+
input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
|
|
3925
|
+
continue;
|
|
3926
|
+
}
|
|
3927
|
+
const op = input.ops.get(g);
|
|
3928
|
+
if (rejected.has(g) || !op) {
|
|
3929
|
+
await insert(g, false, vf);
|
|
3930
|
+
input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
|
|
3931
|
+
continue;
|
|
3932
|
+
}
|
|
3933
|
+
if (op.op === "NOOP") continue;
|
|
3934
|
+
if (op.op === "ADD") {
|
|
3935
|
+
await insert(g, false, vf);
|
|
3936
|
+
continue;
|
|
3937
|
+
}
|
|
3938
|
+
const target = g.neighbours.find((n) => n.id === op.targetId);
|
|
3939
|
+
if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
|
|
3940
|
+
const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
|
|
3941
|
+
if (existing) {
|
|
3942
|
+
const newTitle = op.title ?? g.candidate.fact.title;
|
|
3943
|
+
await deps.entryRepo.upsert({
|
|
3944
|
+
...existing,
|
|
3945
|
+
title: newTitle,
|
|
3946
|
+
body: op.body ?? g.candidate.fact.body,
|
|
3947
|
+
updated_at: now
|
|
3948
|
+
}, tx);
|
|
3949
|
+
titleIndex.delete(normalizeTitleKey(existing.title));
|
|
3950
|
+
titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
|
|
3951
|
+
updatedIds.push(existing.id);
|
|
3952
|
+
continue;
|
|
3953
|
+
}
|
|
3954
|
+
}
|
|
3955
|
+
const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
|
|
3956
|
+
const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
|
|
3957
|
+
if (target.source_type === "immutable_document") {
|
|
3958
|
+
const id = await insert(g, true, supersedeFrom, overrides);
|
|
3959
|
+
input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
|
|
3960
|
+
continue;
|
|
3961
|
+
}
|
|
3962
|
+
const newId = await insert(g, false, supersedeFrom, overrides);
|
|
3963
|
+
try {
|
|
3964
|
+
await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
|
|
3965
|
+
} catch (err) {
|
|
3966
|
+
if (!(err instanceof WikiSupersedeError)) throw err;
|
|
3967
|
+
input.diagnostics.push({
|
|
3968
|
+
...diagBase,
|
|
3969
|
+
code: "librarian_op_rejected",
|
|
3970
|
+
detail: { factId: newId, itemIndex: idx, reason: err.reason }
|
|
3971
|
+
});
|
|
3972
|
+
}
|
|
3973
|
+
}
|
|
3974
|
+
for (const item of pendingEdges) {
|
|
3975
|
+
const resolveDrops = [];
|
|
3976
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
3977
|
+
entityId,
|
|
3978
|
+
item.sourceId,
|
|
3979
|
+
item.sourceType,
|
|
3980
|
+
item.edges,
|
|
3981
|
+
manifest,
|
|
3982
|
+
titleIndex,
|
|
3983
|
+
tx,
|
|
3984
|
+
now,
|
|
3985
|
+
resolveDrops
|
|
3986
|
+
);
|
|
3987
|
+
for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
|
|
3988
|
+
}
|
|
3989
|
+
for (const task of input.tasks) {
|
|
3990
|
+
const taskObj = {
|
|
3991
|
+
id: generateId("task_"),
|
|
3992
|
+
entity_id: entityId,
|
|
3993
|
+
description: task.description,
|
|
3994
|
+
status: "pending",
|
|
3995
|
+
priority: task.priority,
|
|
3996
|
+
created_at: now,
|
|
3997
|
+
updated_at: now,
|
|
3998
|
+
resolved_at: null,
|
|
3999
|
+
deleted_at: null
|
|
4000
|
+
};
|
|
4001
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
4002
|
+
}
|
|
4003
|
+
});
|
|
4004
|
+
return { written, updatedIds };
|
|
4005
|
+
}
|
|
4006
|
+
|
|
4007
|
+
// src/services/librarian/ops/index.ts
|
|
4008
|
+
var runOpsLibrarianPass = async (deps, ctx) => {
|
|
4009
|
+
const { entityId, trigger } = ctx;
|
|
4010
|
+
const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
|
|
4011
|
+
await deps.seedWatermark(entityId);
|
|
4012
|
+
const wm = await deps.getWatermark(entityId);
|
|
4013
|
+
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
4014
|
+
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
4015
|
+
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
4016
|
+
let ex;
|
|
4017
|
+
try {
|
|
4018
|
+
ex = await runExtract(deps, ctx, sel.batch);
|
|
4019
|
+
} catch (err) {
|
|
4020
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
4021
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
4022
|
+
}
|
|
4023
|
+
throw err;
|
|
4024
|
+
}
|
|
4025
|
+
const diagnostics = new DiagnosticBuffer();
|
|
4026
|
+
for (const d of ex.diagnostics) diagnostics.push(d);
|
|
4027
|
+
const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
|
|
4028
|
+
for (const dup of inBatchDuplicates) {
|
|
4029
|
+
diagnostics.push({
|
|
4030
|
+
code: "fact_deduplicated",
|
|
4031
|
+
operation: "librarian",
|
|
4032
|
+
trigger,
|
|
4033
|
+
entityId,
|
|
4034
|
+
detail: { itemIndex: dup.index, reason: "in_batch" }
|
|
4035
|
+
});
|
|
4036
|
+
}
|
|
4037
|
+
diagnostics.push(gateDiagnostic);
|
|
4038
|
+
const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
|
|
4039
|
+
if (res.budgetStop) {
|
|
4040
|
+
diagnostics.discard();
|
|
4041
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
|
|
4042
|
+
}
|
|
4043
|
+
const { written, updatedIds } = await applyOps(deps, ctx, {
|
|
4044
|
+
gated,
|
|
4045
|
+
ops: res.ops,
|
|
4046
|
+
rejected: res.rejected,
|
|
4047
|
+
failed: res.failed,
|
|
4048
|
+
tasks: ex.tasks,
|
|
4049
|
+
ontologyUpdates: ex.ontologyUpdates,
|
|
4050
|
+
groundingCorpus: ex.groundingCorpus,
|
|
4051
|
+
diagnostics
|
|
4052
|
+
});
|
|
4053
|
+
diagnostics.flush(deps.options);
|
|
4054
|
+
await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
|
|
4055
|
+
const embedCtx = { operation: "librarian", trigger };
|
|
4056
|
+
for (const w of written) {
|
|
4057
|
+
if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
|
|
4058
|
+
else await deps.embeddingService.embedFact(w.fact, embedCtx);
|
|
4059
|
+
}
|
|
4060
|
+
if (updatedIds.length > 0) {
|
|
4061
|
+
for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
|
|
4062
|
+
await deps.embeddingService.embedFact(
|
|
4063
|
+
{ id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
|
|
4064
|
+
embedCtx
|
|
4065
|
+
);
|
|
4066
|
+
}
|
|
4067
|
+
}
|
|
4068
|
+
deps.searchService.evictCache(entityId);
|
|
4069
|
+
const last = sel.batch[sel.batch.length - 1].event;
|
|
4070
|
+
return {
|
|
4071
|
+
processedThrough: { at: last.created_at, id: last.id },
|
|
4072
|
+
factsWritten: written.length + updatedIds.length
|
|
4073
|
+
};
|
|
4074
|
+
};
|
|
4075
|
+
|
|
4076
|
+
// src/services/librarian/index.ts
|
|
4077
|
+
var STRATEGIES = /* @__PURE__ */ new Map([
|
|
4078
|
+
["legacy", runLegacyLibrarianPass],
|
|
4079
|
+
["ops", runOpsLibrarianPass]
|
|
4080
|
+
]);
|
|
4081
|
+
async function runLibrarianStrategy(deps, ctx) {
|
|
4082
|
+
const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
|
|
4083
|
+
const fn = STRATEGIES.get(strategy);
|
|
4084
|
+
if (!fn) {
|
|
4085
|
+
throw new Error(`Unknown librarian strategy: ${strategy}`);
|
|
4086
|
+
}
|
|
4087
|
+
return fn(deps, ctx);
|
|
4088
|
+
}
|
|
4089
|
+
|
|
4090
|
+
// src/services/MaintenanceService.ts
|
|
3163
4091
|
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
3164
4092
|
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
3165
4093
|
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
@@ -3232,10 +4160,23 @@ var MaintenanceService = class {
|
|
|
3232
4160
|
const vacuum = options?.vacuum ?? false;
|
|
3233
4161
|
this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
|
|
3234
4162
|
this._validatePruneDuration(retainEventsFor, "retainEventsFor");
|
|
4163
|
+
const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
|
|
4164
|
+
this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
|
|
3235
4165
|
const now = Date.now();
|
|
4166
|
+
let syncedIds = [];
|
|
3236
4167
|
let deletedEntries = 0;
|
|
3237
4168
|
let deletedTasks = 0;
|
|
3238
4169
|
let deletedEvents = 0;
|
|
4170
|
+
if (pruneSupersededAfter !== null) {
|
|
4171
|
+
const expiredCutoff = now - pruneSupersededAfter * 864e5;
|
|
4172
|
+
const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
|
|
4173
|
+
if (expiredIds.length > 0) {
|
|
4174
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
4175
|
+
await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
|
|
4176
|
+
});
|
|
4177
|
+
syncedIds = [...expiredIds];
|
|
4178
|
+
}
|
|
4179
|
+
}
|
|
3239
4180
|
if (retainSoftDeletedFor !== null) {
|
|
3240
4181
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
3241
4182
|
const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
|
|
@@ -3251,6 +4192,7 @@ var MaintenanceService = class {
|
|
|
3251
4192
|
}
|
|
3252
4193
|
}
|
|
3253
4194
|
const succeededIds = succeeded.map((r) => r.id);
|
|
4195
|
+
syncedIds = [...syncedIds, ...succeededIds];
|
|
3254
4196
|
await this.db.withTransactionAsync(async (tx) => {
|
|
3255
4197
|
if (succeededIds.length > 0) {
|
|
3256
4198
|
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
@@ -3258,7 +4200,7 @@ var MaintenanceService = class {
|
|
|
3258
4200
|
deletedTasks = await this.taskRepo.bulkDeletePruned(entityId, cutoff, tx);
|
|
3259
4201
|
});
|
|
3260
4202
|
if (failure) {
|
|
3261
|
-
await this.searchService.
|
|
4203
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
3262
4204
|
const remaining = entriesToDelete.length - succeeded.length - 1;
|
|
3263
4205
|
const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
|
|
3264
4206
|
if (isTimeout) {
|
|
@@ -3286,13 +4228,23 @@ var MaintenanceService = class {
|
|
|
3286
4228
|
}
|
|
3287
4229
|
if (retainEventsFor !== null) {
|
|
3288
4230
|
const cutoff = now - retainEventsFor * 864e5;
|
|
3289
|
-
const
|
|
3290
|
-
|
|
4231
|
+
const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
|
|
4232
|
+
if (guarded) {
|
|
4233
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
4234
|
+
deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
|
|
4235
|
+
const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
|
|
4236
|
+
if (held > 0) {
|
|
4237
|
+
emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
|
|
4238
|
+
}
|
|
4239
|
+
} else {
|
|
4240
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
4241
|
+
deletedEvents = eventResult.changes;
|
|
4242
|
+
}
|
|
3291
4243
|
}
|
|
3292
4244
|
if (vacuum) {
|
|
3293
4245
|
await this.metadataRepo.vacuum();
|
|
3294
4246
|
}
|
|
3295
|
-
await this.searchService.
|
|
4247
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
3296
4248
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
3297
4249
|
} finally {
|
|
3298
4250
|
this.jobManager.releaseLock("prune", entityId);
|
|
@@ -3462,8 +4414,8 @@ var MaintenanceService = class {
|
|
|
3462
4414
|
deletedEntries += refResult;
|
|
3463
4415
|
}
|
|
3464
4416
|
});
|
|
3465
|
-
await this.searchService.sync(entityId);
|
|
3466
4417
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
4418
|
+
await this.searchService.syncEntries(entityId, uniqueDeletedIds);
|
|
3467
4419
|
for (const factId of uniqueDeletedIds) {
|
|
3468
4420
|
try {
|
|
3469
4421
|
await this.embeddingService.notifyEmbeddingPersistedOrThrow(entityId, factId, null);
|
|
@@ -3517,155 +4469,72 @@ var MaintenanceService = class {
|
|
|
3517
4469
|
}
|
|
3518
4470
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
3519
4471
|
async doRunLibrarian(entityId, promptOverride, trigger = "call") {
|
|
3520
|
-
|
|
3521
|
-
|
|
3522
|
-
|
|
3523
|
-
|
|
3524
|
-
|
|
3525
|
-
|
|
3526
|
-
|
|
3527
|
-
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
|
|
3538
|
-
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
3539
|
-
const result = parseJsonResponse(responseText);
|
|
3540
|
-
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
3541
|
-
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
3542
|
-
const ontologyUpdates = result.ontology_updates;
|
|
3543
|
-
const diagBuffer = new DiagnosticBuffer();
|
|
3544
|
-
const diagBase = { entityId, operation: "librarian", trigger };
|
|
3545
|
-
const validFacts = [];
|
|
3546
|
-
const validFactItemIndexes = [];
|
|
3547
|
-
facts.forEach((raw, itemIndex) => {
|
|
3548
|
-
const valid = validateFact(raw);
|
|
3549
|
-
if (valid) {
|
|
3550
|
-
validFacts.push(valid);
|
|
3551
|
-
validFactItemIndexes.push(itemIndex);
|
|
3552
|
-
} else {
|
|
3553
|
-
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
3554
|
-
}
|
|
3555
|
-
});
|
|
3556
|
-
const validTasks = [];
|
|
3557
|
-
tasks.forEach((raw, itemIndex) => {
|
|
3558
|
-
const valid = validateTask(raw);
|
|
3559
|
-
if (valid) validTasks.push(valid);
|
|
3560
|
-
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
3561
|
-
});
|
|
3562
|
-
const now = Date.now();
|
|
3563
|
-
const insertedFacts = [];
|
|
3564
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
3565
|
-
let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
3566
|
-
if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
|
|
3567
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
3568
|
-
}
|
|
3569
|
-
const titleIndex = /* @__PURE__ */ new Map();
|
|
3570
|
-
for (const existing of currentFactsRows) {
|
|
3571
|
-
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
3572
|
-
id: existing.id,
|
|
3573
|
-
okf_type: existing.okf_type ?? null
|
|
3574
|
-
});
|
|
3575
|
-
}
|
|
3576
|
-
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
3577
|
-
const pendingEdges = [];
|
|
3578
|
-
for (const [k, fact] of validFacts.entries()) {
|
|
3579
|
-
const newTokens = titleTokens(fact.title);
|
|
3580
|
-
let skip = false;
|
|
3581
|
-
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3582
|
-
for (const existing of factsForDedupe) {
|
|
3583
|
-
if (existing.source_type !== "librarian_inferred") continue;
|
|
3584
|
-
const existingTokens = titleTokens(existing.title);
|
|
3585
|
-
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
3586
|
-
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
3587
|
-
skip = true;
|
|
3588
|
-
break;
|
|
3589
|
-
}
|
|
3590
|
-
}
|
|
3591
|
-
}
|
|
3592
|
-
}
|
|
3593
|
-
if (skip) {
|
|
3594
|
-
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
3595
|
-
continue;
|
|
3596
|
-
}
|
|
3597
|
-
const ontologyFact = fact;
|
|
3598
|
-
const id = generateId("fact_");
|
|
3599
|
-
const validationDrops = [];
|
|
3600
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
3601
|
-
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
3602
|
-
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
3603
|
-
if (grounding?.diagnostic) {
|
|
3604
|
-
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
3605
|
-
}
|
|
3606
|
-
const factObj = {
|
|
3607
|
-
id,
|
|
3608
|
-
entity_id: entityId,
|
|
3609
|
-
title: fact.title,
|
|
3610
|
-
body: fact.body,
|
|
3611
|
-
tags: fact.tags,
|
|
3612
|
-
confidence: fact.confidence,
|
|
3613
|
-
source_type: "librarian_inferred",
|
|
3614
|
-
source_hash: null,
|
|
3615
|
-
source_ref: null,
|
|
3616
|
-
created_at: now,
|
|
3617
|
-
updated_at: now,
|
|
3618
|
-
last_accessed_at: null,
|
|
3619
|
-
access_count: 0,
|
|
3620
|
-
deleted_at: null,
|
|
3621
|
-
okf_type: normalized.okf_type,
|
|
3622
|
-
...grounding?.trust
|
|
3623
|
-
};
|
|
3624
|
-
await this.entryRepo.upsert(factObj, tx);
|
|
3625
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3626
|
-
factsForDedupe.push(factObj);
|
|
3627
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3628
|
-
if (normalized.edges.length > 0) {
|
|
3629
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3630
|
-
}
|
|
3631
|
-
}
|
|
3632
|
-
for (const item of pendingEdges) {
|
|
3633
|
-
const resolveDrops = [];
|
|
3634
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
3635
|
-
entityId,
|
|
3636
|
-
item.sourceId,
|
|
3637
|
-
item.sourceType,
|
|
3638
|
-
item.edges ?? [],
|
|
3639
|
-
manifest,
|
|
3640
|
-
titleIndex,
|
|
3641
|
-
tx,
|
|
3642
|
-
now,
|
|
3643
|
-
resolveDrops
|
|
3644
|
-
);
|
|
3645
|
-
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
3646
|
-
}
|
|
3647
|
-
for (const task of validTasks) {
|
|
3648
|
-
const id = generateId("task_");
|
|
3649
|
-
const taskObj = {
|
|
3650
|
-
id,
|
|
3651
|
-
entity_id: entityId,
|
|
3652
|
-
description: task.description,
|
|
3653
|
-
status: "pending",
|
|
3654
|
-
priority: task.priority,
|
|
3655
|
-
created_at: now,
|
|
3656
|
-
updated_at: now,
|
|
3657
|
-
resolved_at: null,
|
|
3658
|
-
deleted_at: null
|
|
3659
|
-
};
|
|
3660
|
-
await this.taskRepo.upsert(taskObj, tx);
|
|
3661
|
-
}
|
|
3662
|
-
});
|
|
3663
|
-
diagBuffer.flush(this.options);
|
|
3664
|
-
await this.searchService.sync(entityId);
|
|
3665
|
-
for (const fact of insertedFacts) {
|
|
3666
|
-
await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
4472
|
+
await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
|
|
4473
|
+
}
|
|
4474
|
+
/**
|
|
4475
|
+
* Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
|
|
4476
|
+
* §5.1). Builds the strategy deps from `this` and dispatches via
|
|
4477
|
+
* `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
|
|
4478
|
+
* class. Returns the strategy's `LibrarianResult` so deferred/budgeted
|
|
4479
|
+
* schedulers (PR-C) can advance watermarks; the existing void-returning
|
|
4480
|
+
* `doRunLibrarian` discards it.
|
|
4481
|
+
*
|
|
4482
|
+
* Spec §10.3: one place advances the watermark, for every strategy, only
|
|
4483
|
+
* after the pass resolved (its transaction committed) and never backwards.
|
|
4484
|
+
* A budget-stopped pass wrote nothing and must not move the watermark.
|
|
4485
|
+
*/
|
|
4486
|
+
async runLibrarianPass(ctx) {
|
|
4487
|
+
const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
|
|
4488
|
+
if (result.processedThrough && !result.budgetStop) {
|
|
4489
|
+
await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
|
|
3667
4490
|
}
|
|
3668
|
-
|
|
4491
|
+
return result;
|
|
4492
|
+
}
|
|
4493
|
+
/**
|
|
4494
|
+
* Strategy deps shared between legacy and ops librarians (spec §5.1). The
|
|
4495
|
+
* ops strategy (PR-B Task 8) reads/anchors the watermark through these two
|
|
4496
|
+
* hooks so the dispatcher itself never reaches into the metadata table —
|
|
4497
|
+
* the watermark invariant (one place advances, never backwards) lives here.
|
|
4498
|
+
*/
|
|
4499
|
+
librarianDeps() {
|
|
4500
|
+
return {
|
|
4501
|
+
db: this.db,
|
|
4502
|
+
options: this.options,
|
|
4503
|
+
entryRepo: this.entryRepo,
|
|
4504
|
+
taskRepo: this.taskRepo,
|
|
4505
|
+
eventRepo: this.eventRepo,
|
|
4506
|
+
searchService: this.searchService,
|
|
4507
|
+
embeddingService: this.embeddingService,
|
|
4508
|
+
promptService: this.promptService,
|
|
4509
|
+
ontologyService: this.ontologyService,
|
|
4510
|
+
seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
|
|
4511
|
+
getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
|
|
4512
|
+
};
|
|
4513
|
+
}
|
|
4514
|
+
/**
|
|
4515
|
+
* Persist `next` as the entity's librarian watermark when it is strictly
|
|
4516
|
+
* newer than the stored value, under the (created_at, id) ordering used by
|
|
4517
|
+
* the op-based librarian (spec §10.3). Never moves backwards.
|
|
4518
|
+
*/
|
|
4519
|
+
async advanceLibrarianWatermark(entityId, next) {
|
|
4520
|
+
const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
4521
|
+
const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
|
|
4522
|
+
if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
|
|
4523
|
+
}
|
|
4524
|
+
/**
|
|
4525
|
+
* Spec §10.3: first ops pass on a legacy DB starts after what the count
|
|
4526
|
+
* checkpoint already covered. No-op when a watermark already exists or the
|
|
4527
|
+
* memory checkpoint is 0; otherwise anchor the watermark to the event at
|
|
4528
|
+
* offset `min(memory_checkpoint, totalEvents) - 1`.
|
|
4529
|
+
*/
|
|
4530
|
+
async seedLibrarianWatermark(entityId) {
|
|
4531
|
+
if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
|
|
4532
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
4533
|
+
const memory = cp.memory ?? 0;
|
|
4534
|
+
if (memory <= 0) return;
|
|
4535
|
+
const total = await this.eventRepo.count(entityId);
|
|
4536
|
+
const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
|
|
4537
|
+
if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
|
|
3669
4538
|
}
|
|
3670
4539
|
/**
|
|
3671
4540
|
* Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
|
|
@@ -3719,7 +4588,7 @@ var MaintenanceService = class {
|
|
|
3719
4588
|
}
|
|
3720
4589
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
3721
4590
|
if (healCandidates.length === 0) {
|
|
3722
|
-
await this.searchService.
|
|
4591
|
+
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
3723
4592
|
this.searchService.evictCache(entityId);
|
|
3724
4593
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
3725
4594
|
return {
|
|
@@ -3771,7 +4640,7 @@ var MaintenanceService = class {
|
|
|
3771
4640
|
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
3772
4641
|
return prompts;
|
|
3773
4642
|
},
|
|
3774
|
-
call: (prompts) => this.options
|
|
4643
|
+
call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
|
|
3775
4644
|
parse: (responseText, batch, prompts) => {
|
|
3776
4645
|
const result = parseJsonResponse(responseText);
|
|
3777
4646
|
return {
|
|
@@ -3875,9 +4744,10 @@ var MaintenanceService = class {
|
|
|
3875
4744
|
const callErrorIds = new Set(
|
|
3876
4745
|
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
3877
4746
|
);
|
|
4747
|
+
const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
|
|
3878
4748
|
await this.entryRepo.markHealChecked(
|
|
3879
4749
|
[
|
|
3880
|
-
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
4750
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
|
|
3881
4751
|
...insertedFacts.map((f) => f.id)
|
|
3882
4752
|
],
|
|
3883
4753
|
entityId,
|
|
@@ -3889,7 +4759,14 @@ var MaintenanceService = class {
|
|
|
3889
4759
|
diagBuffer.push({ ...diagBase, code: "heal_skipped", detail: { factId: item.id, reason } });
|
|
3890
4760
|
}
|
|
3891
4761
|
diagBuffer.flush(this.options);
|
|
3892
|
-
|
|
4762
|
+
const healSyncIds = Array.from(/* @__PURE__ */ new Set([
|
|
4763
|
+
...orphanedIds,
|
|
4764
|
+
...staleDowngradedIds,
|
|
4765
|
+
...safeDowngraded,
|
|
4766
|
+
...uniqueDeletedFactIds,
|
|
4767
|
+
...insertedFacts.map((f) => f.id)
|
|
4768
|
+
]));
|
|
4769
|
+
await this.searchService.syncEntries(entityId, healSyncIds);
|
|
3893
4770
|
for (const factId of uniqueDeletedFactIds) {
|
|
3894
4771
|
try {
|
|
3895
4772
|
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
@@ -3922,7 +4799,8 @@ var MaintenanceService = class {
|
|
|
3922
4799
|
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
3923
4800
|
degraded: healedDegraded,
|
|
3924
4801
|
remaining: counts.eligible,
|
|
3925
|
-
deferred: counts.deferred
|
|
4802
|
+
deferred: counts.deferred,
|
|
4803
|
+
...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
|
|
3926
4804
|
};
|
|
3927
4805
|
}
|
|
3928
4806
|
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
@@ -3966,7 +4844,7 @@ var MaintenanceService = class {
|
|
|
3966
4844
|
const outcome = await runBatched({
|
|
3967
4845
|
items: candidates,
|
|
3968
4846
|
buildPrompt,
|
|
3969
|
-
call: (prompts) => this.options
|
|
4847
|
+
call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
|
|
3970
4848
|
parse: (responseText, batch) => {
|
|
3971
4849
|
const parsed = parseJsonResponse(responseText);
|
|
3972
4850
|
return {
|
|
@@ -4497,6 +5375,12 @@ var ImportExportService = class {
|
|
|
4497
5375
|
})()
|
|
4498
5376
|
};
|
|
4499
5377
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
5378
|
+
await this.entryRepo.setTemporal(fact.id, entityId, {
|
|
5379
|
+
valid_from: fact.valid_from ?? null,
|
|
5380
|
+
valid_to: fact.valid_to ?? null,
|
|
5381
|
+
superseded_by: fact.superseded_by ?? null,
|
|
5382
|
+
superseded_at: fact.superseded_at ?? null
|
|
5383
|
+
}, tx, { outbox: false });
|
|
4500
5384
|
if (blobData != null) {
|
|
4501
5385
|
factsWithPreservedBlob.set(fact.id, blobData);
|
|
4502
5386
|
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
@@ -4575,7 +5459,8 @@ var ImportExportService = class {
|
|
|
4575
5459
|
event_type: event.event_type,
|
|
4576
5460
|
summary: event.summary,
|
|
4577
5461
|
related_entry_id: event.related_entry_id ?? null,
|
|
4578
|
-
created_at: event.created_at
|
|
5462
|
+
created_at: event.created_at,
|
|
5463
|
+
occurred_at: event.occurred_at ?? null
|
|
4579
5464
|
},
|
|
4580
5465
|
tx
|
|
4581
5466
|
);
|
|
@@ -4822,6 +5707,21 @@ var EmbeddingService = class {
|
|
|
4822
5707
|
}
|
|
4823
5708
|
}
|
|
4824
5709
|
async tryEmbedFact(fact, ctx) {
|
|
5710
|
+
const r = await this.embedTextForFact(fact, ctx);
|
|
5711
|
+
return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
|
|
5712
|
+
}
|
|
5713
|
+
/**
|
|
5714
|
+
* Compute half: build the embed text, call embed(), validate the result,
|
|
5715
|
+
* mark any embed-time failure, and return the validated Float32 vector.
|
|
5716
|
+
*
|
|
5717
|
+
* Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
|
|
5718
|
+
* `provider_error`. All embed-time failures are marked via `markFailure`;
|
|
5719
|
+
* `no_provider` deliberately does not mark (spec §2.4).
|
|
5720
|
+
*
|
|
5721
|
+
* Storage errors are NOT possible here — that is `storeFactVector`'s domain.
|
|
5722
|
+
*/
|
|
5723
|
+
async embedTextForFact(fact, ctx, opts) {
|
|
5724
|
+
const markFailures = opts?.markFailures !== false;
|
|
4825
5725
|
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
4826
5726
|
let tagsStr;
|
|
4827
5727
|
if (Array.isArray(fact.tags)) {
|
|
@@ -4842,8 +5742,8 @@ var EmbeddingService = class {
|
|
|
4842
5742
|
const vector = await this.options.llmProvider.embed(text);
|
|
4843
5743
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
4844
5744
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
4845
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
4846
|
-
await this.markFailure(fact.id, "invalid_vector");
|
|
5745
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
5746
|
+
if (markFailures) await this.markFailure(fact.id, "invalid_vector");
|
|
4847
5747
|
return { ok: false, kind: "invalid_vector" };
|
|
4848
5748
|
}
|
|
4849
5749
|
float32Vector = new Float32Array(vector);
|
|
@@ -4856,16 +5756,24 @@ var EmbeddingService = class {
|
|
|
4856
5756
|
}
|
|
4857
5757
|
if (hasNonFinite) {
|
|
4858
5758
|
console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
|
|
4859
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
4860
|
-
await this.markFailure(fact.id, "float32_overflow");
|
|
5759
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
5760
|
+
if (markFailures) await this.markFailure(fact.id, "float32_overflow");
|
|
4861
5761
|
return { ok: false, kind: "float32_overflow" };
|
|
4862
5762
|
}
|
|
4863
5763
|
} catch (err) {
|
|
4864
5764
|
console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
|
|
4865
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
4866
|
-
await this.markFailure(fact.id, "provider_error");
|
|
5765
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
5766
|
+
if (markFailures) await this.markFailure(fact.id, "provider_error");
|
|
4867
5767
|
return { ok: false, kind: "provider_error" };
|
|
4868
5768
|
}
|
|
5769
|
+
return { ok: true, vector: float32Vector };
|
|
5770
|
+
}
|
|
5771
|
+
/**
|
|
5772
|
+
* Storage half: persist the computed vector to the entry row, then fire the
|
|
5773
|
+
* `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
|
|
5774
|
+
* failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
|
|
5775
|
+
*/
|
|
5776
|
+
async storeFactVector(fact, float32Vector, ctx) {
|
|
4869
5777
|
try {
|
|
4870
5778
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
4871
5779
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
@@ -5050,6 +5958,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
5050
5958
|
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
5051
5959
|
}
|
|
5052
5960
|
|
|
5961
|
+
// src/utils/budget.ts
|
|
5962
|
+
function factTokenCost(f) {
|
|
5963
|
+
return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
|
|
5964
|
+
}
|
|
5965
|
+
function packFactsByBudget(items, budget, scores) {
|
|
5966
|
+
const candidates = items.length;
|
|
5967
|
+
if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
|
|
5968
|
+
let top = items[0];
|
|
5969
|
+
let topCost = factTokenCost(top);
|
|
5970
|
+
if (topCost > budget) {
|
|
5971
|
+
const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
|
|
5972
|
+
top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
|
|
5973
|
+
topCost = factTokenCost(top);
|
|
5974
|
+
}
|
|
5975
|
+
const kept = [{ rank: 0, item: top }];
|
|
5976
|
+
let used = topCost;
|
|
5977
|
+
const rest = items.slice(1).map((item, i) => {
|
|
5978
|
+
const rank = i + 1;
|
|
5979
|
+
const cost = factTokenCost(item);
|
|
5980
|
+
const s = scores?.get(item.id);
|
|
5981
|
+
const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
|
|
5982
|
+
return { rank, item, cost, density: value / Math.max(1, cost) };
|
|
5983
|
+
});
|
|
5984
|
+
rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
|
|
5985
|
+
for (const r of rest) {
|
|
5986
|
+
if (used + r.cost <= budget) {
|
|
5987
|
+
kept.push({ rank: r.rank, item: r.item });
|
|
5988
|
+
used += r.cost;
|
|
5989
|
+
}
|
|
5990
|
+
}
|
|
5991
|
+
kept.sort((a, b) => a.rank - b.rank);
|
|
5992
|
+
return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
|
|
5993
|
+
}
|
|
5994
|
+
|
|
5053
5995
|
// src/services/RetrievalService.ts
|
|
5054
5996
|
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
5055
5997
|
var RetrievalService = class {
|
|
@@ -5085,6 +6027,28 @@ var RetrievalService = class {
|
|
|
5085
6027
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
5086
6028
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
5087
6029
|
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
6030
|
+
const rawAsOf = options?.asOf;
|
|
6031
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
6032
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
6033
|
+
}
|
|
6034
|
+
const tokenBudget = options?.tokenBudget;
|
|
6035
|
+
if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
|
|
6036
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
6037
|
+
}
|
|
6038
|
+
const applyBudget = (input) => {
|
|
6039
|
+
if (tokenBudget === void 0) return input;
|
|
6040
|
+
const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
|
|
6041
|
+
emitDiagnostic(this.options, {
|
|
6042
|
+
code: "read_budget",
|
|
6043
|
+
operation: "read",
|
|
6044
|
+
trigger: "call",
|
|
6045
|
+
entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
|
|
6046
|
+
detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
|
|
6047
|
+
});
|
|
6048
|
+
return packed.items;
|
|
6049
|
+
};
|
|
6050
|
+
const liveMode = rawAsOf === void 0 ? "current" : "asOf";
|
|
6051
|
+
const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
|
|
5088
6052
|
const trimmedQuery = query.trim();
|
|
5089
6053
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
5090
6054
|
entityIds,
|
|
@@ -5104,8 +6068,10 @@ var RetrievalService = class {
|
|
|
5104
6068
|
let usedEmbed = false;
|
|
5105
6069
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
5106
6070
|
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
5107
|
-
const
|
|
5108
|
-
const
|
|
6071
|
+
const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
|
|
6072
|
+
const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
|
|
6073
|
+
const excludedPad = excludedIds.size;
|
|
6074
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
|
|
5109
6075
|
if (scoredEntityIds.length === 0) {
|
|
5110
6076
|
usedEmbed = true;
|
|
5111
6077
|
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
@@ -5139,7 +6105,7 @@ var RetrievalService = class {
|
|
|
5139
6105
|
let miniSearchScores;
|
|
5140
6106
|
if (effectivePreFilterLimit !== void 0) {
|
|
5141
6107
|
populateCache = false;
|
|
5142
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !
|
|
6108
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
|
|
5143
6109
|
if (preResults.length === 0) {
|
|
5144
6110
|
candidateRows = null;
|
|
5145
6111
|
} else {
|
|
@@ -5161,9 +6127,9 @@ var RetrievalService = class {
|
|
|
5161
6127
|
}
|
|
5162
6128
|
} else {
|
|
5163
6129
|
if (useRanker) {
|
|
5164
|
-
candidateRows = this.
|
|
6130
|
+
candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
|
|
5165
6131
|
} else {
|
|
5166
|
-
candidateRows = this.
|
|
6132
|
+
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
5167
6133
|
}
|
|
5168
6134
|
if (weight !== void 0 && weight < 1) {
|
|
5169
6135
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -5448,7 +6414,7 @@ var RetrievalService = class {
|
|
|
5448
6414
|
// read() re-sorts after applying tier weights
|
|
5449
6415
|
});
|
|
5450
6416
|
}
|
|
5451
|
-
if (
|
|
6417
|
+
if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
|
|
5452
6418
|
if (scored.length > 0) {
|
|
5453
6419
|
scored = scored.map((row) => ({
|
|
5454
6420
|
...row,
|
|
@@ -5512,7 +6478,7 @@ var RetrievalService = class {
|
|
|
5512
6478
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
5513
6479
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
5514
6480
|
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
5515
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !
|
|
6481
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
|
|
5516
6482
|
const candidates = results.map((r) => ({
|
|
5517
6483
|
id: r.id,
|
|
5518
6484
|
entity_id: r.entity_id,
|
|
@@ -5530,13 +6496,18 @@ var RetrievalService = class {
|
|
|
5530
6496
|
}
|
|
5531
6497
|
}
|
|
5532
6498
|
}
|
|
6499
|
+
facts = applyBudget(facts);
|
|
5533
6500
|
if (facts.length > 0) {
|
|
5534
6501
|
const ids = facts.map((f) => f.id);
|
|
5535
6502
|
const now = Date.now();
|
|
5536
6503
|
await this.entryRepo.trackAccess(ids, now);
|
|
5537
6504
|
}
|
|
5538
6505
|
} else {
|
|
5539
|
-
facts =
|
|
6506
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
|
|
6507
|
+
...excludeDrafts ? { excludeDrafts: true } : {},
|
|
6508
|
+
live: { mode: liveMode, t: liveAt }
|
|
6509
|
+
});
|
|
6510
|
+
facts = applyBudget(facts);
|
|
5540
6511
|
}
|
|
5541
6512
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
5542
6513
|
const [tasks, events] = await Promise.all([
|
|
@@ -5572,8 +6543,8 @@ var RetrievalService = class {
|
|
|
5572
6543
|
_tieBreakSort(items) {
|
|
5573
6544
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
5574
6545
|
}
|
|
5575
|
-
|
|
5576
|
-
return
|
|
6546
|
+
_withoutIds(rows, excludeIds) {
|
|
6547
|
+
return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
|
|
5577
6548
|
}
|
|
5578
6549
|
/**
|
|
5579
6550
|
* Comparator for score + deterministic tie-break fields.
|
|
@@ -5655,6 +6626,18 @@ var WriteService = class {
|
|
|
5655
6626
|
this.metadataRepo = metadataRepo;
|
|
5656
6627
|
this.jobManager = jobManager;
|
|
5657
6628
|
this.maintenanceService = maintenanceService;
|
|
6629
|
+
this.inflight = /* @__PURE__ */ new Set();
|
|
6630
|
+
}
|
|
6631
|
+
track(p) {
|
|
6632
|
+
this.inflight.add(p);
|
|
6633
|
+
p.finally(() => this.inflight.delete(p)).catch(() => {
|
|
6634
|
+
});
|
|
6635
|
+
}
|
|
6636
|
+
/** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
|
|
6637
|
+
async drain() {
|
|
6638
|
+
while (this.inflight.size > 0) {
|
|
6639
|
+
await Promise.allSettled([...this.inflight]);
|
|
6640
|
+
}
|
|
5658
6641
|
}
|
|
5659
6642
|
async write(entityId, event) {
|
|
5660
6643
|
if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
|
|
@@ -5681,6 +6664,7 @@ var WriteService = class {
|
|
|
5681
6664
|
}
|
|
5682
6665
|
const id = generateId("evt_");
|
|
5683
6666
|
const now = Date.now();
|
|
6667
|
+
const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
|
|
5684
6668
|
let eventType = event.event_type;
|
|
5685
6669
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
5686
6670
|
eventType = "observation";
|
|
@@ -5691,8 +6675,14 @@ var WriteService = class {
|
|
|
5691
6675
|
event_type: eventType,
|
|
5692
6676
|
summary,
|
|
5693
6677
|
related_entry_id: relatedEntryId,
|
|
5694
|
-
created_at: now
|
|
6678
|
+
created_at: now,
|
|
6679
|
+
occurred_at: occurredAt
|
|
5695
6680
|
};
|
|
6681
|
+
const deferred = this.options.config?.maintenance === "deferred";
|
|
6682
|
+
if (deferred) {
|
|
6683
|
+
await this.eventRepo.add(newEvent);
|
|
6684
|
+
return;
|
|
6685
|
+
}
|
|
5696
6686
|
let shouldRunLibrarian = false;
|
|
5697
6687
|
let librarianCount = 0;
|
|
5698
6688
|
let prevMemoryCheckpoint = 0;
|
|
@@ -5714,27 +6704,41 @@ var WriteService = class {
|
|
|
5714
6704
|
prevMemoryCheckpoint = memoryCheckpoint;
|
|
5715
6705
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
5716
6706
|
}
|
|
6707
|
+
} else {
|
|
6708
|
+
const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
|
|
6709
|
+
if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
|
|
6710
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
|
|
6711
|
+
const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
|
|
6712
|
+
if (Math.ceil(chars / 4) >= tokenThreshold) {
|
|
6713
|
+
shouldRunLibrarian = true;
|
|
6714
|
+
librarianCount = count;
|
|
6715
|
+
prevMemoryCheckpoint = memoryCheckpoint;
|
|
6716
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
6717
|
+
}
|
|
6718
|
+
}
|
|
5717
6719
|
}
|
|
5718
6720
|
});
|
|
5719
6721
|
if (shouldRunLibrarian) {
|
|
5720
6722
|
try {
|
|
5721
6723
|
this.jobManager.acquireLock("librarian", entityId);
|
|
5722
6724
|
const stage = { job: "librarian" };
|
|
5723
|
-
this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6725
|
+
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
5724
6726
|
console.error(err);
|
|
5725
6727
|
this.reportBackgroundFailure(entityId, stage.job);
|
|
5726
6728
|
}).finally(() => {
|
|
5727
6729
|
this.jobManager.releaseLock("librarian", entityId);
|
|
5728
6730
|
});
|
|
6731
|
+
this.track(job);
|
|
5729
6732
|
} catch (e) {
|
|
5730
6733
|
if (!(e instanceof WikiBusyError)) throw e;
|
|
5731
6734
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
|
|
5732
6735
|
}
|
|
5733
6736
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
5734
|
-
this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6737
|
+
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
5735
6738
|
console.error(err);
|
|
5736
6739
|
this.reportBackgroundFailure(entityId, "heal");
|
|
5737
6740
|
});
|
|
6741
|
+
this.track(healJob);
|
|
5738
6742
|
}
|
|
5739
6743
|
}
|
|
5740
6744
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {
|
|
@@ -5781,6 +6785,6 @@ var WriteService = class {
|
|
|
5781
6785
|
}
|
|
5782
6786
|
};
|
|
5783
6787
|
|
|
5784
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
5785
|
-
//# sourceMappingURL=chunk-
|
|
5786
|
-
//# sourceMappingURL=chunk-
|
|
6788
|
+
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
6789
|
+
//# sourceMappingURL=chunk-M76CF2Q6.mjs.map
|
|
6790
|
+
//# sourceMappingURL=chunk-M76CF2Q6.mjs.map
|