@equationalapplications/core-llm-wiki 7.7.7 → 7.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/testing.js CHANGED
@@ -153,6 +153,15 @@ var WikiInvalidReadOptions = class _WikiInvalidReadOptions extends Error {
153
153
  Object.setPrototypeOf(this, _WikiInvalidReadOptions.prototype);
154
154
  }
155
155
  };
156
+ var WikiSupersedeError = class _WikiSupersedeError extends Error {
157
+ constructor(reason) {
158
+ super(`supersede() rejected: ${reason}`);
159
+ this.code = "WIKI_SUPERSEDE_REJECTED";
160
+ this.name = "WikiSupersedeError";
161
+ this.reason = reason;
162
+ Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
163
+ }
164
+ };
156
165
 
157
166
  // src/utils/pure.ts
158
167
  var MAX_REPAIR_CANDIDATES = 5;
@@ -686,7 +695,14 @@ var SEVERITY = {
686
695
  grounding_missing: "warn",
687
696
  grounding_failed: "warn",
688
697
  classification_low_confidence: "info",
689
- classification_invalid: "warn"
698
+ classification_invalid: "warn",
699
+ llm_usage: "info",
700
+ librarian_gate: "info",
701
+ librarian_op_rejected: "warn",
702
+ contradicts_document: "warn",
703
+ resolve_failed: "warn",
704
+ read_budget: "info",
705
+ event_retention_held: "info"
690
706
  };
691
707
  var MESSAGE = {
692
708
  ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
@@ -701,7 +717,14 @@ var MESSAGE = {
701
717
  grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
702
718
  grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
703
719
  classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
704
- classification_invalid: "A classifier answer was invalid and was not applied."
720
+ classification_invalid: "A classifier answer was invalid and was not applied.",
721
+ llm_usage: "An LLM call completed; token usage attached.",
722
+ librarian_gate: "Librarian gate outcome counts for one pass.",
723
+ librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
724
+ contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
725
+ resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
726
+ read_budget: "A read was packed to a token budget.",
727
+ event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
705
728
  };
706
729
  var warnedNonFunction = /* @__PURE__ */ new WeakSet();
707
730
  function copyDetail(detail) {
@@ -851,6 +874,21 @@ var EmbeddingService = class {
851
874
  }
852
875
  }
853
876
  async tryEmbedFact(fact, ctx) {
877
+ const r = await this.embedTextForFact(fact, ctx);
878
+ return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
879
+ }
880
+ /**
881
+ * Compute half: build the embed text, call embed(), validate the result,
882
+ * mark any embed-time failure, and return the validated Float32 vector.
883
+ *
884
+ * Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
885
+ * `provider_error`. All embed-time failures are marked via `markFailure`;
886
+ * `no_provider` deliberately does not mark (spec §2.4).
887
+ *
888
+ * Storage errors are NOT possible here — that is `storeFactVector`'s domain.
889
+ */
890
+ async embedTextForFact(fact, ctx, opts) {
891
+ const markFailures = opts?.markFailures !== false;
854
892
  if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
855
893
  let tagsStr;
856
894
  if (Array.isArray(fact.tags)) {
@@ -871,8 +909,8 @@ var EmbeddingService = class {
871
909
  const vector = await this.options.llmProvider.embed(text);
872
910
  if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
873
911
  console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
874
- this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
875
- await this.markFailure(fact.id, "invalid_vector");
912
+ if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
913
+ if (markFailures) await this.markFailure(fact.id, "invalid_vector");
876
914
  return { ok: false, kind: "invalid_vector" };
877
915
  }
878
916
  float32Vector = new Float32Array(vector);
@@ -885,16 +923,24 @@ var EmbeddingService = class {
885
923
  }
886
924
  if (hasNonFinite) {
887
925
  console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
888
- this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
889
- await this.markFailure(fact.id, "float32_overflow");
926
+ if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
927
+ if (markFailures) await this.markFailure(fact.id, "float32_overflow");
890
928
  return { ok: false, kind: "float32_overflow" };
891
929
  }
892
930
  } catch (err) {
893
931
  console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
894
- this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
895
- await this.markFailure(fact.id, "provider_error");
932
+ if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
933
+ if (markFailures) await this.markFailure(fact.id, "provider_error");
896
934
  return { ok: false, kind: "provider_error" };
897
935
  }
936
+ return { ok: true, vector: float32Vector };
937
+ }
938
+ /**
939
+ * Storage half: persist the computed vector to the entry row, then fire the
940
+ * `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
941
+ * failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
942
+ */
943
+ async storeFactVector(fact, float32Vector, ctx) {
898
944
  try {
899
945
  await this.storeEmbeddingDimension(float32Vector.length);
900
946
  const blob = new Uint8Array(float32Vector.buffer);
@@ -1246,6 +1292,12 @@ var ImportExportService = class {
1246
1292
  })()
1247
1293
  };
1248
1294
  await this.entryRepo.upsertForImport(factObj, tx);
1295
+ await this.entryRepo.setTemporal(fact.id, entityId, {
1296
+ valid_from: fact.valid_from ?? null,
1297
+ valid_to: fact.valid_to ?? null,
1298
+ superseded_by: fact.superseded_by ?? null,
1299
+ superseded_at: fact.superseded_at ?? null
1300
+ }, tx, { outbox: false });
1249
1301
  if (blobData != null) {
1250
1302
  factsWithPreservedBlob.set(fact.id, blobData);
1251
1303
  if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
@@ -1324,7 +1376,8 @@ var ImportExportService = class {
1324
1376
  event_type: event.event_type,
1325
1377
  summary: event.summary,
1326
1378
  related_entry_id: event.related_entry_id ?? null,
1327
- created_at: event.created_at
1379
+ created_at: event.created_at,
1380
+ occurred_at: event.occurred_at ?? null
1328
1381
  },
1329
1382
  tx
1330
1383
  );
@@ -1557,6 +1610,21 @@ function groundingEvidenceBlock(writer, cfg) {
1557
1610
  return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
1558
1611
  "evidence": ["exact substring copied from the SOURCE section"]`;
1559
1612
  }
1613
+ var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
1614
+ Return ONLY a valid JSON object matching this schema:
1615
+ {
1616
+ "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
1617
+ "tasks": [{ "description": "string", "priority": "number (0-10)" }]
1618
+ }
1619
+ One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
1620
+ var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
1621
+ - "ADD": the candidate is new information; the existing facts are about something else.
1622
+ - "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
1623
+ - "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
1624
+ - "NOOP": an existing fact already states the candidate.
1625
+ Return ONLY a valid JSON object matching this schema:
1626
+ { "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
1627
+ "target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
1560
1628
 
1561
1629
  // src/utils/healConstants.ts
1562
1630
  var HEAL_MAX_ANCHORS = 50;
@@ -1795,6 +1863,22 @@ ${JSON.stringify(facts, null, 2)}`
1795
1863
  ${JSON.stringify(facts, null, 2)}`
1796
1864
  };
1797
1865
  }
1866
+ /** Ops extract (spec §5.2): events only — no existing facts. */
1867
+ buildOpsExtractPrompt(events, ontologyContext) {
1868
+ const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
1869
+ const grounding = this.groundingFor("librarian");
1870
+ return {
1871
+ systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
1872
+ userPrompt: `Events:
1873
+ ${lines.join("\n")}`,
1874
+ ...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
1875
+ };
1876
+ }
1877
+ /** Ops resolve (spec §5.4): only each item's own neighbours. */
1878
+ buildOpsResolvePrompt(items) {
1879
+ return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
1880
+ ${JSON.stringify({ items })}` };
1881
+ }
1798
1882
  /**
1799
1883
  * The system prompt each writer would send, without hydrating data (spec
1800
1884
  * §8.3). `buildSystemPrompt` with no variables hydrates only ontology
@@ -1859,6 +1943,59 @@ function toGroundingAnchor(anchor) {
1859
1943
  var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
1860
1944
  var DEFAULT_CHUNK_OVERLAP = 400;
1861
1945
 
1946
+ // src/utils/usage.ts
1947
+ function estimateTokens(text) {
1948
+ return Math.ceil(text.length / 4);
1949
+ }
1950
+ var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
1951
+ constructor(requiredEstimate, remaining) {
1952
+ super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
1953
+ this.name = "WikiBudgetExhausted";
1954
+ this.requiredEstimate = requiredEstimate;
1955
+ this.remaining = remaining;
1956
+ Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
1957
+ }
1958
+ };
1959
+
1960
+ // src/utils/llmCall.ts
1961
+ function validUsage(u) {
1962
+ if (!u || typeof u !== "object") return false;
1963
+ const { inputTokens, outputTokens } = u;
1964
+ return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
1965
+ }
1966
+ async function callLlm(options, ctx, prompts) {
1967
+ const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
1968
+ if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
1969
+ throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
1970
+ }
1971
+ const provider = options.llmProvider;
1972
+ const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
1973
+ let text;
1974
+ let reported;
1975
+ if (typeof provider.generateTextWithUsage === "function") {
1976
+ const result = await provider.generateTextWithUsage(params);
1977
+ if (!result || typeof result.text !== "string") {
1978
+ throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
1979
+ }
1980
+ text = result.text;
1981
+ reported = validUsage(result.usage) ? result.usage : void 0;
1982
+ } else {
1983
+ text = await provider.generateText(params);
1984
+ }
1985
+ const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
1986
+ ctx.meter?.record(record);
1987
+ if (options.config?.reportLlmUsage === true) {
1988
+ emitDiagnostic(options, {
1989
+ code: "llm_usage",
1990
+ operation: ctx.operation,
1991
+ trigger: ctx.trigger,
1992
+ entityId: ctx.entityId,
1993
+ detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
1994
+ });
1995
+ }
1996
+ return text;
1997
+ }
1998
+
1862
1999
  // src/services/IngestionService.ts
1863
2000
  var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
1864
2001
  function dedupDiagnostic(entityId, sourceRef, locator) {
@@ -1935,7 +2072,7 @@ var IngestionService = class {
1935
2072
  ontologyContext
1936
2073
  );
1937
2074
  try {
1938
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
2075
+ const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
1939
2076
  const result2 = parseJsonResponse(responseText);
1940
2077
  const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
1941
2078
  const facts = [];
@@ -2619,6 +2756,8 @@ async function runBatched(args) {
2619
2756
  const results = [];
2620
2757
  const skipped = [];
2621
2758
  let batches = 0;
2759
+ const settled = /* @__PURE__ */ new Set();
2760
+ let budgetStop;
2622
2761
  let batchSize = initialBatchSize(maxOutputTokens);
2623
2762
  const trim = async (candidate) => {
2624
2763
  const whole = await buildPrompt(candidate);
@@ -2652,6 +2791,7 @@ async function runBatched(args) {
2652
2791
  } else {
2653
2792
  const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
2654
2793
  skipped.push({ item: batch[0], reason });
2794
+ settled.add(batch[0]);
2655
2795
  onSkip?.(batch[0], err);
2656
2796
  }
2657
2797
  return;
@@ -2660,7 +2800,7 @@ async function runBatched(args) {
2660
2800
  const mid = Math.ceil(batch.length / 2);
2661
2801
  if (mid < batchSize) batchSize = mid;
2662
2802
  let i = 0;
2663
- while (i < batch.length) {
2803
+ while (i < batch.length && !budgetStop) {
2664
2804
  const size = Math.min(batchSize, batch.length - i);
2665
2805
  const trimmed = await trim(batch.slice(i, i + size));
2666
2806
  await attempt(trimmed.batch, trimmed.prompts);
@@ -2668,6 +2808,7 @@ async function runBatched(args) {
2668
2808
  }
2669
2809
  };
2670
2810
  const attempt = async (batch, prebuilt, attemptLevel = 0) => {
2811
+ if (budgetStop) return;
2671
2812
  if (batch.length === 0) return;
2672
2813
  const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
2673
2814
  batches++;
@@ -2675,6 +2816,10 @@ async function runBatched(args) {
2675
2816
  try {
2676
2817
  responseText = await call(prompts);
2677
2818
  } catch (err) {
2819
+ if (err instanceof WikiBudgetExhausted) {
2820
+ budgetStop = { requiredEstimate: err.requiredEstimate };
2821
+ return;
2822
+ }
2678
2823
  await onFailure(batch, err, attemptLevel, true);
2679
2824
  return;
2680
2825
  }
@@ -2686,19 +2831,784 @@ async function runBatched(args) {
2686
2831
  return;
2687
2832
  }
2688
2833
  results.push(result);
2834
+ for (const item of batch) settled.add(item);
2689
2835
  };
2690
2836
  let index = 0;
2691
- while (index < items.length) {
2837
+ while (index < items.length && !budgetStop) {
2692
2838
  const { batch, prompts } = await trim(items.slice(index, index + batchSize));
2693
2839
  index += batch.length;
2694
2840
  await attempt(batch, prompts, 0);
2695
2841
  }
2696
- return { results, skipped, batches };
2842
+ const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
2843
+ return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
2697
2844
  }
2698
2845
 
2699
- // src/services/MaintenanceService.ts
2846
+ // src/services/librarian/constants.ts
2700
2847
  var FUZZY_THRESHOLD = 0.5;
2701
2848
  var MIN_TOKENS_TO_QUALIFY = 3;
2849
+
2850
+ // src/services/librarian/legacy.ts
2851
+ var runLegacyLibrarianPass = async (deps, ctx) => {
2852
+ const { entityId, trigger } = ctx;
2853
+ const promptOverride = ctx.promptOverride;
2854
+ const events = await deps.eventRepo.getRecent(entityId, 50);
2855
+ const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
2856
+ const currentFacts = currentFactsRows.map((f) => {
2857
+ const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
2858
+ return {
2859
+ ...rest,
2860
+ tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
2861
+ };
2862
+ });
2863
+ const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
2864
+ const promptEvents = events.reverse();
2865
+ const librarianGrounding = deps.promptService.groundingFor("librarian");
2866
+ const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
2867
+ promptEvents,
2868
+ currentFacts,
2869
+ promptOverride,
2870
+ ontologyContext
2871
+ );
2872
+ let responseText;
2873
+ try {
2874
+ responseText = await callLlm(
2875
+ deps.options,
2876
+ { operation: "librarian", entityId, trigger, meter: ctx.meter },
2877
+ { systemPrompt, userPrompt }
2878
+ );
2879
+ } catch (err) {
2880
+ if (err instanceof WikiBudgetExhausted) {
2881
+ return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
2882
+ }
2883
+ throw err;
2884
+ }
2885
+ const result = parseJsonResponse(responseText);
2886
+ const facts = Array.isArray(result.facts) ? result.facts : [];
2887
+ const tasks = Array.isArray(result.tasks) ? result.tasks : [];
2888
+ const ontologyUpdates = result.ontology_updates;
2889
+ const diagBuffer = new DiagnosticBuffer();
2890
+ const diagBase = { entityId, operation: "librarian", trigger };
2891
+ const validFacts = [];
2892
+ const validFactItemIndexes = [];
2893
+ facts.forEach((raw, itemIndex) => {
2894
+ const valid = validateFact(raw);
2895
+ if (valid) {
2896
+ validFacts.push(valid);
2897
+ validFactItemIndexes.push(itemIndex);
2898
+ } else {
2899
+ diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
2900
+ }
2901
+ });
2902
+ const validTasks = [];
2903
+ tasks.forEach((raw, itemIndex) => {
2904
+ const valid = validateTask(raw);
2905
+ if (valid) validTasks.push(valid);
2906
+ else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
2907
+ });
2908
+ const now = Date.now();
2909
+ const insertedFacts = [];
2910
+ await deps.db.withTransactionAsync(async (tx) => {
2911
+ let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
2912
+ if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
2913
+ manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
2914
+ }
2915
+ const titleIndex = /* @__PURE__ */ new Map();
2916
+ for (const existing of currentFactsRows) {
2917
+ titleIndex.set(normalizeTitleKey(existing.title), {
2918
+ id: existing.id,
2919
+ okf_type: existing.okf_type ?? null
2920
+ });
2921
+ }
2922
+ const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
2923
+ const pendingEdges = [];
2924
+ for (const [k, fact] of validFacts.entries()) {
2925
+ const newTokens = titleTokens(fact.title);
2926
+ let skip = false;
2927
+ if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
2928
+ for (const existing of factsForDedupe) {
2929
+ if (existing.source_type !== "librarian_inferred") continue;
2930
+ const existingTokens = titleTokens(existing.title);
2931
+ if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
2932
+ if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
2933
+ skip = true;
2934
+ break;
2935
+ }
2936
+ }
2937
+ }
2938
+ }
2939
+ if (skip) {
2940
+ diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
2941
+ continue;
2942
+ }
2943
+ const ontologyFact = fact;
2944
+ const id = generateId("fact_");
2945
+ const validationDrops = [];
2946
+ const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
2947
+ for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
2948
+ const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
2949
+ if (grounding?.diagnostic) {
2950
+ diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
2951
+ }
2952
+ const factObj = {
2953
+ id,
2954
+ entity_id: entityId,
2955
+ title: fact.title,
2956
+ body: fact.body,
2957
+ tags: fact.tags,
2958
+ confidence: fact.confidence,
2959
+ source_type: "librarian_inferred",
2960
+ source_hash: null,
2961
+ source_ref: null,
2962
+ created_at: now,
2963
+ updated_at: now,
2964
+ last_accessed_at: null,
2965
+ access_count: 0,
2966
+ deleted_at: null,
2967
+ okf_type: normalized.okf_type,
2968
+ ...grounding?.trust
2969
+ };
2970
+ await deps.entryRepo.upsert(factObj, tx);
2971
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
2972
+ factsForDedupe.push(factObj);
2973
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
2974
+ if (normalized.edges.length > 0) {
2975
+ pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
2976
+ }
2977
+ }
2978
+ for (const item of pendingEdges) {
2979
+ const resolveDrops = [];
2980
+ await deps.ontologyService?.resolveAndPersistEdges(
2981
+ entityId,
2982
+ item.sourceId,
2983
+ item.sourceType,
2984
+ item.edges ?? [],
2985
+ manifest,
2986
+ titleIndex,
2987
+ tx,
2988
+ now,
2989
+ resolveDrops
2990
+ );
2991
+ for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
2992
+ }
2993
+ for (const task of validTasks) {
2994
+ const id = generateId("task_");
2995
+ const taskObj = {
2996
+ id,
2997
+ entity_id: entityId,
2998
+ description: task.description,
2999
+ status: "pending",
3000
+ priority: task.priority,
3001
+ created_at: now,
3002
+ updated_at: now,
3003
+ resolved_at: null,
3004
+ deleted_at: null
3005
+ };
3006
+ await deps.taskRepo.upsert(taskObj, tx);
3007
+ }
3008
+ });
3009
+ diagBuffer.flush(deps.options);
3010
+ await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
3011
+ for (const fact of insertedFacts) {
3012
+ await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
3013
+ }
3014
+ deps.searchService.evictCache(entityId);
3015
+ const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
3016
+ const out = { processedThrough, factsWritten: insertedFacts.length };
3017
+ return out;
3018
+ };
3019
+
3020
+ // src/services/librarian/ops/text.ts
3021
+ var OPS_NEIGHBOUR_BODY_CHARS = 400;
3022
+ function normalizeFactText(title, body) {
3023
+ return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
3024
+ }
3025
+ function labelEvents(events) {
3026
+ return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
3027
+ }
3028
+ function parseValidFrom(v) {
3029
+ if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
3030
+ if (typeof v === "string") {
3031
+ const t = Date.parse(v);
3032
+ return Number.isFinite(t) && t >= 0 ? t : void 0;
3033
+ }
3034
+ return void 0;
3035
+ }
3036
+
3037
+ // src/services/librarian/ops/extract.ts
3038
+ var OPS_MAX_EVENTS_PER_PASS = 50;
3039
+ var OPS_MAX_EXTRACT_CHARS = 24e3;
3040
+ var RESOLVE_RESERVE_FACTOR = 2;
3041
+ var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
3042
+ async function selectBatch(deps, entityId, wm, meter) {
3043
+ const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
3044
+ if (events.length === 0) return { batch: [] };
3045
+ let n = 0;
3046
+ let chars = 0;
3047
+ for (const e of events) {
3048
+ if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
3049
+ chars += e.summary.length;
3050
+ n++;
3051
+ }
3052
+ let batch = labelEvents(events.slice(0, n));
3053
+ if (meter && meter.budget !== void 0) {
3054
+ const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
3055
+ for (; ; ) {
3056
+ const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
3057
+ if (meter.fits(need)) break;
3058
+ if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
3059
+ batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
3060
+ }
3061
+ }
3062
+ return { batch };
3063
+ }
3064
+ async function runExtract(deps, ctx, batch) {
3065
+ const { entityId, trigger } = ctx;
3066
+ const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
3067
+ const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
3068
+ const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
3069
+ const parsed = parseJsonResponse(text);
3070
+ const base = { entityId, operation: "librarian", trigger };
3071
+ const diagnostics = [];
3072
+ const byLabel = new Map(batch.map((l) => [l.label, l]));
3073
+ const newestAt = batch[batch.length - 1].at;
3074
+ const candidates = [];
3075
+ (Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
3076
+ const fact = validateFact(raw);
3077
+ if (!fact) {
3078
+ diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
3079
+ return;
3080
+ }
3081
+ const r = raw;
3082
+ const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
3083
+ const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
3084
+ candidates.push({ index, fact, sourceLabel, validFrom });
3085
+ });
3086
+ const tasks = [];
3087
+ (Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
3088
+ const task = validateTask(raw);
3089
+ if (task) tasks.push(task);
3090
+ else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
3091
+ });
3092
+ return {
3093
+ batch,
3094
+ candidates,
3095
+ tasks,
3096
+ ...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
3097
+ groundingCorpus: prompt.groundingCorpus ?? null,
3098
+ diagnostics
3099
+ };
3100
+ }
3101
+
3102
+ // src/utils/cosine.ts
3103
+ function cosineSimilarity(a, b) {
3104
+ let dot = 0, normA = 0, normB = 0;
3105
+ const len = Math.min(a.length, b.length);
3106
+ for (let i = 0; i < len; i++) {
3107
+ dot += a[i] * b[i];
3108
+ normA += a[i] * a[i];
3109
+ normB += b[i] * b[i];
3110
+ }
3111
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
3112
+ return denom === 0 ? 0 : dot / denom;
3113
+ }
3114
+
3115
+ // src/services/librarian/ops/gate.ts
3116
+ var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
3117
+ function resolveGateConfig(cfg) {
3118
+ const kRaw = cfg?.k;
3119
+ const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
3120
+ const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
3121
+ const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
3122
+ if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
3123
+ return { k, dupThreshold, novelThreshold };
3124
+ }
3125
+ function titlesMatch(a, b) {
3126
+ const ta = titleTokens(a);
3127
+ const tb = titleTokens(b);
3128
+ if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
3129
+ return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
3130
+ }
3131
+ function classifyCandidate(candidate, neighbours, cfg, mode) {
3132
+ const key = normalizeFactText(candidate.title, candidate.body);
3133
+ const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
3134
+ if (same) return { kind: "noop", target: same.id };
3135
+ if (neighbours.length === 0) return { kind: "add" };
3136
+ if (mode === "keyword") return { kind: "ambiguous" };
3137
+ const best = neighbours[0];
3138
+ const score = best.score ?? 0;
3139
+ if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
3140
+ if (score < cfg.novelThreshold) return { kind: "add" };
3141
+ return { kind: "ambiguous" };
3142
+ }
3143
+ async function gateCandidates(deps, ctx, candidates, cfg) {
3144
+ const { entityId, trigger } = ctx;
3145
+ const seen = /* @__PURE__ */ new Set();
3146
+ const unique = [];
3147
+ const inBatchDuplicates = [];
3148
+ for (const c of candidates) {
3149
+ const key = normalizeFactText(c.fact.title, c.fact.body);
3150
+ if (seen.has(key)) inBatchDuplicates.push(c);
3151
+ else {
3152
+ seen.add(key);
3153
+ unique.push(c);
3154
+ }
3155
+ }
3156
+ const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
3157
+ const canEmbed = typeof deps.options.llmProvider.embed === "function";
3158
+ const stored = [];
3159
+ if (canEmbed && unique.length > 0) {
3160
+ for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
3161
+ if (nonLive.has(row.id)) continue;
3162
+ const vec = parseEmbedding(row.embedding_blob, row.embedding);
3163
+ if (vec) stored.push({ id: row.id, vec });
3164
+ }
3165
+ }
3166
+ const gated = [];
3167
+ for (const c of unique) {
3168
+ let ranked = null;
3169
+ let vector = null;
3170
+ let mode = "keyword";
3171
+ if (canEmbed) {
3172
+ const r = await deps.embeddingService.embedTextForFact(
3173
+ { id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
3174
+ void 0,
3175
+ { markFailures: false }
3176
+ );
3177
+ if (r.ok) {
3178
+ vector = r.vector;
3179
+ if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
3180
+ ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
3181
+ mode = "vector";
3182
+ }
3183
+ }
3184
+ }
3185
+ if (!ranked) {
3186
+ ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
3187
+ }
3188
+ const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
3189
+ const byId = new Map(facts.map((f) => [f.id, f]));
3190
+ const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
3191
+ const f = byId.get(r.id);
3192
+ return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
3193
+ });
3194
+ gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
3195
+ }
3196
+ const count = (k) => gated.filter((g) => g.decision.kind === k).length;
3197
+ const gateDiagnostic = {
3198
+ code: "librarian_gate",
3199
+ operation: "librarian",
3200
+ trigger,
3201
+ entityId,
3202
+ detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
3203
+ };
3204
+ return { gated, inBatchDuplicates, gateDiagnostic };
3205
+ }
3206
+
3207
+ // src/services/librarian/ops/resolve.ts
3208
+ var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
3209
+ function validateOp(raw, aliases) {
3210
+ if (!raw || typeof raw !== "object") return null;
3211
+ const r = raw;
3212
+ if (r.op === "ADD") return { op: "ADD" };
3213
+ if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
3214
+ const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
3215
+ if (!targetId) return null;
3216
+ if (r.op === "NOOP") return { op: "NOOP", targetId };
3217
+ if (r.op === "SUPERSEDE") {
3218
+ const validFrom = parseValidFrom(r.valid_from);
3219
+ return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
3220
+ }
3221
+ const update = { op: "UPDATE", targetId };
3222
+ if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
3223
+ if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
3224
+ return update;
3225
+ }
3226
+ var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
3227
+ async function runResolve(deps, ctx, ambiguous) {
3228
+ const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
3229
+ if (ambiguous.length === 0) return out;
3230
+ const { entityId, trigger } = ctx;
3231
+ const outcome = await runBatched({
3232
+ items: ambiguous,
3233
+ buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
3234
+ batch.map((g, i) => ({
3235
+ item: i,
3236
+ candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
3237
+ existing: g.neighbours.map((n) => ({
3238
+ ref: n.ref,
3239
+ title: n.title,
3240
+ body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
3241
+ }))
3242
+ }))
3243
+ ),
3244
+ call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
3245
+ parse: (text, batch) => {
3246
+ const parsed = parseJsonResponse(text);
3247
+ if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
3248
+ return { batch, ops: parsed.ops };
3249
+ },
3250
+ maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
3251
+ maxOutputTokens: deps.options.llmProvider.maxOutputTokens
3252
+ });
3253
+ if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
3254
+ for (const { batch, ops } of outcome.results) {
3255
+ const byItem = /* @__PURE__ */ new Map();
3256
+ for (const raw of ops) {
3257
+ const item = raw?.item;
3258
+ if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
3259
+ byItem.set(item, raw);
3260
+ }
3261
+ }
3262
+ batch.forEach((g, i) => {
3263
+ const op = validateOp(byItem.get(i), aliasMapOf(g));
3264
+ if (op) out.ops.set(g, op);
3265
+ else out.rejected.push(g);
3266
+ });
3267
+ }
3268
+ if (outcome.skipped.some((s) => s.reason === "call_error")) {
3269
+ throw new Error("resolve: transient provider error; retry the pass");
3270
+ }
3271
+ for (const s of outcome.skipped) out.failed.push(s.item);
3272
+ return out;
3273
+ }
3274
+
3275
+ // src/utils/temporal.ts
3276
+ function assertEpochMs(name, v) {
3277
+ if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
3278
+ throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
3279
+ }
3280
+ return Math.trunc(v);
3281
+ }
3282
+
3283
+ // src/services/TemporalService.ts
3284
+ var HISTORY_MAX_DEPTH = 100;
3285
+ var TemporalService = class {
3286
+ constructor(db, options, entryRepo, searchService, embeddingService) {
3287
+ this.db = db;
3288
+ this.options = options;
3289
+ this.entryRepo = entryRepo;
3290
+ this.searchService = searchService;
3291
+ this.embeddingService = embeddingService;
3292
+ }
3293
+ /**
3294
+ * Link `oldId` → `newId` inside the caller's transaction. `newId` must
3295
+ * already exist. Throws WikiSupersedeError on any rule violation; the
3296
+ * caller's transaction then rolls back.
3297
+ */
3298
+ async supersedeInTx(tx, entityId, oldId, newId, t, now) {
3299
+ const old = await this.entryRepo.findTemporalRow(oldId, tx);
3300
+ if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
3301
+ if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
3302
+ if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
3303
+ if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
3304
+ if (newId === oldId) throw new WikiSupersedeError("cycle");
3305
+ const repl = await this.entryRepo.findTemporalRow(newId, tx);
3306
+ if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
3307
+ if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
3308
+ if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
3309
+ const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
3310
+ if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
3311
+ await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
3312
+ if (repl.valid_from == null) {
3313
+ await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
3314
+ }
3315
+ }
3316
+ async supersede(entityId, oldId, replacement, options) {
3317
+ const now = Date.now();
3318
+ const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
3319
+ let created = null;
3320
+ if (typeof replacement !== "string") {
3321
+ if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
3322
+ throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
3323
+ }
3324
+ created = {
3325
+ id: generateId("fact_"),
3326
+ entity_id: entityId,
3327
+ title: replacement.title,
3328
+ body: replacement.body,
3329
+ tags: Array.isArray(replacement.tags) ? replacement.tags : [],
3330
+ confidence: replacement.confidence ?? "certain",
3331
+ source_type: replacement.source_type ?? "user_stated",
3332
+ source_hash: null,
3333
+ source_ref: null,
3334
+ created_at: now,
3335
+ updated_at: now,
3336
+ last_accessed_at: null,
3337
+ access_count: 0,
3338
+ deleted_at: null
3339
+ };
3340
+ }
3341
+ const newId = typeof replacement === "string" ? replacement : created.id;
3342
+ const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
3343
+ await this.db.withTransactionAsync(async (tx) => {
3344
+ if (created) {
3345
+ await this.entryRepo.upsert(created, tx);
3346
+ if (newValidFrom !== void 0) {
3347
+ await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
3348
+ }
3349
+ }
3350
+ await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
3351
+ });
3352
+ await this.searchService.syncEntries(entityId, [oldId, newId]);
3353
+ if (created) {
3354
+ await this.embeddingService.embedFact(
3355
+ { id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
3356
+ { operation: "supersede", trigger: "call" }
3357
+ );
3358
+ }
3359
+ this.searchService.evictCache(entityId);
3360
+ return { newId };
3361
+ }
3362
+ async history(entityId, factId) {
3363
+ const start = await this.entryRepo.findTemporalRow(factId);
3364
+ if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
3365
+ const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
3366
+ const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
3367
+ return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
3368
+ }
3369
+ };
3370
+
3371
+ // src/services/librarian/ops/apply.ts
3372
+ async function applyOps(deps, ctx, input) {
3373
+ const { entityId, trigger } = ctx;
3374
+ const diagBase = { entityId, operation: "librarian", trigger };
3375
+ const grounding = deps.promptService.groundingFor("librarian");
3376
+ const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
3377
+ const rejected = new Set(input.rejected);
3378
+ const failed = new Set(input.failed);
3379
+ const written = [];
3380
+ const updatedIds = [];
3381
+ const now = Date.now();
3382
+ await deps.db.withTransactionAsync(async (tx) => {
3383
+ let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
3384
+ if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
3385
+ manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
3386
+ }
3387
+ const titleIndex = /* @__PURE__ */ new Map();
3388
+ for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
3389
+ titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
3390
+ }
3391
+ const pendingEdges = [];
3392
+ const insert = async (g, draft, validFrom, overrides) => {
3393
+ const fact = g.candidate.fact;
3394
+ const title = overrides?.title ?? fact.title;
3395
+ const body = overrides?.body ?? fact.body;
3396
+ const id = generateId("fact_");
3397
+ const drops = [];
3398
+ const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
3399
+ for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
3400
+ const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
3401
+ if (g2?.diagnostic) {
3402
+ input.diagnostics.push({
3403
+ ...diagBase,
3404
+ code: g2.diagnostic.code,
3405
+ detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
3406
+ });
3407
+ }
3408
+ const row = {
3409
+ id,
3410
+ entity_id: entityId,
3411
+ title,
3412
+ body,
3413
+ tags: fact.tags,
3414
+ confidence: fact.confidence,
3415
+ source_type: "librarian_inferred",
3416
+ source_hash: null,
3417
+ source_ref: null,
3418
+ created_at: now,
3419
+ updated_at: now,
3420
+ last_accessed_at: null,
3421
+ access_count: 0,
3422
+ deleted_at: null,
3423
+ okf_type: normalized.okf_type,
3424
+ ...g2?.trust,
3425
+ ...draft ? { lifecycle_status: "draft" } : {}
3426
+ };
3427
+ await deps.entryRepo.upsert(row, tx);
3428
+ await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
3429
+ titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
3430
+ if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
3431
+ const textChanged = title !== fact.title || body !== fact.body;
3432
+ written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
3433
+ return id;
3434
+ };
3435
+ for (const g of input.gated) {
3436
+ const idx = g.candidate.index;
3437
+ const vf = g.candidate.validFrom;
3438
+ if (g.decision.kind === "noop") continue;
3439
+ if (g.decision.kind === "add") {
3440
+ await insert(g, false, vf);
3441
+ continue;
3442
+ }
3443
+ if (failed.has(g)) {
3444
+ const id = await insert(g, true, vf);
3445
+ input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
3446
+ continue;
3447
+ }
3448
+ const op = input.ops.get(g);
3449
+ if (rejected.has(g) || !op) {
3450
+ await insert(g, false, vf);
3451
+ input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
3452
+ continue;
3453
+ }
3454
+ if (op.op === "NOOP") continue;
3455
+ if (op.op === "ADD") {
3456
+ await insert(g, false, vf);
3457
+ continue;
3458
+ }
3459
+ const target = g.neighbours.find((n) => n.id === op.targetId);
3460
+ if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
3461
+ const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
3462
+ if (existing) {
3463
+ const newTitle = op.title ?? g.candidate.fact.title;
3464
+ await deps.entryRepo.upsert({
3465
+ ...existing,
3466
+ title: newTitle,
3467
+ body: op.body ?? g.candidate.fact.body,
3468
+ updated_at: now
3469
+ }, tx);
3470
+ titleIndex.delete(normalizeTitleKey(existing.title));
3471
+ titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
3472
+ updatedIds.push(existing.id);
3473
+ continue;
3474
+ }
3475
+ }
3476
+ const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
3477
+ const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
3478
+ if (target.source_type === "immutable_document") {
3479
+ const id = await insert(g, true, supersedeFrom, overrides);
3480
+ input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
3481
+ continue;
3482
+ }
3483
+ const newId = await insert(g, false, supersedeFrom, overrides);
3484
+ try {
3485
+ await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
3486
+ } catch (err) {
3487
+ if (!(err instanceof WikiSupersedeError)) throw err;
3488
+ input.diagnostics.push({
3489
+ ...diagBase,
3490
+ code: "librarian_op_rejected",
3491
+ detail: { factId: newId, itemIndex: idx, reason: err.reason }
3492
+ });
3493
+ }
3494
+ }
3495
+ for (const item of pendingEdges) {
3496
+ const resolveDrops = [];
3497
+ await deps.ontologyService?.resolveAndPersistEdges(
3498
+ entityId,
3499
+ item.sourceId,
3500
+ item.sourceType,
3501
+ item.edges,
3502
+ manifest,
3503
+ titleIndex,
3504
+ tx,
3505
+ now,
3506
+ resolveDrops
3507
+ );
3508
+ for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
3509
+ }
3510
+ for (const task of input.tasks) {
3511
+ const taskObj = {
3512
+ id: generateId("task_"),
3513
+ entity_id: entityId,
3514
+ description: task.description,
3515
+ status: "pending",
3516
+ priority: task.priority,
3517
+ created_at: now,
3518
+ updated_at: now,
3519
+ resolved_at: null,
3520
+ deleted_at: null
3521
+ };
3522
+ await deps.taskRepo.upsert(taskObj, tx);
3523
+ }
3524
+ });
3525
+ return { written, updatedIds };
3526
+ }
3527
+
3528
+ // src/services/librarian/ops/index.ts
3529
+ var runOpsLibrarianPass = async (deps, ctx) => {
3530
+ const { entityId, trigger } = ctx;
3531
+ const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
3532
+ await deps.seedWatermark(entityId);
3533
+ const wm = await deps.getWatermark(entityId);
3534
+ const sel = await selectBatch(deps, entityId, wm, ctx.meter);
3535
+ if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
3536
+ if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
3537
+ let ex;
3538
+ try {
3539
+ ex = await runExtract(deps, ctx, sel.batch);
3540
+ } catch (err) {
3541
+ if (err instanceof WikiBudgetExhausted) {
3542
+ return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
3543
+ }
3544
+ throw err;
3545
+ }
3546
+ const diagnostics = new DiagnosticBuffer();
3547
+ for (const d of ex.diagnostics) diagnostics.push(d);
3548
+ const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
3549
+ for (const dup of inBatchDuplicates) {
3550
+ diagnostics.push({
3551
+ code: "fact_deduplicated",
3552
+ operation: "librarian",
3553
+ trigger,
3554
+ entityId,
3555
+ detail: { itemIndex: dup.index, reason: "in_batch" }
3556
+ });
3557
+ }
3558
+ diagnostics.push(gateDiagnostic);
3559
+ const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
3560
+ if (res.budgetStop) {
3561
+ diagnostics.discard();
3562
+ return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
3563
+ }
3564
+ const { written, updatedIds } = await applyOps(deps, ctx, {
3565
+ gated,
3566
+ ops: res.ops,
3567
+ rejected: res.rejected,
3568
+ failed: res.failed,
3569
+ tasks: ex.tasks,
3570
+ ontologyUpdates: ex.ontologyUpdates,
3571
+ groundingCorpus: ex.groundingCorpus,
3572
+ diagnostics
3573
+ });
3574
+ diagnostics.flush(deps.options);
3575
+ await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
3576
+ const embedCtx = { operation: "librarian", trigger };
3577
+ for (const w of written) {
3578
+ if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
3579
+ else await deps.embeddingService.embedFact(w.fact, embedCtx);
3580
+ }
3581
+ if (updatedIds.length > 0) {
3582
+ for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
3583
+ await deps.embeddingService.embedFact(
3584
+ { id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
3585
+ embedCtx
3586
+ );
3587
+ }
3588
+ }
3589
+ deps.searchService.evictCache(entityId);
3590
+ const last = sel.batch[sel.batch.length - 1].event;
3591
+ return {
3592
+ processedThrough: { at: last.created_at, id: last.id },
3593
+ factsWritten: written.length + updatedIds.length
3594
+ };
3595
+ };
3596
+
3597
+ // src/services/librarian/index.ts
3598
+ var STRATEGIES = /* @__PURE__ */ new Map([
3599
+ ["legacy", runLegacyLibrarianPass],
3600
+ ["ops", runOpsLibrarianPass]
3601
+ ]);
3602
+ async function runLibrarianStrategy(deps, ctx) {
3603
+ const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
3604
+ const fn = STRATEGIES.get(strategy);
3605
+ if (!fn) {
3606
+ throw new Error(`Unknown librarian strategy: ${strategy}`);
3607
+ }
3608
+ return fn(deps, ctx);
3609
+ }
3610
+
3611
+ // src/services/MaintenanceService.ts
2702
3612
  var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
2703
3613
  var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
2704
3614
  var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
@@ -2771,11 +3681,23 @@ var MaintenanceService = class {
2771
3681
  const vacuum = options?.vacuum ?? false;
2772
3682
  this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
2773
3683
  this._validatePruneDuration(retainEventsFor, "retainEventsFor");
3684
+ const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
3685
+ this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
2774
3686
  const now = Date.now();
2775
3687
  let syncedIds = [];
2776
3688
  let deletedEntries = 0;
2777
3689
  let deletedTasks = 0;
2778
3690
  let deletedEvents = 0;
3691
+ if (pruneSupersededAfter !== null) {
3692
+ const expiredCutoff = now - pruneSupersededAfter * 864e5;
3693
+ const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
3694
+ if (expiredIds.length > 0) {
3695
+ await this.db.withTransactionAsync(async (tx) => {
3696
+ await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
3697
+ });
3698
+ syncedIds = [...expiredIds];
3699
+ }
3700
+ }
2779
3701
  if (retainSoftDeletedFor !== null) {
2780
3702
  const cutoff = now - retainSoftDeletedFor * 864e5;
2781
3703
  const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
@@ -2791,7 +3713,7 @@ var MaintenanceService = class {
2791
3713
  }
2792
3714
  }
2793
3715
  const succeededIds = succeeded.map((r) => r.id);
2794
- syncedIds = succeededIds;
3716
+ syncedIds = [...syncedIds, ...succeededIds];
2795
3717
  await this.db.withTransactionAsync(async (tx) => {
2796
3718
  if (succeededIds.length > 0) {
2797
3719
  deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
@@ -2827,8 +3749,18 @@ var MaintenanceService = class {
2827
3749
  }
2828
3750
  if (retainEventsFor !== null) {
2829
3751
  const cutoff = now - retainEventsFor * 864e5;
2830
- const eventResult = await this.eventRepo.prune(entityId, cutoff);
2831
- deletedEvents = eventResult.changes;
3752
+ const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
3753
+ if (guarded) {
3754
+ const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
3755
+ deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
3756
+ const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
3757
+ if (held > 0) {
3758
+ emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
3759
+ }
3760
+ } else {
3761
+ const eventResult = await this.eventRepo.prune(entityId, cutoff);
3762
+ deletedEvents = eventResult.changes;
3763
+ }
2832
3764
  }
2833
3765
  if (vacuum) {
2834
3766
  await this.metadataRepo.vacuum();
@@ -3058,155 +3990,72 @@ var MaintenanceService = class {
3058
3990
  }
3059
3991
  /** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
3060
3992
  async doRunLibrarian(entityId, promptOverride, trigger = "call") {
3061
- const events = await this.eventRepo.getRecent(entityId, 50);
3062
- const currentFactsRows = await this.entryRepo.findRecentByEntityId(entityId, 100);
3063
- const currentFacts = currentFactsRows.map((f) => {
3064
- const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
3065
- return {
3066
- ...rest,
3067
- tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
3068
- };
3069
- });
3070
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
3071
- const promptEvents = events.reverse();
3072
- const librarianGrounding = this.promptService.groundingFor("librarian");
3073
- const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = this.promptService.buildLibrarianPrompt(
3074
- promptEvents,
3075
- currentFacts,
3076
- promptOverride,
3077
- ontologyContext
3078
- );
3079
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
3080
- const result = parseJsonResponse(responseText);
3081
- const facts = Array.isArray(result.facts) ? result.facts : [];
3082
- const tasks = Array.isArray(result.tasks) ? result.tasks : [];
3083
- const ontologyUpdates = result.ontology_updates;
3084
- const diagBuffer = new DiagnosticBuffer();
3085
- const diagBase = { entityId, operation: "librarian", trigger };
3086
- const validFacts = [];
3087
- const validFactItemIndexes = [];
3088
- facts.forEach((raw, itemIndex) => {
3089
- const valid = validateFact(raw);
3090
- if (valid) {
3091
- validFacts.push(valid);
3092
- validFactItemIndexes.push(itemIndex);
3093
- } else {
3094
- diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
3095
- }
3096
- });
3097
- const validTasks = [];
3098
- tasks.forEach((raw, itemIndex) => {
3099
- const valid = validateTask(raw);
3100
- if (valid) validTasks.push(valid);
3101
- else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
3102
- });
3103
- const now = Date.now();
3104
- const insertedFacts = [];
3105
- await this.db.withTransactionAsync(async (tx) => {
3106
- let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
3107
- if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
3108
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
3109
- }
3110
- const titleIndex = /* @__PURE__ */ new Map();
3111
- for (const existing of currentFactsRows) {
3112
- titleIndex.set(normalizeTitleKey(existing.title), {
3113
- id: existing.id,
3114
- okf_type: existing.okf_type ?? null
3115
- });
3116
- }
3117
- const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
3118
- const pendingEdges = [];
3119
- for (const [k, fact] of validFacts.entries()) {
3120
- const newTokens = titleTokens(fact.title);
3121
- let skip = false;
3122
- if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
3123
- for (const existing of factsForDedupe) {
3124
- if (existing.source_type !== "librarian_inferred") continue;
3125
- const existingTokens = titleTokens(existing.title);
3126
- if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
3127
- if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
3128
- skip = true;
3129
- break;
3130
- }
3131
- }
3132
- }
3133
- }
3134
- if (skip) {
3135
- diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
3136
- continue;
3137
- }
3138
- const ontologyFact = fact;
3139
- const id = generateId("fact_");
3140
- const validationDrops = [];
3141
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
3142
- for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
3143
- const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
3144
- if (grounding?.diagnostic) {
3145
- diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
3146
- }
3147
- const factObj = {
3148
- id,
3149
- entity_id: entityId,
3150
- title: fact.title,
3151
- body: fact.body,
3152
- tags: fact.tags,
3153
- confidence: fact.confidence,
3154
- source_type: "librarian_inferred",
3155
- source_hash: null,
3156
- source_ref: null,
3157
- created_at: now,
3158
- updated_at: now,
3159
- last_accessed_at: null,
3160
- access_count: 0,
3161
- deleted_at: null,
3162
- okf_type: normalized.okf_type,
3163
- ...grounding?.trust
3164
- };
3165
- await this.entryRepo.upsert(factObj, tx);
3166
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
3167
- factsForDedupe.push(factObj);
3168
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
3169
- if (normalized.edges.length > 0) {
3170
- pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
3171
- }
3172
- }
3173
- for (const item of pendingEdges) {
3174
- const resolveDrops = [];
3175
- await this.ontologyService?.resolveAndPersistEdges(
3176
- entityId,
3177
- item.sourceId,
3178
- item.sourceType,
3179
- item.edges ?? [],
3180
- manifest,
3181
- titleIndex,
3182
- tx,
3183
- now,
3184
- resolveDrops
3185
- );
3186
- for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
3187
- }
3188
- for (const task of validTasks) {
3189
- const id = generateId("task_");
3190
- const taskObj = {
3191
- id,
3192
- entity_id: entityId,
3193
- description: task.description,
3194
- status: "pending",
3195
- priority: task.priority,
3196
- created_at: now,
3197
- updated_at: now,
3198
- resolved_at: null,
3199
- deleted_at: null
3200
- };
3201
- await this.taskRepo.upsert(taskObj, tx);
3202
- }
3203
- });
3204
- diagBuffer.flush(this.options);
3205
- await this.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
3206
- for (const fact of insertedFacts) {
3207
- await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
3993
+ await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
3994
+ }
3995
+ /**
3996
+ * Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
3997
+ * §5.1). Builds the strategy deps from `this` and dispatches via
3998
+ * `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
3999
+ * class. Returns the strategy's `LibrarianResult` so deferred/budgeted
4000
+ * schedulers (PR-C) can advance watermarks; the existing void-returning
4001
+ * `doRunLibrarian` discards it.
4002
+ *
4003
+ * Spec §10.3: one place advances the watermark, for every strategy, only
4004
+ * after the pass resolved (its transaction committed) and never backwards.
4005
+ * A budget-stopped pass wrote nothing and must not move the watermark.
4006
+ */
4007
+ async runLibrarianPass(ctx) {
4008
+ const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
4009
+ if (result.processedThrough && !result.budgetStop) {
4010
+ await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
3208
4011
  }
3209
- this.searchService.evictCache(entityId);
4012
+ return result;
4013
+ }
4014
+ /**
4015
+ * Strategy deps shared between legacy and ops librarians (spec §5.1). The
4016
+ * ops strategy (PR-B Task 8) reads/anchors the watermark through these two
4017
+ * hooks so the dispatcher itself never reaches into the metadata table —
4018
+ * the watermark invariant (one place advances, never backwards) lives here.
4019
+ */
4020
+ librarianDeps() {
4021
+ return {
4022
+ db: this.db,
4023
+ options: this.options,
4024
+ entryRepo: this.entryRepo,
4025
+ taskRepo: this.taskRepo,
4026
+ eventRepo: this.eventRepo,
4027
+ searchService: this.searchService,
4028
+ embeddingService: this.embeddingService,
4029
+ promptService: this.promptService,
4030
+ ontologyService: this.ontologyService,
4031
+ seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
4032
+ getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
4033
+ };
4034
+ }
4035
+ /**
4036
+ * Persist `next` as the entity's librarian watermark when it is strictly
4037
+ * newer than the stored value, under the (created_at, id) ordering used by
4038
+ * the op-based librarian (spec §10.3). Never moves backwards.
4039
+ */
4040
+ async advanceLibrarianWatermark(entityId, next) {
4041
+ const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
4042
+ const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
4043
+ if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
4044
+ }
4045
+ /**
4046
+ * Spec §10.3: first ops pass on a legacy DB starts after what the count
4047
+ * checkpoint already covered. No-op when a watermark already exists or the
4048
+ * memory checkpoint is 0; otherwise anchor the watermark to the event at
4049
+ * offset `min(memory_checkpoint, totalEvents) - 1`.
4050
+ */
4051
+ async seedLibrarianWatermark(entityId) {
4052
+ if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
4053
+ const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
4054
+ const memory = cp.memory ?? 0;
4055
+ if (memory <= 0) return;
4056
+ const total = await this.eventRepo.count(entityId);
4057
+ const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
4058
+ if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
3210
4059
  }
3211
4060
  /**
3212
4061
  * Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
@@ -3312,7 +4161,7 @@ var MaintenanceService = class {
3312
4161
  if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
3313
4162
  return prompts;
3314
4163
  },
3315
- call: (prompts) => this.options.llmProvider.generateText(prompts),
4164
+ call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
3316
4165
  parse: (responseText, batch, prompts) => {
3317
4166
  const result = parseJsonResponse(responseText);
3318
4167
  return {
@@ -3416,9 +4265,10 @@ var MaintenanceService = class {
3416
4265
  const callErrorIds = new Set(
3417
4266
  outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
3418
4267
  );
4268
+ const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
3419
4269
  await this.entryRepo.markHealChecked(
3420
4270
  [
3421
- ...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
4271
+ ...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
3422
4272
  ...insertedFacts.map((f) => f.id)
3423
4273
  ],
3424
4274
  entityId,
@@ -3470,7 +4320,8 @@ var MaintenanceService = class {
3470
4320
  skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
3471
4321
  degraded: healedDegraded,
3472
4322
  remaining: counts.eligible,
3473
- deferred: counts.deferred
4323
+ deferred: counts.deferred,
4324
+ ...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
3474
4325
  };
3475
4326
  }
3476
4327
  /** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
@@ -3514,7 +4365,7 @@ var MaintenanceService = class {
3514
4365
  const outcome = await runBatched({
3515
4366
  items: candidates,
3516
4367
  buildPrompt,
3517
- call: (prompts) => this.options.llmProvider.generateText(prompts),
4368
+ call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
3518
4369
  parse: (responseText, batch) => {
3519
4370
  const parsed = parseJsonResponse(responseText);
3520
4371
  return {
@@ -3899,6 +4750,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
3899
4750
  return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
3900
4751
  }
3901
4752
 
4753
+ // src/utils/budget.ts
4754
+ function factTokenCost(f) {
4755
+ return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
4756
+ }
4757
+ function packFactsByBudget(items, budget, scores) {
4758
+ const candidates = items.length;
4759
+ if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
4760
+ let top = items[0];
4761
+ let topCost = factTokenCost(top);
4762
+ if (topCost > budget) {
4763
+ const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
4764
+ top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
4765
+ topCost = factTokenCost(top);
4766
+ }
4767
+ const kept = [{ rank: 0, item: top }];
4768
+ let used = topCost;
4769
+ const rest = items.slice(1).map((item, i) => {
4770
+ const rank = i + 1;
4771
+ const cost = factTokenCost(item);
4772
+ const s = scores?.get(item.id);
4773
+ const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
4774
+ return { rank, item, cost, density: value / Math.max(1, cost) };
4775
+ });
4776
+ rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
4777
+ for (const r of rest) {
4778
+ if (used + r.cost <= budget) {
4779
+ kept.push({ rank: r.rank, item: r.item });
4780
+ used += r.cost;
4781
+ }
4782
+ }
4783
+ kept.sort((a, b) => a.rank - b.rank);
4784
+ return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
4785
+ }
4786
+
3902
4787
  // src/services/RetrievalService.ts
3903
4788
  var EMPTY_ID_SET = /* @__PURE__ */ new Set();
3904
4789
  var RetrievalService = class {
@@ -3934,6 +4819,28 @@ var RetrievalService = class {
3934
4819
  const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
3935
4820
  const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
3936
4821
  const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
4822
+ const rawAsOf = options?.asOf;
4823
+ if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
4824
+ throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
4825
+ }
4826
+ const tokenBudget = options?.tokenBudget;
4827
+ if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
4828
+ throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
4829
+ }
4830
+ const applyBudget = (input) => {
4831
+ if (tokenBudget === void 0) return input;
4832
+ const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
4833
+ emitDiagnostic(this.options, {
4834
+ code: "read_budget",
4835
+ operation: "read",
4836
+ trigger: "call",
4837
+ entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
4838
+ detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
4839
+ });
4840
+ return packed.items;
4841
+ };
4842
+ const liveMode = rawAsOf === void 0 ? "current" : "asOf";
4843
+ const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
3937
4844
  const trimmedQuery = query.trim();
3938
4845
  const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
3939
4846
  entityIds,
@@ -3953,8 +4860,10 @@ var RetrievalService = class {
3953
4860
  let usedEmbed = false;
3954
4861
  const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
3955
4862
  const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
3956
- const draftPad = draftIds.size;
3957
- const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + draftPad;
4863
+ const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
4864
+ const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
4865
+ const excludedPad = excludedIds.size;
4866
+ const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
3958
4867
  if (scoredEntityIds.length === 0) {
3959
4868
  usedEmbed = true;
3960
4869
  } else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
@@ -3988,7 +4897,7 @@ var RetrievalService = class {
3988
4897
  let miniSearchScores;
3989
4898
  if (effectivePreFilterLimit !== void 0) {
3990
4899
  populateCache = false;
3991
- const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !draftIds.has(r.id));
4900
+ const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
3992
4901
  if (preResults.length === 0) {
3993
4902
  candidateRows = null;
3994
4903
  } else {
@@ -4010,9 +4919,9 @@ var RetrievalService = class {
4010
4919
  }
4011
4920
  } else {
4012
4921
  if (useRanker) {
4013
- candidateRows = this._withoutDrafts(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), draftIds);
4922
+ candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
4014
4923
  } else {
4015
- candidateRows = this._withoutDrafts(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), draftIds);
4924
+ candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
4016
4925
  }
4017
4926
  if (weight !== void 0 && weight < 1) {
4018
4927
  miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
@@ -4297,7 +5206,7 @@ var RetrievalService = class {
4297
5206
  // read() re-sorts after applying tier weights
4298
5207
  });
4299
5208
  }
4300
- if (draftPad > 0) scored = scored.filter((s) => !draftIds.has(s.id));
5209
+ if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
4301
5210
  if (scored.length > 0) {
4302
5211
  scored = scored.map((row) => ({
4303
5212
  ...row,
@@ -4361,7 +5270,7 @@ var RetrievalService = class {
4361
5270
  if (!usedEmbed && scoredEntityIds.length > 0) {
4362
5271
  const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
4363
5272
  const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
4364
- const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !draftIds.has(r.id));
5273
+ const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
4365
5274
  const candidates = results.map((r) => ({
4366
5275
  id: r.id,
4367
5276
  entity_id: r.entity_id,
@@ -4379,13 +5288,18 @@ var RetrievalService = class {
4379
5288
  }
4380
5289
  }
4381
5290
  }
5291
+ facts = applyBudget(facts);
4382
5292
  if (facts.length > 0) {
4383
5293
  const ids = facts.map((f) => f.id);
4384
5294
  const now = Date.now();
4385
5295
  await this.entryRepo.trackAccess(ids, now);
4386
5296
  }
4387
5297
  } else {
4388
- facts = excludeDrafts ? await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, { excludeDrafts: true }) : await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
5298
+ facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
5299
+ ...excludeDrafts ? { excludeDrafts: true } : {},
5300
+ live: { mode: liveMode, t: liveAt }
5301
+ });
5302
+ facts = applyBudget(facts);
4389
5303
  }
4390
5304
  const eventsLimit = Math.min(10 * entityIds.length, 100);
4391
5305
  const [tasks, events] = await Promise.all([
@@ -4421,8 +5335,8 @@ var RetrievalService = class {
4421
5335
  _tieBreakSort(items) {
4422
5336
  items.sort((a, b) => this._compareScoredRows(a, b));
4423
5337
  }
4424
- _withoutDrafts(rows, draftIds) {
4425
- return draftIds.size === 0 ? rows : rows.filter((row) => !draftIds.has(row.id));
5338
+ _withoutIds(rows, excludeIds) {
5339
+ return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
4426
5340
  }
4427
5341
  /**
4428
5342
  * Comparator for score + deterministic tie-break fields.
@@ -4493,21 +5407,6 @@ var RetrievalService = class {
4493
5407
  return scored;
4494
5408
  }
4495
5409
  };
4496
-
4497
- // src/utils/cosine.ts
4498
- function cosineSimilarity(a, b) {
4499
- let dot = 0, normA = 0, normB = 0;
4500
- const len = Math.min(a.length, b.length);
4501
- for (let i = 0; i < len; i++) {
4502
- dot += a[i] * b[i];
4503
- normA += a[i] * a[i];
4504
- normB += b[i] * b[i];
4505
- }
4506
- const denom = Math.sqrt(normA) * Math.sqrt(normB);
4507
- return denom === 0 ? 0 : dot / denom;
4508
- }
4509
-
4510
- // src/services/SearchService.ts
4511
5410
  var _SearchService = class _SearchService {
4512
5411
  constructor(entryRepo) {
4513
5412
  this.entryRepo = entryRepo;
@@ -5290,6 +6189,18 @@ var WriteService = class {
5290
6189
  this.metadataRepo = metadataRepo;
5291
6190
  this.jobManager = jobManager;
5292
6191
  this.maintenanceService = maintenanceService;
6192
+ this.inflight = /* @__PURE__ */ new Set();
6193
+ }
6194
+ track(p) {
6195
+ this.inflight.add(p);
6196
+ p.finally(() => this.inflight.delete(p)).catch(() => {
6197
+ });
6198
+ }
6199
+ /** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
6200
+ async drain() {
6201
+ while (this.inflight.size > 0) {
6202
+ await Promise.allSettled([...this.inflight]);
6203
+ }
5293
6204
  }
5294
6205
  async write(entityId, event) {
5295
6206
  if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
@@ -5316,6 +6227,7 @@ var WriteService = class {
5316
6227
  }
5317
6228
  const id = generateId("evt_");
5318
6229
  const now = Date.now();
6230
+ const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
5319
6231
  let eventType = event.event_type;
5320
6232
  if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
5321
6233
  eventType = "observation";
@@ -5326,8 +6238,14 @@ var WriteService = class {
5326
6238
  event_type: eventType,
5327
6239
  summary,
5328
6240
  related_entry_id: relatedEntryId,
5329
- created_at: now
6241
+ created_at: now,
6242
+ occurred_at: occurredAt
5330
6243
  };
6244
+ const deferred = this.options.config?.maintenance === "deferred";
6245
+ if (deferred) {
6246
+ await this.eventRepo.add(newEvent);
6247
+ return;
6248
+ }
5331
6249
  let shouldRunLibrarian = false;
5332
6250
  let librarianCount = 0;
5333
6251
  let prevMemoryCheckpoint = 0;
@@ -5349,27 +6267,41 @@ var WriteService = class {
5349
6267
  prevMemoryCheckpoint = memoryCheckpoint;
5350
6268
  await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
5351
6269
  }
6270
+ } else {
6271
+ const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
6272
+ if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
6273
+ const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
6274
+ const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
6275
+ if (Math.ceil(chars / 4) >= tokenThreshold) {
6276
+ shouldRunLibrarian = true;
6277
+ librarianCount = count;
6278
+ prevMemoryCheckpoint = memoryCheckpoint;
6279
+ await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
6280
+ }
6281
+ }
5352
6282
  }
5353
6283
  });
5354
6284
  if (shouldRunLibrarian) {
5355
6285
  try {
5356
6286
  this.jobManager.acquireLock("librarian", entityId);
5357
6287
  const stage = { job: "librarian" };
5358
- this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
6288
+ const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
5359
6289
  console.error(err);
5360
6290
  this.reportBackgroundFailure(entityId, stage.job);
5361
6291
  }).finally(() => {
5362
6292
  this.jobManager.releaseLock("librarian", entityId);
5363
6293
  });
6294
+ this.track(job);
5364
6295
  } catch (e) {
5365
6296
  if (!(e instanceof WikiBusyError)) throw e;
5366
6297
  await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
5367
6298
  }
5368
6299
  } else if (!this.jobManager.isBlocked("librarian", entityId)) {
5369
- this.maybeRunHeal(entityId, eventCount).catch((err) => {
6300
+ const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
5370
6301
  console.error(err);
5371
6302
  this.reportBackgroundFailure(entityId, "heal");
5372
6303
  });
6304
+ this.track(healJob);
5373
6305
  }
5374
6306
  }
5375
6307
  async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {