dsh-trivium 0.4.21 → 0.5.0

package/lib/context-budget.js

@@ -0,0 +1,348 @@
+/**
+ * Unified per-turn context budget.
+ *
+ * Chips (pins) and recall used to clip to 300 tokens each, so one turn could
+ * carry 600 and the same node could appear twice. This module owns a single
+ * decision: which structured records go into this turn, in what order, and
+ * what the exact text will be. It never re-parses a clipped string.
+ *
+ * Only `estimateTokens` is used for accounting — it is a CJK/ASCII heuristic,
+ * not a model tokenizer, so every surface must label its numbers as estimates.
+ */
+
+import { estimateTokens, clipToTokenBudget } from "./tokens.js";
+
+/** Session map: once per session, labels only. */
+export const MAP_TOKEN_BUDGET = 400;
+
+/** One user turn: pins + corrections + recall share this, headers included. */
+export const TURN_ADDITIONAL_BUDGET = 300;
+
+/** Why a candidate did not make it. Kept short so it can be shown as a tag. */
+export const DROP_REASONS = Object.freeze({
+  invalid: "invalid",
+  archived: "archived",
+  dirty: "dirty",
+  stale: "stale",
+  empty: "empty",
+  duplicateNode: "duplicate-node",
+  duplicateTurn: "duplicate-in-turn",
+  alreadyInMap: "already-in-map",
+  noCurrent: "no-current",
+  overBudget: "over-budget",
+});
+
+const PIN_HEADER = "dsh-trivium pins (L0 only; use ctx_read for full text):";
+const RECALL_HEADER = (form) =>
+  `dsh-trivium ${form || "recall"} (L0 only; use ctx_read for full text):`;
+
+const MIN_MAP_MATCH_CHARS = 12;
+
+let turnSeq = 0;
+const objectTurnKeys = new WeakMap();
+
+function usableTurnValue(value) {
+  if (value === undefined || value === null) return "";
+  const raw = String(value).trim();
+  return raw ? raw : "";
+}
+
+/**
+ * Stable identity for one user turn.
+ *
+ * Host ids only — never `Date.now()`, the query text, or a pre-step counter.
+ * A tool loop calls pre-step several times for one user message; that reuses
+ * the same message object, so the identity stays put. Two messages with
+ * identical text but different host ids are two turns.
+ *
+ * @param messages - messages that make up the upcoming turn.
+ * @returns `uid:<host id>`, `obj:<n>` for hosts that expose no id, or "".
+ */
+export function turnKeyOf(messages = []) {
+  const users = (messages || []).filter((m) => m?.source?.kind === "user");
+  const last = users[users.length - 1];
+  if (!last) return "";
+  for (const candidate of [
+    last.id,
+    last.messageId,
+    last.seq,
+    last.source?.messageId,
+    last.source?.id,
+    last.source?.seq,
+    last.source?.turn,
+    last.turn,
+  ]) {
+    const value = usableTurnValue(candidate);
+    if (value) return `uid:${value}`;
+  }
+  let token = objectTurnKeys.get(last);
+  if (!token) {
+    token = `obj:${++turnSeq}`;
+    objectTurnKeys.set(last, token);
+  }
+  return token;
+}
+
+function normaliseText(value, max = 200) {
+  return String(value ?? "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, max);
+}
+
+function normalisePath(path) {
+  if (!Array.isArray(path)) return [];
+  return path.map((p) => normaliseText(p, 60)).filter(Boolean).slice(0, 2);
+}
+
+/**
+ * One structured candidate. Producers (pins.js, store.js) already know the
+ * payload; they must hand over fields, not a rendered string.
+ */
+function shapeCandidate(raw, channel) {
+  if (!raw) return null;
+  const id = Number(raw.id);
+  if (!Number.isFinite(id)) return null;
+  return {
+    id,
+    channel,
+    type: normaliseText(raw.type, 24) || "unknown",
+    l0: normaliseText(raw.l0 ?? raw.text ?? raw.name, 200),
+    until: normaliseText(raw.until, 32),
+    stale: raw.stale === true,
+    archived: raw.archived === true || raw.status === "archived",
+    dirty: raw.dirty === true,
+    path: normalisePath(raw.path),
+    historical: raw.historical === true,
+    supersededBy: Number.isFinite(Number(raw.supersededBy)) ? Number(raw.supersededBy) : null,
+    matchedHistory: raw.matchedHistory || null,
+    note: normaliseText(raw.note, 120),
+    fingerprint: "",
+  };
+}
+
+function fingerprintOf(id, l0) {
+  // Cheap stable content fingerprint: node id plus its normalised L0. Enough to
+  // tell "same record delivered twice" from "record changed within the turn".
+  let hash = 5381;
+  const s = String(l0 || "");
+  for (let i = 0; i < s.length; i += 1) hash = ((hash * 33) ^ s.charCodeAt(i)) >>> 0;
+  return `${id}:${hash.toString(36)}`;
+}
+
+function lineFor(record) {
+  const flags = [];
+  if (record.stale) flags.push(record.until ? `stale until=${record.until}` : "stale");
+  if (record.historical) flags.push(record.supersededBy ? `superseded->#${record.supersededBy}` : "superseded");
+  if (record.matchedHistory) {
+    flags.push(`matched_history=#${record.matchedHistory.from} → current=#${record.matchedHistory.current}`);
+  }
+  if (record.note) flags.push(record.note);
+  const suffix = flags.length ? ` ${flags.join(" ")}` : "";
+  const body = record.channel === "pin" ? record.l0 : withPath(record);
+  return `- [${record.type}#${record.id}${suffix}] ${body}`;
+}
+
+function withPath(record) {
+  if (!record.path.length) return record.l0;
+  return `${record.l0} (${record.path.join(" | ")})`;
+}
+
+function blockTextOf(records, channel, form) {
+  if (!records.length) return "";
+  const header = channel === "pin" ? PIN_HEADER : RECALL_HEADER(form);
+  return `${header}\n${records.map((r) => r.line).join("\n")}`;
+}
+
+/**
+ * Render the current selection. Pins and recall stay separate messages (the
+ * client labels them differently) but they are measured together against the
+ * one shared budget.
+ */
+export function renderContextRecords(plan, { form = "recall" } = {}) {
+  const pinRecords = plan.records.filter((r) => r.channel === "pin");
+  const recallRecords = plan.records.filter((r) => r.channel !== "pin");
+  const pinText = blockTextOf(pinRecords, "pin", form);
+  const recallText = blockTextOf(recallRecords, "recall", form);
+  const pinTokens = pinText ? estimateTokens(pinText) : 0;
+  const recallTokens = recallText ? estimateTokens(recallText) : 0;
+  return {
+    pins: { text: pinText, tokens: pinTokens },
+    recall: { text: recallText, tokens: recallTokens },
+    total: pinTokens + recallTokens,
+    estimatedTokens: pinTokens + recallTokens,
+  };
+}
+
+function measure(records, form) {
+  return renderContextRecords({ records }, { form }).total;
+}
+
+/**
+ * Turn one round of candidates into the exact records to inject.
+ *
+ * Order is pins → corrections → recall. Pins win the budget because the user
+ * chose them by hand; recall only gets what is left. A record that does not
+ * fit is dropped with `over-budget` instead of being clipped mid-record.
+ *
+ * @param pins - structured pin candidates, user order.
+ * @param recall - structured recall candidates, ranked.
+ * @param corrections - short staleness/supersession notices for this turn.
+ * @param resolveCurrent - optional `(id) => ({id, payload, archived, stale})`;
+ *   M2 plugs the supersedes chain in here. Without it nothing is remapped.
+ * @param delivered - fingerprints already delivered in this turn.
+ * @param mapText - the short map text when it arrived in this same turn.
+ */
+export function selectContextRecords({
+  pins = [],
+  recall = [],
+  corrections = [],
+  budget = TURN_ADDITIONAL_BUDGET,
+  form = "recall",
+  now,
+  resolveCurrent = null,
+  delivered = null,
+  mapText = "",
+} = {}) {
+  const cap = Math.max(1, Number(budget) || TURN_ADDITIONAL_BUDGET);
+  const at = now instanceof Date ? now : new Date();
+  const deliveredSet = delivered instanceof Set ? delivered : new Set();
+  const mapBlob = String(mapText || "");
+  const dropped = [];
+  const deferred = [];
+  const byId = new Map();
+  const ordered = [];
+
+  const drop = (candidate, reason, channel) =>
+    dropped.push({ id: bestId(candidate), channel, reason });
+
+  const accept = (raw, channel) => {
+    const record = shapeCandidate(raw, channel);
+    if (!record) {
+      drop(raw, DROP_REASONS.invalid, channel);
+      return;
+    }
+    if (!record.l0) {
+      drop(record, DROP_REASONS.empty, channel);
+      return;
+    }
+    if (record.archived) {
+      drop(record, DROP_REASONS.archived, channel);
+      return;
+    }
+    if (record.dirty) {
+      drop(record, DROP_REASONS.dirty, channel);
+      return;
+    }
+
+    let targetId = record.id;
+    if (record.channel !== "pin" && typeof resolveCurrent === "function") {
+      let resolved = null;
+      try {
+        resolved = resolveCurrent(record.id);
+      } catch {
+        resolved = null;
+      }
+      if (resolved === null || resolved === undefined) {
+        drop(record, DROP_REASONS.noCurrent, channel);
+        return;
+      }
+      const nextId = Number(resolved.currentId ?? resolved.id ?? record.id);
+      if (!Number.isFinite(nextId)) {
+        drop(record, DROP_REASONS.noCurrent, channel);
+        return;
+      }
+      // The successor is gone, archived or expired: do not resurrect the old
+      // node, and do not inject the dead chain either.
+      if (nextId !== record.id) {
+        if (resolved.archived === true || resolved.stale === true) {
+          drop(record, resolved.archived === true ? DROP_REASONS.archived : DROP_REASONS.stale, channel);
+          return;
+        }
+        record.matchedHistory = { from: record.id, current: nextId };
+        targetId = nextId;
+        record.id = nextId;
+      }
+    } else if (record.channel !== "pin" && record.stale) {
+      // Recall never injects expired nodes; pinned chips may be stale (the user
+      // chose them) and carry a `stale until=` flag instead.
+      drop(record, DROP_REASONS.stale, channel);
+      return;
+    }
+
+    const existing = byId.get(targetId);
+    if (existing) {
+      if (!existing.channels.includes(record.channel)) existing.channels.push(record.channel);
+      // A node already present keeps its first (higher-priority) rendering.
+      drop(record, DROP_REASONS.duplicateNode, channel);
+      return;
+    }
+
+    record.channel = record.channel === "pin" ? "pin" : channel;
+    record.channels = [record.channel];
+    record.fingerprint = fingerprintOf(record.id, record.l0);
+    record.line = lineFor(record);
+    if (deliveredSet.has(record.fingerprint)) {
+      drop(record, DROP_REASONS.duplicateTurn, record.channel);
+      return;
+    }
+    if (mapBlob && mapCoversRecord(mapBlob, record.l0)) {
+      drop(record, DROP_REASONS.alreadyInMap, record.channel);
+      return;
+    }
+    byId.set(targetId, record);
+    ordered.push(record);
+  };
+
+  for (const pin of pins) accept(pin, "pin");
+  for (const correction of corrections) accept(correction, "recall");
+  for (const hit of recall) accept(hit, "recall");
+
+  // Greedy fill in priority order: a later, shorter record may still fit after
+  // a long one is rejected, so keep trying instead of stopping at the first miss.
+  const kept = [];
+  for (const record of ordered) {
+    const attempt = [...kept, record];
+    if (measure(attempt, form) <= cap) kept.push(record);
+    else if (record.channel === "pin") {
+      // An explicit pin that cannot fit is deferred, not silently mixed in.
+      drop(record, DROP_REASONS.overBudget, record.channel);
+      deferred.push({ id: record.id, channel: record.channel, reason: DROP_REASONS.overBudget });
+    } else {
+      drop(record, DROP_REASONS.overBudget, record.channel);
+      deferred.push({ id: record.id, channel: record.channel, reason: DROP_REASONS.overBudget });
+    }
+  }
+
+  const rendered = renderContextRecords({ records: kept }, { form });
+  return {
+    budget: cap,
+    tokens: rendered.total,
+    estimatedTokens: rendered.total,
+    records: kept,
+    dropped,
+    deferred,
+    at: at.toISOString(),
+  };
+}
+
+function bestId(candidate) {
+  const id = Number(candidate?.id);
+  return Number.isFinite(id) ? id : null;
+}
+
+/**
+ * Cross-stage duplicate: the map is labels only, so an identical long body is
+ * the same fact and must not be repeated. A short label match is not "full
+ * text already in context" and still gets its summary.
+ */
+function mapCoversRecord(mapText, l0) {
+  const body = normaliseText(l0, 400);
+  if (body.length < MIN_MAP_MATCH_CHARS) return false;
+  return mapText.includes(body);
+}
+
+/** Clip one already-selected block; used by the debug/UI surface, not injection. */
+export function clipBlock(text, budget = TURN_ADDITIONAL_BUDGET) {
+  return clipToTokenBudget(text, budget);
+}

package/lib/extract.js

@@ -9,7 +9,7 @@ import {
   mergeInto,
   ensureLink,
 } from "./store.js";
-import { parseUntilFromText, resolveUntilAt } from "./until.js";
+import { resolveUntilAt } from "./until.js";
 
 export const PREFERENCE_CUES =
   /记住|以后都|别再|不要再|从现在起|always do|from now on|never again|remember (?:this|that|to)\b/i;
@@ -114,10 +114,6 @@ export function textFromContent(content) {
   return parts.join("\n");
 }
 
-function untilOf(text) {
-  return parseUntilFromText(text);
-}
-
 function isToolDriveTurn(text) {
   return TOOL_DRIVE_CUES.test(String(text || ""));
 }
@@ -256,7 +252,6 @@ export function ruleCandidates(turns) {
         type: "decision",
         name: object,
         text: clip(span, 240),
-        until: untilOf(span),
         linkName: object,
         linkLabel: EDGE_LABELS.decided,
         quote: clip(span, 120),
@@ -377,9 +372,10 @@ export function buildExtractPrompt(turns) {
   const transcript = clip(turnsText(turns), 2400);
   return [
     "Extract durable memory candidates as JSON only.",
-    'Shape: {"candidates":[{"type":"preference|entity|decision|experience","name":"","text":"","linkName":"","linkLabel":"about|decided|broke|fixed","fail":"","fix":"","until":"","quote":""}]}',
+    'Shape: {"candidates":[{"type":"preference|entity|decision|experience","name":"","text":"","linkName":"","linkLabel":"about|decided|broke|fixed","fail":"","fix":"","quote":""}]}',
     "Whitelist: 记住/以后都/别再 → preference; repeated proper names → entity; decision with an object (先别动/下周再/采用方案) → decision; tool fail then success in the same turn → experience.",
     "Do not write chitchat or one-off file edits as preference. Do not store secrets, tool dumps, or whole conversations.",
+    "Keep dates in the fact text. Do not infer expiry or emit until/untilAt; expiry is set explicitly through the memory tools or settings.",
     "If nothing matches, return {\"candidates\":[]}.",
     "",
     transcript,
@@ -410,8 +406,17 @@ export function capWriteBatch(candidates, { maxChars = MAX_WRITE_CHARS, maxItems
   return { kept, used, dropped: Math.max(0, (candidates || []).length - kept.length) };
 }
 
-export function applyCandidates(db, candidates, { sessionId, via } = {}) {
-  const { kept: list } = capWriteBatch(
+/** Legacy channel name → the stable provenance kind the UI reports. */
+function sourceKindOfVia(via) {
+  const value = String(via || "");
+  if (value === "chip-add") return "chip";
+  if (value === "workbuddy-import" || value === "external-import") return "import";
+  if (value === "tool") return "tool";
+  if (value === "manual") return "manual";
+  return "extract";
+}
+
+export function applyCandidates(db, candidates, { sessionId, via } = {}) {  const { kept: list } = capWriteBatch(
     filterCandidates(candidates, {
       importFile: via === "workbuddy-import" || via === "external-import",
       explicit: via === "chip-add",
@@ -434,9 +439,14 @@ export function applyCandidates(db, candidates, { sessionId, via } = {}) {
       fix: cand.fix,
       until: cand.until,
       source: {
+        // `via` is the legacy channel name; `kind` is the newer stable field the
+        // provenance UI reads. Only what this path actually knows is recorded —
+        // no message id, author or quote is invented.
+        kind: sourceKindOfVia(via || cand.via),
+        via: via || cand.via || "extract",
         sessionId: sessionId ? String(sessionId) : "",
         quote: cand.quote,
-        via: via || cand.via || "extract",
+        capturedAt: nowIso(),
       },
     };
     const untilAt = cand.until ? resolveUntilAt(cand.until) : undefined;
@@ -481,7 +491,11 @@ export async function distill({ db, turns, sessionId, llmCall, log }) {
   if (typeof llmCall === "function") {
     try {
       const raw = await llmCall(buildExtractPrompt(usable));
-      model = parseModelJson(raw);
+      // Automatic extraction cannot silently turn an event date into expiry,
+      // even when the model emits fields outside the requested schema.
+      model = parseModelJson(raw).map((candidate) => ({
+        ...candidate, until: undefined, untilAt: undefined,
+      }));
     } catch (err) {
       llmFailed = true;
       log?.warn?.(`[dsh-trivium] extract prompt failed: ${err.message}`);