@modusensus/dsh-mneme 0.8.5 → 0.8.7

package/src/document-index.js

@@ -0,0 +1,94 @@
+// dsh-mneme/src/document-index.js
+// documentDir 的索引（issue #296 第二批）。`<documentDir>/index.md` 整文件机器所有，
+// 可从库随时重建：内容里没有任何「生成时间」，所以删掉再同步一次得到逐字节相同的
+// 文件（test/document-dir.test.js 钉住这一条）。
+//
+// 与镜像侧 documents.md 的分工（#296 第 2 节表格）：这里是 document 子系统自己的
+// 索引（id + 标题 + 定位 + managed 标记），那份是带 frontmatter 的镜像只读视图
+// （多一个摘要首句）。两份都只列指针行，都不含正文——正文永远在 doc_path 指向的
+// 文件里，那是 agent 的东西。
+import { readFileSync, mkdirSync, writeFileSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { STR } from "./lang.js";
+import { isManagedDocumentPath } from "./document.js";
+
+function oneLine(text) {
+  return String(text ?? "").replace(/\s+/g, " ").trim();
+}
+
+/**
+ * @param {string} dir resolved documentDir (see resolveDocumentDir)
+ * @param {"zh"|"en"} language instance language, same as the mirror
+ */
+export function createDocumentIndex(dir, language = "zh") {
+  const filePath = join(dir, "index.md");
+
+  /**
+   * Render the whole file from the given rows. Sorted updated_at DESC, id ASC:
+   * the same store state must always render the same bytes (rebuildable).
+   */
+  function render(memories) {
+    const items = (memories ?? []).slice().sort((a, b) => {
+      if (a.updated_at !== b.updated_at) return a.updated_at < b.updated_at ? 1 : -1;
+      return a.id < b.id ? -1 : 1;
+    });
+    const lines = [STR.documentIndexHeader[language]()];
+    for (const m of items) {
+      const where = isManagedDocumentPath(dir, m.doc_path) ? "managed" : "external";
+      // doc_path 理论上必带（store.saveDocument 是唯一铸造口），缺了就只写定位
+      // 标记，不编造路径。
+      const path = m.doc_path ? ` · \`${oneLine(m.doc_path)}\`` : "";
+      lines.push(`- \`${m.id}\` **${oneLine(m.title)}** · ${where}${path}`);
+    }
+    return `${lines.join("\n")}\n`;
+  }
+
+  /**
+   * Ensure the directory exists (recursive). Never throws: callers warn and move
+   * on — a managed directory that cannot be created must not fail plugin boot.
+   */
+  function ensure() {
+    try {
+      mkdirSync(dir, { recursive: true });
+      return { ok: true };
+    } catch (error) {
+      return { ok: false, error: error?.message ?? String(error) };
+    }
+  }
+
+  /**
+   * Write the index if it changed. Never throws: a failed index write must not
+   * fail the business write that triggered it (same fail-safe shape as
+   * mirror.sync). Returns { ok, changed } or { ok: false, error }.
+   */
+  function sync(memories) {
+    const text = render(memories);
+    let prev = null;
+    try { prev = readFileSync(filePath, "utf8"); } catch { /* first write */ }
+    if (prev === text) return { ok: true, changed: false };
+    const ready = ensure();
+    if (!ready.ok) return { ok: false, error: ready.error };
+    try {
+      writeFileSync(filePath, text, "utf8");
+      return { ok: true, changed: true };
+    } catch (error) {
+      return { ok: false, error: error?.message ?? String(error) };
+    }
+  }
+
+  /**
+   * Drop the index file (闸关时用）。闸关意味着 document 子系统整体退出，留一份
+   * 陈旧索引会列出已归档、已遗忘的指针行——正是 documents.md 在闸关时被镜像删掉
+   * 要避免的那种「看着还在、其实已关」的视图。删掉不从库取数，O(1)，也不抛。
+   */
+  function remove() {
+    try {
+      rmSync(filePath, { force: true });
+      return { ok: true };
+    } catch (error) {
+      return { ok: false, error: error?.message ?? String(error) };
+    }
+  }
+
+  return { filePath, render, sync, ensure, remove };
+}

package/src/document.js

@@ -13,7 +13,7 @@
 
 import { statSync } from "node:fs";
 import { homedir } from "node:os";
-import { isAbsolute, resolve } from "node:path";
+import { isAbsolute, resolve, sep } from "node:path";
 import { scopeKeyOf } from "./scope.js";
 import { cosineSimilarity } from "./dream/clustering.js";
 
@@ -23,26 +23,71 @@ import { cosineSimilarity } from "./dream/clustering.js";
 // 新版判成「无近重复」，留下两行活跃摘要漂移——supersede 探测必须全时段。
 const DOCUMENT_MIN_SIM = 0.92;
 const CANDIDATE_LIMIT = 200;
+const DOCUMENT_DIR_NAME = "documents";
+
+/**
+ * 解析 managed 文档目录（#296 第二批）。`configured` 为空 = 跟随 memoryDir 的
+ * `<memoryDir>/documents/`；`~` / `~/` / `~\` 展开到用户 home；相对路径落在
+ * memoryDir 下（本键自己的规则：memoryDir 的相对路径原样留着走 cwd 语义）；
+ * 绝对路径归一化后原样用。纯函数，路径解析集中在 index.js 调用一次。
+ */
+export function resolveDocumentDir(memoryDir, configured = "") {
+  const raw = String(configured ?? "").trim();
+  if (!raw) return resolve(memoryDir, DOCUMENT_DIR_NAME);
+  if (raw === "~" || raw.startsWith("~/") || raw.startsWith("~\\")) {
+    return resolve(homedir(), raw.slice(2));
+  }
+  return isAbsolute(raw) ? resolve(raw) : resolve(memoryDir, raw);
+}
+
+/**
+ * 这个路径是不是落在 managed 目录里（`documentDir` 内 = mneme 管的树，`index.md`
+ * 与镜像 md 同权；之外 = 只登记指针行、正文一个字节都不碰）。比较前两边都归一化，
+ * Windows 上大小写不敏感（文件系统就是），否则 `C:\a\documents\x.md` 与
+ * `c:\A\Documents\x.md` 会被判成两类。
+ */
+export function isManagedDocumentPath(dir, path) {
+  if (!dir || !path) return false;
+  const norm = (p) => (process.platform === "win32" ? String(p).toLowerCase() : String(p));
+  const base = norm(resolve(dir));
+  const target = norm(resolve(path));
+  const prefix = base.endsWith(sep) ? base : base + sep;
+  return target === base || target.startsWith(prefix);
+}
 
 // 追加到被取代行正文的指针注记。刻意不带语言分支：id 是跨语言稳定键（用户
 // 对 tag/章节名建议用英文同理——指针性内容不翻译）；可追溯链 = 注记里的新行
 // id + 旧行 content_history 里的旧摘要（source=superseded）。
 const supersededByPointer = (id) => `\n\n[superseded by ${id}]`;
 
+// 升格吸收不碰的类型（#275 拍板 5 的边界）：吸收的对象是「原子条」，而下面这两种
+// 行各有自己的生命周期，被归档会各自留下第二行——
+//   document：本体是磁盘上的文件。归档它等于替用户撤下外档；而且 supersede 探测
+//     只看活跃行（store.list 默认排除归档），同一 path 下次注册会在旧行还挂着
+//     「新版本」语义时再铸一行，指针注记与 content_history 记账也走不到。
+//   summary：dream 总览与叙述按 source 身份去重、并按注入档位常驻。归档它会让下一
+//     次做梦把它当不存在（候选集同样只取活跃行），于是库里多出一行同源总览。
+// 口径是保守方向的：拿不准的行一律不吸收——少做一步没有代价，收错一步是内容事故。
+const ABSORB_EXEMPT_TYPES = new Set(["document", "summary"]);
+
 /**
  * Build the document registrar. Injected deps are service.js closure members
  * (store / embedQuery / transaction / finalize) so this module stays free of
  * service-internal wiring; `finalize` is the single write epilogue (mirror
  * sync + notify + re-embed) so callers never duplicate it.
  *
- * @returns {(payload: object) => Promise<object>}
+ * `pinnedTypes` 是注入的 #249 逐字保真池（service.js 的 PINNED_MEMORY_TYPES）：
+ * 这里不 import 它在 service.js 里，是因为 service.js 反向 import 本模块——
+ * 方向反了会成环。
+ *
+ * @returns {(payload: object, opts?: object) => Promise<object>}
  *   `{ action: "created"|"superseded", memory, superseded?, evidence_kept,
- *      evidence_dropped, degraded }`
+ *      evidence_dropped, evidence_archived, degraded }`
  * @throws flag 关 / 路径非法或文件缺失 / title 或 summary 为空 / evidence 全部
  *   捏造 / 仅 vector 近重复（疑似重复外档，交 agent 裁决而不是静默二选一）。
  */
-export function createDocumentRegistrar({ store, config, embedQuery, pushContentHistory, transaction, finalize }) {
-  return async function registerDocument(payload, { hiddenEvidenceIds = [] } = {}) {
+export function createDocumentRegistrar({ store, config, embedQuery, pushContentHistory, transaction, finalize, pinnedTypes }) {
+  return async function registerDocument(payload, { hiddenEvidenceIds = [], archiveEvidence = true } = {}) {
     // opt-in 总闸（#230 对齐 #228 形态）：关 = 注册入口整体不存在，错误信息
     // 指回配置键，agent 能准确转告用户去开。
     if (config?.documentMemoryEnabled !== true) {
@@ -90,15 +135,14 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
     const hidden = new Set((Array.isArray(hiddenEvidenceIds) ? hiddenEvidenceIds : []).map((id) => String(id)));
     const kept = [];
     const dropped = [];
+    // 存在但已归档的引用另记一份：它可能是被上一版文档升格吸收走的行（见下方
+    // supersede 目标探测后的「认回」），不能一律按「不可用」处理。
+    const archivedRefs = new Set();
     for (const id of wanted) {
       const row = hidden.has(id) ? null : store.getById(id);
+      if (row?.archived) archivedRefs.add(id);
       (row && !row.archived ? kept : dropped).push(id);
     }
-    if (wanted.length > 0 && kept.length === 0) {
-      throw new Error(
-        `registerDocument: all ${wanted.length} evidence ids are unknown, archived or out of scope — fabricated evidence is rejected`
-      );
-    }
 
     const title = String(payload?.title ?? "").trim();
     if (!title) throw new Error("registerDocument: title is required");
@@ -131,6 +175,25 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
       .filter((m) => scopeMatches(m));
     const explicitTarget = scopeMatched.find((m) => samePath(m.doc_path, expanded))
       ?? scopeMatched.find((m) => m.title.trim() === title);
+    // 升格吸收过的引用认回（#275 拍板 5 的配套）：evidence 行在文档注册成功后就翻了
+    // archived，于是「同一份文档出新版、evidence 照旧」这一最常规的路径会整批落在
+    // dropped——那会被下面的捏造判据误报成「证据都是编的」。口径：已归档行仍可为
+    // **吸收它的那份文档**背书（loser.evidence 就是它吸收走的名单），不为别的文档
+    // 背书；捏造与跨 scope 照旧拒绝。
+    if (explicitTarget) {
+      const own = new Set(explicitTarget.evidence ?? []);
+      for (const id of archivedRefs) {
+        if (!own.has(id)) continue;
+        const at = dropped.indexOf(id);
+        if (at >= 0) dropped.splice(at, 1);
+        kept.push(id);
+      }
+    }
+    if (wanted.length > 0 && kept.length === 0) {
+      throw new Error(
+        `registerDocument: all ${wanted.length} evidence ids are unknown, archived or out of scope — fabricated evidence is rejected`
+      );
+    }
     if (!explicitTarget) {
       // vector 档：embedder 不可用/向量缺失一律跳过——去重是增强不是写入依赖
       // （findSessionDuplicate 同原则）。probe 用 title+summary，与行向量
@@ -206,7 +269,25 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
         store.setArchived(loser.id, true);
         superseded = store.getById(loser.id);
       }
-      return { created, superseded };
+      // #275 拍板 5（#230 验收口径的补丁）：升格吸收的 evidence 行随之退出活跃面——
+      // 一次吸收 20 条，库里就是「21 行不是 1 行」。四条口径：①与本行同一事务（要么
+      // 都成、要么都不成）；②pinned（constraint / preference，即 #249 的逐字保真池）
+      // 永不自动归档——升格不能绕过它；③自有生命周期的类型（document / summary，见
+      // ABSORB_EXEMPT_TYPES）不吸收，吸收的是原子条；④只翻标志位、内容与审计全留
+      // （可恢复），opt-out 走 archiveEvidence。
+      // pinnedTypes 拿不到就不做这一步：宁可少做，也不能把保真池当普通行收走。
+      let evidenceArchived = 0;
+      if (archiveEvidence && pinnedTypes && typeof pinnedTypes.has === "function") {
+        for (const id of kept) {
+          // 事务内重读：期间被别的进程归档/遗忘的行不重复计数，也不误伤 pinned。
+          const row = store.getById(id);
+          if (!row || row.archived || row.forgotten) continue;
+          if (pinnedTypes.has(row.type) || ABSORB_EXEMPT_TYPES.has(row.type)) continue;
+          store.setArchived(id, true);
+          evidenceArchived += 1;
+        }
+      }
+      return { created, superseded, evidenceArchived };
     });
     finalize(result.superseded ? [result.created, result.superseded] : [result.created]);
     return {
@@ -215,6 +296,7 @@ export function createDocumentRegistrar({ store, config, embedQuery, pushContent
       ...(result.superseded ? { superseded: result.superseded } : {}),
       evidence_kept: kept.length,
       evidence_dropped: dropped.length,
+      evidence_archived: result.evidenceArchived,
       degraded: dropped.length > 0
     };
   };

package/src/dream/sleep.js

@@ -49,12 +49,22 @@ function parseJsonArray(text) {
   }
 }
 
-/** Same stream consumption contract as dream.js. */
-async function streamText(ctx, options, onStreamError) {
+/**
+ * Same stream consumption contract as dream.js.
+ *
+ * `onUsage`（optional, issue #250）receives any usage chunk for token accounting.
+ * dream.js 那份一直有它；sleep 这份副本没有，导致 conflict / pattern 两条链路的
+ * token 从未进 llm_audit_logs（面板因此只看到 autoDream 与 autoSummarize）。
+ */
+async function streamText(ctx, options, onUsage, onStreamError) {
   if (!ctx?.llm?.stream) return undefined;
   let text = "";
   for await (const chunk of ctx.llm.stream(options)) {
     if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
+    // DSH 的 StreamChunk 契约把用量嵌在 chunk.usage（TokenUsage：inputTokens /
+    // outputTokens / totalTokens）；兼容直接平铺在 chunk 上的旧形态。归一放在
+    // 这里，调用侧的 reporter 只需认平铺形状——与 dream.js:239 同一处理。
+    if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
     if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
       // Same rc.1 error-as-finish-chunk behavior as dream.js — surface the
       // cause instead of discarding it.
@@ -67,6 +77,79 @@ async function streamText(ctx, options, onStreamError) {
   return text;
 }
 
+/**
+ * Issue #250：给 sleep 的一条 LLM 调用记一笔 llm_audit_logs（token / 时长 / 状态 /
+ * 触发源）。与 dream.js 的 runAuditedLlm 同口径但各自独立一份——dream 那两个
+ * checker 的调用形状绑定着它自己的排程细节，硬抽共享抽象只会收敛回 dream 的形态。
+ *
+ * 记账是 best-effort：写审计行失败只 warn，绝不反噬 sleep 本体（CONTRIBUTING 的
+ * fail-safe 硬约定）。`body(reportUsage)` 负责真实消费流，usage chunk 经
+ * reportUsage 上报。
+ *
+ * 审计诚实性：流被中止/报错时 body 返回 undefined，记 status='error'；流式成功但
+ * 输出不可用时 `spec.auditError` 给出原因，同样记 error——否则同一轮调用会在
+ * 「phase 报 failed」与「audit 报 success」之间自相矛盾（CONTRIBUTING 的审计诚实性
+ * 硬约定）。`auditError` 必须是**纯判据函数**：调用方的控制流不能依赖它的副作用，
+ * 因为审计关闭时这里整段早退、钩子根本不会被调用（dream 正是那样踩的，见 PR 正文）。
+ */
+async function auditedSleepLlm(ctx, service, config, logger, spec, body) {
+  if (config?.llmAudit?.enabled === false || typeof service?.saveLlmAudit !== "function") {
+    return body(() => {});
+  }
+  const startedAt = Date.now();
+  const timestamp = new Date(startedAt).toISOString();
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let status = "success";
+  let errorMessage = null;
+  try {
+    const result = await body((usage) => {
+      if (!usage) return;
+      const i = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens ?? usage.promptTokens;
+      const o = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens ?? usage.completionTokens;
+      if (Number.isFinite(i)) inputTokens = i;
+      if (Number.isFinite(o)) outputTokens = o;
+    });
+    if (result === undefined) {
+      status = "error";
+      const streamErr = typeof spec.streamError === "function" ? String(spec.streamError() ?? "") : "";
+      errorMessage = streamErr ? `llm stream aborted or errored (${streamErr})` : "llm stream aborted or errored";
+    } else if (typeof spec.auditError === "function") {
+      // 流式成功但输出不可用，仍然是一次失败的调用——记 error，不与调用方的
+      // failed 返回值打架。
+      const message = spec.auditError(result);
+      if (message) {
+        status = "error";
+        errorMessage = message;
+      }
+    }
+    return result;
+  } catch (error) {
+    status = "error";
+    errorMessage = String(error?.message ?? error);
+    throw error;
+  } finally {
+    try {
+      service.saveLlmAudit({
+        timestamp,
+        trigger_source: spec.triggerSource,
+        operation_type: spec.operationType,
+        model_id: spec.modelId,
+        input_tokens: inputTokens,
+        output_tokens: outputTokens,
+        total_tokens: inputTokens + outputTokens,
+        cost_usd: 0,
+        duration_ms: Date.now() - startedAt,
+        status,
+        error_message: errorMessage,
+        related_memory_ids: spec.relatedMemoryIds ?? []
+      });
+    } catch (auditError) {
+      logger?.warn?.(`dsh-mneme sleep: llm audit write failed: ${String(auditError)}`);
+    }
+  }
+}
+
 /** LLM route (Issue #25): explicit sleepProvider/Model wins, then the dream
  *  route as a shared explicit fallback, then the agent default model. Sleep
  *  can pin a cheaper model for its bulk passes without disturbing the dream
@@ -232,7 +315,19 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
   const conflictPrompt = actionSet === "full" ? STR.prompts.conflictFull[language] : STR.prompts.conflict[language];
   const runConflict = (withEffort) => {
     conflictStreamFailure = "";
-    return streamText(ctx, {
+    return auditedSleepLlm(ctx, service, config, logger, {
+      triggerSource: "sleep",
+      operationType: "sleep_conflict",
+      modelId: `${route.provider}:${route.model}`,
+      // 候选对的双方 id：面板的活动流用 related_memory_ids 解析「沉淀条数」。
+      relatedMemoryIds: samePairs.flatMap((p) => [p.a.id, p.b.id]),
+      streamError: () => conflictStreamFailure,
+      // 纯判据：输出里没有 JSON 数组 ⇒ 这轮是失败的调用，审计记 error。**不要**把
+      // 解析结果存进闭包给下面的控制流用——审计关闭时 auditedSleepLlm 整段早退、
+      // 这个钩子不会被调用，控制流会跟着失效（dream 的既有坑）。代价是审计开启时
+      // 多解析一次，换来两条路径彻底解耦。
+      auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
+    }, (reportUsage) => streamText(ctx, {
     provider: route.provider,
     model: route.model,
     purpose: "sleep-conflict",
@@ -242,7 +337,7 @@ async function phaseConflicts(ctx, service, config, logger, runId, semantic = nu
       { role: "system", content: [{ type: "text", text: conflictPrompt }], source: { kind: "plugin", plugin: "dsh-mneme" } },
       { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
     ]
-  }, (reason) => { conflictStreamFailure = describeStreamFailure(reason); });
+  }, reportUsage, (reason) => { conflictStreamFailure = describeStreamFailure(reason); }));
   };
   const text = await withEffortFallback(ctx, sleepEffort, () => runConflict(true), () => runConflict(false), () => conflictStreamFailure);
   if (text === undefined) {
@@ -385,7 +480,16 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
   let patternStreamFailure = "";
   const runPattern = (withEffort) => {
     patternStreamFailure = "";
-    return streamText(ctx, {
+    return auditedSleepLlm(ctx, service, config, logger, {
+      triggerSource: "sleep",
+      operationType: "sleep_pattern",
+      modelId: `${route.provider}:${route.model}`,
+      // 本轮扫描的记忆 id：与 dream 的 relatedMemoryIds 同义，供面板解析沉淀条数。
+      relatedMemoryIds: memories.map((m) => m.id),
+      streamError: () => patternStreamFailure,
+      // 纯判据，同 conflict 阶段：不把解析结果存进闭包供控制流使用。
+      auditError: (text) => (parseJsonArray(text) ? null : "no json array in llm output")
+    }, (reportUsage) => streamText(ctx, {
     provider: route.provider,
     model: route.model,
     purpose: "sleep-pattern",
@@ -395,7 +499,7 @@ async function phasePatterns(ctx, service, config, logger, runId, signal = null)
       { role: "system", content: [{ type: "text", text: STR.prompts.pattern[language].replace("N", String(maxPatterns)) }], source: { kind: "plugin", plugin: "dsh-mneme" } },
       { role: "user", content: [{ type: "text", text: listText }], source: { kind: "plugin", plugin: "dsh-mneme" } }
     ]
-  }, (reason) => { patternStreamFailure = describeStreamFailure(reason); });
+  }, reportUsage, (reason) => { patternStreamFailure = describeStreamFailure(reason); }));
   };
   const text = await withEffortFallback(ctx, sleepEffort, () => runPattern(true), () => runPattern(false), () => patternStreamFailure);
   if (text === undefined) {
@@ -507,6 +611,9 @@ function deriveStatus(phases) {
  */
 export async function runSleep(ctx, service, config, logger, semantic = null, signal = null) {
   const runId = randomUUID();
+  // Issue #89：开跑时刻——审计行 created_at 的基准（= 冷却种子），run 耗时
+  // （LLM 各 phase）不应计入重启后的冷却窗口。
+  const sleepStartedAt = Date.now();
   const phases = {};
   const attempt = async (name, fn) => {
     if (signal?.aborted) return; // user resumed activity — stop before next phase
@@ -543,6 +650,9 @@ export async function runSleep(ctx, service, config, logger, semantic = null, si
   try {
     service.saveDreamRun({
       id: runId,
+      // Issue #89：审计行记开跑时刻而非完成时刻——run 耗时（含 LLM 各 phase）
+      // 不该计入重启后的 sleep 冷却种子。
+      created_at: new Date(sleepStartedAt).toISOString(),
       status,
       provider: route?.provider,
       model: route?.model,
@@ -582,10 +692,14 @@ export function createSleepScheduler({
   onRun = null,
   now = () => Date.now(),
   setTimeoutFn = setTimeout,
-  clearTimeoutFn = clearTimeout
+  clearTimeoutFn = clearTimeout,
+  // Issue #89 同族：冷却时刻由调用方从 dream_runs 审计表恢复（run_type='sleep'）——
+  // 内存变量进程重启即归零，冷却闸对新实例放行 → 重启后立即进入 sleep 冷却盲区。
+  // 测试注入 0（默认）即保持旧行为。
+  lastRunAtSeed = 0
 }) {
   let lastWriteAt = now();
-  let lastRunAt = 0;
+  let lastRunAt = lastRunAtSeed;
   let running = false;
   let disposed = false;
   let idleTimer = null;