@modusensus/dsh-mneme 0.9.1 → 0.9.2

package/test/dream-maxtokens-fallback.test.js

@@ -0,0 +1,132 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { withMaxTokensFallback, parseMaxTokensCap, MAX_TOKENS_REJECT_RE } from "../src/dream.js";
+
+// Issue #478：dreamMaxTokens 默认 131072 原样发给 OpenAI 兼容网关，很多网关硬
+// 上限更小（实测 dream_runs 180 连败的报错："max_tokens 不能超过 65536"），
+// 400 直接拒、推理没开始。本套锁两件事：上限提取认得出显式上限、认不出时不猜；
+// withMaxTokensFallback 在 stream（result===undefined + getStreamError）与
+// throw 两条路径上都按提取到的上限降级重试恰一次，AbortError 永不被降级吞掉。
+
+function makeLogger() {
+  const warns = [];
+  return { warns, logger: { warn: (...a) => warns.push(a.join(" ")), info() {}, error() {} } };
+}
+
+test("parseMaxTokensCap: 显式上限认得出（中/英/符号三种说法）", () => {
+  assert.equal(parseMaxTokensCap("max_tokens 不能超过 65536"), 65536);
+  assert.equal(parseMaxTokensCap("Maximum value is 65536"), 65536);
+  assert.equal(parseMaxTokensCap("max_tokens must be at most 32768"), 32768);
+  assert.equal(parseMaxTokensCap("max_tokens <= 8192"), 8192);
+  assert.equal(parseMaxTokensCap("max_tokens 不得超过 4096"), 4096);
+});
+
+test("parseMaxTokensCap: 认不出显式上限时返回 null，绝不猜", () => {
+  // 只有请求值、没有上限值——把 131072 误当上限会让重试原样再炸一次
+  assert.equal(parseMaxTokensCap("max_tokens 131072 is too large for this model"), null);
+  assert.equal(parseMaxTokensCap("bad request"), null);
+  assert.equal(parseMaxTokensCap(""), null);
+  assert.equal(parseMaxTokensCap(null), null);
+});
+
+test("MAX_TOKENS_REJECT_RE: 命中 max_tokens/maxTokens 两种写法，不误伤无关报错", () => {
+  assert.ok(MAX_TOKENS_REJECT_RE.test("max_tokens 不能超过 65536"));
+  assert.ok(MAX_TOKENS_REJECT_RE.test("maxTokens exceed limit"));
+  assert.ok(!MAX_TOKENS_REJECT_RE.test("reasoning effort unsupported"));
+});
+
+test("withMaxTokensFallback: stream 路径（undefined + getStreamError）按上限降级重试恰一次", async () => {
+  const { logger, warns } = makeLogger();
+  const ctx = { logger };
+  const calls = [];
+  const attempt = async (tokens) => {
+    calls.push(tokens);
+    // 第一次发大值 → 网关拒收折叠成 undefined；重试用 65536 → 成功
+    return tokens > 65536 ? undefined : `ok@${tokens}`;
+  };
+  const out = await withMaxTokensFallback(ctx, 131072, attempt, () =>
+    "400 max_tokens 不能超过 65536");
+  assert.equal(out, "ok@65536");
+  assert.deepEqual(calls, [131072, 65536], "大值先发，拒收后按上限重试");
+  assert.equal(warns.length, 1, "降级必须留痕");
+  assert.match(warns[0], /131072.*65536/);
+});
+
+test("withMaxTokensFallback: throw 路径同样降级，AbortError 永远直通", async () => {
+  const { logger } = makeLogger();
+  const ctx = { logger };
+  // throw 路径：第一次抛 400，重试成功
+  const calls = [];
+  const out = await withMaxTokensFallback(ctx, 131072, async (tokens) => {
+    calls.push(tokens);
+    if (tokens > 65536) throw new Error("invalid max_tokens: max_tokens 不能超过 65536");
+    return `ok@${tokens}`;
+  }, () => "");
+  assert.equal(out, "ok@65536");
+  assert.deepEqual(calls, [131072, 65536]);
+
+  // AbortError：即使消息里带 max_tokens 字样也绝不触发重试（取消意图不可吞）
+  const abort = new Error("max_tokens rejected during dispose");
+  abort.name = "AbortError";
+  const abCalls = [];
+  await assert.rejects(
+    withMaxTokensFallback(ctx, 131072, async (tokens) => { abCalls.push(tokens); throw abort; }, () => ""),
+    /dispose/
+  );
+  assert.deepEqual(abCalls, [131072], "AbortError 不重试");
+});
+
+test("withMaxTokensFallback: 无上限可提取或上限不小于当前值时不重试", async () => {
+  const { logger } = makeLogger();
+  const ctx = { logger };
+  // 报错只有请求值没有上限 → 不猜，原样 undefined 返回
+  let calls = 0;
+  const out1 = await withMaxTokensFallback(ctx, 131072, async () => { calls++; return undefined; }, () =>
+    "max_tokens 131072 is too large for this model");
+  assert.equal(out1, undefined);
+  assert.equal(calls, 1, "认不出上限就不重试");
+
+  // 上限 >= 当前值（报错自相矛盾）→ 不重试，throw 路径原样抛
+  const boom = new Error("max_tokens 不能超过 131072");
+  await assert.rejects(
+    withMaxTokensFallback(ctx, 131072, async () => { throw boom; }, () => ""),
+    (e) => e === boom,
+    "cap >= maxTokens 时不重试且原样抛"
+  );
+});
+
+test("withMaxTokensFallback: 无关报错不触发重试（stream 与 throw 双路径）", async () => {
+  const { logger } = makeLogger();
+  const ctx = { logger };
+  let calls = 0;
+  const out = await withMaxTokensFallback(ctx, 131072, async () => { calls++; return undefined; }, () =>
+    "reasoning effort UNSUPPORTED_REASONING_EFFORT");
+  assert.equal(out, undefined);
+  assert.equal(calls, 1);
+
+  await assert.rejects(
+    withMaxTokensFallback(ctx, 131072, async () => { throw new Error("quota exhausted"); }, () => ""),
+    /quota exhausted/
+  );
+});
+
+// 接线锁：单测绿不代表真接上了。#478 的第一次实现只挂了 dream 侧三个点，
+// sleep.js 两处仍原样发 config.sleepMaxTokens，而面板 hint 已对外承诺
+// 「超上限同样自动降级」——这种「函数有、没人用」的缺口只靠行为测试抓不到。
+test("接线锁：dreamMaxTokens/sleepMaxTokens 的五个使用点全部包在 withMaxTokensFallback 里", async () => {
+  const { readFile } = await import("node:fs/promises");
+  const srcDir = new URL("../src/", import.meta.url);
+  const files = { "dream.js": await readFile(new URL("dream.js", srcDir), "utf8"), "dream/sleep.js": await readFile(new URL("dream/sleep.js", srcDir), "utf8") };
+  for (const [name, source] of Object.entries(files)) {
+    // 任何一处还把 config.dreamMaxTokens/sleepMaxTokens 直接当 maxTokens 发，就是漏接线。
+    assert.ok(
+      !/maxTokens:\s*config\.(dream|sleep)MaxTokens/.test(source),
+      `${name}: still passes config.*MaxTokens straight into maxTokens — wrap it in withMaxTokensFallback`
+    );
+  }
+  // 五个调用点：dream 三条（叙述/巩固/摘要）+ sleep 两条（冲突/模式）。
+  const dreamHits = files["dream.js"].match(/withMaxTokensFallback\(ctx, config\.dreamMaxTokens/g) ?? [];
+  const sleepHits = files["dream/sleep.js"].match(/withMaxTokensFallback\(ctx, config\.sleepMaxTokens/g) ?? [];
+  assert.equal(dreamHits.length, 3, `dream.js must wrap all 3 dreamMaxTokens sites (got ${dreamHits.length})`);
+  assert.equal(sleepHits.length, 2, `dream/sleep.js must wrap both sleepMaxTokens sites (got ${sleepHits.length})`);
+});

package/test/embed-text-reasons.test.js

@@ -0,0 +1,112 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { embedText, embedTextDetailed, createEmbedder } from "../src/embedding.js";
+
+// distill-tx-embed-gap 的"坏了为什么看不出来"半边：embedText 此前把「配置缺」
+// 「网络挂」「超时」「HTTP 4xx/5xx」「响应不是 JSON」「响应没有向量」全部压成
+// null，回执账本只能统一写 embedding-request-returned-no-vector——分诊不可能。
+// embedTextDetailed 把失败模式拆开；embedText 保留旧 null 契约（调用方不变）。
+
+const FULL = {
+  enabled: true,
+  baseUrl: "https://api.example.com/v1",
+  apiKey: "sk-test",
+  model: "text-embedding-3-small"
+};
+
+async function withFetch(impl, fn) {
+  const original = globalThis.fetch;
+  globalThis.fetch = impl;
+  try {
+    return await fn();
+  } finally {
+    globalThis.fetch = original;
+  }
+}
+
+test("detailed: 成功时 reason 为 null，vector 与旧契约一致", async () => {
+  await withFetch(async () => ({ ok: true, json: async () => ({ data: [{ embedding: [1, 2, 3] }] }) }), async () => {
+    const out = await embedTextDetailed(FULL, "猫咪");
+    assert.deepEqual(out.vector, [1, 2, 3]);
+    assert.equal(out.reason, null);
+    // 旧包装的 null 契约不变
+    assert.deepEqual(await embedText(FULL, "猫咪"), [1, 2, 3]);
+  });
+});
+
+test("detailed: 配置缺失时逐字段点名，不伪装成网络故障", async () => {
+  assert.match((await embedTextDetailed({ ...FULL, model: "" }, "x")).reason, /^config-incomplete:model$/);
+  assert.match((await embedTextDetailed({ ...FULL, apiKey: "" }, "x")).reason, /^config-incomplete:apiKey$/);
+  assert.match((await embedTextDetailed({ ...FULL, baseUrl: "" }, "x")).reason, /^config-incomplete:baseUrl$/);
+  assert.match(await embedTextDetailed(FULL, "").then((o) => o.reason), /config-incomplete:text/);
+});
+
+test("detailed: HTTP 状态码进 reason（401 钥匙 / 404 路径 / 429 限流）", async () => {
+  await withFetch(async () => ({ ok: false, status: 404, text: async () => "Not Found" }), async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.equal(out.vector, null);
+    assert.equal(out.reason, "http-404:Not Found");
+  });
+  await withFetch(async () => ({ ok: false, status: 429, text: async () => "" }), async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.equal(out.reason, "http-429", "空响应体不拼尾巴");
+  });
+  // 旧包装在这些场景全部 null，不抛
+  await withFetch(async () => ({ ok: false, status: 401, text: async () => "denied" }), async () => {
+    assert.equal(await embedText(FULL, "x"), null);
+  });
+});
+
+test("detailed: 超时与网络错误分型，不混为一谈", async () => {
+  await withFetch(async () => {
+    const err = new Error("socket hang up");
+    err.name = "TypeError";
+    throw err;
+  }, async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.match(out.reason, /^request-failed:network:socket hang up/);
+  });
+  await withFetch(async () => {
+    const err = new Error("The operation was aborted");
+    err.name = "TimeoutError";
+    throw err;
+  }, async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.match(out.reason, /^request-failed:timeout:/, "慢和不通是两种病");
+  });
+});
+
+test("detailed: 响应不是 JSON 与响应没有向量各自有名", async () => {
+  await withFetch(async () => ({ ok: true, json: async () => { throw new Error("bad"); } }), async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.equal(out.reason, "response-not-json");
+  });
+  await withFetch(async () => ({ ok: true, json: async () => ({ data: [] }) }), async () => {
+    const out = await embedTextDetailed(FULL, "x");
+    assert.equal(out.reason, "response-no-vector");
+  });
+});
+
+test("embedFor 回执账本带上可区分 reason（不再是统一的 no-vector）", async () => {
+  const receipts = [];
+  const embedder = createEmbedder({
+    store: { setEmbedding() {} },
+    settings: { getVectorConfig: () => FULL },
+    logger: { info() {}, warn() {} }
+  });
+  // embedReceipts 在账本上；rejection 已在 schedule() 内消化，这里直接等 schedule promise。
+  await withFetch(async () => ({ ok: false, status: 404, text: async () => "no route" }), async () => {
+    const r = await embedder.schedule({ id: "m1", title: "t", content: "c" });
+    assert.equal(r.status, "failed");
+    assert.equal(r.reason, "http-404:no route");
+  });
+  await withFetch(async () => ({ ok: true, json: async () => ({ data: [{ embedding: [1, 2, 3] }] }) }), async () => {
+    const r = await embedder.schedule({ id: "m2", title: "t", content: "c" });
+    assert.equal(r.status, "ok");
+    assert.equal(r.dim, 3);
+  });
+  const snap = embedder.embedReceipts();
+  assert.equal(snap.totals.failed, 1);
+  assert.equal(snap.totals.ok, 1);
+  assert.equal(snap.lastFailReason, "http-404:no route", "账本记的是分诊后的原因");
+});

package/test/readme-claims.test.js

@@ -0,0 +1,68 @@
+// 两份 README 曾经都写着「默认全离线」「offline by default」「默认零网络依赖」——
+// 这与代码默认值相反：autoDream 与 autoSummarize 默认开（src/config.js），
+// 两者都把记忆正文交给 ctx.llm.stream(...)（src/dream.js / src/summarize.js）；
+// embedProvider 默认还是 "openai"，只有显式改成 "local"（ONNX）才真正离线。
+// 一句假话比不写更坏：读者会据此以为装完就能断网用，而它默认会把正文发给宿主模型。
+// 所以这里锁两件事：一、绝对离线声明不许回来；二、隐私节必须写出「巩固走宿主模型」这个出站事实——
+// 删掉假话后如果什么都不说，误导是一样的。
+import test from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync, existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import path from "node:path";
+
+const pkgRoot = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
+const repoRoot = path.dirname(pkgRoot);
+const rootReadmePath = path.join(repoRoot, "README.md");
+
+// npm 的 files 清单把整个 test/ 打进包里，所以在 packed / installed 包里跑 npm test 时，
+// 这个文件照样会被执行——但那里没有仓库根 README（只有包内 dsh-mneme/README.md）。
+// 直接 readFileSync 会让整个测试文件在加载阶段 ENOENT 崩掉，把「不是这个仓」报成「断言失败」。
+// 所以仓库根不在时显式 skip：这条守卫的适用面本来就只是仓库，不该在包内假红。
+const hasRepoRoot = existsSync(rootReadmePath);
+const rootReadme = hasRepoRoot ? readFileSync(rootReadmePath, "utf8") : "";
+const pkgReadme = readFileSync(path.join(pkgRoot, "README.md"), "utf8");
+const outsideRepo = hasRepoRoot ? {} : { skip: "not a repo checkout: 仓库根 README 不存在（包内安装态）" };
+
+// 只锁「默认 / 无条件」的说法。带条件的离线表述（把 embedProvider 改成 local、离线巩固论文、
+// 运行时离线取件）都是真话，不在这个列表里，否则守卫会把正确写法也判红。
+const ABSOLUTE_CLAIMS = [
+  "默认全离线",
+  "默认零网络依赖",
+  "offline by default",
+  "Zero network dependency by default",
+  "完全离线的语义记忆引擎",
+];
+
+function section(text, heading) {
+  const start = text.indexOf(heading);
+  assert.ok(start >= 0, `README 里应有「${heading}」一节`);
+  const rest = text.slice(start + heading.length);
+  const next = rest.search(/\r?\n## /);
+  return next === -1 ? rest : rest.slice(0, next);
+}
+
+test("两份 README 都不许再把「离线」写成默认行为", outsideRepo, () => {
+  for (const [name, text] of [
+    ["README.md", rootReadme],
+    ["dsh-mneme/README.md", pkgReadme],
+  ]) {
+    for (const claim of ABSOLUTE_CLAIMS) {
+      assert.ok(!text.includes(claim), `${name} 仍含绝对离线声明：${claim}`);
+    }
+  }
+});
+
+test("隐私节必须写出「巩固与摘要走宿主模型」这个出站事实", outsideRepo, () => {
+  const zh = section(rootReadme, "## 隐私承诺");
+  assert.ok(zh.includes("宿主模型"), "中文隐私节应说明巩固与摘要走宿主模型");
+  assert.ok(
+    zh.includes("autoDream") || zh.includes("autoSummarize"),
+    "中文隐私节应点名 autoDream / autoSummarize，否则读者不知道是谁在出站",
+  );
+  const en = section(rootReadme, "## Privacy");
+  assert.ok(
+    en.includes("host model"),
+    "English Privacy section should disclose that consolidation uses the host model",
+  );
+});