@kenz1117/dsh-engram 0.13.10 → 0.13.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -5,7 +5,7 @@ import { BlockAssembler, createUserMessage } from "@deepseek-ai/dsh-llm";
5
5
  import { homedir } from "node:os";
6
6
  import z from "@deepseek-ai/schemastery";
7
7
  import { execFileSync } from "node:child_process";
8
- import { fileURLToPath } from "node:url";
8
+ import { fileURLToPath, pathToFileURL } from "node:url";
9
9
  import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
10
10
  import { createGzip, gunzip } from "node:zlib";
11
11
  import { promisify } from "node:util";
@@ -195,6 +195,7 @@ const zh$5 = {
195
195
  "core.curatedHeader": "Curated profile (user-maintained via engram_profile_edit; authoritative over derived facts):",
196
196
  "core.observationsHeader": "Consolidated beliefs (distilled from multiple stored memories; verify against sources before high-stakes use):",
197
197
  "core.observationItem": "- {belief} ({count} sources)",
198
+ "core.profileFolded": "(另含 {count} 条相似条目)",
198
199
  "core.reviewDue": "\nPalace review due today: {count}{plus} memories. Use engram_review_queue for active recall (recall beats re-reading).",
199
200
  "core.reflectPending": "\n巩固待处理:有 {count} 组会话登记的记忆还没蒸馏。调用 engram_reflect 整理——只摄取不整理,记忆就是散的。",
200
201
  "core.compressSystem": "把记忆条目压缩为更短的一句话表述(每条不超过 40 个字符),保留可跨会话复用的关键信息(事实、偏好、决策、方法)。\n只输出一个 JSON 数组,每项形如 {\"id\": \"原样返回的id\", \"content\": \"压缩后表述\"},条目数量与 id 必须与输入一一对应。\n不要输出 JSON 以外的任何内容。",
@@ -255,6 +256,7 @@ const en$5 = {
255
256
  "core.curatedHeader": "Curated profile (user-maintained via engram_profile_edit; authoritative over derived facts):",
256
257
  "core.observationsHeader": "Consolidated beliefs (distilled from multiple stored memories; verify against sources before high-stakes use):",
257
258
  "core.observationItem": "- {belief} ({count} sources)",
259
+ "core.profileFolded": "(+{count} folded)",
258
260
  "core.reviewDue": "\nPalace review due today: {count}{plus} memories. Use engram_review_queue for active recall (recall beats re-reading).",
259
261
  "core.reflectPending": "\nConsolidation pending: {count} session registration(s) hold memories not yet distilled. Call engram_reflect to consolidate them — ingested memories stay scattered until reflected.",
260
262
  "core.warnIngestWithoutReflect": "[dsh-engram] ingest is on but reflect is off — memories accumulate without consolidation (they read scattered). Set reflect: auto (or suggest), or run engram_reflect regularly.",
@@ -554,6 +556,7 @@ const zh$2 = {
554
556
  "store.migrationAlreadyMigrated": "[dsh-engram] 旧库已有迁移记录({names},先前归属 {cwd}):{pointer};先前迁移的库可能含有当前目录的记忆。{manual}",
555
557
  "store.mergeTargetMissing": "write-disposition: 并入目标 {id} 不存在",
556
558
  "store.embedderDownloadFailed": "嵌入模型下载/加载失败;检索已降级为纯关键词模式,可检查网络后重试",
559
+ "store.embedderEngineMissing": "本地嵌入引擎未随插件安装(lib/engine 缺失)——安装包不完整或更新被中途截断,请重新安装插件;检索已降级为纯关键词模式",
557
560
  "store.embedderDimensionMismatch": "嵌入向量维度不一致:库里是 {expected} 维,嵌入器给出 {found} 维。比较与写入直接拒绝,而不是逐维截断——请把模型改回去,或用新模型重建库中向量",
558
561
  "store.embedderInvalidVector": "嵌入向量含非有限分量(NaN/∞),写入被拒——这样的向量落库后余弦恒不可比,整条记忆会从语义检索里静默消失",
559
562
  "store.restoreBusyWriters": "检测到 {file} 仍有活动连接在写入(可能是另一个打开着的 DeepSeek 窗口),已取消恢复——请先关闭其他窗口再试,否则那些窗口的写入会丢",
@@ -606,6 +609,7 @@ const en$2 = {
606
609
  "store.migrationAlreadyMigrated": "[dsh-engram] the legacy database already has a migration record ({names}, claimed earlier by {cwd}): {pointer}; the database migrated earlier may hold memories of the current directory. {manual}",
607
610
  "store.mergeTargetMissing": "write-disposition: merge target {id} does not exist",
608
611
  "store.embedderDownloadFailed": "embedding model download/load failed; retrieval has degraded to keyword-only mode — check the network and retry",
612
+ "store.embedderEngineMissing": "the local embedding engine was not installed with the plugin (lib/engine is missing) — the package is incomplete or the update was cut short; reinstall the plugin; retrieval has degraded to keyword-only mode",
609
613
  "store.embedderDimensionMismatch": "embedding dimension mismatch: the store holds {expected}-dimensional vectors, the embedder returned {found}. Comparison and writes are refused instead of being silently truncated — switch the model back, or re-embed the store with the new one",
610
614
  "store.embedderInvalidVector": "embedding vector contains a non-finite component (NaN/∞); the write is refused — such a vector makes cosine comparisons meaningless and the memory would silently vanish from semantic retrieval",
611
615
  "store.restoreBusyWriters": "{file} still has a live connection writing (likely another open DeepSeek window); restore cancelled — close the other windows first, otherwise their writes would be lost",
@@ -860,7 +864,7 @@ const zh$1 = {
860
864
  "tool.reviewSectionRelated": "关联",
861
865
  "tool.reviewRevisions": "\n修订历史:\n{entries}",
862
866
  "tool.reviewOperations": "\n最近操作:\n{entries}",
863
- "tool.statsDescription": "记忆库统计:各状态与种类数量、关系边数、信噪比、操作日志量,以及房间目录(房名/桩位数/最新门牌——检索前先看目录决定进哪个房间,配 engram_search 的 room 参数)。scope=all 时合并两库。",
867
+ "tool.statsDescription": "记忆库统计:各状态与种类数量、关系边数、信噪比、操作日志量,以及房间目录(房名/桩位数/最新门牌——检索前先看目录决定进哪个房间,配 engram_search 的 room 参数)。scope=all 时合并三库。",
864
868
  "tool.statsRoomLine": " {room}({label}): {count}/{maxIndex} 桩{placard}",
865
869
  "tool.statsRoomPlacard": " · 最新门牌「{placard}」",
866
870
  "tool.statsTotal": "[{scope}] 总数 {total}(active {active} / archived {archived} / forgotten {forgotten})",
@@ -899,6 +903,7 @@ const zh$1 = {
899
903
  "tool.distillNoRoute": "engram_distill: 无法确定模型路由(会话尚无模型请求),请在 cordis.yml 配置 provider/model",
900
904
  "tool.distillDone": "蒸馏完成:取材 {input} 条,产出 {distilled} 条高层规律,归档 {superseded} 条旧记忆(supersedes 链已建立,可 engram_review 审计)。",
901
905
  "tool.distillNoVectors": "{count} 条产物没有写入向量:嵌入器当时不可用,这些记录将在维护卡片中显示为「有条目缺向量」,可用「执行」补齐。",
906
+ "tool.distillFailedLine": "engram_distill:本轮有 {count} 条因 LLM 输出上限未能提炼;条目保持 active,下次蒸馏会再次取材。",
902
907
  "tool.reflectDescription": "信念巩固:把多条原始记忆经辅助 LLM 巩固为带证据数的高层信念(form 新建/refine 细化/still 维持/refute 否定),同主题自动归并;原始记忆保留不归档。反复出现的模式或新增一批记忆后调用。",
903
908
  "tool.reflectNoLlm": "engram_reflect: 辅助 LLM 不可用(宿主未提供 llm 服务),无法巩固",
904
909
  "tool.reflectNoRoute": "engram_reflect: 无法确定模型路由(会话尚无模型请求),请在 cordis.yml 配置 provider/model",
@@ -1194,7 +1199,7 @@ const en$1 = {
1194
1199
  "tool.reviewSectionRelated": "Related",
1195
1200
  "tool.reviewRevisions": "\nRevision history:\n{entries}",
1196
1201
  "tool.reviewOperations": "\nRecent operations:\n{entries}",
1197
- "tool.statsDescription": "Memory store statistics: counts by status and kind, number of relation edges, signal-to-noise ratio, operation-log volume, and the room guide (room name / slot count / latest placard — check the guide before retrieval to decide which room to enter, together with the room parameter of engram_search). With scope=all the two stores are merged.",
1202
+ "tool.statsDescription": "Memory store statistics: counts by status and kind, number of relation edges, signal-to-noise ratio, operation-log volume, and the room guide (room name / slot count / latest placard — check the guide before retrieval to decide which room to enter, together with the room parameter of engram_search). With scope=all all three stores are merged.",
1198
1203
  "tool.statsRoomLine": " {room} ({label}): {count}/{maxIndex} slots{placard}",
1199
1204
  "tool.statsRoomPlacard": " · latest placard “{placard}”",
1200
1205
  "tool.statsTotal": "[{scope}] total {total} (active {active} / archived {archived} / forgotten {forgotten})",
@@ -1233,6 +1238,7 @@ const en$1 = {
1233
1238
  "tool.distillNoRoute": "engram_distill: cannot determine the model route (the session has no model request yet); configure provider/model in cordis.yml",
1234
1239
  "tool.distillDone": "Distillation complete: drew on {input} memories, produced {distilled} higher-level rules and archived {superseded} old memories (the supersedes chain is in place and can be audited with engram_review).",
1235
1240
  "tool.distillNoVectors": "{count} produced records were written without a vector: the embedder was unavailable at that moment, so they show up as “records without a vector” on the maintenance card and the Run button fills them in.",
1241
+ "tool.distillFailedLine": "engram_distill: {count} entries left undistilled this run (LLM output budget); they stay active and are retried next run.",
1236
1242
  "tool.reflectDescription": "Belief consolidation: use the auxiliary LLM to consolidate several raw memories into a higher-level belief carrying an evidence count (form / refine / still / refute), merging same-topic beliefs automatically; the raw memories are kept, not archived. Call it after recurring patterns appear or a new batch of memories is added.",
1237
1243
  "tool.reflectNoLlm": "engram_reflect: the auxiliary LLM is unavailable (the host provides no llm service), cannot consolidate",
1238
1244
  "tool.reflectNoRoute": "engram_reflect: cannot determine the model route (the session has no model request yet); configure provider/model in cordis.yml",
@@ -2142,7 +2148,7 @@ const en = {
2142
2148
  logFilterIngest: "Capture",
2143
2149
  logFilterRetrieve: "Retrieval",
2144
2150
  logFilterOrganize: "Organize",
2145
- logCount: "{n} entries · both stores merged, newest first",
2151
+ logCount: "{n} entries · all stores merged, newest first",
2146
2152
  palaceAskTitle: "Which language will you use to talk to the agent?",
2147
2153
  palaceAskBody: "We will set the Memory Palace up to work efficiently in that language. You can change the interface language of the Memory Palace plugin at any time yourself.",
2148
2154
  palaceAskFallback: "The palace has no bundled set of texts for your language yet: it will be recorded as {language} and will take its service texts from English — it will still understand notes written in your language. Ask the agent to print the instruction we keep for it inside the plugin in your native language: it will help you set the Memory Palace up for comfortable work in a couple of minutes.",
@@ -2462,7 +2468,7 @@ const en = {
2462
2468
  pageHeaderEpisodesSubtitle: "Sessions grouped by episode, with adjacent-window and summary recall",
2463
2469
  pageHeaderEntitiesSubtitle: "Entity dictionary (people/projects/tools/concepts), filter by kind and name",
2464
2470
  pageHeaderBeliefsSubtitle: "Consolidated beliefs with evidence chain, click \"n evidence\" for read-only recall",
2465
- pageHeaderLogSubtitle: "Merged operation log across both DBs, filterable by category",
2471
+ pageHeaderLogSubtitle: "Merged operation log across all DBs, filterable by category",
2466
2472
  pageHeaderBackfillSubtitle: "Distil historical sessions into the palace, estimate first then run, resumable",
2467
2473
  pageHeaderJevSubtitle: "Jev system-one adjudication, config overrides and recent decisions log",
2468
2474
  heatmapTitle: "Last 26 weeks activity",
@@ -3458,7 +3464,7 @@ const zh = {
3458
3464
  logFilterIngest: "发掘",
3459
3465
  logFilterRetrieve: "检索",
3460
3466
  logFilterOrganize: "整理",
3461
- logCount: "{n} 条 · 两库合并倒序",
3467
+ logCount: "{n} 条 · 三库合并倒序",
3462
3468
  palaceAskTitle: "您打算用哪种语言与代理对话?",
3463
3469
  palaceAskBody: "我们会把记忆宫殿配置成在该语言下高效工作。记忆宫殿插件的界面语言您可以随时自行切换。",
3464
3470
  palaceAskFallback: "宫殿目前还没有您语言的内置文本:它会被记录为 {language},服务性文字先用英文——用您的语言写下的记录它仍然看得懂。请让代理用您的母语输出我们为它保存在插件中的说明:它会在几分钟内帮您把记忆宫殿配置到舒适可用的状态。",
@@ -3778,7 +3784,7 @@ const zh = {
3778
3784
  pageHeaderEpisodesSubtitle: "按会话分组的经历时间线,支持邻近扩展与摘要回看",
3779
3785
  pageHeaderEntitiesSubtitle: "实体词典:人物/项目/工具/概念,按类别与名称筛选",
3780
3786
  pageHeaderBeliefsSubtitle: "巩固信念与证据链,点证据 n 条展开只读回查",
3781
- pageHeaderLogSubtitle: "两库合并的完整操作日志,按类别筛选",
3787
+ pageHeaderLogSubtitle: "三库合并的完整操作日志,按类别筛选",
3782
3788
  pageHeaderBackfillSubtitle: "把历史会话逐轮提炼进宫殿,先估算再执行,可暂停续做",
3783
3789
  pageHeaderJevSubtitle: "Jev 系统一裁决,配置覆盖与近期裁决观测",
3784
3790
  heatmapTitle: "近半年活动热力",
@@ -5806,10 +5812,174 @@ function resolveRoomInput(input) {
5806
5812
  }
5807
5813
  }
5808
5814
  }
5815
+ // src/embedder/engine.ts
5816
+ /**
5817
+ * WASM 嵌入引擎的解析与装载(0.13.11「引擎出树」)。
5818
+ *
5819
+ * 引擎四个文件随包放在 `lib/engine/`(scripts/copy-engine.mjs 从 devDependencies
5820
+ * 拷入):transformers.js 的 web 构建、自含 wasm 后端的 ORT bundle、ORT 的
5821
+ * emscripten 工厂与 wasm 二进制。技术路线来自 /tmp/wasm-spike/spike-web.mjs 的
5822
+ * 实跑验证,四步注入缺一不可:
5823
+ *
5824
+ * 1. Node 下 web 构建的 ONNX 后端是空壳(node:fs 被 stub),必须先把完整的
5825
+ * ORT 挂到 `globalThis[Symbol.for('onnxruntime')]`,再 import 引擎入口;
5826
+ * 2. ORT 不能按包名 import(exports map 会解析到 onnxruntime-node 原生构建),
5827
+ * 只能从 ort.bundle.min.mjs 的 file: URL 进;
5828
+ * 3. wasm 二进制 readFileSync 后直接注入 `ORT.env.wasm.wasmBinary`,
5829
+ * 运行期不发起任何 wasm 网络加载;
5830
+ * 4. 工厂 mjs 以 file: URL 动态 import。
5831
+ *
5832
+ * 模块级只注入一次:globalThis 挂载与 wasmBinary 都是幂等写,重复注入徒增
5833
+ * 28MB 拷贝;同一 URL 的 ESM import 只求值一次,后续调用拿到的是同一模块。
5834
+ * @module @kenz1117/dsh-engram/embedder/engine
5835
+ */
5836
+ /**
5837
+ * 引擎目录的完备清单(与 scripts/copy-engine.mjs 的产出一致)。末两项是解析
5838
+ * shim:transformers web 构建的真实静态 import("onnxruntime-web/webgpu" 与
5839
+ * "onnxruntime-common" 的 { Tensor })需要包名 Specifier 可解析,缺了它们引擎
5840
+ * 在 import 时就 ERR_MODULE_NOT_FOUND。
5841
+ */
5842
+ const ENGINE_ENTRY = "transformers.web.js";
5843
+ const ORT_MODULE_FILE = "ort.bundle.min.mjs";
5844
+ const ORT_FACTORY_FILE = "ort-wasm-simd-threaded.jsep.mjs";
5845
+ const ORT_WASM_FILE = "ort-wasm-simd-threaded.jsep.wasm";
5846
+ const ENGINE_FILES = [
5847
+ ENGINE_ENTRY,
5848
+ ORT_MODULE_FILE,
5849
+ ORT_FACTORY_FILE,
5850
+ ORT_WASM_FILE,
5851
+ ...["onnxruntime-web", "onnxruntime-common"].map((packageName) => `node_modules/${packageName}/package.json`)
5852
+ ];
5853
+ /** 推导的引擎目录在「文件全在」时才成立:半个引擎比没有引擎更难排查。 */
5854
+ function isCompleteEngineDir(dir) {
5855
+ return ENGINE_FILES.every((name) => existsSync(join(dir, name)));
5856
+ }
5857
+ /** 把 file: URL 换成 fs 路径:候选目录一律落回路径再检查,不带尾随分隔符。 */
5858
+ function dirOf(url) {
5859
+ const dir = fileURLToPath(url);
5860
+ return dir.endsWith(sep) && dir.length > 1 ? dir.slice(0, -sep.length) : dir;
5861
+ }
5862
+ /**
5863
+ * 定位随包引擎。顺序:
5864
+ * 1. `DSH_ENGRAM_ENGINE_DIR` 显式指定(tests/CI 可注入);显式路径是明确的
5865
+ * 意图,缺文件直接报错而不是悄悄回退推导——否则配置错误会被掩盖;
5866
+ * 2. `moduleUrl` 相对推导:打包后本模块整体落在 `lib/index.js`,引擎在
5867
+ * 同级的 `lib/engine/`;开发态(vitest 直接跑 `src/embedder/local.ts`)
5868
+ * 则在仓库根的 `lib/engine/`。哪个候选四个文件齐全就用哪个。
5869
+ * 3. 都不成立 → EMBEDDER_DOWNLOAD_FAILED(调用方按既定链路降级),
5870
+ * 文案说清是引擎没随包,而不是网络问题。
5871
+ * @param explicitDir - 环境变量 `DSH_ENGRAM_ENGINE_DIR` 的值(可能为空串)。
5872
+ * @param moduleUrl - 调用方的 `import.meta.url`,推导锚点。
5873
+ * @throws EngramError(code=EMBEDDER_DOWNLOAD_FAILED) 引擎文件找不到。
5874
+ */
5875
+ function resolveEngine(explicitDir, moduleUrl) {
5876
+ const candidates = [];
5877
+ const trimmed = explicitDir?.trim();
5878
+ if (trimmed !== void 0 && trimmed !== "") {
5879
+ if (!isCompleteEngineDir(trimmed)) throw missingEngineError(trimmed);
5880
+ return engineAt(trimmed);
5881
+ }
5882
+ candidates.push(dirOf(new URL("./engine/", moduleUrl)), dirOf(new URL("../../lib/engine/", moduleUrl)));
5883
+ for (const dir of candidates) if (isCompleteEngineDir(dir)) return engineAt(dir);
5884
+ throw missingEngineError(candidates.at(-1));
5885
+ }
5886
+ /** 组装 EnginePaths:目录里四个文件名固定,URL 一律经 pathToFileURL 生成。 */
5887
+ function engineAt(dir) {
5888
+ const url = (name) => pathToFileURL(join(dir, name)).href;
5889
+ return {
5890
+ dir,
5891
+ entryUrl: url(ENGINE_ENTRY),
5892
+ ortModuleUrl: url(ORT_MODULE_FILE),
5893
+ ortFactoryUrl: url(ORT_FACTORY_FILE),
5894
+ ortWasmFile: join(dir, ORT_WASM_FILE)
5895
+ };
5896
+ }
5897
+ /** 引擎缺位的人话错误:码不变(调用方降级逻辑只看码),文案单说这件事。 */
5898
+ function missingEngineError(detail) {
5899
+ const cause = detail === void 0 ? void 0 : /* @__PURE__ */ new Error(`引擎目录候选/指定路径:${detail}`);
5900
+ return new EngramError("EMBEDDER_DOWNLOAD_FAILED", message("store.embedderEngineMissing"), cause === void 0 ? void 0 : { cause });
5901
+ }
5902
+ /** 注入只做一次的守卫:28MB wasmBinary 的拷贝不值得重复付。 */
5903
+ let injected = false;
5904
+ /**
5905
+ * 装载引擎模块:先注入 ORT(仅一次),再 import 引擎入口。
5906
+ * 前提:调用方已确保注入先于引擎入口的首次 import(两步都在这里)。
5907
+ */
5908
+ async function loadEngineModule(paths) {
5909
+ if (!injected) {
5910
+ await injectOnnxRuntime(paths);
5911
+ injected = true;
5912
+ }
5913
+ return await import(paths.entryUrl);
5914
+ }
5915
+ /**
5916
+ * 四步注入(见文件头)。任一步失败都在 injected 置位前抛出:下次调用会重试,
5917
+ * 不会把半套注入钉死在全局上。
5918
+ */
5919
+ async function injectOnnxRuntime(paths) {
5920
+ const ort = await import(paths.ortModuleUrl);
5921
+ ort.env.wasm.wasmPaths = {
5922
+ mjs: paths.ortFactoryUrl,
5923
+ wasm: pathToFileURL(paths.ortWasmFile).href
5924
+ };
5925
+ ort.env.wasm.wasmBinary = new Uint8Array(readFileSync(paths.ortWasmFile));
5926
+ ort.env.wasm.numThreads = 1;
5927
+ ort.env.wasm.proxy = false;
5928
+ globalThis[Symbol.for("onnxruntime")] = ort;
5929
+ await import(paths.ortFactoryUrl);
5930
+ }
5931
+ // src/embedder/engine-cache.ts
5932
+ /**
5933
+ * Web Cache API 的 fs 实现:transformers.js 的 web 构建里 `env.cacheDir` 无效
5934
+ * (node:fs 被构建 stub 掉,文件操作全部落空),模型文件只能走
5935
+ * `env.useCustomCache` + `env.customCache`。这个 shim 把缓存落到真文件系统,
5936
+ * 让「首次联网下载、之后完全离线」的契约在 WASM 引擎下继续成立。
5937
+ * @module @kenz1117/dsh-engram/embedder/engine-cache
5938
+ */
5939
+ /**
5940
+ * 缓存键 → 落盘文件名。键是模型文件的下载 URL(可能带 query),直接当文件名
5941
+ * 既超长又含路径分隔符;取 sha256 前 32 个十六进制位加原始扩展名(截 12 字符),
5942
+ * 不同文件互不覆盖,排查时还能认出类型。
5943
+ */
5944
+ function keyToPath(dir, key) {
5945
+ const name = String(key);
5946
+ const hash = createHash("sha256").update(name).digest("hex").slice(0, 32);
5947
+ const extension = (name.split("?")[0]?.match(/\.[A-Za-z0-9]+$/) ?? [""])[0]?.slice(0, 12) ?? "";
5948
+ return join(dir, `${hash}${extension}`);
5949
+ }
5950
+ /**
5951
+ * 建 fs 落盘的模型缓存。
5952
+ * @param dir - 缓存目录;createLocalEmbedder 已保证它存在(0o700)。
5953
+ */
5954
+ function createFsModelCache(dir) {
5955
+ return {
5956
+ async match(key) {
5957
+ try {
5958
+ const data = await readFile(keyToPath(dir, key));
5959
+ return new Response(data, {
5960
+ status: 200,
5961
+ headers: { "content-length": String(data.byteLength) }
5962
+ });
5963
+ } catch {
5964
+ return;
5965
+ }
5966
+ },
5967
+ async put(key, response) {
5968
+ const data = await response.clone().arrayBuffer();
5969
+ await writeFile(keyToPath(dir, key), Buffer.from(data));
5970
+ }
5971
+ };
5972
+ }
5809
5973
  // src/embedder/local.ts
5810
5974
  /**
5811
- * transformers.js 本地嵌入实现:离线推理,模型首次使用需联网下载到 cacheDir;
5812
- * 之后完全离线,数据不出机。
5975
+ * transformers.js 本地嵌入实现(0.13.11「引擎出树」):引擎以纯 WASM 形式随包
5976
+ * 放在 lib/engine/(scripts/copy-engine.mjs 拷入,spike-web.mjs 实跑验证的路线,
5977
+ * 见 src/embedder/engine.ts),模型首次使用需联网下载到 cacheDir;之后完全
5978
+ * 离线,数据不出机。安装树里不再有 transformers/onnxruntime 依赖,弱网机器的
5979
+ * 市场更新不会再被百 MB 的原生件卡死(issue #50/#54)。
5980
+ *
5981
+ * 性能换特点是这条路线的代价:WASM 单线程推理约 90ms/条(原生 onnxruntime-node
5982
+ * 约 10-20ms/条),批量回填相应变慢;换来的是零原生依赖、零构建脚本许可。
5813
5983
  *
5814
5984
  * Ни модель, ни способ сведения токенов, ни вид весов здесь не задаются: все три
5815
5985
  * приходят строкой каталога моделей (`registry/models.json`). В коде вшитых
@@ -5825,31 +5995,49 @@ function sameSpec(left, right) {
5825
5995
  let cached;
5826
5996
  /**
5827
5997
  * 创建本地嵌入器。
5828
- * @param cacheDir - 模型缓存目录;不存在会自动创建(0o700)。
5998
+ * @param cacheDir - 模型缓存目录;不存在会自动创建(0o700)。web 构建的
5999
+ * `env.cacheDir` 无效(node:fs 被 stub),模型文件经 engine-cache 的 fs shim
6000
+ * 落在这个目录。
5829
6001
  * @param remoteHost - 模型下载端点;默认 huggingface.co,网络受限环境可配镜像(如 https://hf-mirror.com)。
5830
6002
  * @param spec - строка каталога моделей: идентификатор, сведение токенов, вид весов. Смена любого поля обесценивает закэшированный экземпляр, и модель загружается заново.
5831
6003
  * @returns готовый EngramEmbedder.
5832
- * @throws EngramError(code=EMBEDDER_DOWNLOAD_FAILED) 模型下载/加载失败(调用方据此降级)。
6004
+ * @throws EngramError(code=EMBEDDER_DOWNLOAD_FAILED) 引擎未随包安装,或模型下载/加载失败(调用方据此降级)。
5833
6005
  */
5834
6006
  async function createLocalEmbedder(cacheDir, remoteHost, spec) {
5835
6007
  if (cached?.dir === cacheDir && sameSpec(cached.spec, spec)) return cached.embedder;
6008
+ const engine = resolveEngine(process.env.DSH_ENGRAM_ENGINE_DIR, import.meta.url);
5836
6009
  await mkdir(cacheDir, {
5837
6010
  recursive: true,
5838
6011
  mode: 448
5839
6012
  });
5840
6013
  try {
5841
- const { pipeline, env } = await import("@huggingface/transformers");
5842
- env.cacheDir = cacheDir;
5843
- if (remoteHost !== void 0 && remoteHost !== "") env.remoteHost = remoteHost;
5844
- const extractor = await pipeline("feature-extraction", spec.id, { dtype: spec.dtype });
6014
+ const { pipeline, env } = await loadEngineModule(engine);
6015
+ const modelEnv = env;
6016
+ modelEnv.allowLocalModels = false;
6017
+ modelEnv.useCustomCache = true;
6018
+ modelEnv.customCache = createFsModelCache(cacheDir);
6019
+ if (remoteHost !== void 0 && remoteHost !== "") modelEnv.remoteHost = remoteHost;
6020
+ const extractor = await pipeline("feature-extraction", spec.id, {
6021
+ dtype: spec.dtype,
6022
+ device: "auto"
6023
+ });
6024
+ for (let round = 0; round < 2; round += 1) (await extractor(["模型热身"], {
6025
+ pooling: spec.pooling,
6026
+ normalize: true
6027
+ })).dispose?.();
5845
6028
  const embedder = {
5846
6029
  model: spec.id,
5847
6030
  async embed(texts) {
5848
6031
  if (texts.length === 0) return [];
5849
- return (await extractor([...texts], {
6032
+ const output = await extractor([...texts], {
5850
6033
  pooling: spec.pooling,
5851
6034
  normalize: true
5852
- })).tolist().map((list) => new Float32Array(list));
6035
+ });
6036
+ try {
6037
+ return output.tolist().map((list) => new Float32Array(list));
6038
+ } finally {
6039
+ output.dispose?.();
6040
+ }
5853
6041
  },
5854
6042
  close: async () => {
5855
6043
  if (cached?.embedder === embedder) cached = void 0;
@@ -6515,15 +6703,49 @@ async function resolveBand(judge, neighbor, similarity, kind, content, store) {
6515
6703
  }
6516
6704
  }
6517
6705
  /**
6706
+ * 字符 bigram Jaccard:与余弦互补的表面积信号——余弦说「语义像」,bigram Jaccard 说
6707
+ * 「字面像」;改写复述两者皆高,主题相似而内容不同的句子 bigram 低(#55 带内兜底的
6708
+ * 第二个证据)。归一化 = 小写 + 去全部空白;任一侧不足两个字符(无 bigram,含单字符
6709
+ * 串)返回 0。
6710
+ */
6711
+ function charBigramJaccard(a, b) {
6712
+ const bigrams = (raw) => {
6713
+ const text = raw.toLowerCase().replace(/\s+/g, "");
6714
+ const set = /* @__PURE__ */ new Set();
6715
+ for (let i = 0; i < text.length - 1; i += 1) set.add(text.slice(i, i + 2));
6716
+ return set;
6717
+ };
6718
+ const left = bigrams(a);
6719
+ const right = bigrams(b);
6720
+ if (left.size === 0 || right.size === 0) return 0;
6721
+ let intersection = 0;
6722
+ for (const gram of left) if (right.has(gram)) intersection += 1;
6723
+ return intersection / (left.size + right.size - intersection);
6724
+ }
6725
+ /**
6726
+ * 数字令牌守卫:「端口是 8080」与「端口是 9090」字面重叠高、余弦也高,但绝不能兜底
6727
+ * 合并;任何数字差异 = 修正而非复述,保持 defer。令牌取 `/\d+(?:\.\d+)?/g`,按排序后
6728
+ * 序列比较(与位置无关)。
6729
+ */
6730
+ function numbersDiffer(a, b) {
6731
+ const tokens = (raw) => (raw.match(/\d+(?:\.\d+)?/g) ?? []).sort();
6732
+ const left = tokens(a);
6733
+ const right = tokens(b);
6734
+ return left.length !== right.length || left.some((token, index) => token !== right[index]);
6735
+ }
6736
+ /**
6518
6737
  * 判定一条新内容的处置。
6519
6738
  * 嵌入不可用 / 库内无向量条目 / 最近邻低于 DEFER_COSINE → accept;
6520
6739
  * 最近邻同 kind 且 ≥ MERGE_COSINE → merge;
6521
6740
  * 其余达到 DEFER_COSINE 的进模糊带:注入 judge 时自动裁决(高置信同 → merge、
6522
- * 高置信异 → accept、低置信或失败 → defer);未注入 judge → defer(纯规则原行为)。
6741
+ * 高置信异 → accept、低置信或失败 → 落到兜底);judge 缺席、judge 失败或低置信时,
6742
+ * 带内高余弦 + 高字面重叠 + 数字一致也判 merge(免 LLM 兜底,#55——JEV 未配置时
6743
+ * 改写复述同样坍缩),其余 → defer(疑似矛盾照旧落库建边待裁决)。
6523
6744
  * @param store - 目标分库
6524
6745
  * @param kind - 新内容的记忆种类(MERGE 要求同 kind,跨 kind 高度相似按 defer 处理)
6525
6746
  * @param embedding - 新内容的向量;undefined = 嵌入不可用,直接 accept
6526
- * @param options - 可选 judge 注入;judge 与 content 齐备时模糊带走自动裁决
6747
+ * @param options - 可选 judge 注入;judge 与 content 齐备时模糊带先走自动裁决,
6748
+ * content 另供带内兜底比对字面重叠(调用方始终带上)
6527
6749
  */
6528
6750
  async function decideWrite(store, kind, embedding, options = {}) {
6529
6751
  if (embedding === void 0) return { disposition: "accept" };
@@ -6539,6 +6761,11 @@ async function decideWrite(store, kind, embedding, options = {}) {
6539
6761
  const verdict = await resolveBand(judge, nearest.record, nearest.similarity, kind, content, store);
6540
6762
  if (verdict !== void 0) return verdict;
6541
6763
  }
6764
+ if (content !== void 0 && nearest.record.kind === kind && nearest.similarity >= .9 && charBigramJaccard(nearest.record.content, content) >= .5 && !numbersDiffer(nearest.record.content, content)) return {
6765
+ disposition: "merge",
6766
+ into: nearest.record,
6767
+ similarity: nearest.similarity
6768
+ };
6542
6769
  return {
6543
6770
  disposition: "defer",
6544
6771
  neighbor: nearest.record,
@@ -7142,8 +7369,8 @@ async function ingestPreviousTurn(deps) {
7142
7369
  } catch {}
7143
7370
  };
7144
7371
  const vector = embedder === void 0 ? void 0 : (await embedder.embed([content]))[0];
7145
- const decision = await decideWrite(target, kind, vector, judge === void 0 ? {} : {
7146
- judge,
7372
+ const decision = await decideWrite(target, kind, vector, {
7373
+ ...judge === void 0 ? {} : { judge },
7147
7374
  content
7148
7375
  });
7149
7376
  if (decision.disposition === "merge") {
@@ -10912,7 +11139,11 @@ function registerEngramRoutes(ctx, deps) {
10912
11139
  return;
10913
11140
  }
10914
11141
  const scopeParam = url.searchParams.get("scope");
10915
- const scopes = scopeParam === "all" || scopeParam === null || scopeParam === "" ? ["user", "project"] : [scopeOf$1(scopeParam, "user")];
11142
+ const scopes = scopeParam === "all" || scopeParam === null || scopeParam === "" ? [
11143
+ "user",
11144
+ "project",
11145
+ "shared"
11146
+ ] : [scopeOf$1(scopeParam, "user")];
10916
11147
  const kind = url.searchParams.get("kind");
10917
11148
  const limit = Math.min(20, Math.max(1, Number(url.searchParams.get("limit") ?? 10) || 10));
10918
11149
  const embedder = deps.embedder === void 0 ? void 0 : await deps.embedder;
@@ -10960,6 +11191,7 @@ function registerEngramRoutes(ctx, deps) {
10960
11191
  if (!guardWrite(req, res, deps.panelAccess === "lan")) return;
10961
11192
  const body = await readBody(req, res, deps.language);
10962
11193
  if (body === null) return;
11194
+ adoptBodySelector(body);
10963
11195
  const scope = scopeOf$1(typeof body.scope === "string" ? body.scope : null, "user");
10964
11196
  const action = body.action;
10965
11197
  if (action !== "dismiss" && action !== "forget-loser") {
@@ -11242,7 +11474,11 @@ function registerEngramRoutes(ctx, deps) {
11242
11474
  }
11243
11475
  if (req.method === "GET" && route === "activity") {
11244
11476
  const limit = Math.min(50, Math.max(1, Number(url.searchParams.get("limit") ?? 20) || 20));
11245
- json(res, 200, { operations: (await Promise.all(["user", "project"].map(async (scope) => {
11477
+ json(res, 200, { operations: (await Promise.all([
11478
+ "user",
11479
+ "project",
11480
+ "shared"
11481
+ ].map(async (scope) => {
11246
11482
  return (await (await storeFor(scope)).recentOps(limit)).map((op) => ({
11247
11483
  ...op,
11248
11484
  scope
@@ -11416,10 +11652,15 @@ function registerEngramRoutes(ctx, deps) {
11416
11652
  }
11417
11653
  if (req.method === "GET" && route === "health") {
11418
11654
  const rawScope = url.searchParams.get("scope");
11419
- const scopes = rawScope === "user" || rawScope === "project" || rawScope === "shared" ? [rawScope] : ["user", "project"];
11420
- const parts = await Promise.all(scopes.map(async (scope) => {
11655
+ const scopes = rawScope === "user" || rawScope === "project" || rawScope === "shared" ? [rawScope] : [
11656
+ "user",
11657
+ "project",
11658
+ "shared"
11659
+ ];
11660
+ const scored = (await Promise.all(scopes.map(async (scope) => {
11421
11661
  const store = await storeFor(scope);
11422
11662
  const s = await store.stats();
11663
+ if (scope === "shared" && s.total === 0) return null;
11423
11664
  const edgeCount = (await store.exportAll()).edges.length;
11424
11665
  const total = Math.max(s.total, 1);
11425
11666
  const signal = s.signalRatio;
@@ -11443,10 +11684,10 @@ function registerEngramRoutes(ctx, deps) {
11443
11684
  redacted: s.redacted,
11444
11685
  archivedRatio
11445
11686
  };
11446
- }));
11687
+ }))).filter((part) => part !== null);
11447
11688
  json(res, 200, {
11448
- overall: Math.round(parts.reduce((sum, part) => sum + part.score, 0) / parts.length),
11449
- parts,
11689
+ overall: Math.round(scored.reduce((sum, part) => sum + part.score, 0) / scored.length),
11690
+ parts: scored,
11450
11691
  evaluatedAt: Date.now()
11451
11692
  });
11452
11693
  return;
@@ -15028,6 +15269,15 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
15028
15269
  bumpVectorMatrix();
15029
15270
  return attached;
15030
15271
  },
15272
+ async vectorsFor(ids) {
15273
+ const vectors = /* @__PURE__ */ new Map();
15274
+ for (const id of ids) {
15275
+ if (vectors.has(id)) continue;
15276
+ const row = sqlGetMemory.get(id) ?? sqlGet.get(id);
15277
+ if (row?.embedding != null && row.embedding.byteLength >= Float32Array.BYTES_PER_ELEMENT) vectors.set(id, blobToVec(row.embedding));
15278
+ }
15279
+ return vectors;
15280
+ },
15031
15281
  async rewriteVectors(targets) {
15032
15282
  let written = 0;
15033
15283
  withTransaction(() => {
@@ -15465,8 +15715,10 @@ async function openEngramStore(path, rankBoost = NO_BOOST, automation = DEFAULT_
15465
15715
  * 置信度均值。由 `engram_distill` 工具显式触发。
15466
15716
  * @module @kenz1117/dsh-engram/flywheel/distill
15467
15717
  */
15468
- /** 蒸馏输入条数上限(协议内常量:单次辅助调用的可控上下文)。 */
15718
+ /** 蒸馏输入条数上限(协议内常量:单次蒸馏的可控取材窗口)。 */
15469
15719
  const DISTILL_MAX_INPUT = 30;
15720
+ /** 每批进一次辅助调用的条数上限(#55:30 条一次调用必然顶穿输出预算,蒸馏整次失败)。 */
15721
+ const DISTILL_BATCH = 10;
15470
15722
  /** 蒸馏输出 token 上限。 */
15471
15723
  const DISTILL_MAX_TOKENS = 900;
15472
15724
  /**
@@ -15477,8 +15729,24 @@ function distillSystemPrompt() {
15477
15729
  return message("llm.distillSystem");
15478
15730
  }
15479
15731
  /**
15480
- * 执行一次蒸馏。
15481
- * @returns 结果摘要;LLM/解析失败抛错(工具层报给模型),单组写入失败中断本次。
15732
+ * 单批蒸馏的确定性失败(#55 分批自愈的内部信号):max-tokens 由调用层对半递归重试,
15733
+ * unparseable 由调用层把整批计入 failed。网络等瞬时错误不用它承载、原样上抛
15734
+ * (整批失败仍由 distillMemories 抛给工具层报给模型)。
15735
+ */
15736
+ var DistillBatchFailure = class extends Error {
15737
+ reason;
15738
+ constructor(reason) {
15739
+ super("distill-batch-failure");
15740
+ this.reason = reason;
15741
+ this.name = "DistillBatchFailure";
15742
+ }
15743
+ };
15744
+ /**
15745
+ * 执行一次蒸馏。取材按 DISTILL_BATCH 分批依次过辅助 LLM(#55:库稍大时整份计划
15746
+ * 必然顶穿输出预算、蒸馏整次失败,整理永不发生);单批输出超预算对半递归重试,
15747
+ * 切到单片仍超限或解析失败只丢该批(计入 failed,不再空烧整次)。网络等瞬时错误
15748
+ * 原样上抛(工具层报给模型),单组写入失败中断本次。
15749
+ * @returns 结果摘要。
15482
15750
  */
15483
15751
  async function distillMemories(deps) {
15484
15752
  const rows = await deps.store.topActive(deps.scope, DISTILL_MAX_INPUT);
@@ -15487,85 +15755,118 @@ async function distillMemories(deps) {
15487
15755
  distilled: 0,
15488
15756
  superseded: 0,
15489
15757
  vectorsAttached: 0,
15490
- vectorsWithout: 0
15758
+ vectorsWithout: 0,
15759
+ failed: 0
15491
15760
  };
15492
- const lines = rows.map((record) => `[${record.id}] (${record.kind}, importance ${record.importance.toFixed(2)}) ${record.content}`);
15493
- const userText = `${message("llm.distillUserPrompt")}\n${lines.join("\n")}`;
15494
- deps.logRequest({
15495
- route: deps.route,
15496
- scope: deps.scope,
15497
- userText,
15498
- maxTokens: DISTILL_MAX_TOKENS
15499
- });
15500
- const parsed = parseJsonArray(await deps.call({
15501
- route: deps.route,
15502
- system: distillSystemPrompt(),
15503
- userText,
15504
- maxTokens: DISTILL_MAX_TOKENS,
15505
- purpose: "engram-distill",
15506
- signal: deps.signal
15507
- }));
15508
- if (parsed === void 0 || parsed.length === 0) return {
15761
+ const outcome = {
15509
15762
  input: rows.length,
15510
15763
  distilled: 0,
15511
15764
  superseded: 0,
15512
15765
  vectorsAttached: 0,
15513
- vectorsWithout: 0
15766
+ vectorsWithout: 0,
15767
+ failed: 0
15514
15768
  };
15515
- const validIds = new Set(rows.map((record) => String(record.id)));
15516
- const byId = new Map(rows.map((record) => [String(record.id), record]));
15517
- let distilled = 0;
15518
- let superseded = 0;
15519
- let vectorsAttached = 0;
15520
- let vectorsWithout = 0;
15521
- for (const item of parsed) {
15522
- const candidate = item;
15523
- if (typeof candidate.content !== "string" || candidate.content.trim() === "") continue;
15524
- if (!Array.isArray(candidate.supersedes) || candidate.supersedes.length === 0) continue;
15525
- const oldIds = [];
15526
- for (const rawId of candidate.supersedes) {
15527
- if (typeof rawId !== "string" || !validIds.has(rawId)) continue;
15528
- const row = await deps.store.get(asMemoryId(rawId));
15529
- if (row !== void 0 && row.status === "active") oldIds.push(asMemoryId(rawId));
15769
+ /**
15770
+ * 单批蒸馏:组装 → 调用 → 解析 → 逐项校验落库。输出超预算抛
15771
+ * DistillBatchFailure('max-tokens') 供上层对半递归;解析失败抛
15772
+ * DistillBatchFailure('unparseable') 由上层把整批计入 failed。LLM 只见过本批的
15773
+ * id,validIds/byId 也只覆盖本批:跨批 supersedes 引用一律无效。
15774
+ */
15775
+ const applyBatch = async (batch) => {
15776
+ const lines = batch.map((record) => `[${record.id}] (${record.kind}, importance ${record.importance.toFixed(2)}) ${record.content}`);
15777
+ const userText = `${message("llm.distillUserPrompt")}\n${lines.join("\n")}`;
15778
+ deps.logRequest({
15779
+ route: deps.route,
15780
+ scope: deps.scope,
15781
+ userText,
15782
+ maxTokens: DISTILL_MAX_TOKENS
15783
+ });
15784
+ let raw;
15785
+ try {
15786
+ raw = await deps.call({
15787
+ route: deps.route,
15788
+ system: distillSystemPrompt(),
15789
+ userText,
15790
+ maxTokens: DISTILL_MAX_TOKENS,
15791
+ purpose: "engram-distill",
15792
+ signal: deps.signal
15793
+ });
15794
+ } catch (error) {
15795
+ if (isMaxTokensError(error)) throw new DistillBatchFailure("max-tokens");
15796
+ throw error;
15530
15797
  }
15531
- if (oldIds.length === 0) continue;
15532
- const kind = candidate.kind === "skill" || candidate.kind === "fact" ? candidate.kind : "skill";
15533
- const importance = typeof candidate.importance === "number" && Number.isFinite(candidate.importance) ? Math.min(1, Math.max(0, candidate.importance)) : .7;
15534
- const confidences = oldIds.map((oldId) => byId.get(String(oldId))?.confidence ?? .5).filter((value) => value > 0);
15535
- const confidence = confidences.length === 0 ? .5 : confidences.reduce((sum, value) => sum + value, 0) / confidences.length;
15536
- let embedding;
15537
- if (deps.embedder !== void 0) {
15538
- const vectors = [];
15539
- for (const oldId of oldIds) {
15540
- const vector = (await deps.embedder.embed([byId.get(String(oldId))?.content ?? ""]))[0];
15541
- if (vector !== void 0) vectors.push(vector);
15798
+ const parsed = parseJsonArray(raw);
15799
+ if (parsed === void 0) throw new DistillBatchFailure("unparseable");
15800
+ const validIds = new Set(batch.map((record) => String(record.id)));
15801
+ const byId = new Map(batch.map((record) => [String(record.id), record]));
15802
+ for (const item of parsed) {
15803
+ const candidate = item;
15804
+ if (typeof candidate.content !== "string" || candidate.content.trim() === "") continue;
15805
+ if (!Array.isArray(candidate.supersedes) || candidate.supersedes.length === 0) continue;
15806
+ const oldIds = [];
15807
+ for (const rawId of candidate.supersedes) {
15808
+ if (typeof rawId !== "string" || !validIds.has(rawId)) continue;
15809
+ const row = await deps.store.get(asMemoryId(rawId));
15810
+ if (row !== void 0 && row.status === "active") oldIds.push(asMemoryId(rawId));
15542
15811
  }
15543
- if (vectors.length > 0) {
15544
- const average = new Float32Array(vectors[0].length);
15545
- for (const vector of vectors) for (let i = 0; i < vector.length; i++) average[i] = (average[i] ?? 0) + vector[i] / vectors.length;
15546
- embedding = average;
15812
+ if (oldIds.length === 0) continue;
15813
+ const kind = candidate.kind === "skill" || candidate.kind === "fact" ? candidate.kind : "skill";
15814
+ const importance = typeof candidate.importance === "number" && Number.isFinite(candidate.importance) ? Math.min(1, Math.max(0, candidate.importance)) : .7;
15815
+ const confidences = oldIds.map((oldId) => byId.get(String(oldId))?.confidence ?? .5).filter((value) => value > 0);
15816
+ const confidence = confidences.length === 0 ? .5 : confidences.reduce((sum, value) => sum + value, 0) / confidences.length;
15817
+ let embedding;
15818
+ if (deps.embedder !== void 0) {
15819
+ const vectors = [];
15820
+ for (const oldId of oldIds) {
15821
+ const vector = (await deps.embedder.embed([byId.get(String(oldId))?.content ?? ""]))[0];
15822
+ if (vector !== void 0) vectors.push(vector);
15823
+ }
15824
+ if (vectors.length > 0) {
15825
+ const average = new Float32Array(vectors[0].length);
15826
+ for (const vector of vectors) for (let i = 0; i < vector.length; i++) average[i] = (average[i] ?? 0) + vector[i] / vectors.length;
15827
+ embedding = average;
15828
+ }
15547
15829
  }
15830
+ await deps.store.supersedeMany({
15831
+ scope: deps.scope,
15832
+ kind,
15833
+ content: candidate.content.trim(),
15834
+ importance,
15835
+ confidence,
15836
+ ...embedding === void 0 ? {} : { embedding }
15837
+ }, oldIds);
15838
+ outcome.distilled += 1;
15839
+ outcome.superseded += oldIds.length;
15840
+ if (embedding === void 0) outcome.vectorsWithout += 1;
15841
+ else outcome.vectorsAttached += 1;
15548
15842
  }
15549
- await deps.store.supersedeMany({
15550
- scope: deps.scope,
15551
- kind,
15552
- content: candidate.content.trim(),
15553
- importance,
15554
- confidence,
15555
- ...embedding === void 0 ? {} : { embedding }
15556
- }, oldIds);
15557
- distilled += 1;
15558
- superseded += oldIds.length;
15559
- if (embedding === void 0) vectorsWithout += 1;
15560
- else vectorsAttached += 1;
15561
- }
15562
- return {
15563
- input: rows.length,
15564
- distilled,
15565
- superseded,
15566
- vectorsAttached,
15567
- vectorsWithout
15568
15843
  };
15844
+ /**
15845
+ * 单批失败自愈(#55):输出超预算对半递归重试(两半各自独立调用——合回同一请求体积
15846
+ * 几乎不减、必然再超限);切到单片仍超限则该条计入 failed(保持 active,下次蒸馏
15847
+ * 再次取材);解析失败整批计入 failed;其余错误原样上抛。
15848
+ */
15849
+ const runBatch = async (batch) => {
15850
+ try {
15851
+ await applyBatch(batch);
15852
+ return;
15853
+ } catch (error) {
15854
+ if (!(error instanceof DistillBatchFailure)) throw error;
15855
+ if (error.reason === "unparseable") {
15856
+ outcome.failed += batch.length;
15857
+ return;
15858
+ }
15859
+ }
15860
+ if (batch.length <= 1) {
15861
+ outcome.failed += batch.length;
15862
+ return;
15863
+ }
15864
+ const mid = Math.ceil(batch.length / 2);
15865
+ await runBatch(batch.slice(0, mid));
15866
+ await runBatch(batch.slice(mid));
15867
+ };
15868
+ for (let start = 0; start < rows.length; start += DISTILL_BATCH) await runBatch(rows.slice(start, start + DISTILL_BATCH));
15869
+ return outcome;
15569
15870
  }
15570
15871
  /**
15571
15872
  * 改写系统指令:只输出 JSON 字符串数组,不回答问题。
@@ -16143,8 +16444,8 @@ function createEngramTools(baseDeps) {
16143
16444
  */
16144
16445
  async function writeWithDisposition(store, item, judge) {
16145
16446
  const { embedding } = item;
16146
- const decision = await decideWrite(store, item.kind, embedding, judge === void 0 ? {} : {
16147
- judge,
16447
+ const decision = await decideWrite(store, item.kind, embedding, {
16448
+ ...judge === void 0 ? {} : { judge },
16148
16449
  content: item.content
16149
16450
  });
16150
16451
  if (decision.disposition === "merge") return {
@@ -18150,12 +18451,14 @@ function createEngramTools(baseDeps) {
18150
18451
  scope
18151
18452
  }));
18152
18453
  })().catch(() => {});
18153
- const done = message("tool.distillDone", {
18454
+ const lines = [message("tool.distillDone", {
18154
18455
  input: outcome.input,
18155
18456
  distilled: outcome.distilled,
18156
18457
  superseded: outcome.superseded
18157
- });
18158
- return { text: outcome.vectorsWithout === 0 ? done : `${done}\n${message("tool.distillNoVectors", { count: outcome.vectorsWithout })}` };
18458
+ })];
18459
+ if (outcome.vectorsWithout > 0) lines.push(message("tool.distillNoVectors", { count: outcome.vectorsWithout }));
18460
+ if (outcome.failed > 0) lines.push(message("tool.distillFailedLine", { count: outcome.failed }));
18461
+ return { text: lines.join("\n") };
18159
18462
  }
18160
18463
  }),
18161
18464
  defineTool({
@@ -18781,8 +19084,49 @@ function estimateTokens(text) {
18781
19084
  else narrow += 1;
18782
19085
  return Math.ceil(wide * WIDE_TOKENS_PER_CHAR + narrow / 4);
18783
19086
  }
19087
+ // src/fold.ts
19088
+ /** 折叠阈值:同 kind 余弦达到该值判为同一事实的改写(报告人建议值)。 */
19089
+ const FOLD_COSINE = .9;
19090
+ /**
19091
+ * 同 kind 近似重复折叠。无向量条目不可比较,一律保留不折叠;跨 kind 不折叠;
19092
+ * 混维(换过模型的存量库)视作不可比较,不折叠也不炸注入。
19093
+ * @param records - 候选(topActive 顺序 = importance 降序)。
19094
+ * @param vectors - id → 存储向量(键取各条目的 memoryId;缺向量条目一律保留)。
19095
+ * @param threshold - 折叠阈值(缺省 FOLD_COSINE)。
19096
+ */
19097
+ function foldNearDuplicates(records, vectors, threshold = FOLD_COSINE) {
19098
+ const heads = [];
19099
+ for (const record of records) {
19100
+ const vector = vectors.get(record.memoryId);
19101
+ const host = vector === void 0 ? void 0 : heads.find((head) => {
19102
+ if (head.record.kind !== record.kind) return false;
19103
+ const headVector = vectors.get(head.record.memoryId);
19104
+ if (headVector === void 0) return false;
19105
+ try {
19106
+ return cosine(vector, headVector) >= threshold;
19107
+ } catch (error) {
19108
+ if (isDimensionMismatch(error)) return false;
19109
+ throw error;
19110
+ }
19111
+ });
19112
+ if (host === void 0) heads.push({
19113
+ record,
19114
+ foldedIds: []
19115
+ });
19116
+ else host.foldedIds.push(record.id);
19117
+ }
19118
+ const kept = heads.map((head) => ({
19119
+ record: head.record,
19120
+ foldedCount: head.foldedIds.length,
19121
+ foldedIds: head.foldedIds
19122
+ }));
19123
+ return {
19124
+ kept,
19125
+ foldedCount: kept.reduce((sum, head) => sum + head.foldedCount, 0)
19126
+ };
19127
+ }
18784
19128
  // package.json
18785
- var version = "0.13.10";
19129
+ var version = "0.13.12";
18786
19130
  // src/index.ts
18787
19131
  /**
18788
19132
  * dsh-engram:DeepSeek Harness 跨会话长期记忆插件(host 半)。
@@ -18808,7 +19152,8 @@ const DEFAULT_ITEM_BUDGET = {
18808
19152
  * 整条降级索引行);token 预算(估算见 estimateTokens:中文按 1.5 token/字、
18809
19153
  * 其余按 4 字符/token)是外层硬约束:截断后仍装不下的条目先以 40 字索引行
18810
19154
  * 兜底,索引行也装不下的折成末尾 `+N more; use engram_search` 计数行;计数行同样占用预算。
18811
- * @param records - 候选条目(调用方已按重要性排序、按条数截断)。
19155
+ * @param records - 候选条目(调用方已按重要性排序、按条数截断;folded 为该条目
19156
+ * 折叠掉的近似重复条数,> 0 时行尾追加折叠注记)。
18812
19157
  * @param tokenBudget - 整段画像的 token 预算(含首尾固定行)。
18813
19158
  * @param language - 房间名的显示语言(房间里存的是标识符,不是文本)。
18814
19159
  * @param itemBudget - 分级递减的单条正文预算(缺省用 DEFAULT_ITEM_BUDGET;配置面在 resolveConfig)。
@@ -18825,7 +19170,8 @@ function renderProfileDetailed(records, tokenBudget, language, itemBudget = DEFA
18825
19170
  const budget = Math.max(itemBudget.floor, Math.round(itemBudget.start * itemBudget.decay ** index));
18826
19171
  const content = record.content.length > budget ? `${record.content.slice(0, budget)}…` : record.content;
18827
19172
  const slot = record.slot === void 0 ? "" : ` ${roomLabel(record.slot.room, language)}#${record.slot.index}`;
18828
- const line = `- [${record.kind}]${slot} ${content}`;
19173
+ const foldedNote = record.folded === void 0 || record.folded <= 0 ? "" : ` ${messageIn(language, "core.profileFolded", { count: record.folded })}`;
19174
+ const line = `- [${record.kind}]${slot} ${content}${foldedNote}`;
18829
19175
  const cost = estimate(line);
18830
19176
  if (cost <= remaining) {
18831
19177
  lines.push(line);
@@ -18854,7 +19200,7 @@ function renderProfileDetailed(records, tokenBudget, language, itemBudget = DEFA
18854
19200
  }
18855
19201
  /**
18856
19202
  * 会话开始注入的画像渲染(只取文本;溢出明细见 renderProfileDetailed)。
18857
- * @param records - 候选条目(调用方已按重要性排序、按条数截断)。
19203
+ * @param records - 候选条目(调用方已按重要性排序、按条数截断;folded 见 renderProfileDetailed)。
18858
19204
  * @param tokenBudget - 整段画像的 token 预算(含首尾固定行)。
18859
19205
  * @returns 注入文本。
18860
19206
  */
@@ -19053,6 +19399,11 @@ async function preStep(ctx, openStore, openStoreForProjectCwd, resolved, embedde
19053
19399
  const curated = await store.getProfileBlock("user");
19054
19400
  const top = await store.topActive("user", resolved.profileTopN);
19055
19401
  if (top.length === 0 && curated === void 0) return decision;
19402
+ const folded = foldNearDuplicates(top, await store.vectorsFor(top.map((record) => record.memoryId)));
19403
+ const keptRecords = folded.kept.map((head) => head.foldedCount > 0 ? {
19404
+ ...head.record,
19405
+ folded: head.foldedCount
19406
+ } : head.record);
19056
19407
  const dueTotal = resolved.reviewScheduling ? (await Promise.all(["user", "project"].map(async (scope) => (await openStore(scope)).dueReviews(Date.now(), 50)))).reduce((sum, rows) => sum + rows.length, 0) : 0;
19057
19408
  let reflectPending = 0;
19058
19409
  if (resolved.reflect === "suggest" && resolved.reflectReminder) reflectPending = (await Promise.all(["user", "project"].map(async (scope) => {
@@ -19069,13 +19420,13 @@ async function preStep(ctx, openStore, openStoreForProjectCwd, resolved, embedde
19069
19420
  decay: resolved.injectItemBudgetDecay,
19070
19421
  floor: resolved.injectItemBudgetFloor
19071
19422
  };
19072
- let parts = renderProfileParts(curated, top, resolved.injectTokenBudget, resolved.language, itemBudget);
19423
+ let parts = renderProfileParts(curated, keptRecords, resolved.injectTokenBudget, resolved.language, itemBudget);
19073
19424
  if (parts.derived.overflow.length > 0) {
19074
19425
  const overflow = parts.derived.overflow;
19075
19426
  const compressed = await compressProfileOverflow(ctx, agent, resolved.routeOverride, overflow, signal);
19076
19427
  if (compressed !== void 0) {
19077
19428
  openStore("user").then((auditStore) => auditStore.audit("compress-request", "AUX", JSON.stringify({ count: overflow.length }))).catch(() => {});
19078
- parts = renderProfileParts(curated, top.map((record) => {
19429
+ parts = renderProfileParts(curated, keptRecords.map((record) => {
19079
19430
  const shorter = compressed.get(record.id);
19080
19431
  return shorter === void 0 ? record : {
19081
19432
  ...record,
@@ -19101,7 +19452,7 @@ async function preStep(ctx, openStore, openStoreForProjectCwd, resolved, embedde
19101
19452
  plus: dueTotal >= 50 ? "+" : ""
19102
19453
  });
19103
19454
  const reflectPendingLine = reflectPending === 0 ? "" : message("core.reflectPending", { count: reflectPending });
19104
- const textWithRationale = wrapWithRationale(text, top) + dueLine + reflectPendingLine;
19455
+ const textWithRationale = wrapWithRationale(text, keptRecords) + dueLine + reflectPendingLine;
19105
19456
  const hash = createHash("sha256").update(`${parts.curated ?? ""}\n${textWithRationale}`).digest("hex");
19106
19457
  if (!compacted && hash === sessionState.lastProfileHash) return decision;
19107
19458
  sessionState.lastProfileHash = hash;
@@ -19114,7 +19465,7 @@ async function preStep(ctx, openStore, openStoreForProjectCwd, resolved, embedde
19114
19465
  sessionId,
19115
19466
  packetHash,
19116
19467
  packet,
19117
- memoryIds: top.map((record) => record.memoryId),
19468
+ memoryIds: folded.kept.map((head) => head.record.memoryId),
19118
19469
  hasCurated: parts.curated !== void 0,
19119
19470
  compactionCount
19120
19471
  });