@a9i5k4/dsh-auto-memory 2.3.1 → 2.4.0

package/lib/python-setup.js

@@ -4,9 +4,11 @@
  * 四步链路(全部走本模块,UI 只管展示状态与点击):
  *   ①detect  — 探测系统 Python(≥3.9)/既有 venv/模型本体;全部只读。
  *   ②venv    — python -m venv <userDir>/python-engine/.venv(幂等:已存在直接跳过)。
- *   ③deps    — venv 内 pip 安装运行依赖(transformers/onnxruntime/torch,清华镜像兜底)。
+ *   ③deps    — venv 内 pip 安装运行依赖(transformers/onnxruntime,清华镜像兜底;int8 档不需要 torch)。
  *   ④model   — BGE-M3 int8(~539MB)下载到 <userDir>/python-engine/models/,
  *              cn(hf-mirror)/intl(hf 官方)双通道+SHA256 校验,复用 JS 档下载器的状态机形态。
+ *   ④'       — 模型齐备后**必须回写 embedding-config.json**(provider/modelDir/onnxFile/dimension),
+ *              否则 worker load_embedder 抛 unknown embedding provider —— 这是"装了用不了"的头号断点。
  *
  * 设计约束:
  *   - 一切落盘在用户目录(~/.dsh/python-engine/),npm 包目录升级会被覆盖,绝不放模型。
@@ -18,7 +20,7 @@ import { promisify } from 'node:util'
 import { createHash } from 'node:crypto'
 import { homedir } from 'node:os'
 import path from 'node:path'
-import { existsSync, mkdirSync, writeFileSync, statSync, createWriteStream } from 'node:fs'
+import { existsSync, mkdirSync, writeFileSync, readFileSync, statSync, createWriteStream } from 'node:fs'
 import { readFile, writeFile, rm } from 'node:fs/promises'
 
 const execFileP = promisify(execFile)
@@ -45,6 +47,15 @@ const TOKENIZER_FILES = ['config.json', 'tokenizer.json', 'tokenizer_config.json
 const PIP_DEPS_CPU = ['transformers', 'onnxruntime']
 const PIP_DEPS_GPU_EXTRA = ['onnxruntime-gpu']
 
+/** 探针要与安装清单**同口径**:int8 档 load_embedder 只 import numpy+onnxruntime+transformers;
+ *  历史上探针多要一个 torch,而 PIP_DEPS_CPU 从不装 torch → depsOk 恒 false,"全部就绪"永不出现。 */
+const DEPS_PROBE = 'import transformers, onnxruntime, numpy; print("deps-ok")'
+
+/** worker 侧 load_embedder 的档位名(m7_embedding_v1.PROVIDER_REAL_INT8);发布构建会把 -pre-v1 折成 -v1。 */
+const PROVIDER_ID_INT8 = 'bge-m3-onnx-int8-v1'
+/** dense 维度(BGE-M3 = 1024),写进 embedding-config.json 的 dimension。 */
+const EMBED_DIMENSION = 1024
+
 export function createPythonSetupPre(opts = {}) {
   const dshHomeOf = typeof opts.dshHome === 'function' ? opts.dshHome : () => opts.dshHome || path.join(homedir(), '.dsh')
   const diagOf = typeof opts.diag === 'function' ? opts.diag : () => {}
@@ -64,6 +75,7 @@ export function createPythonSetupPre(opts = {}) {
     chosenPython: '',
     venvOk: false,
     depsOk: false,
+    configOk: false,
     dl: { bytesDone: 0, bytesTotal: MODEL_SPEC.bytes, mirror: '', startedAt: 0, etaSec: 0 },
     cancelled: false,
   }
@@ -86,6 +98,43 @@ export function createPythonSetupPre(opts = {}) {
     return m ? { major: Number(m[1]), minor: Number(m[2]) } : null
   }
 
+  // ---------- 引擎侧 embedding-config.json(D1/D2:向导产物必须落在 worker 真正读取的位置与键上) ----------
+  /** worker 读 <dsh-home>/memory/semantic/embedding-config.json(见 worker_semantic_v1.load_embedding_config_from_env);
+   *  发布构建会把 `semantic` 折成 `semantic`,故这里写带 -pre 的规范名。
+   *  历史缺陷:向导曾写 <dsh-home>/memory/semantic/(缺 -pre),pre 线 worker 从不读该路径 → 装完即"找不到配置"。 */
+  const embeddingConfigPath = () => path.join(dshHomeOf(), 'memory', 'semantic', 'embedding-config.json')
+
+  /** read-modify-write:只覆盖本次传入的键,保留引擎运行期自管的 search/activationPolicy/activationEmitMode 等。 */
+  async function patchEmbeddingConfig(patch) {
+    const cfgPath = embeddingConfigPath()
+    mkdirSync(path.dirname(cfgPath), { recursive: true })
+    let cfg = {}
+    try { cfg = JSON.parse(await readFile(cfgPath, 'utf8')) } catch (_) {}
+    if (!cfg || typeof cfg !== 'object' || Array.isArray(cfg)) cfg = {}
+    Object.assign(cfg, patch)
+    await writeFile(cfgPath, JSON.stringify(cfg, null, 2), 'utf8')
+    return cfgPath
+  }
+
+/** 就绪口径(D4):配置声明的 provider/modelDir 必须指向磁盘上真实存在的 onnx + tokenizer;
+ *  只看"onnx 已下载"会给出假绿 —— worker 起来照样抛 unknown embedding provider / KeyError modelDir。
+ *  注意与下载清单 TOKENIZER_FILES 的区别:那是"下全"的清单(slow/fast 两条路径都覆盖),
+ *  这里判的是**能否加载** —— AutoTokenizer 默认 fast 路径有 tokenizer.json 即可,缺 sentencepiece.bpe.model 不算坏。 */
+function configReadyForModels() {
+  try {
+    const cfg = JSON.parse(readFileSync(embeddingConfigPath(), 'utf8'))
+    if (!cfg || typeof cfg !== 'object' || Array.isArray(cfg)) return false
+    if (String(cfg.provider || '') !== PROVIDER_ID_INT8) return false
+    const base = String(cfg.modelDir || '').trim()
+    if (!base) return false
+    const rel = String(cfg.onnxFile || 'onnx/model_int8.onnx')
+    if (!existsSync(path.join(base, ...rel.split('/')))) return false
+    const hasTokenizer = ['tokenizer.json', 'sentencepiece.bpe.model'].some((f) => existsSync(path.join(base, f)))
+    if (!hasTokenizer) return false
+    return ['config.json', 'tokenizer_config.json'].every((f) => existsSync(path.join(base, f)))
+  } catch (_) { return false }
+}
+
   /** ①环境探测:venv(推荐)→ 系统 python/python3/py launcher;版本 ≥3.9 且 <3.13(onnxruntime 兼容上界)。 */
   async function detect() {
     st.phase = 'detecting'; st.error = ''
@@ -106,7 +155,8 @@ export function createPythonSetupPre(opts = {}) {
     }
     // 既有成果快照
     st.venvOk = existsSync(venvPython())
-    st.depsOk = st.venvOk ? (await probe(venvPython(), ['-c', 'import transformers, onnxruntime, torch; print("deps-ok")'])).ok : false
+    st.depsOk = st.venvOk ? (await probe(venvPython(), ['-c', DEPS_PROBE])).ok : false
+    st.configOk = configReadyForModels()
     // modelReady 判定(2026-09-09):onnx + tokenizer 5 件全齐才算就绪,避免 UI 显示 ✓ 但 sidecar 起不来
     st.modelReady = existsSync(modelPath()) && TOKENIZER_FILES.every((f) => existsSync(path.join(modelsDir(), f)))
     st.pythons = out
@@ -118,7 +168,7 @@ export function createPythonSetupPre(opts = {}) {
     return {
       phase: st.phase, error: st.error,
       pythons: st.pythons, chosenPython: st.chosenPython,
-      venvOk: st.venvOk, depsOk: st.depsOk, modelReady: st.modelReady, wantGpu: !!st.wantGpu,
+      venvOk: st.venvOk, depsOk: st.depsOk, configOk: st.configOk, modelReady: st.modelReady, wantGpu: !!st.wantGpu,
       modelPath: modelPath(), venvPython: venvPython(),
       modelBytes: st.modelReady ? statSync(modelPath()).size : 0, modelExpectedBytes: MODEL_SPEC.bytes,
       dl: { ...st.dl },
@@ -163,16 +213,7 @@ export function createPythonSetupPre(opts = {}) {
     st.depsOk = !!r.ok
     if (!r.ok) { st.error = '依赖安装失败: ' + (r.tail || ''); st.phase = 'error'; return snapshot() }
     // GPU 偏好写入 embedding-config.json(worker 读 config['gpu'] 选 CUDA/CPU provider;read-modify-write 不覆盖既有键)
-    try {
-      const cfgDir = path.join(dshHomeOf(), 'memory', 'semantic')
-      mkdirSync(cfgDir, { recursive: true })
-      const cfgPath = path.join(cfgDir, 'embedding-config.json')
-      let cfg = {}
-      try { cfg = JSON.parse(await readFile(cfgPath, 'utf8')) } catch (_) {}
-      if (!cfg || typeof cfg !== 'object') cfg = {}
-      cfg.gpu = wantGpu
-      await writeFile(cfgPath, JSON.stringify(cfg, null, 2), 'utf8')
-    } catch (eCfg) { diagOf('python-setup: gpu pref write failed: ' + String(eCfg && eCfg.message || eCfg)) }
+    try { await patchEmbeddingConfig({ gpu: wantGpu }) } catch (eCfg) { diagOf('python-setup: gpu pref write failed: ' + String(eCfg && eCfg.message || eCfg)) }
     diagOf('python-setup: deps installed into venv (gpu=' + wantGpu + ')')
     return snapshot()
   }
@@ -201,6 +242,18 @@ export function createPythonSetupPre(opts = {}) {
           await downloadWithResume(tu, path.join(modelsDir(), tf), () => {}, () => st.cancelled)
         }
         st.modelReady = true; st.phase = 'ready'
+        // D1/D2:模型齐备后把**引擎消费所需的键**写全 —— provider(否则 load_embedder 抛 unknown embedding provider)、
+        // modelDir(否则 KeyError)、onnxFile(落位是平铺 models/model_int8.onnx,而 worker 默认找 modelDir/onnx/model_int8.onnx)、
+        // dimension(向量 identity 块)。tokenizer 五件与 onnx 同在 modelDir 根,AutoTokenizer.from_pretrained(modelDir) 因此可用。
+        try {
+          await patchEmbeddingConfig({
+            provider: PROVIDER_ID_INT8,
+            modelDir: modelsDir().replace(/\\/g, '/'),
+            onnxFile: path.basename(MODEL_SPEC.file),
+            dimension: EMBED_DIMENSION,
+          })
+          st.configOk = true
+        } catch (eCfg) { diagOf('python-setup: embedding-config write failed: ' + String(eCfg && eCfg.message || eCfg)) }
         diagOf('python-setup: model+tokens ready at ' + modelsDir() + ' (model=' + size + ' bytes, mirror=' + mirror.id + ')')
         return snapshot()
       } catch (e) {

package/lib/water-window.js

@@ -80,7 +80,7 @@ export function pickWindowPre(windows, provider, model) {
  * @returns {{provider:string,model:string,contextWindow:number}} 未找到时字段为空/0。
  */
 export function findSessionModelPre(events, maxScan = 0) {
-  const out = { provider: '', model: '', contextWindow: 0 }
+  const out = { provider: '', model: '', contextWindow: 0, maxTokens: 0 }
   if (!Array.isArray(events) || !events.length) return out
   const floor = maxScan > 0 ? Math.max(0, events.length - maxScan) : 0
   for (let i = events.length - 1; i >= floor; i--) {
@@ -92,13 +92,70 @@ export function findSessionModelPre(events, maxScan = 0) {
     const provider = String(cfg.provider || d.provider || '')
     const model = String(cfg.model || d.model || '')
     const cw = Number(d.contextWindow) || 0
+    const mt = Number(cfg.maxTokens) || 0
     if (!out.model && model) { out.model = model; out.provider = provider }
     if (!out.contextWindow && cw > 0) out.contextWindow = cw
-    if (out.model && out.contextWindow) break
+    // maxTokens = 路由为**本次请求预留的输出预算**,provider 会把它计入 "requested tokens"(实测:
+    // 消息 666,044 + 预留 384,000 = 1,050,044 > 上限 1,048,576 → 400 CONTEXT_WINDOW_EXCEEDED)。
+    if (!out.maxTokens && mt > 0) out.maxTokens = mt
+    if (out.model && out.contextWindow && out.maxTokens) break
   }
   return out
 }
 
+/**
+ * 水位「硬信号」扫描(2026-09-10,实机取证后新增)。
+ *
+ * 背景(用户实测:官方压缩又抢在 75% 接续之前):本机 provider 的硬上限是 1,048,576 token,
+ * 而**预留输出预算 384,000 也算在请求里** ⇒ 消息实际只能用到约 664,576。插件此前拿
+ * `contextWindow`(1,000,000)当分母、拿本地计量当分子,两边都偏乐观,于是 75% 永远到不了,
+ * 直到上游直接 400:
+ *   "This model's maximum context length is 1048576 tokens. However, you requested 1050044 tokens
+ *    (666044 in the messages, 384000 in the completion)."
+ * 这条错误是**唯一权威**的口径来源。本函数把它与 compaction 事件一起扫出来,供上层:
+ *   ① 解析真实窗口/预留额度/消息实占 → 自我校准分子;
+ *   ② 一旦发生 overflow 或 compaction,直接当作「已到阈值」触发接续(不再依赖估算)。
+ *
+ * @param {object[]} events - 会话事件数组。
+ * @returns {{reservedTokens:number,overflow:{seq:number,windowTokens:number,requestedTokens:number,messageTokens:number,completionTokens:number,code:string}|null,compactionSeq:number}}
+ */
+export function scanPressureSignalsPre(events) {
+  const out = { reservedTokens: 0, overflow: null, compactionSeq: 0 }
+  if (!Array.isArray(events) || !events.length) return out
+  for (let i = events.length - 1; i >= 0; i--) {
+    const ev = events[i]
+    if (!ev) continue
+    const seq = Number(ev.seq) || 0
+    const t = ev.type
+    if (!out.reservedTokens && (t === 'request/header' || t === 'request/context')) {
+      const d = (ev.data || {})
+      const cfg = (d.header && d.header.config) || d.config || {}
+      const mt = Number(cfg.maxTokens) || 0
+      if (mt > 0) out.reservedTokens = mt
+    }
+    if (!out.compactionSeq && (t === 'compaction/start' || t === 'compaction/summary')) out.compactionSeq = seq
+    if (out.overflow && out.compactionSeq && out.reservedTokens) break
+    if (out.overflow || t !== 'assistant/attempt') continue
+    const stream = (ev.data && ev.data.stream) || []
+    for (const item of stream) {
+      const chunk = (item && item.chunk) || {}
+      if (chunk.type !== 'finish') continue
+      const failure = (chunk.reason && chunk.reason.failure) || {}
+      const code = String(failure.code || '')
+      const msg = String(failure.message || '')
+      if (code !== 'CONTEXT_WINDOW_EXCEEDED' && !/maximum context length/i.test(msg)) continue
+      const win = Number((msg.match(/maximum context length is\s+(\d+)\s+tokens/i) || [])[1]) || 0
+      const req = Number((msg.match(/you requested\s+(\d+)\s+tokens/i) || [])[1]) || 0
+      const inMsg = Number((msg.match(/\((\d+)\s+in the messages/i) || [])[1]) || 0
+      const inComp = Number((msg.match(/(\d+)\s+in the completion\)/i) || [])[1]) || 0
+      out.overflow = { seq, windowTokens: win, requestedTokens: req, messageTokens: inMsg, completionTokens: inComp, code: code || 'CONTEXT_WINDOW_EXCEEDED' }
+      break
+    }
+  }
+  return out
+}
+
+
 /**
  * 从会话事件里取最近一条 request/context 的 contextWindow(路由权威值)。
  * 不限制扫描窗口:该事件只在请求头部变化时追加,长会话里往往只存在于开头。

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@a9i5k4/dsh-auto-memory",
   "description": "Proactive associative memory for DSH: zero-prompt recall injected before the model speaks, three-layer auto-consolidation, skill crystallization, and Astra-style context management - handoff ledgers, PLAN whiteboard, water-level sensing. Local-first, model-agnostic, zero deps. 主动联想记忆+Astra 式上下文管理:自动唤回/自动沉淀/技能固化/交接账本与白板跨窗口续命/水位感知。",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "type": "module",
   "main": "lib/index.js",
   "exports": {