@a9i5k4/dsh-auto-memory 2.1.8 → 2.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/client.js +1 -0
- package/lib/index.js +87 -1
- package/lib/semantic-js.js +491 -485
- package/package.json +1 -1
package/lib/client.js
CHANGED
|
@@ -3062,6 +3062,7 @@ window.__ModuleLoader__.load({
|
|
|
3062
3062
|
: 'The built-in JS semantic model (~135MB, one-time) can be downloaded in the background; the semantic tier enables itself once done:'))
|
|
3063
3063
|
sdActions.push(h('button', { 'data-dam-btn': '', 'data-primary': 'true', style: close, onClick: function () {
|
|
3064
3064
|
apiPost(API.semanticDownload, { action: 'start', mirror: 'auto' }).catch(function () {})
|
|
3065
|
+
try { localStorage.setItem('dsh-auto-memory.semDetectSnoozeUntil', String(Date.now() + 24 * 3600 * 1000)) } catch (eSn2) {}
|
|
3065
3066
|
closeDialog()
|
|
3066
3067
|
} }, sdZh ? '立即下载' : 'Download now'))
|
|
3067
3068
|
}
|
package/lib/index.js
CHANGED
|
@@ -2335,6 +2335,78 @@ class MemoryEngine {
|
|
|
2335
2335
|
return true
|
|
2336
2336
|
}
|
|
2337
2337
|
|
|
2338
|
+
/**
|
|
2339
|
+
* F1(M6-CONTRACT 外补丁,2026-08-23):anchor 开启时的记录级压缩。
|
|
2340
|
+
* 按 sidecar 记录序整条保留/移除——今天(标题日期=插件今天)记录无条件保留,其余按字节配额
|
|
2341
|
+
* 从最新往回保留(layer 上限 user 4000/note 3000);被移除记录整条原文写入归档文件
|
|
2342
|
+
* (writeFullRaw,不经 anchor 事务,归档非检索语料)。重组经 store.replace 原子执行:
|
|
2343
|
+
* 候选预检(clean)→被移除 id 自动删除→保留 id/版本不变。永不字符切片。
|
|
2344
|
+
* @returns {Promise<boolean|'not-anchored'>} true=已腾位 false=无法腾位 'not-anchored'=无 sidecar 走旧路径
|
|
2345
|
+
*/
|
|
2346
|
+
async compactAnchoredLayer(store, filePath, layer, today, paths) {
|
|
2347
|
+
const cur = await this.readTextSafe(filePath)
|
|
2348
|
+
if (!cur || !cur.trim()) return false
|
|
2349
|
+
let parsed
|
|
2350
|
+
try { parsed = parseAnchors(Buffer.from(cur, 'utf8')) } catch (_) { return 'not-anchored' }
|
|
2351
|
+
if (!parsed || parsed.status !== 'clean') return false // 脏文件 fail closed:不压缩、不切片
|
|
2352
|
+
const anchored = parsed.records.filter((r) => r.kind === 'anchored')
|
|
2353
|
+
if (!anchored.length) return 'not-anchored'
|
|
2354
|
+
const limit = layer === 'user' ? 4000 : 3000
|
|
2355
|
+
const isTodayHeading = (heading) => /^\d{4}-\d{2}-\d{2}/.test(String(heading || '')) && String(heading).slice(0, 10) === today
|
|
2356
|
+
const buf = Buffer.from(cur, 'utf8')
|
|
2357
|
+
// 配额从尾回溯(F1):新的留;今天记录无条件保留
|
|
2358
|
+
const keep = new Array(anchored.length).fill(false)
|
|
2359
|
+
let keptBytes = 0
|
|
2360
|
+
for (let i = anchored.length - 1; i >= 0; i--) {
|
|
2361
|
+
const rec = anchored[i]
|
|
2362
|
+
const spanBytes = rec.byteEnd - rec.markerByteStart
|
|
2363
|
+
if (isTodayHeading(rec.heading) || keptBytes + spanBytes <= limit) { keep[i] = true; keptBytes += spanBytes }
|
|
2364
|
+
}
|
|
2365
|
+
const removedIdx = []
|
|
2366
|
+
anchored.forEach((rec, i) => { if (!keep[i]) removedIdx.push(i) })
|
|
2367
|
+
if (!removedIdx.length) return false // 无可移除 → 无法腾位(预算照常拒绝)
|
|
2368
|
+
// 有序块重组:legacy 块与 preamble/tail 原样保留,gap 字节逐段拷贝,仅丢弃被移除记录的 [marker..contentEnd]
|
|
2369
|
+
const blocks = []
|
|
2370
|
+
for (const r of parsed.records) {
|
|
2371
|
+
if (r.kind === 'anchored') blocks.push({ start: r.markerByteStart, end: r.byteEnd, keep: keep[anchored.indexOf(r)] })
|
|
2372
|
+
else blocks.push({ start: r.byteStart, end: r.byteEnd, keep: true })
|
|
2373
|
+
}
|
|
2374
|
+
blocks.sort((a, b) => a.start - b.start)
|
|
2375
|
+
const parts = []
|
|
2376
|
+
let cursorB = 0
|
|
2377
|
+
for (let i = 0; i < blocks.length; i++) {
|
|
2378
|
+
const b = blocks[i]
|
|
2379
|
+
if (b.start > cursorB) parts.push(buf.slice(cursorB, b.start).toString('utf8'))
|
|
2380
|
+
if (b.keep) parts.push(buf.slice(b.start, b.end).toString('utf8'))
|
|
2381
|
+
cursorB = b.end
|
|
2382
|
+
}
|
|
2383
|
+
if (cursorB < buf.length) parts.push(buf.slice(cursorB).toString('utf8'))
|
|
2384
|
+
const newText = parts.join('')
|
|
2385
|
+
// 归档:整条原文(writeFullRaw 绕开 anchor 事务;归档文件从不参与解析/索引)
|
|
2386
|
+
const archiveFile = layer === 'user'
|
|
2387
|
+
? path.join(dshHome(), 'memory', 'archived-user.md')
|
|
2388
|
+
: path.join(paths.projectDir, 'archive', 'notes-archived.md')
|
|
2389
|
+
const archiveText = removedIdx.map((i2) => {
|
|
2390
|
+
const r = anchored[i2]
|
|
2391
|
+
return buf.slice(r.markerByteStart, r.byteEnd).toString('utf8')
|
|
2392
|
+
}).join('\n')
|
|
2393
|
+
const existingArchive = await this.readTextSafe(archiveFile)
|
|
2394
|
+
await this.writeFullRaw(archiveFile, (existingArchive ? existingArchive.replace(/\s+$/, '') + '\n' : '') + archiveText + '\n')
|
|
2395
|
+
const rr = await store.replace(filePath, newText)
|
|
2396
|
+
if (!rr.ok) { console.error('[compacted-anchored] replace failed: ' + rr.reason); return false }
|
|
2397
|
+
const freshText = await this.readTextSafe(filePath)
|
|
2398
|
+
if (layer === 'user') this.state.userText = freshText
|
|
2399
|
+
else this.state.notesText = freshText
|
|
2400
|
+
if (this._budgets) {
|
|
2401
|
+
const b = this._budgets.get('today')
|
|
2402
|
+
if (b && b.date === this.memToday()) b[layer] = 0
|
|
2403
|
+
}
|
|
2404
|
+
this._lastCompactAt = Date.now()
|
|
2405
|
+
this.state.loadedAt = Date.now()
|
|
2406
|
+
console.log('[compacted-anchored] ' + layer + ': kept=' + (anchored.length - removedIdx.length) + '/' + anchored.length + ' archived=' + removedIdx.length + ' bytes=' + cur.length + '->' + newText.length)
|
|
2407
|
+
return true
|
|
2408
|
+
}
|
|
2409
|
+
|
|
2338
2410
|
// ---------- 注入渲染(同步,基于缓存) ----------
|
|
2339
2411
|
// 动态记忆 → ctx.systemPrompt.context()(user-role 快照):内容变化才追加新快照,内容不变不重复注入(dsh-agent-loop project() 去重),
|
|
2340
2412
|
// system prompt 不再包含动态内容 → 字节级稳定 → DeepSeek 前缀缓存全程命中(对比 section 方案:动态内容任何变化都从变化点起击穿整个前缀,含全部历史)
|
|
@@ -2381,6 +2453,17 @@ class MemoryEngine {
|
|
|
2381
2453
|
if (cfg.waterLevelAdvisory !== false && !this.isUnattendedNow() && (s.waterLevelRatio || 0) >= (Number(cfg.waterLevelThreshold) || 0.8) && Date.now() - (s.waterLevelAt || 0) < 600000) {
|
|
2382
2454
|
lines.push('\n' + L('snapshotWaterTitle') + '\n' + L('snapshotWaterBody', { pct: Math.round((s.waterLevelRatio || 0) * 100) }))
|
|
2383
2455
|
}
|
|
2456
|
+
// M-CM1 交接白板:PLAN.md 全貌+最新交接账本,动态快照首位(动态层内容,不碰静态字节;敏感段清洗+行边界硬预算截断)
|
|
2457
|
+
if (cfg.handoffEnabled !== false && (s.planText || s.latestHandoffText)) {
|
|
2458
|
+
const planPart = truncateLinesBounded(stripSensitiveSections(sanitizeForInjection(s.planText || '')), Math.max(Number(cfg.handoffPlanChars) || 1200, 200))
|
|
2459
|
+
if (planPart) lines.push('\n' + L('snapshotPlanTitle') + '\n' + planPart)
|
|
2460
|
+
const ledgerPart = truncateLinesBounded(stripSensitiveSections(sanitizeForInjection(s.latestHandoffText || '')), Math.max(Number(cfg.handoffLedgerChars) || 800, 200))
|
|
2461
|
+
if (ledgerPart) lines.push('\n' + L('snapshotHandoffTitle') + '\n' + ledgerPart)
|
|
2462
|
+
}
|
|
2463
|
+
// M-CM4/M-CM5 水位建议:越阈注入交接+派子代理建议(advisory;10 分钟新鲜度;无人值守静默)
|
|
2464
|
+
if (cfg.waterLevelAdvisory !== false && !this.isUnattendedNow() && (s.waterLevelRatio || 0) >= (Number(cfg.waterLevelThreshold) || 0.8) && Date.now() - (s.waterLevelAt || 0) < 600000) {
|
|
2465
|
+
lines.push('\n' + L('snapshotWaterTitle') + '\n' + L('snapshotWaterBody', { pct: Math.round((s.waterLevelRatio || 0) * 100) }))
|
|
2466
|
+
}
|
|
2384
2467
|
let used = 0
|
|
2385
2468
|
const part = (title, text, max) => {
|
|
2386
2469
|
if (!text) return
|
|
@@ -4829,7 +4912,10 @@ export function apply(ctx, config) {
|
|
|
4829
4912
|
{
|
|
4830
4913
|
const pluginDir = path.dirname(fileURLToPath(import.meta.url))
|
|
4831
4914
|
engine._jsSemantic = createJsSemanticEnginePre({ pluginDir })
|
|
4832
|
-
|
|
4915
|
+
// #15 后续/B:JS 模型下载落位用户目录(~/.dsh/models/js-semantic/)——包目录在 npm 更新时被重装,130MB 曾被冲掉
|
|
4916
|
+
const userModelsRoot = path.join((process.env.DSH_HOME && process.env.DSH_HOME.trim()) || path.join(homedir(), '.dsh'), 'models', 'js-semantic')
|
|
4917
|
+
mkdirSync(userModelsRoot, { recursive: true })
|
|
4918
|
+
engine._jsDownload = createSemanticDownloaderPre({ modelsRoot: userModelsRoot })
|
|
4833
4919
|
// 资产探测(semantic-status / 档位解析 / 引导卡共用):共享实现见 semantic-js-pre 的
|
|
4834
4920
|
// probeJsSemanticAssets —— peer 探测与引擎加载走同一套 Node 解析(2026-09-02 issue 修正)。
|
|
4835
4921
|
// _peerExtraDirs = 深度扫描热接入位(semanticDeepDetect 命中后 probe/加载/档位即时生效)。
|
package/lib/semantic-js.js
CHANGED
|
@@ -1,485 +1,491 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* C2 内置语义引擎宿主(js_semantic_engine_v1)+ 资产下载器(js_semantic_dl_v1)。
|
|
3
|
-
* 2026-08-26 用户裁定:C2 是默认主路径,本模块把 js-semantic-trial 试验模块接入生产。
|
|
4
|
-
*
|
|
5
|
-
* 组成:
|
|
6
|
-
* 1) fuseD6Pre —— 与 Python sidecar 同一契约的 minmax 加权融合(dense 0.7 / lexical 0.3,D6);
|
|
7
|
-
* 2) createJsSemanticEnginePre —— 懒加载 e5-small q8(peer 多候选路径解析)、miv 键索引缓存
|
|
8
|
-
* (trial 版每次全库重嵌,此处修复)、cosine 排名;任何失败 → degraded + null,调用方回退词法;
|
|
9
|
-
* 3) createSemanticDownloaderPre —— 五文件资产清单(SHA256 冻结)双源下载
|
|
10
|
-
* (cn=hf-mirror 国内 / intl=huggingface 国际;auto=国内优先),流式进度 + 哈希校验 + 原子落位;
|
|
11
|
-
*
|
|
12
|
-
* 边界(与 M7-CLOSED-LOOP-WIRING.md 一致):本模块只做检索排序,绝不做激活决策;
|
|
13
|
-
* 「要不要打断」仍属两车道策略(Python sidecar 在场时)。全部函数对非法输入 fail closed。
|
|
14
|
-
*/
|
|
15
|
-
import { createHash } from 'node:crypto'
|
|
16
|
-
import { existsSync, readFileSync, mkdirSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, rmSync } from 'node:fs'
|
|
17
|
-
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
18
|
-
import { createRequire } from 'node:module'
|
|
19
|
-
import path from 'node:path'
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
export const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
const
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (a.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
//
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
*
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
return
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
*
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
const
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
}
|
|
314
|
-
},
|
|
315
|
-
/**
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
const
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
})
|
|
369
|
-
|
|
370
|
-
/**
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
const
|
|
395
|
-
const
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
return { ok:
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* C2 内置语义引擎宿主(js_semantic_engine_v1)+ 资产下载器(js_semantic_dl_v1)。
|
|
3
|
+
* 2026-08-26 用户裁定:C2 是默认主路径,本模块把 js-semantic-trial 试验模块接入生产。
|
|
4
|
+
*
|
|
5
|
+
* 组成:
|
|
6
|
+
* 1) fuseD6Pre —— 与 Python sidecar 同一契约的 minmax 加权融合(dense 0.7 / lexical 0.3,D6);
|
|
7
|
+
* 2) createJsSemanticEnginePre —— 懒加载 e5-small q8(peer 多候选路径解析)、miv 键索引缓存
|
|
8
|
+
* (trial 版每次全库重嵌,此处修复)、cosine 排名;任何失败 → degraded + null,调用方回退词法;
|
|
9
|
+
* 3) createSemanticDownloaderPre —— 五文件资产清单(SHA256 冻结)双源下载
|
|
10
|
+
* (cn=hf-mirror 国内 / intl=huggingface 国际;auto=国内优先),流式进度 + 哈希校验 + 原子落位;
|
|
11
|
+
*
|
|
12
|
+
* 边界(与 M7-CLOSED-LOOP-WIRING.md 一致):本模块只做检索排序,绝不做激活决策;
|
|
13
|
+
* 「要不要打断」仍属两车道策略(Python sidecar 在场时)。全部函数对非法输入 fail closed。
|
|
14
|
+
*/
|
|
15
|
+
import { createHash } from 'node:crypto'
|
|
16
|
+
import { existsSync, readFileSync, mkdirSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync, rmSync } from 'node:fs'
|
|
17
|
+
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
18
|
+
import { createRequire } from 'node:module'
|
|
19
|
+
import path from 'node:path'
|
|
20
|
+
import { homedir } from 'node:os'
|
|
21
|
+
|
|
22
|
+
export const JS_SEMANTIC_ENGINE_VERSION = 'js_semantic_engine_v1'
|
|
23
|
+
export const JS_SEMANTIC_DL_VERSION = 'js_semantic_dl_v1'
|
|
24
|
+
/** D6 冻结融合权重(与 worker DEFAULT_SEARCH_POLICY 一致)。 */
|
|
25
|
+
export const D6_FUSION_WEIGHTS_V1 = Object.freeze({ dense: 0.7, lexical: 0.3 })
|
|
26
|
+
|
|
27
|
+
// ========== 1) D6 minmax 融合(纯函数) ==========
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* pairs: [{memoryId, dense:number|null, lex:number|null}]
|
|
31
|
+
* 两臂各自在非空值域上 minmax 归一(单值/零极差 → 该臂非空值记 0.5);缺失记 0。
|
|
32
|
+
* 返回 [{memoryId, fused, denseN, lexN}] 按 fused 降序、平局 memoryId 升序(确定性)。
|
|
33
|
+
*/
|
|
34
|
+
export function fuseD6Pre(pairs) {
|
|
35
|
+
const list = Array.isArray(pairs) ? pairs.filter((p) => p && typeof p.memoryId === 'string') : []
|
|
36
|
+
const normArm = (key) => {
|
|
37
|
+
const vals = list.map((p) => (typeof p[key] === 'number' && Number.isFinite(p[key]) ? p[key] : null)).filter((v) => v !== null)
|
|
38
|
+
if (!vals.length) return { lo: 0, hi: -1 } // 空臂:全部归 0(下方 missing 分支覆盖)
|
|
39
|
+
const lo = Math.min(...vals)
|
|
40
|
+
const hi = Math.max(...vals)
|
|
41
|
+
return hi > lo ? { lo, hi } : { lo, hi: lo, flat: true }
|
|
42
|
+
}
|
|
43
|
+
const dn = normArm('dense')
|
|
44
|
+
const ln = normArm('lex')
|
|
45
|
+
const normOf = (v, arm) => {
|
|
46
|
+
if (typeof v !== 'number' || !Number.isFinite(v)) return 0
|
|
47
|
+
const a = arm === 'dense' ? dn : ln
|
|
48
|
+
if (a.hi < a.lo) return 0
|
|
49
|
+
if (a.flat) return 0.5
|
|
50
|
+
return (v - a.lo) / (a.hi - a.lo)
|
|
51
|
+
}
|
|
52
|
+
return list
|
|
53
|
+
.map((p) => {
|
|
54
|
+
const denseN = normOf(p.dense, 'dense')
|
|
55
|
+
const lexN = normOf(p.lex, 'lex')
|
|
56
|
+
return {
|
|
57
|
+
memoryId: p.memoryId,
|
|
58
|
+
denseN,
|
|
59
|
+
lexN,
|
|
60
|
+
fused: D6_FUSION_WEIGHTS_V1.dense * denseN + D6_FUSION_WEIGHTS_V1.lexical * lexN,
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
.sort((x, y) => (y.fused !== x.fused ? y.fused - x.fused : (x.memoryId < y.memoryId ? -1 : 1)))
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ========== 2) 引擎宿主 ==========
|
|
67
|
+
|
|
68
|
+
const E5_MODELS_SUBDIR = 'multilingual-e5-small'
|
|
69
|
+
|
|
70
|
+
function defaultModelsDirCandidates(pluginDir) {
|
|
71
|
+
// 用户目录优先(#15 后续/B 修复):~/.dsh/models/js-semantic/ 跨插件升级存活——
|
|
72
|
+
// 包目录(lib/models)在 npm 更新时会被整体重装,下载的 130MB 模型曾被冲掉。
|
|
73
|
+
const dshHome = (process.env.DSH_HOME && process.env.DSH_HOME.trim()) || path.join(homedir(), '.dsh')
|
|
74
|
+
return [
|
|
75
|
+
path.join(dshHome, 'models', 'js-semantic'),
|
|
76
|
+
path.join(pluginDir, 'models'),
|
|
77
|
+
path.join(pluginDir, '..', 'artifacts', 'm7-live-pre', 'js-semantic-trial', 'models'),
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function defaultPeerDirCandidates(pluginDir) {
|
|
82
|
+
// pluginDir = <pkg>/lib(2026-09-02 issue 复核修正:此前上溯层级错位 → peerPresent 恒 false)。
|
|
83
|
+
// 静态候选仅作兜底,主路径是 resolvePeerTransformersDir 的真实解析:
|
|
84
|
+
// 1) lib/node_modules —— issue 临时 junction 绕过位(向后兼容)
|
|
85
|
+
// 2) <pkg>/node_modules —— 包内邻接位(lib 上 1 级再进 node_modules)
|
|
86
|
+
// 3) lib 上 3 级直拼 @huggingface —— 标准布局即 <root>/node_modules(pnpm hoisted/npm 提升),
|
|
87
|
+
// pnpm isolated 下即虚拟存储包内位 .pnpm/<hash>/node_modules
|
|
88
|
+
// 4) 开发树 artifacts
|
|
89
|
+
return [
|
|
90
|
+
path.join(pluginDir, 'node_modules', '@huggingface', 'transformers'),
|
|
91
|
+
path.join(pluginDir, '..', 'node_modules', '@huggingface', 'transformers'),
|
|
92
|
+
path.join(pluginDir, '..', '..', '..', '@huggingface', 'transformers'),
|
|
93
|
+
path.join(pluginDir, '..', 'artifacts', 'm7-live-pre', 'js-semantic-trial', 'node_modules', '@huggingface', 'transformers'),
|
|
94
|
+
]
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* peer 探测(2026-09-02 issue 修正):真实解析优先 —— createRequire 从 lib/ 起走 Node 标准
|
|
99
|
+
* 解析,天然覆盖 npm/pnpm(hoisted/isolated 虚拟存储)/junction 全部布局,且不执行模块本体
|
|
100
|
+
* (require.resolve 只解析不加载);静态候选仅作 exports 损坏等极端情形的兜底;
|
|
101
|
+
* extraDirs = 深度扫描(semanticDeepDetect)热接入的额外目录,最后兜底检查。
|
|
102
|
+
* 返回命中的包目录,''=未找到。
|
|
103
|
+
*/
|
|
104
|
+
export function resolvePeerTransformersDir(pluginDir, extraDirs) {
|
|
105
|
+
try {
|
|
106
|
+
const req = createRequire(path.join(pluginDir, 'noop.js'))
|
|
107
|
+
const entry = req.resolve('@huggingface/transformers')
|
|
108
|
+
// entry 是包 main 文件(可能在任意子目录,如 dist/transformers.node.cjs):
|
|
109
|
+
// 从 entry 向上走到 package.json 所在层 = 包目录
|
|
110
|
+
if (entry && existsSync(entry)) {
|
|
111
|
+
let dir = path.dirname(entry)
|
|
112
|
+
while (dir && !existsSync(path.join(dir, 'package.json'))) {
|
|
113
|
+
const up = path.dirname(dir)
|
|
114
|
+
if (up === dir) { dir = ''; break }
|
|
115
|
+
dir = up
|
|
116
|
+
}
|
|
117
|
+
if (dir) return dir
|
|
118
|
+
}
|
|
119
|
+
} catch (_) { /* 静态候选兜底 */ }
|
|
120
|
+
const hit = defaultPeerDirCandidates(pluginDir).find((c) => existsSync(c))
|
|
121
|
+
if (hit) return hit
|
|
122
|
+
for (const d of Array.isArray(extraDirs) ? extraDirs : []) {
|
|
123
|
+
if (typeof d === 'string' && d && existsSync(d)) return d
|
|
124
|
+
}
|
|
125
|
+
return ''
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* 深度扫描(0.1.37「打开即自动检测」;issue #14 后续):零依赖手扫常见安装根。
|
|
130
|
+
* 每个 root 探两层:root 自身的 node_modules(直传 profile 根/包根时命中)+ 一级子目录
|
|
131
|
+
* (直传 ~/.dsh/profiles 时枚举各 profile);每个落位再探 node_modules/@huggingface/
|
|
132
|
+
* transformers、pnpm 虚拟存储 .pnpm/node_modules/... 与 .pnpm/@huggingface+transformers@版本
|
|
133
|
+
* /node_modules/...(isolated 模式包体落位)。返回存在的包目录列表(去重,保持扫描序)。
|
|
134
|
+
*/
|
|
135
|
+
export function deepScanPeerTransformers(roots) {
|
|
136
|
+
const out = []
|
|
137
|
+
const seen = new Set()
|
|
138
|
+
const push = (p) => { if (p && !seen.has(p) && existsSync(p)) { seen.add(p); out.push(p) } }
|
|
139
|
+
const probeDir = (base) => {
|
|
140
|
+
if (!base || typeof base !== 'string') return
|
|
141
|
+
push(path.join(base, 'node_modules', '@huggingface', 'transformers'))
|
|
142
|
+
push(path.join(base, 'node_modules', '.pnpm', 'node_modules', '@huggingface', 'transformers'))
|
|
143
|
+
const pnpmDir = path.join(base, 'node_modules', '.pnpm')
|
|
144
|
+
let entries = []
|
|
145
|
+
try { entries = readdirSync(pnpmDir) } catch (_) { /* base 无 .pnpm 虚拟存储 */ }
|
|
146
|
+
for (const e of entries) {
|
|
147
|
+
if (/^@huggingface\+transformers@/.test(e)) push(path.join(pnpmDir, e, 'node_modules', '@huggingface', 'transformers'))
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
for (const root of Array.isArray(roots) ? roots : []) {
|
|
151
|
+
if (!root || typeof root !== 'string') continue
|
|
152
|
+
probeDir(root)
|
|
153
|
+
let children = []
|
|
154
|
+
try { children = readdirSync(root) } catch (_) { /* root 不可枚举 */ }
|
|
155
|
+
for (const c of children.slice(0, 64)) probeDir(path.join(root, c))
|
|
156
|
+
}
|
|
157
|
+
return out
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* semanticAssetProbe 共享实现(index.js 的 status API / 档位解析 / 引导卡同源):
|
|
162
|
+
* 模型资产=发行包 lib/models 优先,其次开发树;peer=resolvePeerTransformersDir。
|
|
163
|
+
* extraDirs 透传给 peer 探测(深度扫描热接入位)。返回形状与 v0.1.36 完全一致。
|
|
164
|
+
*/
|
|
165
|
+
export function probeJsSemanticAssets(pluginDir, extraDirs) {
|
|
166
|
+
const dshHome = (process.env.DSH_HOME && process.env.DSH_HOME.trim()) || path.join(homedir(), '.dsh')
|
|
167
|
+
const modelCands = [
|
|
168
|
+
path.join(dshHome, 'models', 'js-semantic', E5_MODELS_SUBDIR, 'onnx', 'model_quantized.onnx'),
|
|
169
|
+
path.join(pluginDir, 'models', E5_MODELS_SUBDIR, 'onnx', 'model_quantized.onnx'),
|
|
170
|
+
path.join(pluginDir, '..', 'artifacts', 'm7-live-pre', 'js-semantic-trial', 'models', E5_MODELS_SUBDIR, 'onnx', 'model_quantized.onnx'),
|
|
171
|
+
]
|
|
172
|
+
const modelOnnx = modelCands.find((c) => existsSync(c)) || modelCands[0]
|
|
173
|
+
const assetPresent = existsSync(modelOnnx)
|
|
174
|
+
const peerDir = resolvePeerTransformersDir(pluginDir, extraDirs)
|
|
175
|
+
return {
|
|
176
|
+
assetPresent,
|
|
177
|
+
peerPresent: Boolean(peerDir),
|
|
178
|
+
ready: Boolean(assetPresent && peerDir),
|
|
179
|
+
assetBytes: assetPresent ? statSync(modelOnnx).size : 0,
|
|
180
|
+
assetPath: modelOnnx,
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function importPeerTransformers(peerDirs) {
|
|
185
|
+
try {
|
|
186
|
+
return await import('@huggingface/transformers') // 发行包:peer 邻接安装
|
|
187
|
+
} catch (_) { /* 开发树:按路径解析 */ }
|
|
188
|
+
for (const dir of peerDirs || []) {
|
|
189
|
+
try {
|
|
190
|
+
const pkgPath = path.join(dir, 'package.json')
|
|
191
|
+
if (!existsSync(pkgPath)) continue
|
|
192
|
+
const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'))
|
|
193
|
+
const mainFile = String(pkg.main || 'dist/transformers.js')
|
|
194
|
+
const entry = path.join(dir, mainFile)
|
|
195
|
+
if (!existsSync(entry)) continue
|
|
196
|
+
return await import(pathToFileURL(entry).href)
|
|
197
|
+
} catch (_) { /* 尝试下一候选 */ }
|
|
198
|
+
}
|
|
199
|
+
const e = new Error('js-semantic-engine: optional peer @huggingface/transformers not installed; tier stays disabled')
|
|
200
|
+
e.code = 'JS_SEMANTIC_PEER_MISSING'
|
|
201
|
+
throw e
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* opts: { pluginDir, modelsDirCandidates?, peerDirCandidates?, injectEmbedder? }
|
|
206
|
+
* injectEmbedder: 测试注入 {embedQuery(text)→Float32Array, embedPassages(texts)→Float32Array[]};
|
|
207
|
+
* 注入时跳过真实模型加载(冒烟测试离线跑全逻辑)。
|
|
208
|
+
*/
|
|
209
|
+
export function createJsSemanticEnginePre(opts = {}) {
|
|
210
|
+
const pluginDir = opts.pluginDir
|
|
211
|
+
const modelDirCands = opts.modelsDirCandidates || defaultModelsDirCandidates(pluginDir)
|
|
212
|
+
const peerDirCands = opts.peerDirCandidates || defaultPeerDirCandidates(pluginDir)
|
|
213
|
+
let tier = null // {embedQuery, embedPassages, model}
|
|
214
|
+
let tierPromise = null
|
|
215
|
+
let degraded = '' // 非空 = 初始化失败原因(调用方回退词法)
|
|
216
|
+
let lastRankError = '' // 运行期排名失败原因(诊断用;rank 恒返回 null 由调用方回退)
|
|
217
|
+
let idx = { miv: null, entries: [] } // entries [{memoryId, vec}]
|
|
218
|
+
let rebuilding = null // 单飞行重建 promise
|
|
219
|
+
|
|
220
|
+
function modelsDir() {
|
|
221
|
+
return (modelDirCands.find((c) => existsSync(path.join(c, E5_MODELS_SUBDIR))) || modelDirCands[0])
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async function ensureTier() {
|
|
225
|
+
if (tier) return tier
|
|
226
|
+
if (degraded) throw new Error(degraded)
|
|
227
|
+
if (!tierPromise) {
|
|
228
|
+
tierPromise = (async () => {
|
|
229
|
+
if (opts.injectEmbedder) return opts.injectEmbedder
|
|
230
|
+
const md = modelsDir()
|
|
231
|
+
if (!existsSync(path.join(md, E5_MODELS_SUBDIR, 'onnx', 'model_quantized.onnx'))) {
|
|
232
|
+
throw new Error('js-semantic-engine: model asset missing under ' + md)
|
|
233
|
+
}
|
|
234
|
+
const mod = await importPeerTransformers(peerDirCands)
|
|
235
|
+
const { pipeline, env } = mod
|
|
236
|
+
env.allowRemoteModels = false // 全离线:记忆不出电脑
|
|
237
|
+
env.localModelPath = md
|
|
238
|
+
const extractor = await pipeline('feature-extraction', E5_MODELS_SUBDIR, { dtype: 'q8' })
|
|
239
|
+
const built = {
|
|
240
|
+
model: E5_MODELS_SUBDIR + '/q8',
|
|
241
|
+
async embedQuery(text) {
|
|
242
|
+
const out = await extractor('query: ' + clean(String(text || '')), { pooling: 'mean', normalize: true, truncation: true })
|
|
243
|
+
return Float32Array.from(out.data)
|
|
244
|
+
},
|
|
245
|
+
async embedPassages(texts) {
|
|
246
|
+
const out = []
|
|
247
|
+
for (const t of texts) {
|
|
248
|
+
const o = await extractor('passage: ' + clean(String(t || '')), { pooling: 'mean', normalize: true, truncation: true })
|
|
249
|
+
out.push(Float32Array.from(o.data))
|
|
250
|
+
}
|
|
251
|
+
return out
|
|
252
|
+
},
|
|
253
|
+
}
|
|
254
|
+
// 规范 F3 最小推理自检(RELEASE-SEMANTIC-OPTION.md):加载后立即一次真实编码,
|
|
255
|
+
// 校验维度(384)与非退化输出(模长≈1 且非全零)。失败=资产损坏 → 整体 degraded,
|
|
256
|
+
// 调用方回退 C1,不给下游喂坏向量。
|
|
257
|
+
const probe = await built.embedQuery('semantic self-test 探针')
|
|
258
|
+
if (!(probe.length === 384)) throw new Error('js-semantic-engine: self-test dimension mismatch: ' + probe.length)
|
|
259
|
+
let sum = 0
|
|
260
|
+
for (let i = 0; i < probe.length; i++) sum += probe[i] * probe[i]
|
|
261
|
+
const norm = Math.sqrt(sum)
|
|
262
|
+
if (!(norm > 0.9 && norm < 1.1)) throw new Error('js-semantic-engine: self-test degenerate output (norm=' + norm.toFixed(3) + ')')
|
|
263
|
+
return built
|
|
264
|
+
})()
|
|
265
|
+
}
|
|
266
|
+
try {
|
|
267
|
+
tier = await tierPromise
|
|
268
|
+
return tier
|
|
269
|
+
} catch (e) {
|
|
270
|
+
degraded = String(e && e.message || e).slice(0, 160)
|
|
271
|
+
tierPromise = null
|
|
272
|
+
throw e
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
async function buildIndexIfStale(miv, records) {
|
|
277
|
+
if (idx.miv === miv) return idx
|
|
278
|
+
if (rebuilding) return rebuilding
|
|
279
|
+
rebuilding = (async () => {
|
|
280
|
+
const t = await ensureTier()
|
|
281
|
+
const seen = new Set()
|
|
282
|
+
const items = []
|
|
283
|
+
for (const r of Array.isArray(records) ? records : []) {
|
|
284
|
+
if (!r || typeof r.memoryId !== 'string' || seen.has(r.memoryId)) continue
|
|
285
|
+
seen.add(r.memoryId)
|
|
286
|
+
items.push({ memoryId: r.memoryId, text: String(r.text || '') })
|
|
287
|
+
}
|
|
288
|
+
const vecs = await t.embedPassages(items.map((i) => i.text.slice(0, 1200)))
|
|
289
|
+
idx = { miv: String(miv || ''), entries: items.map((it, i) => ({ memoryId: it.memoryId, vec: vecs[i] })) }
|
|
290
|
+
return idx
|
|
291
|
+
})()
|
|
292
|
+
try {
|
|
293
|
+
return await rebuilding
|
|
294
|
+
} finally {
|
|
295
|
+
rebuilding = null
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
return {
|
|
300
|
+
version: JS_SEMANTIC_ENGINE_VERSION,
|
|
301
|
+
/** 探测状态(不触发加载):ready 仅表示「可尝试」,真实可用性以 rank 成功为准。 */
|
|
302
|
+
status() {
|
|
303
|
+
return {
|
|
304
|
+
version: JS_SEMANTIC_ENGINE_VERSION,
|
|
305
|
+
assetPresent: existsSync(path.join(modelsDir(), E5_MODELS_SUBDIR, 'onnx', 'model_quantized.onnx')),
|
|
306
|
+
ready: !!tier,
|
|
307
|
+
degraded,
|
|
308
|
+
lastRankError,
|
|
309
|
+
model: tier ? tier.model : null,
|
|
310
|
+
indexedRecords: idx.entries.length,
|
|
311
|
+
miv: idx.miv,
|
|
312
|
+
embedding: !!rebuilding,
|
|
313
|
+
}
|
|
314
|
+
},
|
|
315
|
+
/** 深度扫描热接入(0.1.37):把扫描发现的额外 peer 目录并入加载候选(去重,立即生效)。 */
|
|
316
|
+
addPeerDirCandidates(dirs) {
|
|
317
|
+
for (const d of Array.isArray(dirs) ? dirs : []) {
|
|
318
|
+
if (typeof d === 'string' && d && !peerDirCands.includes(d)) peerDirCands.push(d)
|
|
319
|
+
}
|
|
320
|
+
},
|
|
321
|
+
/**
|
|
322
|
+
* 对语料做稠密排名。返回 {miv, scores:Map(memoryId→cosine)};失败返回 null(调用方回退词法)。
|
|
323
|
+
*/
|
|
324
|
+
async rank(corpusSnap, queryText) {
|
|
325
|
+
try {
|
|
326
|
+
const miv = String((corpusSnap && corpusSnap.memoryIndexVersion) || '')
|
|
327
|
+
if (!miv || !/^idx_[0-9a-f]{32}$/.test(miv)) return null
|
|
328
|
+
const t = await ensureTier()
|
|
329
|
+
const built = await buildIndexIfStale(miv, corpusSnap && corpusSnap.records)
|
|
330
|
+
if (!built.entries.length) return { miv, scores: new Map() }
|
|
331
|
+
const qv = await t.embedQuery(String(queryText || '').slice(0, 2000))
|
|
332
|
+
const scores = new Map()
|
|
333
|
+
for (const en of built.entries) {
|
|
334
|
+
if (!en.vec || en.vec.length !== qv.length) continue
|
|
335
|
+
let d = 0
|
|
336
|
+
for (let i = 0; i < qv.length; i++) d += qv[i] * en.vec[i]
|
|
337
|
+
scores.set(en.memoryId, d)
|
|
338
|
+
}
|
|
339
|
+
return { miv, scores }
|
|
340
|
+
} catch (e) {
|
|
341
|
+
lastRankError = String(e && e.message || e).slice(0, 160)
|
|
342
|
+
return null // fail closed:词法回退由调用方自然发生
|
|
343
|
+
}
|
|
344
|
+
},
|
|
345
|
+
_resetForTest() { tier = null; tierPromise = null; degraded = ''; idx = { miv: null, entries: [] }; rebuilding = null },
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
function clean(t) {
|
|
350
|
+
return String(t || '').replace(/\s+/g, ' ').trim().slice(0, 1200)
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
// ========== 3) 资产下载器 ==========
|
|
354
|
+
|
|
355
|
+
/** 资产清单:SHA256 于 2026-08-26 从已验证开发树副本冻结(Xenova/multilingual-e5-small q8)。 */
|
|
356
|
+
const E5_SMALL_Q8_FILES_V1 = Object.freeze([
|
|
357
|
+
Object.freeze({ rel: 'multilingual-e5-small/onnx/model_quantized.onnx', bytes: 118308185, sha256: 'f80102d3f2a1229f387d3c81909990d8945513e347b0eab049f7de3c6f98c193' }),
|
|
358
|
+
Object.freeze({ rel: 'multilingual-e5-small/tokenizer.json', bytes: 17082730, sha256: '0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39' }),
|
|
359
|
+
Object.freeze({ rel: 'multilingual-e5-small/config.json', bytes: 658, sha256: 'cb99455288675345e1a4f411438d5d0adbba5fbd3a67ea4fb03c015433b996c1' }),
|
|
360
|
+
Object.freeze({ rel: 'multilingual-e5-small/special_tokens_map.json', bytes: 167, sha256: 'd05497f1da52c5e09554c0cd874037a083e1dc1b9cfd48034d1c717f1afc07a7' }),
|
|
361
|
+
Object.freeze({ rel: 'multilingual-e5-small/tokenizer_config.json', bytes: 443, sha256: 'a1d6bc8734a6f635dc158508bef000f8e2e5a759c7d92f984b2c86e5ff53425b' }),
|
|
362
|
+
])
|
|
363
|
+
export const E5_SMALL_Q8_MANIFEST_V1 = Object.freeze({
|
|
364
|
+
model: E5_MODELS_SUBDIR,
|
|
365
|
+
dtype: 'q8',
|
|
366
|
+
files: E5_SMALL_Q8_FILES_V1,
|
|
367
|
+
totalBytes: E5_SMALL_Q8_FILES_V1.reduce((s, f) => s + f.bytes, 0),
|
|
368
|
+
})
|
|
369
|
+
|
|
370
|
+
/** 双通道:cn=hf-mirror(国内可达)/ intl=huggingface 官方。auto=国内优先(用户群主体)。 */
|
|
371
|
+
export const SEMANTIC_DL_MIRRORS_V1 = Object.freeze({
|
|
372
|
+
cn: 'https://hf-mirror.com/Xenova/multilingual-e5-small/resolve/main/',
|
|
373
|
+
intl: 'https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/',
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* opts: { modelsRoot(下载落位根目录), manifest?, mirrors?, fetchImpl?, chunkBytes? }
|
|
378
|
+
* 状态机: idle → downloading → verifying → done | error | cancelled;单飞行。
|
|
379
|
+
*/
|
|
380
|
+
export function createSemanticDownloaderPre(opts = {}) {
|
|
381
|
+
const modelsRoot = String(opts.modelsRoot || '')
|
|
382
|
+
const manifest = opts.manifest || E5_SMALL_Q8_MANIFEST_V1
|
|
383
|
+
const doFetch = opts.fetchImpl || ((u, o) => fetch(u, o))
|
|
384
|
+
let st = { phase: 'idle', file: '', bytesDone: 0, bytesTotal: manifest.totalBytes, mirrorUsed: '', error: '', startedAt: 0 }
|
|
385
|
+
let cancelFlag = false
|
|
386
|
+
|
|
387
|
+
function setState(patch) { st = Object.assign({}, st, patch) }
|
|
388
|
+
|
|
389
|
+
function tmpDir() { return path.join(modelsRoot, '.tmp-dl') }
|
|
390
|
+
|
|
391
|
+
async function fetchToFile(fileRec, base) {
|
|
392
|
+
// 镜像 base 已含模型子目录(Xenova/multilingual-e5-small/resolve/main/),
|
|
393
|
+
// rel 带 multilingual-e5-small/ 前缀用于落位;URL 拼接时剥掉前缀避免路径重复 404。
|
|
394
|
+
const urlRel = fileRec.rel.indexOf(E5_MODELS_SUBDIR + '/') === 0 ? fileRec.rel.slice(E5_MODELS_SUBDIR.length + 1) : fileRec.rel
|
|
395
|
+
const res = await doFetch(base + fileRelUrl(urlRel), { redirect: 'follow' })
|
|
396
|
+
if (!res.ok) throw new Error('http-' + res.status)
|
|
397
|
+
if (!res.body) throw new Error('no-body')
|
|
398
|
+
const total = Number(res.headers.get('content-length')) || fileRec.bytes
|
|
399
|
+
mkdirSync(tmpDir(), { recursive: true })
|
|
400
|
+
const dst = path.join(tmpDir(), path.basename(fileRec.rel))
|
|
401
|
+
const hash = createHash('sha256')
|
|
402
|
+
const reader = res.body.getReader()
|
|
403
|
+
const fd = writeFileSync // 占位避免未用告警;真正写入走手动缓冲
|
|
404
|
+
let buf = Buffer.alloc(0)
|
|
405
|
+
let n = 0
|
|
406
|
+
for (;;) {
|
|
407
|
+
if (cancelFlag) throw new Error('cancelled')
|
|
408
|
+
const { done, value } = await reader.read()
|
|
409
|
+
if (done) break
|
|
410
|
+
buf = Buffer.concat([buf, Buffer.from(value)])
|
|
411
|
+
n += value.byteLength
|
|
412
|
+
hash.update(Buffer.from(value))
|
|
413
|
+
setState({ file: fileRec.rel, bytesDone: st.bytesDoneBase + n, bytesTotal: st.bytesTotalBase + Math.max(0, total - fileRec.bytes) })
|
|
414
|
+
if (buf.length >= (opts.chunkBytes || 1 << 20)) {
|
|
415
|
+
appendChunk(dst, buf); buf = Buffer.alloc(0)
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
if (buf.length) appendChunk(dst, buf)
|
|
419
|
+
void fd
|
|
420
|
+
const hex = hash.digest('hex')
|
|
421
|
+
if (hex !== fileRec.sha256) {
|
|
422
|
+
try { unlinkSync(dst) } catch (_) {}
|
|
423
|
+
const e = new Error('sha256-mismatch:' + hex.slice(0, 12))
|
|
424
|
+
e.code = 'SHA256_MISMATCH'
|
|
425
|
+
throw e
|
|
426
|
+
}
|
|
427
|
+
return { dst, got: n }
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
function appendChunk(dst, buf) {
|
|
431
|
+
// writeFileSync flag 'a':追加;首块前由调用方确保不存在
|
|
432
|
+
writeFileSync(dst, buf, { flag: 'a' })
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function fileRelUrl(rel) { return rel.split('\\').join('/') }
|
|
436
|
+
|
|
437
|
+
async function run(order) {
|
|
438
|
+
setState({ phase: 'downloading', bytesDone: 0, error: '', startedAt: Date.now() })
|
|
439
|
+
rmSync(tmpDir(), { recursive: true, force: true })
|
|
440
|
+
let cum = 0
|
|
441
|
+
for (const f of manifest.files) {
|
|
442
|
+
st.bytesDoneBase = cum
|
|
443
|
+
st.bytesTotalBase = cum + f.bytes
|
|
444
|
+
let ok = false
|
|
445
|
+
let lastErr = ''
|
|
446
|
+
for (const m of order) {
|
|
447
|
+
if (cancelFlag) break
|
|
448
|
+
try {
|
|
449
|
+
setState({ mirrorUsed: m })
|
|
450
|
+
const r = await fetchToFile(f, SEMANTIC_DL_MIRRORS_V1[m])
|
|
451
|
+
const final = path.join(modelsRoot, f.rel)
|
|
452
|
+
mkdirSync(path.dirname(final), { recursive: true })
|
|
453
|
+
try { unlinkSync(final) } catch (_) {}
|
|
454
|
+
renameSync(r.dst, final)
|
|
455
|
+
ok = true
|
|
456
|
+
break
|
|
457
|
+
} catch (e) {
|
|
458
|
+
lastErr = (e && e.code === 'SHA256_MISMATCH' ? e.message : String(e && e.message || e)).slice(0, 140)
|
|
459
|
+
if (e && e.message === 'cancelled') break
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
if (cancelFlag) { setState({ phase: 'cancelled', error: 'user-cancelled', file: f.rel }); return }
|
|
463
|
+
if (!ok) { setState({ phase: 'error', error: (f.rel + ' ← ' + lastErr) }); return }
|
|
464
|
+
setState({ phase: 'verifying' })
|
|
465
|
+
cum += f.bytes
|
|
466
|
+
setState({ bytesDone: cum, bytesTotalBase: cum, bytesDoneBase: cum })
|
|
467
|
+
}
|
|
468
|
+
rmSync(tmpDir(), { recursive: true, force: true })
|
|
469
|
+
setState({ phase: 'done', mirrorUsed: st.mirrorUsed, error: '' })
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return {
|
|
473
|
+
version: JS_SEMANTIC_DL_VERSION,
|
|
474
|
+
start(mirror) {
|
|
475
|
+
const m = String(mirror || 'auto')
|
|
476
|
+
if (!['auto', 'cn', 'intl'].includes(m)) return { ok: false, reason: 'bad-mirror' }
|
|
477
|
+
if (st.phase === 'downloading' || st.phase === 'verifying') return { ok: false, reason: 'already-running' }
|
|
478
|
+
if (!modelsRoot) return { ok: false, reason: 'no-models-root' }
|
|
479
|
+
cancelFlag = false
|
|
480
|
+
const order = m === 'auto' ? ['cn', 'intl'] : [m]
|
|
481
|
+
void run(order).catch((e) => setState({ phase: 'error', error: String(e && e.message || e).slice(0, 160) }))
|
|
482
|
+
return { ok: true }
|
|
483
|
+
},
|
|
484
|
+
cancel() {
|
|
485
|
+
if (st.phase !== 'downloading' && st.phase !== 'verifying') return { ok: false, reason: 'not-running' }
|
|
486
|
+
cancelFlag = true
|
|
487
|
+
return { ok: true }
|
|
488
|
+
},
|
|
489
|
+
state() { return Object.assign({}, st) },
|
|
490
|
+
}
|
|
491
|
+
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@a9i5k4/dsh-auto-memory",
|
|
3
3
|
"description": "Proactive associative memory for DSH: zero-prompt recall injected before the model speaks, three-layer auto-consolidation, skill crystallization, and Astra-style context management - handoff ledgers, PLAN whiteboard, water-level sensing. Local-first, model-agnostic, zero deps. 主动联想记忆+Astra 式上下文管理:自动唤回/自动沉淀/技能固化/交接账本与白板跨窗口续命/水位感知。",
|
|
4
|
-
"version": "2.1.
|
|
4
|
+
"version": "2.1.9",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"exports": {
|