dsh-layered-memory 0.7.1 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +165 -75
- package/README.md +174 -104
- package/assets/img/EmbeddingSource.png +0 -0
- package/assets/img/MemoryTools.png +0 -0
- package/assets/img/ToolTrajectory.png +0 -0
- package/assets/readme/bench-dialog.svg +87 -0
- package/assets/readme/bench-workflow.svg +79 -0
- package/assets/readme/flow.svg +74 -74
- package/assets/readme/storage.svg +56 -56
- package/dist/client.js +380 -2
- package/dist/config.d.ts +41 -1
- package/dist/config.js +10 -3
- package/dist/hooks/recall.d.ts +11 -8
- package/dist/hooks/recall.js +70 -51
- package/dist/index.d.ts +20 -0
- package/dist/index.js +15 -4
- package/dist/llm.d.ts +28 -0
- package/dist/llm.js +33 -0
- package/dist/pipeline/l1.js +5 -4
- package/dist/pipeline/l2.js +7 -2
- package/dist/pipeline/l3.js +7 -2
- package/dist/pipeline/rebuild.js +2 -2
- package/dist/pipeline/runner.d.ts +45 -8
- package/dist/pipeline/runner.js +185 -39
- package/dist/pipeline/trigger.d.ts +38 -0
- package/dist/pipeline/trigger.js +64 -0
- package/dist/settings.d.ts +14 -0
- package/dist/settings.js +41 -2
- package/dist/stats.js +113 -1
- package/dist/store/bm25.js +2 -1
- package/dist/store/download-queue.d.ts +35 -2
- package/dist/store/download-queue.js +102 -5
- package/dist/store/embedding.d.ts +12 -6
- package/dist/store/embedding.js +10 -7
- package/dist/store/l0.d.ts +2 -0
- package/dist/store/l0.js +4 -0
- package/dist/store/l1.d.ts +2 -0
- package/dist/store/l1.js +2 -2
- package/dist/store/model-catalog.js +1 -1
- package/dist/store/pending.d.ts +27 -6
- package/dist/store/pending.js +49 -9
- package/dist/store/search-utils.js +3 -2
- package/dist/store/session-modes.d.ts +4 -0
- package/dist/store/session-modes.js +16 -0
- package/dist/store/sqlite.d.ts +12 -0
- package/dist/store/sqlite.js +103 -5
- package/dist/util/recall-budget.d.ts +32 -0
- package/dist/util/recall-budget.js +85 -0
- package/dist/util/text.d.ts +8 -3
- package/dist/util/text.js +41 -15
- package/dist/util/tokenizer.d.ts +14 -0
- package/dist/util/tokenizer.js +53 -0
- package/package.json +6 -3
package/dist/stats.js
CHANGED
|
@@ -10,6 +10,9 @@ import { createRequire } from 'node:module';
|
|
|
10
10
|
import { closeSync, openSync, readSync, statSync } from 'node:fs';
|
|
11
11
|
import { join } from 'node:path';
|
|
12
12
|
import { resolveDataDir } from './config.js';
|
|
13
|
+
import { effectiveCfg } from './pipeline/runner.js';
|
|
14
|
+
import { LAYER_DEFAULT_BUDGETS, resolveModelRoute } from './llm.js';
|
|
15
|
+
import { errDetail } from './util/filelog.js';
|
|
13
16
|
const require = createRequire(import.meta.url);
|
|
14
17
|
export const PLUGIN_VERSION = require('../package.json').version;
|
|
15
18
|
/** 注册状态 RPC(web 侧 connection 服务可选,缺失时跳过,不影响插件主体)。 */
|
|
@@ -30,6 +33,7 @@ export function registerMemoryRpc(ctx, cfg, stores, logger, status, live, modes,
|
|
|
30
33
|
const dispose = connection.rpc.handle('/rpc', async (endpoint, payload) => {
|
|
31
34
|
try {
|
|
32
35
|
const value = await handleEndpoint(endpoint, payload, {
|
|
36
|
+
ctx,
|
|
33
37
|
cfg,
|
|
34
38
|
stores,
|
|
35
39
|
status,
|
|
@@ -157,9 +161,14 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
157
161
|
}
|
|
158
162
|
case 'dsh-memory/settings-get': {
|
|
159
163
|
const s = live?.get();
|
|
164
|
+
const budgets = s?.distillBudgets ?? { extract: 0, dedup: 0, l2: 0, l3: 0 };
|
|
160
165
|
return {
|
|
161
166
|
supported: live?.supported ?? false,
|
|
162
|
-
settings: s ?? {
|
|
167
|
+
settings: s ?? {
|
|
168
|
+
enabled: true, capture: true, distill: true, recall: true,
|
|
169
|
+
reasoningEffort: '', distillProvider: '', distillModel: '',
|
|
170
|
+
distillBudgets: { extract: 0, dedup: 0, l2: 0, l3: 0 }, distillMaxInputChars: 0,
|
|
171
|
+
},
|
|
163
172
|
// 静态部署上限(cordis.patch.yml):运行时开关与它取 AND
|
|
164
173
|
ceilings: { capture: cfg.capture.enabled, distill: cfg.extract.enabled, recall: cfg.recall.enabled },
|
|
165
174
|
// 蒸馏思考档位:current 是运行时覆盖('' = 跟随配置),effective 是实际生效值
|
|
@@ -168,6 +177,23 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
168
177
|
effective: s?.reasoningEffort || cfg.llm.reasoningEffort,
|
|
169
178
|
fallback: cfg.llm.reasoningEffort,
|
|
170
179
|
},
|
|
180
|
+
// 分层输出预算:current 是运行时覆盖(0 = 跟随默认),defaults 是内置默认(UI 占位/提示用)
|
|
181
|
+
budgets: {
|
|
182
|
+
current: budgets,
|
|
183
|
+
defaults: { ...LAYER_DEFAULT_BUDGETS },
|
|
184
|
+
effective: {
|
|
185
|
+
extract: budgets.extract > 0 ? budgets.extract : LAYER_DEFAULT_BUDGETS.extract,
|
|
186
|
+
dedup: budgets.dedup > 0 ? budgets.dedup : LAYER_DEFAULT_BUDGETS.dedup,
|
|
187
|
+
l2: budgets.l2 > 0 ? budgets.l2 : LAYER_DEFAULT_BUDGETS.l2,
|
|
188
|
+
l3: budgets.l3 > 0 ? budgets.l3 : LAYER_DEFAULT_BUDGETS.l3,
|
|
189
|
+
},
|
|
190
|
+
},
|
|
191
|
+
// 输入预算(字符):current 是运行时覆盖(0 = 跟随配置),fallback 是静态配置值
|
|
192
|
+
inputBudget: {
|
|
193
|
+
current: s?.distillMaxInputChars ?? 0,
|
|
194
|
+
fallback: cfg.llm.maxInputChars,
|
|
195
|
+
effective: s && s.distillMaxInputChars > 0 ? s.distillMaxInputChars : cfg.llm.maxInputChars,
|
|
196
|
+
},
|
|
171
197
|
};
|
|
172
198
|
}
|
|
173
199
|
case 'dsh-memory/settings-set': {
|
|
@@ -186,6 +212,37 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
186
212
|
}
|
|
187
213
|
clean.reasoningEffort = v;
|
|
188
214
|
}
|
|
215
|
+
// 蒸馏模型运行时覆盖:供应商/模型 id 原样接受(不在此校验存在性——
|
|
216
|
+
// 供应商可被用户随后删除,解析侧按存在性回退并提示)
|
|
217
|
+
for (const key of ['distillProvider', 'distillModel']) {
|
|
218
|
+
if (patch[key] !== undefined) {
|
|
219
|
+
const v = String(patch[key]);
|
|
220
|
+
if (v.length > 200)
|
|
221
|
+
throw new Error(`${key} 过长(≤200 字符)`);
|
|
222
|
+
clean[key] = v;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
// 分层输出预算:四键一起校验,非负整数 ≤ 100 万;0 = 跟随内置默认
|
|
226
|
+
if (patch.distillBudgets !== undefined) {
|
|
227
|
+
const raw = (patch.distillBudgets ?? {});
|
|
228
|
+
const budgets = {};
|
|
229
|
+
for (const key of ['extract', 'dedup', 'l2', 'l3']) {
|
|
230
|
+
const n = Number(raw[key] ?? 0);
|
|
231
|
+
if (!Number.isInteger(n) || n < 0 || n > 1_000_000) {
|
|
232
|
+
throw new Error(`distillBudgets.${key} 须为 0~1000000 的整数(0 = 跟随默认)`);
|
|
233
|
+
}
|
|
234
|
+
budgets[key] = n;
|
|
235
|
+
}
|
|
236
|
+
clean.distillBudgets = budgets;
|
|
237
|
+
}
|
|
238
|
+
// 输入预算(字符):0 = 跟随静态配置;正值须落在静态 schema 同款范围(1000~100 万)
|
|
239
|
+
if (patch.distillMaxInputChars !== undefined) {
|
|
240
|
+
const n = Number(patch.distillMaxInputChars);
|
|
241
|
+
if (!Number.isInteger(n) || n < 0 || n > 1_000_000 || (n > 0 && n < 1000)) {
|
|
242
|
+
throw new Error('distillMaxInputChars 须为 0 或 1000~1000000 的整数(0 = 跟随配置)');
|
|
243
|
+
}
|
|
244
|
+
clean.distillMaxInputChars = n;
|
|
245
|
+
}
|
|
189
246
|
if (Object.keys(clean).length === 0)
|
|
190
247
|
throw new Error('开关更新载荷为空');
|
|
191
248
|
await live.update(clean);
|
|
@@ -270,6 +327,61 @@ async function handleEndpoint(endpoint, payload, deps) {
|
|
|
270
327
|
throw new Error('重建控制器未初始化');
|
|
271
328
|
return rebuild.requestCancel();
|
|
272
329
|
}
|
|
330
|
+
// ── 蒸馏模型选择器(用户已配置的供应商路由) ──
|
|
331
|
+
case 'dsh-memory/llm-providers': {
|
|
332
|
+
// 供应商目录(已注册适配器的活动路由)+ 默认选择 + 当前覆盖与实际生效路由
|
|
333
|
+
let providers = [];
|
|
334
|
+
try {
|
|
335
|
+
providers = deps.ctx.llm.listProviders();
|
|
336
|
+
}
|
|
337
|
+
catch (err) {
|
|
338
|
+
deps.logger.warn(`[memory] 供应商列表读取失败: ${errDetail(err)}`);
|
|
339
|
+
}
|
|
340
|
+
let def = null;
|
|
341
|
+
try {
|
|
342
|
+
const sel = deps.ctx.get('agentDefaultModel')?.currentSelection?.();
|
|
343
|
+
if (sel?.provider && sel?.model)
|
|
344
|
+
def = { provider: sel.provider, model: sel.model };
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
/* 可选服务缺失 = 无默认选择 */
|
|
348
|
+
}
|
|
349
|
+
const s = live?.get();
|
|
350
|
+
const current = { provider: s?.distillProvider ?? '', model: s?.distillModel ?? '' };
|
|
351
|
+
let effective = null;
|
|
352
|
+
try {
|
|
353
|
+
effective = await resolveModelRoute(deps.ctx, effectiveCfg(cfg, live));
|
|
354
|
+
}
|
|
355
|
+
catch {
|
|
356
|
+
effective = null; // 无法解析(无默认选择且未覆盖)时 UI 显示占位
|
|
357
|
+
}
|
|
358
|
+
return {
|
|
359
|
+
supported: true,
|
|
360
|
+
providers,
|
|
361
|
+
default: def,
|
|
362
|
+
// 部署静态 pin(provider+model 双字段)优先于运行时选择,UI 据此禁用选择器
|
|
363
|
+
pinned: Boolean(cfg.llm.provider && cfg.llm.model),
|
|
364
|
+
current,
|
|
365
|
+
// 所选供应商是否仍在已注册路由中(用户删掉供应商后提示回退)
|
|
366
|
+
currentRegistered: current.provider === '' || providers.some((p) => p.id === current.provider),
|
|
367
|
+
effective,
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
case 'dsh-memory/llm-models': {
|
|
371
|
+
const p = (payload ?? {});
|
|
372
|
+
if (typeof p.provider !== 'string' || !p.provider)
|
|
373
|
+
throw new Error('provider 缺失');
|
|
374
|
+
// 两个内置适配器(deepseek/pi-ai)的 listModels 都读本地快照不触网;
|
|
375
|
+
// 仍加超时兜底,防第三方适配器实现为远端查询拖死 RPC 轮询
|
|
376
|
+
const models = await Promise.race([
|
|
377
|
+
deps.ctx.llm.listModels(p.provider),
|
|
378
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error('模型列表查询超时')), 8000)),
|
|
379
|
+
]);
|
|
380
|
+
return {
|
|
381
|
+
provider: p.provider,
|
|
382
|
+
models: models.map((m) => ({ id: m.id, name: m.name, description: m.description ?? null })),
|
|
383
|
+
};
|
|
384
|
+
}
|
|
273
385
|
// ── 嵌入源(远程/本地/关闭 三态)与模型管理 ──
|
|
274
386
|
case 'dsh-memory/embedding-state-get': {
|
|
275
387
|
if (!embedManager)
|
package/dist/store/bm25.js
CHANGED
|
@@ -31,10 +31,18 @@ export interface DownloaderOptions {
|
|
|
31
31
|
/** 下载镜像根(默认 https://hf-mirror.com,可配回 https://huggingface.co)。 */
|
|
32
32
|
mirror: string;
|
|
33
33
|
logger?: MemoryLogger;
|
|
34
|
-
/**
|
|
34
|
+
/** 测试注入;默认 undici fetch(按需挂代理 dispatcher)。 */
|
|
35
35
|
fetchImpl?: FetchLike;
|
|
36
36
|
/** 测试注入磁盘剩余字节;默认 statfs。 */
|
|
37
37
|
freeBytes?: () => Promise<number | null>;
|
|
38
|
+
/** 单文件失败的自动重试间隔(毫秒),长度即重试次数;默认 [1000, 3000]。
|
|
39
|
+
* sha256 失配重试从零开始(污染断点已删除);数量不吻合/网络错误保留断点续传。 */
|
|
40
|
+
retryDelaysMs?: number[];
|
|
41
|
+
/** 显式代理三态:''(默认)= 探测代理环境变量;'none' = 禁用强制直连;
|
|
42
|
+
* 其他值 = 代理 URL(如 http://127.0.0.1:7890)。镜像直连在国内网络
|
|
43
|
+
* 间歇不可达(真实事故:直连超时与污染字节交替),与 curl/npm 同语义地
|
|
44
|
+
* 走用户已配置的代理是可达性的底线。 */
|
|
45
|
+
proxy?: string;
|
|
38
46
|
}
|
|
39
47
|
export declare class ModelDownloadQueue {
|
|
40
48
|
private readonly dataDir;
|
|
@@ -42,7 +50,15 @@ export declare class ModelDownloadQueue {
|
|
|
42
50
|
private progress;
|
|
43
51
|
private busy;
|
|
44
52
|
private abort;
|
|
53
|
+
/** 代理 dispatcher(按需创建;dispose 关闭连接池)。 */
|
|
54
|
+
private agent;
|
|
55
|
+
/** 默认 fetch:undici(可挂代理 dispatcher);测试注入优先。 */
|
|
56
|
+
private readonly defaultFetch;
|
|
45
57
|
constructor(dataDir: string, opts: DownloaderOptions);
|
|
58
|
+
/** 镜像根(无尾斜杠)。 */
|
|
59
|
+
private mirrorUrl;
|
|
60
|
+
/** 释放代理连接池(插件 dispose 链调用;无代理时幂等无操作)。 */
|
|
61
|
+
dispose(): void;
|
|
46
62
|
/** 当前进度快照(无任务时 null)。 */
|
|
47
63
|
getProgress(): DownloadProgress | null;
|
|
48
64
|
/** 是否有任务在跑(含校验阶段)。 */
|
|
@@ -64,8 +80,25 @@ export declare class ModelDownloadQueue {
|
|
|
64
80
|
/** 取消当前任务:中断 fetch,保留 .part 断点。 */
|
|
65
81
|
cancel(): boolean;
|
|
66
82
|
private run;
|
|
67
|
-
/** 下载单文件到最终路径(含续传与校验),返回该文件贡献的字节数。
|
|
83
|
+
/** 下载单文件到最终路径(含续传与校验),返回该文件贡献的字节数。
|
|
84
|
+
* 单文件失败自动重试(默认 2 次)+ **重试换缓存键**:镜像链路
|
|
85
|
+
* (Caddy×3 → CloudFront → Cloudflare)存在缓存对象污染窗口——同一时间窗内
|
|
86
|
+
* 同一 URL 确定性拿到错误字节(2026-08-19 embeddinggemma generation_config.json
|
|
87
|
+
* 连续错哈希的真实事故),普通重试会全打同一污染缓存;每次重试追加
|
|
88
|
+
* `?dshmem-retry=N` 参数绕开缓存键另取对象,窗口期也能自愈。
|
|
89
|
+
* - sha256 失配:downloadFileOnce 已删除断点 → 从零重下;
|
|
90
|
+
* - 数量不吻合/网络错误:断点保留 → Range 续传重试;
|
|
91
|
+
* - 取消:立即上抛不重试。 */
|
|
68
92
|
private downloadFile;
|
|
93
|
+
/** 单次尝试:续传探测 → fetch(attempt>0 追加缓存键参数)→ 落盘 → 尺寸与 sha256 校验 → rename。 */
|
|
94
|
+
private downloadFileOnce;
|
|
69
95
|
private freeBytes;
|
|
70
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* 解析下载代理(三态):`''`(默认)= 探测代理环境变量(HTTPS_PROXY > ALL_PROXY >
|
|
99
|
+
* HTTP_PROXY,大小写双形态,尊重 NO_PROXY);`'none'` = 禁用代理强制直连;
|
|
100
|
+
* 其他值 = 显式代理 URL。与 curl/npm 同语义——用户配置了代理 env 的机器上
|
|
101
|
+
* 镜像直连往往间歇不可达(真实事故:直连超时与污染字节交替出现)。
|
|
102
|
+
*/
|
|
103
|
+
export declare function resolveProxyUrl(setting: string | undefined, host: string): string;
|
|
71
104
|
export {};
|
|
@@ -6,13 +6,15 @@
|
|
|
6
6
|
* - 断点续传:写 .part 旁车文件,重试从断点 Range 续传;服务器不支持 Range(回 200)
|
|
7
7
|
* 则从头重写;取消保留断点;
|
|
8
8
|
* - 完整性:每文件下满后流式哈希整文件比对目录 sha256(续传无法增量哈希,落盘后
|
|
9
|
-
*
|
|
9
|
+
* 单遍校验最简单且正确);失配删文件整体重下;单文件失败自动重试(默认 2 次,
|
|
10
|
+
* 吸收镜像瞬态污染——sha 失配从零重下、网络类错误保留断点续传);
|
|
10
11
|
* - 磁盘门禁:下载前检查数据目录所在卷剩余空间 ≥ 模型体积 × 1.2(statfs 不可用时跳过);
|
|
11
12
|
* - 同一时刻只跑一个下载任务(串行队列),后续请求直接拒绝并说明。
|
|
12
13
|
*/
|
|
13
14
|
import { createHash } from 'node:crypto';
|
|
14
15
|
import { promises as fs } from 'node:fs';
|
|
15
16
|
import * as path from 'node:path';
|
|
17
|
+
import { fetch as undiciFetch, ProxyAgent } from 'undici';
|
|
16
18
|
import { catalogById, catalogTotalBytes, MODEL_CATALOG } from './model-catalog.js';
|
|
17
19
|
const DISK_HEADROOM = 1.2;
|
|
18
20
|
export class ModelDownloadQueue {
|
|
@@ -21,9 +23,40 @@ export class ModelDownloadQueue {
|
|
|
21
23
|
progress = null;
|
|
22
24
|
busy = false;
|
|
23
25
|
abort = null;
|
|
26
|
+
/** 代理 dispatcher(按需创建;dispose 关闭连接池)。 */
|
|
27
|
+
agent;
|
|
28
|
+
/** 默认 fetch:undici(可挂代理 dispatcher);测试注入优先。 */
|
|
29
|
+
defaultFetch;
|
|
24
30
|
constructor(dataDir, opts) {
|
|
25
31
|
this.dataDir = dataDir;
|
|
26
32
|
this.opts = opts;
|
|
33
|
+
// 畸形 mirror(无 scheme 等)只跳过代理解析,不炸构造器(下载本身还会报清晰的 URL 错)
|
|
34
|
+
let host = '';
|
|
35
|
+
try {
|
|
36
|
+
host = new URL(this.mirrorUrl()).host;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
/* ignore */
|
|
40
|
+
}
|
|
41
|
+
const proxy = resolveProxyUrl(opts.proxy, host);
|
|
42
|
+
if (proxy) {
|
|
43
|
+
this.agent = new ProxyAgent(proxy);
|
|
44
|
+
opts.logger?.info(`[memory] 模型下载走代理 ${proxy}(镜像直连在国内网络间歇不可达)`);
|
|
45
|
+
}
|
|
46
|
+
this.defaultFetch = ((u, init) => {
|
|
47
|
+
const dispatch = this.agent;
|
|
48
|
+
// undici fetch 的 init 接受 dispatcher;RequestInit 类型无此字段,断言透传
|
|
49
|
+
return undiciFetch(u, { ...init, ...(dispatch ? { dispatcher: dispatch } : {}) });
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
/** 镜像根(无尾斜杠)。 */
|
|
53
|
+
mirrorUrl() {
|
|
54
|
+
return this.opts.mirror.replace(/\/+$/, '');
|
|
55
|
+
}
|
|
56
|
+
/** 释放代理连接池(插件 dispose 链调用;无代理时幂等无操作)。 */
|
|
57
|
+
dispose() {
|
|
58
|
+
void this.agent?.close().catch(() => { });
|
|
59
|
+
this.agent = undefined;
|
|
27
60
|
}
|
|
28
61
|
/** 当前进度快照(无任务时 null)。 */
|
|
29
62
|
getProgress() {
|
|
@@ -187,13 +220,46 @@ export class ModelDownloadQueue {
|
|
|
187
220
|
prog.overallReceived = overall;
|
|
188
221
|
}
|
|
189
222
|
}
|
|
190
|
-
/** 下载单文件到最终路径(含续传与校验),返回该文件贡献的字节数。
|
|
223
|
+
/** 下载单文件到最终路径(含续传与校验),返回该文件贡献的字节数。
|
|
224
|
+
* 单文件失败自动重试(默认 2 次)+ **重试换缓存键**:镜像链路
|
|
225
|
+
* (Caddy×3 → CloudFront → Cloudflare)存在缓存对象污染窗口——同一时间窗内
|
|
226
|
+
* 同一 URL 确定性拿到错误字节(2026-08-19 embeddinggemma generation_config.json
|
|
227
|
+
* 连续错哈希的真实事故),普通重试会全打同一污染缓存;每次重试追加
|
|
228
|
+
* `?dshmem-retry=N` 参数绕开缓存键另取对象,窗口期也能自愈。
|
|
229
|
+
* - sha256 失配:downloadFileOnce 已删除断点 → 从零重下;
|
|
230
|
+
* - 数量不吻合/网络错误:断点保留 → Range 续传重试;
|
|
231
|
+
* - 取消:立即上抛不重试。 */
|
|
191
232
|
async downloadFile(entry, f, dir, onBytes) {
|
|
233
|
+
const delays = this.opts.retryDelaysMs ?? [1000, 3000];
|
|
234
|
+
let lastErr;
|
|
235
|
+
for (let attempt = 0;; attempt++) {
|
|
236
|
+
if (this.progress?.phase === 'cancelled')
|
|
237
|
+
throw new Error('已取消');
|
|
238
|
+
try {
|
|
239
|
+
return await this.downloadFileOnce(entry, f, dir, attempt, onBytes);
|
|
240
|
+
}
|
|
241
|
+
catch (err) {
|
|
242
|
+
lastErr = err;
|
|
243
|
+
// as 断言绕开 CFA 窄化——await 期间 cancel() 可能已改写 phase(同 downloadFileOnce 末段)
|
|
244
|
+
if (this.progress?.phase === 'cancelled')
|
|
245
|
+
throw err;
|
|
246
|
+
if (attempt >= delays.length)
|
|
247
|
+
throw err;
|
|
248
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
249
|
+
this.opts.logger?.warn(`[memory] 文件 ${f.path} 第 ${attempt + 1} 次尝试失败(${msg}),${delays[attempt]}ms 后自动重试(换缓存键)`);
|
|
250
|
+
await new Promise((r) => setTimeout(r, delays[attempt]));
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
/** 单次尝试:续传探测 → fetch(attempt>0 追加缓存键参数)→ 落盘 → 尺寸与 sha256 校验 → rename。 */
|
|
255
|
+
async downloadFileOnce(entry, f, dir, attempt, onBytes) {
|
|
192
256
|
const finalPath = path.join(dir, f.path);
|
|
193
257
|
const partPath = finalPath + '.part';
|
|
194
|
-
const base = this.
|
|
195
|
-
|
|
196
|
-
const
|
|
258
|
+
const base = this.mirrorUrl();
|
|
259
|
+
// 重试追加缓存键参数:绕开镜像 CDN 的污染缓存对象(同窗口普通重试全打同一对象)
|
|
260
|
+
const cacheBust = attempt > 0 ? `?dshmem-retry=${attempt}` : '';
|
|
261
|
+
const url = `${base}/${entry.repo}/resolve/${entry.revision}/${f.path}${cacheBust}`;
|
|
262
|
+
const fetchImpl = this.opts.fetchImpl ?? this.defaultFetch;
|
|
197
263
|
const prog = this.progress;
|
|
198
264
|
let resumeFrom = 0;
|
|
199
265
|
const partSize = await fileSize(partPath);
|
|
@@ -303,6 +369,37 @@ async function fileSize(p) {
|
|
|
303
369
|
return null;
|
|
304
370
|
}
|
|
305
371
|
}
|
|
372
|
+
/**
|
|
373
|
+
* 解析下载代理(三态):`''`(默认)= 探测代理环境变量(HTTPS_PROXY > ALL_PROXY >
|
|
374
|
+
* HTTP_PROXY,大小写双形态,尊重 NO_PROXY);`'none'` = 禁用代理强制直连;
|
|
375
|
+
* 其他值 = 显式代理 URL。与 curl/npm 同语义——用户配置了代理 env 的机器上
|
|
376
|
+
* 镜像直连往往间歇不可达(真实事故:直连超时与污染字节交替出现)。
|
|
377
|
+
*/
|
|
378
|
+
export function resolveProxyUrl(setting, host) {
|
|
379
|
+
const value = (setting ?? '').trim();
|
|
380
|
+
if (value.toLowerCase() === 'none')
|
|
381
|
+
return '';
|
|
382
|
+
if (value)
|
|
383
|
+
return value;
|
|
384
|
+
const noProxy = process.env.NO_PROXY ?? process.env.no_proxy ?? '';
|
|
385
|
+
if (noProxy) {
|
|
386
|
+
for (const raw of noProxy.split(',')) {
|
|
387
|
+
const entry = raw.trim().replace(/^\./, '').toLowerCase();
|
|
388
|
+
if (entry && (host.toLowerCase() === entry || host.toLowerCase().endsWith(`.${entry}`)))
|
|
389
|
+
return '';
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
const candidates = [
|
|
393
|
+
process.env.HTTPS_PROXY, process.env.https_proxy,
|
|
394
|
+
process.env.ALL_PROXY, process.env.all_proxy,
|
|
395
|
+
process.env.HTTP_PROXY, process.env.http_proxy,
|
|
396
|
+
];
|
|
397
|
+
for (const c of candidates) {
|
|
398
|
+
if (c && c.trim())
|
|
399
|
+
return c.trim();
|
|
400
|
+
}
|
|
401
|
+
return '';
|
|
402
|
+
}
|
|
306
403
|
async function sha256File(p) {
|
|
307
404
|
const { createReadStream } = await import('node:fs');
|
|
308
405
|
const hash = createHash('sha256');
|
|
@@ -10,9 +10,14 @@ export interface EmbeddingProviderInfo {
|
|
|
10
10
|
model: string;
|
|
11
11
|
dimensions: number;
|
|
12
12
|
}
|
|
13
|
+
/** 单次嵌入调用的可选参数:timeoutMs 只允许缩短服务配置的超时(内层钳制),
|
|
14
|
+
* 永不放大——召回路径用它给 FTS 降级留时间(规格 A 节)。本地实现可忽略。 */
|
|
15
|
+
export interface EmbedCallOptions {
|
|
16
|
+
timeoutMs?: number;
|
|
17
|
+
}
|
|
13
18
|
export interface EmbeddingService {
|
|
14
|
-
embed(text: string): Promise<Float32Array>;
|
|
15
|
-
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
19
|
+
embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
|
|
20
|
+
embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
|
|
16
21
|
getDimensions(): number;
|
|
17
22
|
getProviderInfo(): EmbeddingProviderInfo;
|
|
18
23
|
isReady(): boolean;
|
|
@@ -46,8 +51,8 @@ export declare class RemoteEmbeddingService implements EmbeddingService {
|
|
|
46
51
|
getDimensions(): number;
|
|
47
52
|
getProviderInfo(): EmbeddingProviderInfo;
|
|
48
53
|
isReady(): boolean;
|
|
49
|
-
embed(text: string): Promise<Float32Array>;
|
|
50
|
-
embedBatch(texts: string[]): Promise<Float32Array[]>;
|
|
54
|
+
embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
|
|
55
|
+
embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
|
|
51
56
|
}
|
|
52
57
|
/**
|
|
53
58
|
* 嵌入调用降级助手(L0/L1 Store 共用):查询向量失败 → undefined(调用方降级 FTS);
|
|
@@ -62,8 +67,9 @@ export declare class EmbedHelper {
|
|
|
62
67
|
/** 活切换嵌入源(D4/D5):换掉底层服务并复位一次性告警(新服务重新获得告警机会)。 */
|
|
63
68
|
setService(svc: EmbeddingService): void;
|
|
64
69
|
vectorReady(): boolean;
|
|
65
|
-
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
66
|
-
|
|
70
|
+
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
71
|
+
* timeoutMs 为内层钳制(仅缩短服务超时),召回路径使用。 */
|
|
72
|
+
query(text: string, timeoutMs?: number): Promise<Float32Array | undefined>;
|
|
67
73
|
/** 批量嵌入;服务未就绪或失败时返回全 undefined(不阻断元数据/FTS 写入)。 */
|
|
68
74
|
batch(texts: string[]): Promise<Array<Float32Array | undefined>>;
|
|
69
75
|
private warn;
|
package/dist/store/embedding.js
CHANGED
|
@@ -34,15 +34,17 @@ export class RemoteEmbeddingService {
|
|
|
34
34
|
isReady() {
|
|
35
35
|
return true;
|
|
36
36
|
}
|
|
37
|
-
async embed(text) {
|
|
38
|
-
const [vec] = await this.embedBatch([text]);
|
|
37
|
+
async embed(text, callOpts) {
|
|
38
|
+
const [vec] = await this.embedBatch([text], callOpts);
|
|
39
39
|
return vec;
|
|
40
40
|
}
|
|
41
|
-
async embedBatch(texts) {
|
|
41
|
+
async embedBatch(texts, callOpts) {
|
|
42
42
|
if (texts.length === 0)
|
|
43
43
|
return [];
|
|
44
44
|
const input = texts.map((t) => t.slice(0, this.opts.maxInputChars));
|
|
45
45
|
const base = this.opts.baseUrl.replace(/\/+$/, '');
|
|
46
|
+
// 内层钳制只缩短不放大:召回路径传短超时,给 FTS 降级留时间
|
|
47
|
+
const timeoutMs = Math.min(this.opts.timeoutMs, callOpts?.timeoutMs ?? this.opts.timeoutMs);
|
|
46
48
|
const res = await fetch(`${base}/embeddings`, {
|
|
47
49
|
method: 'POST',
|
|
48
50
|
headers: {
|
|
@@ -50,7 +52,7 @@ export class RemoteEmbeddingService {
|
|
|
50
52
|
authorization: `Bearer ${this.opts.apiKey}`,
|
|
51
53
|
},
|
|
52
54
|
body: JSON.stringify({ model: this.opts.model, input }),
|
|
53
|
-
signal: AbortSignal.timeout(
|
|
55
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
54
56
|
});
|
|
55
57
|
if (!res.ok) {
|
|
56
58
|
const body = await res.text().catch(() => '');
|
|
@@ -103,10 +105,11 @@ export class EmbedHelper {
|
|
|
103
105
|
vectorReady() {
|
|
104
106
|
return this.embed.isReady();
|
|
105
107
|
}
|
|
106
|
-
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
107
|
-
|
|
108
|
+
/** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
|
|
109
|
+
* timeoutMs 为内层钳制(仅缩短服务超时),召回路径使用。 */
|
|
110
|
+
async query(text, timeoutMs) {
|
|
108
111
|
try {
|
|
109
|
-
const vec = await this.embed.embed(text);
|
|
112
|
+
const vec = await this.embed.embed(text, timeoutMs != null ? { timeoutMs } : undefined);
|
|
110
113
|
return vec.length > 0 ? vec : undefined;
|
|
111
114
|
}
|
|
112
115
|
catch (err) {
|
package/dist/store/l0.d.ts
CHANGED
|
@@ -15,6 +15,8 @@ export declare class L0Store {
|
|
|
15
15
|
append(sessionId: string, messages: ConversationMessage[]): Promise<void>;
|
|
16
16
|
/** 今日已捕获消息数(SQL 计数,不再读整文件)。 */
|
|
17
17
|
countToday(): Promise<number>;
|
|
18
|
+
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
19
|
+
recentBySession(sessionId: string, limit: number): Promise<ConversationMessage[]>;
|
|
18
20
|
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
19
21
|
search(query: string, limit: number): Promise<L0MessageRecord[]>;
|
|
20
22
|
/** 活切换嵌入源:同步换底层服务(嵌入源三态切换用)。 */
|
package/dist/store/l0.js
CHANGED
|
@@ -99,6 +99,10 @@ export class L0Store {
|
|
|
99
99
|
const d = new Date();
|
|
100
100
|
return this.db.countL0Since(new Date(d.getFullYear(), d.getMonth(), d.getDate()).toISOString());
|
|
101
101
|
}
|
|
102
|
+
/** 该会话最近 n 条消息(时间升序;蒸馏背景参考用,按会话现查——ADR-0003)。 */
|
|
103
|
+
async recentBySession(sessionId, limit) {
|
|
104
|
+
return this.db.recentL0BySession(sessionId, limit);
|
|
105
|
+
}
|
|
102
106
|
/** 检索:FTS + 向量 hybrid(RRF 融合),返回按相关性排序的消息。 */
|
|
103
107
|
async search(query, limit) {
|
|
104
108
|
const caps = this.db.getCapabilities();
|
package/dist/store/l1.d.ts
CHANGED
|
@@ -10,6 +10,8 @@ export interface L1SearchOptions {
|
|
|
10
10
|
/** 分数阈值(仅召回路径传;keyword/embedding 策略生效,FTS 含小语料例外;
|
|
11
11
|
* hybrid 按官方语义在 RRF 融合前不过滤)。 */
|
|
12
12
|
scoreThreshold?: number;
|
|
13
|
+
/** 嵌入查询内层钳制(ms,只缩短不放大;召回路径传入给 FTS 降级留时间)。 */
|
|
14
|
+
embeddingTimeoutMs?: number;
|
|
13
15
|
}
|
|
14
16
|
export declare class L1Store {
|
|
15
17
|
private readonly db;
|
package/dist/store/l1.js
CHANGED
|
@@ -135,7 +135,7 @@ export class L1Store {
|
|
|
135
135
|
return this.postProcess(applyFtsThreshold(fts, threshold, limit), opts?.type, limit);
|
|
136
136
|
}
|
|
137
137
|
if (strategy === 'embedding') {
|
|
138
|
-
const vec = await this.helper.query(query);
|
|
138
|
+
const vec = await this.helper.query(query, opts?.embeddingTimeoutMs);
|
|
139
139
|
if (!vec) {
|
|
140
140
|
// embedding 调用失败:降级 FTS,不阻断
|
|
141
141
|
const fts = this.db.searchL1Fts(query, candidateK, opts?.family);
|
|
@@ -148,7 +148,7 @@ export class L1Store {
|
|
|
148
148
|
// → 融合分归一化:rank1 双列表命中 = 1.0,单列表命中 ≤ 0.5,保持 0~1 语义
|
|
149
149
|
const [ftsList, vecRaw] = await Promise.all([
|
|
150
150
|
Promise.resolve(this.db.searchL1Fts(query, candidateK, opts?.family)),
|
|
151
|
-
this.helper.query(query),
|
|
151
|
+
this.helper.query(query, opts?.embeddingTimeoutMs),
|
|
152
152
|
]);
|
|
153
153
|
const vecList = vecRaw ? this.db.searchL1Vector(vecRaw, candidateK, opts?.family) : [];
|
|
154
154
|
const merged = rrfMerge([ftsList, vecList], (h) => h.id);
|
|
@@ -43,7 +43,7 @@ export const MODEL_CATALOG = [
|
|
|
43
43
|
{ path: 'added_tokens.json', size: 35, sha256: '50b2f405ba56a26d4913fd772089992252d7f942123cc0a034d96424221ba946' },
|
|
44
44
|
{ path: 'special_tokens_map.json', size: 662, sha256: '2f7b0adf4fb469770bb1490e3e35df87b1dc578246c5e7e6fc76ecf33213a397' },
|
|
45
45
|
{ path: 'tokenizer_config.json', size: 1156830, sha256: '3ca953eea6c3c9fcda9cf3df22949ff18b216f7c74bd6459230f3f1013953f3a' },
|
|
46
|
-
{ path: 'generation_config.json', size: 133, sha256: '
|
|
46
|
+
{ path: 'generation_config.json', size: 133, sha256: '1fb1efd221c1ca88a736d1b36cb47d754c177677e222acb3b1e5424c5d664870' },
|
|
47
47
|
{ path: 'tokenizer.json', size: 20323312, sha256: '4dda02faaf32bc91031dc8c88457ac272b00c1016cc679757d1c441b248b9c47' },
|
|
48
48
|
{ path: 'onnx/model_quantized.onnx', size: 567874, sha256: '172efde319fe1542dc41f31be6154910b05b78f7a861c265c4600eec906bd6d8' },
|
|
49
49
|
{ path: 'onnx/model_quantized.onnx_data', size: 308890624, sha256: '705626e28e4c23c82ade34566b4197d97f534c12275fa406dfb71e9937d388c0' },
|
package/dist/store/pending.d.ts
CHANGED
|
@@ -1,15 +1,36 @@
|
|
|
1
1
|
import type { ConversationMessage, ExtractMode, MemoryLogger } from '../types.js';
|
|
2
|
+
/** 带会话标识的未蒸馏消息(会话切片的成员;CONTEXT.md「会话切片」「捕获档位」)。 */
|
|
3
|
+
export interface PendingMessage extends ConversationMessage {
|
|
4
|
+
/** 捕获该消息的会话;旧格式数据无此字段,加载时归 legacy 组一次性蒸馏。 */
|
|
5
|
+
sessionId: string;
|
|
6
|
+
}
|
|
2
7
|
/** 三档蒸馏缓冲桶(off 在捕获侧已被拦截,永远不到这里)。 */
|
|
3
8
|
export interface PendingBuckets {
|
|
4
|
-
auto:
|
|
5
|
-
chat:
|
|
6
|
-
work:
|
|
9
|
+
auto: PendingMessage[];
|
|
10
|
+
chat: PendingMessage[];
|
|
11
|
+
work: PendingMessage[];
|
|
7
12
|
}
|
|
13
|
+
/** 旧格式(无 sessionId 字段)条目加载时归属的会话组。 */
|
|
14
|
+
export declare const LEGACY_SESSION = "legacy";
|
|
15
|
+
/** 各档位桶的渐进阈值状态(0 = 已毕业用稳态值;1 = 爬坡起点;ADR-0003)。 */
|
|
16
|
+
export type WarmupState = Record<ExtractMode, number>;
|
|
17
|
+
/** 全新起步:三档都从 1 爬坡(首轮即触发抽取)。 */
|
|
18
|
+
export declare function freshWarmup(): WarmupState;
|
|
8
19
|
export declare function emptyPending(): PendingBuckets;
|
|
9
|
-
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
10
|
-
|
|
20
|
+
/** 读取缓冲文件:文件缺失/损坏 → 空桶(不抛出——丢了缓冲 L0 事实源仍在)。
|
|
21
|
+
* 旧格式条目(无 sessionId)归 legacy 组;warmup 缺省 = 全新起步。 */
|
|
22
|
+
export declare function loadPending(file: string, logger?: MemoryLogger): Promise<{
|
|
23
|
+
buckets: PendingBuckets;
|
|
24
|
+
warmup: WarmupState;
|
|
25
|
+
}>;
|
|
26
|
+
/** 按会话分组(会话切片):组按首条时间排序、组内按时间稳定排序——
|
|
27
|
+
* 蒸馏的一切触发都以切片为单位,切片内永不跨会话混装(ADR-0003)。 */
|
|
28
|
+
export declare function groupPendingBySession(messages: PendingMessage[]): Array<{
|
|
29
|
+
sessionId: string;
|
|
30
|
+
messages: PendingMessage[];
|
|
31
|
+
}>;
|
|
11
32
|
/** 全量原子落盘(每次蒸馏尝试后调用;桶有上限,量级为百条级)。 */
|
|
12
|
-
export declare function savePending(file: string, buckets: PendingBuckets): Promise<void>;
|
|
33
|
+
export declare function savePending(file: string, buckets: PendingBuckets, warmup?: WarmupState): Promise<void>;
|
|
13
34
|
export declare function pendingPathFor(dataDir: string): string;
|
|
14
35
|
/** 三档 key(调度/遍历用)。 */
|
|
15
36
|
export declare const PENDING_MODES: readonly ExtractMode[];
|