dsh-layered-memory 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +101 -81
- package/README.md +88 -80
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/dist/client.js +1100 -402
- package/dist/config.d.ts +12 -0
- package/dist/config.js +20 -17
- package/dist/hooks/capture.d.ts +17 -1
- package/dist/hooks/capture.js +45 -11
- package/dist/hooks/recall.d.ts +7 -0
- package/dist/hooks/recall.js +20 -4
- package/dist/index.d.ts +8 -0
- package/dist/index.js +107 -44
- package/dist/pipeline/rebuild.d.ts +2 -0
- package/dist/pipeline/rebuild.js +7 -0
- package/dist/pipeline/runner.d.ts +7 -1
- package/dist/pipeline/runner.js +22 -7
- package/dist/settings.js +79 -20
- package/dist/stats.d.ts +7 -1
- package/dist/stats.js +131 -14
- package/dist/store/download-queue.d.ts +71 -0
- package/dist/store/download-queue.js +313 -0
- package/dist/store/embedding-source.d.ts +160 -0
- package/dist/store/embedding-source.js +421 -0
- package/dist/store/embedding.d.ts +3 -1
- package/dist/store/embedding.js +5 -0
- package/dist/store/l0.d.ts +13 -5
- package/dist/store/l0.js +33 -6
- package/dist/store/l1.d.ts +15 -5
- package/dist/store/l1.js +40 -15
- package/dist/store/local-embedding.d.ts +64 -0
- package/dist/store/local-embedding.js +120 -0
- package/dist/store/model-catalog.d.ts +45 -0
- package/dist/store/model-catalog.js +78 -0
- package/dist/store/runtime-installer.d.ts +60 -0
- package/dist/store/runtime-installer.js +181 -0
- package/dist/store/sqlite.d.ts +53 -7
- package/dist/store/sqlite.js +339 -72
- package/dist/tools/index.js +6 -4
- package/dist/util/filelog.d.ts +2 -0
- package/dist/util/filelog.js +20 -3
- package/package.json +1 -1
- package/assets/readme/hero.svg +0 -58
package/dist/config.d.ts
CHANGED
|
@@ -64,6 +64,10 @@ export interface MemoryConfig {
|
|
|
64
64
|
maxInputChars: number;
|
|
65
65
|
/** 单次调用超时(ms)。 */
|
|
66
66
|
timeoutMs: number;
|
|
67
|
+
/** 允许本地嵌入模型档(部署上限:公司环境可禁下载与本地推理)。 */
|
|
68
|
+
allowLocalModels: boolean;
|
|
69
|
+
/** 模型下载镜像根地址(默认国内可达的 hf-mirror.com,可改回官方)。 */
|
|
70
|
+
mirror: string;
|
|
67
71
|
};
|
|
68
72
|
llm: {
|
|
69
73
|
/** 蒸馏用的 provider 路由;留空用当前默认选择。 */
|
|
@@ -147,6 +151,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
147
151
|
dimensions: Schema<number, number>;
|
|
148
152
|
maxInputChars: Schema<number, number>;
|
|
149
153
|
timeoutMs: Schema<number, number>;
|
|
154
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
155
|
+
mirror: Schema<string, string>;
|
|
150
156
|
}>, Schemastery.ObjectT<{
|
|
151
157
|
enabled: Schema<boolean, boolean>;
|
|
152
158
|
baseUrl: Schema<string, string>;
|
|
@@ -155,6 +161,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
155
161
|
dimensions: Schema<number, number>;
|
|
156
162
|
maxInputChars: Schema<number, number>;
|
|
157
163
|
timeoutMs: Schema<number, number>;
|
|
164
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
165
|
+
mirror: Schema<string, string>;
|
|
158
166
|
}>>;
|
|
159
167
|
llm: Schema<Schemastery.ObjectS<{
|
|
160
168
|
provider: Schema<string, string>;
|
|
@@ -238,6 +246,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
238
246
|
dimensions: Schema<number, number>;
|
|
239
247
|
maxInputChars: Schema<number, number>;
|
|
240
248
|
timeoutMs: Schema<number, number>;
|
|
249
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
250
|
+
mirror: Schema<string, string>;
|
|
241
251
|
}>, Schemastery.ObjectT<{
|
|
242
252
|
enabled: Schema<boolean, boolean>;
|
|
243
253
|
baseUrl: Schema<string, string>;
|
|
@@ -246,6 +256,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
246
256
|
dimensions: Schema<number, number>;
|
|
247
257
|
maxInputChars: Schema<number, number>;
|
|
248
258
|
timeoutMs: Schema<number, number>;
|
|
259
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
260
|
+
mirror: Schema<string, string>;
|
|
249
261
|
}>>;
|
|
250
262
|
llm: Schema<Schemastery.ObjectS<{
|
|
251
263
|
provider: Schema<string, string>;
|
package/dist/config.js
CHANGED
|
@@ -11,53 +11,56 @@ export const memorySchema = Schema.object({
|
|
|
11
11
|
capture: Schema.object({
|
|
12
12
|
enabled: Schema.boolean().default(true),
|
|
13
13
|
stripCodeBlocks: Schema.boolean().default(true),
|
|
14
|
-
maxMessageChars: Schema.number().default(4000),
|
|
14
|
+
maxMessageChars: Schema.number().min(200).max(200_000).default(4000),
|
|
15
15
|
}),
|
|
16
16
|
extract: Schema.object({
|
|
17
17
|
enabled: Schema.boolean().default(true),
|
|
18
|
-
minMessages: Schema.number().default(1),
|
|
19
|
-
backgroundMessages: Schema.number().default(10),
|
|
20
|
-
candidatePool: Schema.number().default(5),
|
|
18
|
+
minMessages: Schema.number().min(1).max(100).default(1),
|
|
19
|
+
backgroundMessages: Schema.number().min(0).max(50).default(10),
|
|
20
|
+
candidatePool: Schema.number().min(1).max(20).default(5),
|
|
21
21
|
}),
|
|
22
22
|
l2: Schema.object({
|
|
23
23
|
enabled: Schema.boolean().default(true),
|
|
24
|
-
minNewMemories: Schema.number().default(5),
|
|
25
|
-
maxScenes: Schema.number().default(12),
|
|
26
|
-
sceneContextLimit: Schema.number().default(3),
|
|
24
|
+
minNewMemories: Schema.number().min(1).max(100).default(5),
|
|
25
|
+
maxScenes: Schema.number().min(1).max(100).default(12),
|
|
26
|
+
sceneContextLimit: Schema.number().min(0).max(20).default(3),
|
|
27
27
|
}),
|
|
28
28
|
l3: Schema.object({
|
|
29
29
|
enabled: Schema.boolean().default(true),
|
|
30
|
-
interval: Schema.number().default(20),
|
|
30
|
+
interval: Schema.number().min(1).max(200).default(20),
|
|
31
31
|
}),
|
|
32
32
|
recall: Schema.object({
|
|
33
33
|
enabled: Schema.boolean().default(true),
|
|
34
|
-
maxResults: Schema.number().default(5),
|
|
34
|
+
maxResults: Schema.number().min(1).max(20).default(5),
|
|
35
35
|
includePersona: Schema.boolean().default(true),
|
|
36
36
|
includeSceneNav: Schema.boolean().default(true),
|
|
37
37
|
strategy: Schema.union(['keyword', 'embedding', 'hybrid']).default('hybrid'),
|
|
38
|
-
scoreThreshold: Schema.number().default(0.3),
|
|
38
|
+
scoreThreshold: Schema.number().min(0).max(1).default(0.3),
|
|
39
39
|
}),
|
|
40
40
|
embedding: Schema.object({
|
|
41
41
|
enabled: Schema.boolean().default(false),
|
|
42
42
|
baseUrl: Schema.string().default(''),
|
|
43
43
|
apiKey: Schema.string().default(''),
|
|
44
44
|
model: Schema.string().default(''),
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
// 0 = 纯 FTS 模式(合法值,勿设 min>0)
|
|
46
|
+
dimensions: Schema.number().min(0).max(8192).default(0),
|
|
47
|
+
maxInputChars: Schema.number().min(100).max(100_000).default(5000),
|
|
48
|
+
timeoutMs: Schema.number().min(1000).max(300_000).default(10_000),
|
|
49
|
+
allowLocalModels: Schema.boolean().default(true),
|
|
50
|
+
mirror: Schema.string().default('https://hf-mirror.com'),
|
|
48
51
|
}),
|
|
49
52
|
llm: Schema.object({
|
|
50
53
|
provider: Schema.string().default(''),
|
|
51
54
|
model: Schema.string().default(''),
|
|
52
55
|
// 推理模型(如 v4-flash)的 reasoning 计入输出预算:预算不足会被思考吃光导致正文 0 字符
|
|
53
|
-
maxTokens: Schema.number().default(256_000),
|
|
56
|
+
maxTokens: Schema.number().min(1024).max(1_000_000).default(256_000),
|
|
54
57
|
// 蒸馏是结构化抽取任务,默认关思考(off):v4-flash 默认 high 档的思考可把任意 maxTokens
|
|
55
58
|
// 预算全部吃光导致正文 0 字符;非推理模型不认识 effort 时会报 UNSUPPORTED_REASONING_EFFORT,设空串跳过
|
|
56
59
|
reasoningEffort: Schema.union(['', 'off', 'high', 'max']).default('off'),
|
|
57
|
-
temperature: Schema.number().default(0.3),
|
|
60
|
+
temperature: Schema.number().min(0).max(2).default(0.3),
|
|
58
61
|
// 模型上下文 1M token,日常压到 ~700k 使用(中文按 1 字≈1 token 保守折算)
|
|
59
|
-
maxInputChars: Schema.number().default(700_000),
|
|
60
|
-
timeoutMs: Schema.number().default(120_000),
|
|
62
|
+
maxInputChars: Schema.number().min(1000).max(1_000_000).default(700_000),
|
|
63
|
+
timeoutMs: Schema.number().min(1000).max(600_000).default(120_000),
|
|
61
64
|
}),
|
|
62
65
|
tools: Schema.boolean().default(true),
|
|
63
66
|
});
|
package/dist/hooks/capture.d.ts
CHANGED
|
@@ -7,7 +7,23 @@ import type { L0Store } from '../store/l0.js';
|
|
|
7
7
|
import type { LiveSettingsHandle } from '../settings.js';
|
|
8
8
|
import type { MemoryLogger } from '../types.js';
|
|
9
9
|
export declare function isCaptureRelevant(type: string): boolean;
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* 按会话缓冲轮次事件。turn 被消费后剩余前缀为空即删除 Map 条目——
|
|
12
|
+
* 条目随会话数累积是慢泄漏(每会话残留一个数组引用,宿主长跑不释放)。
|
|
13
|
+
*/
|
|
14
|
+
export declare class CaptureBuffers {
|
|
15
|
+
private readonly map;
|
|
16
|
+
/** 活跃缓冲条目数(诊断/冒烟用)。 */
|
|
17
|
+
get size(): number;
|
|
18
|
+
push(sid: string, event: SessionEvent): void;
|
|
19
|
+
/** 取出该 turn 的全部事件(不含 turn/start 自身);无匹配 start 时返回整个缓冲。 */
|
|
20
|
+
takeTurn(sid: string, turn: number): SessionEvent[];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* 注册 L0 捕获。返回 L0 串行链的冲刷函数(dispose 序在关库前 await,
|
|
24
|
+
* 排队中的 turn 消息先落盘);capture 关闭时返回 undefined。
|
|
25
|
+
*/
|
|
26
|
+
export declare function registerCapture(ctx: Context, cfg: MemoryConfig, runner: MemoryRunner, l0: L0Store, logger: MemoryLogger, live: LiveSettingsHandle, modes: SessionModeStore): (() => Promise<void>) | undefined;
|
|
11
27
|
/**
|
|
12
28
|
* 缓冲上限裁剪(防御性;RELEVANT_TYPES 过滤后基本不可达)。
|
|
13
29
|
* 铁律:进行中轮次(turn/start 之后、turn/end 之前)的事件绝不裁——
|
package/dist/hooks/capture.js
CHANGED
|
@@ -17,11 +17,49 @@ export function isCaptureRelevant(type) {
|
|
|
17
17
|
return RELEVANT_TYPES.has(type);
|
|
18
18
|
}
|
|
19
19
|
const MAX_BUFFER = 500;
|
|
20
|
+
/**
|
|
21
|
+
* 按会话缓冲轮次事件。turn 被消费后剩余前缀为空即删除 Map 条目——
|
|
22
|
+
* 条目随会话数累积是慢泄漏(每会话残留一个数组引用,宿主长跑不释放)。
|
|
23
|
+
*/
|
|
24
|
+
export class CaptureBuffers {
|
|
25
|
+
map = new Map();
|
|
26
|
+
/** 活跃缓冲条目数(诊断/冒烟用)。 */
|
|
27
|
+
get size() {
|
|
28
|
+
return this.map.size;
|
|
29
|
+
}
|
|
30
|
+
push(sid, event) {
|
|
31
|
+
let buf = this.map.get(sid);
|
|
32
|
+
if (!buf) {
|
|
33
|
+
buf = [];
|
|
34
|
+
this.map.set(sid, buf);
|
|
35
|
+
}
|
|
36
|
+
buf.push(event);
|
|
37
|
+
trimBuffer(buf);
|
|
38
|
+
}
|
|
39
|
+
/** 取出该 turn 的全部事件(不含 turn/start 自身);无匹配 start 时返回整个缓冲。 */
|
|
40
|
+
takeTurn(sid, turn) {
|
|
41
|
+
const buf = this.map.get(sid);
|
|
42
|
+
if (!buf)
|
|
43
|
+
return [];
|
|
44
|
+
const startIdx = findTurnStart(buf, turn);
|
|
45
|
+
const turnEvents = startIdx === -1 ? buf : buf.slice(startIdx + 1);
|
|
46
|
+
const rest = startIdx === -1 ? [] : buf.slice(0, startIdx);
|
|
47
|
+
if (rest.length === 0)
|
|
48
|
+
this.map.delete(sid);
|
|
49
|
+
else
|
|
50
|
+
this.map.set(sid, rest);
|
|
51
|
+
return turnEvents;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* 注册 L0 捕获。返回 L0 串行链的冲刷函数(dispose 序在关库前 await,
|
|
56
|
+
* 排队中的 turn 消息先落盘);capture 关闭时返回 undefined。
|
|
57
|
+
*/
|
|
20
58
|
export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
21
59
|
if (!cfg.capture.enabled)
|
|
22
60
|
return;
|
|
23
61
|
const startFloor = Date.now();
|
|
24
|
-
const buffers = new
|
|
62
|
+
const buffers = new CaptureBuffers();
|
|
25
63
|
// L0 即时落盘链:turn/end 立刻写,不被蒸馏队列(慢 LLM 调用)阻塞;
|
|
26
64
|
// 串行化保证同轮次顺序与单次写入(进程退出时排队中的 L0 不再依赖蒸馏完成)
|
|
27
65
|
let l0Queue = Promise.resolve();
|
|
@@ -43,18 +81,10 @@ export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
|
43
81
|
}
|
|
44
82
|
return;
|
|
45
83
|
}
|
|
46
|
-
|
|
47
|
-
if (!buf) {
|
|
48
|
-
buf = [];
|
|
49
|
-
buffers.set(sid, buf);
|
|
50
|
-
}
|
|
51
|
-
buf.push(event);
|
|
52
|
-
trimBuffer(buf);
|
|
84
|
+
buffers.push(sid, event);
|
|
53
85
|
if (event.type === 'turn/end') {
|
|
54
86
|
const turn = event.data.turn;
|
|
55
|
-
const
|
|
56
|
-
const turnEvents = startIdx === -1 ? buf : buf.slice(startIdx + 1);
|
|
57
|
-
buffers.set(sid, startIdx === -1 ? [] : buf.slice(0, startIdx));
|
|
87
|
+
const turnEvents = buffers.takeTurn(sid, turn);
|
|
58
88
|
const messages = turnEventsToMessages(turnEvents, cfg, logger);
|
|
59
89
|
if (messages.length > 0) {
|
|
60
90
|
const roles = messages.reduce((acc, m) => {
|
|
@@ -84,6 +114,10 @@ export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
|
84
114
|
logger.warn(`[memory] session/event 处理失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
85
115
|
}
|
|
86
116
|
});
|
|
117
|
+
// 冲刷 = 等待串行链排空(链上每环自带 catch,永不 reject)。
|
|
118
|
+
// 注:极小概率在 await 期间又入队的新消息会落到链尾——其 JSONL 事实源照写、
|
|
119
|
+
// DB 写入由 upsert 内部兜底(关库后 warn),下次重建自愈。
|
|
120
|
+
return () => l0Queue;
|
|
87
121
|
}
|
|
88
122
|
/**
|
|
89
123
|
* 缓冲上限裁剪(防御性;RELEVANT_TYPES 过滤后基本不可达)。
|
package/dist/hooks/recall.d.ts
CHANGED
|
@@ -20,6 +20,13 @@ import type { PersonaStore } from '../store/persona.js';
|
|
|
20
20
|
import type { SceneStore } from '../store/scenes.js';
|
|
21
21
|
import type { SessionModeStore } from '../store/session-modes.js';
|
|
22
22
|
import type { MemoryLogger } from '../types.js';
|
|
23
|
+
/**
|
|
24
|
+
* 从会话消息构建召回查询(纯函数):末尾 N 条 + 总长截断,空输入返回空串。
|
|
25
|
+
* 全史拼接会让 MATCH 表达式随会话长度线性膨胀(整会话累计二次方成本)。
|
|
26
|
+
*/
|
|
27
|
+
export declare function buildRecallQuery(messages: Array<{
|
|
28
|
+
content: unknown;
|
|
29
|
+
}>, tailMessages?: number, maxChars?: number): string;
|
|
23
30
|
export interface RecallHooks {
|
|
24
31
|
/** 管线更新后调用:画像/场景缓存立即失效并异步刷新。 */
|
|
25
32
|
invalidateProfile(): void;
|
package/dist/hooks/recall.js
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import { blocksToText } from '../util/text.js';
|
|
2
2
|
const PROFILE_TTL = 60_000;
|
|
3
|
+
/** 召回查询只取会话末尾 N 条消息(长会话每步把全史拼进 FTS MATCH 会让检索成本线性上涨)。 */
|
|
4
|
+
const RECALL_QUERY_TAIL_MESSAGES = 8;
|
|
5
|
+
/** 召回查询总字符上限(保留末尾——最新语境权重最高)。 */
|
|
6
|
+
const RECALL_QUERY_MAX_CHARS = 2_000;
|
|
7
|
+
/**
|
|
8
|
+
* 从会话消息构建召回查询(纯函数):末尾 N 条 + 总长截断,空输入返回空串。
|
|
9
|
+
* 全史拼接会让 MATCH 表达式随会话长度线性膨胀(整会话累计二次方成本)。
|
|
10
|
+
*/
|
|
11
|
+
export function buildRecallQuery(messages, tailMessages = RECALL_QUERY_TAIL_MESSAGES, maxChars = RECALL_QUERY_MAX_CHARS) {
|
|
12
|
+
const tail = messages.slice(-tailMessages);
|
|
13
|
+
let text = tail.map((m) => blocksToText(m.content)).join(' ').trim();
|
|
14
|
+
if (text.length > maxChars)
|
|
15
|
+
text = text.slice(-maxChars);
|
|
16
|
+
return text;
|
|
17
|
+
}
|
|
3
18
|
const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
|
|
4
19
|
## 记忆工具调用指南
|
|
5
20
|
|
|
@@ -57,10 +72,7 @@ export function registerRecall(ctx, cfg, stores, logger, live, modes) {
|
|
|
57
72
|
l1Cache.set(payload.agent.id, []);
|
|
58
73
|
return next();
|
|
59
74
|
}
|
|
60
|
-
const query = payload.messages
|
|
61
|
-
.map((m) => blocksToText(m.content))
|
|
62
|
-
.join(' ')
|
|
63
|
-
.trim();
|
|
75
|
+
const query = buildRecallQuery(payload.messages);
|
|
64
76
|
if (query) {
|
|
65
77
|
const hits = await stores.l1.search(query, cfg.recall.maxResults, {
|
|
66
78
|
scoreThreshold: cfg.recall.scoreThreshold,
|
|
@@ -71,6 +83,10 @@ export function registerRecall(ctx, cfg, stores, logger, live, modes) {
|
|
|
71
83
|
logger.info(`[memory] 召回命中 ${hits.length} 条 L1(mode=${mode},query="${query.slice(0, 30).replace(/\n/g, ' ')}…",agent=${payload.agent.id})`);
|
|
72
84
|
}
|
|
73
85
|
}
|
|
86
|
+
else {
|
|
87
|
+
// 空查询清缓存:不把上一步的命中注入到与本步无关的上下文
|
|
88
|
+
l1Cache.set(payload.agent.id, []);
|
|
89
|
+
}
|
|
74
90
|
}
|
|
75
91
|
catch (err) {
|
|
76
92
|
logger.warn(`[memory] 召回检索失败: ${err instanceof Error ? err.message : String(err)}`);
|
package/dist/index.d.ts
CHANGED
|
@@ -72,6 +72,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
72
72
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
73
73
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
74
74
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
75
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
76
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
75
77
|
}>, Schemastery.ObjectT<{
|
|
76
78
|
enabled: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
77
79
|
baseUrl: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -80,6 +82,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
80
82
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
81
83
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
82
84
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
85
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
86
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
83
87
|
}>>;
|
|
84
88
|
llm: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<{
|
|
85
89
|
provider: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -163,6 +167,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
163
167
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
164
168
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
165
169
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
170
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
171
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
166
172
|
}>, Schemastery.ObjectT<{
|
|
167
173
|
enabled: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
168
174
|
baseUrl: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -171,6 +177,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
171
177
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
172
178
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
173
179
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
180
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
181
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
174
182
|
}>>;
|
|
175
183
|
llm: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<{
|
|
176
184
|
provider: import("@deepseek-ai/schemastery").default<string, string>;
|
package/dist/index.js
CHANGED
|
@@ -24,7 +24,10 @@ import { MemoryRunner } from './pipeline/runner.js';
|
|
|
24
24
|
import { RebuildController } from './pipeline/rebuild.js';
|
|
25
25
|
import { registerMemoryRpc, PLUGIN_VERSION } from './stats.js';
|
|
26
26
|
import { registerLiveSettings } from './settings.js';
|
|
27
|
-
import { NoopEmbeddingService
|
|
27
|
+
import { NoopEmbeddingService } from './store/embedding.js';
|
|
28
|
+
import { EmbeddingManager, EmbeddingSourceStore, makeLocalServiceFactory, resolveInitialEmbedding, } from './store/embedding-source.js';
|
|
29
|
+
import { ModelDownloadQueue } from './store/download-queue.js';
|
|
30
|
+
import { PINNED_TRANSFORMERS_VERSION, RuntimeInstaller } from './store/runtime-installer.js';
|
|
28
31
|
import { ensureDir } from './store/io.js';
|
|
29
32
|
import { L0Store } from './store/l0.js';
|
|
30
33
|
import { L1Store } from './store/l1.js';
|
|
@@ -45,25 +48,6 @@ export const inject = ['llm', 'tools', 'systemPrompt'];
|
|
|
45
48
|
* 导致 config 里嵌套对象为 undefined、apply 抛错、fiber FAILED 拖垮宿主启动。
|
|
46
49
|
*/
|
|
47
50
|
export const Config = memorySchema;
|
|
48
|
-
/** 按配置构造 embedding 服务:未启用或配置不完整 → Noop(纯 FTS 运行)。 */
|
|
49
|
-
function createEmbeddingService(cfg, logger) {
|
|
50
|
-
if (!cfg.embedding.enabled)
|
|
51
|
-
return new NoopEmbeddingService();
|
|
52
|
-
const { baseUrl, apiKey, model, dimensions } = cfg.embedding;
|
|
53
|
-
if (!baseUrl || !apiKey || !model || dimensions <= 0) {
|
|
54
|
-
logger.error('[memory] embedding.enabled=true 但 baseUrl/apiKey/model/dimensions 未配置完整,向量检索停用(纯 FTS 运行)');
|
|
55
|
-
return new NoopEmbeddingService();
|
|
56
|
-
}
|
|
57
|
-
return new RemoteEmbeddingService({
|
|
58
|
-
baseUrl,
|
|
59
|
-
apiKey,
|
|
60
|
-
model,
|
|
61
|
-
dimensions,
|
|
62
|
-
maxInputChars: cfg.embedding.maxInputChars,
|
|
63
|
-
timeoutMs: cfg.embedding.timeoutMs,
|
|
64
|
-
logger,
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
51
|
export async function apply(ctx, config) {
|
|
68
52
|
let logger = {
|
|
69
53
|
debug: (m) => ctx.logger.debug(m),
|
|
@@ -87,6 +71,8 @@ export async function apply(ctx, config) {
|
|
|
87
71
|
}
|
|
88
72
|
// ── 记忆模式运行时开关(官方 settings 服务,live 生效;缺失时恒开) ──
|
|
89
73
|
const live = registerLiveSettings(ctx, logger);
|
|
74
|
+
/** 插件停机标志:置位后不再发起后台 embeddings 调用(dispose 序最先设置)。 */
|
|
75
|
+
let disposed = false;
|
|
90
76
|
// ── 会话档位存储(sessionId → auto/chat/work/off;默认档 = config.family) ──
|
|
91
77
|
const modes = new SessionModeStore(dataDir, config.family, logger);
|
|
92
78
|
if (storageOk) {
|
|
@@ -97,15 +83,34 @@ export async function apply(ctx, config) {
|
|
|
97
83
|
logger.warn(`[memory] 会话档位载入失败(降级为默认档内存态): ${err instanceof Error ? err.message : String(err)}`);
|
|
98
84
|
}
|
|
99
85
|
}
|
|
86
|
+
// ── 嵌入源三态(D4:远程/本地/关闭)——状态文件优先于静态配置 ──
|
|
87
|
+
const sourceStore = new EmbeddingSourceStore(dataDir, logger);
|
|
88
|
+
const installer = new RuntimeInstaller(dataDir, PINNED_TRANSFORMERS_VERSION, { logger });
|
|
89
|
+
const downloader = new ModelDownloadQueue(dataDir, { mirror: config.embedding.mirror, logger });
|
|
90
|
+
const makeLocalService = makeLocalServiceFactory(installer, downloader, logger);
|
|
91
|
+
let initial = { svc: new NoopEmbeddingService(), dims: 0 };
|
|
92
|
+
/** 管理器引用:启动重嵌链/backfill 闭包在运行期解引用(声明早于创建避免 TDZ)。 */
|
|
93
|
+
let embedManagerRef;
|
|
94
|
+
if (storageOk) {
|
|
95
|
+
try {
|
|
96
|
+
await sourceStore.init();
|
|
97
|
+
initial = await resolveInitialEmbedding(config, sourceStore, downloader, makeLocalService, logger);
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
logger.warn(`[memory] 嵌入源初始解析失败(降级纯 FTS): ${errDetail(err)}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/** meta 写入永远用当前目标(杜绝启动期闭包里的陈旧 providerInfo 腐蚀 embedding_meta)。 */
|
|
104
|
+
const currentProviderInfo = () => embedManagerRef?.currentProviderInfo() ?? initial.providerInfo;
|
|
100
105
|
// ── 检索引擎(memory.db)与向量服务 ──
|
|
101
|
-
const embed =
|
|
102
|
-
const db = new MemoryDb(path.join(dataDir, 'memory.db'),
|
|
106
|
+
const embed = initial.svc;
|
|
107
|
+
const db = new MemoryDb(path.join(dataDir, 'memory.db'), initial.dims, logger);
|
|
103
108
|
// 插件卸载时关闭连接(WAL 落盘),注册一次即可
|
|
104
109
|
ctx.effect(() => () => db.close());
|
|
105
110
|
let dbInit = { needsReindex: false };
|
|
106
111
|
if (storageOk) {
|
|
107
112
|
try {
|
|
108
|
-
dbInit = db.init(
|
|
113
|
+
dbInit = db.init(initial.providerInfo);
|
|
109
114
|
if (db.isDegraded()) {
|
|
110
115
|
storageOk = false;
|
|
111
116
|
logger.error('[memory] 检索库初始化失败(schema/FTS 不可用),记忆功能停用');
|
|
@@ -161,21 +166,30 @@ export async function apply(ctx, config) {
|
|
|
161
166
|
logger.warn(`[memory] 蒸馏模型路由解析失败: ${errDetail(err)}`);
|
|
162
167
|
}
|
|
163
168
|
// embedding 配置变化 → 后台全量重嵌入(不阻塞启动)。
|
|
164
|
-
// meta
|
|
165
|
-
if (storageOk &&
|
|
166
|
-
const info =
|
|
169
|
+
// meta 标记条件 = 失败为 0 且缺失复查为 0(服务未就绪的空转不算成功)。
|
|
170
|
+
if (storageOk && initial.providerInfo && !db.isDegraded()) {
|
|
171
|
+
const info = initial.providerInfo;
|
|
167
172
|
if (dbInit.needsReindex) {
|
|
168
173
|
if (caps.vectorSearch) {
|
|
169
174
|
void (async () => {
|
|
170
175
|
try {
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
+
if (disposed || embedManagerRef?.isBusy())
|
|
177
|
+
return;
|
|
178
|
+
// 增量:配置变化路径向量表已被 drop,全部记录均"缺失",等效全量;
|
|
179
|
+
// 活切换发起时让路(swap 会 drop 表,并发跑只是白烧嵌入调用)
|
|
180
|
+
const shouldCancel = () => disposed || !!embedManagerRef?.isBusy();
|
|
181
|
+
const r1 = await stores.l1.reindex({ shouldCancel });
|
|
182
|
+
const r0 = await stores.l0.reindex({ shouldCancel });
|
|
183
|
+
// missing 复查:嵌入服务未就绪时 reindex 短路 0/0/0——那是"没跑"不是"成功",
|
|
184
|
+
// 不得标记 meta(否则启动链不再补,补齐被无限推迟到 backfill)
|
|
185
|
+
const missingAfter = db.countL1VecMissing(db.getVecSkipSet('l1')) + db.countL0VecMissing(db.getVecSkipSet('l0'));
|
|
186
|
+
if (r1.failed === 0 && r0.failed === 0 && missingAfter === 0) {
|
|
187
|
+
db.markEmbeddingSynced(currentProviderInfo() ?? info);
|
|
188
|
+
const skipNote = r1.skipped + r0.skipped > 0 ? `,跳过不可嵌入 ${r1.skipped + r0.skipped} 条` : '';
|
|
189
|
+
logger.info(`[memory] 向量重建完成:L1 ${r1.written} 条,L0 ${r0.written} 条${skipNote}(原因:${dbInit.reason ?? '配置变化'})`);
|
|
176
190
|
}
|
|
177
191
|
else {
|
|
178
|
-
logger.warn(`[memory] 向量重建未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed}
|
|
192
|
+
logger.warn(`[memory] 向量重建未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed},剩缺 ${Math.max(missingAfter, 0)} 条),下次启动/补齐周期重试`);
|
|
179
193
|
}
|
|
180
194
|
}
|
|
181
195
|
catch (err) {
|
|
@@ -186,22 +200,41 @@ export async function apply(ctx, config) {
|
|
|
186
200
|
}
|
|
187
201
|
else {
|
|
188
202
|
// meta 已匹配或空库(initSchema 已写入),幂等重写一次
|
|
189
|
-
db.markEmbeddingSynced(info);
|
|
203
|
+
db.markEmbeddingSynced(currentProviderInfo() ?? info);
|
|
190
204
|
}
|
|
191
|
-
// 周期性向量补齐:嵌入失败的批次/漏网记录自动补上(30
|
|
192
|
-
|
|
205
|
+
// 周期性向量补齐:嵌入失败的批次/漏网记录自动补上(30 分钟一次)。
|
|
206
|
+
// 判据 = 排除 skip 集后的缺失数(增量语义):缺 1 条只补 1 条,
|
|
207
|
+
// 零向量记录进 skip 集后不再触发——否则补齐永不收敛、每 30 分钟全量重嵌。
|
|
208
|
+
// 注册不再依赖启动期 caps/providerInfo(off 起步后活切回旧源也要能补),
|
|
209
|
+
// 执行期动态查当前目标与能力位。
|
|
210
|
+
{
|
|
193
211
|
const backfill = () => {
|
|
194
212
|
void (async () => {
|
|
195
213
|
try {
|
|
196
|
-
|
|
197
|
-
|
|
214
|
+
if (disposed)
|
|
215
|
+
return;
|
|
216
|
+
const infoNow = currentProviderInfo();
|
|
217
|
+
const capsNow = db.getCapabilities();
|
|
218
|
+
if (!infoNow || !capsNow.vectorSearch)
|
|
219
|
+
return;
|
|
220
|
+
// 活切换链(安装/下载/重嵌)进行中让路——增量 reindex 天然幂等,
|
|
221
|
+
// 但并发跑只是白烧一次嵌入调用
|
|
222
|
+
if (embedManagerRef?.isBusy())
|
|
223
|
+
return;
|
|
224
|
+
const l1Missing = db.countL1VecMissing(db.getVecSkipSet('l1')) > 0;
|
|
225
|
+
const l0Missing = db.countL0VecMissing(db.getVecSkipSet('l0')) > 0;
|
|
198
226
|
if (!l1Missing && !l0Missing)
|
|
199
227
|
return;
|
|
200
|
-
const
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
228
|
+
const shouldCancel = () => disposed || !!embedManagerRef?.isBusy();
|
|
229
|
+
const r1 = await stores.l1.reindex({ shouldCancel });
|
|
230
|
+
const r0 = await stores.l0.reindex({ shouldCancel });
|
|
231
|
+
const missingAfter = db.countL1VecMissing(db.getVecSkipSet('l1')) + db.countL0VecMissing(db.getVecSkipSet('l0'));
|
|
232
|
+
if (r1.failed === 0 && r0.failed === 0 && missingAfter === 0) {
|
|
233
|
+
db.markEmbeddingSynced(infoNow);
|
|
234
|
+
logger.info(`[memory] 向量补齐完成:L1 ${r1.written} 条,L0 ${r0.written} 条(跳过不可嵌入 ${r1.skipped + r0.skipped} 条)`);
|
|
235
|
+
}
|
|
236
|
+
else if (r1.failed > 0 || r0.failed > 0) {
|
|
237
|
+
logger.warn(`[memory] 向量补齐未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed}),下个周期重试`);
|
|
205
238
|
}
|
|
206
239
|
}
|
|
207
240
|
catch (err) {
|
|
@@ -219,14 +252,32 @@ export async function apply(ctx, config) {
|
|
|
219
252
|
});
|
|
220
253
|
}
|
|
221
254
|
}
|
|
255
|
+
// 嵌入管理器(活切换/下载/运行时安装;存储降级时不建——RPC 走 supported=false)
|
|
256
|
+
const embedManager = storageOk && !db.isDegraded()
|
|
257
|
+
? new EmbeddingManager({
|
|
258
|
+
dataDir,
|
|
259
|
+
cfg: config,
|
|
260
|
+
db,
|
|
261
|
+
l0: stores.l0,
|
|
262
|
+
l1: stores.l1,
|
|
263
|
+
sourceStore,
|
|
264
|
+
installer,
|
|
265
|
+
downloader,
|
|
266
|
+
initial,
|
|
267
|
+
logger,
|
|
268
|
+
makeLocal: makeLocalService,
|
|
269
|
+
})
|
|
270
|
+
: undefined;
|
|
271
|
+
embedManagerRef = embedManager;
|
|
222
272
|
const runner = new MemoryRunner(ctx, config, stores, logger, live);
|
|
223
273
|
await runner.init();
|
|
224
274
|
// 重建控制器(存储降级时不建——RPC 端点走 supported=false 分支)
|
|
225
275
|
const rebuild = storageOk && !db.isDegraded()
|
|
226
276
|
? new RebuildController(ctx, config, stores, db, runner, logger, live)
|
|
227
277
|
: undefined;
|
|
278
|
+
let flushL0;
|
|
228
279
|
if (storageOk) {
|
|
229
|
-
registerCapture(ctx, config, runner, stores.l0, logger, live, modes);
|
|
280
|
+
flushL0 = registerCapture(ctx, config, runner, stores.l0, logger, live, modes);
|
|
230
281
|
}
|
|
231
282
|
const recall = registerRecall(ctx, config, stores, logger, live, modes);
|
|
232
283
|
runner.setAfterRun(recall.invalidateProfile);
|
|
@@ -234,6 +285,18 @@ export async function apply(ctx, config) {
|
|
|
234
285
|
registerMemoryRpc(ctx, config, stores, logger, {
|
|
235
286
|
degraded: () => !storageOk || db.isDegraded(),
|
|
236
287
|
pending: () => runner.pendingCount,
|
|
237
|
-
}, live, modes, dataDir, rebuild);
|
|
288
|
+
}, live, modes, dataDir, rebuild, embedManager);
|
|
238
289
|
logger.info(`[memory] L0~L3 分层蒸馏记忆插件就绪(L1 记忆 ${storageOk ? stores.l1.size : 0} 条 | 捕获=${storageOk && config.capture.enabled} | 蒸馏=${storageOk && config.extract.enabled} | 召回=${config.recall.enabled})`);
|
|
290
|
+
// 停机顺序(M7):置停机标志(后台 embeddings 不再发起)→ 停蒸馏取新任务 →
|
|
291
|
+
// 冲刷 L0 串行链(排队消息先落盘)→ 关库。cordis disposer 为 LIFO 逐个 await,
|
|
292
|
+
// 本 effect 晚注册故先执行;apply 中途失败时由上方早注册的 db.close 兜底(双关安全)。
|
|
293
|
+
ctx.effect(() => () => {
|
|
294
|
+
disposed = true;
|
|
295
|
+
runner.stop();
|
|
296
|
+
embedManager?.dispose();
|
|
297
|
+
return (async () => {
|
|
298
|
+
await flushL0?.();
|
|
299
|
+
db.close();
|
|
300
|
+
})();
|
|
301
|
+
});
|
|
239
302
|
}
|
|
@@ -59,6 +59,8 @@ export declare class RebuildController {
|
|
|
59
59
|
constructor(ctx: Context, cfg: MemoryConfig, stores: RebuildStores, db: MemoryDb, runner: Pick<MemoryRunner, 'enqueueRebuildTask' | 'runRebuildTurn' | 'states'>, logger: MemoryLogger, live: LiveSettingsHandle);
|
|
60
60
|
/** 状态快照(idle 时附带实时 L0 预估,供确认弹窗显示成本)。 */
|
|
61
61
|
getStatus(): RebuildStatus;
|
|
62
|
+
/** 内存中尚未处理的会话快照块数(收尾后应为 0——快照即弃,诊断/冒烟用)。 */
|
|
63
|
+
get chunkCount(): number;
|
|
62
64
|
/** 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。 */
|
|
63
65
|
start(): RebuildStatus;
|
|
64
66
|
/** 请求取消:当前块完成后停止,已重建部分保留并照常收尾 L2/L3。 */
|
package/dist/pipeline/rebuild.js
CHANGED
|
@@ -104,6 +104,10 @@ export class RebuildController {
|
|
|
104
104
|
}
|
|
105
105
|
return { ...this.status };
|
|
106
106
|
}
|
|
107
|
+
/** 内存中尚未处理的会话快照块数(收尾后应为 0——快照即弃,诊断/冒烟用)。 */
|
|
108
|
+
get chunkCount() {
|
|
109
|
+
return this.chunks.length;
|
|
110
|
+
}
|
|
107
111
|
/** 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。 */
|
|
108
112
|
start() {
|
|
109
113
|
if (this.status.running)
|
|
@@ -270,6 +274,9 @@ export class RebuildController {
|
|
|
270
274
|
this.status.phase = phase;
|
|
271
275
|
this.status.error = error;
|
|
272
276
|
this.status.finishedAt = Date.now();
|
|
277
|
+
// 快照即弃:全量 L0 消息(可能几十 MB)在重建结束/取消/失败后必须释放,
|
|
278
|
+
// 不能滞留到下一次 start() 覆盖(M6——宿主长跑内存只增不减)
|
|
279
|
+
this.chunks = [];
|
|
273
280
|
const cost = this.status.finishedAt - (this.status.startedAt ?? this.status.finishedAt);
|
|
274
281
|
this.logger.info(`[memory] 重建结束(${phase}):${this.status.done}/${this.status.total} 会话,产出 ${this.status.recordsBuilt} 条记录,耗时 ${Math.round(cost / 1000)}s` +
|
|
275
282
|
(error ? `,错误:${error}` : ''));
|
|
@@ -48,6 +48,8 @@ export declare class MemoryRunner {
|
|
|
48
48
|
private readonly live;
|
|
49
49
|
private tasks;
|
|
50
50
|
private draining;
|
|
51
|
+
/** 停止标志(dispose 序置位):不再取新任务;进行中任务自然收尾,其 DB 写入失败由各层兜底捕获。 */
|
|
52
|
+
private stopped;
|
|
51
53
|
private pending;
|
|
52
54
|
private readonly pendingFile;
|
|
53
55
|
private background;
|
|
@@ -68,9 +70,13 @@ export declare class MemoryRunner {
|
|
|
68
70
|
enqueueRebuildTask(run: () => Promise<unknown>): void;
|
|
69
71
|
/** 重建蒸馏轮:统一 auto 档,不受缓冲 200 上限(历史会话全量入桶,由 char 预算分块)。 */
|
|
70
72
|
runRebuildTurn(sessionId: string, messages: ConversationMessage[]): Promise<number>;
|
|
73
|
+
/** 停止取新任务(插件 dispose 序调用;进行中任务照常跑完但不 await——LLM 慢调用不拖住宿主卸载)。 */
|
|
74
|
+
stop(): void;
|
|
71
75
|
private pushTask;
|
|
72
76
|
private drain;
|
|
73
|
-
/** 缓冲落盘(每次蒸馏尝试后调用;失败只告警不阻断管线)。
|
|
77
|
+
/** 缓冲落盘(每次蒸馏尝试后调用;失败只告警不阻断管线)。
|
|
78
|
+
* 非重建轮持久化前按桶截断到上限:重建取消后的大桶不至于在后续每次
|
|
79
|
+
* 蒸馏尝试时反复整量序列化落盘(多 MB 级 IO);重建轮豁免维持。 */
|
|
74
80
|
private persistPending;
|
|
75
81
|
private runTurn;
|
|
76
82
|
}
|