dsh-layered-memory 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +101 -81
- package/README.md +91 -71
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/dist/client.js +1242 -325
- package/dist/config.d.ts +12 -0
- package/dist/config.js +20 -17
- package/dist/hooks/capture.d.ts +17 -1
- package/dist/hooks/capture.js +45 -11
- package/dist/hooks/recall.d.ts +7 -0
- package/dist/hooks/recall.js +20 -4
- package/dist/index.d.ts +8 -0
- package/dist/index.js +112 -44
- package/dist/pipeline/rebuild.d.ts +80 -0
- package/dist/pipeline/rebuild.js +307 -0
- package/dist/pipeline/runner.d.ts +42 -4
- package/dist/pipeline/runner.js +125 -21
- package/dist/settings.js +79 -20
- package/dist/stats.d.ts +8 -1
- package/dist/stats.js +156 -14
- package/dist/store/download-queue.d.ts +71 -0
- package/dist/store/download-queue.js +313 -0
- package/dist/store/embedding-source.d.ts +160 -0
- package/dist/store/embedding-source.js +421 -0
- package/dist/store/embedding.d.ts +3 -1
- package/dist/store/embedding.js +5 -0
- package/dist/store/l0.d.ts +13 -5
- package/dist/store/l0.js +33 -6
- package/dist/store/l1.d.ts +15 -5
- package/dist/store/l1.js +40 -15
- package/dist/store/local-embedding.d.ts +64 -0
- package/dist/store/local-embedding.js +120 -0
- package/dist/store/model-catalog.d.ts +45 -0
- package/dist/store/model-catalog.js +78 -0
- package/dist/store/pending.d.ts +15 -0
- package/dist/store/pending.js +56 -0
- package/dist/store/runtime-installer.d.ts +60 -0
- package/dist/store/runtime-installer.js +181 -0
- package/dist/store/sqlite.d.ts +68 -7
- package/dist/store/sqlite.js +414 -71
- package/dist/store/state.d.ts +6 -0
- package/dist/store/state.js +11 -0
- package/dist/tools/index.js +6 -4
- package/dist/util/filelog.d.ts +2 -0
- package/dist/util/filelog.js +20 -3
- package/package.json +1 -1
- package/assets/readme/hero.svg +0 -58
package/dist/config.d.ts
CHANGED
|
@@ -64,6 +64,10 @@ export interface MemoryConfig {
|
|
|
64
64
|
maxInputChars: number;
|
|
65
65
|
/** 单次调用超时(ms)。 */
|
|
66
66
|
timeoutMs: number;
|
|
67
|
+
/** 允许本地嵌入模型档(部署上限:公司环境可禁下载与本地推理)。 */
|
|
68
|
+
allowLocalModels: boolean;
|
|
69
|
+
/** 模型下载镜像根地址(默认国内可达的 hf-mirror.com,可改回官方)。 */
|
|
70
|
+
mirror: string;
|
|
67
71
|
};
|
|
68
72
|
llm: {
|
|
69
73
|
/** 蒸馏用的 provider 路由;留空用当前默认选择。 */
|
|
@@ -147,6 +151,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
147
151
|
dimensions: Schema<number, number>;
|
|
148
152
|
maxInputChars: Schema<number, number>;
|
|
149
153
|
timeoutMs: Schema<number, number>;
|
|
154
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
155
|
+
mirror: Schema<string, string>;
|
|
150
156
|
}>, Schemastery.ObjectT<{
|
|
151
157
|
enabled: Schema<boolean, boolean>;
|
|
152
158
|
baseUrl: Schema<string, string>;
|
|
@@ -155,6 +161,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
155
161
|
dimensions: Schema<number, number>;
|
|
156
162
|
maxInputChars: Schema<number, number>;
|
|
157
163
|
timeoutMs: Schema<number, number>;
|
|
164
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
165
|
+
mirror: Schema<string, string>;
|
|
158
166
|
}>>;
|
|
159
167
|
llm: Schema<Schemastery.ObjectS<{
|
|
160
168
|
provider: Schema<string, string>;
|
|
@@ -238,6 +246,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
238
246
|
dimensions: Schema<number, number>;
|
|
239
247
|
maxInputChars: Schema<number, number>;
|
|
240
248
|
timeoutMs: Schema<number, number>;
|
|
249
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
250
|
+
mirror: Schema<string, string>;
|
|
241
251
|
}>, Schemastery.ObjectT<{
|
|
242
252
|
enabled: Schema<boolean, boolean>;
|
|
243
253
|
baseUrl: Schema<string, string>;
|
|
@@ -246,6 +256,8 @@ export declare const memorySchema: Schema<Schemastery.ObjectS<{
|
|
|
246
256
|
dimensions: Schema<number, number>;
|
|
247
257
|
maxInputChars: Schema<number, number>;
|
|
248
258
|
timeoutMs: Schema<number, number>;
|
|
259
|
+
allowLocalModels: Schema<boolean, boolean>;
|
|
260
|
+
mirror: Schema<string, string>;
|
|
249
261
|
}>>;
|
|
250
262
|
llm: Schema<Schemastery.ObjectS<{
|
|
251
263
|
provider: Schema<string, string>;
|
package/dist/config.js
CHANGED
|
@@ -11,53 +11,56 @@ export const memorySchema = Schema.object({
|
|
|
11
11
|
capture: Schema.object({
|
|
12
12
|
enabled: Schema.boolean().default(true),
|
|
13
13
|
stripCodeBlocks: Schema.boolean().default(true),
|
|
14
|
-
maxMessageChars: Schema.number().default(4000),
|
|
14
|
+
maxMessageChars: Schema.number().min(200).max(200_000).default(4000),
|
|
15
15
|
}),
|
|
16
16
|
extract: Schema.object({
|
|
17
17
|
enabled: Schema.boolean().default(true),
|
|
18
|
-
minMessages: Schema.number().default(1),
|
|
19
|
-
backgroundMessages: Schema.number().default(10),
|
|
20
|
-
candidatePool: Schema.number().default(5),
|
|
18
|
+
minMessages: Schema.number().min(1).max(100).default(1),
|
|
19
|
+
backgroundMessages: Schema.number().min(0).max(50).default(10),
|
|
20
|
+
candidatePool: Schema.number().min(1).max(20).default(5),
|
|
21
21
|
}),
|
|
22
22
|
l2: Schema.object({
|
|
23
23
|
enabled: Schema.boolean().default(true),
|
|
24
|
-
minNewMemories: Schema.number().default(5),
|
|
25
|
-
maxScenes: Schema.number().default(12),
|
|
26
|
-
sceneContextLimit: Schema.number().default(3),
|
|
24
|
+
minNewMemories: Schema.number().min(1).max(100).default(5),
|
|
25
|
+
maxScenes: Schema.number().min(1).max(100).default(12),
|
|
26
|
+
sceneContextLimit: Schema.number().min(0).max(20).default(3),
|
|
27
27
|
}),
|
|
28
28
|
l3: Schema.object({
|
|
29
29
|
enabled: Schema.boolean().default(true),
|
|
30
|
-
interval: Schema.number().default(20),
|
|
30
|
+
interval: Schema.number().min(1).max(200).default(20),
|
|
31
31
|
}),
|
|
32
32
|
recall: Schema.object({
|
|
33
33
|
enabled: Schema.boolean().default(true),
|
|
34
|
-
maxResults: Schema.number().default(5),
|
|
34
|
+
maxResults: Schema.number().min(1).max(20).default(5),
|
|
35
35
|
includePersona: Schema.boolean().default(true),
|
|
36
36
|
includeSceneNav: Schema.boolean().default(true),
|
|
37
37
|
strategy: Schema.union(['keyword', 'embedding', 'hybrid']).default('hybrid'),
|
|
38
|
-
scoreThreshold: Schema.number().default(0.3),
|
|
38
|
+
scoreThreshold: Schema.number().min(0).max(1).default(0.3),
|
|
39
39
|
}),
|
|
40
40
|
embedding: Schema.object({
|
|
41
41
|
enabled: Schema.boolean().default(false),
|
|
42
42
|
baseUrl: Schema.string().default(''),
|
|
43
43
|
apiKey: Schema.string().default(''),
|
|
44
44
|
model: Schema.string().default(''),
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
// 0 = 纯 FTS 模式(合法值,勿设 min>0)
|
|
46
|
+
dimensions: Schema.number().min(0).max(8192).default(0),
|
|
47
|
+
maxInputChars: Schema.number().min(100).max(100_000).default(5000),
|
|
48
|
+
timeoutMs: Schema.number().min(1000).max(300_000).default(10_000),
|
|
49
|
+
allowLocalModels: Schema.boolean().default(true),
|
|
50
|
+
mirror: Schema.string().default('https://hf-mirror.com'),
|
|
48
51
|
}),
|
|
49
52
|
llm: Schema.object({
|
|
50
53
|
provider: Schema.string().default(''),
|
|
51
54
|
model: Schema.string().default(''),
|
|
52
55
|
// 推理模型(如 v4-flash)的 reasoning 计入输出预算:预算不足会被思考吃光导致正文 0 字符
|
|
53
|
-
maxTokens: Schema.number().default(
|
|
56
|
+
maxTokens: Schema.number().min(1024).max(1_000_000).default(256_000),
|
|
54
57
|
// 蒸馏是结构化抽取任务,默认关思考(off):v4-flash 默认 high 档的思考可把任意 maxTokens
|
|
55
58
|
// 预算全部吃光导致正文 0 字符;非推理模型不认识 effort 时会报 UNSUPPORTED_REASONING_EFFORT,设空串跳过
|
|
56
59
|
reasoningEffort: Schema.union(['', 'off', 'high', 'max']).default('off'),
|
|
57
|
-
temperature: Schema.number().default(0.3),
|
|
60
|
+
temperature: Schema.number().min(0).max(2).default(0.3),
|
|
58
61
|
// 模型上下文 1M token,日常压到 ~700k 使用(中文按 1 字≈1 token 保守折算)
|
|
59
|
-
maxInputChars: Schema.number().default(700_000),
|
|
60
|
-
timeoutMs: Schema.number().default(120_000),
|
|
62
|
+
maxInputChars: Schema.number().min(1000).max(1_000_000).default(700_000),
|
|
63
|
+
timeoutMs: Schema.number().min(1000).max(600_000).default(120_000),
|
|
61
64
|
}),
|
|
62
65
|
tools: Schema.boolean().default(true),
|
|
63
66
|
});
|
package/dist/hooks/capture.d.ts
CHANGED
|
@@ -7,7 +7,23 @@ import type { L0Store } from '../store/l0.js';
|
|
|
7
7
|
import type { LiveSettingsHandle } from '../settings.js';
|
|
8
8
|
import type { MemoryLogger } from '../types.js';
|
|
9
9
|
export declare function isCaptureRelevant(type: string): boolean;
|
|
10
|
-
|
|
10
|
+
/**
|
|
11
|
+
* 按会话缓冲轮次事件。turn 被消费后剩余前缀为空即删除 Map 条目——
|
|
12
|
+
* 条目随会话数累积是慢泄漏(每会话残留一个数组引用,宿主长跑不释放)。
|
|
13
|
+
*/
|
|
14
|
+
export declare class CaptureBuffers {
|
|
15
|
+
private readonly map;
|
|
16
|
+
/** 活跃缓冲条目数(诊断/冒烟用)。 */
|
|
17
|
+
get size(): number;
|
|
18
|
+
push(sid: string, event: SessionEvent): void;
|
|
19
|
+
/** 取出该 turn 的全部事件(不含 turn/start 自身);无匹配 start 时返回整个缓冲。 */
|
|
20
|
+
takeTurn(sid: string, turn: number): SessionEvent[];
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* 注册 L0 捕获。返回 L0 串行链的冲刷函数(dispose 序在关库前 await,
|
|
24
|
+
* 排队中的 turn 消息先落盘);capture 关闭时返回 undefined。
|
|
25
|
+
*/
|
|
26
|
+
export declare function registerCapture(ctx: Context, cfg: MemoryConfig, runner: MemoryRunner, l0: L0Store, logger: MemoryLogger, live: LiveSettingsHandle, modes: SessionModeStore): (() => Promise<void>) | undefined;
|
|
11
27
|
/**
|
|
12
28
|
* 缓冲上限裁剪(防御性;RELEVANT_TYPES 过滤后基本不可达)。
|
|
13
29
|
* 铁律:进行中轮次(turn/start 之后、turn/end 之前)的事件绝不裁——
|
package/dist/hooks/capture.js
CHANGED
|
@@ -17,11 +17,49 @@ export function isCaptureRelevant(type) {
|
|
|
17
17
|
return RELEVANT_TYPES.has(type);
|
|
18
18
|
}
|
|
19
19
|
const MAX_BUFFER = 500;
|
|
20
|
+
/**
|
|
21
|
+
* 按会话缓冲轮次事件。turn 被消费后剩余前缀为空即删除 Map 条目——
|
|
22
|
+
* 条目随会话数累积是慢泄漏(每会话残留一个数组引用,宿主长跑不释放)。
|
|
23
|
+
*/
|
|
24
|
+
export class CaptureBuffers {
|
|
25
|
+
map = new Map();
|
|
26
|
+
/** 活跃缓冲条目数(诊断/冒烟用)。 */
|
|
27
|
+
get size() {
|
|
28
|
+
return this.map.size;
|
|
29
|
+
}
|
|
30
|
+
push(sid, event) {
|
|
31
|
+
let buf = this.map.get(sid);
|
|
32
|
+
if (!buf) {
|
|
33
|
+
buf = [];
|
|
34
|
+
this.map.set(sid, buf);
|
|
35
|
+
}
|
|
36
|
+
buf.push(event);
|
|
37
|
+
trimBuffer(buf);
|
|
38
|
+
}
|
|
39
|
+
/** 取出该 turn 的全部事件(不含 turn/start 自身);无匹配 start 时返回整个缓冲。 */
|
|
40
|
+
takeTurn(sid, turn) {
|
|
41
|
+
const buf = this.map.get(sid);
|
|
42
|
+
if (!buf)
|
|
43
|
+
return [];
|
|
44
|
+
const startIdx = findTurnStart(buf, turn);
|
|
45
|
+
const turnEvents = startIdx === -1 ? buf : buf.slice(startIdx + 1);
|
|
46
|
+
const rest = startIdx === -1 ? [] : buf.slice(0, startIdx);
|
|
47
|
+
if (rest.length === 0)
|
|
48
|
+
this.map.delete(sid);
|
|
49
|
+
else
|
|
50
|
+
this.map.set(sid, rest);
|
|
51
|
+
return turnEvents;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* 注册 L0 捕获。返回 L0 串行链的冲刷函数(dispose 序在关库前 await,
|
|
56
|
+
* 排队中的 turn 消息先落盘);capture 关闭时返回 undefined。
|
|
57
|
+
*/
|
|
20
58
|
export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
21
59
|
if (!cfg.capture.enabled)
|
|
22
60
|
return;
|
|
23
61
|
const startFloor = Date.now();
|
|
24
|
-
const buffers = new
|
|
62
|
+
const buffers = new CaptureBuffers();
|
|
25
63
|
// L0 即时落盘链:turn/end 立刻写,不被蒸馏队列(慢 LLM 调用)阻塞;
|
|
26
64
|
// 串行化保证同轮次顺序与单次写入(进程退出时排队中的 L0 不再依赖蒸馏完成)
|
|
27
65
|
let l0Queue = Promise.resolve();
|
|
@@ -43,18 +81,10 @@ export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
|
43
81
|
}
|
|
44
82
|
return;
|
|
45
83
|
}
|
|
46
|
-
|
|
47
|
-
if (!buf) {
|
|
48
|
-
buf = [];
|
|
49
|
-
buffers.set(sid, buf);
|
|
50
|
-
}
|
|
51
|
-
buf.push(event);
|
|
52
|
-
trimBuffer(buf);
|
|
84
|
+
buffers.push(sid, event);
|
|
53
85
|
if (event.type === 'turn/end') {
|
|
54
86
|
const turn = event.data.turn;
|
|
55
|
-
const
|
|
56
|
-
const turnEvents = startIdx === -1 ? buf : buf.slice(startIdx + 1);
|
|
57
|
-
buffers.set(sid, startIdx === -1 ? [] : buf.slice(0, startIdx));
|
|
87
|
+
const turnEvents = buffers.takeTurn(sid, turn);
|
|
58
88
|
const messages = turnEventsToMessages(turnEvents, cfg, logger);
|
|
59
89
|
if (messages.length > 0) {
|
|
60
90
|
const roles = messages.reduce((acc, m) => {
|
|
@@ -84,6 +114,10 @@ export function registerCapture(ctx, cfg, runner, l0, logger, live, modes) {
|
|
|
84
114
|
logger.warn(`[memory] session/event 处理失败: ${err instanceof Error ? err.message : String(err)}`);
|
|
85
115
|
}
|
|
86
116
|
});
|
|
117
|
+
// 冲刷 = 等待串行链排空(链上每环自带 catch,永不 reject)。
|
|
118
|
+
// 注:极小概率在 await 期间又入队的新消息会落到链尾——其 JSONL 事实源照写、
|
|
119
|
+
// DB 写入由 upsert 内部兜底(关库后 warn),下次重建自愈。
|
|
120
|
+
return () => l0Queue;
|
|
87
121
|
}
|
|
88
122
|
/**
|
|
89
123
|
* 缓冲上限裁剪(防御性;RELEVANT_TYPES 过滤后基本不可达)。
|
package/dist/hooks/recall.d.ts
CHANGED
|
@@ -20,6 +20,13 @@ import type { PersonaStore } from '../store/persona.js';
|
|
|
20
20
|
import type { SceneStore } from '../store/scenes.js';
|
|
21
21
|
import type { SessionModeStore } from '../store/session-modes.js';
|
|
22
22
|
import type { MemoryLogger } from '../types.js';
|
|
23
|
+
/**
|
|
24
|
+
* 从会话消息构建召回查询(纯函数):末尾 N 条 + 总长截断,空输入返回空串。
|
|
25
|
+
* 全史拼接会让 MATCH 表达式随会话长度线性膨胀(整会话累计二次方成本)。
|
|
26
|
+
*/
|
|
27
|
+
export declare function buildRecallQuery(messages: Array<{
|
|
28
|
+
content: unknown;
|
|
29
|
+
}>, tailMessages?: number, maxChars?: number): string;
|
|
23
30
|
export interface RecallHooks {
|
|
24
31
|
/** 管线更新后调用:画像/场景缓存立即失效并异步刷新。 */
|
|
25
32
|
invalidateProfile(): void;
|
package/dist/hooks/recall.js
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
import { blocksToText } from '../util/text.js';
|
|
2
2
|
const PROFILE_TTL = 60_000;
|
|
3
|
+
/** 召回查询只取会话末尾 N 条消息(长会话每步把全史拼进 FTS MATCH 会让检索成本线性上涨)。 */
|
|
4
|
+
const RECALL_QUERY_TAIL_MESSAGES = 8;
|
|
5
|
+
/** 召回查询总字符上限(保留末尾——最新语境权重最高)。 */
|
|
6
|
+
const RECALL_QUERY_MAX_CHARS = 2_000;
|
|
7
|
+
/**
|
|
8
|
+
* 从会话消息构建召回查询(纯函数):末尾 N 条 + 总长截断,空输入返回空串。
|
|
9
|
+
* 全史拼接会让 MATCH 表达式随会话长度线性膨胀(整会话累计二次方成本)。
|
|
10
|
+
*/
|
|
11
|
+
export function buildRecallQuery(messages, tailMessages = RECALL_QUERY_TAIL_MESSAGES, maxChars = RECALL_QUERY_MAX_CHARS) {
|
|
12
|
+
const tail = messages.slice(-tailMessages);
|
|
13
|
+
let text = tail.map((m) => blocksToText(m.content)).join(' ').trim();
|
|
14
|
+
if (text.length > maxChars)
|
|
15
|
+
text = text.slice(-maxChars);
|
|
16
|
+
return text;
|
|
17
|
+
}
|
|
3
18
|
const MEMORY_TOOLS_GUIDE = `<memory-tools-guide>
|
|
4
19
|
## 记忆工具调用指南
|
|
5
20
|
|
|
@@ -57,10 +72,7 @@ export function registerRecall(ctx, cfg, stores, logger, live, modes) {
|
|
|
57
72
|
l1Cache.set(payload.agent.id, []);
|
|
58
73
|
return next();
|
|
59
74
|
}
|
|
60
|
-
const query = payload.messages
|
|
61
|
-
.map((m) => blocksToText(m.content))
|
|
62
|
-
.join(' ')
|
|
63
|
-
.trim();
|
|
75
|
+
const query = buildRecallQuery(payload.messages);
|
|
64
76
|
if (query) {
|
|
65
77
|
const hits = await stores.l1.search(query, cfg.recall.maxResults, {
|
|
66
78
|
scoreThreshold: cfg.recall.scoreThreshold,
|
|
@@ -71,6 +83,10 @@ export function registerRecall(ctx, cfg, stores, logger, live, modes) {
|
|
|
71
83
|
logger.info(`[memory] 召回命中 ${hits.length} 条 L1(mode=${mode},query="${query.slice(0, 30).replace(/\n/g, ' ')}…",agent=${payload.agent.id})`);
|
|
72
84
|
}
|
|
73
85
|
}
|
|
86
|
+
else {
|
|
87
|
+
// 空查询清缓存:不把上一步的命中注入到与本步无关的上下文
|
|
88
|
+
l1Cache.set(payload.agent.id, []);
|
|
89
|
+
}
|
|
74
90
|
}
|
|
75
91
|
catch (err) {
|
|
76
92
|
logger.warn(`[memory] 召回检索失败: ${err instanceof Error ? err.message : String(err)}`);
|
package/dist/index.d.ts
CHANGED
|
@@ -72,6 +72,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
72
72
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
73
73
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
74
74
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
75
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
76
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
75
77
|
}>, Schemastery.ObjectT<{
|
|
76
78
|
enabled: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
77
79
|
baseUrl: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -80,6 +82,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
80
82
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
81
83
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
82
84
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
85
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
86
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
83
87
|
}>>;
|
|
84
88
|
llm: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<{
|
|
85
89
|
provider: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -163,6 +167,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
163
167
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
164
168
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
165
169
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
170
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
171
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
166
172
|
}>, Schemastery.ObjectT<{
|
|
167
173
|
enabled: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
168
174
|
baseUrl: import("@deepseek-ai/schemastery").default<string, string>;
|
|
@@ -171,6 +177,8 @@ export declare const Config: import("@deepseek-ai/schemastery").default<Schemast
|
|
|
171
177
|
dimensions: import("@deepseek-ai/schemastery").default<number, number>;
|
|
172
178
|
maxInputChars: import("@deepseek-ai/schemastery").default<number, number>;
|
|
173
179
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number>;
|
|
180
|
+
allowLocalModels: import("@deepseek-ai/schemastery").default<boolean, boolean>;
|
|
181
|
+
mirror: import("@deepseek-ai/schemastery").default<string, string>;
|
|
174
182
|
}>>;
|
|
175
183
|
llm: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<{
|
|
176
184
|
provider: import("@deepseek-ai/schemastery").default<string, string>;
|
package/dist/index.js
CHANGED
|
@@ -21,9 +21,13 @@ import { memorySchema, resolveDataDir } from './config.js';
|
|
|
21
21
|
import { registerCapture } from './hooks/capture.js';
|
|
22
22
|
import { registerRecall } from './hooks/recall.js';
|
|
23
23
|
import { MemoryRunner } from './pipeline/runner.js';
|
|
24
|
+
import { RebuildController } from './pipeline/rebuild.js';
|
|
24
25
|
import { registerMemoryRpc, PLUGIN_VERSION } from './stats.js';
|
|
25
26
|
import { registerLiveSettings } from './settings.js';
|
|
26
|
-
import { NoopEmbeddingService
|
|
27
|
+
import { NoopEmbeddingService } from './store/embedding.js';
|
|
28
|
+
import { EmbeddingManager, EmbeddingSourceStore, makeLocalServiceFactory, resolveInitialEmbedding, } from './store/embedding-source.js';
|
|
29
|
+
import { ModelDownloadQueue } from './store/download-queue.js';
|
|
30
|
+
import { PINNED_TRANSFORMERS_VERSION, RuntimeInstaller } from './store/runtime-installer.js';
|
|
27
31
|
import { ensureDir } from './store/io.js';
|
|
28
32
|
import { L0Store } from './store/l0.js';
|
|
29
33
|
import { L1Store } from './store/l1.js';
|
|
@@ -44,25 +48,6 @@ export const inject = ['llm', 'tools', 'systemPrompt'];
|
|
|
44
48
|
* 导致 config 里嵌套对象为 undefined、apply 抛错、fiber FAILED 拖垮宿主启动。
|
|
45
49
|
*/
|
|
46
50
|
export const Config = memorySchema;
|
|
47
|
-
/** 按配置构造 embedding 服务:未启用或配置不完整 → Noop(纯 FTS 运行)。 */
|
|
48
|
-
function createEmbeddingService(cfg, logger) {
|
|
49
|
-
if (!cfg.embedding.enabled)
|
|
50
|
-
return new NoopEmbeddingService();
|
|
51
|
-
const { baseUrl, apiKey, model, dimensions } = cfg.embedding;
|
|
52
|
-
if (!baseUrl || !apiKey || !model || dimensions <= 0) {
|
|
53
|
-
logger.error('[memory] embedding.enabled=true 但 baseUrl/apiKey/model/dimensions 未配置完整,向量检索停用(纯 FTS 运行)');
|
|
54
|
-
return new NoopEmbeddingService();
|
|
55
|
-
}
|
|
56
|
-
return new RemoteEmbeddingService({
|
|
57
|
-
baseUrl,
|
|
58
|
-
apiKey,
|
|
59
|
-
model,
|
|
60
|
-
dimensions,
|
|
61
|
-
maxInputChars: cfg.embedding.maxInputChars,
|
|
62
|
-
timeoutMs: cfg.embedding.timeoutMs,
|
|
63
|
-
logger,
|
|
64
|
-
});
|
|
65
|
-
}
|
|
66
51
|
export async function apply(ctx, config) {
|
|
67
52
|
let logger = {
|
|
68
53
|
debug: (m) => ctx.logger.debug(m),
|
|
@@ -86,6 +71,8 @@ export async function apply(ctx, config) {
|
|
|
86
71
|
}
|
|
87
72
|
// ── 记忆模式运行时开关(官方 settings 服务,live 生效;缺失时恒开) ──
|
|
88
73
|
const live = registerLiveSettings(ctx, logger);
|
|
74
|
+
/** 插件停机标志:置位后不再发起后台 embeddings 调用(dispose 序最先设置)。 */
|
|
75
|
+
let disposed = false;
|
|
89
76
|
// ── 会话档位存储(sessionId → auto/chat/work/off;默认档 = config.family) ──
|
|
90
77
|
const modes = new SessionModeStore(dataDir, config.family, logger);
|
|
91
78
|
if (storageOk) {
|
|
@@ -96,15 +83,34 @@ export async function apply(ctx, config) {
|
|
|
96
83
|
logger.warn(`[memory] 会话档位载入失败(降级为默认档内存态): ${err instanceof Error ? err.message : String(err)}`);
|
|
97
84
|
}
|
|
98
85
|
}
|
|
86
|
+
// ── 嵌入源三态(D4:远程/本地/关闭)——状态文件优先于静态配置 ──
|
|
87
|
+
const sourceStore = new EmbeddingSourceStore(dataDir, logger);
|
|
88
|
+
const installer = new RuntimeInstaller(dataDir, PINNED_TRANSFORMERS_VERSION, { logger });
|
|
89
|
+
const downloader = new ModelDownloadQueue(dataDir, { mirror: config.embedding.mirror, logger });
|
|
90
|
+
const makeLocalService = makeLocalServiceFactory(installer, downloader, logger);
|
|
91
|
+
let initial = { svc: new NoopEmbeddingService(), dims: 0 };
|
|
92
|
+
/** 管理器引用:启动重嵌链/backfill 闭包在运行期解引用(声明早于创建避免 TDZ)。 */
|
|
93
|
+
let embedManagerRef;
|
|
94
|
+
if (storageOk) {
|
|
95
|
+
try {
|
|
96
|
+
await sourceStore.init();
|
|
97
|
+
initial = await resolveInitialEmbedding(config, sourceStore, downloader, makeLocalService, logger);
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
logger.warn(`[memory] 嵌入源初始解析失败(降级纯 FTS): ${errDetail(err)}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/** meta 写入永远用当前目标(杜绝启动期闭包里的陈旧 providerInfo 腐蚀 embedding_meta)。 */
|
|
104
|
+
const currentProviderInfo = () => embedManagerRef?.currentProviderInfo() ?? initial.providerInfo;
|
|
99
105
|
// ── 检索引擎(memory.db)与向量服务 ──
|
|
100
|
-
const embed =
|
|
101
|
-
const db = new MemoryDb(path.join(dataDir, 'memory.db'),
|
|
106
|
+
const embed = initial.svc;
|
|
107
|
+
const db = new MemoryDb(path.join(dataDir, 'memory.db'), initial.dims, logger);
|
|
102
108
|
// 插件卸载时关闭连接(WAL 落盘),注册一次即可
|
|
103
109
|
ctx.effect(() => () => db.close());
|
|
104
110
|
let dbInit = { needsReindex: false };
|
|
105
111
|
if (storageOk) {
|
|
106
112
|
try {
|
|
107
|
-
dbInit = db.init(
|
|
113
|
+
dbInit = db.init(initial.providerInfo);
|
|
108
114
|
if (db.isDegraded()) {
|
|
109
115
|
storageOk = false;
|
|
110
116
|
logger.error('[memory] 检索库初始化失败(schema/FTS 不可用),记忆功能停用');
|
|
@@ -160,21 +166,30 @@ export async function apply(ctx, config) {
|
|
|
160
166
|
logger.warn(`[memory] 蒸馏模型路由解析失败: ${errDetail(err)}`);
|
|
161
167
|
}
|
|
162
168
|
// embedding 配置变化 → 后台全量重嵌入(不阻塞启动)。
|
|
163
|
-
// meta
|
|
164
|
-
if (storageOk &&
|
|
165
|
-
const info =
|
|
169
|
+
// meta 标记条件 = 失败为 0 且缺失复查为 0(服务未就绪的空转不算成功)。
|
|
170
|
+
if (storageOk && initial.providerInfo && !db.isDegraded()) {
|
|
171
|
+
const info = initial.providerInfo;
|
|
166
172
|
if (dbInit.needsReindex) {
|
|
167
173
|
if (caps.vectorSearch) {
|
|
168
174
|
void (async () => {
|
|
169
175
|
try {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
176
|
+
if (disposed || embedManagerRef?.isBusy())
|
|
177
|
+
return;
|
|
178
|
+
// 增量:配置变化路径向量表已被 drop,全部记录均"缺失",等效全量;
|
|
179
|
+
// 活切换发起时让路(swap 会 drop 表,并发跑只是白烧嵌入调用)
|
|
180
|
+
const shouldCancel = () => disposed || !!embedManagerRef?.isBusy();
|
|
181
|
+
const r1 = await stores.l1.reindex({ shouldCancel });
|
|
182
|
+
const r0 = await stores.l0.reindex({ shouldCancel });
|
|
183
|
+
// missing 复查:嵌入服务未就绪时 reindex 短路 0/0/0——那是"没跑"不是"成功",
|
|
184
|
+
// 不得标记 meta(否则启动链不再补,补齐被无限推迟到 backfill)
|
|
185
|
+
const missingAfter = db.countL1VecMissing(db.getVecSkipSet('l1')) + db.countL0VecMissing(db.getVecSkipSet('l0'));
|
|
186
|
+
if (r1.failed === 0 && r0.failed === 0 && missingAfter === 0) {
|
|
187
|
+
db.markEmbeddingSynced(currentProviderInfo() ?? info);
|
|
188
|
+
const skipNote = r1.skipped + r0.skipped > 0 ? `,跳过不可嵌入 ${r1.skipped + r0.skipped} 条` : '';
|
|
189
|
+
logger.info(`[memory] 向量重建完成:L1 ${r1.written} 条,L0 ${r0.written} 条${skipNote}(原因:${dbInit.reason ?? '配置变化'})`);
|
|
175
190
|
}
|
|
176
191
|
else {
|
|
177
|
-
logger.warn(`[memory] 向量重建未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed}
|
|
192
|
+
logger.warn(`[memory] 向量重建未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed},剩缺 ${Math.max(missingAfter, 0)} 条),下次启动/补齐周期重试`);
|
|
178
193
|
}
|
|
179
194
|
}
|
|
180
195
|
catch (err) {
|
|
@@ -185,22 +200,41 @@ export async function apply(ctx, config) {
|
|
|
185
200
|
}
|
|
186
201
|
else {
|
|
187
202
|
// meta 已匹配或空库(initSchema 已写入),幂等重写一次
|
|
188
|
-
db.markEmbeddingSynced(info);
|
|
203
|
+
db.markEmbeddingSynced(currentProviderInfo() ?? info);
|
|
189
204
|
}
|
|
190
|
-
// 周期性向量补齐:嵌入失败的批次/漏网记录自动补上(30
|
|
191
|
-
|
|
205
|
+
// 周期性向量补齐:嵌入失败的批次/漏网记录自动补上(30 分钟一次)。
|
|
206
|
+
// 判据 = 排除 skip 集后的缺失数(增量语义):缺 1 条只补 1 条,
|
|
207
|
+
// 零向量记录进 skip 集后不再触发——否则补齐永不收敛、每 30 分钟全量重嵌。
|
|
208
|
+
// 注册不再依赖启动期 caps/providerInfo(off 起步后活切回旧源也要能补),
|
|
209
|
+
// 执行期动态查当前目标与能力位。
|
|
210
|
+
{
|
|
192
211
|
const backfill = () => {
|
|
193
212
|
void (async () => {
|
|
194
213
|
try {
|
|
195
|
-
|
|
196
|
-
|
|
214
|
+
if (disposed)
|
|
215
|
+
return;
|
|
216
|
+
const infoNow = currentProviderInfo();
|
|
217
|
+
const capsNow = db.getCapabilities();
|
|
218
|
+
if (!infoNow || !capsNow.vectorSearch)
|
|
219
|
+
return;
|
|
220
|
+
// 活切换链(安装/下载/重嵌)进行中让路——增量 reindex 天然幂等,
|
|
221
|
+
// 但并发跑只是白烧一次嵌入调用
|
|
222
|
+
if (embedManagerRef?.isBusy())
|
|
223
|
+
return;
|
|
224
|
+
const l1Missing = db.countL1VecMissing(db.getVecSkipSet('l1')) > 0;
|
|
225
|
+
const l0Missing = db.countL0VecMissing(db.getVecSkipSet('l0')) > 0;
|
|
197
226
|
if (!l1Missing && !l0Missing)
|
|
198
227
|
return;
|
|
199
|
-
const
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
228
|
+
const shouldCancel = () => disposed || !!embedManagerRef?.isBusy();
|
|
229
|
+
const r1 = await stores.l1.reindex({ shouldCancel });
|
|
230
|
+
const r0 = await stores.l0.reindex({ shouldCancel });
|
|
231
|
+
const missingAfter = db.countL1VecMissing(db.getVecSkipSet('l1')) + db.countL0VecMissing(db.getVecSkipSet('l0'));
|
|
232
|
+
if (r1.failed === 0 && r0.failed === 0 && missingAfter === 0) {
|
|
233
|
+
db.markEmbeddingSynced(infoNow);
|
|
234
|
+
logger.info(`[memory] 向量补齐完成:L1 ${r1.written} 条,L0 ${r0.written} 条(跳过不可嵌入 ${r1.skipped + r0.skipped} 条)`);
|
|
235
|
+
}
|
|
236
|
+
else if (r1.failed > 0 || r0.failed > 0) {
|
|
237
|
+
logger.warn(`[memory] 向量补齐未完成(L1 失败 ${r1.failed},L0 失败 ${r0.failed}),下个周期重试`);
|
|
204
238
|
}
|
|
205
239
|
}
|
|
206
240
|
catch (err) {
|
|
@@ -218,10 +252,32 @@ export async function apply(ctx, config) {
|
|
|
218
252
|
});
|
|
219
253
|
}
|
|
220
254
|
}
|
|
255
|
+
// 嵌入管理器(活切换/下载/运行时安装;存储降级时不建——RPC 走 supported=false)
|
|
256
|
+
const embedManager = storageOk && !db.isDegraded()
|
|
257
|
+
? new EmbeddingManager({
|
|
258
|
+
dataDir,
|
|
259
|
+
cfg: config,
|
|
260
|
+
db,
|
|
261
|
+
l0: stores.l0,
|
|
262
|
+
l1: stores.l1,
|
|
263
|
+
sourceStore,
|
|
264
|
+
installer,
|
|
265
|
+
downloader,
|
|
266
|
+
initial,
|
|
267
|
+
logger,
|
|
268
|
+
makeLocal: makeLocalService,
|
|
269
|
+
})
|
|
270
|
+
: undefined;
|
|
271
|
+
embedManagerRef = embedManager;
|
|
221
272
|
const runner = new MemoryRunner(ctx, config, stores, logger, live);
|
|
222
273
|
await runner.init();
|
|
274
|
+
// 重建控制器(存储降级时不建——RPC 端点走 supported=false 分支)
|
|
275
|
+
const rebuild = storageOk && !db.isDegraded()
|
|
276
|
+
? new RebuildController(ctx, config, stores, db, runner, logger, live)
|
|
277
|
+
: undefined;
|
|
278
|
+
let flushL0;
|
|
223
279
|
if (storageOk) {
|
|
224
|
-
registerCapture(ctx, config, runner, stores.l0, logger, live, modes);
|
|
280
|
+
flushL0 = registerCapture(ctx, config, runner, stores.l0, logger, live, modes);
|
|
225
281
|
}
|
|
226
282
|
const recall = registerRecall(ctx, config, stores, logger, live, modes);
|
|
227
283
|
runner.setAfterRun(recall.invalidateProfile);
|
|
@@ -229,6 +285,18 @@ export async function apply(ctx, config) {
|
|
|
229
285
|
registerMemoryRpc(ctx, config, stores, logger, {
|
|
230
286
|
degraded: () => !storageOk || db.isDegraded(),
|
|
231
287
|
pending: () => runner.pendingCount,
|
|
232
|
-
}, live, modes, dataDir);
|
|
288
|
+
}, live, modes, dataDir, rebuild, embedManager);
|
|
233
289
|
logger.info(`[memory] L0~L3 分层蒸馏记忆插件就绪(L1 记忆 ${storageOk ? stores.l1.size : 0} 条 | 捕获=${storageOk && config.capture.enabled} | 蒸馏=${storageOk && config.extract.enabled} | 召回=${config.recall.enabled})`);
|
|
290
|
+
// 停机顺序(M7):置停机标志(后台 embeddings 不再发起)→ 停蒸馏取新任务 →
|
|
291
|
+
// 冲刷 L0 串行链(排队消息先落盘)→ 关库。cordis disposer 为 LIFO 逐个 await,
|
|
292
|
+
// 本 effect 晚注册故先执行;apply 中途失败时由上方早注册的 db.close 兜底(双关安全)。
|
|
293
|
+
ctx.effect(() => () => {
|
|
294
|
+
disposed = true;
|
|
295
|
+
runner.stop();
|
|
296
|
+
embedManager?.dispose();
|
|
297
|
+
return (async () => {
|
|
298
|
+
await flushL0?.();
|
|
299
|
+
db.close();
|
|
300
|
+
})();
|
|
301
|
+
});
|
|
234
302
|
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import { type MemoryConfig } from '../config.js';
|
|
3
|
+
import type { L1Store } from '../store/l1.js';
|
|
4
|
+
import type { PersonaStore } from '../store/persona.js';
|
|
5
|
+
import type { SceneStore } from '../store/scenes.js';
|
|
6
|
+
import type { StateStore } from '../store/state.js';
|
|
7
|
+
import type { MemoryDb } from '../store/sqlite.js';
|
|
8
|
+
import type { LiveSettingsHandle } from '../settings.js';
|
|
9
|
+
import type { ConversationMessage, L0MessageRecord, MemoryFamily, MemoryLogger } from '../types.js';
|
|
10
|
+
import { type MemoryRunner } from './runner.js';
|
|
11
|
+
/** 重建所需的存储子集(l0 不需要——快照直接走 db)。 */
|
|
12
|
+
export interface RebuildStores {
|
|
13
|
+
l1: L1Store;
|
|
14
|
+
scenes: Record<MemoryFamily, SceneStore>;
|
|
15
|
+
persona: Record<MemoryFamily, PersonaStore>;
|
|
16
|
+
state: StateStore;
|
|
17
|
+
}
|
|
18
|
+
export type RebuildPhase = 'idle' | 'preparing' | 'distilling' | 'finalizing' | 'done' | 'cancelled' | 'failed';
|
|
19
|
+
export interface RebuildStatus {
|
|
20
|
+
running: boolean;
|
|
21
|
+
phase: RebuildPhase;
|
|
22
|
+
/** 已完成的会话块数 / 总块数。 */
|
|
23
|
+
done: number;
|
|
24
|
+
total: number;
|
|
25
|
+
/** L0 体量(idle 时为实时预估,运行中为快照值)。 */
|
|
26
|
+
sessionCount: number;
|
|
27
|
+
messageCount: number;
|
|
28
|
+
/** 预计 LLM 抽取调用次数(下界估算:块数与字符预算取大)。 */
|
|
29
|
+
estCalls: number;
|
|
30
|
+
/** 重建产出的 L1 记录累计条数。 */
|
|
31
|
+
recordsBuilt: number;
|
|
32
|
+
cancelRequested: boolean;
|
|
33
|
+
startedAt: number | null;
|
|
34
|
+
finishedAt: number | null;
|
|
35
|
+
error: string | null;
|
|
36
|
+
/** 归档产物名(提示用户可手工找回)。 */
|
|
37
|
+
archiveNote: string | null;
|
|
38
|
+
}
|
|
39
|
+
export interface RebuildChunk {
|
|
40
|
+
sessionId: string;
|
|
41
|
+
messages: ConversationMessage[];
|
|
42
|
+
}
|
|
43
|
+
export declare function groupL0Sessions(records: L0MessageRecord[]): RebuildChunk[];
|
|
44
|
+
/** 抽取调用数下界估算(与 l1.ts 的 perChunk 同式;会话数与字符预算取大)。 */
|
|
45
|
+
export declare function estimateCalls(sessions: number, messages: number, chars: number, maxInputChars: number): number;
|
|
46
|
+
export declare class RebuildController {
|
|
47
|
+
private readonly ctx;
|
|
48
|
+
private readonly cfg;
|
|
49
|
+
private readonly stores;
|
|
50
|
+
private readonly db;
|
|
51
|
+
private readonly runner;
|
|
52
|
+
private readonly logger;
|
|
53
|
+
private readonly live;
|
|
54
|
+
private status;
|
|
55
|
+
private chunks;
|
|
56
|
+
private cancelRequested;
|
|
57
|
+
/** 快照时刻(收尾时按它区分重建产物与重建后新对话的记录)。 */
|
|
58
|
+
private rebuildStartMs;
|
|
59
|
+
constructor(ctx: Context, cfg: MemoryConfig, stores: RebuildStores, db: MemoryDb, runner: Pick<MemoryRunner, 'enqueueRebuildTask' | 'runRebuildTurn' | 'states'>, logger: MemoryLogger, live: LiveSettingsHandle);
|
|
60
|
+
/** 状态快照(idle 时附带实时 L0 预估,供确认弹窗显示成本)。 */
|
|
61
|
+
getStatus(): RebuildStatus;
|
|
62
|
+
/** 内存中尚未处理的会话快照块数(收尾后应为 0——快照即弃,诊断/冒烟用)。 */
|
|
63
|
+
get chunkCount(): number;
|
|
64
|
+
/** 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。 */
|
|
65
|
+
start(): RebuildStatus;
|
|
66
|
+
/** 请求取消:当前块完成后停止,已重建部分保留并照常收尾 L2/L3。 */
|
|
67
|
+
requestCancel(): RebuildStatus;
|
|
68
|
+
private prepare;
|
|
69
|
+
/** 分块链:一次只挂一个重建块,跑完再挂下一块——正常轮次可随时插队。 */
|
|
70
|
+
private scheduleChunk;
|
|
71
|
+
private finalize;
|
|
72
|
+
/**
|
|
73
|
+
* 收集重建窗口内某族的记录:重建产物全部是新插入(updated==created),
|
|
74
|
+
* 按 updated_time 倒序翻页、越过 rebuildStartMs 即停。
|
|
75
|
+
*/
|
|
76
|
+
private collectRebuildRecords;
|
|
77
|
+
private finish;
|
|
78
|
+
/** 归档旧派生层:records/ scenes/ persona-*.md 改名 .bak.<ts>。不存在则跳过。 */
|
|
79
|
+
private archiveDerived;
|
|
80
|
+
}
|