dsh-prime-memory 0.26.9 → 0.26.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.de.md +1638 -0
- package/CHANGELOG.en.md +15 -0
- package/CHANGELOG.es.md +1641 -0
- package/CHANGELOG.fr.md +1643 -0
- package/CHANGELOG.it.md +1643 -0
- package/CHANGELOG.ja.md +15 -0
- package/CHANGELOG.ko.md +15 -0
- package/CHANGELOG.md +62 -0
- package/CHANGELOG.ru.md +1640 -0
- package/README.de.md +1 -1
- package/README.en.md +3 -3
- package/README.es.md +1 -1
- package/README.fr.md +1 -1
- package/README.it.md +1 -1
- package/README.ja.md +1 -1
- package/README.ko.md +1 -1
- package/README.md +17 -4
- package/README.ru.md +1 -1
- package/dist/bench-control.js +3 -1
- package/dist/client.js +121 -12
- package/dist/clients/drsc.d.ts +34 -0
- package/dist/clients/drsc.js +56 -11
- package/dist/config.d.ts +10 -1
- package/dist/config.js +37 -18
- package/dist/contract.d.ts +48 -1
- package/dist/graph/gate.d.ts +17 -0
- package/dist/graph/gate.js +11 -0
- package/dist/graph/preview.d.ts +5 -0
- package/dist/graph/preview.js +2 -0
- package/dist/index.js +1 -1
- package/dist/pipeline/fact-source.d.ts +127 -0
- package/dist/pipeline/fact-source.js +192 -0
- package/dist/pipeline/rebuild-preserve.d.ts +31 -2
- package/dist/pipeline/rebuild-preserve.js +40 -14
- package/dist/pipeline/rebuild.d.ts +47 -13
- package/dist/pipeline/rebuild.js +153 -57
- package/dist/pipeline/runner.d.ts +4 -1
- package/dist/pipeline/runner.js +58 -21
- package/dist/pipeline/workspace-relink.js +7 -2
- package/dist/room-review.d.ts +1 -1
- package/dist/room-review.js +10 -5
- package/dist/settings.js +4 -0
- package/dist/stats.js +10 -3
- package/dist/store/graph-store.d.ts +20 -0
- package/dist/store/graph-store.js +50 -1
- package/dist/store/sqlite.d.ts +42 -0
- package/dist/store/sqlite.js +98 -0
- package/dist/tools/index.js +4 -3
- package/dsh.plugin.json +2 -2
- package/package.json +26 -21
|
@@ -44,6 +44,19 @@ export function isRederivable(record) {
|
|
|
44
44
|
const src = record.source_message_ids;
|
|
45
45
|
return Array.isArray(src) && src.length > 0;
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* 来源是否**全部**落在本次快照内。
|
|
49
|
+
*
|
|
50
|
+
* `every` 而不是 `some`:只要有一张原料缺失,重蒸馏就只会产出一个缺角的新记录,
|
|
51
|
+
* 不如原样保留。无来源时返回 `false`——但那一类由 `no-source` 先行接住,
|
|
52
|
+
* 不应依赖本函数的返回值。
|
|
53
|
+
*/
|
|
54
|
+
export function sourcesCoveredBySnapshot(record, snapshotIds) {
|
|
55
|
+
const src = record.source_message_ids;
|
|
56
|
+
if (!Array.isArray(src) || src.length === 0)
|
|
57
|
+
return false;
|
|
58
|
+
return src.every((id) => snapshotIds.has(id));
|
|
59
|
+
}
|
|
47
60
|
/**
|
|
48
61
|
* 读取事实源目录(`<dataDir>/records/*.jsonl`)。
|
|
49
62
|
*
|
|
@@ -89,16 +102,7 @@ export async function readFactSource(recordsDir) {
|
|
|
89
102
|
}
|
|
90
103
|
return { ok: true, records: [...byId.values()], files: names.length };
|
|
91
104
|
}
|
|
92
|
-
|
|
93
|
-
* 出保留计划。纯函数:输入事实源记录 + 检索库现存记录,输出要保留什么。
|
|
94
|
-
*
|
|
95
|
-
* 两个方向都保守:事实源说"这条没来源"要留,**检索库里查不到出处**也要留
|
|
96
|
-
* (`unprovenanced`)—— 判据缺失时留,而不是丢。
|
|
97
|
-
*
|
|
98
|
-
* `liveDbRecords` 由调用方从**检索库**取(它没有 `source_message_ids` 列,
|
|
99
|
-
* 只用来回答"这条还在不在"),恢复时以检索库副本为准 —— 那是当前检索态。
|
|
100
|
-
*/
|
|
101
|
-
export function planPreserve(factRecords, liveDbRecords) {
|
|
105
|
+
export function planPreserve(factRecords, liveDbRecords, opts = {}) {
|
|
102
106
|
const live = new Map();
|
|
103
107
|
for (const r of liveDbRecords)
|
|
104
108
|
if (r && typeof r.id === 'string')
|
|
@@ -112,6 +116,21 @@ export function planPreserve(factRecords, liveDbRecords) {
|
|
|
112
116
|
continue;
|
|
113
117
|
seen.add(f.id);
|
|
114
118
|
if (isRederivable(f)) {
|
|
119
|
+
// 顺序纪律:「外部可重蒸」必须在「post-watermark 保留」**之后**判。
|
|
120
|
+
// 停写后新增的记录 `source_message_ids` 非空,但指向的消息在本地
|
|
121
|
+
// `l0_conversations` 已查不到 —— 清库后蒸不出来,等于丢失;
|
|
122
|
+
// 而按旧口径(只看非空)它们恰好会被归进 `rederivable` 然后被清掉。
|
|
123
|
+
if (opts.snapshotIds && !sourcesCoveredBySnapshot(f, opts.snapshotIds)) {
|
|
124
|
+
const current = live.get(f.id);
|
|
125
|
+
// 已退场的不复活(与 no-source 分支同口径)
|
|
126
|
+
if (current) {
|
|
127
|
+
preserved.push({
|
|
128
|
+
record: current,
|
|
129
|
+
reason: String(current.content ?? '').trim() ? 'post-watermark' : 'empty-content',
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
115
134
|
// 有来源但已退场(合并掉了)的不必计数——重建会按 L0 再造。
|
|
116
135
|
if (live.has(f.id))
|
|
117
136
|
rederivable++;
|
|
@@ -175,8 +194,8 @@ export function gateClear(read, plan) {
|
|
|
175
194
|
allowed: true,
|
|
176
195
|
code: 'ok',
|
|
177
196
|
note: keep > 0
|
|
178
|
-
? `保留 ${keep}
|
|
179
|
-
:
|
|
197
|
+
? `保留 ${keep} 条不可重蒸的记忆(导入/手工写入/来源不在本次快照),重建只重造 ${plan.rederivable} 条有来源的记忆`
|
|
198
|
+
: `无不可重蒸记忆需要保留,重建重造 ${plan.rederivable} 条有来源的记忆`,
|
|
180
199
|
};
|
|
181
200
|
}
|
|
182
201
|
/**
|
|
@@ -195,12 +214,19 @@ export async function restorePreserved(l1, plan) {
|
|
|
195
214
|
const missing = ids.filter((id) => !back.has(id));
|
|
196
215
|
return { attempted: ids.length, restored: ids.length - missing.length, missing };
|
|
197
216
|
}
|
|
217
|
+
/** reason → 中文标签(日志/状态栏直接给人看,不再打印英文枚举)。 */
|
|
218
|
+
const REASON_LABEL = {
|
|
219
|
+
'no-source': '无来源',
|
|
220
|
+
'empty-content': '内容为空',
|
|
221
|
+
unprovenanced: '来源无从证明',
|
|
222
|
+
'post-watermark': '来源不在本次快照',
|
|
223
|
+
};
|
|
198
224
|
/** 面向日志/状态栏的一行摘要。 */
|
|
199
225
|
export function describePlan(plan) {
|
|
200
226
|
const byReason = new Map();
|
|
201
227
|
for (const p of plan.preserved)
|
|
202
228
|
byReason.set(p.reason, (byReason.get(p.reason) ?? 0) + 1);
|
|
203
|
-
const parts = [...byReason].map(([r, n]) => `${r} ${n}`);
|
|
204
|
-
return (`保留 ${plan.preserved.length}
|
|
229
|
+
const parts = [...byReason].map(([r, n]) => `${REASON_LABEL[r] ?? r} ${n}`);
|
|
230
|
+
return (`保留 ${plan.preserved.length} 条不可重蒸记忆${parts.length ? `(${parts.join(' / ')})` : ''};` +
|
|
205
231
|
`可重蒸馏 ${plan.rederivable} 条;事实源 ${plan.factCount} 条(其中无来源但已退场 ${plan.retiredNoSource} 条,不恢复)`);
|
|
206
232
|
}
|
|
@@ -6,24 +6,36 @@ import type { SceneStore } from '../store/scenes.js';
|
|
|
6
6
|
import type { StateStore } from '../store/state.js';
|
|
7
7
|
import type { MemoryDb } from '../store/sqlite.js';
|
|
8
8
|
import type { LiveSettingsHandle } from '../settings.js';
|
|
9
|
-
import type {
|
|
9
|
+
import type { MemoryFamily, MemoryLogger } from '../types.js';
|
|
10
10
|
import { type MemoryRunner } from './runner.js';
|
|
11
|
+
import type { DrscClient } from '../clients/drsc.js';
|
|
12
|
+
export type { FactCoverage, FactEstimate, FactMessage, FactSourceProvider, RebuildChunk } from './fact-source.js';
|
|
13
|
+
export { DrscFactSource, groupL0Sessions, LocalL0FactSource } from './fact-source.js';
|
|
11
14
|
/** 重建所需的存储子集(l0 不需要——快照直接走 db)。 */
|
|
12
15
|
export interface RebuildStores {
|
|
13
16
|
l1: L1Store;
|
|
14
17
|
scenes: Record<MemoryFamily, SceneStore>;
|
|
15
18
|
persona: Record<MemoryFamily, PersonaStore>;
|
|
16
19
|
state: StateStore;
|
|
20
|
+
/**
|
|
21
|
+
* DRSC 客户端(issue #14-B):仅当 `rebuild.factSource === 'drsc'` 时用到。
|
|
22
|
+
* 缺席时选 drsc 会**显式中止**,不静默回退本地表。
|
|
23
|
+
*/
|
|
24
|
+
drsc?: DrscClient;
|
|
17
25
|
}
|
|
18
26
|
import type { RebuildStatus } from '../contract.js';
|
|
19
27
|
export type { RebuildPhase, RebuildStatus } from '../contract.js';
|
|
20
|
-
export interface RebuildChunk {
|
|
21
|
-
sessionId: string;
|
|
22
|
-
messages: ConversationMessage[];
|
|
23
|
-
}
|
|
24
|
-
export declare function groupL0Sessions(records: L0MessageRecord[]): RebuildChunk[];
|
|
25
28
|
/** 抽取调用数下界估算(与 l1.ts 的 perChunk 同式;会话数与字符预算取大)。 */
|
|
26
29
|
export declare function estimateCalls(sessions: number, messages: number, chars: number, maxInputChars: number): number;
|
|
30
|
+
/**
|
|
31
|
+
* 水位落差**提示**阀值:检索库里晚于 L0 水位的占比超过此值,就要求二次确认。
|
|
32
|
+
*
|
|
33
|
+
* 为什么叫“提示阀值”而不是“拒绝阀值”:L0 写停后这个比例只会单调上升,
|
|
34
|
+
* 拿它做硬门槛等于把重建**永久锁死**(issue #14 原提案的致命缺陷)。
|
|
35
|
+
* 所以它只负责“提醒你正在做什么”,放行靠 `force`;真正的**不丢**由
|
|
36
|
+
* `planPreserve` 的集合判据(`post-watermark`)结构性保证。
|
|
37
|
+
*/
|
|
38
|
+
export declare const STALE_RATIO_THRESHOLD = 0.5;
|
|
27
39
|
export declare class RebuildController {
|
|
28
40
|
private readonly ctx;
|
|
29
41
|
private readonly cfg;
|
|
@@ -33,22 +45,44 @@ export declare class RebuildController {
|
|
|
33
45
|
private readonly logger;
|
|
34
46
|
private readonly live;
|
|
35
47
|
private status;
|
|
36
|
-
|
|
48
|
+
/**
|
|
49
|
+
* 会话级事实源迭代器(issue #14-B):不再把全量 L0 正文驻留内存。
|
|
50
|
+
* null = 尚未进入蒸馏 / 已收尾。
|
|
51
|
+
*/
|
|
52
|
+
private chunkIter;
|
|
37
53
|
private cancelRequested;
|
|
38
54
|
/** 快照时刻(收尾时按它区分重建产物与重建后新对话的记录)。 */
|
|
39
55
|
private rebuildStartMs;
|
|
40
56
|
constructor(ctx: Context, cfg: MemoryConfig, stores: RebuildStores, db: MemoryDb, runner: Pick<MemoryRunner, 'enqueueRebuildTask' | 'runRebuildTurn' | 'states'>, logger: MemoryLogger, live: LiveSettingsHandle);
|
|
41
|
-
/** 状态快照(idle
|
|
57
|
+
/** 状态快照(idle 时附带实时预估与事实源水位,供确认弹窗讲清"会发生什么")。 */
|
|
42
58
|
getStatus(): RebuildStatus;
|
|
43
|
-
/**
|
|
59
|
+
/** 尚未处理的会话块数(流式下为估算 `total - done`;未运行时为 0,诊断/冒烟用)。 */
|
|
44
60
|
get chunkCount(): number;
|
|
45
|
-
/**
|
|
46
|
-
|
|
61
|
+
/**
|
|
62
|
+
* 按 `cfg.rebuild.factSource` 选事实源(issue #14-B)。
|
|
63
|
+
*
|
|
64
|
+
* 选 drsc 但客户端未装配时**显式抛错**,不静默回退本地表 —— “以为用了新源其实还是旧表”
|
|
65
|
+
* 会让用户拿重建结果去判断 DRSC 数据是否完整,那是最坏的误判。
|
|
66
|
+
*/
|
|
67
|
+
private provider;
|
|
68
|
+
/**
|
|
69
|
+
* 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。
|
|
70
|
+
*
|
|
71
|
+
* `force` 只用于跨过**提示门槛**(水位落差超过 `STALE_RATIO_THRESHOLD` 时的二次确认),
|
|
72
|
+
* 不能跨过唯一那条硬拒:「L0 一条消息都没有、检索库却有记录」—— 那种形状下重建
|
|
73
|
+
* 只会清空、一条也重造不出来,属于纯亏。
|
|
74
|
+
*/
|
|
75
|
+
start(force?: boolean): RebuildStatus;
|
|
47
76
|
/** 请求取消:当前块完成后停止,已重建部分保留并照常收尾 L2/L3。 */
|
|
48
77
|
requestCancel(): RebuildStatus;
|
|
49
78
|
private prepare;
|
|
50
|
-
/**
|
|
51
|
-
|
|
79
|
+
/**
|
|
80
|
+
* 分块链:一次只挂一个重建块,跑完再挂下一块——正常轮次可随时插队。
|
|
81
|
+
*
|
|
82
|
+
* 流式版(issue #14-B):从事实源迭代器拉下一个会话,不再按下标从内存数组取。
|
|
83
|
+
* 内存峰值从"全量正文"降到"单会话"。
|
|
84
|
+
*/
|
|
85
|
+
private scheduleNext;
|
|
52
86
|
private finalize;
|
|
53
87
|
/**
|
|
54
88
|
* 收集重建窗口内某族的记录:重建产物全部是新插入(updated==created),
|
package/dist/pipeline/rebuild.js
CHANGED
|
@@ -26,29 +26,9 @@ import { runSceneConsolidation } from './l2.js';
|
|
|
26
26
|
import { runPersona } from './l3.js';
|
|
27
27
|
import { describePlan, gateClear, planPreserve, readFactSource, restorePreserved } from './rebuild-preserve.js';
|
|
28
28
|
import { effectiveCfg } from './runner.js';
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
if (!r || typeof r.id !== 'string' || typeof r.content !== 'string')
|
|
33
|
-
continue;
|
|
34
|
-
if (r.role !== 'user' && r.role !== 'assistant')
|
|
35
|
-
continue;
|
|
36
|
-
if (!r.content.trim())
|
|
37
|
-
continue;
|
|
38
|
-
const key = r.sessionId || 'default';
|
|
39
|
-
const arr = bySession.get(key) ?? [];
|
|
40
|
-
arr.push({ id: r.id, role: r.role, content: r.content, timestamp: r.timestamp ?? 0 });
|
|
41
|
-
bySession.set(key, arr);
|
|
42
|
-
}
|
|
43
|
-
const chunks = [];
|
|
44
|
-
for (const [sessionId, messages] of bySession) {
|
|
45
|
-
messages.sort((a, b) => a.timestamp - b.timestamp);
|
|
46
|
-
chunks.push({ sessionId, messages });
|
|
47
|
-
}
|
|
48
|
-
// 会话按首条消息时间升序:情境链按时间顺序衔接(与原始发生顺序一致)
|
|
49
|
-
chunks.sort((a, b) => a.messages[0].timestamp - b.messages[0].timestamp);
|
|
50
|
-
return chunks;
|
|
51
|
-
}
|
|
29
|
+
import { DrscFactSource, LocalL0FactSource, } from './fact-source.js';
|
|
30
|
+
export { DrscFactSource, groupL0Sessions, LocalL0FactSource } from './fact-source.js';
|
|
31
|
+
// `RebuildChunk` 与 `groupL0Sessions` 已迁到 pipeline/fact-source.ts(上面已转出)。
|
|
52
32
|
/** 抽取调用数下界估算(与 l1.ts 的 perChunk 同式;会话数与字符预算取大)。 */
|
|
53
33
|
export function estimateCalls(sessions, messages, chars, maxInputChars) {
|
|
54
34
|
if (messages === 0)
|
|
@@ -56,6 +36,15 @@ export function estimateCalls(sessions, messages, chars, maxInputChars) {
|
|
|
56
36
|
const perChunk = Math.max(20_000, maxInputChars - 42_000);
|
|
57
37
|
return Math.max(sessions, Math.ceil((chars + 64 * messages) / perChunk));
|
|
58
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* 水位落差**提示**阀值:检索库里晚于 L0 水位的占比超过此值,就要求二次确认。
|
|
41
|
+
*
|
|
42
|
+
* 为什么叫“提示阀值”而不是“拒绝阀值”:L0 写停后这个比例只会单调上升,
|
|
43
|
+
* 拿它做硬门槛等于把重建**永久锁死**(issue #14 原提案的致命缺陷)。
|
|
44
|
+
* 所以它只负责“提醒你正在做什么”,放行靠 `force`;真正的**不丢**由
|
|
45
|
+
* `planPreserve` 的集合判据(`post-watermark`)结构性保证。
|
|
46
|
+
*/
|
|
47
|
+
export const STALE_RATIO_THRESHOLD = 0.5;
|
|
59
48
|
function idleStatus() {
|
|
60
49
|
return {
|
|
61
50
|
running: false,
|
|
@@ -72,6 +61,9 @@ function idleStatus() {
|
|
|
72
61
|
error: null,
|
|
73
62
|
archiveNote: null,
|
|
74
63
|
preserveNote: null,
|
|
64
|
+
rederivableCount: null,
|
|
65
|
+
preserveCount: null,
|
|
66
|
+
factWatermark: null,
|
|
75
67
|
};
|
|
76
68
|
}
|
|
77
69
|
/** 时间戳后缀(归档命名,秒级防撞)。 */
|
|
@@ -88,7 +80,11 @@ export class RebuildController {
|
|
|
88
80
|
logger;
|
|
89
81
|
live;
|
|
90
82
|
status = idleStatus();
|
|
91
|
-
|
|
83
|
+
/**
|
|
84
|
+
* 会话级事实源迭代器(issue #14-B):不再把全量 L0 正文驻留内存。
|
|
85
|
+
* null = 尚未进入蒸馏 / 已收尾。
|
|
86
|
+
*/
|
|
87
|
+
chunkIter = null;
|
|
92
88
|
cancelRequested = false;
|
|
93
89
|
/** 快照时刻(收尾时按它区分重建产物与重建后新对话的记录)。 */
|
|
94
90
|
rebuildStartMs = 0;
|
|
@@ -101,32 +97,88 @@ export class RebuildController {
|
|
|
101
97
|
this.logger = logger;
|
|
102
98
|
this.live = live;
|
|
103
99
|
}
|
|
104
|
-
/** 状态快照(idle
|
|
100
|
+
/** 状态快照(idle 时附带实时预估与事实源水位,供确认弹窗讲清"会发生什么")。 */
|
|
105
101
|
getStatus() {
|
|
106
102
|
if (this.status.phase === 'idle') {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
103
|
+
try {
|
|
104
|
+
const provider = this.provider();
|
|
105
|
+
const est = provider.estimate();
|
|
106
|
+
const cov = provider.coverage();
|
|
107
|
+
return {
|
|
108
|
+
...this.status,
|
|
109
|
+
sessionCount: est.sessions,
|
|
110
|
+
messageCount: est.messages,
|
|
111
|
+
estCalls: estimateCalls(est.sessions, est.messages, est.chars, effectiveCfg(this.cfg, this.live).llm.maxInputChars),
|
|
112
|
+
factWatermark: cov.untilIso,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
// 事实源未就绪(如选了 drsc 但 collector 未装配):面板不能因此整块挂,
|
|
117
|
+
// 只把水位置空;真正的错误留给 start() 显式报。
|
|
118
|
+
return { ...this.status, factWatermark: null };
|
|
119
|
+
}
|
|
114
120
|
}
|
|
115
121
|
return { ...this.status };
|
|
116
122
|
}
|
|
117
|
-
/**
|
|
123
|
+
/** 尚未处理的会话块数(流式下为估算 `total - done`;未运行时为 0,诊断/冒烟用)。 */
|
|
118
124
|
get chunkCount() {
|
|
119
|
-
return this.
|
|
125
|
+
return this.status.running ? Math.max(0, this.status.total - this.status.done) : 0;
|
|
120
126
|
}
|
|
121
|
-
/**
|
|
122
|
-
|
|
127
|
+
/**
|
|
128
|
+
* 按 `cfg.rebuild.factSource` 选事实源(issue #14-B)。
|
|
129
|
+
*
|
|
130
|
+
* 选 drsc 但客户端未装配时**显式抛错**,不静默回退本地表 —— “以为用了新源其实还是旧表”
|
|
131
|
+
* 会让用户拿重建结果去判断 DRSC 数据是否完整,那是最坏的误判。
|
|
132
|
+
*/
|
|
133
|
+
provider() {
|
|
134
|
+
if (this.cfg.rebuild?.factSource === 'drsc') {
|
|
135
|
+
const drsc = this.stores.drsc;
|
|
136
|
+
if (!drsc) {
|
|
137
|
+
throw new Error('重建事实源配置为 drsc,但 collector 客户端未装配 —— 已中止且**未动任何数据**。'
|
|
138
|
+
+ '请先启用 reference-session-collector,或把 rebuild.factSource 改回 local。');
|
|
139
|
+
}
|
|
140
|
+
return new DrscFactSource(drsc, this.logger);
|
|
141
|
+
}
|
|
142
|
+
return new LocalL0FactSource(this.db, this.logger);
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。
|
|
146
|
+
*
|
|
147
|
+
* `force` 只用于跨过**提示门槛**(水位落差超过 `STALE_RATIO_THRESHOLD` 时的二次确认),
|
|
148
|
+
* 不能跨过唯一那条硬拒:「L0 一条消息都没有、检索库却有记录」—— 那种形状下重建
|
|
149
|
+
* 只会清空、一条也重造不出来,属于纯亏。
|
|
150
|
+
*/
|
|
151
|
+
start(force = false) {
|
|
123
152
|
if (this.status.running)
|
|
124
153
|
throw new Error('重建已在进行中');
|
|
125
|
-
|
|
126
|
-
|
|
154
|
+
// drsc 但客户端未装配 → provider() 在这里显式抛(未动任何数据)
|
|
155
|
+
const provider = this.provider();
|
|
156
|
+
const est = provider.estimate();
|
|
157
|
+
const wm = this.db.rebuildWatermark();
|
|
158
|
+
// 同步门只对“能同步报数”的事实源成立(本地 L0)。DRSC 要发请求才知道会话数,
|
|
159
|
+
// known=false 时必须跳过 —— 否则会把“未知”当成“空”直接拒掉。
|
|
160
|
+
if (est.known && est.messages === 0) {
|
|
161
|
+
if (wm.l1Total > 0) {
|
|
162
|
+
throw new Error(`本地 L0 一条消息都没有,而检索库有 ${wm.l1Total} 条记忆 ——`
|
|
163
|
+
+ 'L0 正文自 T8.4 起已停写,用它重建只会清空检索库、一条也重造不出来。'
|
|
164
|
+
+ '已中止重建且**未动任何数据**。若确实需要以其他事实源重建,请先把该事实源接上后重试。');
|
|
165
|
+
}
|
|
127
166
|
throw new Error('L0 无任何消息,无需重建');
|
|
167
|
+
}
|
|
168
|
+
// 水位落差**只做提示**:L0 停写后这个比例会单调上升,硬拒等于永久锁死重建。
|
|
169
|
+
// 真正的“不丢”由 planPreserve 的集合判据结构性保证,这里只负责让用户看清楚。
|
|
170
|
+
if (est.known && !force && wm.l1Total > 0) {
|
|
171
|
+
const ratio = wm.l1After / wm.l1Total;
|
|
172
|
+
if (ratio > STALE_RATIO_THRESHOLD) {
|
|
173
|
+
const pct = Math.round(ratio * 100);
|
|
174
|
+
throw new Error(`检索库有 ${wm.l1Total} 条记忆,其中 ${wm.l1After} 条(${pct}%)晚于本地 L0 水位`
|
|
175
|
+
+ `${wm.l0Iso ? ` ${wm.l0Iso}` : '(L0 为空)'} —— 这些记忆的原材料已不在本次快照里,`
|
|
176
|
+
+ '重建会把它们**原样保留但不再重蒸馏**(不会丢,只是不重造)。'
|
|
177
|
+
+ '确认要继续请带 force 重试。');
|
|
178
|
+
}
|
|
179
|
+
}
|
|
128
180
|
this.cancelRequested = false;
|
|
129
|
-
this.
|
|
181
|
+
this.chunkIter = null;
|
|
130
182
|
this.status = {
|
|
131
183
|
...idleStatus(),
|
|
132
184
|
running: true,
|
|
@@ -135,9 +187,12 @@ export class RebuildController {
|
|
|
135
187
|
messageCount: est.messages,
|
|
136
188
|
estCalls: estimateCalls(est.sessions, est.messages, est.chars, effectiveCfg(this.cfg, this.live).llm.maxInputChars),
|
|
137
189
|
startedAt: Date.now(),
|
|
190
|
+
factWatermark: provider.coverage().untilIso,
|
|
138
191
|
};
|
|
139
192
|
this.runner.enqueueRebuildTask(() => this.prepare());
|
|
140
|
-
this.logger.info(`[memory] 重建开始:${est.sessions} 个会话 / ${est.messages} 条 L0
|
|
193
|
+
this.logger.info(`[memory] 重建开始:${est.sessions} 个会话 / ${est.messages} 条 L0 消息`
|
|
194
|
+
+ `(预计 ≥${this.status.estCalls} 次抽取调用;事实源水位 ${wm.l0Iso ?? '空'};`
|
|
195
|
+
+ `检索库 ${wm.l1Total} 条,其中晚于水位 ${wm.l1After} 条)`);
|
|
141
196
|
return { ...this.status };
|
|
142
197
|
}
|
|
143
198
|
/** 请求取消:当前块完成后停止,已重建部分保留并照常收尾 L2/L3。 */
|
|
@@ -151,18 +206,28 @@ export class RebuildController {
|
|
|
151
206
|
}
|
|
152
207
|
async prepare() {
|
|
153
208
|
try {
|
|
154
|
-
//
|
|
209
|
+
// 事实源(issue #14-B):默认本地 L0;drsc 需显式配置且必须有客户端。
|
|
210
|
+
const provider = this.provider();
|
|
211
|
+
// 快照时刻:重建期间新捕获的消息走正常轮次,天然不重不漏
|
|
155
212
|
this.rebuildStartMs = Date.now();
|
|
156
|
-
|
|
157
|
-
if (
|
|
158
|
-
this.finish('failed',
|
|
213
|
+
const est = await provider.prepare();
|
|
214
|
+
if (est.sessions === 0) {
|
|
215
|
+
this.finish('failed', `事实源(${provider.kind})为空,无可重建内容`);
|
|
159
216
|
return;
|
|
160
217
|
}
|
|
161
|
-
this.status.total =
|
|
218
|
+
this.status.total = est.sessions;
|
|
219
|
+
if (est.messages > 0)
|
|
220
|
+
this.status.messageCount = est.messages;
|
|
221
|
+
// 本次事实源的**消息 id 集**:判"来源是否还在快照里"的唯一依据。
|
|
222
|
+
// 不取时间:时间差对停写的 L0 会永久走扩(详见 rebuild-preserve 的 post-watermark)。
|
|
223
|
+
// 注意这里**不读正文**:2.5 万条 id 约 1.5MB,而全量正文是 88MB。
|
|
224
|
+
const snapshotIds = await provider.snapshotIds();
|
|
162
225
|
const dataDir = resolveDataDir(this.cfg);
|
|
163
226
|
// ── ① 读事实源 + 判定保留集(必须在 archiveDerived 之前:归档会把 records/ 搬走)──
|
|
164
227
|
const factRead = await readFactSource(path.join(dataDir, 'records'));
|
|
165
|
-
const plan = planPreserve(factRead.records, this.stores.l1.all());
|
|
228
|
+
const plan = planPreserve(factRead.records, this.stores.l1.all(), { snapshotIds });
|
|
229
|
+
this.status.rederivableCount = plan.rederivable;
|
|
230
|
+
this.status.preserveCount = plan.preserved.length;
|
|
166
231
|
// ── ② 过闸门:判不出该保留什么时**拒绝清空**(见 rebuild-preserve.ts 文件头)──
|
|
167
232
|
const gate = gateClear(factRead, plan);
|
|
168
233
|
if (!gate.allowed) {
|
|
@@ -197,26 +262,53 @@ export class RebuildController {
|
|
|
197
262
|
this.stores.persona.work.init(),
|
|
198
263
|
]);
|
|
199
264
|
this.status.phase = 'distilling';
|
|
200
|
-
this.logger.info(`[memory] 重建准备完成(归档:${archiveNote ?? '无旧产物'},保留:${this.status.preserveNote},${
|
|
201
|
-
this.
|
|
265
|
+
this.logger.info(`[memory] 重建准备完成(归档:${archiveNote ?? '无旧产物'},保留:${this.status.preserveNote},${est.sessions} 个会话块,事实源=${provider.kind})`);
|
|
266
|
+
this.chunkIter = provider.chunks()[Symbol.asyncIterator]();
|
|
267
|
+
this.scheduleNext();
|
|
202
268
|
}
|
|
203
269
|
catch (err) {
|
|
204
270
|
this.finish('failed', `准备阶段失败: ${errDetail(err)}`);
|
|
205
271
|
}
|
|
206
272
|
}
|
|
207
|
-
/**
|
|
208
|
-
|
|
209
|
-
|
|
273
|
+
/**
|
|
274
|
+
* 分块链:一次只挂一个重建块,跑完再挂下一块——正常轮次可随时插队。
|
|
275
|
+
*
|
|
276
|
+
* 流式版(issue #14-B):从事实源迭代器拉下一个会话,不再按下标从内存数组取。
|
|
277
|
+
* 内存峰值从"全量正文"降到"单会话"。
|
|
278
|
+
*/
|
|
279
|
+
scheduleNext() {
|
|
280
|
+
if (this.cancelRequested) {
|
|
281
|
+
this.runner.enqueueRebuildTask(() => this.finalize());
|
|
282
|
+
return;
|
|
283
|
+
}
|
|
284
|
+
const iter = this.chunkIter;
|
|
285
|
+
if (!iter) {
|
|
210
286
|
this.runner.enqueueRebuildTask(() => this.finalize());
|
|
211
287
|
return;
|
|
212
288
|
}
|
|
213
|
-
const chunk = this.chunks[i];
|
|
214
289
|
this.runner.enqueueRebuildTask(async () => {
|
|
215
290
|
// 入队后开跑前可能已收到取消(等待插队的正常轮次期间),直接跳到收尾
|
|
216
291
|
if (this.cancelRequested) {
|
|
217
292
|
this.runner.enqueueRebuildTask(() => this.finalize());
|
|
218
293
|
return;
|
|
219
294
|
}
|
|
295
|
+
let next;
|
|
296
|
+
try {
|
|
297
|
+
next = await iter.next();
|
|
298
|
+
}
|
|
299
|
+
catch (err) {
|
|
300
|
+
// 事实源中途失败**不能**当“跑完了”:那会把“只重建了一部分”报成成功,
|
|
301
|
+
// 而用户会拿这个结论去判断数据是否重新生成。
|
|
302
|
+
this.logger.error(`[memory] 重建事实源中断(已完成 ${this.status.done}/${this.status.total} 个会话): ${errDetail(err)}`);
|
|
303
|
+
this.finish('failed', `事实源读取中断: ${errDetail(err)}`);
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
if (next.done === true) {
|
|
307
|
+
this.chunkIter = null;
|
|
308
|
+
this.runner.enqueueRebuildTask(() => this.finalize());
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
const chunk = next.value;
|
|
220
312
|
try {
|
|
221
313
|
const n = await this.runner.runRebuildTurn(chunk.sessionId, chunk.messages);
|
|
222
314
|
this.status.recordsBuilt += n;
|
|
@@ -224,8 +316,8 @@ export class RebuildController {
|
|
|
224
316
|
catch (err) {
|
|
225
317
|
this.logger.warn(`[memory] 重建块失败(session=${chunk.sessionId},跳过继续): ${errDetail(err)}`);
|
|
226
318
|
}
|
|
227
|
-
this.status.done
|
|
228
|
-
this.
|
|
319
|
+
this.status.done += 1;
|
|
320
|
+
this.scheduleNext();
|
|
229
321
|
});
|
|
230
322
|
}
|
|
231
323
|
async finalize() {
|
|
@@ -307,9 +399,13 @@ export class RebuildController {
|
|
|
307
399
|
this.status.phase = phase;
|
|
308
400
|
this.status.error = error;
|
|
309
401
|
this.status.finishedAt = Date.now();
|
|
310
|
-
//
|
|
311
|
-
//
|
|
312
|
-
|
|
402
|
+
// 事实源释放:迭代器持有单会话内存,收尾必须断开(不能滞留到下一次 start 覆盖)。
|
|
403
|
+
// 若它实现了 `return`(异步生成器),顺手关掉,避免上游资源悬着。
|
|
404
|
+
const iter = this.chunkIter;
|
|
405
|
+
this.chunkIter = null;
|
|
406
|
+
if (iter !== null && typeof iter.return === 'function') {
|
|
407
|
+
void iter.return().catch(() => { });
|
|
408
|
+
}
|
|
313
409
|
const cost = this.status.finishedAt - (this.status.startedAt ?? this.status.finishedAt);
|
|
314
410
|
this.logger.info(`[memory] 重建结束(${phase}):${this.status.done}/${this.status.total} 会话,产出 ${this.status.recordsBuilt} 条记录,耗时 ${Math.round(cost / 1000)}s` +
|
|
315
411
|
(error ? `,错误:${error}` : ''));
|
|
@@ -16,6 +16,7 @@ import { type MemoryConfig } from '../config.js';
|
|
|
16
16
|
import type { LiveSettingsHandle } from '../settings.js';
|
|
17
17
|
import type { DrscClient } from '../clients/drsc.js';
|
|
18
18
|
import type { GraphStore } from '../store/graph-store.js';
|
|
19
|
+
import { graphFeatureOn } from '../graph/gate.js';
|
|
19
20
|
import type { L1Store } from '../store/l1.js';
|
|
20
21
|
import type { PersonaStore } from '../store/persona.js';
|
|
21
22
|
import type { SceneStore } from '../store/scenes.js';
|
|
@@ -73,6 +74,7 @@ export declare function pickNextTaskIndex(tasks: PipelineTask[]): number;
|
|
|
73
74
|
* primaryEffort;非链模式旧键对静态回退条目盖章接管(存量兼容)。
|
|
74
75
|
* - 输出预算/输入预算/通道覆盖/远程嵌入覆盖:非零/非空运行时值直接注入对应子树。
|
|
75
76
|
*/
|
|
77
|
+
export { graphFeatureOn };
|
|
76
78
|
export declare function effectiveCfg(cfg: MemoryConfig, live?: LiveSettingsHandle): MemoryConfig;
|
|
77
79
|
export declare class MemoryRunner {
|
|
78
80
|
private readonly ctx;
|
|
@@ -192,7 +194,8 @@ export declare class MemoryRunner {
|
|
|
192
194
|
};
|
|
193
195
|
/**
|
|
194
196
|
* 图谱投影泵入队判定:
|
|
195
|
-
* -
|
|
197
|
+
* - 双门:`graphFeatureOn`(部署 cfg.graph.enabled **或** 面板 live.graphEnabled)
|
|
198
|
+
* + 运行时 live.enabled/distill,任一为假不入队;
|
|
196
199
|
* - 占位:同一时刻至多一个 graph 任务在队列(靠 graphPumpQueued 标志),永不与
|
|
197
200
|
* live 抢位(pickNextTaskIndex 保证 live > graph);
|
|
198
201
|
* - 退避/attempts 封顶:由 GraphStore.claimNext 的 WHERE 过滤与 fail 转 dead 承接,
|