dsh-prime-memory 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.en.md +28 -0
- package/CHANGELOG.ja.md +30 -0
- package/CHANGELOG.ko.md +30 -0
- package/CHANGELOG.md +1220 -0
- package/ENGINEERING-NOTES.md +452 -0
- package/INSTALL.en.md +92 -0
- package/INSTALL.ja.md +92 -0
- package/INSTALL.ko.md +92 -0
- package/INSTALL.md +92 -0
- package/LICENSE +21 -0
- package/README.en.md +458 -0
- package/README.ja.md +306 -0
- package/README.ko.md +306 -0
- package/README.md +425 -0
- package/assets/changelog/0.8.10/01-write-only-pill.png +0 -0
- package/assets/changelog/0.8.9/01-panel.png +0 -0
- package/assets/changelog/0.8.9/02-halo.png +0 -0
- package/assets/changelog/0.8.9/03-layer-segmented-panel.png +0 -0
- package/assets/changelog/0.8.9/04-layer-l1-panel.png +0 -0
- package/assets/img/EmbeddingSource.png +0 -0
- package/assets/img/Hero.png +0 -0
- package/assets/img/Layers.png +0 -0
- package/assets/img/MemoryTools.png +0 -0
- package/assets/img/Modes.png +0 -0
- package/assets/img/ToolTrajectory.png +0 -0
- package/assets/img/ui-dark.jpg +0 -0
- package/assets/img/ui-light.jpg +0 -0
- package/assets/readme/bench-dialog.svg +70 -0
- package/assets/readme/bench-workflow.svg +79 -0
- package/assets/readme/flow.svg +189 -0
- package/assets/readme/storage.svg +115 -0
- package/cordis.patch.yml +16 -0
- package/dist/bench-control.d.ts +34 -0
- package/dist/bench-control.js +16 -0
- package/dist/client.js +4293 -0
- package/dist/config.d.ts +683 -0
- package/dist/config.js +129 -0
- package/dist/contract.d.ts +820 -0
- package/dist/contract.js +1 -0
- package/dist/embedding-worker.cjs +176 -0
- package/dist/graph/apply.d.ts +37 -0
- package/dist/graph/apply.js +270 -0
- package/dist/graph/constraints.d.ts +47 -0
- package/dist/graph/constraints.js +38 -0
- package/dist/graph/search.d.ts +16 -0
- package/dist/graph/search.js +115 -0
- package/dist/graph/types.d.ts +142 -0
- package/dist/graph/types.js +14 -0
- package/dist/hooks/capture.d.ts +32 -0
- package/dist/hooks/capture.js +194 -0
- package/dist/hooks/recall.d.ts +63 -0
- package/dist/hooks/recall.js +429 -0
- package/dist/index.d.ts +534 -0
- package/dist/index.js +344 -0
- package/dist/llm-usage.d.ts +26 -0
- package/dist/llm-usage.js +37 -0
- package/dist/llm.d.ts +153 -0
- package/dist/llm.js +530 -0
- package/dist/pipeline/graph.d.ts +35 -0
- package/dist/pipeline/graph.js +104 -0
- package/dist/pipeline/l1.d.ts +19 -0
- package/dist/pipeline/l1.js +271 -0
- package/dist/pipeline/l2.d.ts +13 -0
- package/dist/pipeline/l2.js +83 -0
- package/dist/pipeline/l3.d.ts +15 -0
- package/dist/pipeline/l3.js +78 -0
- package/dist/pipeline/rebuild.d.ts +61 -0
- package/dist/pipeline/rebuild.js +307 -0
- package/dist/pipeline/ruminate.d.ts +89 -0
- package/dist/pipeline/ruminate.js +298 -0
- package/dist/pipeline/runner.d.ts +167 -0
- package/dist/pipeline/runner.js +638 -0
- package/dist/pipeline/trigger.d.ts +40 -0
- package/dist/pipeline/trigger.js +75 -0
- package/dist/prompts/graph-projection.d.ts +70 -0
- package/dist/prompts/graph-projection.js +167 -0
- package/dist/prompts/l1-dedup.d.ts +22 -0
- package/dist/prompts/l1-dedup.js +251 -0
- package/dist/prompts/l1-extraction.d.ts +22 -0
- package/dist/prompts/l1-extraction.js +457 -0
- package/dist/prompts/persona.d.ts +23 -0
- package/dist/prompts/persona.js +240 -0
- package/dist/prompts/scene.d.ts +32 -0
- package/dist/prompts/scene.js +414 -0
- package/dist/runtime-package-lock.json +982 -0
- package/dist/settings.d.ts +50 -0
- package/dist/settings.js +355 -0
- package/dist/stats.d.ts +109 -0
- package/dist/stats.js +929 -0
- package/dist/store/bm25.d.ts +19 -0
- package/dist/store/bm25.js +63 -0
- package/dist/store/cost-ledger.d.ts +75 -0
- package/dist/store/cost-ledger.js +171 -0
- package/dist/store/download-queue.d.ts +79 -0
- package/dist/store/download-queue.js +424 -0
- package/dist/store/embedding-source.d.ts +118 -0
- package/dist/store/embedding-source.js +443 -0
- package/dist/store/embedding.d.ts +90 -0
- package/dist/store/embedding.js +206 -0
- package/dist/store/graph-store.d.ts +94 -0
- package/dist/store/graph-store.js +641 -0
- package/dist/store/l0.d.ts +40 -0
- package/dist/store/l0.js +197 -0
- package/dist/store/l1.d.ts +93 -0
- package/dist/store/l1.js +297 -0
- package/dist/store/local-embedding.d.ts +89 -0
- package/dist/store/local-embedding.js +227 -0
- package/dist/store/model-catalog.d.ts +48 -0
- package/dist/store/model-catalog.js +81 -0
- package/dist/store/occupancy.d.ts +30 -0
- package/dist/store/occupancy.js +134 -0
- package/dist/store/pending.d.ts +36 -0
- package/dist/store/pending.js +103 -0
- package/dist/store/persona.d.ts +15 -0
- package/dist/store/persona.js +60 -0
- package/dist/store/recall-dedupe.d.ts +26 -0
- package/dist/store/recall-dedupe.js +138 -0
- package/dist/store/runtime-installer.d.ts +59 -0
- package/dist/store/runtime-installer.js +243 -0
- package/dist/store/scenes.d.ts +24 -0
- package/dist/store/scenes.js +160 -0
- package/dist/store/search-utils.d.ts +38 -0
- package/dist/store/search-utils.js +100 -0
- package/dist/store/session-modes.d.ts +35 -0
- package/dist/store/session-modes.js +144 -0
- package/dist/store/sqlite.d.ts +246 -0
- package/dist/store/sqlite.js +1491 -0
- package/dist/store/state.d.ts +41 -0
- package/dist/store/state.js +72 -0
- package/dist/token-cost.d.ts +23 -0
- package/dist/token-cost.js +185 -0
- package/dist/tools/index.d.ts +34 -0
- package/dist/tools/index.js +758 -0
- package/dist/types.d.ts +139 -0
- package/dist/types.js +38 -0
- package/dist/util/context-occupancy.d.ts +68 -0
- package/dist/util/context-occupancy.js +92 -0
- package/dist/util/filelog.d.ts +6 -0
- package/dist/util/filelog.js +108 -0
- package/dist/util/io.d.ts +18 -0
- package/dist/util/io.js +97 -0
- package/dist/util/recall-budget.d.ts +31 -0
- package/dist/util/recall-budget.js +84 -0
- package/dist/util/sanitize.d.ts +11 -0
- package/dist/util/sanitize.js +67 -0
- package/dist/util/text.d.ts +16 -0
- package/dist/util/text.js +61 -0
- package/dist/util/tokenizer.d.ts +9 -0
- package/dist/util/tokenizer.js +50 -0
- package/dsh.plugin.json +22 -0
- package/package.json +118 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import type { MemoryConfig } from '../config.js';
|
|
3
|
+
import type { L1Store } from '../store/l1.js';
|
|
4
|
+
import type { MemoryState } from '../store/state.js';
|
|
5
|
+
import type { ConversationMessage, ExtractMode, MemoryFamily, MemoryLogger, MemoryRecord } from '../types.js';
|
|
6
|
+
export interface ExtractionResult {
|
|
7
|
+
stored: number;
|
|
8
|
+
skipped: boolean;
|
|
9
|
+
sceneName: string;
|
|
10
|
+
newRecords: MemoryRecord[];
|
|
11
|
+
}
|
|
12
|
+
/** 分族 checkpoint 桶(活引用,改动由调用方 save 落盘)。 */
|
|
13
|
+
export type FamilyStates = Record<MemoryFamily, MemoryState>;
|
|
14
|
+
/**
|
|
15
|
+
* 按字符预算把消息切成多块(保持顺序,单条超预算独占一块,由 callLLM 兜底截断)。
|
|
16
|
+
* 每条消息按 content 长度 + 64 字符脚手架开销(id/时间戳行)计。
|
|
17
|
+
*/
|
|
18
|
+
export declare function chunkByCharBudget(messages: ConversationMessage[], budgetChars: number): ConversationMessage[][];
|
|
19
|
+
export declare function runExtraction(ctx: Context, cfg: MemoryConfig, store: L1Store, states: FamilyStates, pending: ConversationMessage[], background: ConversationMessage[], logger: MemoryLogger, mode: ExtractMode): Promise<ExtractionResult>;
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L1 蒸馏:抽取(情境切分 + 记忆提取)→ 去重(冲突检测 + 合并)→ 写入。
|
|
3
|
+
*
|
|
4
|
+
* 流程(官方管线语义):抽取 prompt 分块跑(按 llm.maxInputChars 预算,情境链式
|
|
5
|
+
* 衔接)→ 去重候选按族召回 → 批量冲突检测 prompt → 决策应用:新记录追加进事实源,
|
|
6
|
+
* 被替换目标只从检索库删除。 族隔离贯穿始终:去重候选只在同族内,合并产物保持
|
|
7
|
+
* 新记忆的族标签。
|
|
8
|
+
*/
|
|
9
|
+
import { randomBytes } from 'node:crypto';
|
|
10
|
+
import { callLLM, parseJsonLogged, resolveLayerTokens } from '../llm.js';
|
|
11
|
+
import { formatExtractionPrompt, getExtractMemoriesSystemPrompt } from '../prompts/l1-extraction.js';
|
|
12
|
+
import { formatBatchConflictPrompt, getConflictDetectionSystemPrompt } from '../prompts/l1-dedup.js';
|
|
13
|
+
import { familyForType, normPersistence, resolveRecordFamily } from '../types.js';
|
|
14
|
+
/** 解析 ISO/epoch 时间证据,非法或非正值一律 undefined——不猜测。 */
|
|
15
|
+
function parseTimeEvidence(raw) {
|
|
16
|
+
const t = typeof raw === 'string' ? Date.parse(raw) : typeof raw === 'number' ? raw : Number.NaN;
|
|
17
|
+
return Number.isFinite(t) && t > 0 ? t : undefined;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* 抽取产出的 metadata → 记录的时间轴。
|
|
21
|
+
*
|
|
22
|
+
* `activity_start_time`/`activity_end_time` 是这两列的前身(prompt 已在产出),
|
|
23
|
+
* 这里把它们提升为结构化字段;`persistence` 由 prompt 产出,缺省即未判定。
|
|
24
|
+
*/
|
|
25
|
+
function temporalOf(meta) {
|
|
26
|
+
return {
|
|
27
|
+
validFrom: parseTimeEvidence(meta?.activity_start_time),
|
|
28
|
+
validTo: parseTimeEvidence(meta?.activity_end_time),
|
|
29
|
+
persistence: normPersistence(meta?.persistence),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* 合并(merge/update)时的有效期并集:
|
|
34
|
+
* 起 = 两侧最早;止 = 任一侧未闭合则仍未闭合(undefined),否则取最晚。
|
|
35
|
+
* 持续性以新记忆为准(合并产物描述的是当前认知),新记忆未判定时继承旧记录。
|
|
36
|
+
*/
|
|
37
|
+
function mergeTemporal(self, targets) {
|
|
38
|
+
const froms = [self.validFrom, ...targets.map((r) => r.validFrom)].filter((t) => t !== undefined);
|
|
39
|
+
const tos = [self.validTo, ...targets.map((r) => r.validTo)];
|
|
40
|
+
const anyOpen = tos.some((t) => t === undefined);
|
|
41
|
+
const closed = tos.filter((t) => t !== undefined);
|
|
42
|
+
return {
|
|
43
|
+
validFrom: froms.length > 0 ? Math.min(...froms) : undefined,
|
|
44
|
+
validTo: anyOpen || closed.length === 0 ? undefined : Math.max(...closed),
|
|
45
|
+
persistence: self.persistence ?? targets.find((r) => r.persistence)?.persistence,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function newId(prefix) {
|
|
49
|
+
return `${prefix}_${Date.now()}_${randomBytes(3).toString('hex')}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* 按字符预算把消息切成多块(保持顺序,单条超预算独占一块,由 callLLM 兜底截断)。
|
|
53
|
+
* 每条消息按 content 长度 + 64 字符脚手架开销(id/时间戳行)计。
|
|
54
|
+
*/
|
|
55
|
+
export function chunkByCharBudget(messages, budgetChars) {
|
|
56
|
+
if (messages.length === 0)
|
|
57
|
+
return [];
|
|
58
|
+
const chunks = [];
|
|
59
|
+
let cur = [];
|
|
60
|
+
let curChars = 0;
|
|
61
|
+
for (const m of messages) {
|
|
62
|
+
const len = m.content.length + 64;
|
|
63
|
+
if (cur.length > 0 && curChars + len > budgetChars) {
|
|
64
|
+
chunks.push(cur);
|
|
65
|
+
cur = [];
|
|
66
|
+
curChars = 0;
|
|
67
|
+
}
|
|
68
|
+
cur.push(m);
|
|
69
|
+
curChars += len;
|
|
70
|
+
}
|
|
71
|
+
if (cur.length > 0)
|
|
72
|
+
chunks.push(cur);
|
|
73
|
+
return chunks;
|
|
74
|
+
}
|
|
75
|
+
export async function runExtraction(ctx, cfg, store, states, pending, background, logger, mode) {
|
|
76
|
+
if (!cfg.extract.enabled)
|
|
77
|
+
return { stored: 0, skipped: true, sceneName: chainHead(states, mode), newRecords: [] };
|
|
78
|
+
// 触发阈值(渐进爬坡 + 按会话切片计数)由 runner 判定(trigger.ts);
|
|
79
|
+
// 此处不再重复 gate——重建轮 force 与 warmup 早期轮次都会传入小于稳态值的切片。
|
|
80
|
+
// 纯档强制族 = 档位族;auto 档抽取后的记录族按 type 前缀判定
|
|
81
|
+
const forcedFamily = mode === 'auto' ? undefined : mode;
|
|
82
|
+
// 情境链锚点桶:纯档用本族;auto 用最近活跃的族(chainHead 同源)
|
|
83
|
+
const chainState = mode === 'auto' ? activeState(states) : states[mode];
|
|
84
|
+
// ── Step 1: 抽取(输入按 llm.maxInputChars 预算分块,情境链式衔接,不丢消息) ──
|
|
85
|
+
const backgroundMsgs = background.slice(-cfg.extract.backgroundMessages);
|
|
86
|
+
// 预留:背景消息(≤10 条 ×4000 字)+ prompt 脚手架
|
|
87
|
+
const perChunk = Math.max(20_000, cfg.llm.maxInputChars - 42_000);
|
|
88
|
+
const chunks = chunkByCharBudget(pending, perChunk);
|
|
89
|
+
if (chunks.length > 1) {
|
|
90
|
+
logger.info(`[memory] L1 输入超预算(${pending.length} 条消息),分 ${chunks.length} 块抽取(每块 ≤${perChunk} 字符)`);
|
|
91
|
+
}
|
|
92
|
+
const extracted = [];
|
|
93
|
+
let lastScene = chainState.lastSceneName;
|
|
94
|
+
let sceneCount = 0;
|
|
95
|
+
for (const chunk of chunks) {
|
|
96
|
+
const userPrompt = formatExtractionPrompt({
|
|
97
|
+
newMessages: chunk,
|
|
98
|
+
backgroundMessages: backgroundMsgs,
|
|
99
|
+
previousSceneName: lastScene || '无',
|
|
100
|
+
halls: cfg.hall?.enabled,
|
|
101
|
+
});
|
|
102
|
+
const raw = await callLLM(ctx, cfg, {
|
|
103
|
+
system: getExtractMemoriesSystemPrompt(mode),
|
|
104
|
+
user: userPrompt,
|
|
105
|
+
maxTokens: resolveLayerTokens(cfg, 'extract'),
|
|
106
|
+
layer: 'l1-extract',
|
|
107
|
+
logger,
|
|
108
|
+
});
|
|
109
|
+
const scenes = parseJsonLogged(raw, 'L1 抽取', logger);
|
|
110
|
+
if (!Array.isArray(scenes))
|
|
111
|
+
throw new Error('L1 抽取输出不是 JSON 数组');
|
|
112
|
+
for (const scene of scenes) {
|
|
113
|
+
if (!scene || typeof scene.scene_name !== 'string')
|
|
114
|
+
continue;
|
|
115
|
+
lastScene = scene.scene_name;
|
|
116
|
+
sceneCount++;
|
|
117
|
+
for (const m of scene.memories ?? []) {
|
|
118
|
+
if (!m || typeof m.content !== 'string' || !m.content.trim())
|
|
119
|
+
continue;
|
|
120
|
+
extracted.push({
|
|
121
|
+
...m,
|
|
122
|
+
record_id: newId('mem'),
|
|
123
|
+
scene_name: scene.scene_name,
|
|
124
|
+
family: resolveRecordFamily(forcedFamily, m.family, m.type ?? ''),
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (extracted.length === 0) {
|
|
130
|
+
logger.info(`[memory] L1 抽取完成:无可提取记忆(mode=${mode},${pending.length} 条消息,${sceneCount} 个情境)`);
|
|
131
|
+
// 成功但零产出:同样推进时间戳/场景,与失败(lastExtractAt 保持 0)区分开
|
|
132
|
+
markExtracted(states, mode, lastScene);
|
|
133
|
+
return { stored: 0, skipped: false, sceneName: lastScene, newRecords: [] };
|
|
134
|
+
}
|
|
135
|
+
// ── Step 2: 去重(batch 冲突检测;候选只在本族内召回,去重永不跨族) ──
|
|
136
|
+
const matches = await Promise.all(extracted.map(async (m) => ({
|
|
137
|
+
newMemory: m,
|
|
138
|
+
candidates: await store.searchCandidates(m.content, cfg.extract.candidatePool, m.family),
|
|
139
|
+
})));
|
|
140
|
+
const dedupPrompt = formatBatchConflictPrompt(matches);
|
|
141
|
+
const dedupRaw = await callLLM(ctx, cfg, {
|
|
142
|
+
system: getConflictDetectionSystemPrompt(mode),
|
|
143
|
+
user: dedupPrompt,
|
|
144
|
+
maxTokens: resolveLayerTokens(cfg, 'dedup'),
|
|
145
|
+
layer: 'l1-dedup',
|
|
146
|
+
logger,
|
|
147
|
+
});
|
|
148
|
+
const decisions = parseJsonLogged(dedupRaw, 'L1 去重判定', logger);
|
|
149
|
+
const byRecord = new Map();
|
|
150
|
+
for (const d of Array.isArray(decisions) ? decisions : []) {
|
|
151
|
+
if (d && typeof d.record_id === 'string')
|
|
152
|
+
byRecord.set(d.record_id, d);
|
|
153
|
+
}
|
|
154
|
+
// 去重决策统计:无决策的记录按 skip 处理,聚合成单行日志便于排查
|
|
155
|
+
const actionCount = {};
|
|
156
|
+
for (const m of extracted) {
|
|
157
|
+
const action = byRecord.get(m.record_id)?.action ?? 'skip(未返回)';
|
|
158
|
+
actionCount[action] = (actionCount[action] ?? 0) + 1;
|
|
159
|
+
}
|
|
160
|
+
const candidateTotal = matches.reduce((n, m) => n + m.candidates.length, 0);
|
|
161
|
+
logger.info(`[memory] L1 去重判定:${extracted.length} 条候选记忆召回 ${candidateTotal} 条已有记录,决策 ${Object.entries(actionCount)
|
|
162
|
+
.map(([k, v]) => `${k}=${v}`)
|
|
163
|
+
.join(' ')}`);
|
|
164
|
+
// ── Step 3: 应用决策(官方语义:新记录追加进事实源,被替换目标只从检索库删除) ──
|
|
165
|
+
// 只按需取决策涉及的记录(候选 + 目标 id 并集),避免每轮全表扫描
|
|
166
|
+
const relatedIds = new Set();
|
|
167
|
+
for (const d of byRecord.values()) {
|
|
168
|
+
for (const id of d.target_ids ?? [])
|
|
169
|
+
relatedIds.add(id);
|
|
170
|
+
}
|
|
171
|
+
for (const m of matches) {
|
|
172
|
+
for (const c of m.candidates)
|
|
173
|
+
relatedIds.add(c.id);
|
|
174
|
+
}
|
|
175
|
+
const byId = new Map(store.getByIds([...relatedIds]).map((r) => [r.id, r]));
|
|
176
|
+
const deletedIds = new Set();
|
|
177
|
+
const added = [];
|
|
178
|
+
const now = Date.now();
|
|
179
|
+
for (const m of extracted) {
|
|
180
|
+
const decision = byRecord.get(m.record_id);
|
|
181
|
+
if (!decision || decision.action === 'skip')
|
|
182
|
+
continue;
|
|
183
|
+
const action = decision.action;
|
|
184
|
+
const ts = m.metadata?.activity_start_time ? Date.parse(String(m.metadata.activity_start_time)) : now;
|
|
185
|
+
if (action === 'store') {
|
|
186
|
+
added.push({
|
|
187
|
+
id: m.record_id,
|
|
188
|
+
content: m.content,
|
|
189
|
+
type: m.type,
|
|
190
|
+
priority: Number(m.priority) || 60,
|
|
191
|
+
scene_name: m.scene_name,
|
|
192
|
+
timestamps: [Number.isNaN(ts) ? now : ts],
|
|
193
|
+
createdAt: now,
|
|
194
|
+
updatedAt: now,
|
|
195
|
+
version: 0,
|
|
196
|
+
source_message_ids: m.source_message_ids ?? [],
|
|
197
|
+
metadata: m.metadata ?? {},
|
|
198
|
+
family: m.family,
|
|
199
|
+
...temporalOf(m.metadata),
|
|
200
|
+
});
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
// update / merge:目标记录从检索库删除,合并结果作为新记录追加(版本 +1)
|
|
204
|
+
// 候选召回按族隔离,合并产物保持新记忆的族标签
|
|
205
|
+
const targets = (decision.target_ids ?? []).filter((id) => byId.has(id));
|
|
206
|
+
for (const id of targets)
|
|
207
|
+
deletedIds.add(id);
|
|
208
|
+
const targetVersion = targets.reduce((max, id) => Math.max(max, byId.get(id)?.version ?? 0), 0);
|
|
209
|
+
const mergedTs = (decision.merged_timestamps ?? [])
|
|
210
|
+
.map((t) => Date.parse(t))
|
|
211
|
+
.filter((t) => !Number.isNaN(t))
|
|
212
|
+
.concat([now]);
|
|
213
|
+
added.push({
|
|
214
|
+
id: m.record_id,
|
|
215
|
+
content: decision.merged_content && decision.merged_content.trim()
|
|
216
|
+
? decision.merged_content
|
|
217
|
+
: m.content,
|
|
218
|
+
type: decision.merged_type || m.type,
|
|
219
|
+
priority: Number(decision.merged_priority) || Number(m.priority) || 60,
|
|
220
|
+
scene_name: m.scene_name,
|
|
221
|
+
timestamps: Array.from(new Set(mergedTs)).sort((a, b) => a - b),
|
|
222
|
+
createdAt: now,
|
|
223
|
+
updatedAt: now,
|
|
224
|
+
version: targetVersion + 1,
|
|
225
|
+
source_message_ids: m.source_message_ids ?? [],
|
|
226
|
+
metadata: m.metadata ?? {},
|
|
227
|
+
family: m.family,
|
|
228
|
+
// 合并的有效期取并集:起 = 两侧最早;止 = 任一侧未闭合则仍未闭合(undefined)。
|
|
229
|
+
...mergeTemporal(temporalOf(m.metadata), targets.map((id) => byId.get(id)).filter((r) => r !== undefined)),
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
await store.appendNew(added);
|
|
233
|
+
if (deletedIds.size > 0)
|
|
234
|
+
await store.deleteBatch([...deletedIds]);
|
|
235
|
+
// 状态按记录族分桶推进(阈值计数各自独立)
|
|
236
|
+
const addedByFamily = { chat: 0, work: 0 };
|
|
237
|
+
for (const r of added)
|
|
238
|
+
addedByFamily[r.family ?? familyForType(r.type)]++;
|
|
239
|
+
for (const f of ['chat', 'work']) {
|
|
240
|
+
if (addedByFamily[f] === 0)
|
|
241
|
+
continue;
|
|
242
|
+
states[f].totalExtracted += addedByFamily[f];
|
|
243
|
+
states[f].newMemoriesSinceL2 += addedByFamily[f];
|
|
244
|
+
states[f].memoriesSinceL3 += addedByFamily[f];
|
|
245
|
+
}
|
|
246
|
+
markExtracted(states, mode, lastScene);
|
|
247
|
+
logger.info(`[memory] L1 抽取完成(mode=${mode}):消息 ${pending.length} 条,抽取 ${extracted.length} 条,去重后新增 ${added.length} 条(替换 ${deletedIds.size} 条,chat=${addedByFamily.chat}/work=${addedByFamily.work}),累计 chat=${states.chat.totalExtracted}/work=${states.work.totalExtracted}`);
|
|
248
|
+
return { stored: added.length, skipped: false, sceneName: lastScene, newRecords: added };
|
|
249
|
+
}
|
|
250
|
+
/** auto 档取最近活跃的族 checkpoint(情境链/计数锚点)。 */
|
|
251
|
+
function activeState(states) {
|
|
252
|
+
return states.chat.lastExtractAt >= states.work.lastExtractAt ? states.chat : states.work;
|
|
253
|
+
}
|
|
254
|
+
/** 情境链读取:auto → 最近活跃族;纯档 → 本族。 */
|
|
255
|
+
function chainHead(states, mode) {
|
|
256
|
+
return (mode === 'auto' ? activeState(states) : states[mode]).lastSceneName;
|
|
257
|
+
}
|
|
258
|
+
/** 抽取成功后推进时间戳/情境链:auto → 两族都推进(链只写锚点桶);纯档 → 本族。 */
|
|
259
|
+
function markExtracted(states, mode, lastScene) {
|
|
260
|
+
const now = Date.now();
|
|
261
|
+
if (mode === 'auto') {
|
|
262
|
+
// 两族的 lastExtractAt 都推进(去重候选各自族内判断"新消息");情境链只维护锚点桶
|
|
263
|
+
states.chat.lastExtractAt = now;
|
|
264
|
+
states.work.lastExtractAt = now;
|
|
265
|
+
activeState(states).lastSceneName = lastScene;
|
|
266
|
+
}
|
|
267
|
+
else {
|
|
268
|
+
states[mode].lastExtractAt = now;
|
|
269
|
+
states[mode].lastSceneName = lastScene;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L2 场景整合:把新记忆交给 LLM,输出文件操作 JSON,工程侧执行。
|
|
3
|
+
* (UPDATE 优先 / MERGE / 上限预警 / heat 管理;Bm25 选相关场景全文作上下文。)
|
|
4
|
+
*/
|
|
5
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
6
|
+
import type { MemoryConfig } from '../config.js';
|
|
7
|
+
import type { SceneStore } from '../store/scenes.js';
|
|
8
|
+
import type { MemoryFamily, MemoryLogger, MemoryRecord } from '../types.js';
|
|
9
|
+
export interface SceneResult {
|
|
10
|
+
changed: number;
|
|
11
|
+
personaRequestedReason?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare function runSceneConsolidation(ctx: Context, cfg: MemoryConfig, scenes: SceneStore, newMemories: MemoryRecord[], logger: MemoryLogger, family: MemoryFamily): Promise<SceneResult>;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { callLLM, parseJsonLogged, resolveLayerTokens } from '../llm.js';
|
|
2
|
+
import { buildScenePrompt, formatSceneSummaries } from '../prompts/scene.js';
|
|
3
|
+
import { Bm25Index } from '../store/bm25.js';
|
|
4
|
+
const REQUEST_RE = /\[PERSONA_UPDATE_REQUEST\]\s*reason:\s*([\s\S]*?)\[\/PERSONA_UPDATE_REQUEST\]/;
|
|
5
|
+
export async function runSceneConsolidation(ctx, cfg, scenes, newMemories, logger, family) {
|
|
6
|
+
if (!cfg.l2.enabled || newMemories.length === 0) {
|
|
7
|
+
return { changed: 0 };
|
|
8
|
+
}
|
|
9
|
+
const summaries = await scenes.list();
|
|
10
|
+
// ── 组装 prompt ──
|
|
11
|
+
const memoriesJson = JSON.stringify(newMemories.map((m) => ({
|
|
12
|
+
record_id: m.id,
|
|
13
|
+
content: m.content,
|
|
14
|
+
type: m.type,
|
|
15
|
+
priority: m.priority,
|
|
16
|
+
scene_name: m.scene_name,
|
|
17
|
+
timestamps: m.timestamps.map((t) => new Date(t).toISOString()),
|
|
18
|
+
})), null, 2);
|
|
19
|
+
const sceneSummaries = formatSceneSummaries(summaries);
|
|
20
|
+
const sceneContents = await pickSceneContents(scenes, summaries, newMemories, cfg.l2.sceneContextLimit);
|
|
21
|
+
const { systemPrompt, userPrompt } = buildScenePrompt({
|
|
22
|
+
memoriesJson,
|
|
23
|
+
sceneSummaries,
|
|
24
|
+
sceneContents,
|
|
25
|
+
currentTimestamp: new Date().toISOString(),
|
|
26
|
+
existingSceneFiles: summaries.map((s) => s.path),
|
|
27
|
+
maxScenes: cfg.l2.maxScenes,
|
|
28
|
+
family,
|
|
29
|
+
});
|
|
30
|
+
const raw = await callLLM(ctx, cfg, {
|
|
31
|
+
system: systemPrompt,
|
|
32
|
+
user: userPrompt,
|
|
33
|
+
maxTokens: resolveLayerTokens(cfg, 'l2'),
|
|
34
|
+
layer: 'l2',
|
|
35
|
+
logger,
|
|
36
|
+
});
|
|
37
|
+
// ── 解析:先取 PERSONA_UPDATE_REQUEST 标记,再解析操作数组 ──
|
|
38
|
+
const reqMatch = REQUEST_RE.exec(raw);
|
|
39
|
+
const personaRequestedReason = reqMatch?.[1]?.trim() || undefined;
|
|
40
|
+
const opsRaw = reqMatch ? raw.replace(REQUEST_RE, '') : raw;
|
|
41
|
+
const ops = parseJsonLogged(opsRaw, 'L2 场景操作', logger);
|
|
42
|
+
if (!Array.isArray(ops))
|
|
43
|
+
throw new Error('L2 输出不是操作数组');
|
|
44
|
+
let changed = 0;
|
|
45
|
+
for (const op of ops) {
|
|
46
|
+
if (!op || typeof op.path !== 'string')
|
|
47
|
+
continue;
|
|
48
|
+
try {
|
|
49
|
+
if (op.op === 'write' && typeof op.content === 'string') {
|
|
50
|
+
await scenes.write(op.path, op.content);
|
|
51
|
+
changed++;
|
|
52
|
+
logger.debug?.(`[memory] L2 写入场景 ${op.path} (${op.content.length} 字符)`);
|
|
53
|
+
}
|
|
54
|
+
else if (op.op === 'delete') {
|
|
55
|
+
await scenes.write(op.path, '[DELETED]');
|
|
56
|
+
changed++;
|
|
57
|
+
logger.debug?.(`[memory] L2 删除场景 ${op.path}`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
logger.warn(`[memory] L2 操作失败 ${op.path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
logger.info(`[memory] L2 场景整合完成(family=${family}):${newMemories.length} 条新记忆 → ${changed} 个文件操作(场景总数 ${summaries.length})` +
|
|
65
|
+
(personaRequestedReason ? `,请求 L3 更新:${personaRequestedReason}` : ''));
|
|
66
|
+
return { changed, personaRequestedReason };
|
|
67
|
+
}
|
|
68
|
+
/** 选出与本次新记忆最相关的场景全文(供 LLM 参考,避免全量塞入)。 */
|
|
69
|
+
async function pickSceneContents(scenes, summaries, newMemories, limit) {
|
|
70
|
+
if (summaries.length === 0 || limit <= 0)
|
|
71
|
+
return '';
|
|
72
|
+
const index = new Bm25Index();
|
|
73
|
+
index.rebuild(summaries.map((s) => ({ id: s.path, text: `${s.summary} ${s.path}` })));
|
|
74
|
+
const query = newMemories.map((m) => m.content).join(' ');
|
|
75
|
+
const hits = index.search(query, limit);
|
|
76
|
+
const parts = [];
|
|
77
|
+
for (const h of hits) {
|
|
78
|
+
const content = await scenes.read(h.id);
|
|
79
|
+
if (content)
|
|
80
|
+
parts.push(`### 文件: ${h.id}\n\`\`\`markdown\n${content}\n\`\`\``);
|
|
81
|
+
}
|
|
82
|
+
return parts.join('\n\n');
|
|
83
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L3 画像蒸馏:把变化场景交给 LLM,直接产出 persona-<family>.md 完整内容。
|
|
3
|
+
* 触发条件(persona-trigger):L2 主动请求 / 冷启动 / 恢复 / 间隔阈值。
|
|
4
|
+
*/
|
|
5
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
6
|
+
import type { MemoryConfig } from '../config.js';
|
|
7
|
+
import type { PersonaStore } from '../store/persona.js';
|
|
8
|
+
import type { SceneStore } from '../store/scenes.js';
|
|
9
|
+
import type { MemoryState } from '../store/state.js';
|
|
10
|
+
import type { MemoryFamily, MemoryLogger } from '../types.js';
|
|
11
|
+
export interface PersonaResult {
|
|
12
|
+
generated: boolean;
|
|
13
|
+
reason: string;
|
|
14
|
+
}
|
|
15
|
+
export declare function runPersona(ctx: Context, cfg: MemoryConfig, scenes: SceneStore, persona: PersonaStore, state: MemoryState, logger: MemoryLogger, family: MemoryFamily): Promise<PersonaResult>;
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { callLLM, resolveLayerTokens } from '../llm.js';
|
|
2
|
+
import { buildPersonaPrompt } from '../prompts/persona.js';
|
|
3
|
+
export async function runPersona(ctx, cfg, scenes, persona, state, logger, family) {
|
|
4
|
+
if (!cfg.l3.enabled)
|
|
5
|
+
return { generated: false, reason: 'l3 disabled' };
|
|
6
|
+
// ── 触发判定 ──
|
|
7
|
+
const existingPersona = await persona.read();
|
|
8
|
+
let reason;
|
|
9
|
+
if (state.personaRequestedReason) {
|
|
10
|
+
reason = `主动请求: ${state.personaRequestedReason}`;
|
|
11
|
+
}
|
|
12
|
+
else if (!state.hasPersona || !existingPersona) {
|
|
13
|
+
reason = '冷启动/恢复:首次生成或画像缺失';
|
|
14
|
+
}
|
|
15
|
+
else if (state.memoriesSinceL3 >= cfg.l3.interval) {
|
|
16
|
+
reason = `达到阈值: ${state.memoriesSinceL3} >= ${cfg.l3.interval}`;
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
logger.debug?.(`[memory] L3 未触发(自上次蒸馏以来 ${state.memoriesSinceL3}/${cfg.l3.interval} 条新记忆)`);
|
|
20
|
+
return { generated: false, reason: 'no trigger' };
|
|
21
|
+
}
|
|
22
|
+
// ── 收集变化场景(自上次蒸馏以来 updated 变化 / 首次则全部) ──
|
|
23
|
+
const all = await scenes.list();
|
|
24
|
+
const changed = state.hasPersona
|
|
25
|
+
? all.filter((s) => {
|
|
26
|
+
const t = Date.parse(s.updated);
|
|
27
|
+
return !Number.isNaN(t) && t > state.lastL3At;
|
|
28
|
+
})
|
|
29
|
+
: all;
|
|
30
|
+
if (changed.length === 0 && state.hasPersona) {
|
|
31
|
+
logger.debug?.('[memory] L3 触发但无变化场景,跳过');
|
|
32
|
+
return { generated: false, reason: 'no changed scenes' };
|
|
33
|
+
}
|
|
34
|
+
const changedContents = [];
|
|
35
|
+
for (const s of changed) {
|
|
36
|
+
const content = await scenes.read(s.path);
|
|
37
|
+
if (content)
|
|
38
|
+
changedContents.push(`### 场景: ${s.path}\n\`\`\`markdown\n${content}\n\`\`\``);
|
|
39
|
+
}
|
|
40
|
+
const { systemPrompt, userPrompt } = buildPersonaPrompt({
|
|
41
|
+
mode: existingPersona ? 'incremental' : 'first',
|
|
42
|
+
family,
|
|
43
|
+
currentTime: new Date().toISOString(),
|
|
44
|
+
totalProcessed: state.totalExtracted,
|
|
45
|
+
sceneCount: all.length,
|
|
46
|
+
changedSceneCount: changed.length,
|
|
47
|
+
changedScenesContent: changedContents.join('\n\n'),
|
|
48
|
+
existingPersona,
|
|
49
|
+
triggerInfo: reason,
|
|
50
|
+
});
|
|
51
|
+
const raw = await callLLM(ctx, cfg, {
|
|
52
|
+
system: systemPrompt,
|
|
53
|
+
user: userPrompt,
|
|
54
|
+
maxTokens: resolveLayerTokens(cfg, 'l3'),
|
|
55
|
+
layer: 'l3',
|
|
56
|
+
logger,
|
|
57
|
+
});
|
|
58
|
+
const body = unwrapFence(raw);
|
|
59
|
+
if (!body) {
|
|
60
|
+
logger.error(`[memory] L3 输出为空,原始输出前 400 字符: ${raw.slice(0, 400)}`);
|
|
61
|
+
throw new Error('L3 输出为空');
|
|
62
|
+
}
|
|
63
|
+
await persona.write(body);
|
|
64
|
+
state.hasPersona = true;
|
|
65
|
+
state.lastL3At = Date.now();
|
|
66
|
+
state.memoriesSinceL3 = 0;
|
|
67
|
+
state.personaRequestedReason = undefined;
|
|
68
|
+
logger.info(`[memory] L3 画像蒸馏完成(family=${family},${reason}):${body.length} 字符,场景 ${changed.length}/${all.length} 个`);
|
|
69
|
+
return { generated: true, reason };
|
|
70
|
+
}
|
|
71
|
+
/** 剥掉可能的 ```markdown 围栏。 */
|
|
72
|
+
function unwrapFence(text) {
|
|
73
|
+
let s = text.trim();
|
|
74
|
+
if (s.startsWith('```')) {
|
|
75
|
+
s = s.replace(/^```[a-zA-Z]*\n?/, '').replace(/```$/, '').trim();
|
|
76
|
+
}
|
|
77
|
+
return s;
|
|
78
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import type { Context } from '@deepseek-ai/cordis';
|
|
2
|
+
import { type MemoryConfig } from '../config.js';
|
|
3
|
+
import type { L1Store } from '../store/l1.js';
|
|
4
|
+
import type { PersonaStore } from '../store/persona.js';
|
|
5
|
+
import type { SceneStore } from '../store/scenes.js';
|
|
6
|
+
import type { StateStore } from '../store/state.js';
|
|
7
|
+
import type { MemoryDb } from '../store/sqlite.js';
|
|
8
|
+
import type { LiveSettingsHandle } from '../settings.js';
|
|
9
|
+
import type { ConversationMessage, L0MessageRecord, MemoryFamily, MemoryLogger } from '../types.js';
|
|
10
|
+
import { type MemoryRunner } from './runner.js';
|
|
11
|
+
/** 重建所需的存储子集(l0 不需要——快照直接走 db)。 */
|
|
12
|
+
export interface RebuildStores {
|
|
13
|
+
l1: L1Store;
|
|
14
|
+
scenes: Record<MemoryFamily, SceneStore>;
|
|
15
|
+
persona: Record<MemoryFamily, PersonaStore>;
|
|
16
|
+
state: StateStore;
|
|
17
|
+
}
|
|
18
|
+
import type { RebuildStatus } from '../contract.js';
|
|
19
|
+
export type { RebuildPhase, RebuildStatus } from '../contract.js';
|
|
20
|
+
export interface RebuildChunk {
|
|
21
|
+
sessionId: string;
|
|
22
|
+
messages: ConversationMessage[];
|
|
23
|
+
}
|
|
24
|
+
export declare function groupL0Sessions(records: L0MessageRecord[]): RebuildChunk[];
|
|
25
|
+
/** 抽取调用数下界估算(与 l1.ts 的 perChunk 同式;会话数与字符预算取大)。 */
|
|
26
|
+
export declare function estimateCalls(sessions: number, messages: number, chars: number, maxInputChars: number): number;
|
|
27
|
+
export declare class RebuildController {
|
|
28
|
+
private readonly ctx;
|
|
29
|
+
private readonly cfg;
|
|
30
|
+
private readonly stores;
|
|
31
|
+
private readonly db;
|
|
32
|
+
private readonly runner;
|
|
33
|
+
private readonly logger;
|
|
34
|
+
private readonly live;
|
|
35
|
+
private status;
|
|
36
|
+
private chunks;
|
|
37
|
+
private cancelRequested;
|
|
38
|
+
/** 快照时刻(收尾时按它区分重建产物与重建后新对话的记录)。 */
|
|
39
|
+
private rebuildStartMs;
|
|
40
|
+
constructor(ctx: Context, cfg: MemoryConfig, stores: RebuildStores, db: MemoryDb, runner: Pick<MemoryRunner, 'enqueueRebuildTask' | 'runRebuildTurn' | 'states'>, logger: MemoryLogger, live: LiveSettingsHandle);
|
|
41
|
+
/** 状态快照(idle 时附带实时 L0 预估,供确认弹窗显示成本)。 */
|
|
42
|
+
getStatus(): RebuildStatus;
|
|
43
|
+
/** 内存中尚未处理的会话快照块数(收尾后应为 0——快照即弃,诊断/冒烟用)。 */
|
|
44
|
+
get chunkCount(): number;
|
|
45
|
+
/** 启动重建(校验后入队准备任务;真正的清库/归档在管线队列里串行执行,避开并发竞态)。 */
|
|
46
|
+
start(): RebuildStatus;
|
|
47
|
+
/** 请求取消:当前块完成后停止,已重建部分保留并照常收尾 L2/L3。 */
|
|
48
|
+
requestCancel(): RebuildStatus;
|
|
49
|
+
private prepare;
|
|
50
|
+
/** 分块链:一次只挂一个重建块,跑完再挂下一块——正常轮次可随时插队。 */
|
|
51
|
+
private scheduleChunk;
|
|
52
|
+
private finalize;
|
|
53
|
+
/**
|
|
54
|
+
* 收集重建窗口内某族的记录:重建产物全部是新插入(updated==created),
|
|
55
|
+
* 按 updated_time 倒序翻页、越过 rebuildStartMs 即停。
|
|
56
|
+
*/
|
|
57
|
+
private collectRebuildRecords;
|
|
58
|
+
private finish;
|
|
59
|
+
/** 归档旧派生层:records/ scenes/ persona-*.md 改名 .bak.<ts>。不存在则跳过。 */
|
|
60
|
+
private archiveDerived;
|
|
61
|
+
}
|