@kira905/dsh-memory-toolkit 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/README.en.md +225 -2
- package/README.md +180 -2
- package/lib/deploy-guard.mjs +463 -275
- package/lib/guard-http.mjs +59 -62
- package/lib/pitfall-form-map.mjs +94 -84
- package/lib/pitfall-score.mjs +262 -185
- package/lib/session-file-name.mjs +116 -100
- package/lib/zstd-frames.mjs +88 -67
- package/memory-db-tools/README.md +29 -1
- package/memory-db-tools/audit-analyze.mjs +1 -1
- package/memory-db-tools/audit-export.mjs +60 -16
- package/memory-db-tools/audit-struct-probe.mjs +29 -8
- package/memory-db-tools/lib/known-devices.mjs +137 -0
- package/memory-engine/README.md +58 -4
- package/memory-engine/bench/bench.mjs +1 -1
- package/memory-engine/cli/band-sentinel.mjs +164 -0
- package/memory-engine/cli/distill-rollback.mjs +112 -44
- package/memory-engine/cli/distill-sessions.mjs +237 -0
- package/memory-engine/cli/distill.mjs +196 -95
- package/memory-engine/cli/doc-index-rebuild.mjs +142 -108
- package/memory-engine/cli/doc-index.mjs +47 -38
- package/memory-engine/cli/domain-tags.mjs +246 -162
- package/memory-engine/cli/layer-migrate.mjs +176 -0
- package/memory-engine/cli/layer-move.mjs +189 -0
- package/memory-engine/cli/pitfall-hook.mjs +136 -70
- package/memory-engine/cli/recall.mjs +243 -110
- package/memory-engine/cli/selfcheck.mjs +93 -44
- package/memory-engine/cli/serve-start.mjs +221 -140
- package/memory-engine/cli/sweep.mjs +83 -44
- package/memory-engine/cli/sync.mjs +93 -57
- package/memory-engine/config/audit-devices.json +6 -0
- package/memory-engine/config/bands.json +7 -0
- package/memory-engine/config/layers.json +7 -0
- package/memory-engine/dict/LICENSE-/350/257/215/345/205/270.md +53 -0
- package/memory-engine/dict/lexicon-config.json +11 -0
- package/memory-engine/dict//350/257/215/345/205/270.txt +109750 -0
- package/memory-engine/dict//350/257/215/351/242/221.txt +20000 -0
- package/memory-engine/lib/band-sentinel.mjs +417 -0
- package/memory-engine/lib/cli-strict.mjs +128 -0
- package/memory-engine/lib/cli-util.mjs +147 -125
- package/memory-engine/lib/config.mjs +83 -96
- package/memory-engine/lib/dirty-ledger.mjs +127 -94
- package/memory-engine/lib/distill-rollback.mjs +173 -133
- package/memory-engine/lib/distill.mjs +878 -585
- package/memory-engine/lib/doc-index-store.mjs +208 -235
- package/memory-engine/lib/doc-index.mjs +351 -350
- package/memory-engine/lib/doc-scan.mjs +178 -156
- package/memory-engine/lib/doc-summarize.mjs +220 -205
- package/memory-engine/lib/domain-tags.mjs +358 -277
- package/memory-engine/lib/embedder.mjs +175 -194
- package/memory-engine/lib/hashes.mjs +22 -7
- package/memory-engine/lib/layer-migrate.mjs +330 -0
- package/memory-engine/lib/layer-move.mjs +307 -0
- package/memory-engine/lib/layers.mjs +191 -0
- package/memory-engine/lib/lexicon-recall.mjs +473 -0
- package/memory-engine/lib/memory-source.mjs +78 -80
- package/memory-engine/lib/pitfall-source.mjs +115 -114
- package/memory-engine/lib/policy-source.mjs +279 -261
- package/memory-engine/lib/recall-log.mjs +285 -0
- package/memory-engine/lib/recall.mjs +565 -240
- package/memory-engine/lib/selfcheck.mjs +142 -106
- package/memory-engine/lib/session-distill.mjs +603 -0
- package/memory-engine/lib/session-source.mjs +421 -0
- package/memory-engine/lib/sweep.mjs +107 -130
- package/memory-engine/lib/sync.mjs +110 -140
- package/memory-engine/lib/tick-hook.mjs +123 -148
- package/memory-engine/lib/vectors-store.mjs +213 -188
- package/memory-engine/test/12-layering.test.mjs +539 -0
- package/memory-engine/test/13-session-distill.test.mjs +616 -0
- package/memory-engine/tools/bridge-smoke.mjs +94 -55
- package/memory-engine/tools/fetch-model.mjs +167 -87
- package/memory-engine/tools/smoke-recall.mjs +94 -38
- package/memory-engine/tools/verify-recall-chain.mjs +413 -0
- package/package.json +1 -1
|
@@ -0,0 +1,603 @@
|
|
|
1
|
+
// 会话日志 → 记忆条的提炼管线(零依赖)。
|
|
2
|
+
//
|
|
3
|
+
// 分工:session-source.mjs 负责「挑句子 + 记来源」,这里负责「预算闸 → 调一次模型 → 写回 → 记账」。
|
|
4
|
+
//
|
|
5
|
+
// 三条不变式(写在这里,因为它们是这张卡的全部意义):
|
|
6
|
+
// ① **没配每日预算 ⇒ 一次模型都不调**:闸在读凭证之前,也在 openai 之前。
|
|
7
|
+
// ② 写库前必须拍快照(VACUUM INTO),没快照不写。
|
|
8
|
+
// ③ 一批一个来源键(srcKey)⇒ 撤回器一条 `--key` 命令就能整批撤掉。
|
|
9
|
+
//
|
|
10
|
+
// 写回形态与内核一致:tags 以 `distilled` 开头(撤回器认这个前缀),
|
|
11
|
+
// 另加 `sess=` 与 `run=session:<runId>` 两个属性位,方便"按场次/按批次"回指。
|
|
12
|
+
|
|
13
|
+
import fs from 'node:fs';
|
|
14
|
+
import path from 'node:path';
|
|
15
|
+
import crypto from 'node:crypto';
|
|
16
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
17
|
+
import { extractCandidates, summarizeSkipped } from './session-source.mjs';
|
|
18
|
+
import {
|
|
19
|
+
DISTILL_PREFIX, DEFAULT_LLM, shortKey, distilledTagOf, withBusyRetry,
|
|
20
|
+
localDateIso,
|
|
21
|
+
} from './distill.mjs';
|
|
22
|
+
import { extractJson, vacuumInto, readCredential } from './domain-tags.mjs';
|
|
23
|
+
|
|
24
|
+
export const STATE_SCHEMA = 1;
|
|
25
|
+
|
|
26
|
+
/** 一轮最多写几条(卡面口径:≤5)。 */
|
|
27
|
+
export const DEFAULT_LIMIT = 5;
|
|
28
|
+
|
|
29
|
+
/** 标签里记批次的属性位:`run=session:<runId>`。 */
|
|
30
|
+
export const RUN_TAG_PREFIX = 'run=session:';
|
|
31
|
+
|
|
32
|
+
/** 「没有预算」的那句人话 —— 逐字出现在 CLI、README、测试里,避免各处口径漂移。 */
|
|
33
|
+
export const NO_BUDGET_MESSAGE = '未配置每日预算 ⇒ 本轮不跑(这是故意的)';
|
|
34
|
+
|
|
35
|
+
const TAG_RUN_RE = /(?:^|\s)run=session:([0-9A-Za-z._-]+)(?:\s|$)/;
|
|
36
|
+
const TAG_SESS_RE = /(?:^|\s)sess=([^\s]+)(?:\s|$)/;
|
|
37
|
+
|
|
38
|
+
// ── 小工具 ─────────────────────────────────────────────────────────────────
|
|
39
|
+
|
|
40
|
+
const textHash = (text) => crypto.createHash('sha256').update(String(text ?? ''), 'utf8').digest('hex');
|
|
41
|
+
|
|
42
|
+
/** 一批一个来源键:同一个 runId 永远映射到同一个键 ⇒ 幂等 + 整批撤回。 */
|
|
43
|
+
export function runKeyOf(runId) {
|
|
44
|
+
return `d${textHash(`session-distill:${runId}`).slice(0, 16)}`;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** 一轮的 runId 缺省值:本地时间戳 + 6 位随机,肉眼可读、也不撞车。 */
|
|
48
|
+
export function makeRunId(now = Date.now()) {
|
|
49
|
+
const stamp = new Date(Number(now)).toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
|
50
|
+
return `run-${stamp}-${crypto.randomBytes(3).toString('hex')}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** 估算一次调用的 token(保守:提示按 1.4 字/token,输出按每条 120 token 留余量)。 */
|
|
54
|
+
export function estimateSessionTokens(promptChars, limit = DEFAULT_LIMIT) {
|
|
55
|
+
const promptTokens = Math.ceil(Number(promptChars) / 1.4);
|
|
56
|
+
const completionTokens = Math.max(240, Number(limit) * 120);
|
|
57
|
+
return promptTokens + completionTokens;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ── 台账(--state) ────────────────────────────────────────────────────────
|
|
61
|
+
|
|
62
|
+
export function blankState(nowIso) {
|
|
63
|
+
return {
|
|
64
|
+
schemaVersion: STATE_SCHEMA,
|
|
65
|
+
budget: { tokensPerDay: null, source: null, updatedAt: nowIso },
|
|
66
|
+
cost: { date: localDateIso(Date.parse(nowIso) || Date.now()), calls: 0, promptTokens: 0, completionTokens: 0 },
|
|
67
|
+
processed: {},
|
|
68
|
+
runs: [],
|
|
69
|
+
updatedAt: nowIso,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function readState(stateFile, nowIso = new Date().toISOString()) {
|
|
74
|
+
if (!stateFile || !fs.existsSync(stateFile)) return blankState(nowIso);
|
|
75
|
+
try {
|
|
76
|
+
const parsed = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
|
|
77
|
+
const base = blankState(nowIso);
|
|
78
|
+
return {
|
|
79
|
+
...base,
|
|
80
|
+
...parsed,
|
|
81
|
+
budget: { ...base.budget, ...(parsed.budget ?? {}) },
|
|
82
|
+
cost: { ...base.cost, ...(parsed.cost ?? {}) },
|
|
83
|
+
processed: { ...(parsed.processed ?? {}) },
|
|
84
|
+
runs: Array.isArray(parsed.runs) ? parsed.runs : [],
|
|
85
|
+
};
|
|
86
|
+
} catch (failure) {
|
|
87
|
+
throw new Error(`台账读不动(${stateFile}):${failure && failure.message ? failure.message : failure}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** 原子写:同目录临时文件 + rename(同盘,不动跨盘那条路)。 */
|
|
92
|
+
export function writeState(stateFile, state) {
|
|
93
|
+
const abs = path.resolve(stateFile);
|
|
94
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
95
|
+
const tmp = `${abs}.tmp-${process.pid}`;
|
|
96
|
+
fs.writeFileSync(tmp, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
|
|
97
|
+
fs.renameSync(tmp, abs);
|
|
98
|
+
return abs;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** 当日已花 token(跨天自动清零,但不清 processed —— 那是"这条来源句做过了")。 */
|
|
102
|
+
export function spendOf(state, now = Date.now()) {
|
|
103
|
+
const today = localDateIso(Number(now));
|
|
104
|
+
if (String(state?.cost?.date ?? '') !== today) return 0;
|
|
105
|
+
return Number(state?.cost?.promptTokens ?? 0) + Number(state?.cost?.completionTokens ?? 0);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* 预算闸:先看命令行给的,再看台账里配的;都没有 ⇒ tokens=null(这就是"没配预算")。
|
|
110
|
+
* ⚠️ 绝不回落到内核那个 150000 —— 没配就是没配,不是"无限"。
|
|
111
|
+
*/
|
|
112
|
+
export function resolveBudget({ flagTokens = null, state = null, now = Date.now() } = {}) {
|
|
113
|
+
const pick = (value) => {
|
|
114
|
+
const n = Number(value);
|
|
115
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
116
|
+
};
|
|
117
|
+
const fromFlag = pick(flagTokens);
|
|
118
|
+
const fromState = pick(state?.budget?.tokensPerDay);
|
|
119
|
+
const tokens = fromFlag ?? fromState;
|
|
120
|
+
const spent = spendOf(state, now);
|
|
121
|
+
return {
|
|
122
|
+
tokens,
|
|
123
|
+
source: fromFlag !== null ? 'cli' : (fromState !== null ? 'state' : null),
|
|
124
|
+
day: localDateIso(Number(now)),
|
|
125
|
+
spent,
|
|
126
|
+
remaining: tokens === null ? null : Math.max(0, tokens - spent),
|
|
127
|
+
exhausted: tokens === null ? false : spent >= tokens,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ── 提示词与模型调用 ───────────────────────────────────────────────────────
|
|
132
|
+
|
|
133
|
+
export const SESSION_SYSTEM_RULES = Object.freeze([
|
|
134
|
+
'你在做「会话日志 → 长期记忆」的提炼。输入是一场会话里挑出来的候选句(编号 + 线索类别 + 原文)。',
|
|
135
|
+
'任务:从中挑出真正值得长期记住的,合并成若干条独立结论。',
|
|
136
|
+
'硬约束:',
|
|
137
|
+
'· 只能使用候选句里已经写明的信息;不许补常识、不许推断动机、不许加评价。',
|
|
138
|
+
'· 每条结论必须报出它引用了哪几条候选(refs 写候选编号,从 1 开始)。',
|
|
139
|
+
'· 同一件事的候选合并成一条;不是同一件事就分开写,不要硬凑。',
|
|
140
|
+
'· 结论是中文一整段,不超过 120 字,不分点、不带 markdown;不要出现「今天/昨天/刚才/本次」这类相对时间。',
|
|
141
|
+
'· 候选里挑不出值得长期记住的 ⇒ items 给空数组(宁缺勿滥)。',
|
|
142
|
+
'输出:严格 JSON,形如 {"items":[{"refs":[2,5],"conclusion":"……"}]},按重要性降序;',
|
|
143
|
+
'不要代码围栏、不要解释文字、不要输出候选编号以外的字段。',
|
|
144
|
+
]);
|
|
145
|
+
|
|
146
|
+
export function buildSessionSystemPrompt() {
|
|
147
|
+
return SESSION_SYSTEM_RULES.join('\n');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function buildSessionUserPrompt(candidates, limit = DEFAULT_LIMIT) {
|
|
151
|
+
const blocks = candidates.map((candidate, index) => {
|
|
152
|
+
const body = String(candidate.text).replace(/\s+/g, ' ').trim();
|
|
153
|
+
return `[${index + 1}] (${candidate.kind}) ${body}`;
|
|
154
|
+
});
|
|
155
|
+
return [
|
|
156
|
+
`共 ${candidates.length} 条候选,最多输出 ${limit} 条结论。`,
|
|
157
|
+
'每条结论都要用 refs 指明依据的候选编号(可多条)。',
|
|
158
|
+
'',
|
|
159
|
+
blocks.join('\n'),
|
|
160
|
+
].join('\n');
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* 调一次提炼模型。`fetchImpl` 可注入 ⇒ 单测不联网。
|
|
165
|
+
* 只调一次、不重试:重试等于偷花第二份钱。
|
|
166
|
+
*/
|
|
167
|
+
export async function callSessionModel({
|
|
168
|
+
candidates, limit = DEFAULT_LIMIT, apiKey, baseUrl = DEFAULT_LLM.baseUrl,
|
|
169
|
+
model = DEFAULT_LLM.model, fetchImpl = fetch, timeoutMs = DEFAULT_LLM.timeoutMs,
|
|
170
|
+
maxTokens = DEFAULT_LLM.maxTokens, temperature = 0.2,
|
|
171
|
+
} = {}) {
|
|
172
|
+
const payload = {
|
|
173
|
+
model,
|
|
174
|
+
messages: [
|
|
175
|
+
{ role: 'system', content: buildSessionSystemPrompt() },
|
|
176
|
+
{ role: 'user', content: buildSessionUserPrompt(candidates, limit) },
|
|
177
|
+
],
|
|
178
|
+
thinking: { type: 'disabled' },
|
|
179
|
+
temperature,
|
|
180
|
+
max_tokens: maxTokens,
|
|
181
|
+
stream: false,
|
|
182
|
+
};
|
|
183
|
+
const controller = new AbortController();
|
|
184
|
+
const timer = setTimeout(() => controller.abort(), Number(timeoutMs));
|
|
185
|
+
const startedAt = Date.now();
|
|
186
|
+
try {
|
|
187
|
+
const response = await fetchImpl(`${baseUrl}/chat/completions`, {
|
|
188
|
+
method: 'POST',
|
|
189
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` },
|
|
190
|
+
body: JSON.stringify(payload),
|
|
191
|
+
signal: controller.signal,
|
|
192
|
+
});
|
|
193
|
+
const raw = await response.text();
|
|
194
|
+
const ms = Date.now() - startedAt;
|
|
195
|
+
if (!response.ok) return { ok: false, error: `HTTP ${response.status}:${raw.slice(0, 240)}`, ms };
|
|
196
|
+
let envelope;
|
|
197
|
+
try {
|
|
198
|
+
envelope = JSON.parse(raw);
|
|
199
|
+
} catch (failure) {
|
|
200
|
+
return { ok: false, error: `响应不是 JSON:${failure.message}`, raw: raw.slice(0, 240), ms };
|
|
201
|
+
}
|
|
202
|
+
const content = envelope?.choices?.[0]?.message?.content ?? '';
|
|
203
|
+
const usage = envelope?.usage ?? {};
|
|
204
|
+
const parsed = extractJson(content);
|
|
205
|
+
if (!parsed) return { ok: false, error: '模型输出里没有可解析的 JSON', raw: String(content).slice(0, 240), usage, ms };
|
|
206
|
+
const items = Array.isArray(parsed) ? parsed : (Array.isArray(parsed.items) ? parsed.items : null);
|
|
207
|
+
if (!items) return { ok: false, error: 'JSON 里没有 items 数组', raw: String(content).slice(0, 240), usage, ms };
|
|
208
|
+
return { ok: true, items, usage, ms, promptChars: buildSessionSystemPrompt().length + buildSessionUserPrompt(candidates, limit).length };
|
|
209
|
+
} catch (failure) {
|
|
210
|
+
return { ok: false, error: `请求失败:${failure && failure.message ? failure.message : failure}`, ms: Date.now() - startedAt };
|
|
211
|
+
} finally {
|
|
212
|
+
clearTimeout(timer);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* 收口模型输出:refs 落在 1..candidates.length 之外的、conclusion 太短的、撞车的全部剔掉,
|
|
218
|
+
* 并把每条结论挂回候选的 sourceRef(这样"能回指"不靠模型自觉)。
|
|
219
|
+
*/
|
|
220
|
+
export function attachSourceRefs(items, candidates, { limit = DEFAULT_LIMIT, minChars = 8 } = {}) {
|
|
221
|
+
const kept = [];
|
|
222
|
+
const rejected = [];
|
|
223
|
+
const seen = new Set();
|
|
224
|
+
for (const item of items ?? []) {
|
|
225
|
+
const text = String(item?.conclusion ?? '').trim();
|
|
226
|
+
const refs = Array.isArray(item?.refs) ? item.refs.map((n) => Number(n)).filter((n) => Number.isInteger(n)) : [];
|
|
227
|
+
const valid = refs.filter((n) => n >= 1 && n <= candidates.length);
|
|
228
|
+
if (!text || text.length < minChars) {
|
|
229
|
+
rejected.push({ why: 'too-short-or-empty', text: text.slice(0, 40) });
|
|
230
|
+
continue;
|
|
231
|
+
}
|
|
232
|
+
if (!valid.length) {
|
|
233
|
+
rejected.push({ why: 'no-valid-ref', text: text.slice(0, 40) });
|
|
234
|
+
continue;
|
|
235
|
+
}
|
|
236
|
+
if (text.length > 120) {
|
|
237
|
+
rejected.push({ why: 'over-120-chars', text: text.slice(0, 40) });
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
const fingerprint = text.replace(/\s+/g, '');
|
|
241
|
+
if (seen.has(fingerprint)) {
|
|
242
|
+
rejected.push({ why: 'duplicate', text: text.slice(0, 40) });
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
seen.add(fingerprint);
|
|
246
|
+
if (kept.length >= limit) {
|
|
247
|
+
rejected.push({ why: 'over-limit', text: text.slice(0, 40) });
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
const refsOut = valid.map((n) => ({ index: n, kind: candidates[n - 1].kind, sourceRef: candidates[n - 1].sourceRef }));
|
|
251
|
+
kept.push({
|
|
252
|
+
conclusion: text,
|
|
253
|
+
chars: text.length,
|
|
254
|
+
refs: valid,
|
|
255
|
+
candidateIndices: valid.map((n) => n - 1),
|
|
256
|
+
lines: [...new Set(refsOut.map((ref) => ref.sourceRef.line))].sort((a, b) => a - b),
|
|
257
|
+
sessionIds: [...new Set(refsOut.map((ref) => ref.sourceRef.sessionId))],
|
|
258
|
+
anchors: refsOut.map((ref) => ({ line: ref.sourceRef.line, anchor: ref.sourceRef.anchor })),
|
|
259
|
+
sources: refsOut,
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
return { items: kept, rejected };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// ── 写库 ───────────────────────────────────────────────────────────────────
|
|
266
|
+
|
|
267
|
+
function hasOwnerColumns(handle) {
|
|
268
|
+
try {
|
|
269
|
+
const columns = handle.prepare('PRAGMA table_info(memories);').all().map((row) => row.name);
|
|
270
|
+
return columns.includes('owner_user') && columns.includes('visibility');
|
|
271
|
+
} catch {
|
|
272
|
+
return false;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** 这一批的标签:内核那四个属性位 + 场次 + 批次。 */
|
|
277
|
+
export function sessionTagOf({ srcKey, lines, sessionId, runId, now }) {
|
|
278
|
+
const base = distilledTagOf({ srcKey: shortKey(srcKey), srcIds: lines ?? [], now });
|
|
279
|
+
const sess = sessionId && sessionId !== 'unknown' ? ` sess=${sessionId}` : '';
|
|
280
|
+
return `${base}${sess} ${RUN_TAG_PREFIX}${runId}`;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export function tagsOfRun(tags) {
|
|
284
|
+
const raw = String(tags ?? '');
|
|
285
|
+
if (!raw.startsWith(DISTILL_PREFIX)) return null;
|
|
286
|
+
const run = TAG_RUN_RE.exec(raw);
|
|
287
|
+
const sess = TAG_SESS_RE.exec(raw);
|
|
288
|
+
return { runId: run ? run[1] : null, sessionId: sess ? sess[1] : null };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* 写回一批(幂等:同一 runId 已写过就跳过;同正文已存在也跳过)。
|
|
293
|
+
* 事务里逐条插,库被占用按内核那套退避重试。
|
|
294
|
+
*/
|
|
295
|
+
export async function writeSessionItems({ dbPath, items, runId, srcKey, now = Date.now(), ownerUser = null, visibility = 'private_owner', deps = {} } = {}) {
|
|
296
|
+
const stamp = Number(now);
|
|
297
|
+
const handle = new DatabaseSync(dbPath);
|
|
298
|
+
const ids = [];
|
|
299
|
+
const details = [];
|
|
300
|
+
let written = 0;
|
|
301
|
+
let skippedExisting = 0;
|
|
302
|
+
try {
|
|
303
|
+
handle.exec('PRAGMA busy_timeout = 5000;');
|
|
304
|
+
const withOwner = hasOwnerColumns(handle);
|
|
305
|
+
const existing = handle.prepare("SELECT id, text, tags FROM memories WHERE tags LIKE 'distilled%';").all();
|
|
306
|
+
const sameRun = new Set();
|
|
307
|
+
const seenText = new Set();
|
|
308
|
+
for (const row of existing) {
|
|
309
|
+
const meta = tagsOfRun(row.tags);
|
|
310
|
+
if (meta && meta.runId === runId) sameRun.add(row.id);
|
|
311
|
+
seenText.add(textHash(row.text));
|
|
312
|
+
}
|
|
313
|
+
const insert = withOwner
|
|
314
|
+
? handle.prepare('INSERT INTO memories (text, tags, pinned, created_at, updated_at, owner_user, visibility) VALUES (?,?,0,?,?,?,?);')
|
|
315
|
+
: handle.prepare('INSERT INTO memories (text, tags, pinned, created_at, updated_at) VALUES (?,?,0,?,?);');
|
|
316
|
+
|
|
317
|
+
const outcome = await withBusyRetry(handle, () => {
|
|
318
|
+
for (const item of items) {
|
|
319
|
+
const digest = textHash(item.conclusion);
|
|
320
|
+
if (seenText.has(digest)) {
|
|
321
|
+
skippedExisting += 1;
|
|
322
|
+
details.push({ action: 'already-in-db', chars: item.chars });
|
|
323
|
+
continue;
|
|
324
|
+
}
|
|
325
|
+
const tags = sessionTagOf({ srcKey, lines: item.lines, sessionId: item.sessionIds?.[0], runId, now: stamp });
|
|
326
|
+
const info = withOwner
|
|
327
|
+
? insert.run(item.conclusion, tags, stamp, stamp, ownerUser ?? 'local', visibility)
|
|
328
|
+
: insert.run(item.conclusion, tags, stamp, stamp);
|
|
329
|
+
const newId = Number(info.lastInsertRowid ?? 0);
|
|
330
|
+
ids.push(newId);
|
|
331
|
+
written += 1;
|
|
332
|
+
seenText.add(digest);
|
|
333
|
+
details.push({ action: 'written', id: newId, chars: item.chars, lines: item.lines, tags });
|
|
334
|
+
}
|
|
335
|
+
return { written, skippedExisting };
|
|
336
|
+
}, deps);
|
|
337
|
+
|
|
338
|
+
return { written, skippedExisting, ids, details, ok: outcome.ok, error: outcome.error ?? null, sameRunRows: sameRun.size, ownerColumns: withOwner };
|
|
339
|
+
} finally {
|
|
340
|
+
try { handle.close(); } catch { /* 关不掉不影响结果 */ }
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// ── 主流程 ─────────────────────────────────────────────────────────────────
|
|
345
|
+
|
|
346
|
+
const uniq = (list) => [...new Set(list.filter(Boolean))];
|
|
347
|
+
|
|
348
|
+
/**
|
|
349
|
+
* 跑一轮「会话 → 记忆」。缺省 dry-run:只抽、只规划,不调模型、不写库、不写台账。
|
|
350
|
+
*
|
|
351
|
+
* @returns {object} 结果对象(含 stopping 与人话 message),CLI 只负责把它讲出来。
|
|
352
|
+
*/
|
|
353
|
+
export async function runSessionDistill(opts = {}) {
|
|
354
|
+
const now = Number(opts.now ?? Date.now());
|
|
355
|
+
const nowIso = new Date(now).toISOString();
|
|
356
|
+
const mode = opts.apply ? 'apply' : 'dry-run';
|
|
357
|
+
const limit = Number.isFinite(Number(opts.limit)) && Number(opts.limit) >= 1 ? Number(opts.limit) : DEFAULT_LIMIT;
|
|
358
|
+
const runId = String(opts.runId ?? makeRunId(now));
|
|
359
|
+
const srcKey = runKeyOf(runId);
|
|
360
|
+
const stateFile = opts.stateFile ? path.resolve(opts.stateFile) : null;
|
|
361
|
+
const warnings = [];
|
|
362
|
+
const result = {
|
|
363
|
+
ok: true, mode, at: nowIso, runId, srcKey: shortKey(srcKey), source: null, extraction: null,
|
|
364
|
+
budget: null, stopping: null, message: null, calls: 0, tokens: null, planned: [], model: null,
|
|
365
|
+
written: 0, writtenIds: [], snapshot: null, stateFile, warnings, exitCode: 0,
|
|
366
|
+
};
|
|
367
|
+
|
|
368
|
+
// ① 抽句(纯本地,不花钱)
|
|
369
|
+
let extraction;
|
|
370
|
+
try {
|
|
371
|
+
extraction = extractCandidates({
|
|
372
|
+
file: opts.source,
|
|
373
|
+
text: opts.text,
|
|
374
|
+
sessionId: opts.sessionId,
|
|
375
|
+
defaults: { maxChars: opts.maxChars ?? undefined },
|
|
376
|
+
});
|
|
377
|
+
} catch (failure) {
|
|
378
|
+
result.ok = false;
|
|
379
|
+
result.exitCode = failure?.code === 'SOURCE_MISSING' ? 3 : 3;
|
|
380
|
+
result.stopping = 'SOURCE_UNREADABLE';
|
|
381
|
+
result.message = String(failure?.message ?? failure);
|
|
382
|
+
return result;
|
|
383
|
+
}
|
|
384
|
+
result.source = { file: extraction.file, sessionId: extraction.sessionId, compressed: extraction.compressed, note: extraction.note };
|
|
385
|
+
result.extraction = {
|
|
386
|
+
lines: extraction.lines, events: extraction.events, candidates: extraction.candidates.length,
|
|
387
|
+
skipped: extraction.skipped, skippedLine: summarizeSkipped(extraction.skipped),
|
|
388
|
+
};
|
|
389
|
+
result.warnings.push(...extraction.warnings);
|
|
390
|
+
if (extraction.note) result.warnings.push(extraction.note);
|
|
391
|
+
const candidates = extraction.candidates;
|
|
392
|
+
|
|
393
|
+
if (!candidates.length) {
|
|
394
|
+
result.stopping = 'NO_CANDIDATES';
|
|
395
|
+
result.message = '这份日志里没有抽到能回指来源的结论句 ⇒ 不调模型、不写库(抽句统计见 skipped)';
|
|
396
|
+
return result;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// ② 预算闸 —— 必须在读凭证、调模型之前
|
|
400
|
+
const state = stateFile ? readState(stateFile, nowIso) : blankState(nowIso);
|
|
401
|
+
const budget = resolveBudget({ flagTokens: opts.maxTokens, state, now });
|
|
402
|
+
result.budget = budget;
|
|
403
|
+
if (budget.tokens === null) {
|
|
404
|
+
result.stopping = 'NO_BUDGET';
|
|
405
|
+
result.message = NO_BUDGET_MESSAGE;
|
|
406
|
+
return result;
|
|
407
|
+
}
|
|
408
|
+
if (budget.exhausted) {
|
|
409
|
+
result.stopping = 'BUDGET_SPENT';
|
|
410
|
+
result.message = `今日已用 ${budget.spent} / 预算 ${budget.tokens} token ⇒ 本轮不跑(等明天或调高预算)`;
|
|
411
|
+
return result;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// ③ 去重:台账记过的来源句不再产(同一句反复进模型 = 重复花钱)
|
|
415
|
+
const pending = candidates.filter((candidate) => {
|
|
416
|
+
const key = processedKey(extraction.file, extraction.sessionId, candidate);
|
|
417
|
+
return !state.processed?.[key];
|
|
418
|
+
});
|
|
419
|
+
const alreadyProcessed = candidates.length - pending.length;
|
|
420
|
+
if (alreadyProcessed) warnings.push(`${alreadyProcessed} 条候选在台账里已产出过(按「文件 + 行号 + 句子哈希」认),本批跳过`);
|
|
421
|
+
if (!pending.length) {
|
|
422
|
+
result.stopping = 'ALL_PROCESSED';
|
|
423
|
+
result.message = '候选句都产出过 ⇒ 本轮无事可做(不调模型)';
|
|
424
|
+
return result;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// ④ 估算过闸:这一次调用会超预算就干脆不调
|
|
428
|
+
const promptChars = buildSessionSystemPrompt().length + buildSessionUserPrompt(pending, limit).length;
|
|
429
|
+
const estimated = estimateSessionTokens(promptChars, limit);
|
|
430
|
+
result.tokens = { estimated, promptChars, reservedCompletion: Math.max(240, limit * 120) };
|
|
431
|
+
if (estimated > budget.remaining) {
|
|
432
|
+
result.stopping = 'ESTIMATE_OVER';
|
|
433
|
+
result.message = `本轮估算 ${estimated} token > 今日剩余 ${budget.remaining}(已用 ${budget.spent}/${budget.tokens})⇒ 不跑`;
|
|
434
|
+
return result;
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
// ⑤ dry-run 到此为止:把"会写什么"讲清楚,一个字节都不动
|
|
438
|
+
result.planned = pending.slice(0, limit * 2).map((candidate, index) => ({
|
|
439
|
+
index: index + 1, kind: candidate.kind, chars: candidate.chars,
|
|
440
|
+
text: candidate.text, anchor: candidate.sourceRef.anchor, line: candidate.sourceRef.line,
|
|
441
|
+
}));
|
|
442
|
+
if (!opts.apply) {
|
|
443
|
+
result.message = `dry-run:抽到 ${candidates.length} 条候选、待产 ${pending.length} 条(最多写 ${limit} 条);`
|
|
444
|
+
+ `估算 ${estimated} token / 今日剩余 ${budget.remaining},库与台账都没动`;
|
|
445
|
+
return result;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// ⑥ 凭证
|
|
449
|
+
const apiKey = opts.apiKey || readCredential(DEFAULT_LLM.credentialKey);
|
|
450
|
+
if (!apiKey) {
|
|
451
|
+
result.ok = false;
|
|
452
|
+
result.exitCode = 1;
|
|
453
|
+
result.stopping = 'NO_CREDENTIAL';
|
|
454
|
+
result.message = `读不到凭证 ${DEFAULT_LLM.credentialKey}(.credentials.yaml)`;
|
|
455
|
+
return result;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// ⑦ 写库前快照(唯一回滚点,没拍成就别写)
|
|
459
|
+
const snapshotFile = opts.snapshotFile
|
|
460
|
+
?? path.join(opts.snapshotDir ?? path.join(path.dirname(opts.dbPath ?? ''), 'snapshots'), `memory-before-session-distill-${runId}.db`);
|
|
461
|
+
try {
|
|
462
|
+
vacuumInto(opts.dbPath, snapshotFile);
|
|
463
|
+
result.snapshot = snapshotFile;
|
|
464
|
+
} catch (failure) {
|
|
465
|
+
result.ok = false;
|
|
466
|
+
result.exitCode = 1;
|
|
467
|
+
result.stopping = 'SNAPSHOT_FAILED';
|
|
468
|
+
result.message = `写库前快照失败 ⇒ 按规矩不写库:${failure && failure.message ? failure.message : failure}`;
|
|
469
|
+
return result;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
// ⑧ 调一次模型
|
|
473
|
+
const call = await callSessionModel({
|
|
474
|
+
candidates: pending, limit, apiKey,
|
|
475
|
+
baseUrl: opts.baseUrl ?? DEFAULT_LLM.baseUrl,
|
|
476
|
+
model: opts.model ?? DEFAULT_LLM.model,
|
|
477
|
+
fetchImpl: opts.fetchImpl ?? fetch,
|
|
478
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_LLM.timeoutMs,
|
|
479
|
+
maxTokens: opts.modelMaxTokens ?? DEFAULT_LLM.maxTokens,
|
|
480
|
+
});
|
|
481
|
+
result.calls = 1;
|
|
482
|
+
const usage = call.usage ?? {};
|
|
483
|
+
const promptTokens = Number(usage.prompt_tokens ?? 0) || Math.ceil(promptChars / 1.4);
|
|
484
|
+
const completionTokens = Number(usage.completion_tokens ?? 0) || Math.max(0, estimated - Math.ceil(promptChars / 1.4));
|
|
485
|
+
result.model = { ok: Boolean(call.ok), error: call.error ?? null, ms: call.ms ?? null, promptTokens, completionTokens, model: opts.model ?? DEFAULT_LLM.model };
|
|
486
|
+
|
|
487
|
+
// 无论成败都记账(失败也花了 token,装看不见等于浪费下一天的额度)
|
|
488
|
+
const charged = {
|
|
489
|
+
date: budget.day,
|
|
490
|
+
calls: 1,
|
|
491
|
+
promptTokens,
|
|
492
|
+
completionTokens,
|
|
493
|
+
};
|
|
494
|
+
state.cost = mergeCost(state, budget, charged);
|
|
495
|
+
|
|
496
|
+
if (!call.ok) {
|
|
497
|
+
result.ok = false;
|
|
498
|
+
result.exitCode = 1;
|
|
499
|
+
result.stopping = 'MODEL_FAILED';
|
|
500
|
+
result.message = `模型调用失败(已记账 ${promptTokens + completionTokens} token):${call.error}`;
|
|
501
|
+
persistRun({ state, stateFile, result, runId, nowIso, mode, extraction, pending, limit, budget, now });
|
|
502
|
+
return result;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// ⑨ 收口 + 写库
|
|
506
|
+
const { items, rejected } = attachSourceRefs(call.items, pending, { limit });
|
|
507
|
+
result.rejected = rejected;
|
|
508
|
+
if (!items.length) {
|
|
509
|
+
result.stopping = 'NOTHING_KEPT';
|
|
510
|
+
result.message = '模型给回来的条目没有一条能挂回候选(或都不值得留)⇒ 不写库(已花的 token 已记账)';
|
|
511
|
+
persistRun({ state, stateFile, result, runId, nowIso, mode, extraction, pending, limit, budget, now });
|
|
512
|
+
return result;
|
|
513
|
+
}
|
|
514
|
+
const write = await writeSessionItems({
|
|
515
|
+
dbPath: opts.dbPath, items, runId, srcKey, now,
|
|
516
|
+
ownerUser: opts.ownerUser ?? null, visibility: opts.visibility ?? 'private_owner', deps: opts.deps ?? {},
|
|
517
|
+
});
|
|
518
|
+
result.ownerColumns = write.ownerColumns;
|
|
519
|
+
if (write.ownerColumns && !opts.ownerUser) warnings.push('库有 owner_user 列但没给 --owner-user ⇒ 按 local 落库(回指仍然靠标签)');
|
|
520
|
+
if (!write.ok) {
|
|
521
|
+
result.ok = false;
|
|
522
|
+
result.exitCode = 1;
|
|
523
|
+
result.stopping = 'WRITE_FAILED';
|
|
524
|
+
result.message = `写库失败(快照在 ${snapshotFile},可整批撤回):${write.error}`;
|
|
525
|
+
persistRun({ state, stateFile, result, runId, nowIso, mode, extraction, pending, limit, budget, now });
|
|
526
|
+
return result;
|
|
527
|
+
}
|
|
528
|
+
result.written = write.written;
|
|
529
|
+
result.writtenIds = write.ids;
|
|
530
|
+
result.skippedExisting = write.skippedExisting;
|
|
531
|
+
result.planned = items.map((item, index) => ({
|
|
532
|
+
index: index + 1, chars: item.chars, lines: item.lines, anchors: item.anchors, conclusion: item.conclusion, refs: item.refs,
|
|
533
|
+
}));
|
|
534
|
+
|
|
535
|
+
// ⑩ 台账:已产出的来源句 + 本轮摘要
|
|
536
|
+
for (const item of items) {
|
|
537
|
+
for (const index of item.candidateIndices ?? []) {
|
|
538
|
+
const candidate = pending[index];
|
|
539
|
+
if (candidate) state.processed[processedKey(extraction.file, extraction.sessionId, candidate)] = nowIso;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
result.message = `已写 ${write.written} 条(跳过已存在 ${write.skippedExisting} 条),批次键 ${result.srcKey} ⇒ 撤回:`
|
|
543
|
+
+ `node memory-engine/cli/distill-rollback.mjs --db <memory.db> --key ${result.srcKey} --apply`;
|
|
544
|
+
persistRun({ state, stateFile, result, runId, nowIso, mode, extraction, pending, limit, budget, now });
|
|
545
|
+
return result;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** 台账里"这条来源句做过了"的键:文件 + 行号 + 句子哈希(换文件/换行都不会误判)。 */
|
|
549
|
+
export function processedKey(file, sessionId, candidate) {
|
|
550
|
+
const where = file ? path.basename(file) : (sessionId ?? 'unknown');
|
|
551
|
+
return `${where}:${candidate?.sourceRef?.line ?? 0}:${textHash(candidate?.text ?? '').slice(0, 12)}`;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function mergeCost(state, budget, charged) {
|
|
555
|
+
const sameDay = String(state?.cost?.date ?? '') === budget.day;
|
|
556
|
+
return {
|
|
557
|
+
date: budget.day,
|
|
558
|
+
calls: (sameDay ? Number(state.cost?.calls ?? 0) : 0) + charged.calls,
|
|
559
|
+
promptTokens: (sameDay ? Number(state.cost?.promptTokens ?? 0) : 0) + charged.promptTokens,
|
|
560
|
+
completionTokens: (sameDay ? Number(state.cost?.completionTokens ?? 0) : 0) + charged.completionTokens,
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
function persistRun({ state, stateFile, result, runId, nowIso, mode, extraction, pending, limit, budget, now }) {
|
|
565
|
+
if (!stateFile) return;
|
|
566
|
+
if (Number(budget?.tokens ?? 0) > 0 && budget.source === 'cli') {
|
|
567
|
+
state.budget = { tokensPerDay: budget.tokens, source: 'cli', updatedAt: nowIso };
|
|
568
|
+
}
|
|
569
|
+
state.lastRun = {
|
|
570
|
+
runId, at: nowIso, mode, srcKey: result.srcKey,
|
|
571
|
+
source: { file: extraction.file, sessionId: extraction.sessionId, candidates: pending.length },
|
|
572
|
+
limit, budget: { tokens: budget.tokens, spent: budget.spent },
|
|
573
|
+
calls: result.calls, tokens: result.model ? result.model.promptTokens + result.model.completionTokens : 0,
|
|
574
|
+
written: result.written, writtenIds: result.writtenIds, stopping: result.stopping, snapshot: result.snapshot,
|
|
575
|
+
};
|
|
576
|
+
state.runs = [...(state.runs ?? []), {
|
|
577
|
+
runId, at: nowIso, mode, written: result.written, calls: result.calls,
|
|
578
|
+
promptTokens: result.model?.promptTokens ?? 0, completionTokens: result.model?.completionTokens ?? 0,
|
|
579
|
+
stopping: result.stopping,
|
|
580
|
+
}].slice(-20);
|
|
581
|
+
state.updatedAt = nowIso;
|
|
582
|
+
writeState(stateFile, state);
|
|
583
|
+
result.stateWritten = stateFile;
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
/** 已写过哪些批次(给"这个 run 写过没"这类问题用;只读)。 */
|
|
587
|
+
export function listSessionRuns(dbPath) {
|
|
588
|
+
const handle = new DatabaseSync(dbPath, { readOnly: true });
|
|
589
|
+
try {
|
|
590
|
+
const rows = handle.prepare("SELECT tags FROM memories WHERE tags LIKE 'distilled%';").all();
|
|
591
|
+
const runs = new Map();
|
|
592
|
+
for (const row of rows) {
|
|
593
|
+
const meta = tagsOfRun(row.tags);
|
|
594
|
+
if (!meta?.runId) continue;
|
|
595
|
+
runs.set(meta.runId, (runs.get(meta.runId) ?? 0) + 1);
|
|
596
|
+
}
|
|
597
|
+
return [...runs.entries()].map(([runId, rows2]) => ({ runId, rows: rows2 }));
|
|
598
|
+
} finally {
|
|
599
|
+
try { handle.close(); } catch { /* ignore */ }
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
|