@ilaomianyang/dsh-memory-graph 0.10.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.en.md +24 -3
- package/README.md +24 -3
- package/client.js +1 -1
- package/package.json +5 -3
- package/src/auto.js +10 -7
- package/src/capture.js +37 -175
- package/src/distillapi.js +4 -1
- package/src/distillrun.js +7 -3
- package/src/extract.js +42 -18
- package/src/graph.js +10 -5
- package/src/graphdata.js +10 -1
- package/src/history.js +440 -0
- package/src/pageapi.js +2 -2
- package/src/tools.js +52 -1
- package/src/turns.js +265 -0
- package/src/vault.js +20 -10
- package/test/fixtures/session.v4.jsonl.zstd +0 -0
- package/test/history.mjs +249 -0
- package/test/tools.mjs +2 -1
package/src/extract.js
CHANGED
|
@@ -121,7 +121,13 @@ function extractJson(text) {
|
|
|
121
121
|
|
|
122
122
|
export function parseCards(text) {
|
|
123
123
|
const parsed = extractJson(text)
|
|
124
|
-
if (parsed === null)
|
|
124
|
+
if (parsed === null) {
|
|
125
|
+
return {
|
|
126
|
+
cards: [],
|
|
127
|
+
skipped: '模型输出不是 JSON · model output was not JSON',
|
|
128
|
+
raw: String(text ?? '').slice(0, 600),
|
|
129
|
+
}
|
|
130
|
+
}
|
|
125
131
|
const list = Array.isArray(parsed) ? parsed : Array.isArray(parsed.cards) ? parsed.cards : []
|
|
126
132
|
const cards = []
|
|
127
133
|
for (const raw of list) {
|
|
@@ -253,14 +259,16 @@ export function resolveRoute(cfg, selection) {
|
|
|
253
259
|
if (!useCurrentModel) {
|
|
254
260
|
if (!provider || !model) {
|
|
255
261
|
throw new Error(
|
|
256
|
-
'extractor.useCurrentModel is off, so extractor.provider and extractor.model are both required.',
|
|
262
|
+
'关掉了 extractor.useCurrentModel,所以 extractor.provider 和 extractor.model 都必须填 · extractor.useCurrentModel is off, so extractor.provider and extractor.model are both required.',
|
|
257
263
|
)
|
|
258
264
|
}
|
|
259
265
|
return { provider, model, effort: reasoningEffort, source: 'config extractor.provider/model' }
|
|
260
266
|
}
|
|
261
267
|
if (!selection?.provider || !selection?.model) {
|
|
262
268
|
throw new Error(
|
|
263
|
-
'
|
|
269
|
+
'设置 → 模型里没有选中任何模型,所以抽取器没有可借用的路由。' +
|
|
270
|
+
'在那里选一个模型,或者填好 extractor.provider 和 extractor.model 再把 extractor.useCurrentModel 关掉 · ' +
|
|
271
|
+
'No model is selected in Settings → Models, so the extractor has no route to borrow. ' +
|
|
264
272
|
'Pick a model there, or set extractor.provider and extractor.model and turn extractor.useCurrentModel off.',
|
|
265
273
|
)
|
|
266
274
|
}
|
|
@@ -270,7 +278,7 @@ export function resolveRoute(cfg, selection) {
|
|
|
270
278
|
// An explicit setting wins, so a big thinking budget can be reserved for
|
|
271
279
|
// distillation without changing the model the conversation itself uses.
|
|
272
280
|
effort: reasoningEffort || selectionEffort(selection),
|
|
273
|
-
source: 'Settings → Models selection',
|
|
281
|
+
source: '设置 → 模型里选中的路由 · Settings → Models selection',
|
|
274
282
|
}
|
|
275
283
|
}
|
|
276
284
|
|
|
@@ -293,7 +301,7 @@ function splitMessages(messages) {
|
|
|
293
301
|
async function callHarnessModel(cfg, messages, { signal, llm, selection } = {}) {
|
|
294
302
|
if (!llm || typeof llm.prepareCall !== 'function') {
|
|
295
303
|
throw new Error(
|
|
296
|
-
'This harness build exposes no llm service to plugins, so set extractor.transport = "http" to post to an endpoint directly.',
|
|
304
|
+
'这个 harness 版本没有把 llm 服务暴露给插件,所以请把 extractor.transport 设成 "http",直接向接口发请求 · This harness build exposes no llm service to plugins, so set extractor.transport = "http" to post to an endpoint directly.',
|
|
297
305
|
)
|
|
298
306
|
}
|
|
299
307
|
const route = resolveRoute(cfg, selection)
|
|
@@ -339,26 +347,33 @@ async function callHarnessModel(cfg, messages, { signal, llm, selection } = {})
|
|
|
339
347
|
if (controller.signal.aborted) {
|
|
340
348
|
// Either the caller cancelled us or our own timer fired; say which,
|
|
341
349
|
// because the fix is different (wait longer vs. run less work).
|
|
342
|
-
const
|
|
350
|
+
const whyZh = signal?.aborted ? '在路由应答之前就被取消了' : `超时了 ${cfg.extractor.timeoutMs} ms`
|
|
351
|
+
const whyEn = signal?.aborted
|
|
343
352
|
? 'was cancelled before the route answered'
|
|
344
353
|
: `timed out after ${cfg.extractor.timeoutMs} ms`
|
|
345
354
|
throw new Error(
|
|
346
|
-
|
|
355
|
+
`抽取器调用 ${route.provider}/${route.model} 时${whyZh};可以调大 extractor.timeoutMs,或者每次少蒸馏几条情节 · ` +
|
|
356
|
+
`Extractor call to ${route.provider}/${route.model} ${whyEn}; raise extractor.timeoutMs or distill fewer episodes per run.`,
|
|
347
357
|
)
|
|
348
358
|
}
|
|
349
359
|
throw new Error(
|
|
350
|
-
|
|
360
|
+
`抽取器调用 ${route.provider}/${route.model} 失败(${failure.code ?? kind}):${failure.message ?? '路由结束了流但没给原因'} · ` +
|
|
361
|
+
`Extractor call failed on ${route.provider}/${route.model} (${failure.code ?? kind}): ${failure.message ?? 'the route ended the stream without a reason'}`,
|
|
351
362
|
)
|
|
352
363
|
}
|
|
353
364
|
if (kind === 'tool-calls') {
|
|
354
|
-
throw new Error(
|
|
365
|
+
throw new Error(
|
|
366
|
+
'抽取器返回的是工具调用而不是 JSON,蒸馏时换一个模型试试 · Extractor answered with a tool call instead of JSON. Try a different model for distillation.',
|
|
367
|
+
)
|
|
355
368
|
}
|
|
356
369
|
if (kind === 'max-tokens') {
|
|
357
370
|
throw truncated(
|
|
358
|
-
|
|
371
|
+
`抽取器输出撞到了 maxOutputTokens 上限(${cfg.extractor.maxOutputTokens});把它调大,或者每次少蒸馏几条情节 · Extractor output reached maxOutputTokens (${cfg.extractor.maxOutputTokens}); raise it or distill fewer episodes per run.`,
|
|
359
372
|
)
|
|
360
373
|
}
|
|
361
|
-
if (text.trim() === '')
|
|
374
|
+
if (text.trim() === '') {
|
|
375
|
+
throw new Error('抽取器返回了空回复 · Extractor returned an empty completion.')
|
|
376
|
+
}
|
|
362
377
|
return {
|
|
363
378
|
content: text,
|
|
364
379
|
usage,
|
|
@@ -377,7 +392,7 @@ export async function callHttpModel(cfg, messages, { signal } = {}) {
|
|
|
377
392
|
if (key === '') {
|
|
378
393
|
const where = looked.length > 0 ? looked.join(' or ') : 'config extractor.apiKey'
|
|
379
394
|
throw new Error(
|
|
380
|
-
|
|
395
|
+
`抽取器没有可用的 API key,找过这些地方:${where}。请设置该环境变量、把名字加进 harness 的凭据库,或者直接填 config extractor.apiKey · No API key for the extractor. Looked in ${where}. Set the env var, add the name to the harness credential store, or set config extractor.apiKey.`,
|
|
381
396
|
)
|
|
382
397
|
}
|
|
383
398
|
const url = `${cfg.extractor.baseUrl}${cfg.extractor.path}`
|
|
@@ -406,13 +421,17 @@ export async function callHttpModel(cfg, messages, { signal } = {}) {
|
|
|
406
421
|
text = await response.text()
|
|
407
422
|
}
|
|
408
423
|
if (!response.ok) {
|
|
409
|
-
throw new Error(
|
|
424
|
+
throw new Error(
|
|
425
|
+
`抽取器请求 ${url} 返回 HTTP ${response.status}:${text.slice(0, 400)} · Extractor HTTP ${response.status} from ${url}: ${text.slice(0, 400)}`,
|
|
426
|
+
)
|
|
410
427
|
}
|
|
411
428
|
let data
|
|
412
429
|
try {
|
|
413
430
|
data = JSON.parse(text)
|
|
414
431
|
} catch {
|
|
415
|
-
throw new Error(
|
|
432
|
+
throw new Error(
|
|
433
|
+
`抽取器返回的不是 JSON:${text.slice(0, 300)} · Extractor returned non-JSON response: ${text.slice(0, 300)}`,
|
|
434
|
+
)
|
|
416
435
|
}
|
|
417
436
|
const content =
|
|
418
437
|
data?.choices?.[0]?.message?.content ?? data?.choices?.[0]?.text ?? data?.output_text ?? ''
|
|
@@ -420,11 +439,11 @@ export async function callHttpModel(cfg, messages, { signal } = {}) {
|
|
|
420
439
|
// Same situation as the harness route's max-tokens finish: the answer is
|
|
421
440
|
// cut off mid-JSON, so ask for less instead of blaming the parser.
|
|
422
441
|
throw truncated(
|
|
423
|
-
|
|
442
|
+
`抽取器输出撞到了 maxOutputTokens 上限(${cfg.extractor.maxOutputTokens});把它调大,或者每次少蒸馏几条情节 · Extractor output reached maxOutputTokens (${cfg.extractor.maxOutputTokens}); raise it or distill fewer episodes per run.`,
|
|
424
443
|
)
|
|
425
444
|
}
|
|
426
445
|
if (typeof content !== 'string' || content.trim() === '') {
|
|
427
|
-
throw new Error('Extractor returned an empty completion.')
|
|
446
|
+
throw new Error('抽取器返回了空回复 · Extractor returned an empty completion.')
|
|
428
447
|
}
|
|
429
448
|
return { content, usage: data?.usage ?? null, model: data?.model ?? cfg.extractor.model, keySource: source }
|
|
430
449
|
} finally {
|
|
@@ -496,13 +515,18 @@ export async function distill({ cfg, vault, redactor, project, dryRun = false, s
|
|
|
496
515
|
return {
|
|
497
516
|
ok: false,
|
|
498
517
|
reason:
|
|
499
|
-
'Extractor is disabled. Set config extractor.enabled = true; the default transport reuses the model selected in Settings → Models.',
|
|
518
|
+
'抽取器没有启用。请把 config extractor.enabled 设成 true;默认传输方式会复用「设置 → 模型」里选中的模型 · Extractor is disabled. Set config extractor.enabled = true; the default transport reuses the model selected in Settings → Models.',
|
|
500
519
|
}
|
|
501
520
|
}
|
|
502
521
|
|
|
503
522
|
const entries = await vault.readUndistilled(cfg.extractor.maxEpisodesPerRun)
|
|
504
523
|
if (entries.length === 0) {
|
|
505
|
-
return {
|
|
524
|
+
return {
|
|
525
|
+
ok: true,
|
|
526
|
+
episodes: 0,
|
|
527
|
+
written: 0,
|
|
528
|
+
reason: '所有情节都已经蒸馏过了 · Every episode has already been distilled.',
|
|
529
|
+
}
|
|
506
530
|
}
|
|
507
531
|
|
|
508
532
|
const known = await vault.listCards({ includeRetracted: true })
|
package/src/graph.js
CHANGED
|
@@ -143,7 +143,7 @@ function buildModel({ cards, entities, projects }) {
|
|
|
143
143
|
key: `c:${card.id}`,
|
|
144
144
|
type: 'card',
|
|
145
145
|
kind: String(card.kind ?? 'note'),
|
|
146
|
-
label: String(card.title ?? '
|
|
146
|
+
label: String(card.title ?? '(无标题 untitled)'),
|
|
147
147
|
id: card.id,
|
|
148
148
|
project: card.project ?? 'shared',
|
|
149
149
|
status: card.status ?? 'active',
|
|
@@ -843,7 +843,7 @@ const BODY = `
|
|
|
843
843
|
<button id="theme" title="主题:跟随系统 / 深色 / 浅色">◐</button>
|
|
844
844
|
<button id="zoom-in" title="放大">+</button>
|
|
845
845
|
<button id="zoom-out" title="缩小">−</button>
|
|
846
|
-
<button id="zoom-fit" title="复位 (f)"
|
|
846
|
+
<button id="zoom-fit" title="复位 (f)">复位</button>
|
|
847
847
|
</div>
|
|
848
848
|
</div>
|
|
849
849
|
</header>
|
|
@@ -907,6 +907,11 @@ function clientScript(dataJson) {
|
|
|
907
907
|
var selected = null;
|
|
908
908
|
var view = { k: 1, x: 0, y: 0 };
|
|
909
909
|
var GRADIENT_KINDS = ['decision', 'fact', 'open_question', 'preference', 'note', 'entity'];
|
|
910
|
+
var CONFIDENCE_LABELS = {
|
|
911
|
+
user_stated: '用户明说 user_stated',
|
|
912
|
+
verified: '已验证 verified',
|
|
913
|
+
inferred: '推断 inferred'
|
|
914
|
+
};
|
|
910
915
|
|
|
911
916
|
/* ------------------------------------------------------------ theme */
|
|
912
917
|
|
|
@@ -1167,7 +1172,7 @@ function clientScript(dataJson) {
|
|
|
1167
1172
|
if (node.type === 'entity') {
|
|
1168
1173
|
var cards = Object.keys(neighbors[node.key]).filter(function (key) { return key.indexOf('c:') === 0; });
|
|
1169
1174
|
tooltip.appendChild(h('div', 'tt-snip', cards.length + ' 张卡片 · ' + (node.projects.length ? node.projects.join(' / ') : '未归档')));
|
|
1170
|
-
tooltip.appendChild(h('div', 'tt-meta', '
|
|
1175
|
+
tooltip.appendChild(h('div', 'tt-meta', '首次 ' + (node.firstSeen || '—') + ' · 最近 ' + (node.lastSeen || '—')));
|
|
1171
1176
|
} else {
|
|
1172
1177
|
if (node.snippet) tooltip.appendChild(h('div', 'tt-snip', node.snippet));
|
|
1173
1178
|
tooltip.appendChild(h('div', 'tt-meta', node.id + ' · ' + node.project + ' · ' + node.created));
|
|
@@ -1273,7 +1278,7 @@ function clientScript(dataJson) {
|
|
|
1273
1278
|
if (node.status !== 'active') {
|
|
1274
1279
|
badges.appendChild(badge('warn', node.status === 'superseded' ? '已替换 superseded' : '已撤回 retracted', true));
|
|
1275
1280
|
}
|
|
1276
|
-
badges.appendChild(badge('plain', node.confidence, true));
|
|
1281
|
+
badges.appendChild(badge('plain', CONFIDENCE_LABELS[node.confidence] || node.confidence, true));
|
|
1277
1282
|
panel.appendChild(badges);
|
|
1278
1283
|
|
|
1279
1284
|
var meta = h('dl');
|
|
@@ -1560,7 +1565,7 @@ export function renderGraphHtml({ cards = [], entities = [], projects = [], gene
|
|
|
1560
1565
|
generatedAt: generatedAt || stamp(),
|
|
1561
1566
|
})
|
|
1562
1567
|
const dataJson = escapeForScript(JSON.stringify(payload))
|
|
1563
|
-
const title = `记忆图 · ${payload.counts.cards}
|
|
1568
|
+
const title = `记忆图 · ${payload.counts.cards} 张卡片 / ${payload.counts.entities} 个实体`
|
|
1564
1569
|
|
|
1565
1570
|
return `<!doctype html>
|
|
1566
1571
|
<html lang="zh-CN">
|
package/src/graphdata.js
CHANGED
|
@@ -462,7 +462,16 @@ export function createGraphHandler({ getVault, getConfig }) {
|
|
|
462
462
|
return async function handle(request) {
|
|
463
463
|
const url = new URL(request.url)
|
|
464
464
|
const session = url.searchParams.get('session') ?? ''
|
|
465
|
-
if (session === '')
|
|
465
|
+
if (session === '') {
|
|
466
|
+
return json(
|
|
467
|
+
{
|
|
468
|
+
ok: false,
|
|
469
|
+
reason: 'missing-session',
|
|
470
|
+
message: '这个会话没有 id,画不出它的记忆图 · this session has no id, so there is no graph to draw',
|
|
471
|
+
},
|
|
472
|
+
400,
|
|
473
|
+
)
|
|
474
|
+
}
|
|
466
475
|
try {
|
|
467
476
|
const cfg = getConfig()
|
|
468
477
|
const vault = getVault()
|
package/src/history.js
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* History: read the sessions that ran before this plugin existed.
|
|
3
|
+
*
|
|
4
|
+
* Capture only sees turns that arrive while the host is running, so a vault
|
|
5
|
+
* installed today starts empty and the person's past is not in it. The harness
|
|
6
|
+
* keeps every closed session on disk as `session.v4.jsonl.zstd`; this module
|
|
7
|
+
* replays those files through the same turn assembler the live listener uses, so
|
|
8
|
+
* a backfilled episode and a captured one are the same kind of record.
|
|
9
|
+
*
|
|
10
|
+
* Two things make it safe to run twice:
|
|
11
|
+
* - a turn is identified by `(session, turn)`, and turns already in the vault
|
|
12
|
+
* are skipped, so re-running after a session grew imports only the new part;
|
|
13
|
+
* - the timestamps come from the session file, not from now, so history lands
|
|
14
|
+
* in the right place on the timeline instead of piling up under today.
|
|
15
|
+
*/
|
|
16
|
+
import fsp from 'node:fs/promises'
|
|
17
|
+
import os from 'node:os'
|
|
18
|
+
import path from 'node:path'
|
|
19
|
+
import zlib from 'node:zlib'
|
|
20
|
+
import { Decompress } from 'fzstd'
|
|
21
|
+
|
|
22
|
+
import { createTurnAssembler, projectOf } from './turns.js'
|
|
23
|
+
|
|
24
|
+
const SESSION_FILE = 'session.v4.jsonl.zstd'
|
|
25
|
+
/** zstd in, JSON out: a cap so one enormous session cannot pin the process. */
|
|
26
|
+
export const DEFAULT_MAX_BYTES = 512 * 1024 * 1024
|
|
27
|
+
/** No single event is this large; a longer line means the stream is corrupt. */
|
|
28
|
+
const MAX_LINE = 4 * 1024 * 1024
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A session file is not one zstd frame: the writer appends one frame per batch,
|
|
32
|
+
* so the file is a *concatenated container*. Both decoders that matter — Node's
|
|
33
|
+
* and fzstd's — stop at a frame boundary, which is why the raw walk of these
|
|
34
|
+
* files used to yield a single header line and nothing else.
|
|
35
|
+
*
|
|
36
|
+
* Node grew a zstd codec in 22.15 / 23.8 and decodes frames with full fidelity;
|
|
37
|
+
* the pure-JS fallback below refuses some high-accuracy FSE tables that real
|
|
38
|
+
* session files here do contain ("FSE accuracy too high"). A plugin cannot choose
|
|
39
|
+
* the runtime it lands in, so both paths exist and the report names the one that
|
|
40
|
+
* ran.
|
|
41
|
+
*/
|
|
42
|
+
const ZSTD_MAGIC = 0xfd2fb528
|
|
43
|
+
const hasNativeZstd = typeof zlib.zstdDecompressSync === 'function'
|
|
44
|
+
export const decoderKind = hasNativeZstd ? 'node-zstd' : 'fzstd'
|
|
45
|
+
|
|
46
|
+
export function defaultSessionsDir() {
|
|
47
|
+
const home = process.env.DSH_HOME
|
|
48
|
+
if (typeof home === 'string' && home.trim() !== '') return path.join(home.trim(), 'sessions')
|
|
49
|
+
return path.join(os.homedir(), '.dsh', 'sessions')
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The profile's session store: every closed session, newest modified first. */
|
|
53
|
+
export async function listSessions(dir) {
|
|
54
|
+
const out = []
|
|
55
|
+
let groups = []
|
|
56
|
+
try {
|
|
57
|
+
groups = await fsp.readdir(dir, { withFileTypes: true })
|
|
58
|
+
} catch {
|
|
59
|
+
return out
|
|
60
|
+
}
|
|
61
|
+
for (const group of groups) {
|
|
62
|
+
if (!group.isDirectory()) continue
|
|
63
|
+
const groupDir = path.join(dir, group.name)
|
|
64
|
+
let entries = []
|
|
65
|
+
try {
|
|
66
|
+
entries = await fsp.readdir(groupDir, { withFileTypes: true })
|
|
67
|
+
} catch {
|
|
68
|
+
continue
|
|
69
|
+
}
|
|
70
|
+
for (const entry of entries) {
|
|
71
|
+
if (!entry.isDirectory()) continue
|
|
72
|
+
const file = path.join(groupDir, entry.name, SESSION_FILE)
|
|
73
|
+
try {
|
|
74
|
+
const stat = await fsp.stat(file)
|
|
75
|
+
if (!stat.isFile()) continue
|
|
76
|
+
out.push({ file, size: stat.size, mtimeMs: stat.mtimeMs, group: group.name, dirName: entry.name })
|
|
77
|
+
} catch {
|
|
78
|
+
// A session directory without a log: the turn never produced anything.
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
// Newest first: with a limit, the recent past is what a person wants back.
|
|
83
|
+
out.sort((a, b) => b.mtimeMs - a.mtimeMs)
|
|
84
|
+
return out
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Cut the inflated byte stream into lines, with a ceiling on how much of one
|
|
89
|
+
* session is read at all.
|
|
90
|
+
*
|
|
91
|
+
* A session log is tens of megabytes compressed and hundreds as text, and a
|
|
92
|
+
* backfill walks a hundred of them: nothing here holds the whole plaintext.
|
|
93
|
+
*/
|
|
94
|
+
function lineCutter(onLine, { maxBytes = DEFAULT_MAX_BYTES } = {}) {
|
|
95
|
+
let inflated = 0
|
|
96
|
+
let pending = ''
|
|
97
|
+
let stopped = false
|
|
98
|
+
let partial = false
|
|
99
|
+
return {
|
|
100
|
+
push(chunk) {
|
|
101
|
+
if (stopped || chunk.length === 0) return
|
|
102
|
+
inflated += chunk.length
|
|
103
|
+
pending += chunk.toString('utf8')
|
|
104
|
+
let cut = pending.indexOf('\n')
|
|
105
|
+
while (cut !== -1) {
|
|
106
|
+
const line = pending.slice(0, cut)
|
|
107
|
+
pending = pending.slice(cut + 1)
|
|
108
|
+
if (line.trim() !== '') onLine(line)
|
|
109
|
+
cut = pending.indexOf('\n')
|
|
110
|
+
}
|
|
111
|
+
if (pending.length > MAX_LINE || inflated > maxBytes) {
|
|
112
|
+
stopped = true
|
|
113
|
+
pending = ''
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
end() {
|
|
117
|
+
if (!stopped && pending.trim() !== '') onLine(pending)
|
|
118
|
+
pending = ''
|
|
119
|
+
},
|
|
120
|
+
/** The caller stopped early — a byte ceiling, not a corrupt file. */
|
|
121
|
+
markPartial() {
|
|
122
|
+
partial = true
|
|
123
|
+
},
|
|
124
|
+
get stopped() {
|
|
125
|
+
return stopped
|
|
126
|
+
},
|
|
127
|
+
get truncated() {
|
|
128
|
+
return stopped || partial
|
|
129
|
+
},
|
|
130
|
+
get inflated() {
|
|
131
|
+
return inflated
|
|
132
|
+
},
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Frame boundaries, without decompressing: a correctness port of the scanner the
|
|
138
|
+
* harness itself uses for these files (zstd frame header, then 3-byte block
|
|
139
|
+
* headers, then an optional 4-byte checksum). A partial trailing frame is the
|
|
140
|
+
* normal state of a session that is still being written; its start is returned
|
|
141
|
+
* so the caller can ignore it rather than call it corruption.
|
|
142
|
+
*/
|
|
143
|
+
export function scanFrames(buffer) {
|
|
144
|
+
const frames = []
|
|
145
|
+
let offset = 0
|
|
146
|
+
while (offset < buffer.length) {
|
|
147
|
+
const start = offset
|
|
148
|
+
const torn = () => ({ frames, tornStart: start })
|
|
149
|
+
if (buffer.length - offset < 4) return torn()
|
|
150
|
+
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) return torn()
|
|
151
|
+
offset += 4
|
|
152
|
+
if (offset === buffer.length) return torn()
|
|
153
|
+
|
|
154
|
+
const descriptor = buffer.readUInt8(offset)
|
|
155
|
+
offset += 1
|
|
156
|
+
// Reserved bits, or a window with no content size where one is required:
|
|
157
|
+
// reason enough to stop trusting the rest of the file.
|
|
158
|
+
if ((descriptor & 0b00011000) !== 0) return torn()
|
|
159
|
+
const contentSizeFlag = descriptor >>> 6
|
|
160
|
+
const singleSegment = (descriptor & 0b00100000) !== 0
|
|
161
|
+
const checksumFlag = (descriptor & 0b00000100) !== 0
|
|
162
|
+
const dictionaryFlag = descriptor & 0b11
|
|
163
|
+
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag
|
|
164
|
+
const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag
|
|
165
|
+
const remainingHeaderBytes = (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
|
|
166
|
+
if (buffer.length - offset < remainingHeaderBytes) return torn()
|
|
167
|
+
offset += remainingHeaderBytes
|
|
168
|
+
|
|
169
|
+
for (;;) {
|
|
170
|
+
if (buffer.length - offset < 3) return torn()
|
|
171
|
+
const blockHeader = buffer.readUIntLE(offset, 3)
|
|
172
|
+
offset += 3
|
|
173
|
+
const lastBlock = (blockHeader & 1) !== 0
|
|
174
|
+
const blockType = (blockHeader >>> 1) & 0b11
|
|
175
|
+
const blockSize = blockHeader >>> 3
|
|
176
|
+
if (blockType === 3) return torn() // reserved block type
|
|
177
|
+
const payloadBytes = blockType === 1 ? 1 : blockSize // RLE blocks carry one byte
|
|
178
|
+
if (buffer.length - offset < payloadBytes) return torn()
|
|
179
|
+
offset += payloadBytes
|
|
180
|
+
if (lastBlock) break
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (checksumFlag) {
|
|
184
|
+
if (buffer.length - offset < 4) return torn()
|
|
185
|
+
offset += 4
|
|
186
|
+
}
|
|
187
|
+
frames.push({ start, end: offset })
|
|
188
|
+
}
|
|
189
|
+
return { frames }
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function readBytes(file, { maxBytes = DEFAULT_MAX_BYTES } = {}) {
|
|
193
|
+
const handle = await fsp.open(file, 'r')
|
|
194
|
+
try {
|
|
195
|
+
const { size } = await handle.stat()
|
|
196
|
+
const want = Math.max(0, Math.min(size, maxBytes))
|
|
197
|
+
const buffer = Buffer.allocUnsafe(want)
|
|
198
|
+
let read = 0
|
|
199
|
+
while (read < want) {
|
|
200
|
+
const { bytesRead } = await handle.read(buffer, read, want - read, read)
|
|
201
|
+
if (bytesRead === 0) break
|
|
202
|
+
read += bytesRead
|
|
203
|
+
}
|
|
204
|
+
return { buffer: buffer.subarray(0, read), size, partial: want < size }
|
|
205
|
+
} finally {
|
|
206
|
+
await handle.close()
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The reference decoder: every frame zstd can produce, one frame at a time. */
|
|
211
|
+
async function readLinesNative(file, onLine, options) {
|
|
212
|
+
const cutter = lineCutter(onLine, options)
|
|
213
|
+
const { buffer, partial } = await readBytes(file, options)
|
|
214
|
+
if (partial) cutter.markPartial()
|
|
215
|
+
|
|
216
|
+
let torn = false
|
|
217
|
+
let failed = 0
|
|
218
|
+
for (const { start, end } of scanFrames(buffer).frames) {
|
|
219
|
+
let plain
|
|
220
|
+
try {
|
|
221
|
+
plain = zlib.zstdDecompressSync(buffer.subarray(start, end))
|
|
222
|
+
} catch {
|
|
223
|
+
// One frame the codec refuses (a truncated append after a crash) should
|
|
224
|
+
// cost that batch, not the session.
|
|
225
|
+
failed += 1
|
|
226
|
+
continue
|
|
227
|
+
}
|
|
228
|
+
cutter.push(plain)
|
|
229
|
+
if (cutter.stopped) break
|
|
230
|
+
}
|
|
231
|
+
torn = failed > 0
|
|
232
|
+
cutter.end()
|
|
233
|
+
if (torn && cutter.inflated === 0) throw new Error('这个会话文件里没有可解码的 zstd 帧')
|
|
234
|
+
return cutter
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** The fallback for a Node without a zstd codec: pure JS, most frames. */
|
|
238
|
+
async function readLinesFallback(file, onLine, options) {
|
|
239
|
+
const cutter = lineCutter(onLine, options)
|
|
240
|
+
const decoder = new Decompress((chunk) => cutter.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)))
|
|
241
|
+
const { buffer, partial } = await readBytes(file, options)
|
|
242
|
+
if (partial) cutter.markPartial()
|
|
243
|
+
decoder.push(new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength), true)
|
|
244
|
+
cutter.end()
|
|
245
|
+
return cutter
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function readLines(file, onLine, options) {
|
|
249
|
+
return hasNativeZstd ? readLinesNative(file, onLine, options) : readLinesFallback(file, onLine, options)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Every `(session, turn)` the vault already has, so a re-run imports nothing twice. */
|
|
253
|
+
export async function existingTurns(vault) {
|
|
254
|
+
const seen = new Set()
|
|
255
|
+
let files = []
|
|
256
|
+
try {
|
|
257
|
+
files = typeof vault.episodeFiles === 'function' ? await vault.episodeFiles() : []
|
|
258
|
+
} catch {
|
|
259
|
+
return seen
|
|
260
|
+
}
|
|
261
|
+
const dir = vault.paths?.episodes
|
|
262
|
+
if (typeof dir !== 'string') return seen
|
|
263
|
+
for (const name of files) {
|
|
264
|
+
let text = ''
|
|
265
|
+
try {
|
|
266
|
+
text = await fsp.readFile(path.join(dir, name), 'utf8')
|
|
267
|
+
} catch {
|
|
268
|
+
continue
|
|
269
|
+
}
|
|
270
|
+
for (const line of text.split('\n')) {
|
|
271
|
+
if (line.trim() === '') continue
|
|
272
|
+
try {
|
|
273
|
+
const record = JSON.parse(line)
|
|
274
|
+
if (typeof record?.session === 'string') seen.add(`${record.session}#${record.turn ?? ''}`)
|
|
275
|
+
} catch {
|
|
276
|
+
// A half-written last line from a crash; the next run sees it whole.
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return seen
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* One replayed session, fed a line at a time.
|
|
285
|
+
*
|
|
286
|
+
* Kept separate from the file reading so the filter, the dedupe and the
|
|
287
|
+
* timestamps can be tested on hand-written events — the part that matters is the
|
|
288
|
+
* policy, not the decompressor.
|
|
289
|
+
*/
|
|
290
|
+
export function createReplay({ sessionId = '', project = '', cfg, redactor, seen }) {
|
|
291
|
+
const assembler = createTurnAssembler({ cfg, redactor })
|
|
292
|
+
const episodes = []
|
|
293
|
+
let id = sessionId
|
|
294
|
+
let cwd = ''
|
|
295
|
+
let closedTurns = 0
|
|
296
|
+
let duplicates = 0
|
|
297
|
+
let malformed = 0
|
|
298
|
+
|
|
299
|
+
const keep = (episode) => {
|
|
300
|
+
const key = `${episode.session}#${episode.turn ?? ''}`
|
|
301
|
+
if (seen.has(key)) {
|
|
302
|
+
duplicates += 1
|
|
303
|
+
return
|
|
304
|
+
}
|
|
305
|
+
seen.add(key)
|
|
306
|
+
episodes.push(episode)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
return {
|
|
310
|
+
feed(line) {
|
|
311
|
+
let event
|
|
312
|
+
try {
|
|
313
|
+
event = JSON.parse(line)
|
|
314
|
+
} catch {
|
|
315
|
+
malformed += 1
|
|
316
|
+
return
|
|
317
|
+
}
|
|
318
|
+
if (event?.type === 'session') {
|
|
319
|
+
if (typeof event.id === 'string' && event.id !== '') id = event.id
|
|
320
|
+
if (typeof event.cwd === 'string') cwd = event.cwd
|
|
321
|
+
return
|
|
322
|
+
}
|
|
323
|
+
if (event?.type === 'turn/end') closedTurns += 1
|
|
324
|
+
for (const episode of assembler.push({
|
|
325
|
+
sessionId: id,
|
|
326
|
+
project: projectOf({ cwd }, cfg.defaultProject),
|
|
327
|
+
event,
|
|
328
|
+
})) {
|
|
329
|
+
keep(episode)
|
|
330
|
+
}
|
|
331
|
+
},
|
|
332
|
+
|
|
333
|
+
/** A session whose last turn never closed still holds a real message. */
|
|
334
|
+
finish() {
|
|
335
|
+
for (const episode of assembler.flushAll()) keep(episode)
|
|
336
|
+
return {
|
|
337
|
+
id,
|
|
338
|
+
project: projectOf({ cwd }, cfg.defaultProject),
|
|
339
|
+
cwd,
|
|
340
|
+
closedTurns,
|
|
341
|
+
episodes,
|
|
342
|
+
duplicates,
|
|
343
|
+
malformed,
|
|
344
|
+
}
|
|
345
|
+
},
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Replay one session file, then write what the vault does not have. */
|
|
350
|
+
export async function importSession({ entry, cfg, vault, redactor, seen, dryRun = false, maxBytes, collect = null }) {
|
|
351
|
+
const replay = createReplay({ sessionId: entry.dirName, cfg, redactor, seen })
|
|
352
|
+
let failure = null
|
|
353
|
+
let inflated = 0
|
|
354
|
+
let truncated = false
|
|
355
|
+
try {
|
|
356
|
+
const cutter = await readLines(entry.file, (line) => replay.feed(line), { maxBytes })
|
|
357
|
+
inflated = cutter.inflated
|
|
358
|
+
truncated = cutter.truncated
|
|
359
|
+
} catch (error) {
|
|
360
|
+
// One unreadable file must not end a backfill that has hundreds of others:
|
|
361
|
+
// the report says which one failed and the run carries on.
|
|
362
|
+
failure = error?.message ?? String(error)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const report = replay.finish()
|
|
366
|
+
|
|
367
|
+
if (Array.isArray(collect)) collect.push(...report.episodes)
|
|
368
|
+
if (!dryRun) {
|
|
369
|
+
for (const episode of report.episodes) await vault.appendEpisode(episode)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return {
|
|
373
|
+
...report,
|
|
374
|
+
episodes: report.episodes.length,
|
|
375
|
+
truncated,
|
|
376
|
+
failure,
|
|
377
|
+
inflated,
|
|
378
|
+
file: entry.file,
|
|
379
|
+
size: entry.size,
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Walk the store, newest session first, and import what the vault does not have.
|
|
385
|
+
*
|
|
386
|
+
* Returns a report rather than logging: the tool formats it, the tests assert on
|
|
387
|
+
* it, and a person reads the same numbers.
|
|
388
|
+
*/
|
|
389
|
+
export async function runBackfill({
|
|
390
|
+
cfg,
|
|
391
|
+
vault,
|
|
392
|
+
redactor,
|
|
393
|
+
dir = defaultSessionsDir(),
|
|
394
|
+
limit = 0,
|
|
395
|
+
dryRun = false,
|
|
396
|
+
maxBytes = DEFAULT_MAX_BYTES,
|
|
397
|
+
onProgress = null,
|
|
398
|
+
collect = null,
|
|
399
|
+
}) {
|
|
400
|
+
const started = Date.now()
|
|
401
|
+
const seen = await existingTurns(vault)
|
|
402
|
+
const all = await listSessions(dir)
|
|
403
|
+
const selected = limit > 0 ? all.slice(0, limit) : all
|
|
404
|
+
|
|
405
|
+
const sessions = []
|
|
406
|
+
let imported = 0
|
|
407
|
+
let duplicates = 0
|
|
408
|
+
let budget = maxBytes
|
|
409
|
+
let exhausted = false
|
|
410
|
+
|
|
411
|
+
for (const entry of selected) {
|
|
412
|
+
if (budget <= 0) {
|
|
413
|
+
exhausted = true
|
|
414
|
+
break
|
|
415
|
+
}
|
|
416
|
+
const report = await importSession({ entry, cfg, vault, redactor, seen, dryRun, maxBytes: budget, collect })
|
|
417
|
+
budget -= entry.size
|
|
418
|
+
imported += report.episodes
|
|
419
|
+
duplicates += report.duplicates
|
|
420
|
+
sessions.push(report)
|
|
421
|
+
onProgress?.(report)
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
return {
|
|
425
|
+
dir,
|
|
426
|
+
dryRun,
|
|
427
|
+
decoder: decoderKind,
|
|
428
|
+
found: all.length,
|
|
429
|
+
scanned: sessions.length,
|
|
430
|
+
imported,
|
|
431
|
+
duplicates,
|
|
432
|
+
exhausted,
|
|
433
|
+
malformed: sessions.reduce((n, s) => n + s.malformed, 0),
|
|
434
|
+
failed: sessions.filter((s) => s.failure !== null).length,
|
|
435
|
+
ms: Date.now() - started,
|
|
436
|
+
sessions: sessions
|
|
437
|
+
.filter((s) => s.episodes > 0 || s.duplicates > 0 || s.closedTurns > 0 || s.failure !== null)
|
|
438
|
+
.sort((a, b) => b.episodes - a.episodes),
|
|
439
|
+
}
|
|
440
|
+
}
|
package/src/pageapi.js
CHANGED
|
@@ -22,14 +22,14 @@ function failure(error) {
|
|
|
22
22
|
<style>body{font:14px/1.6 system-ui,sans-serif;margin:0;display:grid;place-items:center;height:100vh;color:#333}
|
|
23
23
|
main{max-width:34rem;padding:2rem}code{background:#f4f4f5;padding:.1rem .3rem;border-radius:4px}</style>
|
|
24
24
|
<main><h1>记忆图没能画出来</h1><p><code>${escaped}</code></p>
|
|
25
|
-
<p
|
|
25
|
+
<p>图谱渲染失败了,但记忆库还在磁盘上,什么都没写 · The graph could not be rendered. The vault is still on disk; nothing was written.</p></main></html>`
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export function createPageHandler({ getVault }) {
|
|
29
29
|
return async function fetch() {
|
|
30
30
|
const vault = getVault()
|
|
31
31
|
if (!vault || typeof vault.graphPage !== 'function') {
|
|
32
|
-
return html(failure(new Error('vault unavailable')), 500)
|
|
32
|
+
return html(failure(new Error('记忆库不可用 · vault unavailable')), 500)
|
|
33
33
|
}
|
|
34
34
|
try {
|
|
35
35
|
return html(await vault.graphPage())
|