@ilaomianyang/dsh-memory-graph 0.10.4 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.en.md +73 -12
- package/README.md +73 -12
- package/client.js +433 -1
- package/docs/update-settings.png +0 -0
- package/index.js +42 -3
- package/package.json +5 -3
- package/src/auto.js +78 -14
- package/src/capture.js +37 -175
- package/src/config.js +37 -0
- package/src/distillapi.js +4 -1
- package/src/distillrun.js +7 -3
- package/src/extract.js +42 -18
- package/src/graph.js +10 -5
- package/src/graphdata.js +10 -1
- package/src/history.js +467 -0
- package/src/pageapi.js +2 -2
- package/src/schema.js +55 -0
- package/src/tools.js +76 -2
- package/src/turns.js +265 -0
- package/src/update.js +406 -0
- package/src/vault.js +21 -11
- package/test/auto.mjs +68 -0
- package/test/client.mjs +15 -5
- package/test/fixtures/session.v4.jsonl.zstd +0 -0
- package/test/history.mjs +249 -0
- package/test/tools.mjs +2 -1
- package/test/update.mjs +401 -0
package/src/history.js
ADDED
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* History: read the sessions that ran before this plugin existed.
|
|
3
|
+
*
|
|
4
|
+
* Capture only sees turns that arrive while the host is running, so a vault
|
|
5
|
+
* installed today starts empty and the person's past is not in it. The harness
|
|
6
|
+
* keeps every closed session on disk as `session.v4.jsonl.zstd`; this module
|
|
7
|
+
* replays those files through the same turn assembler the live listener uses, so
|
|
8
|
+
* a backfilled episode and a captured one are the same kind of record.
|
|
9
|
+
*
|
|
10
|
+
* Two things make it safe to run twice:
|
|
11
|
+
* - a turn is identified by `(session, turn)`, and turns already in the vault
|
|
12
|
+
* are skipped, so re-running after a session grew imports only the new part;
|
|
13
|
+
* - the timestamps come from the session file, not from now, so history lands
|
|
14
|
+
* in the right place on the timeline instead of piling up under today.
|
|
15
|
+
*/
|
|
16
|
+
import fsp from 'node:fs/promises'
|
|
17
|
+
import os from 'node:os'
|
|
18
|
+
import path from 'node:path'
|
|
19
|
+
import zlib from 'node:zlib'
|
|
20
|
+
import { Decompress } from 'fzstd'
|
|
21
|
+
|
|
22
|
+
import { createTurnAssembler, projectOf } from './turns.js'
|
|
23
|
+
|
|
24
|
+
const SESSION_FILE = 'session.v4.jsonl.zstd'
|
|
25
|
+
/** zstd in, JSON out: a cap so one enormous session cannot pin the process. */
|
|
26
|
+
export const DEFAULT_MAX_BYTES = 512 * 1024 * 1024
|
|
27
|
+
/** No single event is this large; a longer line means the stream is corrupt. */
|
|
28
|
+
const MAX_LINE = 4 * 1024 * 1024
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A session file is not one zstd frame: the writer appends one frame per batch,
|
|
32
|
+
* so the file is a *concatenated container*. Both decoders that matter — Node's
|
|
33
|
+
* and fzstd's — stop at a frame boundary, which is why the raw walk of these
|
|
34
|
+
* files used to yield a single header line and nothing else.
|
|
35
|
+
*
|
|
36
|
+
* Node grew a zstd codec in 22.15 / 23.8 and decodes frames with full fidelity;
|
|
37
|
+
* the pure-JS fallback below refuses some high-accuracy FSE tables that real
|
|
38
|
+
* session files here do contain ("FSE accuracy too high"). A plugin cannot choose
|
|
39
|
+
* the runtime it lands in, so both paths exist and the report names the one that
|
|
40
|
+
* ran.
|
|
41
|
+
*/
|
|
42
|
+
const ZSTD_MAGIC = 0xfd2fb528
|
|
43
|
+
const hasNativeZstd = typeof zlib.zstdDecompressSync === 'function'
|
|
44
|
+
export const decoderKind = hasNativeZstd ? 'node-zstd' : 'fzstd'
|
|
45
|
+
|
|
46
|
+
export function defaultSessionsDir() {
|
|
47
|
+
const home = process.env.DSH_HOME
|
|
48
|
+
if (typeof home === 'string' && home.trim() !== '') return path.join(home.trim(), 'sessions')
|
|
49
|
+
return path.join(os.homedir(), '.dsh', 'sessions')
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The profile's session store: every closed session, newest modified first. */
|
|
53
|
+
export async function listSessions(dir) {
|
|
54
|
+
const out = []
|
|
55
|
+
let groups = []
|
|
56
|
+
try {
|
|
57
|
+
groups = await fsp.readdir(dir, { withFileTypes: true })
|
|
58
|
+
} catch {
|
|
59
|
+
return out
|
|
60
|
+
}
|
|
61
|
+
for (const group of groups) {
|
|
62
|
+
if (!group.isDirectory()) continue
|
|
63
|
+
const groupDir = path.join(dir, group.name)
|
|
64
|
+
let entries = []
|
|
65
|
+
try {
|
|
66
|
+
entries = await fsp.readdir(groupDir, { withFileTypes: true })
|
|
67
|
+
} catch {
|
|
68
|
+
continue
|
|
69
|
+
}
|
|
70
|
+
for (const entry of entries) {
|
|
71
|
+
if (!entry.isDirectory()) continue
|
|
72
|
+
const file = path.join(groupDir, entry.name, SESSION_FILE)
|
|
73
|
+
try {
|
|
74
|
+
const stat = await fsp.stat(file)
|
|
75
|
+
if (!stat.isFile()) continue
|
|
76
|
+
out.push({ file, size: stat.size, mtimeMs: stat.mtimeMs, group: group.name, dirName: entry.name })
|
|
77
|
+
} catch {
|
|
78
|
+
// A session directory without a log: the turn never produced anything.
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
// Newest first: with a limit, the recent past is what a person wants back.
|
|
83
|
+
out.sort((a, b) => b.mtimeMs - a.mtimeMs)
|
|
84
|
+
return out
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Cut the inflated byte stream into lines, with a ceiling on how much of one
|
|
89
|
+
* session is read at all.
|
|
90
|
+
*
|
|
91
|
+
* A session log is tens of megabytes compressed and hundreds as text, and a
|
|
92
|
+
* backfill walks a hundred of them: nothing here holds the whole plaintext.
|
|
93
|
+
*/
|
|
94
|
+
function lineCutter(onLine, { maxBytes = DEFAULT_MAX_BYTES } = {}) {
|
|
95
|
+
let inflated = 0
|
|
96
|
+
let pending = ''
|
|
97
|
+
let stopped = false
|
|
98
|
+
let partial = false
|
|
99
|
+
return {
|
|
100
|
+
push(chunk) {
|
|
101
|
+
if (stopped || chunk.length === 0) return
|
|
102
|
+
inflated += chunk.length
|
|
103
|
+
pending += chunk.toString('utf8')
|
|
104
|
+
let cut = pending.indexOf('\n')
|
|
105
|
+
while (cut !== -1) {
|
|
106
|
+
const line = pending.slice(0, cut)
|
|
107
|
+
pending = pending.slice(cut + 1)
|
|
108
|
+
if (line.trim() !== '') onLine(line)
|
|
109
|
+
cut = pending.indexOf('\n')
|
|
110
|
+
}
|
|
111
|
+
if (pending.length > MAX_LINE || inflated > maxBytes) {
|
|
112
|
+
stopped = true
|
|
113
|
+
pending = ''
|
|
114
|
+
}
|
|
115
|
+
},
|
|
116
|
+
end() {
|
|
117
|
+
if (!stopped && pending.trim() !== '') onLine(pending)
|
|
118
|
+
pending = ''
|
|
119
|
+
},
|
|
120
|
+
/** The caller stopped early — a byte ceiling, not a corrupt file. */
|
|
121
|
+
markPartial() {
|
|
122
|
+
partial = true
|
|
123
|
+
},
|
|
124
|
+
get stopped() {
|
|
125
|
+
return stopped
|
|
126
|
+
},
|
|
127
|
+
get truncated() {
|
|
128
|
+
return stopped || partial
|
|
129
|
+
},
|
|
130
|
+
get inflated() {
|
|
131
|
+
return inflated
|
|
132
|
+
},
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Frame boundaries, without decompressing: a correctness port of the scanner the
|
|
138
|
+
* harness itself uses for these files (zstd frame header, then 3-byte block
|
|
139
|
+
* headers, then an optional 4-byte checksum). A partial trailing frame is the
|
|
140
|
+
* normal state of a session that is still being written; its start is returned
|
|
141
|
+
* so the caller can ignore it rather than call it corruption.
|
|
142
|
+
*/
|
|
143
|
+
export function scanFrames(buffer) {
|
|
144
|
+
const frames = []
|
|
145
|
+
let offset = 0
|
|
146
|
+
while (offset < buffer.length) {
|
|
147
|
+
const start = offset
|
|
148
|
+
const torn = () => ({ frames, tornStart: start })
|
|
149
|
+
if (buffer.length - offset < 4) return torn()
|
|
150
|
+
if (buffer.readUInt32LE(offset) !== ZSTD_MAGIC) return torn()
|
|
151
|
+
offset += 4
|
|
152
|
+
if (offset === buffer.length) return torn()
|
|
153
|
+
|
|
154
|
+
const descriptor = buffer.readUInt8(offset)
|
|
155
|
+
offset += 1
|
|
156
|
+
// Reserved bits, or a window with no content size where one is required:
|
|
157
|
+
// reason enough to stop trusting the rest of the file.
|
|
158
|
+
if ((descriptor & 0b00011000) !== 0) return torn()
|
|
159
|
+
const contentSizeFlag = descriptor >>> 6
|
|
160
|
+
const singleSegment = (descriptor & 0b00100000) !== 0
|
|
161
|
+
const checksumFlag = (descriptor & 0b00000100) !== 0
|
|
162
|
+
const dictionaryFlag = descriptor & 0b11
|
|
163
|
+
const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag
|
|
164
|
+
const contentSizeBytes = contentSizeFlag === 0 ? (singleSegment ? 1 : 0) : 1 << contentSizeFlag
|
|
165
|
+
const remainingHeaderBytes = (singleSegment ? 0 : 1) + dictionaryBytes + contentSizeBytes
|
|
166
|
+
if (buffer.length - offset < remainingHeaderBytes) return torn()
|
|
167
|
+
offset += remainingHeaderBytes
|
|
168
|
+
|
|
169
|
+
for (;;) {
|
|
170
|
+
if (buffer.length - offset < 3) return torn()
|
|
171
|
+
const blockHeader = buffer.readUIntLE(offset, 3)
|
|
172
|
+
offset += 3
|
|
173
|
+
const lastBlock = (blockHeader & 1) !== 0
|
|
174
|
+
const blockType = (blockHeader >>> 1) & 0b11
|
|
175
|
+
const blockSize = blockHeader >>> 3
|
|
176
|
+
if (blockType === 3) return torn() // reserved block type
|
|
177
|
+
const payloadBytes = blockType === 1 ? 1 : blockSize // RLE blocks carry one byte
|
|
178
|
+
if (buffer.length - offset < payloadBytes) return torn()
|
|
179
|
+
offset += payloadBytes
|
|
180
|
+
if (lastBlock) break
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (checksumFlag) {
|
|
184
|
+
if (buffer.length - offset < 4) return torn()
|
|
185
|
+
offset += 4
|
|
186
|
+
}
|
|
187
|
+
frames.push({ start, end: offset })
|
|
188
|
+
}
|
|
189
|
+
return { frames }
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async function readBytes(file, { maxBytes = DEFAULT_MAX_BYTES } = {}) {
|
|
193
|
+
const handle = await fsp.open(file, 'r')
|
|
194
|
+
try {
|
|
195
|
+
const { size } = await handle.stat()
|
|
196
|
+
const want = Math.max(0, Math.min(size, maxBytes))
|
|
197
|
+
const buffer = Buffer.allocUnsafe(want)
|
|
198
|
+
let read = 0
|
|
199
|
+
while (read < want) {
|
|
200
|
+
const { bytesRead } = await handle.read(buffer, read, want - read, read)
|
|
201
|
+
if (bytesRead === 0) break
|
|
202
|
+
read += bytesRead
|
|
203
|
+
}
|
|
204
|
+
return { buffer: buffer.subarray(0, read), size, partial: want < size }
|
|
205
|
+
} finally {
|
|
206
|
+
await handle.close()
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The reference decoder: every frame zstd can produce, one frame at a time. */
|
|
211
|
+
async function readLinesNative(file, onLine, options) {
|
|
212
|
+
const cutter = lineCutter(onLine, options)
|
|
213
|
+
const { buffer, partial } = await readBytes(file, options)
|
|
214
|
+
if (partial) cutter.markPartial()
|
|
215
|
+
|
|
216
|
+
let torn = false
|
|
217
|
+
let failed = 0
|
|
218
|
+
for (const { start, end } of scanFrames(buffer).frames) {
|
|
219
|
+
let plain
|
|
220
|
+
try {
|
|
221
|
+
plain = zlib.zstdDecompressSync(buffer.subarray(start, end))
|
|
222
|
+
} catch {
|
|
223
|
+
// One frame the codec refuses (a truncated append after a crash) should
|
|
224
|
+
// cost that batch, not the session.
|
|
225
|
+
failed += 1
|
|
226
|
+
continue
|
|
227
|
+
}
|
|
228
|
+
cutter.push(plain)
|
|
229
|
+
if (cutter.stopped) break
|
|
230
|
+
}
|
|
231
|
+
torn = failed > 0
|
|
232
|
+
cutter.end()
|
|
233
|
+
if (torn && cutter.inflated === 0) throw new Error('这个会话文件里没有可解码的 zstd 帧')
|
|
234
|
+
return cutter
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** The fallback for a Node without a zstd codec: pure JS, most frames. */
|
|
238
|
+
async function readLinesFallback(file, onLine, options) {
|
|
239
|
+
const cutter = lineCutter(onLine, options)
|
|
240
|
+
const decoder = new Decompress((chunk) => cutter.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength)))
|
|
241
|
+
const { buffer, partial } = await readBytes(file, options)
|
|
242
|
+
if (partial) cutter.markPartial()
|
|
243
|
+
decoder.push(new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength), true)
|
|
244
|
+
cutter.end()
|
|
245
|
+
return cutter
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
async function readLines(file, onLine, options) {
|
|
249
|
+
return hasNativeZstd ? readLinesNative(file, onLine, options) : readLinesFallback(file, onLine, options)
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Every `(session, turn)` the vault already has, so a re-run imports nothing twice. */
|
|
253
|
+
export async function existingTurns(vault) {
|
|
254
|
+
const seen = new Set()
|
|
255
|
+
let files = []
|
|
256
|
+
try {
|
|
257
|
+
files = typeof vault.episodeFiles === 'function' ? await vault.episodeFiles() : []
|
|
258
|
+
} catch {
|
|
259
|
+
return seen
|
|
260
|
+
}
|
|
261
|
+
const dir = vault.paths?.episodes
|
|
262
|
+
if (typeof dir !== 'string') return seen
|
|
263
|
+
for (const name of files) {
|
|
264
|
+
let text = ''
|
|
265
|
+
try {
|
|
266
|
+
text = await fsp.readFile(path.join(dir, name), 'utf8')
|
|
267
|
+
} catch {
|
|
268
|
+
continue
|
|
269
|
+
}
|
|
270
|
+
for (const line of text.split('\n')) {
|
|
271
|
+
if (line.trim() === '') continue
|
|
272
|
+
try {
|
|
273
|
+
const record = JSON.parse(line)
|
|
274
|
+
if (typeof record?.session === 'string') seen.add(`${record.session}#${record.turn ?? ''}`)
|
|
275
|
+
} catch {
|
|
276
|
+
// A half-written last line from a crash; the next run sees it whole.
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return seen
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* One replayed session, fed a line at a time.
|
|
285
|
+
*
|
|
286
|
+
* Kept separate from the file reading so the filter, the dedupe and the
|
|
287
|
+
* timestamps can be tested on hand-written events — the part that matters is the
|
|
288
|
+
* policy, not the decompressor.
|
|
289
|
+
*/
|
|
290
|
+
export function createReplay({ sessionId = '', project = '', cfg, redactor, seen }) {
|
|
291
|
+
const assembler = createTurnAssembler({ cfg, redactor })
|
|
292
|
+
const episodes = []
|
|
293
|
+
let id = sessionId
|
|
294
|
+
let cwd = ''
|
|
295
|
+
let closedTurns = 0
|
|
296
|
+
let duplicates = 0
|
|
297
|
+
let malformed = 0
|
|
298
|
+
|
|
299
|
+
const keep = (episode) => {
|
|
300
|
+
const key = `${episode.session}#${episode.turn ?? ''}`
|
|
301
|
+
if (seen.has(key)) {
|
|
302
|
+
duplicates += 1
|
|
303
|
+
return
|
|
304
|
+
}
|
|
305
|
+
seen.add(key)
|
|
306
|
+
episodes.push(episode)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
return {
|
|
310
|
+
feed(line) {
|
|
311
|
+
let event
|
|
312
|
+
try {
|
|
313
|
+
event = JSON.parse(line)
|
|
314
|
+
} catch {
|
|
315
|
+
malformed += 1
|
|
316
|
+
return
|
|
317
|
+
}
|
|
318
|
+
if (event?.type === 'session') {
|
|
319
|
+
if (typeof event.id === 'string' && event.id !== '') id = event.id
|
|
320
|
+
if (typeof event.cwd === 'string') cwd = event.cwd
|
|
321
|
+
return
|
|
322
|
+
}
|
|
323
|
+
if (event?.type === 'turn/end') closedTurns += 1
|
|
324
|
+
for (const episode of assembler.push({
|
|
325
|
+
sessionId: id,
|
|
326
|
+
project: projectOf({ cwd }, cfg.defaultProject),
|
|
327
|
+
event,
|
|
328
|
+
})) {
|
|
329
|
+
keep(episode)
|
|
330
|
+
}
|
|
331
|
+
},
|
|
332
|
+
|
|
333
|
+
/** A session whose last turn never closed still holds a real message. */
|
|
334
|
+
finish() {
|
|
335
|
+
for (const episode of assembler.flushAll()) keep(episode)
|
|
336
|
+
return {
|
|
337
|
+
id,
|
|
338
|
+
project: projectOf({ cwd }, cfg.defaultProject),
|
|
339
|
+
cwd,
|
|
340
|
+
closedTurns,
|
|
341
|
+
episodes,
|
|
342
|
+
duplicates,
|
|
343
|
+
malformed,
|
|
344
|
+
}
|
|
345
|
+
},
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Replay one session file, then write what the vault does not have. */
|
|
350
|
+
export async function importSession({ entry, cfg, vault, redactor, seen, dryRun = false, maxBytes, collect = null }) {
|
|
351
|
+
const replay = createReplay({ sessionId: entry.dirName, cfg, redactor, seen })
|
|
352
|
+
let failure = null
|
|
353
|
+
let inflated = 0
|
|
354
|
+
let truncated = false
|
|
355
|
+
try {
|
|
356
|
+
const cutter = await readLines(entry.file, (line) => replay.feed(line), { maxBytes })
|
|
357
|
+
inflated = cutter.inflated
|
|
358
|
+
truncated = cutter.truncated
|
|
359
|
+
} catch (error) {
|
|
360
|
+
// One unreadable file must not end a backfill that has hundreds of others:
|
|
361
|
+
// the report says which one failed and the run carries on.
|
|
362
|
+
failure = error?.message ?? String(error)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const report = replay.finish()
|
|
366
|
+
|
|
367
|
+
if (Array.isArray(collect)) collect.push(...report.episodes)
|
|
368
|
+
if (!dryRun) {
|
|
369
|
+
for (const episode of report.episodes) await vault.appendEpisode(episode)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
return {
|
|
373
|
+
...report,
|
|
374
|
+
episodes: report.episodes.length,
|
|
375
|
+
truncated,
|
|
376
|
+
failure,
|
|
377
|
+
inflated,
|
|
378
|
+
file: entry.file,
|
|
379
|
+
size: entry.size,
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* Walk the store, newest session first, and import what the vault does not have.
|
|
385
|
+
*
|
|
386
|
+
* Returns a report rather than logging: the tool formats it, the tests assert on
|
|
387
|
+
* it, and a person reads the same numbers.
|
|
388
|
+
*/
|
|
389
|
+
export async function runBackfill({
|
|
390
|
+
cfg,
|
|
391
|
+
vault,
|
|
392
|
+
redactor,
|
|
393
|
+
dir = defaultSessionsDir(),
|
|
394
|
+
limit = 0,
|
|
395
|
+
dryRun = false,
|
|
396
|
+
maxBytes = DEFAULT_MAX_BYTES,
|
|
397
|
+
onProgress = null,
|
|
398
|
+
collect = null,
|
|
399
|
+
}) {
|
|
400
|
+
const started = Date.now()
|
|
401
|
+
const seen = await existingTurns(vault)
|
|
402
|
+
const all = await listSessions(dir)
|
|
403
|
+
const selected = limit > 0 ? all.slice(0, limit) : all
|
|
404
|
+
|
|
405
|
+
const sessions = []
|
|
406
|
+
let imported = 0
|
|
407
|
+
let duplicates = 0
|
|
408
|
+
let budget = maxBytes
|
|
409
|
+
let exhausted = false
|
|
410
|
+
|
|
411
|
+
for (const entry of selected) {
|
|
412
|
+
if (budget <= 0) {
|
|
413
|
+
exhausted = true
|
|
414
|
+
break
|
|
415
|
+
}
|
|
416
|
+
const report = await importSession({ entry, cfg, vault, redactor, seen, dryRun, maxBytes: budget, collect })
|
|
417
|
+
budget -= entry.size
|
|
418
|
+
imported += report.episodes
|
|
419
|
+
duplicates += report.duplicates
|
|
420
|
+
sessions.push(report)
|
|
421
|
+
onProgress?.(report)
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
// Views last, and only when something changed: `README.md` and `graph.html` are
|
|
425
|
+
// rendered from the cards, so an import that adds episodes but no cards changes
|
|
426
|
+
// nothing on either. The *reason* to do it here is that a backfill is the one
|
|
427
|
+
// operation that can add hundreds of episodes after the last render, and a stale
|
|
428
|
+
// view is indistinguishable from an empty vault when a person opens the file.
|
|
429
|
+
let refreshed = false
|
|
430
|
+
const previous = { readme: true, graph: true }
|
|
431
|
+
if (!dryRun && imported > 0) {
|
|
432
|
+
for (const [key, render] of [
|
|
433
|
+
['readme', vault.regenerateReadme],
|
|
434
|
+
['graph', vault.regenerateGraph],
|
|
435
|
+
]) {
|
|
436
|
+
if (typeof render !== 'function') {
|
|
437
|
+
previous[key] = false
|
|
438
|
+
continue
|
|
439
|
+
}
|
|
440
|
+
try {
|
|
441
|
+
await render.call(vault)
|
|
442
|
+
} catch {
|
|
443
|
+
// A view that will not render must not lose the episodes that just landed.
|
|
444
|
+
previous[key] = false
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
refreshed = Object.values(previous).some(Boolean)
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
return {
|
|
451
|
+
dir,
|
|
452
|
+
dryRun,
|
|
453
|
+
decoder: decoderKind,
|
|
454
|
+
found: all.length,
|
|
455
|
+
scanned: sessions.length,
|
|
456
|
+
imported,
|
|
457
|
+
duplicates,
|
|
458
|
+
exhausted,
|
|
459
|
+
refreshed,
|
|
460
|
+
malformed: sessions.reduce((n, s) => n + s.malformed, 0),
|
|
461
|
+
failed: sessions.filter((s) => s.failure !== null).length,
|
|
462
|
+
ms: Date.now() - started,
|
|
463
|
+
sessions: sessions
|
|
464
|
+
.filter((s) => s.episodes > 0 || s.duplicates > 0 || s.closedTurns > 0 || s.failure !== null)
|
|
465
|
+
.sort((a, b) => b.episodes - a.episodes),
|
|
466
|
+
}
|
|
467
|
+
}
|
package/src/pageapi.js
CHANGED
|
@@ -22,14 +22,14 @@ function failure(error) {
|
|
|
22
22
|
<style>body{font:14px/1.6 system-ui,sans-serif;margin:0;display:grid;place-items:center;height:100vh;color:#333}
|
|
23
23
|
main{max-width:34rem;padding:2rem}code{background:#f4f4f5;padding:.1rem .3rem;border-radius:4px}</style>
|
|
24
24
|
<main><h1>记忆图没能画出来</h1><p><code>${escaped}</code></p>
|
|
25
|
-
<p
|
|
25
|
+
<p>图谱渲染失败了,但记忆库还在磁盘上,什么都没写 · The graph could not be rendered. The vault is still on disk; nothing was written.</p></main></html>`
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
export function createPageHandler({ getVault }) {
|
|
29
29
|
return async function fetch() {
|
|
30
30
|
const vault = getVault()
|
|
31
31
|
if (!vault || typeof vault.graphPage !== 'function') {
|
|
32
|
-
return html(failure(new Error('vault unavailable')), 500)
|
|
32
|
+
return html(failure(new Error('记忆库不可用 · vault unavailable')), 500)
|
|
33
33
|
}
|
|
34
34
|
try {
|
|
35
35
|
return html(await vault.graphPage())
|
package/src/schema.js
CHANGED
|
@@ -292,6 +292,60 @@ const auto = z.object({
|
|
|
292
292
|
.default('')
|
|
293
293
|
.description('自动蒸馏记到哪个项目;空 = 最近情节的项目 · project for automatic cards; empty = the newest episode\'s project'),
|
|
294
294
|
),
|
|
295
|
+
catchUp: live(
|
|
296
|
+
z
|
|
297
|
+
.boolean()
|
|
298
|
+
.default(false)
|
|
299
|
+
.description('追平模式:只要还攒着一整批就按自己的节奏一批批蒸下去 · catch-up: keep distilling batch after batch while a full batch is still waiting'),
|
|
300
|
+
),
|
|
301
|
+
catchUpEveryMinutes: live(
|
|
302
|
+
z
|
|
303
|
+
.number()
|
|
304
|
+
.min(1)
|
|
305
|
+
.max(1440)
|
|
306
|
+
.step(1)
|
|
307
|
+
.default(10)
|
|
308
|
+
.description('追平模式每批之间隔多少分钟 · minutes between catch-up batches'),
|
|
309
|
+
),
|
|
310
|
+
maxBatchesPerDay: live(
|
|
311
|
+
z
|
|
312
|
+
.number()
|
|
313
|
+
.min(0)
|
|
314
|
+
.max(200)
|
|
315
|
+
.step(1)
|
|
316
|
+
.default(6)
|
|
317
|
+
.description('一天最多自动蒸几批;0 = 不限 · most automatic batches per day; 0 lifts the cap'),
|
|
318
|
+
),
|
|
319
|
+
})
|
|
320
|
+
|
|
321
|
+
const update = z.object({
|
|
322
|
+
enabled: live(
|
|
323
|
+
z
|
|
324
|
+
.boolean()
|
|
325
|
+
.default(true)
|
|
326
|
+
.description('定期检查 npm 上有没有新版本(只查,不装)· check npm for a newer version (checks only, never installs)'),
|
|
327
|
+
),
|
|
328
|
+
checkEveryHours: live(
|
|
329
|
+
z
|
|
330
|
+
.number()
|
|
331
|
+
.min(1)
|
|
332
|
+
.max(720)
|
|
333
|
+
.step(1)
|
|
334
|
+
.default(12)
|
|
335
|
+
.description('查到的结果保持多久 · hours a check result stays good for'),
|
|
336
|
+
),
|
|
337
|
+
autoInstall: live(
|
|
338
|
+
z
|
|
339
|
+
.boolean()
|
|
340
|
+
.default(false)
|
|
341
|
+
.description('启动时发现新版本就自动装(link: 本地安装会拒绝)· install a newer version at startup; a link: install is refused'),
|
|
342
|
+
),
|
|
343
|
+
registry: live(
|
|
344
|
+
z
|
|
345
|
+
.string()
|
|
346
|
+
.default('')
|
|
347
|
+
.description('用哪个 npm 源查版本;空 = 宿主的源,再退到 npmjs · registry to ask; empty = the harness\'s, then npmjs'),
|
|
348
|
+
),
|
|
295
349
|
})
|
|
296
350
|
|
|
297
351
|
export const Config = z.object({
|
|
@@ -317,6 +371,7 @@ export const Config = z.object({
|
|
|
317
371
|
privacy,
|
|
318
372
|
extractor,
|
|
319
373
|
auto,
|
|
374
|
+
update,
|
|
320
375
|
retrieval,
|
|
321
376
|
})
|
|
322
377
|
|
package/src/tools.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The model-facing surface.
|
|
2
|
+
* The model-facing surface. Eight small tools, all explicit.
|
|
3
3
|
*
|
|
4
4
|
* Nothing here runs on its own: retrieval only happens when the agent decides
|
|
5
5
|
* to ask, which keeps memory out of the context (and out of the attention
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import path from 'node:path'
|
|
9
9
|
import { searchCards, timelineCards } from './search.js'
|
|
10
10
|
import { distill, extractorServices, resolveRoute } from './extract.js'
|
|
11
|
+
import { runBackfill, defaultSessionsDir } from './history.js'
|
|
11
12
|
import { CARD_KINDS, CONFIDENCE, slugify } from './vault.js'
|
|
12
13
|
|
|
13
14
|
const OUTPUT_SCHEMA = { type: 'object', additionalProperties: true }
|
|
@@ -56,6 +57,27 @@ function formatHits(hits) {
|
|
|
56
57
|
*/
|
|
57
58
|
|
|
58
59
|
/** One line describing what a distill call would do right now, for memory_stats. */
|
|
60
|
+
/**
|
|
61
|
+
* The version line for memory_stats.
|
|
62
|
+
*
|
|
63
|
+
* Deliberately silent about a newer version when the check has not run yet: "no
|
|
64
|
+
* news" must not read as "you are up to date", and a check is an outbound request
|
|
65
|
+
* that a person did not necessarily just make.
|
|
66
|
+
*/
|
|
67
|
+
function describeUpdate(getUpdate) {
|
|
68
|
+
if (typeof getUpdate !== 'function') return '版本 version: 更新检查不可用 · update check unavailable'
|
|
69
|
+
let state = null
|
|
70
|
+
try {
|
|
71
|
+
state = getUpdate()
|
|
72
|
+
} catch {
|
|
73
|
+
return '版本 version: 更新检查不可用 · update check unavailable'
|
|
74
|
+
}
|
|
75
|
+
if (state === null || typeof state !== 'object' || state.current === null) return '版本 version: 未知 · unknown'
|
|
76
|
+
const current = `版本 version: \`${state.current}\``
|
|
77
|
+
if (state.latest === null) return `${current} · 还没有检查过更新 · no update check yet`
|
|
78
|
+
if (state.latest === state.current) return `${current} · 已是最新 · latest`
|
|
79
|
+
return `${current} · 有新版本 \`${state.latest}\`(设置页可更新)· newer version available`
|
|
80
|
+
}
|
|
59
81
|
function describeExtractor(ctx, cfg) {
|
|
60
82
|
if (!cfg.extractor.enabled) return 'extractor: off (nothing calls a model)'
|
|
61
83
|
if (cfg.extractor.transport === 'http') {
|
|
@@ -72,7 +94,7 @@ function describeExtractor(ctx, cfg) {
|
|
|
72
94
|
}
|
|
73
95
|
}
|
|
74
96
|
|
|
75
|
-
export function registerTools(ctx, { cfg, vault, redactor, runner = null }) {
|
|
97
|
+
export function registerTools(ctx, { cfg, vault, redactor, runner = null, getUpdate = null }) {
|
|
76
98
|
const disposers = []
|
|
77
99
|
|
|
78
100
|
/**
|
|
@@ -524,11 +546,63 @@ export function registerTools(ctx, { cfg, vault, redactor, runner = null }) {
|
|
|
524
546
|
describeExtractor(ctx, cfg),
|
|
525
547
|
`privacy rules: ${stats.privacy.rules}`,
|
|
526
548
|
`views: README.md and graph.html in the vault root (open the HTML directly, no server needed)`,
|
|
549
|
+
describeUpdate(getUpdate),
|
|
527
550
|
].join('\n')
|
|
528
551
|
return { ...stats, text }
|
|
529
552
|
},
|
|
530
553
|
}),
|
|
531
554
|
)
|
|
532
555
|
|
|
556
|
+
/* --------------------------------------------------------------- backfill */
|
|
557
|
+
disposers.push(
|
|
558
|
+
ctx.tools.register({
|
|
559
|
+
name: 'memory_backfill',
|
|
560
|
+
description:
|
|
561
|
+
'把「安装插件之前的」会话历史一次性导入记忆库。采集只在宿主运行时看得见当轮对话,所以新装的记忆库一开始是空的;这个工具回放磁盘上的历史会话文件(~/.dsh/sessions 下的 session.v4.jsonl.zstd),走的是与实时采集完全相同的过滤与脱敏逻辑,因此回填出来的情节和实时采集的是同一种记录。可重复执行:已经导入过的轮次会跳过,只补新的部分;时间戳用会话文件里的原始时间,所以历史落在时间轴的正确位置。默认只统计不写库(dryRun),确认后再真正导入。',
|
|
562
|
+
parameters: {
|
|
563
|
+
type: 'object',
|
|
564
|
+
properties: {
|
|
565
|
+
dryRun: { type: 'boolean', description: '只统计会导入多少条,不写库。默认 true。' },
|
|
566
|
+
limit: { type: 'integer', minimum: 0, description: '最多回填多少个会话,按最近修改时间优先。0 = 全部。默认 0。' },
|
|
567
|
+
sessionsDir: {
|
|
568
|
+
type: 'string',
|
|
569
|
+
description: '会话存储目录。省略则用环境变量 DSH_HOME 下的 sessions,再退到 ~/.dsh/sessions。',
|
|
570
|
+
},
|
|
571
|
+
},
|
|
572
|
+
additionalProperties: false,
|
|
573
|
+
},
|
|
574
|
+
output: { schema: OUTPUT_SCHEMA, render: (args, value) => json(args, value.text ?? value) },
|
|
575
|
+
async execute(args) {
|
|
576
|
+
const dryRun = args.dryRun !== false
|
|
577
|
+
const report = await runBackfill({
|
|
578
|
+
cfg,
|
|
579
|
+
vault,
|
|
580
|
+
redactor,
|
|
581
|
+
dir: args.sessionsDir ?? defaultSessionsDir(),
|
|
582
|
+
limit: args.limit ?? 0,
|
|
583
|
+
dryRun,
|
|
584
|
+
})
|
|
585
|
+
|
|
586
|
+
const lines = [
|
|
587
|
+
`${dryRun ? '试运行' : '已导入'}:会话目录 ${report.dir}(解码器 ${report.decoder})`,
|
|
588
|
+
`扫描 ${report.scanned} / 共找到 ${report.found} 个会话,用时 ${(report.ms / 1000).toFixed(1)}s`,
|
|
589
|
+
`${dryRun ? '将写入' : '已写入'} ${report.imported} 条情节;已存在跳过 ${report.duplicates} 条${report.malformed > 0 ? `;坏行 ${report.malformed} 条` : ''}${report.failed > 0 ? `;读不了 ${report.failed} 个会话` : ''}`,
|
|
590
|
+
]
|
|
591
|
+
if (report.exhausted) lines.push('达到单次读取上限,剩下的会话下次再跑即可(已导入的会自动跳过)。')
|
|
592
|
+
if (report.sessions.length > 0) {
|
|
593
|
+
lines.push('贡献最多的会话:')
|
|
594
|
+
for (const session of report.sessions.slice(0, 5)) {
|
|
595
|
+
lines.push(` ${session.id} ${session.project} ${session.episodes} 条${session.duplicates > 0 ? `(跳过 ${session.duplicates})` : ''}`)
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
if (report.imported > 0 && !dryRun) {
|
|
599
|
+
lines.push('这些情节还没有变成卡片:蒸馏一次(memory_distill 或设置页的「蒸馏一次」)才会进入检索层。')
|
|
600
|
+
if (report.refreshed === true) lines.push('README.md 和 graph.html 已经按新情节重画过一次。')
|
|
601
|
+
}
|
|
602
|
+
return { ...report, text: lines.join('\n') }
|
|
603
|
+
},
|
|
604
|
+
}),
|
|
605
|
+
)
|
|
606
|
+
|
|
533
607
|
return disposers
|
|
534
608
|
}
|