dsh-hypatia-auto-memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +454 -0
- package/cordis.patch.yml +20 -0
- package/lib/client.js +1081 -0
- package/lib/client.js.map +1 -0
- package/package.json +90 -0
- package/skills/hypatia/SKILL.md +585 -0
- package/skills/hypatia-dream/SKILL.md +246 -0
- package/skills/hypatia-dream/evals/evals.json +92 -0
- package/skills/hypatia-memory/SKILL.md +175 -0
- package/src/auto-approve.js +133 -0
- package/src/cascade.js +190 -0
- package/src/client/SettingsCard.module.css +420 -0
- package/src/client/SettingsCard.tsx +700 -0
- package/src/client/card-form.ts +450 -0
- package/src/client/consolidation-models.ts +69 -0
- package/src/client/css-modules.d.ts +4 -0
- package/src/client/index.tsx +106 -0
- package/src/client/locales.ts +111 -0
- package/src/client/shelves.ts +88 -0
- package/src/client/slot-contract.ts +17 -0
- package/src/collector.js +596 -0
- package/src/config.js +225 -0
- package/src/consolidator.js +592 -0
- package/src/content-policy.js +301 -0
- package/src/housekeeping.js +197 -0
- package/src/hypatia-cli.js +294 -0
- package/src/hypatia-client.js +72 -0
- package/src/hypatia-mcp.js +477 -0
- package/src/index.js +418 -0
- package/src/persisted-session.js +99 -0
- package/src/progress.js +56 -0
- package/src/queue.js +466 -0
- package/src/recall.js +148 -0
- package/src/shelf.js +180 -0
- package/src/skills.js +166 -0
- package/src/state.js +87 -0
- package/src/status.js +53 -0
- package/src/writer.js +320 -0
package/src/collector.js
ADDED
|
@@ -0,0 +1,596 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversation collector: listens to the session/event feed, selects the
|
|
3
|
+
* protocol's log-worthy events (human user messages, assistant messages,
|
|
4
|
+
* tool activity for the ledger, turn boundaries for triggers), resolves the
|
|
5
|
+
* project scope, and enqueues idempotent log tasks. Content is ALWAYS
|
|
6
|
+
* re-read from the authoritative session log at execution time — this module
|
|
7
|
+
* never caches message bodies.
|
|
8
|
+
*
|
|
9
|
+
* @module dsh-hypatia-auto-memory/collector
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { execFile } from 'node:child_process'
|
|
13
|
+
import { basename } from 'node:path'
|
|
14
|
+
import { promisify } from 'node:util'
|
|
15
|
+
import {
|
|
16
|
+
absolutizeDates,
|
|
17
|
+
blocksToText,
|
|
18
|
+
capAssistant,
|
|
19
|
+
estimateTokens,
|
|
20
|
+
eventDate,
|
|
21
|
+
flattenToolResult,
|
|
22
|
+
oneLineError,
|
|
23
|
+
projectScope,
|
|
24
|
+
redactSecrets,
|
|
25
|
+
} from './content-policy.js'
|
|
26
|
+
import { EMPTY_PROGRESS, advanceProgress } from './progress.js'
|
|
27
|
+
|
|
28
|
+
const execFileAsync = promisify(execFile)
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Environment variables that point git at a repository other than the one
|
|
32
|
+
* around `-C`. Inherited from whatever launched DSH (a git hook, say), they
|
|
33
|
+
* would decide every session's project.
|
|
34
|
+
*/
|
|
35
|
+
const GIT_LOCATION_VARS = ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR']
|
|
36
|
+
|
|
37
|
+
/** How long one `git rev-parse` may take before the cwd's own name is used. */
|
|
38
|
+
const GIT_TIMEOUT_MS = 3000
|
|
39
|
+
|
|
40
|
+
/** `process.env` without {@link GIT_LOCATION_VARS}. */
|
|
41
|
+
function gitEnv() {
|
|
42
|
+
const env = { ...process.env }
|
|
43
|
+
for (const name of GIT_LOCATION_VARS) delete env[name]
|
|
44
|
+
return env
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Events carrying a host-produced session summary, from which the protocol's
|
|
49
|
+
* `session-<id>` node is built. DSH emits no session-summary event of its own;
|
|
50
|
+
* these two are where a usable one actually appears.
|
|
51
|
+
*/
|
|
52
|
+
const SESSION_SUMMARY_TYPES = new Set(['session/title', 'compaction/summary'])
|
|
53
|
+
|
|
54
|
+
/** Events the log layer records, per the hypatia-memory protocol. */
|
|
55
|
+
const RECORDED_TYPES = new Set([
|
|
56
|
+
'user/message',
|
|
57
|
+
'assistant/message',
|
|
58
|
+
'tool/call',
|
|
59
|
+
'tool/result',
|
|
60
|
+
])
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Whether this event becomes one `msg-*` entry.
|
|
64
|
+
*
|
|
65
|
+
* Plugin-sourced user messages (injected context, including this plugin's own
|
|
66
|
+
* output) are excluded: they are not things the human said, and re-logging them
|
|
67
|
+
* would let memory amplify into itself.
|
|
68
|
+
*
|
|
69
|
+
* @param {any} event
|
|
70
|
+
* @returns {boolean}
|
|
71
|
+
*/
|
|
72
|
+
export function isLoggableMessage(event) {
|
|
73
|
+
if (event?.type === 'assistant/message') return true
|
|
74
|
+
return event?.type === 'user/message' && event.data?.source?.kind === 'user'
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Position of the next logged message, counted over `events`.
|
|
79
|
+
*
|
|
80
|
+
* Entry names are `msg-<session>-<N>` where N is the protocol's "monotonic turn
|
|
81
|
+
* counter within session (increment per logged message)" — a DENSE message
|
|
82
|
+
* ordinal, not a session-log seq. The distinction is load-bearing: DSH appends
|
|
83
|
+
* one `assistant/chunk` event per streamed token delta, so seq counts tokens.
|
|
84
|
+
* Naming entries by seq produced a sparse, unpredictable keyspace, which breaks
|
|
85
|
+
* `$not-summaried`'s FIFO batching and anything that walks `msg-*` in order.
|
|
86
|
+
*
|
|
87
|
+
* Derived by counting rather than stored, so the value cannot drift from the
|
|
88
|
+
* log: the log is append-only, so the same prefix always yields the same index,
|
|
89
|
+
* and a replay therefore reproduces exactly the same names.
|
|
90
|
+
*
|
|
91
|
+
* @param {readonly any[]} events - the prefix preceding the span being written.
|
|
92
|
+
* @returns {number}
|
|
93
|
+
*/
|
|
94
|
+
export function countLoggableMessages(events) {
|
|
95
|
+
let count = 0
|
|
96
|
+
for (const event of events ?? []) {
|
|
97
|
+
if (isLoggableMessage(event)) count += 1
|
|
98
|
+
}
|
|
99
|
+
return count
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* @param {{
|
|
104
|
+
* ctx: import('@deepseek-ai/cordis').Context,
|
|
105
|
+
* queue: ReturnType<import('./queue.js').createQueue>,
|
|
106
|
+
* progress: any,
|
|
107
|
+
* getConfig: () => any,
|
|
108
|
+
* status: import('./status.js').StatusLog,
|
|
109
|
+
* onTurnEnd: (sessionId: string, turn: number) => void,
|
|
110
|
+
* }} deps
|
|
111
|
+
*/
|
|
112
|
+
export function createCollector({ ctx, queue, progress, getConfig, status, onTurnEnd, onSessionEnd }) {
|
|
113
|
+
/** @type {Map<string, string>} sessionId -> resolved project scope. */
|
|
114
|
+
const projects = new Map()
|
|
115
|
+
/** @type {Map<string, number>} sessionId -> unconsolidated token estimate. */
|
|
116
|
+
const pendingTokens = new Map()
|
|
117
|
+
/** @type {Map<string, Promise<string>>} cwd -> project scope promise. */
|
|
118
|
+
const projectByCwd = new Map()
|
|
119
|
+
/**
|
|
120
|
+
* Sessions the store has released but whose queued work has not drained.
|
|
121
|
+
*
|
|
122
|
+
* Executors re-read content from the session log rather than caching it, so
|
|
123
|
+
* they need the object itself. A disposed session is still perfectly readable
|
|
124
|
+
* — `snapshotEvents` works fine — it is merely gone from `ctx.sessions`, so
|
|
125
|
+
* holding the reference across the drain is what lets end-of-session work run
|
|
126
|
+
* at all, rather than being deferred to wait for a session that has closed.
|
|
127
|
+
*
|
|
128
|
+
* @type {Map<string, any>}
|
|
129
|
+
*/
|
|
130
|
+
const detached = new Map()
|
|
131
|
+
|
|
132
|
+
const sessionIdOf = (session) => String(session?.header?.id ?? session?.id ?? '')
|
|
133
|
+
|
|
134
|
+
/** The live session, or one kept alive only until its queued work drains. */
|
|
135
|
+
function sessionFor(sessionId) {
|
|
136
|
+
return ctx.sessions?.get?.(sessionId) ?? detached.get(sessionId)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Drop every per-session cache entry once nothing can need it again. */
|
|
140
|
+
function forget(sessionId) {
|
|
141
|
+
detached.delete(sessionId)
|
|
142
|
+
projects.delete(sessionId)
|
|
143
|
+
pendingTokens.delete(sessionId)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Best-effort git-root basename; falls back to the bare cwd basename. Either
|
|
148
|
+
* way it becomes a scope hypatia stores as given (see `projectScope`).
|
|
149
|
+
*
|
|
150
|
+
* Only git's line ending is cut: trimming is `projectScope`'s, and JS `trim`
|
|
151
|
+
* strips characters hypatia keeps. A timeout says nothing about the
|
|
152
|
+
* directory, so its fallback is not remembered for the next caller.
|
|
153
|
+
*/
|
|
154
|
+
function resolveProjectForCwd(cwd) {
|
|
155
|
+
let cached = projectByCwd.get(cwd)
|
|
156
|
+
if (cached === undefined) {
|
|
157
|
+
cached = execFileAsync('git', ['-C', cwd, 'rev-parse', '--show-toplevel'], {
|
|
158
|
+
timeout: GIT_TIMEOUT_MS,
|
|
159
|
+
env: gitEnv(),
|
|
160
|
+
})
|
|
161
|
+
.then(({ stdout }) => {
|
|
162
|
+
const root = stdout.replace(/\r?\n$/, '')
|
|
163
|
+
// git before 2.25 printed an empty line, exit 0, outside a work tree.
|
|
164
|
+
if (root === '') throw new Error('git printed no top level')
|
|
165
|
+
return basename(root)
|
|
166
|
+
})
|
|
167
|
+
.catch((error) => {
|
|
168
|
+
if (error?.killed) projectByCwd.delete(cwd)
|
|
169
|
+
return basename(cwd)
|
|
170
|
+
})
|
|
171
|
+
.then(projectScope)
|
|
172
|
+
projectByCwd.set(cwd, cached)
|
|
173
|
+
}
|
|
174
|
+
return cached
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Resolve (and cache) the project scope for one session. */
|
|
178
|
+
async function projectFor(session) {
|
|
179
|
+
const id = sessionIdOf(session)
|
|
180
|
+
let project = projects.get(id)
|
|
181
|
+
if (project === undefined) {
|
|
182
|
+
const cwd = session?.header?.cwd ?? session?.meta?.cwd ?? process.cwd()
|
|
183
|
+
project = await resolveProjectForCwd(String(cwd))
|
|
184
|
+
projects.set(id, project)
|
|
185
|
+
}
|
|
186
|
+
return project
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* True when this user/message event is a direct human prompt (as opposed to
|
|
191
|
+
* plugin-injected context — including OUR OWN recall injections, which must
|
|
192
|
+
* never be re-logged: a feedback loop would amplify memory into itself).
|
|
193
|
+
*/
|
|
194
|
+
function isHumanMessage(event) {
|
|
195
|
+
return event.data?.source?.kind === 'user'
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** Extract plain text from a user/message payload. */
|
|
199
|
+
function userText(event) {
|
|
200
|
+
return blocksToText(event.data?.content ?? [])
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Extract plain text from an assistant/message payload. */
|
|
204
|
+
function assistantText(event) {
|
|
205
|
+
return blocksToText(event.data?.message?.content ?? [])
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Accumulate the unconsolidated token estimate for one recorded event. */
|
|
209
|
+
function accrueTokens(session, event) {
|
|
210
|
+
const text = event.type === 'assistant/message'
|
|
211
|
+
? assistantText(event)
|
|
212
|
+
: event.type === 'user/message' ? userText(event) : ''
|
|
213
|
+
if (text.trim() === '') return
|
|
214
|
+
const id = sessionIdOf(session)
|
|
215
|
+
pendingTokens.set(id, (pendingTokens.get(id) ?? 0) + estimateTokens(text))
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Enqueue everything this session has not logged yet, up to `toSeq`.
|
|
220
|
+
*
|
|
221
|
+
* The lower bound is the watermark rather than any particular event, so the
|
|
222
|
+
* span is whole by construction — including messages the loop appended before
|
|
223
|
+
* the first `step/start` of the turn.
|
|
224
|
+
*/
|
|
225
|
+
async function enqueueUnlogged(session, toSeq, { immediate = false } = {}) {
|
|
226
|
+
const id = sessionIdOf(session)
|
|
227
|
+
const current = progress.get(id) ?? EMPTY_PROGRESS
|
|
228
|
+
// A forked child must not re-log the prefix it inherited from its parent.
|
|
229
|
+
const fromSeq = Math.max(current.lastLoggedSeq, session.inheritedEventCount ?? 0)
|
|
230
|
+
if (toSeq <= fromSeq) return
|
|
231
|
+
const project = await projectFor(session)
|
|
232
|
+
await queue.enqueue({ kind: 'log-message', sessionId: id, fromSeq, toSeq, project, immediate })
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Enqueue creation of the `session-<id>` node from the summary at `seq`. */
|
|
236
|
+
async function enqueueSessionNode(session, seq) {
|
|
237
|
+
const id = sessionIdOf(session)
|
|
238
|
+
const project = await projectFor(session)
|
|
239
|
+
await queue.enqueue({
|
|
240
|
+
kind: 'session-node',
|
|
241
|
+
sessionId: id,
|
|
242
|
+
fromSeq: seq,
|
|
243
|
+
toSeq: seq + 1,
|
|
244
|
+
project,
|
|
245
|
+
immediate: true,
|
|
246
|
+
})
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// ---- session/event listener -------------------------------------------------
|
|
250
|
+
|
|
251
|
+
const listener = (session, event) => {
|
|
252
|
+
if (session === undefined || event === undefined) return
|
|
253
|
+
if (getConfig().collector?.enabled === false) return
|
|
254
|
+
if (!RECORDED_TYPES.has(event.type)) {
|
|
255
|
+
if (SESSION_SUMMARY_TYPES.has(event.type)) {
|
|
256
|
+
// The task carries the seq, not the text: content is always re-read
|
|
257
|
+
// from the authoritative log at execution time.
|
|
258
|
+
void enqueueSessionNode(session, event.seq).catch((error) => {
|
|
259
|
+
status.error('collector failed to enqueue the session node', error)
|
|
260
|
+
})
|
|
261
|
+
return
|
|
262
|
+
}
|
|
263
|
+
if (event.type === 'step/end') {
|
|
264
|
+
// The fallback flush is armed here rather than on each message.
|
|
265
|
+
// `step/end` is appended once the step's tools have all run, so a span
|
|
266
|
+
// cut at one never separates an assistant message from its own tool
|
|
267
|
+
// results. Arming it on the message instead meant a turn longer than
|
|
268
|
+
// the window was cut mid-step, and every entry written before the cut
|
|
269
|
+
// kept a partial tool ledger for good (entries are never rewritten).
|
|
270
|
+
const window = getConfig().queue?.flushWindowMs ?? 0
|
|
271
|
+
if (window > 0) {
|
|
272
|
+
void enqueueUnlogged(session, event.seq + 1).catch((error) => {
|
|
273
|
+
status.error('collector failed to arm the fallback flush', error)
|
|
274
|
+
})
|
|
275
|
+
}
|
|
276
|
+
return
|
|
277
|
+
}
|
|
278
|
+
if (event.type === 'turn/end') {
|
|
279
|
+
const id = sessionIdOf(session)
|
|
280
|
+
const turn = event.data?.turn ?? 0
|
|
281
|
+
const previous = progress.get(id) ?? EMPTY_PROGRESS
|
|
282
|
+
// Flush at the turn boundary, never mid-turn. `assistant/message` is
|
|
283
|
+
// appended BEFORE the tools it requested run, so a timer that fired
|
|
284
|
+
// while a slow tool was still working split the assistant entry from
|
|
285
|
+
// its own results — and since `formatSpan` only emits a tool ledger
|
|
286
|
+
// alongside an assistant message, those results were then dropped
|
|
287
|
+
// entirely rather than merely landing late. `turn/end` is appended once
|
|
288
|
+
// the step loop is done, so a turn-aligned span always holds both.
|
|
289
|
+
void enqueueUnlogged(session, event.seq + 1, { immediate: true })
|
|
290
|
+
.catch((error) => status.error('collector failed to enqueue turn span', error))
|
|
291
|
+
// MAX of the two sources, not first-present: the in-memory counter
|
|
292
|
+
// resets on process restart while the durable record keeps the
|
|
293
|
+
// pre-restart accumulation — taking the in-memory value whenever it
|
|
294
|
+
// merely exists would regress the count (e.g. durable 672 -> 220).
|
|
295
|
+
const tokens = Math.max(pendingTokens.get(id) ?? 0, previous.pendingTokens ?? 0)
|
|
296
|
+
// The checkpoint is a minimum-interval anchor, not a polling cursor.
|
|
297
|
+
// Advance it only after consolidation work is durably accepted. When
|
|
298
|
+
// tokens are low, logging lags, or enqueueing fails, retain the anchor
|
|
299
|
+
// so the next turn can retry immediately instead of waiting N turns.
|
|
300
|
+
// `reason` travels with the trigger state so the consolidator can tell a
|
|
301
|
+
// completed turn from a user interrupt or a failure. Its `kind` set is
|
|
302
|
+
// merge-extensible, so it is passed through verbatim rather than
|
|
303
|
+
// interpreted here.
|
|
304
|
+
void Promise.resolve(onTurnEnd(id, turn, {
|
|
305
|
+
...previous,
|
|
306
|
+
pendingTokens: tokens,
|
|
307
|
+
reason: event.data?.reason,
|
|
308
|
+
}))
|
|
309
|
+
.then((decision) => {
|
|
310
|
+
if (decision?.resetTokens === true) pendingTokens.delete(id)
|
|
311
|
+
return advanceProgress(progress, id, (current) => ({
|
|
312
|
+
lastCheckTurn: decision?.advanceCheckpoint === true ? turn : current.lastCheckTurn,
|
|
313
|
+
pendingTokens: decision?.resetTokens === true ? 0 : tokens,
|
|
314
|
+
}))
|
|
315
|
+
})
|
|
316
|
+
.catch((error) => {
|
|
317
|
+
status.error('consolidation trigger failed', error)
|
|
318
|
+
return advanceProgress(progress, id, () => ({ pendingTokens: tokens }))
|
|
319
|
+
})
|
|
320
|
+
}
|
|
321
|
+
return
|
|
322
|
+
}
|
|
323
|
+
if (event.type === 'user/message' && !isHumanMessage(event)) return
|
|
324
|
+
accrueTokens(session, event)
|
|
325
|
+
// No enqueue here: spans are cut at `step/end` (the fallback) and at
|
|
326
|
+
// `turn/end`, both handled above.
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
ctx.on('session/event', listener)
|
|
330
|
+
|
|
331
|
+
// ---- session close ------------------------------------------------------------
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* Final flush and consolidation for a session that is going away.
|
|
335
|
+
*
|
|
336
|
+
* DSH has no `session/end` LOG event; `session/disposed` is the only close
|
|
337
|
+
* signal, and it hands over the session object as it leaves the store. This
|
|
338
|
+
* is the one chance to turn a short exchange into knowledge: the ongoing
|
|
339
|
+
* thresholds (`checkEveryTurns`, `minNewTokens`) are pacing knobs for a live
|
|
340
|
+
* conversation, and a task that finishes under both of them would otherwise
|
|
341
|
+
* be logged and never extracted.
|
|
342
|
+
*/
|
|
343
|
+
async function finishSession(session) {
|
|
344
|
+
const id = sessionIdOf(session)
|
|
345
|
+
if (id === '') return
|
|
346
|
+
detached.set(id, session)
|
|
347
|
+
try {
|
|
348
|
+
await enqueueUnlogged(session, session.seq ?? 0, { immediate: true })
|
|
349
|
+
if (onSessionEnd !== undefined) await onSessionEnd(id, session)
|
|
350
|
+
const drained = await queue.whenIdle(id)
|
|
351
|
+
if (!drained) status.warn(`session ${id} still had queued work when its grace period expired`)
|
|
352
|
+
} catch (error) {
|
|
353
|
+
status.error('session-end consolidation failed', error)
|
|
354
|
+
} finally {
|
|
355
|
+
forget(id)
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
ctx.on('session/disposed', (session) => {
|
|
360
|
+
if (getConfig().collector?.enabled === false) return
|
|
361
|
+
void finishSession(session)
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
// ---- session (re)open -----------------------------------------------------------
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* Resume whatever this session's tasks were waiting on.
|
|
368
|
+
*
|
|
369
|
+
* DSH loads sessions lazily, so work persisted by an earlier run — or queued
|
|
370
|
+
* for a session that closed before it ran — finds no session to read and is
|
|
371
|
+
* deferred rather than failed. `session/created` fires both for a new session
|
|
372
|
+
* and for one reopened from disk, which is the moment that work can run.
|
|
373
|
+
*/
|
|
374
|
+
ctx.on('session/created', (session) => {
|
|
375
|
+
const id = sessionIdOf(session)
|
|
376
|
+
if (id === '' || typeof queue.resumeSession !== 'function') return
|
|
377
|
+
const resumed = queue.resumeSession(id)
|
|
378
|
+
if (resumed > 0) status.info(`session ${id} is live again; resumed ${resumed} deferred task(s)`)
|
|
379
|
+
})
|
|
380
|
+
|
|
381
|
+
// ---- boot backfill ------------------------------------------------------------
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* For every live session, enqueue any log range the watermark has not
|
|
385
|
+
* covered — from plugin crash, kill -9, or HMR reload. Under the same task
|
|
386
|
+
* ids the queue coalesces these with anything already persisted.
|
|
387
|
+
*/
|
|
388
|
+
async function backfillLiveSessions() {
|
|
389
|
+
const live = ctx.sessions?.list?.() ?? []
|
|
390
|
+
for (const session of live) {
|
|
391
|
+
const id = sessionIdOf(session)
|
|
392
|
+
const before = (progress.get(id) ?? EMPTY_PROGRESS).lastLoggedSeq
|
|
393
|
+
const end = session.seq ?? 0
|
|
394
|
+
// Immediate: a boot backfill is recovering a span that is already closed,
|
|
395
|
+
// so there is nothing for a coalescing window to wait for.
|
|
396
|
+
await enqueueUnlogged(session, end, { immediate: true })
|
|
397
|
+
if (end > Math.max(before, session.inheritedEventCount ?? 0)) {
|
|
398
|
+
status.info(`backfill: session ${id} range [${Math.max(before, session.inheritedEventCount ?? 0)}, ${end})`)
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
return {
|
|
404
|
+
backfillLiveSessions,
|
|
405
|
+
projectFor,
|
|
406
|
+
// Startup housekeeping resolves a scope from a persisted session header's
|
|
407
|
+
// cwd, with no Session to hand — `projectFor` needs one, and caches by its
|
|
408
|
+
// id, so it cannot serve that path.
|
|
409
|
+
projectForCwd: resolveProjectForCwd,
|
|
410
|
+
sessionFor,
|
|
411
|
+
/** Drain the in-memory pending-token counter (called after a consolidation trigger). */
|
|
412
|
+
takePendingTokens(sessionId) {
|
|
413
|
+
const value = pendingTokens.get(sessionId) ?? 0
|
|
414
|
+
pendingTokens.delete(sessionId)
|
|
415
|
+
return value
|
|
416
|
+
},
|
|
417
|
+
setPendingTokens(sessionId, value) {
|
|
418
|
+
pendingTokens.set(sessionId, value)
|
|
419
|
+
},
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Format one event span into per-message markdown per the protocol's logging
|
|
425
|
+
* rules (redaction, date absolutization, assistant capping, tool ledger).
|
|
426
|
+
* Reads events from the session log snapshot [fromSeq, toSeq).
|
|
427
|
+
*
|
|
428
|
+
* Exported for the queue executor and for tests.
|
|
429
|
+
*
|
|
430
|
+
* @param {readonly any[]} events - events exactly as recorded (with seq/time/data).
|
|
431
|
+
* @param {{now?: Date, maxAssistantChars?: number, maxUserChars?: number,
|
|
432
|
+
* toolLedger?: boolean, baseIndex?: number, before?: any}} [policy]
|
|
433
|
+
* `baseIndex` is the dense message ordinal the first entry of this span takes;
|
|
434
|
+
* see {@link countLoggableMessages}. `before` is the event just ahead of the
|
|
435
|
+
* span, which decides whether a leading `tool/result` is a compaction
|
|
436
|
+
* replacement (see {@link isCompactionReplacement}). `now` is only the fallback
|
|
437
|
+
* base for relative dates in an event that carries no timestamp.
|
|
438
|
+
* @returns {Array<{seq: number, index: number, role: string, markdown: string, tokens: number}>}
|
|
439
|
+
*/
|
|
440
|
+
export function formatSpan(events, policy = {}) {
|
|
441
|
+
const now = policy.now ?? new Date()
|
|
442
|
+
const maxAssistantChars = policy.maxAssistantChars ?? 8000
|
|
443
|
+
const maxUserChars = policy.maxUserChars ?? 32000
|
|
444
|
+
const toolLedger = policy.toolLedger !== false
|
|
445
|
+
let index = policy.baseIndex ?? 0
|
|
446
|
+
const bySeq = new Map(events.map((e) => [e.seq, e]))
|
|
447
|
+
|
|
448
|
+
// Pass 1: pair tool/call with tool/result inside this span -> ledger by step.
|
|
449
|
+
// A step is one model call plus the tools it requested, so a step is exactly
|
|
450
|
+
// what one assistant message's ledger should cover. Keying by turn instead
|
|
451
|
+
// hung the whole turn's tool activity on every assistant message in it.
|
|
452
|
+
//
|
|
453
|
+
// The ledger records what was called, how long it took and whether it
|
|
454
|
+
// succeeded — never what it returned (the hypatia-memory protocol: "never raw
|
|
455
|
+
// outputs"). Output excerpts were 84% of all stored message bytes, mostly
|
|
456
|
+
// file paths and file contents from `read`, and they made unrelated keyword
|
|
457
|
+
// searches match raw chat logs through text nobody had written.
|
|
458
|
+
/** @type {Map<string, Array<{name: string, ok: boolean, ms: number | undefined, error: string | undefined}>>} */
|
|
459
|
+
const ledgerByStep = new Map()
|
|
460
|
+
if (toolLedger) {
|
|
461
|
+
for (const event of events) {
|
|
462
|
+
if (event.type !== 'tool/result') continue
|
|
463
|
+
if (isCompactionReplacement(event, bySeq, policy.before)) continue
|
|
464
|
+
const callId = event.data?.message?.source?.callId
|
|
465
|
+
const call = callId === undefined ? undefined : bySeq.get(findCallSeq(events, callId))
|
|
466
|
+
const name = call?.data?.name ?? 'tool'
|
|
467
|
+
const failure = event.data?.error
|
|
468
|
+
// A failure keeps one line of what went wrong: that is the part worth
|
|
469
|
+
// remembering, and the protocol asks for it.
|
|
470
|
+
const detail = failure === undefined ? '' : oneLineError(flattenToolResult(event))
|
|
471
|
+
const error = failure === undefined
|
|
472
|
+
? undefined
|
|
473
|
+
: `${failure.name}${failure.code ? ` (${failure.code})` : ''}${detail === '' ? '' : ` — ${detail}`}`
|
|
474
|
+
const ms = typeof call?.time === 'number' && typeof event.time === 'number' && event.time >= call.time
|
|
475
|
+
? event.time - call.time
|
|
476
|
+
: undefined
|
|
477
|
+
const key = stepKey(event)
|
|
478
|
+
const list = ledgerByStep.get(key) ?? []
|
|
479
|
+
list.push({ name, ok: failure === undefined, ms, error })
|
|
480
|
+
ledgerByStep.set(key, list)
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// Pass 2: format user/assistant messages.
|
|
485
|
+
const formatted = []
|
|
486
|
+
for (const event of events) {
|
|
487
|
+
if (!isLoggableMessage(event)) continue
|
|
488
|
+
|
|
489
|
+
const role = event.type === 'user/message' ? 'user' : 'assistant'
|
|
490
|
+
const raw = role === 'user' ? blocksToText(event.data?.content ?? []) : blocksToText(event.data?.message?.content ?? [])
|
|
491
|
+
// Redact BEFORE capping: a cut landing inside a secret would otherwise leave
|
|
492
|
+
// a fragment too short for the patterns to recognise. Relative dates are
|
|
493
|
+
// resolved against when the message was sent, not when it is written.
|
|
494
|
+
const limit = role === 'assistant' ? maxAssistantChars : maxUserChars
|
|
495
|
+
const redacted = absolutizeDates(capAssistant(redactSecrets(raw), limit), eventDate(event, now))
|
|
496
|
+
const interrupted = event.type === 'assistant/message' && event.data?.interrupted === true
|
|
497
|
+
|
|
498
|
+
const sections = [
|
|
499
|
+
'## Role',
|
|
500
|
+
role,
|
|
501
|
+
'',
|
|
502
|
+
'## Timestamp',
|
|
503
|
+
new Date(event.time).toISOString(),
|
|
504
|
+
'',
|
|
505
|
+
]
|
|
506
|
+
// Only assistant/message payloads carry a turn number; user messages are
|
|
507
|
+
// positioned between turns, so stamping them with a fake turn 0 would lie.
|
|
508
|
+
if (role === 'assistant') {
|
|
509
|
+
sections.push('## Turn', String(event.data?.turn ?? 0), '')
|
|
510
|
+
}
|
|
511
|
+
const ledger = role === 'assistant' ? ledgerByStep.get(stepKey(event)) : undefined
|
|
512
|
+
const hasLedger = ledger !== undefined && ledger.length > 0
|
|
513
|
+
// A step that only called tools has no text of its own; say so rather than
|
|
514
|
+
// leave an empty section.
|
|
515
|
+
const body = redacted.trim() === '' && hasLedger ? '(tool calls only)' : redacted
|
|
516
|
+
sections.push('## Content', body + (interrupted ? '\n\n[turn interrupted mid-stream]' : ''))
|
|
517
|
+
|
|
518
|
+
if (hasLedger) {
|
|
519
|
+
sections.push('', '## Tool Calls', ...renderLedger(ledger))
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
const markdown = sections.join('\n')
|
|
523
|
+
formatted.push({ seq: event.seq, index, role, markdown, tokens: estimateTokens(markdown) })
|
|
524
|
+
index += 1
|
|
525
|
+
}
|
|
526
|
+
return formatted
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/** Ledger key: tool activity belongs to the step that requested it. */
|
|
530
|
+
function stepKey(event) {
|
|
531
|
+
return `${event.data?.turn ?? 0}:${event.data?.step ?? 0}`
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
/**
|
|
535
|
+
* Whether a `tool/result` is a compaction replacement rather than a tool outcome.
|
|
536
|
+
*
|
|
537
|
+
* When DSH compaction prunes an old tool result it appends a replacement copy of
|
|
538
|
+
* it IMMEDIATELY after its `compaction/prune` event — the adjacency is
|
|
539
|
+
* contractual in dsh-compaction. The copy carries the original turn/step, often
|
|
540
|
+
* tens of thousands of events after that step ended, so it restates something
|
|
541
|
+
* already recorded. Treating it as new activity would re-add it to a ledger or a
|
|
542
|
+
* transcript, and it is the only way a result ever follows its own `step/end`.
|
|
543
|
+
*
|
|
544
|
+
* @param {any} event
|
|
545
|
+
* @param {Map<number, any>} bySeq - events of the span, by seq.
|
|
546
|
+
* @param {any} [before] - the event just ahead of the span, if known.
|
|
547
|
+
* @returns {boolean}
|
|
548
|
+
*/
|
|
549
|
+
export function isCompactionReplacement(event, bySeq, before) {
|
|
550
|
+
if (event?.type !== 'tool/result') return false
|
|
551
|
+
const previous = bySeq.get(event.seq - 1) ?? (before?.seq === event.seq - 1 ? before : undefined)
|
|
552
|
+
return previous?.type === 'compaction/prune'
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** Locate the tool/call seq for one callId within the span. */
|
|
556
|
+
function findCallSeq(events, callId) {
|
|
557
|
+
for (const event of events) {
|
|
558
|
+
if (event.type === 'tool/call' && event.data?.callId === callId) return event.seq
|
|
559
|
+
}
|
|
560
|
+
return -1
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
/** Human-scale duration: `340ms`, `2.4s`, `3m12s`. */
|
|
564
|
+
export function formatDuration(ms) {
|
|
565
|
+
if (ms < 1000) return `${Math.round(ms)}ms`
|
|
566
|
+
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`
|
|
567
|
+
const minutes = Math.floor(ms / 60_000)
|
|
568
|
+
return `${minutes}m${Math.round((ms - minutes * 60_000) / 1000)}s`
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* Render one step's ledger. Consecutive calls of the same tool with the same
|
|
573
|
+
* outcome collapse into one numbered row carrying a count and their total
|
|
574
|
+
* time; a failure keeps its one-line error.
|
|
575
|
+
*
|
|
576
|
+
* 1. `read` ×5 — ✅ 1.2s total
|
|
577
|
+
* 2. `bash` — ❌ 2.0s — Error (ENOENT) — no such file
|
|
578
|
+
*/
|
|
579
|
+
function renderLedger(ledger) {
|
|
580
|
+
const rows = []
|
|
581
|
+
for (const entry of ledger) {
|
|
582
|
+
const prev = rows[rows.length - 1]
|
|
583
|
+
if (prev !== undefined && prev.name === entry.name && prev.ok === entry.ok && prev.error === entry.error) {
|
|
584
|
+
prev.count += 1
|
|
585
|
+
prev.ms = prev.ms === undefined || entry.ms === undefined ? undefined : prev.ms + entry.ms
|
|
586
|
+
} else {
|
|
587
|
+
rows.push({ ...entry, count: 1 })
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
return rows.map((row, i) => {
|
|
591
|
+
const times = row.count > 1 ? ` ×${row.count}` : ''
|
|
592
|
+
const took = row.ms === undefined ? '' : ` ${formatDuration(row.ms)}${row.count > 1 ? ' total' : ''}`
|
|
593
|
+
const tail = row.ok ? '' : ` — ${row.error}`
|
|
594
|
+
return `${i + 1}. \`${row.name}\`${times} — ${row.ok ? '✅' : '❌'}${took}${tail}`
|
|
595
|
+
})
|
|
596
|
+
}
|