@xiangyixie/dsh-superagent 0.0.0-stage → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/store.js ADDED
@@ -0,0 +1,385 @@
1
+ /**
2
+ * Durable run storage for dsh-superagent.
3
+ *
4
+ * One JSON file per run under `<root>/runs/<runId>.json`. Writes are atomic
5
+ * (temp file + rename) so a host crash mid-write cannot leave a half-written
6
+ * run that blocks recovery.
7
+ *
8
+ * Sibling files carry signals that are safe to write from another process:
9
+ * `<runId>.cancel` asks the owning process to stop the run, `<runId>.lock`
10
+ * claims the right to schedule it so two processes cannot run the same one, and
11
+ * `<runId>.json.corrupt-<ts>` keeps an unparsable state file instead of
12
+ * deleting the evidence.
13
+ *
14
+ * @module dsh-superagent/store
15
+ */
16
+ import fsp from 'node:fs/promises'
17
+ import os from 'node:os'
18
+ import path from 'node:path'
19
+
20
+ /** Statuses for a run that still owns live work. */
21
+ export const ACTIVE_STATUSES = new Set(['queued', 'planning', 'running', 'aggregating'])
22
+
23
+ /** Statuses after which a run will not change on its own. */
24
+ export const TERMINAL_STATUSES = new Set(['completed', 'partial', 'failed', 'cancelled', 'interrupted'])
25
+
26
+ const RUN_ID_PATTERN = /^[0-9A-Za-z][0-9A-Za-z-]{3,63}$/
27
+ const STALE_TMP_MS = 10 * 60 * 1000
28
+
29
+ /**
30
+ * Resolve the state root. An explicit `stateDir` wins, then `$DSH_HOME`, then
31
+ * the harness default `~/.dsh`.
32
+ * @param {string | undefined} configured - plugin config `stateDir`.
33
+ * @returns {string} absolute state root.
34
+ */
35
+ export function resolveStateRoot(configured) {
36
+ if (typeof configured === 'string' && configured.trim()) return path.resolve(configured.trim())
37
+ const home = process.env.DSH_HOME?.trim() || path.join(os.homedir(), '.dsh')
38
+ return path.join(home, 'state', 'superagent')
39
+ }
40
+
41
+ /**
42
+ * Build a store bound to one state root.
43
+ * @param {string} root - absolute state root.
44
+ * @param {{ warn?: (message: string) => void }} [options] - diagnostics sink.
45
+ */
46
+ export function createStore(root, options = {}) {
47
+ const runsDir = path.join(root, 'runs')
48
+ const warn = options.warn ?? (() => {})
49
+ // Two saves of the same run can land in the same millisecond, so the temp
50
+ // name needs a counter as well as a clock.
51
+ let writeSeq = 0
52
+ /** runId -> quarantine path, so status can explain a run it cannot parse. */
53
+ const corrupt = new Map()
54
+
55
+ const runFile = runId => path.join(runsDir, `${runId}.json`)
56
+ const cancelFile = runId => path.join(runsDir, `${runId}.cancel`)
57
+ const lockFile = runId => path.join(runsDir, `${runId}.lock`)
58
+ const ownerFile = path.join(root, '.owner')
59
+
60
+ return {
61
+ root,
62
+ runsDir,
63
+ ownerFile,
64
+ corrupt,
65
+
66
+ async init() {
67
+ await fsp.mkdir(runsDir, { recursive: true })
68
+ await this.cleanup()
69
+ },
70
+
71
+ /**
72
+ * Remove temp files left by a killed process and sibling markers whose run
73
+ * already reached a terminal state, so none of them can affect later runs.
74
+ */
75
+ async cleanup() {
76
+ let names
77
+ try {
78
+ names = await fsp.readdir(runsDir)
79
+ } catch {
80
+ return
81
+ }
82
+ for (const name of names) {
83
+ const file = path.join(runsDir, name)
84
+ if (name.startsWith('.') && name.endsWith('.tmp')) {
85
+ try {
86
+ const stat = await fsp.stat(file)
87
+ if (Date.now() - stat.mtimeMs > STALE_TMP_MS) await fsp.unlink(file)
88
+ } catch {
89
+ // Already gone; nothing to clean.
90
+ }
91
+ continue
92
+ }
93
+ const marker = ['.cancel', '.lock'].find(suffix => name.endsWith(suffix))
94
+ if (!marker) continue
95
+ let run = null
96
+ let readable = true
97
+ try {
98
+ run = await this.read(name.slice(0, -marker.length))
99
+ } catch {
100
+ // An unreadable run file is not a reason to drop its control files.
101
+ readable = false
102
+ }
103
+ if (!readable) continue
104
+ if (!run || TERMINAL_STATUSES.has(run.status)) {
105
+ try {
106
+ await fsp.unlink(file)
107
+ } catch {
108
+ // Nothing to clean.
109
+ }
110
+ }
111
+ }
112
+ },
113
+
114
+ /**
115
+ * Read every readable run, newest first. A corrupt file is quarantined and
116
+ * skipped rather than failing the whole listing; a file that cannot be read
117
+ * at all is reported and skipped for the same reason.
118
+ */
119
+ async list() {
120
+ let names
121
+ try {
122
+ names = await fsp.readdir(runsDir)
123
+ } catch (error) {
124
+ // A missing runs directory means initialization did not happen; saying
125
+ // "no runs recorded" would hide that. Only an empty directory is empty.
126
+ const code = error?.code ? `${error.code}: ` : ''
127
+ throw new Error(
128
+ `cannot read the run state directory ${runsDir} (${code}${error?.message ?? error}); `
129
+ + 'the plugin could not initialize it, so no run can be listed or resumed',
130
+ )
131
+ }
132
+ const runs = []
133
+ for (const name of names) {
134
+ if (!name.endsWith('.json') || name.startsWith('.')) continue
135
+ try {
136
+ const run = await this.read(name.slice(0, -'.json'.length))
137
+ if (run) runs.push(run)
138
+ } catch (error) {
139
+ warn(`skipping run file ${name}: ${error?.message ?? error}`)
140
+ }
141
+ }
142
+ runs.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))
143
+ return runs
144
+ },
145
+
146
+ /**
147
+ * @param {string} runId - run identity.
148
+ * @returns {Promise<object | null>} parsed run, or null when missing/invalid.
149
+ * @throws when the file exists but cannot be read (EPERM, EMFILE, ENOSPC...).
150
+ * Reporting that as "unknown run" would hide a real filesystem problem.
151
+ */
152
+ async read(runId) {
153
+ if (!RUN_ID_PATTERN.test(String(runId))) return null
154
+ // Windows can fail a read with EBUSY/EACCES for a moment while another
155
+ // process replaces the file. Those are transient for a reader, so retry
156
+ // before turning a poll into a hard error; real permission errors still
157
+ // surface after the retries.
158
+ for (let attempt = 1; ; attempt++) {
159
+ try {
160
+ const text = await fsp.readFile(runFile(runId), 'utf8')
161
+ const run = JSON.parse(text)
162
+ if (!run || typeof run !== 'object') {
163
+ await this.quarantine(runId)
164
+ return null
165
+ }
166
+ return run
167
+ } catch (error) {
168
+ if (error?.code === 'ENOENT') return null
169
+ if (error instanceof SyntaxError) {
170
+ const moved = await this.quarantine(runId)
171
+ warn(`state file for run ${runId} was not valid JSON; kept at ${moved ?? 'the runs directory'}`)
172
+ return null
173
+ }
174
+ const transient = ['EPERM', 'EACCES', 'EBUSY', 'EMFILE'].includes(error?.code)
175
+ if (transient && attempt < 3) {
176
+ await new Promise(resolve => setTimeout(resolve, 15 * attempt))
177
+ continue
178
+ }
179
+ const code = error?.code ? `${error.code}: ` : ''
180
+ throw new Error(`cannot read state for run ${runId} (${code}${error?.message ?? error})`)
181
+ }
182
+ }
183
+ },
184
+
185
+ /**
186
+ * Move an unparsable state file aside so it neither blocks recovery nor
187
+ * gets overwritten by the next save of the same id.
188
+ * @param {string} runId - run identity.
189
+ * @returns {Promise<string | null>} new path, or null when nothing moved.
190
+ */
191
+ async quarantine(runId) {
192
+ const target = `${runFile(runId)}.corrupt-${Date.now()}`
193
+ try {
194
+ await fsp.rename(runFile(runId), target)
195
+ corrupt.set(runId, target)
196
+ return target
197
+ } catch {
198
+ return null
199
+ }
200
+ },
201
+
202
+ /**
203
+ * Atomically persist one run.
204
+ * @param {object} run - run record with an `id`.
205
+ */
206
+ async write(run) {
207
+ const file = runFile(run.id)
208
+ const tmp = path.join(runsDir, `.${run.id}.${process.pid}.${++writeSeq}.tmp`)
209
+ await fsp.writeFile(tmp, `${JSON.stringify(run, null, 2)}\n`, 'utf8')
210
+ for (let attempt = 1; ; attempt++) {
211
+ try {
212
+ await fsp.rename(tmp, file)
213
+ return
214
+ } catch (error) {
215
+ // A reader holding the destination open can fail the replace on Windows.
216
+ if (attempt >= 5 || !['EPERM', 'EACCES', 'EBUSY'].includes(error?.code)) throw error
217
+ await new Promise(resolve => setTimeout(resolve, 20 * attempt))
218
+ }
219
+ }
220
+ },
221
+
222
+ /**
223
+ * Ask whichever process owns a run to stop it. This is a file, not IPC, so
224
+ * it works across processes and survives a restart.
225
+ * @param {string} runId - run identity.
226
+ */
227
+ async requestCancel(runId) {
228
+ if (!RUN_ID_PATTERN.test(String(runId))) return false
229
+ await fsp.writeFile(cancelFile(runId), `${JSON.stringify({ requestedAt: new Date().toISOString(), pid: process.pid })}\n`, 'utf8')
230
+ return true
231
+ },
232
+
233
+ /**
234
+ * @param {string} runId - run identity.
235
+ * @returns {Promise<boolean>} true when a cancel was requested by any process.
236
+ */
237
+ async hasCancelRequest(runId) {
238
+ try {
239
+ await fsp.access(cancelFile(runId))
240
+ return true
241
+ } catch {
242
+ return false
243
+ }
244
+ },
245
+
246
+ /**
247
+ * @param {string} runId - run identity.
248
+ */
249
+ async clearCancel(runId) {
250
+ try {
251
+ await fsp.unlink(cancelFile(runId))
252
+ } catch {
253
+ // Nothing to clear.
254
+ }
255
+ },
256
+
257
+ cancelFile,
258
+
259
+ /**
260
+ * Claim the right to schedule one run. The lock is created with `wx`, so
261
+ * exactly one process wins even when two profiles point at the same state
262
+ * directory; the loser gets the holder instead of a second copy of the run.
263
+ * @param {string} runId - run identity.
264
+ * @param {{ pid: number, hostname: string, at: string }} holder - claimant.
265
+ * @returns {Promise<{ acquired: boolean, holder: object | null }>}
266
+ */
267
+ async acquireLock(runId, holder) {
268
+ if (!RUN_ID_PATTERN.test(String(runId))) return { acquired: false, holder: null }
269
+ try {
270
+ await fsp.writeFile(lockFile(runId), `${JSON.stringify(holder)}\n`, { encoding: 'utf8', flag: 'wx' })
271
+ return { acquired: true, holder }
272
+ } catch (error) {
273
+ if (error?.code !== 'EEXIST') throw error
274
+ return { acquired: false, holder: await this.readLock(runId) }
275
+ }
276
+ },
277
+
278
+ /**
279
+ * @param {string} runId - run identity.
280
+ * @returns {Promise<object | null>} recorded holder, or null.
281
+ */
282
+ async readLock(runId) {
283
+ if (!RUN_ID_PATTERN.test(String(runId))) return null
284
+ try {
285
+ const parsed = JSON.parse(await fsp.readFile(lockFile(runId), 'utf8'))
286
+ return parsed && typeof parsed === 'object' ? parsed : null
287
+ } catch {
288
+ return null
289
+ }
290
+ },
291
+
292
+ /**
293
+ * @param {string} runId - run identity.
294
+ */
295
+ async releaseLock(runId) {
296
+ try {
297
+ await fsp.unlink(lockFile(runId))
298
+ } catch {
299
+ // Nothing to release.
300
+ }
301
+ },
302
+
303
+ /**
304
+ * Machine identity of the state directory, written once by the first
305
+ * process that mounts it. Its presence tells later processes whether a pid
306
+ * in a run file belongs to this machine (local) or possibly to another one
307
+ * sharing the directory, where the pid is meaningless.
308
+ * @returns {Promise<{ hostname?: string, createdAt?: string } | null>}
309
+ */
310
+ async readOwner() {
311
+ try {
312
+ const parsed = JSON.parse(await fsp.readFile(ownerFile, 'utf8'))
313
+ return parsed && typeof parsed === 'object' ? parsed : null
314
+ } catch {
315
+ return null
316
+ }
317
+ },
318
+
319
+ /**
320
+ * @param {{ hostname: string, createdAt?: string }} owner - identity to record.
321
+ */
322
+ async writeOwner(owner) {
323
+ try {
324
+ await fsp.writeFile(ownerFile, `${JSON.stringify(owner, null, 2)}\n`, 'utf8')
325
+ } catch (error) {
326
+ warn(`cannot record the state directory owner in ${ownerFile}: ${error?.message ?? error}`)
327
+ }
328
+ },
329
+
330
+ /**
331
+ * Delete terminal runs older than `olderThanDays`, so an install that runs
332
+ * for months does not accumulate state forever. Opt-in: called with a
333
+ * positive number only when the user configured `keepRuns`.
334
+ * @param {{ olderThanDays: number }} options - retention window.
335
+ * @returns {Promise<{ removed: string[] }>} removed run ids.
336
+ */
337
+ async prune({ olderThanDays }) {
338
+ const days = Number(olderThanDays)
339
+ const removed = []
340
+ if (!Number.isFinite(days) || days <= 0) return { removed }
341
+ const cutoff = Date.now() - days * 86400000
342
+ for (const run of await this.list()) {
343
+ if (!TERMINAL_STATUSES.has(run.status)) continue
344
+ const finished = Date.parse(run.finishedAt ?? run.updatedAt ?? run.createdAt)
345
+ if (!Number.isFinite(finished) || finished > cutoff) continue
346
+ try {
347
+ await fsp.unlink(runFile(run.id))
348
+ await this.clearCancel(run.id)
349
+ await this.releaseLock(run.id)
350
+ removed.push(run.id)
351
+ } catch (error) {
352
+ warn(`cannot prune run ${run.id}: ${error?.message ?? error}`)
353
+ }
354
+ }
355
+ return { removed }
356
+ },
357
+
358
+ /**
359
+ * Size of the state directory, for `superagent_list`.
360
+ * @returns {Promise<{ bytes: number, files: number, corrupt: number }>} totals.
361
+ */
362
+ async stats() {
363
+ const totals = { bytes: 0, files: 0, corrupt: 0 }
364
+ let names
365
+ try {
366
+ names = await fsp.readdir(runsDir)
367
+ } catch {
368
+ return totals
369
+ }
370
+ for (const name of names) {
371
+ try {
372
+ const stat = await fsp.stat(path.join(runsDir, name))
373
+ if (!stat.isFile()) continue
374
+ totals.files++
375
+ totals.bytes += stat.size
376
+ if (name.includes('.corrupt-')) totals.corrupt++
377
+ } catch {
378
+ // A file removed mid-scan does not change the answer.
379
+ }
380
+ }
381
+ return totals
382
+ },
383
+ }
384
+ }
385
+
package/lib/usage.js ADDED
@@ -0,0 +1,152 @@
1
+ /**
2
+ * Token accounting for child sessions.
3
+ *
4
+ * The subagent seam does not expose usage on a result, but every model call is
5
+ * recorded in the child's durable session log. This module reads that log and
6
+ * sums the recorded token counts, so a run reports measured numbers instead of
7
+ * estimates.
8
+ *
9
+ * @module dsh-superagent/usage
10
+ */
11
+ import fsp from 'node:fs/promises'
12
+ import path from 'node:path'
13
+ import zlib from 'node:zlib'
14
+
15
+ const ZSTD_MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
16
+
17
+ /**
18
+ * Decompress every zstd frame in a buffer.
19
+ *
20
+ * Session logs append one frame per flush, and the one-shot API only decodes
21
+ * the first frame, so frames are split on the zstd magic number and decoded
22
+ * independently. A slice that fails to decode is counted and skipped.
23
+ *
24
+ * @param {Buffer} buffer - raw session file.
25
+ * @returns {{ text: string, failed: number }} decoded UTF-8 text and failure count.
26
+ */
27
+ function decodeFrames(buffer) {
28
+ if (typeof zlib.zstdDecompressSync !== 'function') return { text: '', failed: 1 }
29
+ const starts = []
30
+ let at = 0
31
+ while ((at = buffer.indexOf(ZSTD_MAGIC, at)) !== -1) {
32
+ starts.push(at)
33
+ at += ZSTD_MAGIC.length
34
+ }
35
+ if (starts.length === 0) return { text: '', failed: 1 }
36
+ const parts = []
37
+ let failed = 0
38
+ for (let index = 0; index < starts.length; index++) {
39
+ const slice = buffer.subarray(starts[index], starts[index + 1] ?? buffer.length)
40
+ try {
41
+ parts.push(zlib.zstdDecompressSync(slice).toString('utf8'))
42
+ } catch {
43
+ failed++
44
+ }
45
+ }
46
+ return { text: parts.join('\n'), failed }
47
+ }
48
+
49
+ /**
50
+ * Sum usage events from one decoded session log.
51
+ * @param {string} text - newline-delimited session events.
52
+ */
53
+ function sumUsage(text) {
54
+ const usage = {
55
+ inputTokens: 0,
56
+ outputTokens: 0,
57
+ cacheReadTokens: 0,
58
+ totalTokens: 0,
59
+ modelCalls: 0,
60
+ models: {},
61
+ }
62
+ for (const line of text.split('\n')) {
63
+ if (!line || !line.includes('usage')) continue
64
+ let event
65
+ try {
66
+ event = JSON.parse(line)
67
+ } catch {
68
+ continue
69
+ }
70
+ const data = event?.data
71
+ const recorded = data?.usage
72
+ if (!recorded || typeof recorded !== 'object') continue
73
+ const inputTokens = Number(recorded.inputTokens) || 0
74
+ const outputTokens = Number(recorded.outputTokens) || 0
75
+ if (inputTokens === 0 && outputTokens === 0) continue
76
+ usage.inputTokens += inputTokens
77
+ usage.outputTokens += outputTokens
78
+ usage.cacheReadTokens += Number(recorded.cacheReadTokens) || 0
79
+ usage.totalTokens += Number(recorded.totalTokens) || inputTokens + outputTokens
80
+ usage.modelCalls++
81
+ const source = data?.message?.source
82
+ if (source?.kind === 'model') {
83
+ const key = `${source.provider ?? '?'}/${source.model ?? '?'}`
84
+ const entry = usage.models[key] ?? (usage.models[key] = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, calls: 0 })
85
+ entry.inputTokens += inputTokens
86
+ entry.outputTokens += outputTokens
87
+ entry.cacheReadTokens += Number(recorded.cacheReadTokens) || 0
88
+ entry.calls++
89
+ }
90
+ }
91
+ return usage
92
+ }
93
+
94
+ /**
95
+ * Build a reader that maps a child session id to its recorded usage.
96
+ *
97
+ * @param {object} options - reader options.
98
+ * @param {string} options.sessionsRoot - absolute `$DSH_HOME/sessions`.
99
+ * @param {(message: string) => void} [options.warn] - diagnostics sink.
100
+ * @returns {(childId: string) => Promise<object | null>} usage reader.
101
+ */
102
+ export function createUsageReader({ sessionsRoot, warn = () => {} }) {
103
+ /** @type {Map<string, string>} */
104
+ const fileCache = new Map()
105
+
106
+ async function locate(childId) {
107
+ if (fileCache.has(childId)) return fileCache.get(childId)
108
+ let workspaces
109
+ try {
110
+ workspaces = await fsp.readdir(sessionsRoot, { withFileTypes: true })
111
+ } catch {
112
+ return null
113
+ }
114
+ for (const entry of workspaces) {
115
+ if (!entry.isDirectory()) continue
116
+ // Root sessions live in `session-<id>`; child sessions use the bare id.
117
+ for (const folder of [`session-${childId}`, childId]) {
118
+ const dir = path.join(sessionsRoot, entry.name, folder)
119
+ let names
120
+ try {
121
+ names = await fsp.readdir(dir)
122
+ } catch {
123
+ continue
124
+ }
125
+ const file = names.find(name => name.startsWith('session.') && name.endsWith('.zstd'))
126
+ if (!file) continue
127
+ const resolved = path.join(dir, file)
128
+ fileCache.set(childId, resolved)
129
+ return resolved
130
+ }
131
+ }
132
+ return null
133
+ }
134
+
135
+ return async function readUsage(childId) {
136
+ if (!childId) return null
137
+ const file = await locate(childId)
138
+ if (!file) return null
139
+ let buffer
140
+ try {
141
+ buffer = await fsp.readFile(file)
142
+ } catch (error) {
143
+ warn(`usage: cannot read session log for ${childId}: ${error?.message ?? error}`)
144
+ return null
145
+ }
146
+ const { text, failed } = decodeFrames(buffer)
147
+ if (failed > 0) warn(`usage: ${failed} session frame(s) failed to decode for ${childId}`)
148
+ const usage = sumUsage(text)
149
+ if (usage.modelCalls === 0) return null
150
+ return usage
151
+ }
152
+ }