@xiangyixie/dsh-superagent 0.9.3 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +210 -0
- package/LICENSE +21 -21
- package/README.md +422 -42
- package/bin/archive.js +185 -0
- package/bin/doctor.js +84 -0
- package/bin/report.js +79 -79
- package/bin/watch.js +57 -57
- package/cordis.patch.yml +9 -9
- package/lib/control.js +495 -494
- package/lib/cost.js +267 -0
- package/lib/doctor.js +876 -0
- package/lib/errors.js +205 -199
- package/lib/index.js +767 -42
- package/lib/ledger.js +270 -0
- package/lib/lock.js +107 -0
- package/lib/memory.js +736 -0
- package/lib/notify.js +257 -0
- package/lib/orchestrator.js +1975 -127
- package/lib/report.js +350 -15
- package/lib/store.js +503 -15
- package/lib/templates.js +690 -0
- package/lib/usage.js +152 -152
- package/lib/watch.js +422 -70
- package/lib/workspace.js +433 -0
- package/package.json +16 -3
package/lib/store.js
CHANGED
|
@@ -13,18 +13,158 @@
|
|
|
13
13
|
*
|
|
14
14
|
* @module dsh-superagent/store
|
|
15
15
|
*/
|
|
16
|
+
import { createHash } from 'node:crypto'
|
|
16
17
|
import fsp from 'node:fs/promises'
|
|
17
18
|
import os from 'node:os'
|
|
18
19
|
import path from 'node:path'
|
|
19
20
|
|
|
21
|
+
/** Format marker written into every exported archive. */
|
|
22
|
+
export const ARCHIVE_FORMAT = 'dsh-superagent/run-archive'
|
|
23
|
+
|
|
24
|
+
/** Archive layout version; `import` refuses a version it cannot read. */
|
|
25
|
+
export const ARCHIVE_FORMAT_VERSION = 1
|
|
26
|
+
|
|
20
27
|
/** Statuses for a run that still owns live work. */
|
|
21
28
|
export const ACTIVE_STATUSES = new Set(['queued', 'planning', 'running', 'aggregating'])
|
|
22
29
|
|
|
23
30
|
/** Statuses after which a run will not change on its own. */
|
|
24
31
|
export const TERMINAL_STATUSES = new Set(['completed', 'partial', 'failed', 'cancelled', 'interrupted'])
|
|
25
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Prefix of a goal guard: the lock a start holds while it decides whether a run
|
|
35
|
+
* for that goal already exists. It is keyed by the goal rather than by a run id,
|
|
36
|
+
* so it has no run file of its own and the mount-time cleanup must leave it
|
|
37
|
+
* alone instead of treating it as a lock left behind by a finished run.
|
|
38
|
+
*/
|
|
39
|
+
export const GOAL_GUARD_PREFIX = 'goal-'
|
|
40
|
+
|
|
26
41
|
const RUN_ID_PATTERN = /^[0-9A-Za-z][0-9A-Za-z-]{3,63}$/
|
|
27
42
|
const STALE_TMP_MS = 10 * 60 * 1000
|
|
43
|
+
const EMBEDDED_ARCHIVE = /<script[^>]*id="superagent-archive"[^>]*>([\s\S]*?)<\/script>/i
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Turn a `--status` filter into a validated set. A typo must fail loudly:
|
|
47
|
+
* silently matching nothing would make an operator believe a prune ran.
|
|
48
|
+
* @param {string | string[] | undefined | null} value - comma list or array.
|
|
49
|
+
* @returns {Set<string> | null} allowed statuses, or null for "all terminal".
|
|
50
|
+
*/
|
|
51
|
+
function normalizeStatuses(value) {
|
|
52
|
+
if (value === undefined || value === null) return null
|
|
53
|
+
const list = Array.isArray(value) ? value : String(value).split(',')
|
|
54
|
+
const statuses = [...new Set(list.map(item => String(item).trim().toLowerCase()).filter(Boolean))]
|
|
55
|
+
const terminal = [...TERMINAL_STATUSES].join(', ')
|
|
56
|
+
if (statuses.length === 0) throw new Error(`a status filter needs at least one status (terminal: ${terminal})`)
|
|
57
|
+
const unknown = statuses.filter(status => !TERMINAL_STATUSES.has(status))
|
|
58
|
+
if (unknown.length > 0) {
|
|
59
|
+
throw new Error(`status filter ${unknown.join(', ')} is not terminal; only ${terminal} runs can be pruned`)
|
|
60
|
+
}
|
|
61
|
+
return new Set(statuses)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Filesystem-safe fragment for a subtask title. */
|
|
65
|
+
function slugOf(text, fallback) {
|
|
66
|
+
const slug = String(text ?? '')
|
|
67
|
+
.toLowerCase()
|
|
68
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
69
|
+
.replace(/^-+|-+$/g, '')
|
|
70
|
+
.slice(0, 40)
|
|
71
|
+
.replace(/-+$/g, '')
|
|
72
|
+
return slug || fallback
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** One subtask as a standalone markdown page inside an export. */
|
|
76
|
+
function renderSubtaskMarkdown(subtask) {
|
|
77
|
+
const lines = [
|
|
78
|
+
`# Subtask ${subtask.index ?? '?'} — ${subtask.title || '(untitled)'}`,
|
|
79
|
+
'',
|
|
80
|
+
`- status: ${subtask.status}`,
|
|
81
|
+
`- attempts: ${subtask.attempts ?? 0}`,
|
|
82
|
+
`- started: ${subtask.startedAt ?? 'unknown'}`,
|
|
83
|
+
`- finished: ${subtask.finishedAt ?? 'not finished'}`,
|
|
84
|
+
`- child: ${subtask.childId ?? 'none'}`,
|
|
85
|
+
`- tokens: ${subtask.usage?.totalTokens ?? 0} across ${subtask.usage?.modelCalls ?? 0} model call(s)`,
|
|
86
|
+
]
|
|
87
|
+
if (subtask.error) lines.push(`- error: ${subtask.error}`)
|
|
88
|
+
lines.push('', '## Prompt', '', subtask.prompt || '_No prompt recorded._')
|
|
89
|
+
lines.push('', '## Result', '', subtask.result || '_No result recorded._')
|
|
90
|
+
return `${lines.join('\n')}\n`
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Guard/attention records as markdown; these are the run's interception trail. */
|
|
94
|
+
function renderGuardsMarkdown(guards, at) {
|
|
95
|
+
const lines = ['# Guard and attention records', '', `Recorded at: ${at ?? 'unknown'}`, '']
|
|
96
|
+
if (guards.issue) lines.push(`Issue: ${guards.issue}`, '')
|
|
97
|
+
if (guards.items.length === 0 && !guards.issue) lines.push('No worker or aggregator asked for a decision during this run.', '')
|
|
98
|
+
guards.items.forEach((item, index) => {
|
|
99
|
+
lines.push(`## ${index + 1}. ${item.source ?? 'unknown source'}`, '')
|
|
100
|
+
lines.push(`Question: ${item.question ?? '(none)'}`)
|
|
101
|
+
if (Array.isArray(item.options) && item.options.length > 0) {
|
|
102
|
+
lines.push('', 'Options:', ...item.options.map(option => `- ${option}`))
|
|
103
|
+
}
|
|
104
|
+
if (item.reason) lines.push('', `Reason: ${item.reason}`)
|
|
105
|
+
lines.push('')
|
|
106
|
+
})
|
|
107
|
+
return `${lines.join('\n').trimEnd()}\n`
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Human-readable cost with the unpriced caveat stated, never guessed. */
|
|
111
|
+
function renderCost(cost) {
|
|
112
|
+
if (!cost || !Number.isFinite(cost.total)) return 'unpriced (no configured price for these models)'
|
|
113
|
+
const amount = `${cost.currency ?? 'USD'} ${cost.total.toFixed(4)}`
|
|
114
|
+
if (cost.complete === false) {
|
|
115
|
+
return `${amount} (partial; unpriced: ${(cost.unpriced ?? []).join(', ') || 'unknown'})`
|
|
116
|
+
}
|
|
117
|
+
return amount
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** One page a reader can open first; the report itself follows inline. */
|
|
121
|
+
function renderReadme(archive) {
|
|
122
|
+
const subtasks = archive.subtasks
|
|
123
|
+
const counts = {}
|
|
124
|
+
for (const subtask of subtasks) counts[subtask.status] = (counts[subtask.status] ?? 0) + 1
|
|
125
|
+
const breakdown = Object.entries(counts).map(([status, count]) => `${count} ${status}`).join(', ')
|
|
126
|
+
const lines = [
|
|
127
|
+
`# Superagent run ${archive.runId}`,
|
|
128
|
+
'',
|
|
129
|
+
`- goal: ${archive.goal || '(empty)'}`,
|
|
130
|
+
`- status: ${archive.status}`,
|
|
131
|
+
`- created: ${archive.createdAt ?? 'unknown'}`,
|
|
132
|
+
`- finished: ${archive.finishedAt ?? 'not finished'}`,
|
|
133
|
+
`- subtasks: ${subtasks.length}${breakdown ? ` (${breakdown})` : ''}`,
|
|
134
|
+
`- guards: ${archive.guards.items.length} decision request(s)${archive.guards.issue ? '; one attention block was unusable' : ''}`,
|
|
135
|
+
`- tokens: ${archive.usage?.totalTokens ?? 0} across ${archive.usage?.modelCalls ?? 0} model call(s)`,
|
|
136
|
+
`- cost: ${renderCost(archive.cost)}`,
|
|
137
|
+
'',
|
|
138
|
+
'## Report',
|
|
139
|
+
'',
|
|
140
|
+
archive.report || '_No merged report was recorded for this run._',
|
|
141
|
+
'',
|
|
142
|
+
'## Subtasks',
|
|
143
|
+
'',
|
|
144
|
+
]
|
|
145
|
+
if (subtasks.length === 0) lines.push('_No subtasks recorded._')
|
|
146
|
+
else {
|
|
147
|
+
lines.push('| # | status | title | tokens |', '| --- | --- | --- | --- |')
|
|
148
|
+
for (const subtask of subtasks) {
|
|
149
|
+
const title = String(subtask.title ?? '').replace(/\|/g, '\\|')
|
|
150
|
+
lines.push(`| ${subtask.index ?? '-'} | ${subtask.status} | ${title} | ${subtask.usage?.totalTokens ?? '-'} |`)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
lines.push(
|
|
154
|
+
'',
|
|
155
|
+
'## Files',
|
|
156
|
+
'',
|
|
157
|
+
'Everything in this directory is part of the archive; nothing is fetched from the network.',
|
|
158
|
+
'',
|
|
159
|
+
'- `run.json` — the full durable run record as the store wrote it',
|
|
160
|
+
'- `report.md` — the final merged report',
|
|
161
|
+
'- `subtasks/` — each subtask as markdown (readable) and JSON (structured)',
|
|
162
|
+
'- `guards.json` / `guards.md` — worker and aggregator decisions the run stopped on',
|
|
163
|
+
'- `usage.json` — token totals per child, plus the cost the operator prices produce',
|
|
164
|
+
'- `manifest.json` — archive metadata, file sizes, and SHA-256 checksums',
|
|
165
|
+
)
|
|
166
|
+
return `${lines.join('\n')}\n`
|
|
167
|
+
}
|
|
28
168
|
|
|
29
169
|
/**
|
|
30
170
|
* Resolve the state root. An explicit `stateDir` wins, then `$DSH_HOME`, then
|
|
@@ -55,8 +195,36 @@ export function createStore(root, options = {}) {
|
|
|
55
195
|
const runFile = runId => path.join(runsDir, `${runId}.json`)
|
|
56
196
|
const cancelFile = runId => path.join(runsDir, `${runId}.cancel`)
|
|
57
197
|
const lockFile = runId => path.join(runsDir, `${runId}.lock`)
|
|
198
|
+
const reportFile = runId => path.join(root, 'reports', `${runId}.md`)
|
|
58
199
|
const ownerFile = path.join(root, '.owner')
|
|
59
200
|
|
|
201
|
+
/** File size, or null when the file does not exist (or is not a file). */
|
|
202
|
+
const sizeOf = async file => {
|
|
203
|
+
try {
|
|
204
|
+
const stat = await fsp.stat(file)
|
|
205
|
+
return stat.isFile() ? stat.size : null
|
|
206
|
+
} catch {
|
|
207
|
+
return null
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* A lock file is the cross-process claim on a run. Even a terminal run must
|
|
213
|
+
* be left alone while one exists: another process may be mid-write.
|
|
214
|
+
*/
|
|
215
|
+
const isHeld = async runId => (await sizeOf(lockFile(runId))) !== null
|
|
216
|
+
|
|
217
|
+
/** The report text, preferring the run record and falling back to its artifact. */
|
|
218
|
+
const reportTextOf = async run => {
|
|
219
|
+
if (typeof run.report === 'string' && run.report.trim()) return run.report.trim()
|
|
220
|
+
try {
|
|
221
|
+
const text = await fsp.readFile(reportFile(run.id), 'utf8')
|
|
222
|
+
return text.trim() ? text.trim() : null
|
|
223
|
+
} catch {
|
|
224
|
+
return null
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
60
228
|
return {
|
|
61
229
|
root,
|
|
62
230
|
runsDir,
|
|
@@ -92,6 +260,9 @@ export function createStore(root, options = {}) {
|
|
|
92
260
|
}
|
|
93
261
|
const marker = ['.cancel', '.lock'].find(suffix => name.endsWith(suffix))
|
|
94
262
|
if (!marker) continue
|
|
263
|
+
// A goal guard is not a run lock and has no run file to read; its
|
|
264
|
+
// holder releases it, or a later start breaks a dead holder's guard.
|
|
265
|
+
if (name.startsWith(GOAL_GUARD_PREFIX) && marker === '.lock') continue
|
|
95
266
|
let run = null
|
|
96
267
|
let readable = true
|
|
97
268
|
try {
|
|
@@ -328,31 +499,348 @@ export function createStore(root, options = {}) {
|
|
|
328
499
|
},
|
|
329
500
|
|
|
330
501
|
/**
|
|
331
|
-
*
|
|
332
|
-
*
|
|
333
|
-
*
|
|
334
|
-
*
|
|
335
|
-
*
|
|
502
|
+
* Decide which runs a prune would remove, without touching anything.
|
|
503
|
+
*
|
|
504
|
+
* A run qualifies only when it is terminal, inside the age window, inside
|
|
505
|
+
* the optional status filter, and not claimed by a lock file. Active runs
|
|
506
|
+
* are never candidates, so the plan is safe to show before deleting.
|
|
507
|
+
*
|
|
508
|
+
* @param {{ olderThanDays?: number, statuses?: string[] | string }} [options] - window and filter.
|
|
509
|
+
* @returns {Promise<{
|
|
510
|
+
* candidates: { id: string, status: string, goal: string, finishedAt: string, bytes: number, files: { kind: string, path: string, bytes: number }[] }[],
|
|
511
|
+
* skipped: { id: string, status: string, reason: string }[],
|
|
512
|
+
* removed: string[], dryRun: boolean, bytes: number, files: number,
|
|
513
|
+
* }>} the plan; `removed` is always empty here.
|
|
336
514
|
*/
|
|
337
|
-
async
|
|
338
|
-
const days = Number(olderThanDays)
|
|
339
|
-
|
|
340
|
-
|
|
515
|
+
async planPrune(options = {}) {
|
|
516
|
+
const days = Number(options.olderThanDays)
|
|
517
|
+
if (!Number.isFinite(days) || days <= 0) {
|
|
518
|
+
throw new Error('olderThanDays must be a positive number of days')
|
|
519
|
+
}
|
|
520
|
+
const wanted = normalizeStatuses(options.statuses)
|
|
341
521
|
const cutoff = Date.now() - days * 86400000
|
|
522
|
+
const candidates = []
|
|
523
|
+
const skipped = []
|
|
342
524
|
for (const run of await this.list()) {
|
|
343
525
|
if (!TERMINAL_STATUSES.has(run.status)) continue
|
|
526
|
+
if (wanted && !wanted.has(run.status)) continue
|
|
344
527
|
const finished = Date.parse(run.finishedAt ?? run.updatedAt ?? run.createdAt)
|
|
345
528
|
if (!Number.isFinite(finished) || finished > cutoff) continue
|
|
529
|
+
if (await isHeld(run.id)) {
|
|
530
|
+
skipped.push({ id: run.id, status: String(run.status), reason: 'held by another process' })
|
|
531
|
+
continue
|
|
532
|
+
}
|
|
533
|
+
const files = []
|
|
534
|
+
for (const [kind, file] of [['run', runFile(run.id)], ['cancel', cancelFile(run.id)], ['report', reportFile(run.id)]]) {
|
|
535
|
+
const bytes = await sizeOf(file)
|
|
536
|
+
if (bytes !== null) files.push({ kind, path: file, bytes })
|
|
537
|
+
}
|
|
538
|
+
candidates.push({
|
|
539
|
+
id: run.id,
|
|
540
|
+
status: String(run.status),
|
|
541
|
+
goal: String(run.goal ?? ''),
|
|
542
|
+
finishedAt: new Date(finished).toISOString(),
|
|
543
|
+
bytes: files.reduce((sum, file) => sum + file.bytes, 0),
|
|
544
|
+
files,
|
|
545
|
+
})
|
|
546
|
+
}
|
|
547
|
+
return {
|
|
548
|
+
candidates,
|
|
549
|
+
skipped,
|
|
550
|
+
removed: [],
|
|
551
|
+
dryRun: true,
|
|
552
|
+
bytes: candidates.reduce((sum, candidate) => sum + candidate.bytes, 0),
|
|
553
|
+
files: candidates.reduce((sum, candidate) => sum + candidate.files.length, 0),
|
|
554
|
+
}
|
|
555
|
+
},
|
|
556
|
+
|
|
557
|
+
/**
|
|
558
|
+
* Delete terminal runs older than `olderThanDays`, so an install that runs
|
|
559
|
+
* for months does not accumulate state forever.
|
|
560
|
+
*
|
|
561
|
+
* The CLI prune defaults to `dryRun: true`; the mount-time `keepRuns` call
|
|
562
|
+
* passes `dryRun: false` because the operator already opted in through the
|
|
563
|
+
* config. A locked or live run is never deleted.
|
|
564
|
+
*
|
|
565
|
+
* @param {{ olderThanDays: number, statuses?: string[] | string, dryRun?: boolean }} options - window, filter, and mode.
|
|
566
|
+
* @returns {Promise<object>} the plan plus `removed`, `failed`, and `deletedBytes`.
|
|
567
|
+
*/
|
|
568
|
+
async prune({ olderThanDays, statuses, dryRun = false } = {}) {
|
|
569
|
+
const days = Number(olderThanDays)
|
|
570
|
+
if (!Number.isFinite(days) || days <= 0) {
|
|
571
|
+
return { candidates: [], skipped: [], removed: [], failed: [], dryRun, bytes: 0, deletedBytes: 0, files: 0 }
|
|
572
|
+
}
|
|
573
|
+
const plan = await this.planPrune({ olderThanDays: days, statuses })
|
|
574
|
+
if (dryRun) return { ...plan, failed: [], deletedBytes: 0 }
|
|
575
|
+
const removed = []
|
|
576
|
+
const failed = []
|
|
577
|
+
let deletedBytes = 0
|
|
578
|
+
for (const candidate of plan.candidates) {
|
|
579
|
+
try {
|
|
580
|
+
// Auxiliary files first: a failure then leaves the run record that
|
|
581
|
+
// names what could not be cleaned up, instead of an orphan report.
|
|
582
|
+
for (const file of candidate.files) {
|
|
583
|
+
if (file.kind === 'run') continue
|
|
584
|
+
await fsp.unlink(file.path).catch(error => {
|
|
585
|
+
if (error?.code !== 'ENOENT') throw error
|
|
586
|
+
})
|
|
587
|
+
}
|
|
588
|
+
await fsp.unlink(runFile(candidate.id)).catch(error => {
|
|
589
|
+
if (error?.code !== 'ENOENT') throw error
|
|
590
|
+
})
|
|
591
|
+
removed.push(candidate.id)
|
|
592
|
+
deletedBytes += candidate.bytes
|
|
593
|
+
} catch (error) {
|
|
594
|
+
failed.push({ id: candidate.id, error: String(error?.message ?? error) })
|
|
595
|
+
warn(`cannot prune run ${candidate.id}: ${error?.message ?? error}`)
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return { ...plan, dryRun: false, removed, failed, deletedBytes }
|
|
599
|
+
},
|
|
600
|
+
|
|
601
|
+
/**
|
|
602
|
+
* Build the portable form of one run: metadata, the full record, the final
|
|
603
|
+
* report, structured subtask results, guard records, and usage/cost.
|
|
604
|
+
* @param {string} runId - run identity.
|
|
605
|
+
* @param {{ exportedAt?: string }} [options] - timestamp override for tests.
|
|
606
|
+
* @returns {Promise<object>} the archive payload.
|
|
607
|
+
* @throws when the run does not exist.
|
|
608
|
+
*/
|
|
609
|
+
async buildArchive(runId, options = {}) {
|
|
610
|
+
const run = await this.read(runId)
|
|
611
|
+
if (!run) throw new Error(`no run ${runId} in ${this.root}`)
|
|
612
|
+
const report = await reportTextOf(run)
|
|
613
|
+
const subtasks = (Array.isArray(run.subtasks) ? run.subtasks : []).map(subtask => ({
|
|
614
|
+
index: subtask.index ?? null,
|
|
615
|
+
title: subtask.title ?? '',
|
|
616
|
+
status: subtask.status ?? 'unknown',
|
|
617
|
+
prompt: subtask.prompt ?? '',
|
|
618
|
+
result: subtask.result ?? null,
|
|
619
|
+
error: subtask.error ?? null,
|
|
620
|
+
attempts: subtask.attempts ?? 0,
|
|
621
|
+
childId: subtask.childId ?? null,
|
|
622
|
+
startedAt: subtask.startedAt ?? null,
|
|
623
|
+
finishedAt: subtask.finishedAt ?? null,
|
|
624
|
+
usage: subtask.usage ?? null,
|
|
625
|
+
}))
|
|
626
|
+
return {
|
|
627
|
+
format: ARCHIVE_FORMAT,
|
|
628
|
+
formatVersion: ARCHIVE_FORMAT_VERSION,
|
|
629
|
+
exportedAt: options.exportedAt ?? new Date().toISOString(),
|
|
630
|
+
generator: 'dsh-superagent',
|
|
631
|
+
runId: run.id,
|
|
632
|
+
goal: String(run.goal ?? ''),
|
|
633
|
+
status: String(run.status ?? 'unknown'),
|
|
634
|
+
createdAt: run.createdAt ?? null,
|
|
635
|
+
finishedAt: run.finishedAt ?? null,
|
|
636
|
+
usage: run.usage ?? null,
|
|
637
|
+
cost: run.cost ?? null,
|
|
638
|
+
guards: {
|
|
639
|
+
at: run.attention?.at ?? null,
|
|
640
|
+
issue: run.attention?.issue ?? null,
|
|
641
|
+
items: Array.isArray(run.attention?.items) ? run.attention.items : [],
|
|
642
|
+
},
|
|
643
|
+
report,
|
|
644
|
+
subtasks,
|
|
645
|
+
run,
|
|
646
|
+
}
|
|
647
|
+
},
|
|
648
|
+
|
|
649
|
+
/**
|
|
650
|
+
* Write one run as a self-contained archive.
|
|
651
|
+
*
|
|
652
|
+
* `dir` writes a readable directory (README, markdown report, per-subtask
|
|
653
|
+
* pages, guard and usage files, manifest with checksums). `bundle` writes a
|
|
654
|
+
* single JSON file, and `html` writes one self-contained page with the same
|
|
655
|
+
* bundle embedded so it can be read and imported offline.
|
|
656
|
+
*
|
|
657
|
+
* @param {string} runId - run identity.
|
|
658
|
+
* @param {string} target - directory for `dir`, file path otherwise.
|
|
659
|
+
* @param {{ format?: 'dir' | 'bundle' | 'html', exportedAt?: string }} [options] - layout.
|
|
660
|
+
* @returns {Promise<{ runId: string, format: string, out: string, bytes: number, files: { path: string, bytes: number }[] }>}
|
|
661
|
+
*/
|
|
662
|
+
async exportRun(runId, target, options = {}) {
|
|
663
|
+
const format = options.format ?? 'dir'
|
|
664
|
+
if (!['dir', 'bundle', 'html'].includes(format)) {
|
|
665
|
+
throw new Error(`unknown archive format ${JSON.stringify(format)}; use dir, bundle, or html`)
|
|
666
|
+
}
|
|
667
|
+
const archive = await this.buildArchive(runId, options)
|
|
668
|
+
const out = path.resolve(String(target))
|
|
669
|
+
|
|
670
|
+
if (format === 'dir') {
|
|
671
|
+
// A stale file from a previous export must not be mistaken for part of
|
|
672
|
+
// this archive, and an unrelated directory must not be written into.
|
|
673
|
+
const existing = await fsp.readdir(out).catch(() => null)
|
|
674
|
+
if (existing && existing.length > 0) {
|
|
675
|
+
throw new Error(`refusing to overwrite ${out}: it already exists and is not empty; choose another --out`)
|
|
676
|
+
}
|
|
677
|
+
await fsp.mkdir(out, { recursive: true })
|
|
678
|
+
const files = []
|
|
679
|
+
const put = async (relative, body) => {
|
|
680
|
+
const buffer = Buffer.isBuffer(body) ? body : Buffer.from(String(body), 'utf8')
|
|
681
|
+
const file = path.join(out, relative)
|
|
682
|
+
await fsp.mkdir(path.dirname(file), { recursive: true })
|
|
683
|
+
await fsp.writeFile(file, buffer)
|
|
684
|
+
files.push({ path: relative, bytes: buffer.length, sha256: createHash('sha256').update(buffer).digest('hex') })
|
|
685
|
+
}
|
|
686
|
+
await put('run.json', `${JSON.stringify(archive.run, null, 2)}\n`)
|
|
687
|
+
if (archive.report) await put('report.md', `${archive.report.trimEnd()}\n`)
|
|
688
|
+
for (const subtask of archive.subtasks) {
|
|
689
|
+
const index = String(subtask.index ?? 0).padStart(2, '0')
|
|
690
|
+
const base = `subtasks/${index}-${slugOf(subtask.title, 'subtask')}`
|
|
691
|
+
await put(`${base}.md`, renderSubtaskMarkdown(subtask))
|
|
692
|
+
await put(`${base}.json`, `${JSON.stringify(subtask, null, 2)}\n`)
|
|
693
|
+
}
|
|
694
|
+
await put('guards.json', `${JSON.stringify(archive.guards, null, 2)}\n`)
|
|
695
|
+
await put('guards.md', renderGuardsMarkdown(archive.guards, archive.guards.at))
|
|
696
|
+
await put('usage.json', `${JSON.stringify({
|
|
697
|
+
total: archive.usage,
|
|
698
|
+
byChild: archive.run.usageByChild ?? {},
|
|
699
|
+
cost: archive.cost,
|
|
700
|
+
}, null, 2)}\n`)
|
|
701
|
+
await put('README.md', renderReadme(archive))
|
|
702
|
+
const manifest = {
|
|
703
|
+
format: ARCHIVE_FORMAT,
|
|
704
|
+
formatVersion: ARCHIVE_FORMAT_VERSION,
|
|
705
|
+
exportedAt: archive.exportedAt,
|
|
706
|
+
generator: archive.generator,
|
|
707
|
+
runId: archive.runId,
|
|
708
|
+
goal: archive.goal,
|
|
709
|
+
status: archive.status,
|
|
710
|
+
createdAt: archive.createdAt,
|
|
711
|
+
finishedAt: archive.finishedAt,
|
|
712
|
+
counts: { subtasks: archive.subtasks.length, guards: archive.guards.items.length },
|
|
713
|
+
totals: { files: files.length, bytes: files.reduce((sum, file) => sum + file.bytes, 0) },
|
|
714
|
+
files,
|
|
715
|
+
}
|
|
716
|
+
const body = `${JSON.stringify(manifest, null, 2)}\n`
|
|
717
|
+
await fsp.writeFile(path.join(out, 'manifest.json'), body, 'utf8')
|
|
718
|
+
const all = [...files, { path: 'manifest.json', bytes: Buffer.byteLength(body) }]
|
|
719
|
+
return { runId: archive.runId, format, out, bytes: all.reduce((sum, file) => sum + file.bytes, 0), files: all }
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
await fsp.mkdir(path.dirname(out), { recursive: true })
|
|
723
|
+
let body
|
|
724
|
+
if (format === 'bundle') {
|
|
725
|
+
body = `${JSON.stringify(archive, null, 2)}\n`
|
|
726
|
+
} else {
|
|
727
|
+
// Dynamic import keeps store.js independent of the renderer at load
|
|
728
|
+
// time; report.js already imports this module.
|
|
729
|
+
const { buildReport } = await import('./report.js')
|
|
730
|
+
const html = buildReport([archive.run], {
|
|
731
|
+
title: `Superagent run ${archive.runId}`,
|
|
732
|
+
generatedAt: Date.now(),
|
|
733
|
+
})
|
|
734
|
+
// `<` is escaped so a report containing "</script>" cannot end the tag.
|
|
735
|
+
const payload = JSON.stringify(archive).replace(/</g, '\\u003c')
|
|
736
|
+
body = html.replace(
|
|
737
|
+
'</body>',
|
|
738
|
+
`<script type="application/json" id="superagent-archive">${payload}</script>\n</body>`,
|
|
739
|
+
)
|
|
740
|
+
}
|
|
741
|
+
await fsp.writeFile(out, body, 'utf8')
|
|
742
|
+
const bytes = Buffer.byteLength(body, 'utf8')
|
|
743
|
+
return { runId: archive.runId, format, out, bytes, files: [{ path: path.basename(out), bytes }] }
|
|
744
|
+
},
|
|
745
|
+
|
|
746
|
+
/**
|
|
747
|
+
* Read an archive back into this store for review.
|
|
748
|
+
*
|
|
749
|
+
* Import never overwrites: a local run with the same id makes this throw
|
|
750
|
+
* before anything is written. The run record is restored byte-for-byte, so
|
|
751
|
+
* an export → import round trip can be compared with deep equality.
|
|
752
|
+
*
|
|
753
|
+
* @param {string} source - archive directory, bundle JSON, or html file.
|
|
754
|
+
* @returns {Promise<{ run: object, out: string, source: string, format: string }>}
|
|
755
|
+
* @throws when the archive is unreadable, incomplete, or collides locally.
|
|
756
|
+
*/
|
|
757
|
+
async importRun(source) {
|
|
758
|
+
const origin = path.resolve(String(source))
|
|
759
|
+
const stat = await fsp.stat(origin).catch(error => {
|
|
760
|
+
throw new Error(`cannot read archive ${origin}: ${error?.message ?? error}`)
|
|
761
|
+
})
|
|
762
|
+
let archive
|
|
763
|
+
let format
|
|
764
|
+
if (stat.isDirectory()) {
|
|
765
|
+
format = 'dir'
|
|
766
|
+
const manifest = JSON.parse(await fsp.readFile(path.join(origin, 'manifest.json'), 'utf8').catch(error => {
|
|
767
|
+
throw new Error(`archive directory ${origin} has no readable manifest.json: ${error?.message ?? error}`)
|
|
768
|
+
}))
|
|
769
|
+
if (manifest?.format !== ARCHIVE_FORMAT) {
|
|
770
|
+
throw new Error(`${path.join(origin, 'manifest.json')} is not a ${ARCHIVE_FORMAT} archive`)
|
|
771
|
+
}
|
|
772
|
+
if (manifest.formatVersion !== ARCHIVE_FORMAT_VERSION) {
|
|
773
|
+
throw new Error(`archive format version ${manifest.formatVersion} is not supported (expected ${ARCHIVE_FORMAT_VERSION})`)
|
|
774
|
+
}
|
|
775
|
+
const run = JSON.parse(await fsp.readFile(path.join(origin, 'run.json'), 'utf8').catch(error => {
|
|
776
|
+
throw new Error(`archive ${origin} has no readable run.json: ${error?.message ?? error}`)
|
|
777
|
+
}))
|
|
778
|
+
// Verify the parts the manifest records, so a truncated copy is caught
|
|
779
|
+
// instead of silently imported as a different run.
|
|
780
|
+
for (const entry of Array.isArray(manifest.files) ? manifest.files : []) {
|
|
781
|
+
const buffer = await fsp.readFile(path.join(origin, entry.path)).catch(() => null)
|
|
782
|
+
if (!buffer) throw new Error(`archive ${origin} is missing ${entry.path}`)
|
|
783
|
+
if (typeof entry.sha256 === 'string') {
|
|
784
|
+
const digest = createHash('sha256').update(buffer).digest('hex')
|
|
785
|
+
if (digest !== entry.sha256) throw new Error(`archive file ${entry.path} does not match its manifest checksum`)
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
let report = null
|
|
789
|
+
try {
|
|
790
|
+
report = (await fsp.readFile(path.join(origin, 'report.md'), 'utf8')).trim() || null
|
|
791
|
+
} catch {
|
|
792
|
+
report = null
|
|
793
|
+
}
|
|
794
|
+
archive = { ...manifest, run, report }
|
|
795
|
+
} else {
|
|
796
|
+
const text = await fsp.readFile(origin, 'utf8')
|
|
797
|
+
if (/\.html?$/i.test(origin)) {
|
|
798
|
+
format = 'html'
|
|
799
|
+
const match = text.match(EMBEDDED_ARCHIVE)
|
|
800
|
+
if (!match) throw new Error(`${origin} has no embedded ${ARCHIVE_FORMAT} payload`)
|
|
801
|
+
archive = JSON.parse(match[1])
|
|
802
|
+
} else {
|
|
803
|
+
format = 'bundle'
|
|
804
|
+
try {
|
|
805
|
+
archive = JSON.parse(text)
|
|
806
|
+
} catch (error) {
|
|
807
|
+
throw new Error(`${origin} is not valid JSON: ${error?.message ?? error}`)
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
if (archive?.format !== ARCHIVE_FORMAT) throw new Error(`${origin} is not a ${ARCHIVE_FORMAT} archive`)
|
|
811
|
+
if (archive.formatVersion !== ARCHIVE_FORMAT_VERSION) {
|
|
812
|
+
throw new Error(`archive format version ${archive.formatVersion} is not supported (expected ${ARCHIVE_FORMAT_VERSION})`)
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
const run = archive.run
|
|
817
|
+
if (!run || typeof run !== 'object') throw new Error(`archive ${origin} carries no run record`)
|
|
818
|
+
const id = String(run.id ?? '')
|
|
819
|
+
if (!RUN_ID_PATTERN.test(id)) throw new Error(`archive ${origin} has no usable run id (got ${JSON.stringify(run.id)})`)
|
|
820
|
+
if (archive.runId && archive.runId !== id) {
|
|
821
|
+
throw new Error(`archive ${origin} names run ${archive.runId} but run.json is ${id}`)
|
|
822
|
+
}
|
|
823
|
+
if (await sizeOf(runFile(id)) !== null) {
|
|
824
|
+
throw new Error(
|
|
825
|
+
`run ${id} already exists in ${this.root}; import never overwrites — `
|
|
826
|
+
+ 'remove it first, or import into another state directory',
|
|
827
|
+
)
|
|
828
|
+
}
|
|
829
|
+
if (typeof archive.report === 'string' && archive.report.trim() && !run.report) {
|
|
830
|
+
run.report = archive.report.trim()
|
|
831
|
+
}
|
|
832
|
+
await this.write(run)
|
|
833
|
+
if (typeof run.report === 'string' && run.report.trim()) {
|
|
834
|
+
// The report is also inside run.json, so a failed artifact write is a
|
|
835
|
+
// warning: the imported record is already complete without it.
|
|
346
836
|
try {
|
|
347
|
-
await fsp.
|
|
348
|
-
await
|
|
349
|
-
await this.releaseLock(run.id)
|
|
350
|
-
removed.push(run.id)
|
|
837
|
+
await fsp.mkdir(path.dirname(reportFile(id)), { recursive: true })
|
|
838
|
+
await fsp.writeFile(reportFile(id), `${run.report.trimEnd()}\n`, 'utf8')
|
|
351
839
|
} catch (error) {
|
|
352
|
-
warn(`
|
|
840
|
+
warn(`imported run ${id} but could not write ${reportFile(id)}: ${error?.message ?? error}`)
|
|
353
841
|
}
|
|
354
842
|
}
|
|
355
|
-
return {
|
|
843
|
+
return { run, out: runFile(id), source: origin, format }
|
|
356
844
|
},
|
|
357
845
|
|
|
358
846
|
/**
|