@llm4ts/runner 2.44.1 → 2.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Connectors.d.ts +1 -1
- package/dist/Connectors.d.ts.map +1 -1
- package/dist/Connectors.js +7 -2
- package/dist/Connectors.js.map +1 -1
- package/dist/Costs.d.ts +8 -4
- package/dist/Costs.d.ts.map +1 -1
- package/dist/Costs.js +32 -13
- package/dist/Costs.js.map +1 -1
- package/dist/Doctor.d.ts +24 -2
- package/dist/Doctor.d.ts.map +1 -1
- package/dist/Doctor.js +32 -12
- package/dist/Doctor.js.map +1 -1
- package/dist/FlowRunner.d.ts +38 -2
- package/dist/FlowRunner.d.ts.map +1 -1
- package/dist/FlowRunner.js +128 -13
- package/dist/FlowRunner.js.map +1 -1
- package/dist/NodeGeminiCliExecutor.js +3 -3
- package/dist/NodeGeminiCliExecutor.js.map +1 -1
- package/dist/RunReaders.d.ts +69 -0
- package/dist/RunReaders.d.ts.map +1 -0
- package/dist/RunReaders.js +277 -0
- package/dist/RunReaders.js.map +1 -0
- package/dist/RunSummary.d.ts +3 -0
- package/dist/RunSummary.d.ts.map +1 -0
- package/dist/RunSummary.js +49 -0
- package/dist/RunSummary.js.map +1 -0
- package/dist/StoryReaders.d.ts +16 -0
- package/dist/StoryReaders.d.ts.map +1 -0
- package/dist/StoryReaders.js +107 -0
- package/dist/StoryReaders.js.map +1 -0
- package/dist/Terminal.d.ts +2 -0
- package/dist/Terminal.d.ts.map +1 -1
- package/dist/Terminal.js +4 -1
- package/dist/Terminal.js.map +1 -1
- package/dist/Watch.d.ts +2 -0
- package/dist/Watch.d.ts.map +1 -1
- package/dist/Watch.js +2 -1
- package/dist/Watch.js.map +1 -1
- package/package.json +6 -3
- package/src/Connectors.ts +8 -2
- package/src/Costs.ts +44 -12
- package/src/Doctor.ts +62 -14
- package/src/FlowRunner.ts +177 -16
- package/src/NodeGeminiCliExecutor.ts +3 -3
- package/src/RunReaders.ts +413 -0
- package/src/RunSummary.ts +62 -0
- package/src/StoryReaders.ts +157 -0
- package/src/Terminal.ts +4 -1
- package/src/Watch.ts +2 -1
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import { join } from "node:path"
|
|
2
|
+
import * as Effect from "effect/Effect"
|
|
3
|
+
import * as Option from "effect/Option"
|
|
4
|
+
import * as Schema from "effect/Schema"
|
|
5
|
+
import { Board, BoardVersion } from "@llm4ts/flow/BoardSync"
|
|
6
|
+
import { loadCostLedger } from "@llm4ts/flow/CostLedger"
|
|
7
|
+
import {
|
|
8
|
+
buildCostReport,
|
|
9
|
+
reconcileWithLedger,
|
|
10
|
+
usageSamplesFromTrace
|
|
11
|
+
} from "@llm4ts/flow/CostReport"
|
|
12
|
+
import { describeFlowError, type FlowError } from "@llm4ts/flow/FlowError"
|
|
13
|
+
import { FlowEvent } from "@llm4ts/flow/FlowEvents"
|
|
14
|
+
import { RunOutcome, type TraceLine } from "@llm4ts/flow/FlowRecorder"
|
|
15
|
+
import {
|
|
16
|
+
CostsOut,
|
|
17
|
+
Diagnosis,
|
|
18
|
+
RunStatus,
|
|
19
|
+
type LaneStatusOut,
|
|
20
|
+
type MeaningfulEvent,
|
|
21
|
+
type OperatorRunState
|
|
22
|
+
} from "@llm4ts/flow/OperatorSchemas"
|
|
23
|
+
import { loadVersioned, type PlainFileStoreShape } from "@llm4ts/flow/Persistence"
|
|
24
|
+
import { parsePlan } from "@llm4ts/flow/Plan"
|
|
25
|
+
import { readTrace } from "@llm4ts/flow/Replay"
|
|
26
|
+
import { emptyTree, reduceTree, treeInputsOfTrace, type TreeState } from "./AgentTree.ts"
|
|
27
|
+
import { summarizeTree } from "./RunSummary.ts"
|
|
28
|
+
import { loadTranscript, tailLinesOf, type TranscriptFiles } from "./Transcripts.ts"
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Read-only answers about a run, from the files it writes: how it stands,
|
|
32
|
+
* what happened since a sequence number, why it failed, what it cost. Every
|
|
33
|
+
* reader takes the file store as a parameter, so a test feeds it a memory
|
|
34
|
+
* store and the MCP operator feeds it the real one.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
export class TraceUnreadable extends Schema.TaggedError<TraceUnreadable>()("TraceUnreadable", {
|
|
38
|
+
path: Schema.String,
|
|
39
|
+
message: Schema.String
|
|
40
|
+
}) {}
|
|
41
|
+
|
|
42
|
+
export interface RunSources {
|
|
43
|
+
readonly files: PlainFileStoreShape
|
|
44
|
+
/**
|
|
45
|
+
* Relative paths of every regular file under a directory, recursively (e.g.
|
|
46
|
+
* "task-2/1-pnpm-test.log"), never directory names; none when it does not exist.
|
|
47
|
+
*/
|
|
48
|
+
readonly listFiles: (directory: string) => ReadonlyArray<string>
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface TraceFold {
|
|
52
|
+
readonly lines: ReadonlyArray<TraceLine>
|
|
53
|
+
readonly state: TreeState
|
|
54
|
+
readonly seq: number
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const decodeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(FlowEvent))
|
|
58
|
+
const isOutcome = Schema.is(RunOutcome)
|
|
59
|
+
|
|
60
|
+
const eventOf = (line: TraceLine): FlowEvent | undefined => {
|
|
61
|
+
const raw = line.fields.event
|
|
62
|
+
return typeof raw === "string" ? Option.getOrUndefined(decodeEvent(raw)) : undefined
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const unreadable =
|
|
66
|
+
(path: string) =>
|
|
67
|
+
(error: FlowError): TraceUnreadable =>
|
|
68
|
+
TraceUnreadable.make({ path, message: describeFlowError(error) })
|
|
69
|
+
|
|
70
|
+
const readLines = (
|
|
71
|
+
files: PlainFileStoreShape,
|
|
72
|
+
path: string
|
|
73
|
+
): Effect.Effect<ReadonlyArray<TraceLine>, TraceUnreadable> =>
|
|
74
|
+
readTrace(files, path).pipe(Effect.mapError(unreadable(path)))
|
|
75
|
+
|
|
76
|
+
const lastSeq = (lines: ReadonlyArray<TraceLine>): number => lines[lines.length - 1]?.seq ?? 0
|
|
77
|
+
|
|
78
|
+
/** The trace folded into the watch TUI's tree, seeded from the epic's board when there is one. */
|
|
79
|
+
export const foldTrace = (
|
|
80
|
+
files: PlainFileStoreShape,
|
|
81
|
+
tracePath: string,
|
|
82
|
+
boardDir?: string
|
|
83
|
+
): Effect.Effect<TraceFold, TraceUnreadable> =>
|
|
84
|
+
Effect.gen(function* () {
|
|
85
|
+
const lines = yield* readLines(files, tracePath)
|
|
86
|
+
const board =
|
|
87
|
+
boardDir === undefined
|
|
88
|
+
? undefined
|
|
89
|
+
: yield* loadVersioned(files, join(boardDir, "board.json"), BoardVersion, Board).pipe(
|
|
90
|
+
Effect.catch(() => Effect.succeed(undefined))
|
|
91
|
+
)
|
|
92
|
+
const start = emptyTree(
|
|
93
|
+
board === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: {
|
|
96
|
+
title: board.title.replace(/^Epic:\s*/u, "epic "),
|
|
97
|
+
stories: board.items.map((item) => ({ id: item.id, status: item.status }))
|
|
98
|
+
}
|
|
99
|
+
)
|
|
100
|
+
const state = treeInputsOfTrace(lines).reduce(reduceTree, start)
|
|
101
|
+
return { lines, state, seq: lastSeq(lines) }
|
|
102
|
+
})
|
|
103
|
+
|
|
104
|
+
export const runStatusOf = (
|
|
105
|
+
fold: TraceFold,
|
|
106
|
+
run: {
|
|
107
|
+
readonly runId: string
|
|
108
|
+
readonly flow: string
|
|
109
|
+
readonly state: OperatorRunState
|
|
110
|
+
},
|
|
111
|
+
now: number
|
|
112
|
+
): RunStatus => {
|
|
113
|
+
const { state } = fold
|
|
114
|
+
const lanes = state.lanes.map((lane): LaneStatusOut => {
|
|
115
|
+
const stage = lane.stages[lane.stages.length - 1]
|
|
116
|
+
return {
|
|
117
|
+
id: lane.id,
|
|
118
|
+
status: lane.status,
|
|
119
|
+
...(stage === undefined ? {} : { stage }),
|
|
120
|
+
...(lane.executor === undefined ? {} : { executor: lane.executor }),
|
|
121
|
+
...(lane.activity === undefined ? {} : { activity: lane.activity.text }),
|
|
122
|
+
lastEventAgeMs: Math.max(0, Math.round(now - lane.lastEventAt)),
|
|
123
|
+
costUsd: lane.costUsd
|
|
124
|
+
}
|
|
125
|
+
})
|
|
126
|
+
const unenforceable = fold.lines.flatMap((line) => {
|
|
127
|
+
const event = eventOf(line)
|
|
128
|
+
return event?._tag === "CapabilityUnenforceable" ? [event.detail] : []
|
|
129
|
+
})
|
|
130
|
+
const end = state.ended === undefined ? now : (state.now ?? now)
|
|
131
|
+
return RunStatus.make({
|
|
132
|
+
runId: run.runId,
|
|
133
|
+
state: run.state,
|
|
134
|
+
...(state.ended !== undefined && isOutcome(state.ended) ? { ended: state.ended } : {}),
|
|
135
|
+
elapsedMs: state.startedAt === undefined ? 0 : Math.max(0, Math.round(end - state.startedAt)),
|
|
136
|
+
lanes,
|
|
137
|
+
stories: state.stories.map((story) => ({ id: story.id, status: story.status })),
|
|
138
|
+
tokens: Math.round(state.tokens),
|
|
139
|
+
costUsd: state.costUsd,
|
|
140
|
+
warnings: [
|
|
141
|
+
...unenforceable,
|
|
142
|
+
...state.exclusions.map((entry) => `${entry.executor} excluded: ${entry.reason}`)
|
|
143
|
+
],
|
|
144
|
+
seq: fold.seq,
|
|
145
|
+
summary: summarizeTree(state, run.flow)
|
|
146
|
+
})
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export const meaningfulKinds: ReadonlySet<string> = new Set([
|
|
150
|
+
"StoryStatusChanged",
|
|
151
|
+
"StageFailed",
|
|
152
|
+
"Aborted",
|
|
153
|
+
"CapabilityUnenforceable",
|
|
154
|
+
"ExecutorExcluded",
|
|
155
|
+
"ExecutorHandedOver",
|
|
156
|
+
"StoryJudged",
|
|
157
|
+
"Began",
|
|
158
|
+
"RunEnded"
|
|
159
|
+
])
|
|
160
|
+
|
|
161
|
+
const textOf = (
|
|
162
|
+
event: FlowEvent
|
|
163
|
+
): { readonly text: string; readonly lane?: string } | undefined => {
|
|
164
|
+
switch (event._tag) {
|
|
165
|
+
case "StoryStatusChanged":
|
|
166
|
+
return { text: `story ${event.id} ${event.status}`, lane: event.id }
|
|
167
|
+
case "StageFailed":
|
|
168
|
+
return {
|
|
169
|
+
text: `${event.stage} failed: ${event.message}`,
|
|
170
|
+
...(event.lane === undefined ? {} : { lane: event.lane })
|
|
171
|
+
}
|
|
172
|
+
case "Aborted":
|
|
173
|
+
return { text: `aborted: ${event.message}` }
|
|
174
|
+
case "CapabilityUnenforceable":
|
|
175
|
+
return { text: `capability unenforceable: ${event.detail}` }
|
|
176
|
+
case "ExecutorExcluded":
|
|
177
|
+
return { text: `${event.executor} excluded: ${event.reason}` }
|
|
178
|
+
case "ExecutorHandedOver":
|
|
179
|
+
return {
|
|
180
|
+
text: `${event.from} handed over ${event.role} (${event.scope}): ${event.reason}`,
|
|
181
|
+
...(event.label === undefined ? {} : { lane: event.label })
|
|
182
|
+
}
|
|
183
|
+
case "StoryJudged":
|
|
184
|
+
return {
|
|
185
|
+
text: `${event.lane} judged round ${event.round}: ${event.cleared ? "cleared" : `${event.issues} issues`}`,
|
|
186
|
+
lane: event.lane
|
|
187
|
+
}
|
|
188
|
+
case "Began":
|
|
189
|
+
return {
|
|
190
|
+
text: `began ${event.kind} ${event.label}`,
|
|
191
|
+
...(event.lane === undefined ? {} : { lane: event.lane })
|
|
192
|
+
}
|
|
193
|
+
default:
|
|
194
|
+
return undefined
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export const meaningfulEventsAfter = (
|
|
199
|
+
lines: ReadonlyArray<TraceLine>,
|
|
200
|
+
sinceSeq: number
|
|
201
|
+
): ReadonlyArray<MeaningfulEvent> =>
|
|
202
|
+
lines.flatMap((line): ReadonlyArray<MeaningfulEvent> => {
|
|
203
|
+
if (line.seq <= sinceSeq || !meaningfulKinds.has(line.kind)) {
|
|
204
|
+
return []
|
|
205
|
+
}
|
|
206
|
+
if (line.kind === "RunEnded") {
|
|
207
|
+
const outcome = line.fields.outcome
|
|
208
|
+
return [
|
|
209
|
+
{
|
|
210
|
+
seq: line.seq,
|
|
211
|
+
at: line.timestamp,
|
|
212
|
+
kind: line.kind,
|
|
213
|
+
text: `run ${typeof outcome === "string" ? outcome : "completed"}`
|
|
214
|
+
}
|
|
215
|
+
]
|
|
216
|
+
}
|
|
217
|
+
const event = eventOf(line)
|
|
218
|
+
const rendered = event === undefined ? undefined : textOf(event)
|
|
219
|
+
return rendered === undefined
|
|
220
|
+
? []
|
|
221
|
+
: [
|
|
222
|
+
{
|
|
223
|
+
seq: line.seq,
|
|
224
|
+
at: line.timestamp,
|
|
225
|
+
kind: line.kind,
|
|
226
|
+
...(rendered.lane === undefined ? {} : { lane: rendered.lane }),
|
|
227
|
+
text: rendered.text
|
|
228
|
+
}
|
|
229
|
+
]
|
|
230
|
+
})
|
|
231
|
+
|
|
232
|
+
export const eventsAfter = (
|
|
233
|
+
lines: ReadonlyArray<TraceLine>,
|
|
234
|
+
sinceSeq: number,
|
|
235
|
+
kinds: ReadonlyArray<string> | undefined,
|
|
236
|
+
limit: number
|
|
237
|
+
): ReadonlyArray<TraceLine> =>
|
|
238
|
+
lines
|
|
239
|
+
.filter(
|
|
240
|
+
(line) =>
|
|
241
|
+
line.seq > sinceSeq && (kinds === undefined || kinds.some((kind) => kind === line.kind))
|
|
242
|
+
)
|
|
243
|
+
.slice(0, Math.max(0, limit))
|
|
244
|
+
|
|
245
|
+
const digestTail = 40
|
|
246
|
+
const tailRows = 30
|
|
247
|
+
const tailWidth = 120
|
|
248
|
+
|
|
249
|
+
const quiet = <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A | undefined> =>
|
|
250
|
+
effect.pipe(Effect.catch(() => Effect.succeed(undefined)))
|
|
251
|
+
|
|
252
|
+
const capText = (text: string, remaining: number): string => text.slice(0, Math.max(0, remaining))
|
|
253
|
+
|
|
254
|
+
/** The last rows of each log in a directory, within what is left of a character budget. */
|
|
255
|
+
export const tailGateLogs = (
|
|
256
|
+
sources: RunSources,
|
|
257
|
+
directory: string | undefined,
|
|
258
|
+
budget: number
|
|
259
|
+
): Effect.Effect<{
|
|
260
|
+
readonly tails: ReadonlyArray<{ readonly path: string; readonly tail: string }>
|
|
261
|
+
readonly remaining: number
|
|
262
|
+
}> =>
|
|
263
|
+
Effect.gen(function* () {
|
|
264
|
+
const tails: Array<{ path: string; tail: string }> = []
|
|
265
|
+
let remaining = budget
|
|
266
|
+
if (directory !== undefined) {
|
|
267
|
+
for (const name of sources.listFiles(directory)) {
|
|
268
|
+
const path = join(directory, name)
|
|
269
|
+
const text = yield* quiet(sources.files.read(path))
|
|
270
|
+
if (text === undefined || remaining <= 0) continue
|
|
271
|
+
const tail = capText(text.trimEnd().split(/\r?\n/u).slice(-tailRows).join("\n"), remaining)
|
|
272
|
+
remaining -= tail.length
|
|
273
|
+
tails.push({ path, tail })
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
return { tails, remaining }
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
/** The last rows of one lane's transcript, capped to what is left of a character budget. */
|
|
280
|
+
export const tailTranscript = (
|
|
281
|
+
sources: RunSources,
|
|
282
|
+
directory: string | undefined,
|
|
283
|
+
lane: string,
|
|
284
|
+
budget: number
|
|
285
|
+
): Effect.Effect<ReadonlyArray<string>> =>
|
|
286
|
+
Effect.gen(function* () {
|
|
287
|
+
const out: Array<string> = []
|
|
288
|
+
if (directory === undefined) return out
|
|
289
|
+
let remaining = budget
|
|
290
|
+
const transcriptFiles: TranscriptFiles = {
|
|
291
|
+
read: (path) => quiet(sources.files.read(path)),
|
|
292
|
+
list: sources.listFiles
|
|
293
|
+
}
|
|
294
|
+
const entries = yield* loadTranscript(transcriptFiles, directory, { lane })
|
|
295
|
+
const rows = tailLinesOf(entries, { lane }, undefined, tailWidth, tailRows) ?? []
|
|
296
|
+
for (const row of rows) {
|
|
297
|
+
if (remaining <= 0) break
|
|
298
|
+
const kept = capText(row, remaining)
|
|
299
|
+
remaining -= kept.length
|
|
300
|
+
out.push(kept)
|
|
301
|
+
}
|
|
302
|
+
return out
|
|
303
|
+
})
|
|
304
|
+
|
|
305
|
+
export const runDiagnosisOf = (
|
|
306
|
+
sources: RunSources,
|
|
307
|
+
args: {
|
|
308
|
+
readonly runId: string
|
|
309
|
+
readonly tracePath: string
|
|
310
|
+
readonly planPath: string | undefined
|
|
311
|
+
readonly transcriptsDir: string | undefined
|
|
312
|
+
readonly gateLogDir: string | undefined
|
|
313
|
+
readonly budget: number
|
|
314
|
+
}
|
|
315
|
+
): Effect.Effect<Diagnosis, TraceUnreadable> =>
|
|
316
|
+
Effect.gen(function* () {
|
|
317
|
+
const lines = yield* readLines(sources.files, args.tracePath)
|
|
318
|
+
const events = lines.flatMap((line) => {
|
|
319
|
+
const event = eventOf(line)
|
|
320
|
+
return event === undefined ? [] : [event]
|
|
321
|
+
})
|
|
322
|
+
const failed = events.filter(
|
|
323
|
+
(event) => event._tag === "StageFailed" || event._tag === "Aborted"
|
|
324
|
+
)
|
|
325
|
+
const last = failed[failed.length - 1]
|
|
326
|
+
const failure =
|
|
327
|
+
last === undefined
|
|
328
|
+
? undefined
|
|
329
|
+
: last._tag === "StageFailed"
|
|
330
|
+
? {
|
|
331
|
+
stage: last.stage,
|
|
332
|
+
message: last.message,
|
|
333
|
+
...(last.lane === undefined ? {} : { lane: last.lane })
|
|
334
|
+
}
|
|
335
|
+
: last._tag === "Aborted"
|
|
336
|
+
? { message: last.message }
|
|
337
|
+
: undefined
|
|
338
|
+
|
|
339
|
+
const markdown =
|
|
340
|
+
args.planPath === undefined ? undefined : yield* quiet(sources.files.read(args.planPath))
|
|
341
|
+
const plan = markdown === undefined ? undefined : yield* quiet(parsePlan(markdown))
|
|
342
|
+
const taskLines = (plan?.tasks ?? []).map(
|
|
343
|
+
(task) => `- [${task.completed ? "x" : " "}] ${task.title}`
|
|
344
|
+
)
|
|
345
|
+
const recent = events
|
|
346
|
+
.flatMap((event) =>
|
|
347
|
+
event._tag === "Info"
|
|
348
|
+
? [event.message]
|
|
349
|
+
: event._tag === "StageFailed"
|
|
350
|
+
? [`${event.stage} failed: ${event.message}`]
|
|
351
|
+
: []
|
|
352
|
+
)
|
|
353
|
+
.slice(-digestTail)
|
|
354
|
+
const digest = capText([...taskLines, ...recent].join("\n"), args.budget)
|
|
355
|
+
const remaining = args.budget - digest.length
|
|
356
|
+
|
|
357
|
+
const gateTails = yield* tailGateLogs(sources, args.gateLogDir, remaining)
|
|
358
|
+
const transcript = yield* tailTranscript(
|
|
359
|
+
sources,
|
|
360
|
+
args.transcriptsDir,
|
|
361
|
+
"run",
|
|
362
|
+
gateTails.remaining
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
return Diagnosis.make({
|
|
366
|
+
runId: args.runId,
|
|
367
|
+
digest,
|
|
368
|
+
...(failure === undefined ? {} : { failure }),
|
|
369
|
+
gateLogTails: gateTails.tails,
|
|
370
|
+
transcriptTail: transcript
|
|
371
|
+
})
|
|
372
|
+
})
|
|
373
|
+
|
|
374
|
+
const ledgerTolerance = 0.005
|
|
375
|
+
|
|
376
|
+
const costOf = (report: {
|
|
377
|
+
readonly measured: { readonly costUsd?: number }
|
|
378
|
+
readonly estimated: { readonly costUsd?: number }
|
|
379
|
+
}): number => (report.measured.costUsd ?? 0) + (report.estimated.costUsd ?? 0)
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* One run's costs: its trace folded and topped up from its `costs.jsonl`
|
|
383
|
+
* line (`CostReport.reconcileWithLedger`), so `total` is what the run itself
|
|
384
|
+
* printed even when the trace lost lines; `fromLedger` is how much of it the
|
|
385
|
+
* ledger supplied. `differs` is left for a trace that says more than the
|
|
386
|
+
* ledger, which no line can fill.
|
|
387
|
+
*/
|
|
388
|
+
export const costsOfRun = (
|
|
389
|
+
files: PlainFileStoreShape,
|
|
390
|
+
tracePath: string,
|
|
391
|
+
ledgerPath: string,
|
|
392
|
+
runId: string
|
|
393
|
+
): Effect.Effect<CostsOut, TraceUnreadable> =>
|
|
394
|
+
Effect.gen(function* () {
|
|
395
|
+
const lines = yield* readLines(files, tracePath)
|
|
396
|
+
const traced = usageSamplesFromTrace(lines)
|
|
397
|
+
const records = yield* loadCostLedger(files, ledgerPath).pipe(
|
|
398
|
+
Effect.mapError(unreadable(ledgerPath))
|
|
399
|
+
)
|
|
400
|
+
const record = records.find((candidate) => candidate.runId === runId)
|
|
401
|
+
const reconciled = record === undefined ? traced : reconcileWithLedger(traced, [record]).samples
|
|
402
|
+
const report = buildCostReport(reconciled)
|
|
403
|
+
const total = costOf(report)
|
|
404
|
+
const fromLedger = total - costOf(buildCostReport(traced))
|
|
405
|
+
const ledger = record?.totalCostUsd
|
|
406
|
+
return CostsOut.make({
|
|
407
|
+
report,
|
|
408
|
+
total,
|
|
409
|
+
...(ledger === undefined ? {} : { ledger }),
|
|
410
|
+
...(fromLedger > ledgerTolerance ? { fromLedger } : {}),
|
|
411
|
+
differs: ledger !== undefined && Math.abs(total - ledger) > ledgerTolerance
|
|
412
|
+
})
|
|
413
|
+
})
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { TreeLane, TreeState } from "./AgentTree.ts"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* One sentence on how a run is going, for a chat agent that asked. Pure over
|
|
5
|
+
* the watch TUI's folded state, so a live run and a replay read the same.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const listed = 3
|
|
9
|
+
|
|
10
|
+
const elapsedOf = (state: TreeState): string => {
|
|
11
|
+
const minutes =
|
|
12
|
+
state.startedAt === undefined || state.now === undefined
|
|
13
|
+
? 0
|
|
14
|
+
: Math.max(0, Math.floor((state.now - state.startedAt) / 60_000))
|
|
15
|
+
return minutes < 60 ? `${minutes} min` : `${Math.floor(minutes / 60)} h ${minutes % 60} min`
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** At most three entries; the rest collapse into `+<n>`. */
|
|
19
|
+
const capped = (items: ReadonlyArray<string>, separator: string): string =>
|
|
20
|
+
items.length > listed
|
|
21
|
+
? [...items.slice(0, listed), `+${items.length - listed}`].join(separator)
|
|
22
|
+
: items.join(separator)
|
|
23
|
+
|
|
24
|
+
const doing = (lane: TreeLane | undefined): string =>
|
|
25
|
+
lane?.stages[lane.stages.length - 1] ?? lane?.activity?.text ?? "starting"
|
|
26
|
+
|
|
27
|
+
const lastLogOf = (state: TreeState, id: string): string => {
|
|
28
|
+
for (let index = state.log.length - 1; index >= 0; index--) {
|
|
29
|
+
const entry = state.log[index]
|
|
30
|
+
if (entry !== undefined && entry.who === id) return entry.what
|
|
31
|
+
}
|
|
32
|
+
return "failed"
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const parenthesized = (items: ReadonlyArray<string>, separator: string): string =>
|
|
36
|
+
items.length === 0 ? "" : ` (${capped(items, separator)})`
|
|
37
|
+
|
|
38
|
+
export const summarizeTree = (state: TreeState, flow: string | undefined): string => {
|
|
39
|
+
const ended = state.ended === undefined ? "" : `${state.ended}, `
|
|
40
|
+
const head = `${ended}${flow ?? state.title}, ${elapsedOf(state)}`
|
|
41
|
+
const cost = `$${state.costUsd.toFixed(2)}`
|
|
42
|
+
const board = state.stories.length > 0
|
|
43
|
+
if (!board && state.lanes.length === 0) return `${head}: no stories yet, ${cost}`
|
|
44
|
+
const ids = (status: "done" | "active" | "failed"): ReadonlyArray<string> =>
|
|
45
|
+
board
|
|
46
|
+
? state.stories.filter((story) => story.status === status).map((story) => story.id)
|
|
47
|
+
: state.lanes
|
|
48
|
+
.filter((lane) => lane.status === (status === "active" ? "running" : status))
|
|
49
|
+
.map((lane) => lane.id)
|
|
50
|
+
const active = ids("active")
|
|
51
|
+
const failed = ids("failed")
|
|
52
|
+
const activeItems = active.map((id) => {
|
|
53
|
+
const lane = state.lanes.find((open) => open.id === id)
|
|
54
|
+
return `${id}, ${lane?.executor ?? "?"}, ${doing(lane)}`
|
|
55
|
+
})
|
|
56
|
+
const failedItems = failed.map((id) => `${id}: ${lastLogOf(state, id)}`)
|
|
57
|
+
return (
|
|
58
|
+
`${head}: ${ids("done").length} stories done, ` +
|
|
59
|
+
`${active.length} active${parenthesized(activeItems, "; ")}, ` +
|
|
60
|
+
`${failed.length} failed${parenthesized(failedItems, "; ")}, ${cost}`
|
|
61
|
+
)
|
|
62
|
+
}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { join } from "node:path"
|
|
2
|
+
import * as Effect from "effect/Effect"
|
|
3
|
+
import * as Option from "effect/Option"
|
|
4
|
+
import * as Schema from "effect/Schema"
|
|
5
|
+
import { describeFlowError, type FlowError } from "@llm4ts/flow/FlowError"
|
|
6
|
+
import { Diagnosis, StoryDetail, UnknownStory } from "@llm4ts/flow/OperatorSchemas"
|
|
7
|
+
import {
|
|
8
|
+
loadEpicFiles,
|
|
9
|
+
loadStoryFiles,
|
|
10
|
+
renderStoryDigest,
|
|
11
|
+
type RetroInputs
|
|
12
|
+
} from "@llm4ts/flow/Retro"
|
|
13
|
+
import { readTrace } from "@llm4ts/flow/Replay"
|
|
14
|
+
import { makeStoryPlanStore, type Story, type StoryPlan } from "@llm4ts/flow/StoryPlan"
|
|
15
|
+
import { TraceUnreadable, tailGateLogs, tailTranscript, type RunSources } from "./RunReaders.ts"
|
|
16
|
+
import { loadTranscript, type TranscriptFiles } from "./Transcripts.ts"
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Read-only answers about one story of an epic run, from the files the story
|
|
20
|
+
* executor keeps under `<stateDir>/stories/<id>.*` and `<stateDir>/board.json`.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const safeId = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u
|
|
24
|
+
|
|
25
|
+
/** A story id is a file-name stem: no separators, no dots-only, nothing that climbs. */
|
|
26
|
+
export const isSafeStoryId = (id: string): boolean => safeId.test(id)
|
|
27
|
+
|
|
28
|
+
const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json))
|
|
29
|
+
const encodeJson = (value: unknown): string | undefined => JSON.stringify(value)
|
|
30
|
+
|
|
31
|
+
const planPathOf = (stateDir: string): string => join(stateDir, "plan.md")
|
|
32
|
+
|
|
33
|
+
const unknownStory = (storyId: string): UnknownStory =>
|
|
34
|
+
UnknownStory.make({ message: `unknown story '${storyId}'` })
|
|
35
|
+
|
|
36
|
+
/** The plan's story, or `UnknownStory` for an unsafe id, a missing plan or an id the plan lacks. */
|
|
37
|
+
const storyOf = (
|
|
38
|
+
sources: RunSources,
|
|
39
|
+
stateDir: string,
|
|
40
|
+
storyId: string
|
|
41
|
+
): Effect.Effect<{ readonly plan: StoryPlan; readonly story: Story }, FlowError | UnknownStory> =>
|
|
42
|
+
Effect.gen(function* () {
|
|
43
|
+
if (!isSafeStoryId(storyId)) {
|
|
44
|
+
return yield* unknownStory(storyId)
|
|
45
|
+
}
|
|
46
|
+
const plan = yield* makeStoryPlanStore(sources.files).load(planPathOf(stateDir))
|
|
47
|
+
const story = plan?.story(storyId)
|
|
48
|
+
if (plan === undefined || story === undefined) {
|
|
49
|
+
return yield* unknownStory(storyId)
|
|
50
|
+
}
|
|
51
|
+
return { plan, story }
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
export const storyDetailOf = (
|
|
55
|
+
sources: RunSources,
|
|
56
|
+
stateDir: string,
|
|
57
|
+
storyId: string
|
|
58
|
+
): Effect.Effect<StoryDetail, FlowError | UnknownStory> =>
|
|
59
|
+
Effect.gen(function* () {
|
|
60
|
+
const { story } = yield* storyOf(sources, stateDir, storyId)
|
|
61
|
+
const stored = yield* loadStoryFiles(sources.files, stateDir, story)
|
|
62
|
+
const epic = yield* loadEpicFiles(sources.files, stateDir)
|
|
63
|
+
const item = epic.board?.items.find((candidate) => candidate.id === storyId)
|
|
64
|
+
const verdictText = stored.verdict === undefined ? undefined : encodeJson(stored.verdict)
|
|
65
|
+
const verdict =
|
|
66
|
+
verdictText === undefined ? undefined : Option.getOrUndefined(decodeJson(verdictText))
|
|
67
|
+
const gateDir = join(stateDir, "stories", storyId, "gates")
|
|
68
|
+
return StoryDetail.make({
|
|
69
|
+
id: story.id,
|
|
70
|
+
title: story.title,
|
|
71
|
+
...(stored.state === undefined
|
|
72
|
+
? {}
|
|
73
|
+
: {
|
|
74
|
+
state: {
|
|
75
|
+
status: stored.state.status,
|
|
76
|
+
branch: stored.state.branch,
|
|
77
|
+
worktree: stored.state.worktree,
|
|
78
|
+
...(stored.state.executor === undefined ? {} : { executor: stored.state.executor })
|
|
79
|
+
}
|
|
80
|
+
}),
|
|
81
|
+
...(item === undefined
|
|
82
|
+
? {}
|
|
83
|
+
: {
|
|
84
|
+
board: {
|
|
85
|
+
status: item.status,
|
|
86
|
+
...(item.detail === undefined ? {} : { detail: item.detail })
|
|
87
|
+
}
|
|
88
|
+
}),
|
|
89
|
+
tasks: (stored.plan?.tasks ?? []).map((task) => ({
|
|
90
|
+
title: task.title,
|
|
91
|
+
completed: task.completed
|
|
92
|
+
})),
|
|
93
|
+
...(stored.findings === undefined ? {} : { findings: stored.findings }),
|
|
94
|
+
...(verdict === undefined ? {} : { verdict }),
|
|
95
|
+
gateLogs: sources.listFiles(gateDir).map((name) => join(gateDir, name))
|
|
96
|
+
})
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
export const storyDiagnosisOf = (
|
|
100
|
+
sources: RunSources,
|
|
101
|
+
args: {
|
|
102
|
+
readonly stateDir: string
|
|
103
|
+
readonly storyId: string
|
|
104
|
+
readonly runId: string
|
|
105
|
+
readonly tracePath: string
|
|
106
|
+
readonly transcriptsDir: string | undefined
|
|
107
|
+
readonly budget: number
|
|
108
|
+
}
|
|
109
|
+
): Effect.Effect<Diagnosis, FlowError | UnknownStory | TraceUnreadable> =>
|
|
110
|
+
Effect.gen(function* () {
|
|
111
|
+
const { plan, story } = yield* storyOf(sources, args.stateDir, args.storyId)
|
|
112
|
+
const trace = yield* readTrace(sources.files, args.tracePath).pipe(
|
|
113
|
+
Effect.mapError((error) =>
|
|
114
|
+
TraceUnreadable.make({ path: args.tracePath, message: describeFlowError(error) })
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
const stored = yield* loadStoryFiles(sources.files, args.stateDir, story)
|
|
118
|
+
const epic = yield* loadEpicFiles(sources.files, args.stateDir)
|
|
119
|
+
const transcriptFiles: TranscriptFiles = {
|
|
120
|
+
read: (path) => Effect.catch(sources.files.read(path), () => Effect.succeed(undefined)),
|
|
121
|
+
list: sources.listFiles
|
|
122
|
+
}
|
|
123
|
+
const entries =
|
|
124
|
+
args.transcriptsDir === undefined
|
|
125
|
+
? undefined
|
|
126
|
+
: yield* loadTranscript(transcriptFiles, args.transcriptsDir, { lane: story.id })
|
|
127
|
+
const inputs: RetroInputs = {
|
|
128
|
+
runId: args.runId,
|
|
129
|
+
plan,
|
|
130
|
+
trace,
|
|
131
|
+
transcripts: entries === undefined ? undefined : new Map([[story.id, entries]]),
|
|
132
|
+
report: epic.report,
|
|
133
|
+
board: epic.board,
|
|
134
|
+
stories: [stored]
|
|
135
|
+
}
|
|
136
|
+
const digest = renderStoryDigest(inputs, story.id, args.budget) ?? ""
|
|
137
|
+
const item = epic.board?.items.find((candidate) => candidate.id === story.id)
|
|
138
|
+
const gates = yield* tailGateLogs(
|
|
139
|
+
sources,
|
|
140
|
+
join(args.stateDir, "stories", story.id, "gates"),
|
|
141
|
+
args.budget - digest.length
|
|
142
|
+
)
|
|
143
|
+
const transcriptTail = yield* tailTranscript(
|
|
144
|
+
sources,
|
|
145
|
+
args.transcriptsDir,
|
|
146
|
+
story.id,
|
|
147
|
+
gates.remaining
|
|
148
|
+
)
|
|
149
|
+
return Diagnosis.make({
|
|
150
|
+
runId: args.runId,
|
|
151
|
+
story: story.id,
|
|
152
|
+
digest,
|
|
153
|
+
...(item?.detail === undefined ? {} : { failure: { message: item.detail, lane: story.id } }),
|
|
154
|
+
gateLogTails: gates.tails,
|
|
155
|
+
transcriptTail
|
|
156
|
+
})
|
|
157
|
+
})
|
package/src/Terminal.ts
CHANGED
|
@@ -460,6 +460,9 @@ export const makeLiveTerminalSurface = Effect.fn("@llm4ts/runner/Terminal.makeLi
|
|
|
460
460
|
const hideCursor = "\u001b[?25l"
|
|
461
461
|
const showCursor = "\u001b[?25h"
|
|
462
462
|
|
|
463
|
+
/** Re-raise only when no other listener remains to handle the signal. */
|
|
464
|
+
export const shouldReraise = (otherListeners: number): boolean => otherListeners === 0
|
|
465
|
+
|
|
463
466
|
/**
|
|
464
467
|
* While the live status block is on screen, keystrokes must not be echoed:
|
|
465
468
|
* an arrow key (or a trackpad scroll the terminal turns into one) echoes as
|
|
@@ -490,7 +493,7 @@ const quietTerminalInput = (
|
|
|
490
493
|
// then let the signal do what it would have done.
|
|
491
494
|
const onSignal = (signal: NodeJS.Signals): void => {
|
|
492
495
|
restore()
|
|
493
|
-
process.kill(process.pid, signal)
|
|
496
|
+
if (shouldReraise(process.listenerCount(signal))) process.kill(process.pid, signal)
|
|
494
497
|
}
|
|
495
498
|
yield* Effect.sync(() => {
|
|
496
499
|
stty("-echo")
|
package/src/Watch.ts
CHANGED
|
@@ -362,7 +362,8 @@ export const makeWatchProgram = Effect.fn("@llm4ts/runner/Watch.make")(function*
|
|
|
362
362
|
|
|
363
363
|
// ── node wiring ─────────────────────────────────────────────────────────────
|
|
364
364
|
|
|
365
|
-
|
|
365
|
+
/** Folder names under an `epics/` directory; none when it is absent. */
|
|
366
|
+
export const nodeListEpics = (directory: string): ReadonlyArray<string> =>
|
|
366
367
|
existsSync(directory)
|
|
367
368
|
? readdirSync(directory).filter((name) => statSync(join(directory, name)).isDirectory())
|
|
368
369
|
: []
|