@llm4ts/runner 2.22.2 → 2.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AgentTree.d.ts +21 -4
- package/dist/AgentTree.d.ts.map +1 -1
- package/dist/AgentTree.js +83 -24
- package/dist/AgentTree.js.map +1 -1
- package/dist/AgentTreeSurface.d.ts +5 -1
- package/dist/AgentTreeSurface.d.ts.map +1 -1
- package/dist/AgentTreeSurface.js +18 -5
- package/dist/AgentTreeSurface.js.map +1 -1
- package/dist/FlowArgs.d.ts +1 -1
- package/dist/FlowArgs.d.ts.map +1 -1
- package/dist/FlowArgs.js +1 -0
- package/dist/FlowArgs.js.map +1 -1
- package/dist/FlowRunner.d.ts +3 -0
- package/dist/FlowRunner.d.ts.map +1 -1
- package/dist/FlowRunner.js +35 -11
- package/dist/FlowRunner.js.map +1 -1
- package/dist/Profile.d.ts +20 -3
- package/dist/Profile.d.ts.map +1 -1
- package/dist/Profile.js +82 -20
- package/dist/Profile.js.map +1 -1
- package/dist/Transcripts.d.ts +55 -0
- package/dist/Transcripts.d.ts.map +1 -0
- package/dist/Transcripts.js +200 -0
- package/dist/Transcripts.js.map +1 -0
- package/dist/Watch.d.ts +5 -0
- package/dist/Watch.d.ts.map +1 -1
- package/dist/Watch.js +74 -5
- package/dist/Watch.js.map +1 -1
- package/package.json +4 -3
- package/src/AgentTree.ts +130 -27
- package/src/AgentTreeSurface.ts +26 -4
- package/src/FlowArgs.ts +1 -0
- package/src/FlowRunner.ts +67 -17
- package/src/Profile.ts +100 -20
- package/src/Transcripts.ts +274 -0
- package/src/Watch.ts +126 -5
package/src/Profile.ts
CHANGED
|
@@ -3,7 +3,9 @@ import * as Schema from "effect/Schema"
|
|
|
3
3
|
import type { FlowError } from "@llm4ts/flow/FlowError"
|
|
4
4
|
import type { FlowEvent, Timed } from "@llm4ts/flow/FlowEvents"
|
|
5
5
|
import { readTrace } from "@llm4ts/flow/Replay"
|
|
6
|
-
import { treeInputsOfTrace, type TreeInput } from "./AgentTree.ts"
|
|
6
|
+
import { duration, treeInputsOfTrace, type TreeInput } from "./AgentTree.ts"
|
|
7
|
+
|
|
8
|
+
export { duration }
|
|
7
9
|
import { formatCount } from "./Terminal.ts"
|
|
8
10
|
import {
|
|
9
11
|
resolveTraceTarget,
|
|
@@ -47,7 +49,11 @@ export const TurnStats = Schema.Struct({
|
|
|
47
49
|
lastPrompt: Schema.optionalKey(Schema.Int),
|
|
48
50
|
maxPrompt: Schema.optionalKey(Schema.Int),
|
|
49
51
|
/** The last prompt is more than three times the first. */
|
|
50
|
-
promptGrowth: Schema.Boolean
|
|
52
|
+
promptGrowth: Schema.Boolean,
|
|
53
|
+
/** Tool calls per finished turn: each is a model round trip. */
|
|
54
|
+
avgSteps: Schema.Number,
|
|
55
|
+
/** The model's own time per round trip: a turn's time less its tools, over its steps + 1. */
|
|
56
|
+
avgStepMs: Ms
|
|
51
57
|
})
|
|
52
58
|
export type TurnStats = typeof TurnStats.Type
|
|
53
59
|
|
|
@@ -118,6 +124,8 @@ export const ProfileReport = Schema.Struct({
|
|
|
118
124
|
models: Schema.Array(ModelRow),
|
|
119
125
|
gates: Schema.Array(GateRow),
|
|
120
126
|
tools: Schema.Array(CountRow),
|
|
127
|
+
/** Coder tool time by kind of work: explore, edit, test, build, install, git, other. */
|
|
128
|
+
toolCategories: Schema.Array(CountRow),
|
|
121
129
|
waits: Schema.Array(CountRow),
|
|
122
130
|
/** Timed on the run's events, in no story: the run's own planning, or a seat no story owns. */
|
|
123
131
|
outside: Schema.Array(CountRow),
|
|
@@ -450,7 +458,20 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
|
|
|
450
458
|
ms: sum(items.map((item) => item.event.ms))
|
|
451
459
|
}))
|
|
452
460
|
const tools = counted("tool")
|
|
461
|
+
const toolCategories = [
|
|
462
|
+
...groupBy(
|
|
463
|
+
allTimed.filter((item) => item.event.kind === "tool" && item.event.category !== undefined),
|
|
464
|
+
(item) => item.event.category ?? ""
|
|
465
|
+
).entries()
|
|
466
|
+
]
|
|
467
|
+
.map(([label, items]) => ({
|
|
468
|
+
label,
|
|
469
|
+
count: items.length,
|
|
470
|
+
ms: sum(items.map((item) => item.event.ms))
|
|
471
|
+
}))
|
|
472
|
+
.sort((left, right) => right.ms - left.ms)
|
|
453
473
|
const waits = counted("wait")
|
|
474
|
+
const steps = stepStats(estimated ? [] : [...lanes.values()].flatMap(turnSteps))
|
|
454
475
|
const outside = [
|
|
455
476
|
...groupBy(
|
|
456
477
|
allTimed.filter((item) => item.event.lane === undefined),
|
|
@@ -485,12 +506,49 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
|
|
|
485
506
|
models,
|
|
486
507
|
gates,
|
|
487
508
|
tools,
|
|
509
|
+
toolCategories,
|
|
488
510
|
waits,
|
|
489
511
|
outside,
|
|
490
|
-
findings: findingsOf({
|
|
512
|
+
findings: findingsOf({
|
|
513
|
+
steps,
|
|
514
|
+
toolCategories,
|
|
515
|
+
stories,
|
|
516
|
+
models,
|
|
517
|
+
gates,
|
|
518
|
+
tools,
|
|
519
|
+
waits,
|
|
520
|
+
merges,
|
|
521
|
+
storyTime,
|
|
522
|
+
estimated
|
|
523
|
+
})
|
|
491
524
|
}
|
|
492
525
|
}
|
|
493
526
|
|
|
527
|
+
const stepStats = (
|
|
528
|
+
turns: ReadonlyArray<{ readonly steps: number; readonly stepMs: number }>
|
|
529
|
+
): { readonly avgSteps: number; readonly avgStepMs: number } => ({
|
|
530
|
+
avgSteps: average(turns.map((turn) => turn.steps)) ?? 0,
|
|
531
|
+
avgStepMs: average(turns.map((turn) => turn.stepMs)) ?? 0
|
|
532
|
+
})
|
|
533
|
+
|
|
534
|
+
/** Each finished coder turn's steps (its tool calls) and model time per step. */
|
|
535
|
+
const turnSteps = (
|
|
536
|
+
lane: Lane
|
|
537
|
+
): ReadonlyArray<{ readonly steps: number; readonly stepMs: number }> => {
|
|
538
|
+
const tools = lane.timed.filter((item) => item.event.kind === "tool")
|
|
539
|
+
return lane.timed
|
|
540
|
+
.filter((item) => item.event.kind === "model" && item.event.label === "coder")
|
|
541
|
+
.map((turn) => {
|
|
542
|
+
const start = turn.at - turn.event.ms
|
|
543
|
+
const inside = tools.filter((tool) => tool.at > start && tool.at <= turn.at)
|
|
544
|
+
const toolMs = sum(inside.map((tool) => tool.event.ms))
|
|
545
|
+
return {
|
|
546
|
+
steps: inside.length,
|
|
547
|
+
stepMs: Math.max(0, turn.event.ms - toolMs) / (inside.length + 1)
|
|
548
|
+
}
|
|
549
|
+
})
|
|
550
|
+
}
|
|
551
|
+
|
|
494
552
|
const storyProfile = (
|
|
495
553
|
lane: Lane,
|
|
496
554
|
lastAt: number,
|
|
@@ -540,6 +598,7 @@ const storyProfile = (
|
|
|
540
598
|
...(firstPrompt === undefined ? {} : { firstPrompt }),
|
|
541
599
|
...(lastPrompt === undefined ? {} : { lastPrompt }),
|
|
542
600
|
...(maxPrompt === undefined ? {} : { maxPrompt }),
|
|
601
|
+
...stepStats(estimated ? [] : turnSteps(lane)),
|
|
543
602
|
promptGrowth:
|
|
544
603
|
firstPrompt !== undefined &&
|
|
545
604
|
lastPrompt !== undefined &&
|
|
@@ -551,24 +610,16 @@ const storyProfile = (
|
|
|
551
610
|
|
|
552
611
|
// ── words ───────────────────────────────────────────────────────────────────
|
|
553
612
|
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
const seconds = Math.round(ms / 1_000)
|
|
557
|
-
if (seconds < 60) {
|
|
558
|
-
return `${seconds}s`
|
|
559
|
-
}
|
|
560
|
-
const minutes = Math.floor(seconds / 60)
|
|
561
|
-
if (minutes < 60) {
|
|
562
|
-
return `${minutes}m${String(seconds % 60).padStart(2, "0")}s`
|
|
563
|
-
}
|
|
564
|
-
return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`
|
|
565
|
-
}
|
|
613
|
+
const stepsText = (steps: number): string =>
|
|
614
|
+
`${Number.isInteger(steps) ? steps : steps.toFixed(1)} step${steps === 1 ? "" : "s"}`
|
|
566
615
|
|
|
567
616
|
const share = (ms: number, of: number): string => (of <= 0 ? "" : `${Math.round((ms / of) * 100)}%`)
|
|
568
617
|
|
|
569
618
|
const plural = (count: number, word: string): string => `${count} ${word}${count === 1 ? "" : "s"}`
|
|
570
619
|
|
|
571
620
|
const findingsOf = (facts: {
|
|
621
|
+
readonly steps: { readonly avgSteps: number; readonly avgStepMs: number }
|
|
622
|
+
readonly toolCategories: ReadonlyArray<CountRow>
|
|
572
623
|
readonly stories: ReadonlyArray<StoryProfile>
|
|
573
624
|
readonly models: ReadonlyArray<ModelRow>
|
|
574
625
|
readonly gates: ReadonlyArray<GateRow>
|
|
@@ -589,7 +640,11 @@ const findingsOf = (facts: {
|
|
|
589
640
|
ms,
|
|
590
641
|
text:
|
|
591
642
|
role === "coder"
|
|
592
|
-
? `coder turns: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}
|
|
643
|
+
? `coder turns: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}${
|
|
644
|
+
facts.steps.avgSteps > 0
|
|
645
|
+
? `, ${stepsText(facts.steps.avgSteps)} per turn, ${duration(facts.steps.avgStepMs)} of model per step`
|
|
646
|
+
: ""
|
|
647
|
+
}`
|
|
593
648
|
: `${role} calls: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}`
|
|
594
649
|
}
|
|
595
650
|
})
|
|
@@ -601,10 +656,16 @@ const findingsOf = (facts: {
|
|
|
601
656
|
row.failed === 0 ? "" : `, ${row.failed} failed`
|
|
602
657
|
}`
|
|
603
658
|
})),
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
659
|
+
// By kind of work when the trace has it (2.23), else by tool name.
|
|
660
|
+
...(facts.toolCategories.length > 0
|
|
661
|
+
? facts.toolCategories.map((row) => ({
|
|
662
|
+
ms: row.ms,
|
|
663
|
+
text: `coder tools, ${row.label}: ${plural(row.count, "call")}, ${ofStories(row.ms)}`
|
|
664
|
+
}))
|
|
665
|
+
: facts.tools.map((row) => ({
|
|
666
|
+
ms: row.ms,
|
|
667
|
+
text: `coder tool \`${row.label}\`: ${plural(row.count, "call")}, ${ofStories(row.ms)}`
|
|
668
|
+
}))),
|
|
608
669
|
...facts.waits.map((row) => ({
|
|
609
670
|
ms: row.ms,
|
|
610
671
|
text: `waiting for the ${row.label}: ${row.count}×, ${ofStories(row.ms)}`
|
|
@@ -708,6 +769,8 @@ export const renderProfile = (report: ProfileReport): string => {
|
|
|
708
769
|
"other",
|
|
709
770
|
"turns",
|
|
710
771
|
"avg turn",
|
|
772
|
+
"steps/turn",
|
|
773
|
+
"model/step",
|
|
711
774
|
"prompt ×"
|
|
712
775
|
],
|
|
713
776
|
report.stories.map((story) => [
|
|
@@ -723,6 +786,8 @@ export const renderProfile = (report: ProfileReport): string => {
|
|
|
723
786
|
duration(story.time.unaccounted),
|
|
724
787
|
String(story.turns.count),
|
|
725
788
|
duration(story.turns.avgMs),
|
|
789
|
+
story.turns.avgSteps === 0 ? "" : story.turns.avgSteps.toFixed(1),
|
|
790
|
+
story.turns.avgSteps === 0 ? "" : duration(story.turns.avgStepMs),
|
|
726
791
|
story.turns.firstPrompt !== undefined &&
|
|
727
792
|
story.turns.lastPrompt !== undefined &&
|
|
728
793
|
story.turns.firstPrompt > 0
|
|
@@ -797,6 +862,21 @@ export const renderProfile = (report: ProfileReport): string => {
|
|
|
797
862
|
String(row.failed)
|
|
798
863
|
])
|
|
799
864
|
)
|
|
865
|
+
]),
|
|
866
|
+
...(report.toolCategories.length === 0
|
|
867
|
+
? []
|
|
868
|
+
: [
|
|
869
|
+
"",
|
|
870
|
+
"Coder tools by kind",
|
|
871
|
+
...table(
|
|
872
|
+
["kind", "calls", "total", "avg"],
|
|
873
|
+
report.toolCategories.map((row) => [
|
|
874
|
+
row.label,
|
|
875
|
+
String(row.count),
|
|
876
|
+
duration(row.ms),
|
|
877
|
+
duration(row.ms / row.count)
|
|
878
|
+
])
|
|
879
|
+
)
|
|
800
880
|
])
|
|
801
881
|
].join("\n")
|
|
802
882
|
}
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import { appendFileSync, mkdirSync, readdirSync, readFileSync } from "node:fs"
|
|
2
|
+
import { join } from "node:path"
|
|
3
|
+
import * as Effect from "effect/Effect"
|
|
4
|
+
import * as Schema from "effect/Schema"
|
|
5
|
+
import { summariseToolArgs } from "@llm4ts/flow/Activity"
|
|
6
|
+
import { TranscriptEntry, type TranscriptSink } from "@llm4ts/flow/Transcript"
|
|
7
|
+
import { duration } from "./AgentTree.ts"
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Transcripts on disk (`llm4ts run --transcript`): one JSON line per entry,
|
|
11
|
+
* one file per story (`<lane>.jsonl`) and `run.jsonl` for the rest, under
|
|
12
|
+
* `.llm4ts/transcripts/<run-id>/`. They hold the customer's code and the
|
|
13
|
+
* model's replies, so only their owner can read them, and a write that fails
|
|
14
|
+
* never fails the run.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** `LLM4TS_TRANSCRIPT=on` (what `--transcript` sets); also `1`, `true`, `yes`. */
|
|
18
|
+
export const transcriptsWanted = (
|
|
19
|
+
environment: Readonly<Record<string, string | undefined>>
|
|
20
|
+
): boolean => /^(on|1|true|yes)$/iu.test(environment.LLM4TS_TRANSCRIPT?.trim() ?? "")
|
|
21
|
+
|
|
22
|
+
export const transcriptFileOf = (lane: string | undefined): string =>
|
|
23
|
+
`${(lane ?? "run").replace(/[^A-Za-z0-9._-]/gu, "_")}.jsonl`
|
|
24
|
+
|
|
25
|
+
const encode = Schema.encodeSync(Schema.fromJsonString(TranscriptEntry))
|
|
26
|
+
|
|
27
|
+
export const nodeTranscriptSink = (directory: string): TranscriptSink => {
|
|
28
|
+
let made = false
|
|
29
|
+
return {
|
|
30
|
+
write: (lane, entry) =>
|
|
31
|
+
Effect.sync(() => {
|
|
32
|
+
try {
|
|
33
|
+
if (!made) {
|
|
34
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 })
|
|
35
|
+
made = true
|
|
36
|
+
}
|
|
37
|
+
appendFileSync(join(directory, transcriptFileOf(lane)), `${encode(entry)}\n`, {
|
|
38
|
+
mode: 0o600
|
|
39
|
+
})
|
|
40
|
+
} catch {
|
|
41
|
+
// A transcript is a window on the run, never a reason to stop it.
|
|
42
|
+
}
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// ── reading ─────────────────────────────────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
export interface TranscriptView {
|
|
50
|
+
readonly width: number
|
|
51
|
+
/** Only calls made in this role (coder, reviewer, judge, …). */
|
|
52
|
+
readonly role?: string
|
|
53
|
+
/** Only calls served by this executor. */
|
|
54
|
+
readonly executor?: string
|
|
55
|
+
/** Keep the last lines only. */
|
|
56
|
+
readonly last?: number
|
|
57
|
+
/** Where the clock in the headers starts; the first entry's time by default. */
|
|
58
|
+
readonly since?: number
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const clockOf = (ms: number): string => {
|
|
62
|
+
const seconds = Math.max(0, Math.floor(ms / 1_000))
|
|
63
|
+
return [Math.floor(seconds / 3_600), Math.floor(seconds / 60) % 60, seconds % 60]
|
|
64
|
+
.map((part) => String(part).padStart(2, "0"))
|
|
65
|
+
.join(":")
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const cut = (line: string, width: number): string =>
|
|
69
|
+
[...line].length <= width ? line : `${[...line].slice(0, Math.max(0, width - 1)).join("")}…`
|
|
70
|
+
|
|
71
|
+
/** `text` as lines of at most `width`, the first after `prefix`, the rest indented. */
|
|
72
|
+
const wrapped = (prefix: string, text: string, width: number, limit = Infinity): Array<string> => {
|
|
73
|
+
const room = Math.max(10, width - prefix.length)
|
|
74
|
+
const lines: Array<string> = []
|
|
75
|
+
for (const paragraph of text.split("\n")) {
|
|
76
|
+
let line = ""
|
|
77
|
+
for (const word of paragraph.split(/\s+/u).filter((part) => part.length > 0)) {
|
|
78
|
+
if (line.length > 0 && line.length + 1 + word.length > room) {
|
|
79
|
+
lines.push(line)
|
|
80
|
+
line = ""
|
|
81
|
+
}
|
|
82
|
+
line = line.length === 0 ? cut(word, room) : `${line} ${word}`
|
|
83
|
+
}
|
|
84
|
+
lines.push(line)
|
|
85
|
+
}
|
|
86
|
+
const kept = lines.filter((line, index) => line.length > 0 || index < lines.length - 1)
|
|
87
|
+
const shown = kept.slice(0, limit)
|
|
88
|
+
const more = kept.length - shown.length
|
|
89
|
+
return [
|
|
90
|
+
...shown.map((line, index) => `${index === 0 ? prefix : " ".repeat(prefix.length)}${line}`),
|
|
91
|
+
...(more > 0 ? [`${" ".repeat(prefix.length)}… (${more} more lines)`] : [])
|
|
92
|
+
]
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const inputLines = 12
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* A transcript as the tail pane and `watch --tail` show it: each call under
|
|
99
|
+
* a header (when, who), its input, the reply as one text, every tool with
|
|
100
|
+
* the gist of its result, and how long the call took.
|
|
101
|
+
*/
|
|
102
|
+
export const renderTranscript = (
|
|
103
|
+
entries: ReadonlyArray<TranscriptEntry>,
|
|
104
|
+
view: TranscriptView
|
|
105
|
+
): ReadonlyArray<string> => {
|
|
106
|
+
const { width } = view
|
|
107
|
+
const start = view.since ?? entries[0]?.at ?? 0
|
|
108
|
+
const wanted = new Set(
|
|
109
|
+
entries.flatMap((entry) =>
|
|
110
|
+
entry._tag === "Call" &&
|
|
111
|
+
(view.role === undefined || entry.role === view.role) &&
|
|
112
|
+
(view.executor === undefined || entry.executor === view.executor)
|
|
113
|
+
? [entry.call]
|
|
114
|
+
: []
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
const lines: Array<string> = []
|
|
118
|
+
let reply: { call: string; text: string } | undefined
|
|
119
|
+
const flushReply = () => {
|
|
120
|
+
if (reply !== undefined && reply.text.trim().length > 0) {
|
|
121
|
+
lines.push(...wrapped("◀ ", reply.text.trim(), width))
|
|
122
|
+
}
|
|
123
|
+
reply = undefined
|
|
124
|
+
}
|
|
125
|
+
// Only a filter needs the calls' headers; without one every entry shows,
|
|
126
|
+
// even one passed alone (the tail prints entries as they arrive).
|
|
127
|
+
const filtering = view.role !== undefined || view.executor !== undefined
|
|
128
|
+
for (const entry of entries) {
|
|
129
|
+
if (filtering && !wanted.has(entry.call)) {
|
|
130
|
+
continue
|
|
131
|
+
}
|
|
132
|
+
if (entry._tag !== "Reply") {
|
|
133
|
+
flushReply()
|
|
134
|
+
}
|
|
135
|
+
switch (entry._tag) {
|
|
136
|
+
case "Call": {
|
|
137
|
+
const title = `── ${clockOf(entry.at - start)} ${entry.role}${
|
|
138
|
+
entry.executor === undefined ? "" : ` · ${entry.executor}`
|
|
139
|
+
} `
|
|
140
|
+
lines.push(cut(title.padEnd(width, "─"), width))
|
|
141
|
+
if (entry.system !== undefined) {
|
|
142
|
+
const [first = "", ...more] = entry.system.split("\n")
|
|
143
|
+
lines.push(
|
|
144
|
+
cut(
|
|
145
|
+
`▶ system: ${first}${more.length === 0 ? "" : ` … (${more.length + 1} lines)`}`,
|
|
146
|
+
width
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
}
|
|
150
|
+
if (entry.earlier !== undefined && entry.earlier > 0) {
|
|
151
|
+
lines.push(`▶ (${entry.earlier} earlier messages)`)
|
|
152
|
+
}
|
|
153
|
+
lines.push(...wrapped("▶ ", entry.input, width, inputLines))
|
|
154
|
+
break
|
|
155
|
+
}
|
|
156
|
+
case "Reply":
|
|
157
|
+
reply =
|
|
158
|
+
reply?.call === entry.call
|
|
159
|
+
? { call: entry.call, text: reply.text + entry.text }
|
|
160
|
+
: { call: entry.call, text: entry.text }
|
|
161
|
+
break
|
|
162
|
+
case "Tool":
|
|
163
|
+
lines.push(cut(`⚙ ${entry.tool} ${summariseToolArgs(entry.args)}`.trimEnd(), width))
|
|
164
|
+
break
|
|
165
|
+
case "ToolResult": {
|
|
166
|
+
const outputLines = entry.output
|
|
167
|
+
.split("\n")
|
|
168
|
+
.map((line) => line.trim())
|
|
169
|
+
.filter((line) => line.length > 0)
|
|
170
|
+
const gist = outputLines.slice(0, 3).join(" · ")
|
|
171
|
+
lines.push(
|
|
172
|
+
cut(
|
|
173
|
+
` ↳ ${entry.failed === true ? "failed" : "ok"}${gist.length === 0 ? "" : `: ${gist}`}${
|
|
174
|
+
outputLines.length > 3 ? " …" : ""
|
|
175
|
+
}`,
|
|
176
|
+
width
|
|
177
|
+
)
|
|
178
|
+
)
|
|
179
|
+
break
|
|
180
|
+
}
|
|
181
|
+
case "End":
|
|
182
|
+
lines.push(` (${duration(entry.ms)}${entry.failed === true ? ", failed" : ""})`)
|
|
183
|
+
break
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
flushReply()
|
|
187
|
+
return view.last === undefined ? lines : lines.slice(-view.last)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/** How a host reaches a run's transcript files. */
|
|
191
|
+
export interface TranscriptFiles {
|
|
192
|
+
readonly read: (path: string) => Effect.Effect<string | undefined>
|
|
193
|
+
/** File names in a directory; none when it does not exist. */
|
|
194
|
+
readonly list: (directory: string) => ReadonlyArray<string>
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const decodeEntry = Schema.decodeUnknownOption(Schema.fromJsonString(TranscriptEntry))
|
|
198
|
+
|
|
199
|
+
/** A transcript file's entries; a torn last line (a write in progress) is skipped. */
|
|
200
|
+
export const parseTranscript = (text: string): ReadonlyArray<TranscriptEntry> =>
|
|
201
|
+
text
|
|
202
|
+
.split("\n")
|
|
203
|
+
.filter((line) => line.trim().length > 0)
|
|
204
|
+
.flatMap((line) => {
|
|
205
|
+
const decoded = decodeEntry(line)
|
|
206
|
+
return decoded._tag === "Some" ? [decoded.value] : []
|
|
207
|
+
})
|
|
208
|
+
|
|
209
|
+
export type TailTarget = { readonly lane: string } | { readonly executor: string }
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* The entries a tail shows: a story's file, or every file for an executor
|
|
213
|
+
* (in time order). `undefined` when the run kept no transcript at all.
|
|
214
|
+
*/
|
|
215
|
+
export const loadTranscript = (
|
|
216
|
+
files: TranscriptFiles,
|
|
217
|
+
directory: string,
|
|
218
|
+
target: TailTarget
|
|
219
|
+
): Effect.Effect<ReadonlyArray<TranscriptEntry> | undefined> =>
|
|
220
|
+
Effect.gen(function* () {
|
|
221
|
+
const names = files.list(directory).filter((name) => name.endsWith(".jsonl"))
|
|
222
|
+
if (names.length === 0) {
|
|
223
|
+
return undefined
|
|
224
|
+
}
|
|
225
|
+
const wanted = "lane" in target ? [transcriptFileOf(target.lane)] : names
|
|
226
|
+
const all: Array<TranscriptEntry> = []
|
|
227
|
+
for (const name of wanted) {
|
|
228
|
+
const text = yield* files.read(join(directory, name))
|
|
229
|
+
all.push(...parseTranscript(text ?? ""))
|
|
230
|
+
}
|
|
231
|
+
return "lane" in target ? all : [...all].sort((left, right) => left.at - right.at)
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* `rows` lines of a tail, ending `back` lines before its end (scrolled), or
|
|
236
|
+
* `undefined` when the run has no transcript.
|
|
237
|
+
*/
|
|
238
|
+
export const tailLinesOf = (
|
|
239
|
+
entries: ReadonlyArray<TranscriptEntry> | undefined,
|
|
240
|
+
target: TailTarget,
|
|
241
|
+
role: string | undefined,
|
|
242
|
+
width: number,
|
|
243
|
+
rows: number,
|
|
244
|
+
back = 0
|
|
245
|
+
): ReadonlyArray<string> | undefined => {
|
|
246
|
+
if (entries === undefined) {
|
|
247
|
+
return undefined
|
|
248
|
+
}
|
|
249
|
+
const lines = renderTranscript(entries, {
|
|
250
|
+
width,
|
|
251
|
+
...(role === undefined ? {} : { role }),
|
|
252
|
+
...("executor" in target ? { executor: target.executor } : {})
|
|
253
|
+
})
|
|
254
|
+
const end = Math.max(Math.min(rows, lines.length), lines.length - back)
|
|
255
|
+
return lines.slice(Math.max(0, end - rows), end)
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export const nodeTranscriptFiles: TranscriptFiles = {
|
|
259
|
+
read: (path) =>
|
|
260
|
+
Effect.sync(() => {
|
|
261
|
+
try {
|
|
262
|
+
return readFileSync(path, "utf8")
|
|
263
|
+
} catch {
|
|
264
|
+
return undefined
|
|
265
|
+
}
|
|
266
|
+
}),
|
|
267
|
+
list: (directory) => {
|
|
268
|
+
try {
|
|
269
|
+
return readdirSync(directory)
|
|
270
|
+
} catch {
|
|
271
|
+
return []
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
package/src/Watch.ts
CHANGED
|
@@ -18,12 +18,22 @@ import {
|
|
|
18
18
|
onTreeKey,
|
|
19
19
|
reduceTree,
|
|
20
20
|
renderTree,
|
|
21
|
+
tailTargetOf,
|
|
21
22
|
treeInputsOfTrace,
|
|
22
23
|
type TreeInput,
|
|
23
24
|
type TreeState,
|
|
24
25
|
type TreeView
|
|
25
26
|
} from "./AgentTree.ts"
|
|
26
27
|
import { enterScreen, leaveScreen, makeScreenPainter, nodeTreeKeys } from "./AgentTreeSurface.ts"
|
|
28
|
+
import {
|
|
29
|
+
loadTranscript,
|
|
30
|
+
nodeTranscriptFiles,
|
|
31
|
+
renderTranscript,
|
|
32
|
+
tailLinesOf,
|
|
33
|
+
transcriptFileOf,
|
|
34
|
+
type TailTarget,
|
|
35
|
+
type TranscriptFiles
|
|
36
|
+
} from "./Transcripts.ts"
|
|
27
37
|
import { nodeListTraces, TraceDirectoryName } from "./Costs.ts"
|
|
28
38
|
import { nodePlainFileStore } from "./NodePlainFileStore.ts"
|
|
29
39
|
|
|
@@ -45,6 +55,8 @@ export interface WatchOptions {
|
|
|
45
55
|
readonly speed?: number
|
|
46
56
|
/** A live lane with no event for this long is marked idle (`LLM4TS_IDLE_AFTER`). */
|
|
47
57
|
readonly idleAfterMs?: number
|
|
58
|
+
/** Print this story's or executor's transcript as it grows, instead of the tree. */
|
|
59
|
+
readonly tail?: string
|
|
48
60
|
}
|
|
49
61
|
|
|
50
62
|
export interface WatchOutput {
|
|
@@ -64,6 +76,8 @@ export interface WatchDependencies {
|
|
|
64
76
|
readonly output: WatchOutput
|
|
65
77
|
/** Key names as `onTreeKey` reads them: "up", "enter", "q", … */
|
|
66
78
|
readonly keys: Stream.Stream<string>
|
|
79
|
+
/** A run's transcript files (`--transcript`); the disk by default. */
|
|
80
|
+
readonly transcripts?: TranscriptFiles
|
|
67
81
|
}
|
|
68
82
|
|
|
69
83
|
export class WatchTargetMissing extends Schema.TaggedError<WatchTargetMissing>()(
|
|
@@ -184,20 +198,42 @@ const readInputs = (
|
|
|
184
198
|
const pollInterval = Duration.millis(500)
|
|
185
199
|
const replayGapCap = 2_000
|
|
186
200
|
|
|
187
|
-
const draw = (
|
|
201
|
+
const draw = (
|
|
202
|
+
state: TreeState,
|
|
203
|
+
view: TreeView,
|
|
204
|
+
output: WatchOutput,
|
|
205
|
+
height?: number,
|
|
206
|
+
tail?: ReadonlyArray<string>
|
|
207
|
+
): string =>
|
|
188
208
|
renderTree(state, {
|
|
189
209
|
width: Math.max(90, output.columns() ?? 90),
|
|
190
210
|
colour: output.colour,
|
|
191
211
|
view,
|
|
192
|
-
...(height === undefined ? {} : { height })
|
|
212
|
+
...(height === undefined ? {} : { height }),
|
|
213
|
+
...(tail === undefined ? {} : { tail })
|
|
193
214
|
}).join("\n")
|
|
194
215
|
|
|
195
216
|
/** A full-screen frame: no taller than the terminal. */
|
|
196
217
|
const screenLines = (
|
|
197
218
|
state: TreeState,
|
|
198
219
|
view: TreeView,
|
|
199
|
-
output: WatchOutput
|
|
200
|
-
|
|
220
|
+
output: WatchOutput,
|
|
221
|
+
tail?: ReadonlyArray<string>
|
|
222
|
+
): ReadonlyArray<string> => draw(state, view, output, output.rows(), tail).split("\n")
|
|
223
|
+
|
|
224
|
+
/** Rows the tail pane shows. */
|
|
225
|
+
const tailRows = 16
|
|
226
|
+
|
|
227
|
+
/** Where a trace's run kept its transcript: `.llm4ts/transcripts/<run-id>/`. */
|
|
228
|
+
const transcriptDirOf = (
|
|
229
|
+
repo: string,
|
|
230
|
+
target: WatchTarget,
|
|
231
|
+
files: PlainFileStoreShape
|
|
232
|
+
): Effect.Effect<string | undefined, FlowError> =>
|
|
233
|
+
Effect.map(readTrace(files, target.tracePath), (lines) => {
|
|
234
|
+
const runId = lines[0]?.runId
|
|
235
|
+
return runId === undefined ? undefined : join(repo, TraceDirectoryName, "transcripts", runId)
|
|
236
|
+
})
|
|
201
237
|
|
|
202
238
|
export const makeWatchProgram = Effect.fn("@llm4ts/runner/Watch.make")(function* (
|
|
203
239
|
options: WatchOptions,
|
|
@@ -210,6 +246,39 @@ export const makeWatchProgram = Effect.fn("@llm4ts/runner/Watch.make")(function*
|
|
|
210
246
|
inputs.reduce(reduceTree, start)
|
|
211
247
|
)
|
|
212
248
|
|
|
249
|
+
const transcripts = dependencies.transcripts ?? nodeTranscriptFiles
|
|
250
|
+
const transcriptDir = yield* transcriptDirOf(options.repo, target, files)
|
|
251
|
+
|
|
252
|
+
if (options.tail !== undefined) {
|
|
253
|
+
if (transcriptDir === undefined || transcripts.list(transcriptDir).length === 0) {
|
|
254
|
+
return yield* WatchTargetMissing.make({
|
|
255
|
+
message: "no transcript for this run — start it with llm4ts run … --transcript"
|
|
256
|
+
})
|
|
257
|
+
}
|
|
258
|
+
return yield* followTail(options.tail, transcriptDir, transcripts, output, () =>
|
|
259
|
+
Effect.map(readInputs(target, files), (inputs) =>
|
|
260
|
+
inputs.some((input) => input._tag === "RunEnded")
|
|
261
|
+
)
|
|
262
|
+
)
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** The open tail's lines, for the view's selection. */
|
|
266
|
+
const tailFor = (view: TreeView): Effect.Effect<ReadonlyArray<string> | undefined> => {
|
|
267
|
+
const selected = tailTargetOf(view)
|
|
268
|
+
return !view.tail || selected === undefined || transcriptDir === undefined
|
|
269
|
+
? Effect.succeed(undefined)
|
|
270
|
+
: Effect.map(loadTranscript(transcripts, transcriptDir, selected), (entries) =>
|
|
271
|
+
tailLinesOf(
|
|
272
|
+
entries,
|
|
273
|
+
selected,
|
|
274
|
+
view.tailRole,
|
|
275
|
+
Math.max(90, output.columns() ?? 90) - 32,
|
|
276
|
+
tailRows,
|
|
277
|
+
view.tailBack
|
|
278
|
+
)
|
|
279
|
+
)
|
|
280
|
+
}
|
|
281
|
+
|
|
213
282
|
if (!output.interactive) {
|
|
214
283
|
const state = yield* Effect.flatMap(readInputs(target, files), fold)
|
|
215
284
|
return yield* output.write(`${draw(state, initialView, output)}\n`)
|
|
@@ -264,7 +333,7 @@ export const makeWatchProgram = Effect.fn("@llm4ts/runner/Watch.make")(function*
|
|
|
264
333
|
? reduceTree(folded, { _tag: "Tick", at: yield* Clock.currentTimeMillis })
|
|
265
334
|
: folded
|
|
266
335
|
const text = yield* painter.paint(
|
|
267
|
-
screenLines(state, yield* Ref.get(view), output),
|
|
336
|
+
screenLines(state, yield* Ref.get(view), output, yield* tailFor(yield* Ref.get(view))),
|
|
268
337
|
`${output.columns()}x${output.rows()}`
|
|
269
338
|
)
|
|
270
339
|
if (text.length > 0) {
|
|
@@ -322,3 +391,55 @@ export const nodeWatchDependencies = (
|
|
|
322
391
|
keys: nodeTreeKeys("detach")
|
|
323
392
|
}
|
|
324
393
|
}
|
|
394
|
+
|
|
395
|
+
// ── tail ────────────────────────────────────────────────────────────────────
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* `llm4ts watch --tail <story|executor>`: the transcript printed as it grows,
|
|
399
|
+
* like `tail -f` — a call's reply streams in as written — until the run ends.
|
|
400
|
+
* A name with its own file is a story; any other is an executor.
|
|
401
|
+
*/
|
|
402
|
+
const followTail = (
|
|
403
|
+
name: string,
|
|
404
|
+
directory: string,
|
|
405
|
+
transcripts: TranscriptFiles,
|
|
406
|
+
output: WatchOutput,
|
|
407
|
+
ended: () => Effect.Effect<boolean, FlowError>
|
|
408
|
+
): Effect.Effect<void, FlowError> =>
|
|
409
|
+
Effect.gen(function* () {
|
|
410
|
+
const target: TailTarget = transcripts.list(directory).includes(transcriptFileOf(name))
|
|
411
|
+
? { lane: name }
|
|
412
|
+
: { executor: name }
|
|
413
|
+
const width = Math.max(60, output.columns() ?? 100)
|
|
414
|
+
let shown = 0
|
|
415
|
+
let start: number | undefined
|
|
416
|
+
const wanted = new Set<string>()
|
|
417
|
+
let replying: string | undefined
|
|
418
|
+
while (true) {
|
|
419
|
+
const done = yield* ended()
|
|
420
|
+
const entries = (yield* loadTranscript(transcripts, directory, target)) ?? []
|
|
421
|
+
for (const entry of entries.slice(shown)) {
|
|
422
|
+
start = start ?? entry.at
|
|
423
|
+
if (entry._tag === "Call" && ("lane" in target || entry.executor === target.executor)) {
|
|
424
|
+
wanted.add(entry.call)
|
|
425
|
+
}
|
|
426
|
+
if (!wanted.has(entry.call)) {
|
|
427
|
+
continue
|
|
428
|
+
}
|
|
429
|
+
if (entry._tag === "Reply") {
|
|
430
|
+
// A reply streams: its text as it arrived, on one growing paragraph.
|
|
431
|
+
yield* output.write(`${replying === entry.call ? "" : "◀ "}${entry.text}`)
|
|
432
|
+
replying = entry.call
|
|
433
|
+
continue
|
|
434
|
+
}
|
|
435
|
+
const lines = renderTranscript([entry], { width, since: start })
|
|
436
|
+
yield* output.write(`${replying === undefined ? "" : "\n"}${lines.join("\n")}\n`)
|
|
437
|
+
replying = undefined
|
|
438
|
+
}
|
|
439
|
+
shown = entries.length
|
|
440
|
+
if (done) {
|
|
441
|
+
return
|
|
442
|
+
}
|
|
443
|
+
yield* Effect.sleep(pollInterval)
|
|
444
|
+
}
|
|
445
|
+
})
|