@stigmer/runner 3.19.1-dev.20260920100552 → 3.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/__test-utils__/execution-record-fixture.d.ts +7 -0
- package/dist/__test-utils__/execution-record-fixture.js +10 -1
- package/dist/__test-utils__/execution-record-fixture.js.map +1 -1
- package/dist/__test-utils__/hermetic-activity.d.ts +7 -4
- package/dist/__test-utils__/hermetic-activity.js +12 -2
- package/dist/__test-utils__/hermetic-activity.js.map +1 -1
- package/dist/__test-utils__/turn-input-fixture.js +0 -1
- package/dist/__test-utils__/turn-input-fixture.js.map +1 -1
- package/dist/activities/execute-cursor/prompt-builder.js +8 -3
- package/dist/activities/execute-cursor/prompt-builder.js.map +1 -1
- package/dist/activities/execute-cursor/turn-settle.js +4 -1
- package/dist/activities/execute-cursor/turn-settle.js.map +1 -1
- package/dist/activities/execute-deep-agent/prompt-builder.d.ts +1 -3
- package/dist/activities/execute-deep-agent/prompt-builder.js +2 -3
- package/dist/activities/execute-deep-agent/prompt-builder.js.map +1 -1
- package/dist/activities/generate-session-subject.d.ts +2 -1
- package/dist/activities/generate-session-subject.js +18 -10
- package/dist/activities/generate-session-subject.js.map +1 -1
- package/dist/harness/run-turn.d.ts +3 -2
- package/dist/harness/run-turn.js +33 -6
- package/dist/harness/run-turn.js.map +1 -1
- package/dist/harness/transcript/builder.d.ts +26 -2
- package/dist/harness/transcript/builder.js +35 -2
- package/dist/harness/transcript/builder.js.map +1 -1
- package/dist/harness/turn-context.js +3 -1
- package/dist/harness/turn-context.js.map +1 -1
- package/dist/harness/turn-timeline.d.ts +132 -0
- package/dist/harness/turn-timeline.js +213 -0
- package/dist/harness/turn-timeline.js.map +1 -0
- package/dist/shared/blueprint-resolver.d.ts +14 -4
- package/dist/shared/blueprint-resolver.js +23 -10
- package/dist/shared/blueprint-resolver.js.map +1 -1
- package/dist/shared/builtin-assistant-prompt.d.ts +30 -0
- package/dist/shared/builtin-assistant-prompt.js +36 -0
- package/dist/shared/builtin-assistant-prompt.js.map +1 -0
- package/dist/shared/cold-start-timing.d.ts +18 -2
- package/dist/shared/cold-start-timing.js +30 -3
- package/dist/shared/cold-start-timing.js.map +1 -1
- package/package.json +5 -5
- package/src/__test-utils__/execution-record-fixture.ts +17 -1
- package/src/__test-utils__/hermetic-activity.ts +19 -6
- package/src/__test-utils__/timing-lines.ts +43 -0
- package/src/__test-utils__/turn-input-fixture.ts +0 -1
- package/src/activities/__tests__/generate-session-subject.test.ts +14 -0
- package/src/activities/execute-cursor/__tests__/goldens/prompt.enhanced.no-instructions.prompt.md +142 -0
- package/src/activities/execute-cursor/__tests__/hermetic/built-in-assistant.test.ts +127 -0
- package/src/activities/execute-cursor/__tests__/hermetic/turn-phases.test.ts +136 -0
- package/src/activities/execute-cursor/__tests__/prompt-goldens.test.ts +14 -1
- package/src/activities/execute-cursor/prompt-builder.ts +8 -3
- package/src/activities/execute-cursor/turn-settle.ts +4 -1
- package/src/activities/execute-deep-agent/__tests__/goldens/system-prompt.minimal.prompt.md +2 -1
- package/src/activities/execute-deep-agent/__tests__/hermetic/built-in-assistant.test.ts +101 -0
- package/src/activities/execute-deep-agent/__tests__/hermetic/turn-phases.test.ts +118 -0
- package/src/activities/execute-deep-agent/__tests__/prompt-goldens.test.ts +1 -1
- package/src/activities/execute-deep-agent/prompt-builder.ts +3 -5
- package/src/activities/generate-session-subject.ts +20 -13
- package/src/harness/README.md +4 -1
- package/src/harness/__tests__/run-turn.test.ts +53 -0
- package/src/harness/__tests__/turn-timeline.test.ts +274 -0
- package/src/harness/run-turn.ts +36 -6
- package/src/harness/transcript/__tests__/builder.test.ts +69 -1
- package/src/harness/transcript/builder.ts +39 -2
- package/src/harness/turn-context.ts +3 -1
- package/src/harness/turn-timeline.ts +248 -0
- package/src/shared/__tests__/blueprint-resolver.test.ts +78 -8
- package/src/shared/__tests__/cold-start-timing.test.ts +53 -5
- package/src/shared/blueprint-resolver.ts +25 -12
- package/src/shared/builtin-assistant-prompt.ts +38 -0
- package/src/shared/cold-start-timing.ts +29 -3
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `TurnTimeline` (`harness/turn-timeline.ts`), driven with `TranscriptEvent`s
|
|
3
|
+
* on a scripted clock — one arm per rule the module's header lists, in the
|
|
4
|
+
* order it lists them, plus the shape of the `turn_phases` line it writes.
|
|
5
|
+
*
|
|
6
|
+
* What these arms protect: the parity comparison reads this line for both
|
|
7
|
+
* harnesses, so a rule that quietly favoured one (a tool span closed at the
|
|
8
|
+
* fold's clock instead of the observed instant; a sub-agent's token counted
|
|
9
|
+
* as the user's first visible token) would bias a published number. Every
|
|
10
|
+
* instant here is asserted exactly, which the scripted clock makes possible;
|
|
11
|
+
* the hermetic proofs under each adapter assert structure only.
|
|
12
|
+
*
|
|
13
|
+
* Fence-clean: names the runtime's timing primitive and the transcript's
|
|
14
|
+
* union, nothing of any engine.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
18
|
+
import { TURN_PHASES_EVENT, TurnTimeline, type TurnTimelineClock } from "../turn-timeline.js";
|
|
19
|
+
import type { TranscriptEvent } from "../transcript/events.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A clock the test advances by hand. `performance.now()` scale starts at the
|
|
23
|
+
* origin 10_000; the wall clock at a fixed epoch, so an `observedAt` stamp can
|
|
24
|
+
* be written as "origin + N ms" and land exactly.
|
|
25
|
+
*/
|
|
26
|
+
const ORIGIN_MS = 10_000;
|
|
27
|
+
const ORIGIN_WALL_MS = Date.UTC(2026, 8, 18, 12, 0, 0, 0);
|
|
28
|
+
|
|
29
|
+
class ScriptedClock implements TurnTimelineClock {
|
|
30
|
+
readonly originMs = ORIGIN_MS;
|
|
31
|
+
readonly originWallMs = ORIGIN_WALL_MS;
|
|
32
|
+
private nowMs = ORIGIN_MS;
|
|
33
|
+
|
|
34
|
+
readonly now = (): number => this.nowMs;
|
|
35
|
+
|
|
36
|
+
/** Move the clock to `offset` ms after the origin. */
|
|
37
|
+
at(offset: number): void {
|
|
38
|
+
this.nowMs = ORIGIN_MS + offset;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The ISO stamp a harness would put on a fact it observed `offset` ms after the origin. */
|
|
43
|
+
function observedWall(offset: number): string {
|
|
44
|
+
return new Date(ORIGIN_WALL_MS + offset).toISOString();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface EmittedLine {
|
|
48
|
+
readonly stigmer_timing: string;
|
|
49
|
+
readonly total_ms: number;
|
|
50
|
+
readonly segments: Array<{ name: string; start_ms: number; duration_ms: number }>;
|
|
51
|
+
readonly [key: string]: unknown;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Drive `steps` (an offset and the event to observe at it), emit, and parse the one line. */
|
|
55
|
+
function run(steps: Array<[number, TranscriptEvent]>, settleAt: number, context: Record<string, string | number> = {}): EmittedLine {
|
|
56
|
+
const clock = new ScriptedClock();
|
|
57
|
+
const timeline = new TurnTimeline(clock);
|
|
58
|
+
for (const [offset, event] of steps) {
|
|
59
|
+
clock.at(offset);
|
|
60
|
+
timeline.observe(event);
|
|
61
|
+
}
|
|
62
|
+
clock.at(settleAt);
|
|
63
|
+
const spy = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
64
|
+
timeline.emit(context);
|
|
65
|
+
expect(spy).toHaveBeenCalledTimes(1);
|
|
66
|
+
return JSON.parse(spy.mock.calls[0]![0] as string) as EmittedLine;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function segment(line: EmittedLine, name: string) {
|
|
70
|
+
const found = line.segments.filter((s) => s.name === name);
|
|
71
|
+
expect(found, `one segment named ${name}`).toHaveLength(1);
|
|
72
|
+
return found[0]!;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const START: TranscriptEvent = { kind: "message_start", runId: "r1" };
|
|
76
|
+
const TEXT: TranscriptEvent = { kind: "text_delta", runId: "r1", text: "hi" };
|
|
77
|
+
const THINK: TranscriptEvent = { kind: "reasoning_delta", runId: "r1", text: "hmm" };
|
|
78
|
+
const FINISH: TranscriptEvent = { kind: "message_finish", runId: "r1" };
|
|
79
|
+
const TOOL_START: TranscriptEvent = { kind: "tool_started", callId: "c1", name: "read_file", input: { path: "a" }, mcpServerSlug: "" };
|
|
80
|
+
const TOOL_DONE: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok" };
|
|
81
|
+
|
|
82
|
+
afterEach(() => {
|
|
83
|
+
vi.restoreAllMocks();
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
describe("TurnTimeline — the three root instants", () => {
|
|
87
|
+
it("first_event is the first event of any kind; visible and text follow the first text delta", () => {
|
|
88
|
+
const line = run([[100, START], [250, TEXT], [400, FINISH]], 500);
|
|
89
|
+
expect(line.first_event_ms).toBe(100);
|
|
90
|
+
expect(line.first_visible_token_ms).toBe(250);
|
|
91
|
+
expect(line.first_text_ms).toBe(250);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("a reasoning delta is visible but not text: thinking on moves the two instants apart", () => {
|
|
95
|
+
const line = run([[100, START], [180, THINK], [420, TEXT]], 500);
|
|
96
|
+
expect(line.first_visible_token_ms).toBe(180);
|
|
97
|
+
expect(line.first_text_ms).toBe(420);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("a sub-agent's tokens set no root instant: the user watches the thread, not the sub-agent's row", () => {
|
|
101
|
+
const sub: TranscriptEvent[] = [
|
|
102
|
+
{ kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "look", input: "look" },
|
|
103
|
+
{ kind: "message_start", runId: "sr", subAgentId: "s1" },
|
|
104
|
+
{ kind: "text_delta", runId: "sr", text: "reading", subAgentId: "s1" },
|
|
105
|
+
];
|
|
106
|
+
const line = run([[50, sub[0]!], [60, sub[1]!], [70, sub[2]!], [900, TEXT]], 1000);
|
|
107
|
+
expect(line.first_event_ms).toBe(50);
|
|
108
|
+
expect(line.first_visible_token_ms).toBe(900);
|
|
109
|
+
expect(line.first_text_ms).toBe(900);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("an instant that never happened is null, and an empty turn has zero of everything", () => {
|
|
113
|
+
const line = run([], 300);
|
|
114
|
+
expect(line.first_event_ms).toBeNull();
|
|
115
|
+
expect(line.first_visible_token_ms).toBeNull();
|
|
116
|
+
expect(line.first_text_ms).toBeNull();
|
|
117
|
+
expect(line.rounds).toBe(0);
|
|
118
|
+
expect(line.tool_calls).toBe(0);
|
|
119
|
+
expect(line.sub_agents).toBe(0);
|
|
120
|
+
expect(line.max_gap_ms).toBeNull();
|
|
121
|
+
expect(line.segments).toEqual([]);
|
|
122
|
+
expect(line.total_ms).toBe(0);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
describe("TurnTimeline — one model_round span per runId", () => {
|
|
127
|
+
it("opens on message_start and closes on message_finish; two runs are two rounds", () => {
|
|
128
|
+
const line = run([
|
|
129
|
+
[100, START], [300, FINISH],
|
|
130
|
+
[350, { kind: "message_start", runId: "r2" }], [700, { kind: "message_finish", runId: "r2" }],
|
|
131
|
+
], 800);
|
|
132
|
+
expect(line.rounds).toBe(2);
|
|
133
|
+
expect(line.segments.filter((s) => s.name === "model_round").map((s) => [s.start_ms, s.duration_ms])).toEqual([[100, 200], [350, 350]]);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("a re-emitted start for a known run keeps the first start", () => {
|
|
137
|
+
const line = run([[100, START], [200, START], [300, FINISH]], 400);
|
|
138
|
+
expect(segment(line, "model_round")).toEqual({ name: "model_round", start_ms: 100, duration_ms: 200 });
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
describe("TurnTimeline — one tool span per callId", () => {
|
|
143
|
+
it("opens on tool_started and closes on the first tool_finished; the name carries the tool", () => {
|
|
144
|
+
const line = run([[100, TOOL_START], [340, TOOL_DONE]], 400);
|
|
145
|
+
expect(line.tool_calls).toBe(1);
|
|
146
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 100, duration_ms: 240 });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("an MCP tool's span names its server, so the MCP phase is legible", () => {
|
|
150
|
+
const start: TranscriptEvent = { kind: "tool_started", callId: "m1", name: "search", input: {}, mcpServerSlug: "github" };
|
|
151
|
+
const line = run([[100, start], [500, { kind: "tool_finished", callId: "m1", result: "" }]], 600);
|
|
152
|
+
expect(segment(line, "tool:github/search").duration_ms).toBe(400);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("a tool_error closes the span like a finish", () => {
|
|
156
|
+
const line = run([[100, TOOL_START], [150, { kind: "tool_error", callId: "c1", message: "boom" }]], 200);
|
|
157
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(50);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("a fact carrying observedAt closes the span at the observed instant, not the fold's", () => {
|
|
161
|
+
// The Cursor shape: the tool finished at +300 on the delta channel, the
|
|
162
|
+
// fold applies it at +2_800, after the model narrated for 2.5 seconds.
|
|
163
|
+
const observed: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok", observedAt: observedWall(300) };
|
|
164
|
+
const line = run([[100, TOOL_START], [2_800, observed]], 3_000);
|
|
165
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 100, duration_ms: 200 });
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("an unparseable observedAt falls back to the fold's clock rather than losing the close", () => {
|
|
169
|
+
const observed: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok", observedAt: "not a date" };
|
|
170
|
+
const line = run([[100, TOOL_START], [900, observed]], 1_000);
|
|
171
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(800);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("a second finish for a settled span changes nothing (Cursor completes every call twice)", () => {
|
|
175
|
+
const line = run([[100, TOOL_START], [300, TOOL_DONE], [900, TOOL_DONE]], 1_000);
|
|
176
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(200);
|
|
177
|
+
expect(line.tool_calls).toBe(1);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it("a finish for a call that never started is ignored", () => {
|
|
181
|
+
const line = run([[300, TOOL_DONE]], 400);
|
|
182
|
+
expect(line.tool_calls).toBe(0);
|
|
183
|
+
expect(line.segments).toEqual([]);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("approval_proposed opens no span: a held call is not a tool phase", () => {
|
|
187
|
+
const proposed: TranscriptEvent = { kind: "approval_proposed", callId: "g1", name: "shell", mcpServerSlug: "", message: "Run?" };
|
|
188
|
+
const line = run([[100, proposed]], 200);
|
|
189
|
+
expect(line.tool_calls).toBe(0);
|
|
190
|
+
expect(line.first_event_ms, "it is still an event").toBe(100);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it("arg and output deltas are events for the gap but open nothing", () => {
|
|
194
|
+
const line = run([
|
|
195
|
+
[100, TOOL_START],
|
|
196
|
+
[150, { kind: "tool_arg_delta", callId: "c1", argsChunk: "{" }],
|
|
197
|
+
[200, { kind: "tool_output_delta", callId: "c1", delta: "partial" }],
|
|
198
|
+
[400, TOOL_DONE],
|
|
199
|
+
], 500);
|
|
200
|
+
expect(line.tool_calls).toBe(1);
|
|
201
|
+
expect(line.segments).toHaveLength(1);
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
describe("TurnTimeline — one sub_agent span per subAgentId", () => {
|
|
206
|
+
it("opens on sub_agent_started, closes on finished or failed, named after the sub-agent", () => {
|
|
207
|
+
const line = run([
|
|
208
|
+
[100, { kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "x", input: "x" }],
|
|
209
|
+
[900, { kind: "sub_agent_finished", subAgentId: "s1", output: "done" }],
|
|
210
|
+
[950, { kind: "sub_agent_started", subAgentId: "s2", name: "writer", subject: "y", input: "y" }],
|
|
211
|
+
[1_000, { kind: "sub_agent_failed", subAgentId: "s2", error: "no" }],
|
|
212
|
+
], 1_100);
|
|
213
|
+
expect(line.sub_agents).toBe(2);
|
|
214
|
+
expect(segment(line, "sub_agent:researcher").duration_ms).toBe(800);
|
|
215
|
+
expect(segment(line, "sub_agent:writer").duration_ms).toBe(50);
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
describe("TurnTimeline — the longest silence", () => {
|
|
220
|
+
it("max_gap is the largest wait between two consecutive events, of any kind", () => {
|
|
221
|
+
const line = run([[100, START], [150, TEXT], [1_350, TOOL_START], [1_400, TOOL_DONE]], 1_500);
|
|
222
|
+
expect(line.max_gap_ms).toBe(1_200);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it("is null under two events", () => {
|
|
226
|
+
expect(run([[100, START]], 200).max_gap_ms).toBeNull();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
describe("TurnTimeline — settle and the line's shape", () => {
|
|
231
|
+
it("a span still open at settle closes at settle", () => {
|
|
232
|
+
const line = run([[100, START], [200, TOOL_START]], 1_000);
|
|
233
|
+
expect(segment(line, "model_round")).toEqual({ name: "model_round", start_ms: 100, duration_ms: 900 });
|
|
234
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 200, duration_ms: 800 });
|
|
235
|
+
expect(line.total_ms).toBe(1_000);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it("segments overlap and come out in start order; total is the latest end, not a sum", () => {
|
|
239
|
+
const line = run([
|
|
240
|
+
[100, START],
|
|
241
|
+
[200, TOOL_START],
|
|
242
|
+
[300, { kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "x", input: "x" }],
|
|
243
|
+
[400, TOOL_DONE],
|
|
244
|
+
[500, FINISH],
|
|
245
|
+
[600, { kind: "message_start", runId: "r2" }],
|
|
246
|
+
[700, { kind: "sub_agent_finished", subAgentId: "s1" }],
|
|
247
|
+
[800, { kind: "message_finish", runId: "r2" }],
|
|
248
|
+
], 850);
|
|
249
|
+
expect(line.segments.map((s) => s.name)).toEqual(["model_round", "tool:read_file", "sub_agent:researcher", "model_round"]);
|
|
250
|
+
const sum = line.segments.reduce((acc, s) => acc + s.duration_ms, 0);
|
|
251
|
+
expect(sum).toBeGreaterThan(line.total_ms);
|
|
252
|
+
expect(line.total_ms).toBe(800);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it("writes the turn_phases selector and the runtime's context beside the fold's facts", () => {
|
|
256
|
+
const line = run([[100, START], [200, TEXT], [300, FINISH]], 400, {
|
|
257
|
+
execution_id: "exe_1",
|
|
258
|
+
session_id: "ses_1",
|
|
259
|
+
turn_seq: 2,
|
|
260
|
+
harness: "deep-agent",
|
|
261
|
+
outcome: "completed",
|
|
262
|
+
});
|
|
263
|
+
expect(line.stigmer_timing).toBe(TURN_PHASES_EVENT);
|
|
264
|
+
expect(line.execution_id).toBe("exe_1");
|
|
265
|
+
expect(line.session_id).toBe("ses_1");
|
|
266
|
+
expect(line.turn_seq).toBe(2);
|
|
267
|
+
expect(line.harness).toBe("deep-agent");
|
|
268
|
+
expect(line.outcome).toBe("completed");
|
|
269
|
+
expect(Object.keys(line).sort()).toEqual([
|
|
270
|
+
"execution_id", "first_event_ms", "first_text_ms", "first_visible_token_ms", "harness", "max_gap_ms",
|
|
271
|
+
"outcome", "rounds", "segments", "session_id", "stigmer_timing", "sub_agents", "tool_calls", "total_ms", "turn_seq",
|
|
272
|
+
]);
|
|
273
|
+
});
|
|
274
|
+
});
|
package/src/harness/run-turn.ts
CHANGED
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* What runs here and nowhere else: the activity's identity and its
|
|
6
6
|
* execution-scoped context; the status, the one transcript builder over it
|
|
7
7
|
* (born with the status, seeded by the reinvocation phase, handed to the
|
|
8
|
-
* adapter as `sink.transcript`, finalized once the adapter returns)
|
|
9
|
-
*
|
|
8
|
+
* adapter as `sink.transcript`, finalized once the adapter returns), the turn
|
|
9
|
+
* timeline folded from that builder's events and written once at settle
|
|
10
|
+
* (`turn-timeline.ts`), and the single persist chokepoint;
|
|
10
11
|
* the periodic heartbeat; the stop controller and its four producers (the
|
|
11
12
|
* stall watchdog, the cost cap, the platform STOP, Temporal's cancellation);
|
|
12
13
|
* the resolution phases (`turn-context.ts`) composed into the `TurnInput`;
|
|
@@ -85,6 +86,7 @@ import {
|
|
|
85
86
|
} from "./terminal-table.js";
|
|
86
87
|
import { TranscriptBuilder } from "./transcript/builder.js";
|
|
87
88
|
import { resolveTurnContext, type ResolutionDeps, type TurnFrame, type TurnSettlement } from "./turn-context.js";
|
|
89
|
+
import { TurnTimeline } from "./turn-timeline.js";
|
|
88
90
|
import type { HarnessAdapter, TurnInput, TurnSink } from "./types.js";
|
|
89
91
|
import { UsageAccumulator } from "./usage-accumulator.js";
|
|
90
92
|
|
|
@@ -151,20 +153,34 @@ async function runTurn(deps: TurnRuntimeDeps, input: NormalizedActivityInput): P
|
|
|
151
153
|
startedAt: utcTimestamp(),
|
|
152
154
|
});
|
|
153
155
|
|
|
156
|
+
// The turn's origin, read once on both clocks so the two timelines below
|
|
157
|
+
// and the harness's observed instants (`transcript/events.ts` `Observed`)
|
|
158
|
+
// all subtract against the same moment.
|
|
159
|
+
const originMs = performance.now();
|
|
160
|
+
const originWallMs = Date.now();
|
|
161
|
+
|
|
162
|
+
// The turn timeline (`turn-timeline.ts`): the runtime's fold over every
|
|
163
|
+
// event the adapter folds through the builder, emitted once at settle as
|
|
164
|
+
// the `turn_phases` line — first visible token, model rounds, tool and
|
|
165
|
+
// sub-agent spans, the longest silence — for every harness alike.
|
|
166
|
+
const timeline = new TurnTimeline({ originMs, originWallMs, now: () => performance.now() });
|
|
167
|
+
|
|
154
168
|
// The one transcript builder of this turn, born with the status and before
|
|
155
169
|
// any phase, so it exists for everything that writes a row or an output
|
|
156
170
|
// onto the status: the write-back coordinator (phase 2c), the seed of a
|
|
157
171
|
// reinvocation's prior rows (phase 3, through `seed()`, which indexes them
|
|
158
172
|
// as it appends), the adapter (`sink.transcript`), and the epilogue. One
|
|
159
173
|
// constructor call per turn; a second builder over the same status would
|
|
160
|
-
// index the same rows twice.
|
|
161
|
-
const transcript = new TranscriptBuilder(executionId, status);
|
|
174
|
+
// index the same rows twice. The timeline is its observer.
|
|
175
|
+
const transcript = new TranscriptBuilder(executionId, status, (event) => timeline.observe(event));
|
|
162
176
|
|
|
163
177
|
// Cold-start timeline of this turn's setup: one mark after each phase, the
|
|
164
178
|
// adapter's own segments marked through the sink, emitted by the adapter
|
|
165
179
|
// once its engine is resolved (an early return skips it — partial setups
|
|
166
|
-
// are not comparable cold-start samples).
|
|
167
|
-
|
|
180
|
+
// are not comparable cold-start samples). On the turn's origin, so
|
|
181
|
+
// `turn_phases.first_visible_token_ms - execution_setup.total_ms` is the
|
|
182
|
+
// wait after the engine was ready, with no join.
|
|
183
|
+
const setupTiming = new TimingRecorder(originMs);
|
|
168
184
|
|
|
169
185
|
// Artifact storage for offloading oversized tool outputs out of the
|
|
170
186
|
// persisted status and for publishing the plan artifact. Resolved once so
|
|
@@ -366,6 +382,20 @@ async function runTurn(deps: TurnRuntimeDeps, input: NormalizedActivityInput): P
|
|
|
366
382
|
// finalizes first, as the Cursor harness does before `run.wait()`.
|
|
367
383
|
transcript.finalize();
|
|
368
384
|
|
|
385
|
+
// The engine's turn is over: write its timeline, once, on every outcome.
|
|
386
|
+
// Here and not in the settles, because this is the one place that sees
|
|
387
|
+
// every path, and before the epilogue, because the epilogue is the
|
|
388
|
+
// runtime's, not the turn the user watched. A turn that settled during
|
|
389
|
+
// resolution never reaches here and gets no line: no engine, no phases.
|
|
390
|
+
// `harness` is the adapter's diagnostic name, the runtime's vocabulary.
|
|
391
|
+
timeline.emit({
|
|
392
|
+
execution_id: executionId,
|
|
393
|
+
session_id: turn.sessionId,
|
|
394
|
+
turn_seq: input.turnSeq,
|
|
395
|
+
harness: adapter.name,
|
|
396
|
+
outcome: outcome.kind,
|
|
397
|
+
});
|
|
398
|
+
|
|
369
399
|
// The capture boundary, ONCE, over whatever the whole turn left on the
|
|
370
400
|
// tree — unless a stop fired mid-turn (`interrupted`: the tree may be
|
|
371
401
|
// mid-edit and the pause or drain reinvokes this same turn). Before the
|
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
* preview, the todo projection, and the rest of the header's rules —
|
|
18
18
|
* the row per id, the message boundary, the thinking row, the sub-agent
|
|
19
19
|
* rows, finalize, artifacts and write-backs, the dirty flag, the error
|
|
20
|
-
* guard — joined commit by commit in the same PR.
|
|
20
|
+
* guard — joined commit by commit in the same PR. The observer arm (the
|
|
21
|
+
* runtime's fold-watcher: told after the fold, guarded apart from the
|
|
22
|
+
* handlers, silent on seed) joined with the turn timeline.
|
|
21
23
|
*/
|
|
22
24
|
|
|
23
25
|
import { describe, it, expect } from "vitest";
|
|
@@ -973,3 +975,69 @@ describe("TranscriptBuilder — the dirty flag and the error guard", () => {
|
|
|
973
975
|
expect(errors[0]).toContain("property access failed");
|
|
974
976
|
});
|
|
975
977
|
});
|
|
978
|
+
|
|
979
|
+
describe("TranscriptBuilder — the observer sees every folded event, after the fold, and can break nothing", () => {
|
|
980
|
+
it("is told every applied event, in order, once the fold has landed", () => {
|
|
981
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
982
|
+
const seen: Array<{ kind: string; rowsAtNotify: number }> = [];
|
|
983
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => {
|
|
984
|
+
seen.push({ kind: event.kind, rowsAtNotify: status.messages.flatMap((m) => m.toolCalls).length });
|
|
985
|
+
});
|
|
986
|
+
for (const e of proposedCall()) sb.apply(e);
|
|
987
|
+
sb.apply({ kind: "tool_finished", callId: "call-1", result: "ok" });
|
|
988
|
+
|
|
989
|
+
expect(seen.map((s) => s.kind)).toEqual(["message_start", "text_delta", "message_finish", "tool_started", "tool_finished"]);
|
|
990
|
+
// "After the fold": when the observer hears tool_started, the row exists.
|
|
991
|
+
expect(seen[3]!.rowsAtNotify).toBe(1);
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
it("a throwing observer is logged with the execution and the event kind, and the fold stands", () => {
|
|
995
|
+
const errors: string[] = [];
|
|
996
|
+
const original = console.error;
|
|
997
|
+
console.error = (msg: unknown) => { errors.push(String(msg)); };
|
|
998
|
+
try {
|
|
999
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
1000
|
+
const sb = new TranscriptBuilder("exec-observe", status, () => { throw new Error("observer fell over"); });
|
|
1001
|
+
expect(() => { for (const e of proposedCall()) sb.apply(e); }).not.toThrow();
|
|
1002
|
+
expect(rowOf(status, "call-1").status, "the transcript is whole").toBe(ToolCallStatus.TOOL_CALL_RUNNING);
|
|
1003
|
+
} finally {
|
|
1004
|
+
console.error = original;
|
|
1005
|
+
}
|
|
1006
|
+
expect(errors).toHaveLength(4);
|
|
1007
|
+
expect(errors[0]).toContain("exec-observe");
|
|
1008
|
+
expect(errors[0]).toContain("kind=message_start");
|
|
1009
|
+
expect(errors[0]).toContain("observer fell over");
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
it("still hears an event whose handler threw: the two guards are independent", () => {
|
|
1013
|
+
const original = console.error;
|
|
1014
|
+
console.error = () => {};
|
|
1015
|
+
const seen: string[] = [];
|
|
1016
|
+
try {
|
|
1017
|
+
const { status } = feed(builder(), proposedCall());
|
|
1018
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => { seen.push(event.kind); });
|
|
1019
|
+
const poisoned = { kind: "tool_finished", callId: "call-1", get result(): string { throw new Error("property access failed"); } } as TranscriptEvent;
|
|
1020
|
+
sb.apply(poisoned);
|
|
1021
|
+
} finally {
|
|
1022
|
+
console.error = original;
|
|
1023
|
+
}
|
|
1024
|
+
expect(seen).toEqual(["tool_finished"]);
|
|
1025
|
+
});
|
|
1026
|
+
|
|
1027
|
+
it("seed() does not notify: a seeded row is last turn's fact, not an event of this turn", () => {
|
|
1028
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
1029
|
+
const seen: string[] = [];
|
|
1030
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => { seen.push(event.kind); });
|
|
1031
|
+
sb.seed(create(AgentExecutionStatusSchema, {
|
|
1032
|
+
messages: [create(AgentMessageSchema, { type: MessageType.MESSAGE_AI, content: "earlier", toolCalls: [create(ToolCallSchema, { id: "old-1", name: "read_file", status: ToolCallStatus.TOOL_CALL_COMPLETED })] })],
|
|
1033
|
+
}));
|
|
1034
|
+
expect(seen).toEqual([]);
|
|
1035
|
+
expect(status.messages).toHaveLength(1);
|
|
1036
|
+
});
|
|
1037
|
+
|
|
1038
|
+
it("is optional: the two-argument constructor folds exactly as before", () => {
|
|
1039
|
+
const { sb, status } = feed(builder(), proposedCall());
|
|
1040
|
+
sb.apply({ kind: "tool_finished", callId: "call-1", result: "ok" });
|
|
1041
|
+
expect(rowOf(status, "call-1").status).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
|
|
1042
|
+
});
|
|
1043
|
+
});
|
|
@@ -82,6 +82,17 @@
|
|
|
82
82
|
* token delta does; the adapter's loop reads it and clears it with
|
|
83
83
|
* {@link markPersisted} as it requests the persist.
|
|
84
84
|
*
|
|
85
|
+
* The runtime may hand the constructor an OBSERVER of the fold: a callback
|
|
86
|
+
* that sees every event `apply` folded, in order, after the fold, and creates
|
|
87
|
+
* nothing (the runtime's turn timeline, `harness/turn-timeline.ts`, is the
|
|
88
|
+
* one observer today). A callback at construction, not a subscribe method,
|
|
89
|
+
* because that is how the runtime tells its collaborators what to do
|
|
90
|
+
* everywhere else (`PersistChokepoint`'s options, the stall watchdog's
|
|
91
|
+
* `onStall`): one observer, fixed at the one constructor call, nothing to
|
|
92
|
+
* add or remove mid-turn. The observer runs in its own try/catch, so it can
|
|
93
|
+
* neither block a fold nor break a harness; `seed()` does not notify it,
|
|
94
|
+
* because a seeded row is last turn's fact.
|
|
95
|
+
*
|
|
85
96
|
* What this builder deliberately does NOT write, and who does (the adapter
|
|
86
97
|
* contract's field ownership, `harness/types.ts` `TurnSink`): the phase,
|
|
87
98
|
* `startedAt`, `streamingUsage` and the epilogue's plan artifact are the
|
|
@@ -127,9 +138,13 @@ import type {
|
|
|
127
138
|
TranscriptEvent,
|
|
128
139
|
} from "./events.js";
|
|
129
140
|
|
|
141
|
+
/** Sees every event the builder folded, in order, after the fold. Creates nothing. */
|
|
142
|
+
export type TranscriptObserver = (event: TranscriptEvent) => void;
|
|
143
|
+
|
|
130
144
|
export class TranscriptBuilder {
|
|
131
145
|
readonly executionId: string;
|
|
132
146
|
private readonly state: TranscriptState;
|
|
147
|
+
private readonly observer: TranscriptObserver | undefined;
|
|
133
148
|
private _dirty = false;
|
|
134
149
|
private _awaitingApproval = false;
|
|
135
150
|
|
|
@@ -146,10 +161,15 @@ export class TranscriptBuilder {
|
|
|
146
161
|
* builder over — `TurnSink.status` in production, the test's own in a
|
|
147
162
|
* test — so the transcript has one read handle and one write handle, and
|
|
148
163
|
* no second spelling of either.
|
|
164
|
+
*
|
|
165
|
+
* `observer`, when given, is told every event after it is folded (the
|
|
166
|
+
* header's rule). The runtime passes its turn timeline; the contract kit
|
|
167
|
+
* and the test doubles pass nothing.
|
|
149
168
|
*/
|
|
150
|
-
constructor(executionId: string, status: AgentExecutionStatus) {
|
|
169
|
+
constructor(executionId: string, status: AgentExecutionStatus, observer?: TranscriptObserver) {
|
|
151
170
|
this.executionId = executionId;
|
|
152
171
|
this.state = new TranscriptState(status);
|
|
172
|
+
this.observer = observer;
|
|
153
173
|
}
|
|
154
174
|
|
|
155
175
|
/**
|
|
@@ -194,8 +214,25 @@ export class TranscriptBuilder {
|
|
|
194
214
|
this._dirty = false;
|
|
195
215
|
}
|
|
196
216
|
|
|
197
|
-
/**
|
|
217
|
+
/**
|
|
218
|
+
* Fold one canonical event into the transcript, then tell the observer.
|
|
219
|
+
* Never throws: a handler's error is logged and the stream goes on, and so
|
|
220
|
+
* is an observer's — each in its own catch, so a failing observer still
|
|
221
|
+
* sees a folded transcript and a failing handler still reaches the observer.
|
|
222
|
+
*/
|
|
198
223
|
apply(event: TranscriptEvent): void {
|
|
224
|
+
this.fold(event);
|
|
225
|
+
if (!this.observer) return;
|
|
226
|
+
try {
|
|
227
|
+
this.observer(event);
|
|
228
|
+
} catch (err) {
|
|
229
|
+
console.error(
|
|
230
|
+
`[TranscriptBuilder] Observer error: execution=${this.executionId} kind=${event.kind}: ${err}`,
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private fold(event: TranscriptEvent): void {
|
|
199
236
|
try {
|
|
200
237
|
switch (event.kind) {
|
|
201
238
|
case "sub_agent_started":
|
|
@@ -759,7 +759,9 @@ export async function resolveMcpServersAndPolicies(
|
|
|
759
759
|
execution.spec?.recalledMemories,
|
|
760
760
|
{
|
|
761
761
|
org: session.metadata?.org ?? "",
|
|
762
|
-
|
|
762
|
+
// Provenance, not scope: empty for the built-in assistant, and the
|
|
763
|
+
// carrier omits an empty field (memory-attachment.ts).
|
|
764
|
+
agentId: blueprint.agent?.metadata?.id ?? "",
|
|
763
765
|
sessionId,
|
|
764
766
|
agentExecutionId: executionId,
|
|
765
767
|
},
|