@stigmer/runner 3.19.1-dev.20260920100552 → 3.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/activities/execute-cursor/turn-settle.js +4 -1
- package/dist/activities/execute-cursor/turn-settle.js.map +1 -1
- package/dist/harness/run-turn.d.ts +3 -2
- package/dist/harness/run-turn.js +33 -6
- package/dist/harness/run-turn.js.map +1 -1
- package/dist/harness/transcript/builder.d.ts +26 -2
- package/dist/harness/transcript/builder.js +35 -2
- package/dist/harness/transcript/builder.js.map +1 -1
- package/dist/harness/turn-timeline.d.ts +132 -0
- package/dist/harness/turn-timeline.js +213 -0
- package/dist/harness/turn-timeline.js.map +1 -0
- package/dist/shared/cold-start-timing.d.ts +18 -2
- package/dist/shared/cold-start-timing.js +30 -3
- package/dist/shared/cold-start-timing.js.map +1 -1
- package/package.json +5 -5
- package/src/__test-utils__/timing-lines.ts +43 -0
- package/src/activities/execute-cursor/__tests__/hermetic/turn-phases.test.ts +136 -0
- package/src/activities/execute-cursor/turn-settle.ts +4 -1
- package/src/activities/execute-deep-agent/__tests__/hermetic/turn-phases.test.ts +118 -0
- package/src/harness/README.md +4 -1
- package/src/harness/__tests__/run-turn.test.ts +53 -0
- package/src/harness/__tests__/turn-timeline.test.ts +274 -0
- package/src/harness/run-turn.ts +36 -6
- package/src/harness/transcript/__tests__/builder.test.ts +69 -1
- package/src/harness/transcript/builder.ts +39 -2
- package/src/harness/turn-timeline.ts +248 -0
- package/src/shared/__tests__/cold-start-timing.test.ts +53 -5
- package/src/shared/cold-start-timing.ts +29 -3
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hermetic proof: the native harness gets the runtime's `turn_phases` timing
|
|
3
|
+
* line without a line of its own.
|
|
4
|
+
*
|
|
5
|
+
* Invariant pinned: driving the REAL deepagents graph through the real native
|
|
6
|
+
* translator, the turn runtime writes exactly one `turn_phases` line for the
|
|
7
|
+
* turn, and the line reads what the translator's events said — one model
|
|
8
|
+
* round per LangGraph run (two here: the proposal and the closing text), one
|
|
9
|
+
* tool span for the `read_file` call, the three root instants present, the
|
|
10
|
+
* adapter's own name as `harness`. The fold and its rules are
|
|
11
|
+
* `harness/__tests__/turn-timeline.test.ts`'s; the runtime's act of writing
|
|
12
|
+
* the line is `harness/__tests__/run-turn.test.ts`'s against the fake. What
|
|
13
|
+
* only this file can prove is that the native translator emits the events the
|
|
14
|
+
* fold needs (`message_start`/`message_finish` per run, `tool_started`/
|
|
15
|
+
* `tool_finished` per call) in the order it needs them, on a real graph.
|
|
16
|
+
*
|
|
17
|
+
* Structural assertions only: the scripted clock makes every number
|
|
18
|
+
* deterministic, but pinning one would pin the clock's tick quantum, not a
|
|
19
|
+
* behaviour. The status goldens beside this file are untouched by design —
|
|
20
|
+
* the line is stdout, never the status — and their byte-identity under this
|
|
21
|
+
* change is proved by running them.
|
|
22
|
+
*
|
|
23
|
+
* Same scenario as `tool-call.test.ts`, so the two files read together.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
|
29
|
+
|
|
30
|
+
vi.mock("../../../../shared/model-client.js", async () =>
|
|
31
|
+
(await import("../../__test-utils__/scripted-model-module.js")).scriptedModelClientModule(),
|
|
32
|
+
);
|
|
33
|
+
vi.mock("../../../../client/stigmer-client.js", async () =>
|
|
34
|
+
(await import("../../../../__test-utils__/hermetic-activity.js")).hermeticStigmerClientModule(),
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
import {
|
|
38
|
+
ScriptedClock,
|
|
39
|
+
createHermeticEnvironment,
|
|
40
|
+
type HermeticEnvironment,
|
|
41
|
+
} from "../../../../__test-utils__/hermetic-activity.js";
|
|
42
|
+
import { stubRegistryFetch } from "../../../../__test-utils__/model-registry-fixture.js";
|
|
43
|
+
import { captureTimingLines } from "../../../../__test-utils__/timing-lines.js";
|
|
44
|
+
import { TURN_PHASES_EVENT } from "../../../../harness/turn-timeline.js";
|
|
45
|
+
import {
|
|
46
|
+
beginDeepAgentScenario,
|
|
47
|
+
deepAgentExecutionRecord,
|
|
48
|
+
runDeepAgentTurn,
|
|
49
|
+
sessionWorkspaceDir,
|
|
50
|
+
} from "../../__test-utils__/hermetic-deep-agent.js";
|
|
51
|
+
|
|
52
|
+
const CALL_ID = "call-hermetic-read-0001";
|
|
53
|
+
const USER_MESSAGE = "What is in README.md?";
|
|
54
|
+
const README = "# Hermetic\n\nA fixture readme.\n";
|
|
55
|
+
const ASSISTANT_TEXT = "README.md holds a one-line fixture description.";
|
|
56
|
+
|
|
57
|
+
describe("ExecuteDeepAgent hermetic — the runtime's turn_phases line", () => {
|
|
58
|
+
let env: HermeticEnvironment;
|
|
59
|
+
let registry: ReturnType<typeof stubRegistryFetch>;
|
|
60
|
+
const clock = new ScriptedClock();
|
|
61
|
+
|
|
62
|
+
beforeAll(() => {
|
|
63
|
+
env = createHermeticEnvironment();
|
|
64
|
+
registry = stubRegistryFetch();
|
|
65
|
+
clock.install();
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
afterAll(() => {
|
|
69
|
+
clock.uninstall();
|
|
70
|
+
registry.restore();
|
|
71
|
+
env.dispose();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it("writes one line for the turn, reading the translator's rounds and tool spans", async () => {
|
|
75
|
+
const record = deepAgentExecutionRecord({ message: USER_MESSAGE });
|
|
76
|
+
const scenario = beginDeepAgentScenario({
|
|
77
|
+
env,
|
|
78
|
+
clock,
|
|
79
|
+
record,
|
|
80
|
+
script: () => ({
|
|
81
|
+
turns: [
|
|
82
|
+
{
|
|
83
|
+
text: "Let me read it.",
|
|
84
|
+
toolCalls: [{ id: CALL_ID, name: "read_file", args: { file_path: "/README.md" } }],
|
|
85
|
+
usage: { inputTokens: 1_500, outputTokens: 60 },
|
|
86
|
+
},
|
|
87
|
+
{ text: ASSISTANT_TEXT, usage: { inputTokens: 1_700, outputTokens: 30 } },
|
|
88
|
+
],
|
|
89
|
+
}),
|
|
90
|
+
});
|
|
91
|
+
const workspace = sessionWorkspaceDir(env);
|
|
92
|
+
mkdirSync(workspace, { recursive: true });
|
|
93
|
+
writeFileSync(join(workspace, "README.md"), README, "utf-8");
|
|
94
|
+
|
|
95
|
+
const lines = await captureTimingLines(TURN_PHASES_EVENT, async () => {
|
|
96
|
+
const invocation = await runDeepAgentTurn(scenario);
|
|
97
|
+
expect(invocation.outcome.kind).toBe("returned");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
expect(lines, "one line per turn").toHaveLength(1);
|
|
101
|
+
const line = lines[0]!;
|
|
102
|
+
expect(line.harness, "the adapter's diagnostic name, the runtime's vocabulary").toBe("deep-agent");
|
|
103
|
+
expect(line.outcome).toBe("completed");
|
|
104
|
+
expect(line.execution_id).toBe(record.execution.metadata!.id);
|
|
105
|
+
expect(line.turn_seq).toBeTypeOf("number");
|
|
106
|
+
|
|
107
|
+
expect(line.rounds, "the proposing run and the closing run").toBe(2);
|
|
108
|
+
expect(line.tool_calls).toBe(1);
|
|
109
|
+
expect(line.sub_agents).toBe(0);
|
|
110
|
+
expect(line.segments.filter((s) => s.name === "model_round")).toHaveLength(2);
|
|
111
|
+
expect(line.segments.filter((s) => s.name === "tool:read_file")).toHaveLength(1);
|
|
112
|
+
|
|
113
|
+
expect(line.first_event_ms).toBeTypeOf("number");
|
|
114
|
+
expect(line.first_visible_token_ms).toBeTypeOf("number");
|
|
115
|
+
expect(line.first_text_ms).toBeTypeOf("number");
|
|
116
|
+
expect(line.total_ms).toBeGreaterThanOrEqual(line.first_text_ms as number);
|
|
117
|
+
});
|
|
118
|
+
});
|
package/src/harness/README.md
CHANGED
|
@@ -47,6 +47,10 @@ pinned somewhere. Read a row across before searching for anything.
|
|
|
47
47
|
translator emits.
|
|
48
48
|
- `harness/transcript/builder.ts` — `TranscriptBuilder`, the one builder every
|
|
49
49
|
harness folds its events through.
|
|
50
|
+
- `harness/turn-timeline.ts` — `TurnTimeline`, the runtime's fold over that
|
|
51
|
+
builder's events into the `turn_phases` timing line every harness gets for
|
|
52
|
+
free: first visible token, model rounds, tool and sub-agent spans, the
|
|
53
|
+
longest silence.
|
|
50
54
|
- `harness/capture.ts` — the file-review capture. The runtime owns all of it; an
|
|
51
55
|
adapter supplies one fact, which CAS-owned paths its engine touched.
|
|
52
56
|
- `harness/turn-context.ts` — the resolution phases that build the `TurnInput`
|
|
@@ -204,7 +208,6 @@ Every hit falls into one of three classes:
|
|
|
204
208
|
| `sdk/react/src/workflow/inspector/forms/AgentCallForm.tsx` | Two radio buttons, placed by hand. |
|
|
205
209
|
| `sdk/react/src/pricing-governance/BaselineEditor.tsx` | A select with one option per harness. |
|
|
206
210
|
| `site/src/components/pages/pricing/ModelPricingTable.tsx` | Harness to display label. |
|
|
207
|
-
| `site/src/data/harness-cost-comparison.json` | Per-harness cost rows. |
|
|
208
211
|
| `docs/concepts/harnesses.mdx`, `docs/concepts/sessions.mdx`, `docs/guides/runners/cursor-harness.mdx`, `docs/guides/workflows/task-types/agent-call.mdx`, `docs/vocabulary.md` | Hand-authored pages that name the harness set; each has an entry in `docs/_inventory/classification.yaml` to keep current. |
|
|
209
212
|
|
|
210
213
|
The cloud composition that serves api.stigmer.ai has sites of the same three
|
|
@@ -53,13 +53,16 @@ import {
|
|
|
53
53
|
assertFailedSurfaceWritesItsCopy,
|
|
54
54
|
assertNonExecutingDecisionSettlesSkipped,
|
|
55
55
|
assertRejectedBindFails,
|
|
56
|
+
assertWorkspaceLockTimeoutFails,
|
|
56
57
|
describeHarnessRuntimeContract,
|
|
57
58
|
RuntimeExecutionDriver,
|
|
58
59
|
slimOf,
|
|
59
60
|
systemMessages,
|
|
60
61
|
type RuntimeContractHarness,
|
|
61
62
|
} from "../../__test-utils__/harness-contract/runtime-contract.js";
|
|
63
|
+
import { captureTimingLines } from "../../__test-utils__/timing-lines.js";
|
|
62
64
|
import { TERMINAL_COPY } from "../terminal-table.js";
|
|
65
|
+
import { TURN_PHASES_EVENT } from "../turn-timeline.js";
|
|
63
66
|
import type { HarnessAdapter } from "../types.js";
|
|
64
67
|
|
|
65
68
|
const TASK_QUEUE = "runtime-test-queue";
|
|
@@ -219,6 +222,56 @@ describe("run-turn: the fake adapter through the real runtime", () => {
|
|
|
219
222
|
});
|
|
220
223
|
});
|
|
221
224
|
|
|
225
|
+
describe("the runtime's turn timeline", () => {
|
|
226
|
+
// `run-turn.ts` emits the `turn_phases` line once, after the adapter
|
|
227
|
+
// returns and the transcript is finalized, on every outcome; a turn that
|
|
228
|
+
// never reached the engine emits none. The line's CONTENT is
|
|
229
|
+
// `turn-timeline.test.ts`'s; what is pinned here is the runtime's act:
|
|
230
|
+
// that the fold is installed on the one builder the adapter is handed,
|
|
231
|
+
// that the line names the adapter and the activity's turn, and that it is
|
|
232
|
+
// written for a failed turn as for a completed one.
|
|
233
|
+
const turnPhasesLines = (act: () => Promise<void>) => captureTimingLines(TURN_PHASES_EVENT, act);
|
|
234
|
+
|
|
235
|
+
it("writes one turn_phases line for a completed turn, naming the adapter, the turn and what the fold saw", async () => {
|
|
236
|
+
clock.reset();
|
|
237
|
+
const driver = new RuntimeExecutionDriver(harness, "fake-timeline-completed");
|
|
238
|
+
const lines = await turnPhasesLines(async () => {
|
|
239
|
+
const invocation = await driver.turn([scenario.say("Reading it."), scenario.read("t1", "notes.md"), scenario.say("Done.")]);
|
|
240
|
+
expect(slimOf(subject, invocation).phase).toBe("EXECUTION_COMPLETED");
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
expect(lines).toHaveLength(1);
|
|
244
|
+
const [line] = lines;
|
|
245
|
+
expect(line!.harness).toBe(subject.adapter.name);
|
|
246
|
+
expect(line!.outcome).toBe("completed");
|
|
247
|
+
expect(line!.turn_seq).toBeTypeOf("number");
|
|
248
|
+
expect(line!.rounds, "two `say`s, two rounds").toBe(2);
|
|
249
|
+
expect(line!.tool_calls, "one `read`").toBe(1);
|
|
250
|
+
expect(line!.first_text_ms).toBeTypeOf("number");
|
|
251
|
+
});
|
|
252
|
+
|
|
253
|
+
it("writes the line for a failed turn too: every outcome is measured", async () => {
|
|
254
|
+
clock.reset();
|
|
255
|
+
const driver = new RuntimeExecutionDriver(harness, "fake-timeline-failed");
|
|
256
|
+
const lines = await turnPhasesLines(async () => {
|
|
257
|
+
const invocation = await driver.turn([scenario.say("Checking."), scenario.fail("engine gave up")]);
|
|
258
|
+
expect(slimOf(subject, invocation).phase).toBe("EXECUTION_FAILED");
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
expect(lines).toHaveLength(1);
|
|
262
|
+
expect(lines[0]!.outcome).toBe("failed");
|
|
263
|
+
expect(lines[0]!.rounds).toBe(1);
|
|
264
|
+
});
|
|
265
|
+
|
|
266
|
+
it("writes no line for a turn that settled during resolution: no engine, no phases", async () => {
|
|
267
|
+
clock.reset();
|
|
268
|
+
const lines = await turnPhasesLines(async () => {
|
|
269
|
+
await assertWorkspaceLockTimeoutFails(harness, "fake-timeline-no-engine");
|
|
270
|
+
});
|
|
271
|
+
expect(lines).toEqual([]);
|
|
272
|
+
});
|
|
273
|
+
});
|
|
274
|
+
|
|
222
275
|
describe("the runtime's write-back rule", () => {
|
|
223
276
|
// `run-turn.ts` calls `frame.writeback.finalize()` in exactly two places:
|
|
224
277
|
// once when a completed turn reaches its epilogue, and once when a resumed
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `TurnTimeline` (`harness/turn-timeline.ts`), driven with `TranscriptEvent`s
|
|
3
|
+
* on a scripted clock — one arm per rule the module's header lists, in the
|
|
4
|
+
* order it lists them, plus the shape of the `turn_phases` line it writes.
|
|
5
|
+
*
|
|
6
|
+
* What these arms protect: the parity comparison reads this line for both
|
|
7
|
+
* harnesses, so a rule that quietly favoured one (a tool span closed at the
|
|
8
|
+
* fold's clock instead of the observed instant; a sub-agent's token counted
|
|
9
|
+
* as the user's first visible token) would bias a published number. Every
|
|
10
|
+
* instant here is asserted exactly, which the scripted clock makes possible;
|
|
11
|
+
* the hermetic proofs under each adapter assert structure only.
|
|
12
|
+
*
|
|
13
|
+
* Fence-clean: names the runtime's timing primitive and the transcript's
|
|
14
|
+
* union, nothing of any engine.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { afterEach, describe, expect, it, vi } from "vitest";
|
|
18
|
+
import { TURN_PHASES_EVENT, TurnTimeline, type TurnTimelineClock } from "../turn-timeline.js";
|
|
19
|
+
import type { TranscriptEvent } from "../transcript/events.js";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A clock the test advances by hand. `performance.now()` scale starts at the
|
|
23
|
+
* origin 10_000; the wall clock at a fixed epoch, so an `observedAt` stamp can
|
|
24
|
+
* be written as "origin + N ms" and land exactly.
|
|
25
|
+
*/
|
|
26
|
+
const ORIGIN_MS = 10_000;
|
|
27
|
+
const ORIGIN_WALL_MS = Date.UTC(2026, 8, 18, 12, 0, 0, 0);
|
|
28
|
+
|
|
29
|
+
class ScriptedClock implements TurnTimelineClock {
|
|
30
|
+
readonly originMs = ORIGIN_MS;
|
|
31
|
+
readonly originWallMs = ORIGIN_WALL_MS;
|
|
32
|
+
private nowMs = ORIGIN_MS;
|
|
33
|
+
|
|
34
|
+
readonly now = (): number => this.nowMs;
|
|
35
|
+
|
|
36
|
+
/** Move the clock to `offset` ms after the origin. */
|
|
37
|
+
at(offset: number): void {
|
|
38
|
+
this.nowMs = ORIGIN_MS + offset;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** The ISO stamp a harness would put on a fact it observed `offset` ms after the origin. */
|
|
43
|
+
function observedWall(offset: number): string {
|
|
44
|
+
return new Date(ORIGIN_WALL_MS + offset).toISOString();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
interface EmittedLine {
|
|
48
|
+
readonly stigmer_timing: string;
|
|
49
|
+
readonly total_ms: number;
|
|
50
|
+
readonly segments: Array<{ name: string; start_ms: number; duration_ms: number }>;
|
|
51
|
+
readonly [key: string]: unknown;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Drive `steps` (an offset and the event to observe at it), emit, and parse the one line. */
|
|
55
|
+
function run(steps: Array<[number, TranscriptEvent]>, settleAt: number, context: Record<string, string | number> = {}): EmittedLine {
|
|
56
|
+
const clock = new ScriptedClock();
|
|
57
|
+
const timeline = new TurnTimeline(clock);
|
|
58
|
+
for (const [offset, event] of steps) {
|
|
59
|
+
clock.at(offset);
|
|
60
|
+
timeline.observe(event);
|
|
61
|
+
}
|
|
62
|
+
clock.at(settleAt);
|
|
63
|
+
const spy = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
64
|
+
timeline.emit(context);
|
|
65
|
+
expect(spy).toHaveBeenCalledTimes(1);
|
|
66
|
+
return JSON.parse(spy.mock.calls[0]![0] as string) as EmittedLine;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function segment(line: EmittedLine, name: string) {
|
|
70
|
+
const found = line.segments.filter((s) => s.name === name);
|
|
71
|
+
expect(found, `one segment named ${name}`).toHaveLength(1);
|
|
72
|
+
return found[0]!;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const START: TranscriptEvent = { kind: "message_start", runId: "r1" };
|
|
76
|
+
const TEXT: TranscriptEvent = { kind: "text_delta", runId: "r1", text: "hi" };
|
|
77
|
+
const THINK: TranscriptEvent = { kind: "reasoning_delta", runId: "r1", text: "hmm" };
|
|
78
|
+
const FINISH: TranscriptEvent = { kind: "message_finish", runId: "r1" };
|
|
79
|
+
const TOOL_START: TranscriptEvent = { kind: "tool_started", callId: "c1", name: "read_file", input: { path: "a" }, mcpServerSlug: "" };
|
|
80
|
+
const TOOL_DONE: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok" };
|
|
81
|
+
|
|
82
|
+
afterEach(() => {
|
|
83
|
+
vi.restoreAllMocks();
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
describe("TurnTimeline — the three root instants", () => {
|
|
87
|
+
it("first_event is the first event of any kind; visible and text follow the first text delta", () => {
|
|
88
|
+
const line = run([[100, START], [250, TEXT], [400, FINISH]], 500);
|
|
89
|
+
expect(line.first_event_ms).toBe(100);
|
|
90
|
+
expect(line.first_visible_token_ms).toBe(250);
|
|
91
|
+
expect(line.first_text_ms).toBe(250);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it("a reasoning delta is visible but not text: thinking on moves the two instants apart", () => {
|
|
95
|
+
const line = run([[100, START], [180, THINK], [420, TEXT]], 500);
|
|
96
|
+
expect(line.first_visible_token_ms).toBe(180);
|
|
97
|
+
expect(line.first_text_ms).toBe(420);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it("a sub-agent's tokens set no root instant: the user watches the thread, not the sub-agent's row", () => {
|
|
101
|
+
const sub: TranscriptEvent[] = [
|
|
102
|
+
{ kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "look", input: "look" },
|
|
103
|
+
{ kind: "message_start", runId: "sr", subAgentId: "s1" },
|
|
104
|
+
{ kind: "text_delta", runId: "sr", text: "reading", subAgentId: "s1" },
|
|
105
|
+
];
|
|
106
|
+
const line = run([[50, sub[0]!], [60, sub[1]!], [70, sub[2]!], [900, TEXT]], 1000);
|
|
107
|
+
expect(line.first_event_ms).toBe(50);
|
|
108
|
+
expect(line.first_visible_token_ms).toBe(900);
|
|
109
|
+
expect(line.first_text_ms).toBe(900);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("an instant that never happened is null, and an empty turn has zero of everything", () => {
|
|
113
|
+
const line = run([], 300);
|
|
114
|
+
expect(line.first_event_ms).toBeNull();
|
|
115
|
+
expect(line.first_visible_token_ms).toBeNull();
|
|
116
|
+
expect(line.first_text_ms).toBeNull();
|
|
117
|
+
expect(line.rounds).toBe(0);
|
|
118
|
+
expect(line.tool_calls).toBe(0);
|
|
119
|
+
expect(line.sub_agents).toBe(0);
|
|
120
|
+
expect(line.max_gap_ms).toBeNull();
|
|
121
|
+
expect(line.segments).toEqual([]);
|
|
122
|
+
expect(line.total_ms).toBe(0);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
describe("TurnTimeline — one model_round span per runId", () => {
|
|
127
|
+
it("opens on message_start and closes on message_finish; two runs are two rounds", () => {
|
|
128
|
+
const line = run([
|
|
129
|
+
[100, START], [300, FINISH],
|
|
130
|
+
[350, { kind: "message_start", runId: "r2" }], [700, { kind: "message_finish", runId: "r2" }],
|
|
131
|
+
], 800);
|
|
132
|
+
expect(line.rounds).toBe(2);
|
|
133
|
+
expect(line.segments.filter((s) => s.name === "model_round").map((s) => [s.start_ms, s.duration_ms])).toEqual([[100, 200], [350, 350]]);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
it("a re-emitted start for a known run keeps the first start", () => {
|
|
137
|
+
const line = run([[100, START], [200, START], [300, FINISH]], 400);
|
|
138
|
+
expect(segment(line, "model_round")).toEqual({ name: "model_round", start_ms: 100, duration_ms: 200 });
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
describe("TurnTimeline — one tool span per callId", () => {
|
|
143
|
+
it("opens on tool_started and closes on the first tool_finished; the name carries the tool", () => {
|
|
144
|
+
const line = run([[100, TOOL_START], [340, TOOL_DONE]], 400);
|
|
145
|
+
expect(line.tool_calls).toBe(1);
|
|
146
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 100, duration_ms: 240 });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("an MCP tool's span names its server, so the MCP phase is legible", () => {
|
|
150
|
+
const start: TranscriptEvent = { kind: "tool_started", callId: "m1", name: "search", input: {}, mcpServerSlug: "github" };
|
|
151
|
+
const line = run([[100, start], [500, { kind: "tool_finished", callId: "m1", result: "" }]], 600);
|
|
152
|
+
expect(segment(line, "tool:github/search").duration_ms).toBe(400);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
it("a tool_error closes the span like a finish", () => {
|
|
156
|
+
const line = run([[100, TOOL_START], [150, { kind: "tool_error", callId: "c1", message: "boom" }]], 200);
|
|
157
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(50);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("a fact carrying observedAt closes the span at the observed instant, not the fold's", () => {
|
|
161
|
+
// The Cursor shape: the tool finished at +300 on the delta channel, the
|
|
162
|
+
// fold applies it at +2_800, after the model narrated for 2.5 seconds.
|
|
163
|
+
const observed: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok", observedAt: observedWall(300) };
|
|
164
|
+
const line = run([[100, TOOL_START], [2_800, observed]], 3_000);
|
|
165
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 100, duration_ms: 200 });
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("an unparseable observedAt falls back to the fold's clock rather than losing the close", () => {
|
|
169
|
+
const observed: TranscriptEvent = { kind: "tool_finished", callId: "c1", result: "ok", observedAt: "not a date" };
|
|
170
|
+
const line = run([[100, TOOL_START], [900, observed]], 1_000);
|
|
171
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(800);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("a second finish for a settled span changes nothing (Cursor completes every call twice)", () => {
|
|
175
|
+
const line = run([[100, TOOL_START], [300, TOOL_DONE], [900, TOOL_DONE]], 1_000);
|
|
176
|
+
expect(segment(line, "tool:read_file").duration_ms).toBe(200);
|
|
177
|
+
expect(line.tool_calls).toBe(1);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
it("a finish for a call that never started is ignored", () => {
|
|
181
|
+
const line = run([[300, TOOL_DONE]], 400);
|
|
182
|
+
expect(line.tool_calls).toBe(0);
|
|
183
|
+
expect(line.segments).toEqual([]);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it("approval_proposed opens no span: a held call is not a tool phase", () => {
|
|
187
|
+
const proposed: TranscriptEvent = { kind: "approval_proposed", callId: "g1", name: "shell", mcpServerSlug: "", message: "Run?" };
|
|
188
|
+
const line = run([[100, proposed]], 200);
|
|
189
|
+
expect(line.tool_calls).toBe(0);
|
|
190
|
+
expect(line.first_event_ms, "it is still an event").toBe(100);
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it("arg and output deltas are events for the gap but open nothing", () => {
|
|
194
|
+
const line = run([
|
|
195
|
+
[100, TOOL_START],
|
|
196
|
+
[150, { kind: "tool_arg_delta", callId: "c1", argsChunk: "{" }],
|
|
197
|
+
[200, { kind: "tool_output_delta", callId: "c1", delta: "partial" }],
|
|
198
|
+
[400, TOOL_DONE],
|
|
199
|
+
], 500);
|
|
200
|
+
expect(line.tool_calls).toBe(1);
|
|
201
|
+
expect(line.segments).toHaveLength(1);
|
|
202
|
+
});
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
describe("TurnTimeline — one sub_agent span per subAgentId", () => {
|
|
206
|
+
it("opens on sub_agent_started, closes on finished or failed, named after the sub-agent", () => {
|
|
207
|
+
const line = run([
|
|
208
|
+
[100, { kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "x", input: "x" }],
|
|
209
|
+
[900, { kind: "sub_agent_finished", subAgentId: "s1", output: "done" }],
|
|
210
|
+
[950, { kind: "sub_agent_started", subAgentId: "s2", name: "writer", subject: "y", input: "y" }],
|
|
211
|
+
[1_000, { kind: "sub_agent_failed", subAgentId: "s2", error: "no" }],
|
|
212
|
+
], 1_100);
|
|
213
|
+
expect(line.sub_agents).toBe(2);
|
|
214
|
+
expect(segment(line, "sub_agent:researcher").duration_ms).toBe(800);
|
|
215
|
+
expect(segment(line, "sub_agent:writer").duration_ms).toBe(50);
|
|
216
|
+
});
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
describe("TurnTimeline — the longest silence", () => {
|
|
220
|
+
it("max_gap is the largest wait between two consecutive events, of any kind", () => {
|
|
221
|
+
const line = run([[100, START], [150, TEXT], [1_350, TOOL_START], [1_400, TOOL_DONE]], 1_500);
|
|
222
|
+
expect(line.max_gap_ms).toBe(1_200);
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
it("is null under two events", () => {
|
|
226
|
+
expect(run([[100, START]], 200).max_gap_ms).toBeNull();
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
describe("TurnTimeline — settle and the line's shape", () => {
|
|
231
|
+
it("a span still open at settle closes at settle", () => {
|
|
232
|
+
const line = run([[100, START], [200, TOOL_START]], 1_000);
|
|
233
|
+
expect(segment(line, "model_round")).toEqual({ name: "model_round", start_ms: 100, duration_ms: 900 });
|
|
234
|
+
expect(segment(line, "tool:read_file")).toEqual({ name: "tool:read_file", start_ms: 200, duration_ms: 800 });
|
|
235
|
+
expect(line.total_ms).toBe(1_000);
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
it("segments overlap and come out in start order; total is the latest end, not a sum", () => {
|
|
239
|
+
const line = run([
|
|
240
|
+
[100, START],
|
|
241
|
+
[200, TOOL_START],
|
|
242
|
+
[300, { kind: "sub_agent_started", subAgentId: "s1", name: "researcher", subject: "x", input: "x" }],
|
|
243
|
+
[400, TOOL_DONE],
|
|
244
|
+
[500, FINISH],
|
|
245
|
+
[600, { kind: "message_start", runId: "r2" }],
|
|
246
|
+
[700, { kind: "sub_agent_finished", subAgentId: "s1" }],
|
|
247
|
+
[800, { kind: "message_finish", runId: "r2" }],
|
|
248
|
+
], 850);
|
|
249
|
+
expect(line.segments.map((s) => s.name)).toEqual(["model_round", "tool:read_file", "sub_agent:researcher", "model_round"]);
|
|
250
|
+
const sum = line.segments.reduce((acc, s) => acc + s.duration_ms, 0);
|
|
251
|
+
expect(sum).toBeGreaterThan(line.total_ms);
|
|
252
|
+
expect(line.total_ms).toBe(800);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it("writes the turn_phases selector and the runtime's context beside the fold's facts", () => {
|
|
256
|
+
const line = run([[100, START], [200, TEXT], [300, FINISH]], 400, {
|
|
257
|
+
execution_id: "exe_1",
|
|
258
|
+
session_id: "ses_1",
|
|
259
|
+
turn_seq: 2,
|
|
260
|
+
harness: "deep-agent",
|
|
261
|
+
outcome: "completed",
|
|
262
|
+
});
|
|
263
|
+
expect(line.stigmer_timing).toBe(TURN_PHASES_EVENT);
|
|
264
|
+
expect(line.execution_id).toBe("exe_1");
|
|
265
|
+
expect(line.session_id).toBe("ses_1");
|
|
266
|
+
expect(line.turn_seq).toBe(2);
|
|
267
|
+
expect(line.harness).toBe("deep-agent");
|
|
268
|
+
expect(line.outcome).toBe("completed");
|
|
269
|
+
expect(Object.keys(line).sort()).toEqual([
|
|
270
|
+
"execution_id", "first_event_ms", "first_text_ms", "first_visible_token_ms", "harness", "max_gap_ms",
|
|
271
|
+
"outcome", "rounds", "segments", "session_id", "stigmer_timing", "sub_agents", "tool_calls", "total_ms", "turn_seq",
|
|
272
|
+
]);
|
|
273
|
+
});
|
|
274
|
+
});
|
package/src/harness/run-turn.ts
CHANGED
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
* What runs here and nowhere else: the activity's identity and its
|
|
6
6
|
* execution-scoped context; the status, the one transcript builder over it
|
|
7
7
|
* (born with the status, seeded by the reinvocation phase, handed to the
|
|
8
|
-
* adapter as `sink.transcript`, finalized once the adapter returns)
|
|
9
|
-
*
|
|
8
|
+
* adapter as `sink.transcript`, finalized once the adapter returns), the turn
|
|
9
|
+
* timeline folded from that builder's events and written once at settle
|
|
10
|
+
* (`turn-timeline.ts`), and the single persist chokepoint;
|
|
10
11
|
* the periodic heartbeat; the stop controller and its four producers (the
|
|
11
12
|
* stall watchdog, the cost cap, the platform STOP, Temporal's cancellation);
|
|
12
13
|
* the resolution phases (`turn-context.ts`) composed into the `TurnInput`;
|
|
@@ -85,6 +86,7 @@ import {
|
|
|
85
86
|
} from "./terminal-table.js";
|
|
86
87
|
import { TranscriptBuilder } from "./transcript/builder.js";
|
|
87
88
|
import { resolveTurnContext, type ResolutionDeps, type TurnFrame, type TurnSettlement } from "./turn-context.js";
|
|
89
|
+
import { TurnTimeline } from "./turn-timeline.js";
|
|
88
90
|
import type { HarnessAdapter, TurnInput, TurnSink } from "./types.js";
|
|
89
91
|
import { UsageAccumulator } from "./usage-accumulator.js";
|
|
90
92
|
|
|
@@ -151,20 +153,34 @@ async function runTurn(deps: TurnRuntimeDeps, input: NormalizedActivityInput): P
|
|
|
151
153
|
startedAt: utcTimestamp(),
|
|
152
154
|
});
|
|
153
155
|
|
|
156
|
+
// The turn's origin, read once on both clocks so the two timelines below
|
|
157
|
+
// and the harness's observed instants (`transcript/events.ts` `Observed`)
|
|
158
|
+
// all subtract against the same moment.
|
|
159
|
+
const originMs = performance.now();
|
|
160
|
+
const originWallMs = Date.now();
|
|
161
|
+
|
|
162
|
+
// The turn timeline (`turn-timeline.ts`): the runtime's fold over every
|
|
163
|
+
// event the adapter folds through the builder, emitted once at settle as
|
|
164
|
+
// the `turn_phases` line — first visible token, model rounds, tool and
|
|
165
|
+
// sub-agent spans, the longest silence — for every harness alike.
|
|
166
|
+
const timeline = new TurnTimeline({ originMs, originWallMs, now: () => performance.now() });
|
|
167
|
+
|
|
154
168
|
// The one transcript builder of this turn, born with the status and before
|
|
155
169
|
// any phase, so it exists for everything that writes a row or an output
|
|
156
170
|
// onto the status: the write-back coordinator (phase 2c), the seed of a
|
|
157
171
|
// reinvocation's prior rows (phase 3, through `seed()`, which indexes them
|
|
158
172
|
// as it appends), the adapter (`sink.transcript`), and the epilogue. One
|
|
159
173
|
// constructor call per turn; a second builder over the same status would
|
|
160
|
-
// index the same rows twice.
|
|
161
|
-
const transcript = new TranscriptBuilder(executionId, status);
|
|
174
|
+
// index the same rows twice. The timeline is its observer.
|
|
175
|
+
const transcript = new TranscriptBuilder(executionId, status, (event) => timeline.observe(event));
|
|
162
176
|
|
|
163
177
|
// Cold-start timeline of this turn's setup: one mark after each phase, the
|
|
164
178
|
// adapter's own segments marked through the sink, emitted by the adapter
|
|
165
179
|
// once its engine is resolved (an early return skips it — partial setups
|
|
166
|
-
// are not comparable cold-start samples).
|
|
167
|
-
|
|
180
|
+
// are not comparable cold-start samples). On the turn's origin, so
|
|
181
|
+
// `turn_phases.first_visible_token_ms - execution_setup.total_ms` is the
|
|
182
|
+
// wait after the engine was ready, with no join.
|
|
183
|
+
const setupTiming = new TimingRecorder(originMs);
|
|
168
184
|
|
|
169
185
|
// Artifact storage for offloading oversized tool outputs out of the
|
|
170
186
|
// persisted status and for publishing the plan artifact. Resolved once so
|
|
@@ -366,6 +382,20 @@ async function runTurn(deps: TurnRuntimeDeps, input: NormalizedActivityInput): P
|
|
|
366
382
|
// finalizes first, as the Cursor harness does before `run.wait()`.
|
|
367
383
|
transcript.finalize();
|
|
368
384
|
|
|
385
|
+
// The engine's turn is over: write its timeline, once, on every outcome.
|
|
386
|
+
// Here and not in the settles, because this is the one place that sees
|
|
387
|
+
// every path, and before the epilogue, because the epilogue is the
|
|
388
|
+
// runtime's, not the turn the user watched. A turn that settled during
|
|
389
|
+
// resolution never reaches here and gets no line: no engine, no phases.
|
|
390
|
+
// `harness` is the adapter's diagnostic name, the runtime's vocabulary.
|
|
391
|
+
timeline.emit({
|
|
392
|
+
execution_id: executionId,
|
|
393
|
+
session_id: turn.sessionId,
|
|
394
|
+
turn_seq: input.turnSeq,
|
|
395
|
+
harness: adapter.name,
|
|
396
|
+
outcome: outcome.kind,
|
|
397
|
+
});
|
|
398
|
+
|
|
369
399
|
// The capture boundary, ONCE, over whatever the whole turn left on the
|
|
370
400
|
// tree — unless a stop fired mid-turn (`interrupted`: the tree may be
|
|
371
401
|
// mid-edit and the pause or drain reinvokes this same turn). Before the
|
|
@@ -17,7 +17,9 @@
|
|
|
17
17
|
* preview, the todo projection, and the rest of the header's rules —
|
|
18
18
|
* the row per id, the message boundary, the thinking row, the sub-agent
|
|
19
19
|
* rows, finalize, artifacts and write-backs, the dirty flag, the error
|
|
20
|
-
* guard — joined commit by commit in the same PR.
|
|
20
|
+
* guard — joined commit by commit in the same PR. The observer arm (the
|
|
21
|
+
* runtime's fold-watcher: told after the fold, guarded apart from the
|
|
22
|
+
* handlers, silent on seed) joined with the turn timeline.
|
|
21
23
|
*/
|
|
22
24
|
|
|
23
25
|
import { describe, it, expect } from "vitest";
|
|
@@ -973,3 +975,69 @@ describe("TranscriptBuilder — the dirty flag and the error guard", () => {
|
|
|
973
975
|
expect(errors[0]).toContain("property access failed");
|
|
974
976
|
});
|
|
975
977
|
});
|
|
978
|
+
|
|
979
|
+
describe("TranscriptBuilder — the observer sees every folded event, after the fold, and can break nothing", () => {
|
|
980
|
+
it("is told every applied event, in order, once the fold has landed", () => {
|
|
981
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
982
|
+
const seen: Array<{ kind: string; rowsAtNotify: number }> = [];
|
|
983
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => {
|
|
984
|
+
seen.push({ kind: event.kind, rowsAtNotify: status.messages.flatMap((m) => m.toolCalls).length });
|
|
985
|
+
});
|
|
986
|
+
for (const e of proposedCall()) sb.apply(e);
|
|
987
|
+
sb.apply({ kind: "tool_finished", callId: "call-1", result: "ok" });
|
|
988
|
+
|
|
989
|
+
expect(seen.map((s) => s.kind)).toEqual(["message_start", "text_delta", "message_finish", "tool_started", "tool_finished"]);
|
|
990
|
+
// "After the fold": when the observer hears tool_started, the row exists.
|
|
991
|
+
expect(seen[3]!.rowsAtNotify).toBe(1);
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
it("a throwing observer is logged with the execution and the event kind, and the fold stands", () => {
|
|
995
|
+
const errors: string[] = [];
|
|
996
|
+
const original = console.error;
|
|
997
|
+
console.error = (msg: unknown) => { errors.push(String(msg)); };
|
|
998
|
+
try {
|
|
999
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
1000
|
+
const sb = new TranscriptBuilder("exec-observe", status, () => { throw new Error("observer fell over"); });
|
|
1001
|
+
expect(() => { for (const e of proposedCall()) sb.apply(e); }).not.toThrow();
|
|
1002
|
+
expect(rowOf(status, "call-1").status, "the transcript is whole").toBe(ToolCallStatus.TOOL_CALL_RUNNING);
|
|
1003
|
+
} finally {
|
|
1004
|
+
console.error = original;
|
|
1005
|
+
}
|
|
1006
|
+
expect(errors).toHaveLength(4);
|
|
1007
|
+
expect(errors[0]).toContain("exec-observe");
|
|
1008
|
+
expect(errors[0]).toContain("kind=message_start");
|
|
1009
|
+
expect(errors[0]).toContain("observer fell over");
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
it("still hears an event whose handler threw: the two guards are independent", () => {
|
|
1013
|
+
const original = console.error;
|
|
1014
|
+
console.error = () => {};
|
|
1015
|
+
const seen: string[] = [];
|
|
1016
|
+
try {
|
|
1017
|
+
const { status } = feed(builder(), proposedCall());
|
|
1018
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => { seen.push(event.kind); });
|
|
1019
|
+
const poisoned = { kind: "tool_finished", callId: "call-1", get result(): string { throw new Error("property access failed"); } } as TranscriptEvent;
|
|
1020
|
+
sb.apply(poisoned);
|
|
1021
|
+
} finally {
|
|
1022
|
+
console.error = original;
|
|
1023
|
+
}
|
|
1024
|
+
expect(seen).toEqual(["tool_finished"]);
|
|
1025
|
+
});
|
|
1026
|
+
|
|
1027
|
+
it("seed() does not notify: a seeded row is last turn's fact, not an event of this turn", () => {
|
|
1028
|
+
const status = create(AgentExecutionStatusSchema, {});
|
|
1029
|
+
const seen: string[] = [];
|
|
1030
|
+
const sb = new TranscriptBuilder("exec-observe", status, (event) => { seen.push(event.kind); });
|
|
1031
|
+
sb.seed(create(AgentExecutionStatusSchema, {
|
|
1032
|
+
messages: [create(AgentMessageSchema, { type: MessageType.MESSAGE_AI, content: "earlier", toolCalls: [create(ToolCallSchema, { id: "old-1", name: "read_file", status: ToolCallStatus.TOOL_CALL_COMPLETED })] })],
|
|
1033
|
+
}));
|
|
1034
|
+
expect(seen).toEqual([]);
|
|
1035
|
+
expect(status.messages).toHaveLength(1);
|
|
1036
|
+
});
|
|
1037
|
+
|
|
1038
|
+
it("is optional: the two-argument constructor folds exactly as before", () => {
|
|
1039
|
+
const { sb, status } = feed(builder(), proposedCall());
|
|
1040
|
+
sb.apply({ kind: "tool_finished", callId: "call-1", result: "ok" });
|
|
1041
|
+
expect(rowOf(status, "call-1").status).toBe(ToolCallStatus.TOOL_CALL_COMPLETED);
|
|
1042
|
+
});
|
|
1043
|
+
});
|