@stigmer/runner 3.19.1-dev.20260920100552 → 3.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build-fingerprint +1 -1
- package/dist/__test-utils__/execution-record-fixture.d.ts +7 -0
- package/dist/__test-utils__/execution-record-fixture.js +10 -1
- package/dist/__test-utils__/execution-record-fixture.js.map +1 -1
- package/dist/__test-utils__/hermetic-activity.d.ts +7 -4
- package/dist/__test-utils__/hermetic-activity.js +12 -2
- package/dist/__test-utils__/hermetic-activity.js.map +1 -1
- package/dist/__test-utils__/turn-input-fixture.js +0 -1
- package/dist/__test-utils__/turn-input-fixture.js.map +1 -1
- package/dist/activities/execute-cursor/prompt-builder.js +8 -3
- package/dist/activities/execute-cursor/prompt-builder.js.map +1 -1
- package/dist/activities/execute-cursor/turn-settle.js +4 -1
- package/dist/activities/execute-cursor/turn-settle.js.map +1 -1
- package/dist/activities/execute-deep-agent/prompt-builder.d.ts +1 -3
- package/dist/activities/execute-deep-agent/prompt-builder.js +2 -3
- package/dist/activities/execute-deep-agent/prompt-builder.js.map +1 -1
- package/dist/activities/generate-session-subject.d.ts +2 -1
- package/dist/activities/generate-session-subject.js +18 -10
- package/dist/activities/generate-session-subject.js.map +1 -1
- package/dist/harness/run-turn.d.ts +3 -2
- package/dist/harness/run-turn.js +33 -6
- package/dist/harness/run-turn.js.map +1 -1
- package/dist/harness/transcript/builder.d.ts +26 -2
- package/dist/harness/transcript/builder.js +35 -2
- package/dist/harness/transcript/builder.js.map +1 -1
- package/dist/harness/turn-context.js +3 -1
- package/dist/harness/turn-context.js.map +1 -1
- package/dist/harness/turn-timeline.d.ts +132 -0
- package/dist/harness/turn-timeline.js +213 -0
- package/dist/harness/turn-timeline.js.map +1 -0
- package/dist/shared/blueprint-resolver.d.ts +14 -4
- package/dist/shared/blueprint-resolver.js +23 -10
- package/dist/shared/blueprint-resolver.js.map +1 -1
- package/dist/shared/builtin-assistant-prompt.d.ts +30 -0
- package/dist/shared/builtin-assistant-prompt.js +36 -0
- package/dist/shared/builtin-assistant-prompt.js.map +1 -0
- package/dist/shared/cold-start-timing.d.ts +18 -2
- package/dist/shared/cold-start-timing.js +30 -3
- package/dist/shared/cold-start-timing.js.map +1 -1
- package/package.json +5 -5
- package/src/__test-utils__/execution-record-fixture.ts +17 -1
- package/src/__test-utils__/hermetic-activity.ts +19 -6
- package/src/__test-utils__/timing-lines.ts +43 -0
- package/src/__test-utils__/turn-input-fixture.ts +0 -1
- package/src/activities/__tests__/generate-session-subject.test.ts +14 -0
- package/src/activities/execute-cursor/__tests__/goldens/prompt.enhanced.no-instructions.prompt.md +142 -0
- package/src/activities/execute-cursor/__tests__/hermetic/built-in-assistant.test.ts +127 -0
- package/src/activities/execute-cursor/__tests__/hermetic/turn-phases.test.ts +136 -0
- package/src/activities/execute-cursor/__tests__/prompt-goldens.test.ts +14 -1
- package/src/activities/execute-cursor/prompt-builder.ts +8 -3
- package/src/activities/execute-cursor/turn-settle.ts +4 -1
- package/src/activities/execute-deep-agent/__tests__/goldens/system-prompt.minimal.prompt.md +2 -1
- package/src/activities/execute-deep-agent/__tests__/hermetic/built-in-assistant.test.ts +101 -0
- package/src/activities/execute-deep-agent/__tests__/hermetic/turn-phases.test.ts +118 -0
- package/src/activities/execute-deep-agent/__tests__/prompt-goldens.test.ts +1 -1
- package/src/activities/execute-deep-agent/prompt-builder.ts +3 -5
- package/src/activities/generate-session-subject.ts +20 -13
- package/src/harness/README.md +4 -1
- package/src/harness/__tests__/run-turn.test.ts +53 -0
- package/src/harness/__tests__/turn-timeline.test.ts +274 -0
- package/src/harness/run-turn.ts +36 -6
- package/src/harness/transcript/__tests__/builder.test.ts +69 -1
- package/src/harness/transcript/builder.ts +39 -2
- package/src/harness/turn-context.ts +3 -1
- package/src/harness/turn-timeline.ts +248 -0
- package/src/shared/__tests__/blueprint-resolver.test.ts +78 -8
- package/src/shared/__tests__/cold-start-timing.test.ts +53 -5
- package/src/shared/blueprint-resolver.ts +25 -12
- package/src/shared/builtin-assistant-prompt.ts +38 -0
- package/src/shared/cold-start-timing.ts +29 -3
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The turn runtime's timeline of one engine turn, folded from the transcript
|
|
3
|
+
* events every harness already emits, and written once at settle as the
|
|
4
|
+
* `turn_phases` `stigmer_timing` line.
|
|
5
|
+
*
|
|
6
|
+
* What it answers, for every harness alike and for any harness added later
|
|
7
|
+
* without a line in an adapter: when the user first saw something happen,
|
|
8
|
+
* how many model rounds and tool calls the turn took, where the sub-agents
|
|
9
|
+
* ran, and where the turn's longest silence was. It is the per-turn,
|
|
10
|
+
* CLIENT-PERCEIVED measurement from the runner's vantage, from the turn's
|
|
11
|
+
* origin (the activity's start, the same origin `execution_setup` counts
|
|
12
|
+
* from, so the two lines subtract without a join) to the adapter's return;
|
|
13
|
+
* the completion epilogue and the terminal persist that follow are the
|
|
14
|
+
* client's own end-to-end, read at `subscribe`. Its counterpart is the cloud
|
|
15
|
+
* proxy's `ProxyTiming` (`agentexecution/v1/usage.proto`): per model CALL,
|
|
16
|
+
* server-side, what the provider did. Neither restates the other; a report
|
|
17
|
+
* that wants both joins them by `execution_id`.
|
|
18
|
+
*
|
|
19
|
+
* Why a fold over the transcript events and not a mark in each adapter: the
|
|
20
|
+
* transcript builder (`transcript/builder.ts`) is the one view the runtime
|
|
21
|
+
* has of every harness's turn — model rounds, text, tools, sub-agents all
|
|
22
|
+
* reach it as a translator's output on one union — so a fold over it is
|
|
23
|
+
* written once and holds for the Cursor adapter, the native adapter and the
|
|
24
|
+
* next one. `UsageAccumulator` (`usage-accumulator.ts`) is the same shape
|
|
25
|
+
* for the usage deltas; this is that shape for the events.
|
|
26
|
+
*
|
|
27
|
+
* The fold's rules, each with an arm in `__tests__/turn-timeline.test.ts`:
|
|
28
|
+
*
|
|
29
|
+
* - Three instants, ROOT scope only (a sub-agent's text is not what the
|
|
30
|
+
* user is watching): `first_event_ms`, the first event of any kind;
|
|
31
|
+
* `first_visible_token_ms`, the first `text_delta` or `reasoning_delta`
|
|
32
|
+
* (both stream into the console as tokens); `first_text_ms`, the first
|
|
33
|
+
* `text_delta`. The two token instants are the cross-harness axes: with
|
|
34
|
+
* extended thinking on, the visible instant moves earlier and the text
|
|
35
|
+
* instant later, and a comparison needs both. `first_event_ms` is
|
|
36
|
+
* harness-relative — on native the first event is `message_start`, the
|
|
37
|
+
* model stream opening; on Cursor it is whatever the SDK delivered first —
|
|
38
|
+
* and is kept as a forensic (engine-ready to first model event), never as
|
|
39
|
+
* an axis. An instant that never happened is `null`.
|
|
40
|
+
* - One `model_round` span per `runId`, `message_start` to `message_finish`.
|
|
41
|
+
* What a `runId` is belongs to the translator: LangGraph's LLM-call run on
|
|
42
|
+
* native, one model round; a minted segment `<run>:<n>` on Cursor, a
|
|
43
|
+
* stretch of assistant output between tool calls, which the SDK does not
|
|
44
|
+
* let us tell apart from a provider round. The line's `rounds` reads the
|
|
45
|
+
* same way.
|
|
46
|
+
* - One `tool:<name>` span per `callId` (`tool:<slug>/<name>` for an MCP
|
|
47
|
+
* tool, so the MCP phase the runner skill's law asks for is legible),
|
|
48
|
+
* `tool_started` to the FIRST `tool_finished` or `tool_error`; a later
|
|
49
|
+
* fact for a settled span changes nothing, the builder's own upsert rule
|
|
50
|
+
* (Cursor completes every call twice). `approval_proposed` opens nothing:
|
|
51
|
+
* a call the engine held before it ran is not a tool phase.
|
|
52
|
+
* - One `sub_agent:<name>` span per `subAgentId`, opened by
|
|
53
|
+
* `sub_agent_started`, closed by `finished` or `failed`.
|
|
54
|
+
* - A fact that carries `observedAt` is placed at that instant, everything
|
|
55
|
+
* else at the fold's clock — the builder's rule for `completedAt`
|
|
56
|
+
* (`transcript/events.ts`, `Observed`), restated here because the
|
|
57
|
+
* alternative repeats the error #1097 fixed, for tool phases and on Cursor
|
|
58
|
+
* alone: a completion the SDK's delta channel reported is folded one
|
|
59
|
+
* stream event later, seconds while the model narrates, and a comparison
|
|
60
|
+
* that read the fold's clock would charge those seconds to Cursor's tools.
|
|
61
|
+
* Placing an observed instant needs a wall-clock origin beside the
|
|
62
|
+
* `performance.now()` one; the runtime captures both together.
|
|
63
|
+
* - `max_gap_ms`, the longest wait between two consecutive events, whatever
|
|
64
|
+
* their kind: the turn's longest silence as the user experienced it.
|
|
65
|
+
* `null` under two events.
|
|
66
|
+
* - A span still open at settle closes at settle: an interrupted turn's
|
|
67
|
+
* spans end where the turn did.
|
|
68
|
+
*
|
|
69
|
+
* The line: `emitTimingLog("turn_phases", context, recorder)` over a
|
|
70
|
+
* `TimingRecorder` on the turn's origin, every span appended in start order
|
|
71
|
+
* through `span()`. This is the first `stigmer_timing` event whose segments
|
|
72
|
+
* OVERLAP (a tool inside a round, a sub-agent across rounds); the recorder's
|
|
73
|
+
* `total_ms` is the latest end, not a sum, and a reader that summed the
|
|
74
|
+
* other timelines' segments must not sum this one's. No histogram mirrors it
|
|
75
|
+
* yet: the benchmark report is its reader and reads the log; an aggregate
|
|
76
|
+
* earns its instrument when a dashboard asks.
|
|
77
|
+
*
|
|
78
|
+
* Failure posture, the timing module's: the fold is arithmetic over the
|
|
79
|
+
* event and cannot affect a turn; `emit` is best-effort through
|
|
80
|
+
* `emitTimingLog`. Nothing here is persisted, and nothing branches on it.
|
|
81
|
+
*/
|
|
82
|
+
|
|
83
|
+
import { TimingRecorder, emitTimingLog } from "../shared/cold-start-timing.js";
|
|
84
|
+
import type { TranscriptEvent } from "./transcript/events.js";
|
|
85
|
+
|
|
86
|
+
/** The `stigmer_timing` selector of the line this module emits. */
|
|
87
|
+
export const TURN_PHASES_EVENT = "turn_phases";
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The two clocks the timeline places facts on, captured together at the
|
|
91
|
+
* turn's origin. `now` is `performance.now` in production; tests inject a
|
|
92
|
+
* scripted one. `originWallMs` is the `Date.now()` of the same instant, the
|
|
93
|
+
* scale `observedAt` stamps are on.
|
|
94
|
+
*/
|
|
95
|
+
export interface TurnTimelineClock {
|
|
96
|
+
/** `performance.now()`-scale origin, the one `execution_setup` shares. */
|
|
97
|
+
readonly originMs: number;
|
|
98
|
+
/** `Date.now()`-scale reading of the same instant. */
|
|
99
|
+
readonly originWallMs: number;
|
|
100
|
+
readonly now: () => number;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The facts the runtime adds to the line beside the fold's own. */
|
|
104
|
+
export type TurnPhasesContext = Record<string, string | number | boolean | null | undefined>;
|
|
105
|
+
|
|
106
|
+
/** One span the fold has closed or is still tracking. Absolute `performance.now()` scale. */
|
|
107
|
+
interface OpenSpan {
|
|
108
|
+
readonly name: string;
|
|
109
|
+
readonly startMs: number;
|
|
110
|
+
endMs: number | undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export class TurnTimeline {
|
|
114
|
+
private readonly clock: TurnTimelineClock;
|
|
115
|
+
|
|
116
|
+
private firstEventMs: number | undefined;
|
|
117
|
+
private firstVisibleTokenMs: number | undefined;
|
|
118
|
+
private firstTextMs: number | undefined;
|
|
119
|
+
private lastEventMs: number | undefined;
|
|
120
|
+
private maxGapMs: number | undefined;
|
|
121
|
+
|
|
122
|
+
private readonly rounds = new Map<string, OpenSpan>();
|
|
123
|
+
private readonly tools = new Map<string, OpenSpan>();
|
|
124
|
+
private readonly subAgents = new Map<string, OpenSpan>();
|
|
125
|
+
|
|
126
|
+
constructor(clock: TurnTimelineClock) {
|
|
127
|
+
this.clock = clock;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** The builder's observer: place one folded event on the timeline. */
|
|
131
|
+
observe(event: TranscriptEvent): void {
|
|
132
|
+
const at = this.clock.now();
|
|
133
|
+
this.noteEvent(at);
|
|
134
|
+
|
|
135
|
+
// Root scope only: a sub-agent's tokens stream into its own row, not the
|
|
136
|
+
// thread the user is watching.
|
|
137
|
+
if ((event.kind === "text_delta" || event.kind === "reasoning_delta") && event.subAgentId === undefined) {
|
|
138
|
+
this.firstVisibleTokenMs ??= at;
|
|
139
|
+
if (event.kind === "text_delta") this.firstTextMs ??= at;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
switch (event.kind) {
|
|
143
|
+
case "message_start":
|
|
144
|
+
this.open(this.rounds, event.runId, "model_round", at);
|
|
145
|
+
return;
|
|
146
|
+
case "message_finish":
|
|
147
|
+
this.close(this.rounds, event.runId, at);
|
|
148
|
+
return;
|
|
149
|
+
case "tool_started":
|
|
150
|
+
this.open(this.tools, event.callId, toolSpanName(event.name, event.mcpServerSlug), at);
|
|
151
|
+
return;
|
|
152
|
+
case "tool_finished":
|
|
153
|
+
case "tool_error":
|
|
154
|
+
this.close(this.tools, event.callId, this.placeObserved(event.observedAt, at));
|
|
155
|
+
return;
|
|
156
|
+
case "sub_agent_started":
|
|
157
|
+
this.open(this.subAgents, event.subAgentId, `sub_agent:${event.name}`, at);
|
|
158
|
+
return;
|
|
159
|
+
case "sub_agent_finished":
|
|
160
|
+
case "sub_agent_failed":
|
|
161
|
+
this.close(this.subAgents, event.subAgentId, at);
|
|
162
|
+
return;
|
|
163
|
+
case "text_delta":
|
|
164
|
+
case "reasoning_delta":
|
|
165
|
+
case "tool_arg_delta":
|
|
166
|
+
case "tool_output_delta":
|
|
167
|
+
case "approval_proposed":
|
|
168
|
+
case "system_note":
|
|
169
|
+
return;
|
|
170
|
+
default: {
|
|
171
|
+
const exhaustive: never = event;
|
|
172
|
+
throw new Error(`unknown transcript event ${String((exhaustive as { kind: string }).kind)}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Write the line, once, at settle. Spans still open close now; the
|
|
179
|
+
* runtime's context (`execution_id`, `session_id`, `turn_seq`, `harness`,
|
|
180
|
+
* `outcome`) rides beside the fold's facts.
|
|
181
|
+
*/
|
|
182
|
+
emit(context: TurnPhasesContext): void {
|
|
183
|
+
const settledAt = this.clock.now();
|
|
184
|
+
const recorder = new TimingRecorder(this.clock.originMs);
|
|
185
|
+
const spans = [...this.rounds.values(), ...this.tools.values(), ...this.subAgents.values()]
|
|
186
|
+
.sort((a, b) => a.startMs - b.startMs);
|
|
187
|
+
for (const span of spans) {
|
|
188
|
+
recorder.span(span.name, span.startMs, span.endMs ?? settledAt);
|
|
189
|
+
}
|
|
190
|
+
emitTimingLog(TURN_PHASES_EVENT, {
|
|
191
|
+
...context,
|
|
192
|
+
first_event_ms: this.offset(this.firstEventMs),
|
|
193
|
+
first_visible_token_ms: this.offset(this.firstVisibleTokenMs),
|
|
194
|
+
first_text_ms: this.offset(this.firstTextMs),
|
|
195
|
+
rounds: this.rounds.size,
|
|
196
|
+
tool_calls: this.tools.size,
|
|
197
|
+
sub_agents: this.subAgents.size,
|
|
198
|
+
max_gap_ms: this.maxGapMs === undefined ? null : round1(this.maxGapMs),
|
|
199
|
+
}, recorder);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
private noteEvent(at: number): void {
|
|
203
|
+
this.firstEventMs ??= at;
|
|
204
|
+
if (this.lastEventMs !== undefined) {
|
|
205
|
+
const gap = at - this.lastEventMs;
|
|
206
|
+
if (this.maxGapMs === undefined || gap > this.maxGapMs) this.maxGapMs = gap;
|
|
207
|
+
}
|
|
208
|
+
this.lastEventMs = at;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** A span per identity: a re-emitted start (a resumed seeded call) keeps the first start. */
|
|
212
|
+
private open(spans: Map<string, OpenSpan>, id: string, name: string, at: number): void {
|
|
213
|
+
if (spans.has(id)) return;
|
|
214
|
+
spans.set(id, { name, startMs: at, endMs: undefined });
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** The FIRST close settles a span; a later fact for it changes nothing. A close for an unknown id is ignored. */
|
|
218
|
+
private close(spans: Map<string, OpenSpan>, id: string, at: number): void {
|
|
219
|
+
const span = spans.get(id);
|
|
220
|
+
if (!span || span.endMs !== undefined) return;
|
|
221
|
+
span.endMs = at;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Where a fact goes on the `performance.now()` scale: its observed instant
|
|
226
|
+
* translated through the paired origins when it carries one, else the fold's
|
|
227
|
+
* clock. An unparseable stamp falls back to the fold's clock rather than
|
|
228
|
+
* losing the fact.
|
|
229
|
+
*/
|
|
230
|
+
private placeObserved(observedAt: string | undefined, foldAt: number): number {
|
|
231
|
+
if (observedAt === undefined) return foldAt;
|
|
232
|
+
const wallMs = Date.parse(observedAt);
|
|
233
|
+
if (Number.isNaN(wallMs)) return foldAt;
|
|
234
|
+
return this.clock.originMs + (wallMs - this.clock.originWallMs);
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
private offset(absoluteMs: number | undefined): number | null {
|
|
238
|
+
return absoluteMs === undefined ? null : round1(absoluteMs - this.clock.originMs);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function toolSpanName(name: string, mcpServerSlug: string): string {
|
|
243
|
+
return mcpServerSlug ? `tool:${mcpServerSlug}/${name}` : `tool:${name}`;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function round1(ms: number): number {
|
|
247
|
+
return Math.round(ms * 10) / 10;
|
|
248
|
+
}
|
|
@@ -1,17 +1,87 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `shared/blueprint-resolver.ts
|
|
2
|
+
* `shared/blueprint-resolver.ts`: `mergeSkillRefs`, the one merge of the
|
|
3
3
|
* agent's and the session's skill refs the turn runtime resolves from
|
|
4
|
-
* (`
|
|
5
|
-
* winning a collision (it may pin a different version)
|
|
4
|
+
* (`resolveBlueprint`) — union, deduplicated by slug, the session's ref
|
|
5
|
+
* winning a collision (it may pin a different version); and
|
|
6
|
+
* `resolveBlueprint`'s two arms: the agent chain walked when the session
|
|
7
|
+
* names an instance, and the built-in assistant when it does not — no
|
|
8
|
+
* instance or agent read, empty instructions, no sub-agents, and the
|
|
9
|
+
* session's own usages and refs as the whole tool set.
|
|
6
10
|
*
|
|
7
|
-
*
|
|
8
|
-
* orchestrator's byte-twin of this function was deleted with its
|
|
9
|
-
* until then the runtime's copy was tested only through the twin.
|
|
11
|
+
* `mergeSkillRefs` carried from `skill-writer.test.ts` in #1096, when the
|
|
12
|
+
* native orchestrator's byte-twin of this function was deleted with its
|
|
13
|
+
* module; until then the runtime's copy was tested only through the twin.
|
|
10
14
|
*/
|
|
11
15
|
|
|
12
|
-
import { describe, it, expect } from "vitest";
|
|
16
|
+
import { describe, it, expect, vi } from "vitest";
|
|
17
|
+
import { create } from "@bufbuild/protobuf";
|
|
18
|
+
import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
|
|
19
|
+
import { AgentInstanceSchema } from "@stigmer/protos/ai/stigmer/agentic/agentinstance/v1/api_pb";
|
|
20
|
+
import { SessionSchema } from "@stigmer/protos/ai/stigmer/agentic/session/v1/api_pb";
|
|
13
21
|
import type { ApiResourceReference } from "@stigmer/protos/ai/stigmer/commons/apiresource/io_pb";
|
|
14
|
-
import {
|
|
22
|
+
import { mockStigmerClient } from "../../__test-utils__/mock-client.js";
|
|
23
|
+
import { mergeSkillRefs, resolveBlueprint } from "../blueprint-resolver.js";
|
|
24
|
+
|
|
25
|
+
describe("resolveBlueprint", () => {
|
|
26
|
+
it("walks session -> instance -> agent and merges with the session's tools", async () => {
|
|
27
|
+
const client = mockStigmerClient({
|
|
28
|
+
getAgentInstance: vi.fn().mockResolvedValue(
|
|
29
|
+
create(AgentInstanceSchema, { metadata: { id: "agi_1" }, spec: { agentId: "agt_1" } }),
|
|
30
|
+
),
|
|
31
|
+
getAgent: vi.fn().mockResolvedValue(
|
|
32
|
+
create(AgentSchema, {
|
|
33
|
+
metadata: { id: "agt_1", name: "builder" },
|
|
34
|
+
spec: {
|
|
35
|
+
instructions: "You build things.",
|
|
36
|
+
mcpServerUsages: [{ mcpServerRef: { org: "acme", slug: "github" } }],
|
|
37
|
+
skillRefs: [{ org: "acme", slug: "release" }],
|
|
38
|
+
},
|
|
39
|
+
}),
|
|
40
|
+
),
|
|
41
|
+
});
|
|
42
|
+
const session = create(SessionSchema, {
|
|
43
|
+
metadata: { id: "ses_1", org: "acme" },
|
|
44
|
+
spec: {
|
|
45
|
+
agentInstanceId: "agi_1",
|
|
46
|
+
mcpServerUsages: [{ mcpServerRef: { org: "acme", slug: "notes" } }],
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const blueprint = await resolveBlueprint(client, session);
|
|
51
|
+
|
|
52
|
+
expect(blueprint.agent?.metadata?.id).toBe("agt_1");
|
|
53
|
+
expect(blueprint.instructions).toBe("You build things.");
|
|
54
|
+
expect(blueprint.mergedMcpServerUsages.map((u) => u.mcpServerRef?.slug).sort()).toEqual(["github", "notes"]);
|
|
55
|
+
expect(blueprint.mergedSkillRefs.map((r) => r.slug)).toEqual(["release"]);
|
|
56
|
+
expect(client.getAgentInstance).toHaveBeenCalledWith("agi_1");
|
|
57
|
+
expect(client.getAgent).toHaveBeenCalledWith("agt_1");
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("answers the built-in assistant for a session with no instance: no agent read, the session's tools alone", async () => {
|
|
61
|
+
const client = mockStigmerClient({
|
|
62
|
+
getAgentInstance: vi.fn().mockRejectedValue(new Error("must not be reached")),
|
|
63
|
+
getAgent: vi.fn().mockRejectedValue(new Error("must not be reached")),
|
|
64
|
+
});
|
|
65
|
+
const session = create(SessionSchema, {
|
|
66
|
+
metadata: { id: "ses_2", org: "acme" },
|
|
67
|
+
spec: {
|
|
68
|
+
agentInstanceId: "",
|
|
69
|
+
mcpServerUsages: [{ mcpServerRef: { org: "acme", slug: "notes" } }],
|
|
70
|
+
skillRefs: [{ org: "acme", slug: "writing" }],
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
const blueprint = await resolveBlueprint(client, session);
|
|
75
|
+
|
|
76
|
+
expect(blueprint.agent).toBeUndefined();
|
|
77
|
+
expect(blueprint.instructions).toBe("");
|
|
78
|
+
expect(blueprint.subAgents).toEqual([]);
|
|
79
|
+
expect(blueprint.mergedMcpServerUsages.map((u) => u.mcpServerRef?.slug)).toEqual(["notes"]);
|
|
80
|
+
expect(blueprint.mergedSkillRefs.map((r) => r.slug)).toEqual(["writing"]);
|
|
81
|
+
expect(client.getAgentInstance).not.toHaveBeenCalled();
|
|
82
|
+
expect(client.getAgent).not.toHaveBeenCalled();
|
|
83
|
+
});
|
|
84
|
+
});
|
|
15
85
|
|
|
16
86
|
function makeRef(slug: string, org = "test-org"): ApiResourceReference {
|
|
17
87
|
return { slug, org, $typeName: "ai.stigmer.commons.apiresource.ApiResourceReference" } as ApiResourceReference;
|
|
@@ -3,11 +3,13 @@
|
|
|
3
3
|
*
|
|
4
4
|
* The warm-agent-surfaces baseline report derives its waterfall from these
|
|
5
5
|
* emissions, so the invariants pinned here are load-bearing for a spend
|
|
6
|
-
* decision: segments must partition the timeline (total == sum, no
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
6
|
+
* decision: `mark` segments must partition the timeline (total == sum, no
|
|
7
|
+
* gaps), `span` segments may overlap and never move the mark cursor (so the
|
|
8
|
+
* two idioms compose on one recorder), the emitted line must keep its stable
|
|
9
|
+
* shape (`stigmer_timing` selector, context spread, `segments` with
|
|
10
|
+
* per-segment `start_ms`/`duration_ms`), and the process-boot timeline must
|
|
11
|
+
* emit exactly once — a worker restart inside one process must never produce
|
|
12
|
+
* a bogus second timeline.
|
|
11
13
|
*
|
|
12
14
|
* The boot singleton is module-level state, so those tests re-import the
|
|
13
15
|
* module fresh via `vi.resetModules()` to get an unlatched instance.
|
|
@@ -101,6 +103,52 @@ describe("TimingRecorder", () => {
|
|
|
101
103
|
});
|
|
102
104
|
});
|
|
103
105
|
|
|
106
|
+
describe("TimingRecorder.span", () => {
|
|
107
|
+
// Explicit bounds are absolute, like the origin; offsets come out relative.
|
|
108
|
+
it("records a segment with explicit bounds, relative to the origin", () => {
|
|
109
|
+
const origin = 1_000;
|
|
110
|
+
const recorder = new TimingRecorder(origin);
|
|
111
|
+
recorder.span("model_round", origin + 100, origin + 350.26);
|
|
112
|
+
|
|
113
|
+
expect(recorder.snapshot()).toEqual([{ name: "model_round", startMs: 100, durationMs: 250.3 }]);
|
|
114
|
+
expect(recorder.totalMs()).toBe(350.3);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it("lets spans overlap: the timeline is not partitioned and total is the latest end", () => {
|
|
118
|
+
const origin = 0;
|
|
119
|
+
const recorder = new TimingRecorder(origin);
|
|
120
|
+
recorder.span("model_round", 10, 500);
|
|
121
|
+
recorder.span("tool:read_file", 200, 300);
|
|
122
|
+
recorder.span("sub_agent:researcher", 250, 900);
|
|
123
|
+
recorder.span("model_round", 600, 700);
|
|
124
|
+
|
|
125
|
+
const segments = recorder.snapshot();
|
|
126
|
+
expect(segments.map((s) => s.name)).toEqual(["model_round", "tool:read_file", "sub_agent:researcher", "model_round"]);
|
|
127
|
+
// The tool sits inside the first round; the sub-agent crosses both rounds.
|
|
128
|
+
expect(segments[1]!.startMs).toBeGreaterThan(segments[0]!.startMs);
|
|
129
|
+
expect(segments[1]!.startMs + segments[1]!.durationMs).toBeLessThan(segments[0]!.startMs + segments[0]!.durationMs);
|
|
130
|
+
// Not the last span's end, and not the sum: the segment that ends latest.
|
|
131
|
+
expect(recorder.totalMs()).toBe(900);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it("never moves the mark cursor: a mark after a span still measures from the previous mark", () => {
|
|
135
|
+
const origin = performance.now();
|
|
136
|
+
const recorder = new TimingRecorder(origin);
|
|
137
|
+
recorder.span("model_round", origin + 100, origin + 250);
|
|
138
|
+
recorder.mark("after");
|
|
139
|
+
|
|
140
|
+
const [span, mark] = recorder.snapshot();
|
|
141
|
+
expect(span!.startMs).toBe(100);
|
|
142
|
+
// The mark closes from the ORIGIN (the previous mark cursor), not from
|
|
143
|
+
// the span's end: a span is a fact placed on the timeline, not progress
|
|
144
|
+
// of the sequential chain.
|
|
145
|
+
expect(mark!.startMs).toBe(0);
|
|
146
|
+
expect(mark!.durationMs).toBeLessThan(100);
|
|
147
|
+
// The span still ends latest, so it is the total.
|
|
148
|
+
expect(recorder.totalMs()).toBe(250);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
|
|
104
152
|
describe("emitTimingLog", () => {
|
|
105
153
|
it("emits one line with the stable shape the baseline report parses", () => {
|
|
106
154
|
const spy = vi.spyOn(console, "log").mockImplementation(() => {});
|
|
@@ -6,6 +6,14 @@
|
|
|
6
6
|
* mcp_server_usages. Also merges agent-level and session-level MCP usages
|
|
7
7
|
* and skill refs following the same merge semantics.
|
|
8
8
|
*
|
|
9
|
+
* A session with an EMPTY agent_instance_id is the built-in assistant
|
|
10
|
+
* (session/v1/spec.proto): the chain stops at the session, `agent` is
|
|
11
|
+
* undefined, the instructions are empty (each harness's prompt builder
|
|
12
|
+
* substitutes the one built-in prompt, shared/builtin-assistant-prompt.ts),
|
|
13
|
+
* there are no sub-agents, and the session's own MCP usages and skill refs
|
|
14
|
+
* are the whole tool set — the same merge with an empty agent side, so no
|
|
15
|
+
* consumer downstream needs an agent-less arm of its own.
|
|
16
|
+
*
|
|
9
17
|
* Workspace isolation: resolved workspace directories are validated to
|
|
10
18
|
* ensure they never point at the runner's own app directory. Paths
|
|
11
19
|
* containing runner-internal markers are rejected with a warning.
|
|
@@ -32,8 +40,8 @@ export interface CloudRepo {
|
|
|
32
40
|
}
|
|
33
41
|
|
|
34
42
|
export interface ResolvedBlueprint {
|
|
35
|
-
agent
|
|
36
|
-
|
|
43
|
+
/** The agent the session is bound to; undefined for the built-in assistant. */
|
|
44
|
+
agent: Agent | undefined;
|
|
37
45
|
session: Session;
|
|
38
46
|
sessionSpec: SessionSpec;
|
|
39
47
|
instructions: string;
|
|
@@ -47,7 +55,9 @@ export interface ResolvedBlueprint {
|
|
|
47
55
|
* Resolve the full agent blueprint from the execution chain.
|
|
48
56
|
*
|
|
49
57
|
* Chain: execution.spec.sessionId -> session.spec.agentInstanceId ->
|
|
50
|
-
* agentInstance.spec.agentId -> agent (with full spec)
|
|
58
|
+
* agentInstance.spec.agentId -> agent (with full spec); an empty
|
|
59
|
+
* agent_instance_id ends the chain at the session (the built-in
|
|
60
|
+
* assistant).
|
|
51
61
|
*/
|
|
52
62
|
export async function resolveBlueprint(
|
|
53
63
|
client: StigmerClient,
|
|
@@ -55,18 +65,22 @@ export async function resolveBlueprint(
|
|
|
55
65
|
): Promise<ResolvedBlueprint> {
|
|
56
66
|
const sessionSpec = session.spec!;
|
|
57
67
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
68
|
+
let agent: Agent | undefined;
|
|
69
|
+
let agentSpec: AgentSpec | undefined;
|
|
70
|
+
if (sessionSpec.agentInstanceId !== "") {
|
|
71
|
+
const agentInstance = await client.getAgentInstance(sessionSpec.agentInstanceId);
|
|
72
|
+
const agentId = agentInstance.spec!.agentId;
|
|
73
|
+
agent = await client.getAgent(agentId);
|
|
74
|
+
agentSpec = agent.spec!;
|
|
75
|
+
}
|
|
62
76
|
|
|
63
77
|
const mergedMcpServerUsages = mergeMcpServerUsages(
|
|
64
|
-
agentSpec
|
|
78
|
+
agentSpec?.mcpServerUsages ?? [],
|
|
65
79
|
sessionSpec.mcpServerUsages,
|
|
66
80
|
);
|
|
67
81
|
|
|
68
82
|
const mergedSkillRefs = mergeSkillRefs(
|
|
69
|
-
agentSpec
|
|
83
|
+
agentSpec?.skillRefs ?? [],
|
|
70
84
|
sessionSpec.skillRefs,
|
|
71
85
|
);
|
|
72
86
|
|
|
@@ -74,11 +88,10 @@ export async function resolveBlueprint(
|
|
|
74
88
|
|
|
75
89
|
return {
|
|
76
90
|
agent,
|
|
77
|
-
agentSpec,
|
|
78
91
|
session,
|
|
79
92
|
sessionSpec,
|
|
80
|
-
instructions: agentSpec
|
|
81
|
-
subAgents: agentSpec
|
|
93
|
+
instructions: agentSpec?.instructions ?? "",
|
|
94
|
+
subAgents: agentSpec?.subAgents ?? [],
|
|
82
95
|
mergedMcpServerUsages,
|
|
83
96
|
mergedSkillRefs,
|
|
84
97
|
cloudRepos,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The built-in assistant's instructions: the ONE text a run reads when its
|
|
3
|
+
* blueprint carries no instructions — because the session names no agent
|
|
4
|
+
* (session/v1/spec.proto, an empty agent_instance_id) or because the agent
|
|
5
|
+
* it names left its instructions empty. Both harnesses' prompt builders
|
|
6
|
+
* substitute these words on that arm and frame them their own way (the
|
|
7
|
+
* `prompt-sections.ts` doctrine: the framing is house style, the words are
|
|
8
|
+
* shared), so a person who picks no agent meets the same assistant on the
|
|
9
|
+
* native and the Cursor harness, and a maintainer changes the assistant in
|
|
10
|
+
* one place.
|
|
11
|
+
*
|
|
12
|
+
* A prompt is code: versioned here, pinned by the prompt goldens of both
|
|
13
|
+
* harnesses (`execute-deep-agent/__tests__/goldens/system-prompt.minimal.prompt.md`,
|
|
14
|
+
* `execute-cursor/__tests__/goldens/*no-instructions*`). There is no stored
|
|
15
|
+
* "default agent" row anywhere whose instructions could drift from these.
|
|
16
|
+
*
|
|
17
|
+
* What the words leave to the builders on purpose: the response rules, the
|
|
18
|
+
* sub-agent rules, the skills protocol and every other section quote an
|
|
19
|
+
* engine's tools and stay per harness. These words say who the assistant
|
|
20
|
+
* is and how it works with a person; nothing here names a tool.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The display name every surface uses for the built-in assistant. */
|
|
24
|
+
export const BUILT_IN_ASSISTANT_NAME = "Assistant";
|
|
25
|
+
|
|
26
|
+
export const BUILT_IN_ASSISTANT_INSTRUCTIONS = [
|
|
27
|
+
"You are Stigmer's assistant. Help with whatever the person brings: answer, reason, write and edit, and use the tools and skills attached to this conversation when they help.",
|
|
28
|
+
"Be direct and concrete. Say what you do not know. Ask when a request is ambiguous. Prefer changing what exists over creating anew.",
|
|
29
|
+
].join("\n");
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The instructions a run reads: the blueprint's own when it has any, the
|
|
33
|
+
* built-in assistant's otherwise. Both prompt builders call this on their
|
|
34
|
+
* instructions arm so neither carries a second copy of the fallback.
|
|
35
|
+
*/
|
|
36
|
+
export function effectiveInstructions(instructions: string): string {
|
|
37
|
+
return instructions === "" ? BUILT_IN_ASSISTANT_INSTRUCTIONS : instructions;
|
|
38
|
+
}
|
|
@@ -31,7 +31,7 @@ export interface TimingSegment {
|
|
|
31
31
|
}
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
|
-
* Records
|
|
34
|
+
* Records named segments on one timeline, two ways.
|
|
35
35
|
*
|
|
36
36
|
* `mark(name)` closes the span from the previous mark (or the origin) and
|
|
37
37
|
* names it — so instrumentation reads as one line placed AFTER the work it
|
|
@@ -42,11 +42,22 @@ export interface TimingSegment {
|
|
|
42
42
|
* const execution = await client.getExecution(executionId);
|
|
43
43
|
* timing.mark("fetch_execution");
|
|
44
44
|
* ```
|
|
45
|
+
*
|
|
46
|
+
* `span(name, startMs, endMs)` records a segment with explicit bounds, for a
|
|
47
|
+
* timeline whose phases OVERLAP (a tool call inside a model round, a
|
|
48
|
+
* sub-agent across two rounds) and so cannot be written as a chain of marks.
|
|
49
|
+
* A span never moves the mark cursor: a `mark` after a `span` still measures
|
|
50
|
+
* from the previous MARK, so the two idioms compose on one recorder without
|
|
51
|
+
* either changing what the other means. `totalMs()` is the latest end on the
|
|
52
|
+
* timeline, whichever way it was recorded — for a mark-only recorder that is
|
|
53
|
+
* the last mark, as it always was.
|
|
45
54
|
*/
|
|
46
55
|
export class TimingRecorder {
|
|
47
56
|
private readonly originMs: number;
|
|
48
57
|
private readonly segments: TimingSegment[] = [];
|
|
49
58
|
private lastMarkMs: number;
|
|
59
|
+
/** Absolute end of the segment that ends latest, by either idiom. */
|
|
60
|
+
private latestEndMs: number;
|
|
50
61
|
|
|
51
62
|
/**
|
|
52
63
|
* @param originMs absolute `performance.now()`-scale origin. Defaults to
|
|
@@ -57,6 +68,7 @@ export class TimingRecorder {
|
|
|
57
68
|
constructor(originMs: number = performance.now()) {
|
|
58
69
|
this.originMs = originMs;
|
|
59
70
|
this.lastMarkMs = originMs;
|
|
71
|
+
this.latestEndMs = originMs;
|
|
60
72
|
}
|
|
61
73
|
|
|
62
74
|
/** Close the span since the previous mark (or origin) and name it. */
|
|
@@ -68,11 +80,25 @@ export class TimingRecorder {
|
|
|
68
80
|
durationMs: round1(now - this.lastMarkMs),
|
|
69
81
|
});
|
|
70
82
|
this.lastMarkMs = now;
|
|
83
|
+
if (now > this.latestEndMs) this.latestEndMs = now;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Record a segment with explicit absolute bounds (`performance.now()`
|
|
88
|
+
* scale, like the origin). Leaves the mark cursor where it was.
|
|
89
|
+
*/
|
|
90
|
+
span(name: string, startMs: number, endMs: number): void {
|
|
91
|
+
this.segments.push({
|
|
92
|
+
name,
|
|
93
|
+
startMs: round1(startMs - this.originMs),
|
|
94
|
+
durationMs: round1(endMs - startMs),
|
|
95
|
+
});
|
|
96
|
+
if (endMs > this.latestEndMs) this.latestEndMs = endMs;
|
|
71
97
|
}
|
|
72
98
|
|
|
73
|
-
/**
|
|
99
|
+
/** Elapsed time from the origin to the latest end on the timeline (ms). */
|
|
74
100
|
totalMs(): number {
|
|
75
|
-
return round1(this.
|
|
101
|
+
return round1(this.latestEndMs - this.originMs);
|
|
76
102
|
}
|
|
77
103
|
|
|
78
104
|
snapshot(): readonly TimingSegment[] {
|