@stigmer/runner 3.19.0 → 3.20.0

package/src/harness/transcript/builder.ts

@@ -82,6 +82,17 @@
  *     token delta does; the adapter's loop reads it and clears it with
  *     {@link markPersisted} as it requests the persist.
  *
+ * The runtime may hand the constructor an OBSERVER of the fold: a callback
+ * that sees every event `apply` folded, in order, after the fold, and creates
+ * nothing (the runtime's turn timeline, `harness/turn-timeline.ts`, is the
+ * one observer today). A callback at construction, not a subscribe method,
+ * because that is how the runtime tells its collaborators what to do
+ * everywhere else (`PersistChokepoint`'s options, the stall watchdog's
+ * `onStall`): one observer, fixed at the one constructor call, nothing to
+ * add or remove mid-turn. The observer runs in its own try/catch, so it can
+ * neither block a fold nor break a harness; `seed()` does not notify it,
+ * because a seeded row is last turn's fact.
+ *
  * What this builder deliberately does NOT write, and who does (the adapter
  * contract's field ownership, `harness/types.ts` `TurnSink`): the phase,
  * `startedAt`, `streamingUsage` and the epilogue's plan artifact are the
@@ -127,9 +138,13 @@ import type {
   TranscriptEvent,
 } from "./events.js";
 
+/** Sees every event the builder folded, in order, after the fold. Creates nothing. */
+export type TranscriptObserver = (event: TranscriptEvent) => void;
+
 export class TranscriptBuilder {
   readonly executionId: string;
   private readonly state: TranscriptState;
+  private readonly observer: TranscriptObserver | undefined;
   private _dirty = false;
   private _awaitingApproval = false;
 
@@ -146,10 +161,15 @@ export class TranscriptBuilder {
    * builder over — `TurnSink.status` in production, the test's own in a
    * test — so the transcript has one read handle and one write handle, and
    * no second spelling of either.
+   *
+   * `observer`, when given, is told every event after it is folded (the
+   * header's rule). The runtime passes its turn timeline; the contract kit
+   * and the test doubles pass nothing.
    */
-  constructor(executionId: string, status: AgentExecutionStatus) {
+  constructor(executionId: string, status: AgentExecutionStatus, observer?: TranscriptObserver) {
     this.executionId = executionId;
     this.state = new TranscriptState(status);
+    this.observer = observer;
   }
 
   /**
@@ -194,8 +214,25 @@ export class TranscriptBuilder {
     this._dirty = false;
   }
 
-  /** Fold one canonical event into the transcript. Never throws: a handler's error is logged and the stream goes on. */
+  /**
+   * Fold one canonical event into the transcript, then tell the observer.
+   * Never throws: a handler's error is logged and the stream goes on, and so
+   * is an observer's — each in its own catch, so a failing observer still
+   * sees a folded transcript and a failing handler still reaches the observer.
+   */
   apply(event: TranscriptEvent): void {
+    this.fold(event);
+    if (!this.observer) return;
+    try {
+      this.observer(event);
+    } catch (err) {
+      console.error(
+        `[TranscriptBuilder] Observer error: execution=${this.executionId} kind=${event.kind}: ${err}`,
+      );
+    }
+  }
+
+  private fold(event: TranscriptEvent): void {
     try {
       switch (event.kind) {
         case "sub_agent_started":

package/src/harness/turn-timeline.ts

@@ -0,0 +1,248 @@
+/**
+ * The turn runtime's timeline of one engine turn, folded from the transcript
+ * events every harness already emits, and written once at settle as the
+ * `turn_phases` `stigmer_timing` line.
+ *
+ * What it answers, for every harness alike and for any harness added later
+ * without a line in an adapter: when the user first saw something happen,
+ * how many model rounds and tool calls the turn took, where the sub-agents
+ * ran, and where the turn's longest silence was. It is the per-turn,
+ * CLIENT-PERCEIVED measurement from the runner's vantage, from the turn's
+ * origin (the activity's start, the same origin `execution_setup` counts
+ * from, so the two lines subtract without a join) to the adapter's return;
+ * the completion epilogue and the terminal persist that follow are the
+ * client's own end-to-end, read at `subscribe`. Its counterpart is the cloud
+ * proxy's `ProxyTiming` (`agentexecution/v1/usage.proto`): per model CALL,
+ * server-side, what the provider did. Neither restates the other; a report
+ * that wants both joins them by `execution_id`.
+ *
+ * Why a fold over the transcript events and not a mark in each adapter: the
+ * transcript builder (`transcript/builder.ts`) is the one view the runtime
+ * has of every harness's turn — model rounds, text, tools, sub-agents all
+ * reach it as a translator's output on one union — so a fold over it is
+ * written once and holds for the Cursor adapter, the native adapter and the
+ * next one. `UsageAccumulator` (`usage-accumulator.ts`) is the same shape
+ * for the usage deltas; this is that shape for the events.
+ *
+ * The fold's rules, each with an arm in `__tests__/turn-timeline.test.ts`:
+ *
+ *   - Three instants, ROOT scope only (a sub-agent's text is not what the
+ *     user is watching): `first_event_ms`, the first event of any kind;
+ *     `first_visible_token_ms`, the first `text_delta` or `reasoning_delta`
+ *     (both stream into the console as tokens); `first_text_ms`, the first
+ *     `text_delta`. The two token instants are the cross-harness axes: with
+ *     extended thinking on, the visible instant moves earlier and the text
+ *     instant later, and a comparison needs both. `first_event_ms` is
+ *     harness-relative — on native the first event is `message_start`, the
+ *     model stream opening; on Cursor it is whatever the SDK delivered first —
+ *     and is kept as a forensic (engine-ready to first model event), never as
+ *     an axis. An instant that never happened is `null`.
+ *   - One `model_round` span per `runId`, `message_start` to `message_finish`.
+ *     What a `runId` is belongs to the translator: LangGraph's LLM-call run on
+ *     native, one model round; a minted segment `<run>:<n>` on Cursor, a
+ *     stretch of assistant output between tool calls, which the SDK does not
+ *     let us tell apart from a provider round. The line's `rounds` reads the
+ *     same way.
+ *   - One `tool:<name>` span per `callId` (`tool:<slug>/<name>` for an MCP
+ *     tool, so the MCP phase the runner skill's law asks for is legible),
+ *     `tool_started` to the FIRST `tool_finished` or `tool_error`; a later
+ *     fact for a settled span changes nothing, the builder's own upsert rule
+ *     (Cursor completes every call twice). `approval_proposed` opens nothing:
+ *     a call the engine held before it ran is not a tool phase.
+ *   - One `sub_agent:<name>` span per `subAgentId`, opened by
+ *     `sub_agent_started`, closed by `finished` or `failed`.
+ *   - A fact that carries `observedAt` is placed at that instant, everything
+ *     else at the fold's clock — the builder's rule for `completedAt`
+ *     (`transcript/events.ts`, `Observed`), restated here because the
+ *     alternative repeats the error #1097 fixed, for tool phases and on Cursor
+ *     alone: a completion the SDK's delta channel reported is folded one
+ *     stream event later, seconds while the model narrates, and a comparison
+ *     that read the fold's clock would charge those seconds to Cursor's tools.
+ *     Placing an observed instant needs a wall-clock origin beside the
+ *     `performance.now()` one; the runtime captures both together.
+ *   - `max_gap_ms`, the longest wait between two consecutive events, whatever
+ *     their kind: the turn's longest silence as the user experienced it.
+ *     `null` under two events.
+ *   - A span still open at settle closes at settle: an interrupted turn's
+ *     spans end where the turn did.
+ *
+ * The line: `emitTimingLog("turn_phases", context, recorder)` over a
+ * `TimingRecorder` on the turn's origin, every span appended in start order
+ * through `span()`. This is the first `stigmer_timing` event whose segments
+ * OVERLAP (a tool inside a round, a sub-agent across rounds); the recorder's
+ * `total_ms` is the latest end, not a sum, and a reader that summed the
+ * other timelines' segments must not sum this one's. No histogram mirrors it
+ * yet: the benchmark report is its reader and reads the log; an aggregate
+ * earns its instrument when a dashboard asks.
+ *
+ * Failure posture, the timing module's: the fold is arithmetic over the
+ * event and cannot affect a turn; `emit` is best-effort through
+ * `emitTimingLog`. Nothing here is persisted, and nothing branches on it.
+ */
+
+import { TimingRecorder, emitTimingLog } from "../shared/cold-start-timing.js";
+import type { TranscriptEvent } from "./transcript/events.js";
+
+/** The `stigmer_timing` selector of the line this module emits. */
+export const TURN_PHASES_EVENT = "turn_phases";
+
+/**
+ * The two clocks the timeline places facts on, captured together at the
+ * turn's origin. `now` is `performance.now` in production; tests inject a
+ * scripted one. `originWallMs` is the `Date.now()` of the same instant, the
+ * scale `observedAt` stamps are on.
+ */
+export interface TurnTimelineClock {
+  /** `performance.now()`-scale origin, the one `execution_setup` shares. */
+  readonly originMs: number;
+  /** `Date.now()`-scale reading of the same instant. */
+  readonly originWallMs: number;
+  readonly now: () => number;
+}
+
+/** The facts the runtime adds to the line beside the fold's own. */
+export type TurnPhasesContext = Record<string, string | number | boolean | null | undefined>;
+
+/** One span the fold has closed or is still tracking. Absolute `performance.now()` scale. */
+interface OpenSpan {
+  readonly name: string;
+  readonly startMs: number;
+  endMs: number | undefined;
+}
+
+export class TurnTimeline {
+  private readonly clock: TurnTimelineClock;
+
+  private firstEventMs: number | undefined;
+  private firstVisibleTokenMs: number | undefined;
+  private firstTextMs: number | undefined;
+  private lastEventMs: number | undefined;
+  private maxGapMs: number | undefined;
+
+  private readonly rounds = new Map<string, OpenSpan>();
+  private readonly tools = new Map<string, OpenSpan>();
+  private readonly subAgents = new Map<string, OpenSpan>();
+
+  constructor(clock: TurnTimelineClock) {
+    this.clock = clock;
+  }
+
+  /** The builder's observer: place one folded event on the timeline. */
+  observe(event: TranscriptEvent): void {
+    const at = this.clock.now();
+    this.noteEvent(at);
+
+    // Root scope only: a sub-agent's tokens stream into its own row, not the
+    // thread the user is watching.
+    if ((event.kind === "text_delta" || event.kind === "reasoning_delta") && event.subAgentId === undefined) {
+      this.firstVisibleTokenMs ??= at;
+      if (event.kind === "text_delta") this.firstTextMs ??= at;
+    }
+
+    switch (event.kind) {
+      case "message_start":
+        this.open(this.rounds, event.runId, "model_round", at);
+        return;
+      case "message_finish":
+        this.close(this.rounds, event.runId, at);
+        return;
+      case "tool_started":
+        this.open(this.tools, event.callId, toolSpanName(event.name, event.mcpServerSlug), at);
+        return;
+      case "tool_finished":
+      case "tool_error":
+        this.close(this.tools, event.callId, this.placeObserved(event.observedAt, at));
+        return;
+      case "sub_agent_started":
+        this.open(this.subAgents, event.subAgentId, `sub_agent:${event.name}`, at);
+        return;
+      case "sub_agent_finished":
+      case "sub_agent_failed":
+        this.close(this.subAgents, event.subAgentId, at);
+        return;
+      case "text_delta":
+      case "reasoning_delta":
+      case "tool_arg_delta":
+      case "tool_output_delta":
+      case "approval_proposed":
+      case "system_note":
+        return;
+      default: {
+        const exhaustive: never = event;
+        throw new Error(`unknown transcript event ${String((exhaustive as { kind: string }).kind)}`);
+      }
+    }
+  }
+
+  /**
+   * Write the line, once, at settle. Spans still open close now; the
+   * runtime's context (`execution_id`, `session_id`, `turn_seq`, `harness`,
+   * `outcome`) rides beside the fold's facts.
+   */
+  emit(context: TurnPhasesContext): void {
+    const settledAt = this.clock.now();
+    const recorder = new TimingRecorder(this.clock.originMs);
+    const spans = [...this.rounds.values(), ...this.tools.values(), ...this.subAgents.values()]
+      .sort((a, b) => a.startMs - b.startMs);
+    for (const span of spans) {
+      recorder.span(span.name, span.startMs, span.endMs ?? settledAt);
+    }
+    emitTimingLog(TURN_PHASES_EVENT, {
+      ...context,
+      first_event_ms: this.offset(this.firstEventMs),
+      first_visible_token_ms: this.offset(this.firstVisibleTokenMs),
+      first_text_ms: this.offset(this.firstTextMs),
+      rounds: this.rounds.size,
+      tool_calls: this.tools.size,
+      sub_agents: this.subAgents.size,
+      max_gap_ms: this.maxGapMs === undefined ? null : round1(this.maxGapMs),
+    }, recorder);
+  }
+
+  private noteEvent(at: number): void {
+    this.firstEventMs ??= at;
+    if (this.lastEventMs !== undefined) {
+      const gap = at - this.lastEventMs;
+      if (this.maxGapMs === undefined || gap > this.maxGapMs) this.maxGapMs = gap;
+    }
+    this.lastEventMs = at;
+  }
+
+  /** A span per identity: a re-emitted start (a resumed seeded call) keeps the first start. */
+  private open(spans: Map<string, OpenSpan>, id: string, name: string, at: number): void {
+    if (spans.has(id)) return;
+    spans.set(id, { name, startMs: at, endMs: undefined });
+  }
+
+  /** The FIRST close settles a span; a later fact for it changes nothing. A close for an unknown id is ignored. */
+  private close(spans: Map<string, OpenSpan>, id: string, at: number): void {
+    const span = spans.get(id);
+    if (!span || span.endMs !== undefined) return;
+    span.endMs = at;
+  }
+
+  /**
+   * Where a fact goes on the `performance.now()` scale: its observed instant
+   * translated through the paired origins when it carries one, else the fold's
+   * clock. An unparseable stamp falls back to the fold's clock rather than
+   * losing the fact.
+   */
+  private placeObserved(observedAt: string | undefined, foldAt: number): number {
+    if (observedAt === undefined) return foldAt;
+    const wallMs = Date.parse(observedAt);
+    if (Number.isNaN(wallMs)) return foldAt;
+    return this.clock.originMs + (wallMs - this.clock.originWallMs);
+  }
+
+  private offset(absoluteMs: number | undefined): number | null {
+    return absoluteMs === undefined ? null : round1(absoluteMs - this.clock.originMs);
+  }
+}
+
+function toolSpanName(name: string, mcpServerSlug: string): string {
+  return mcpServerSlug ? `tool:${mcpServerSlug}/${name}` : `tool:${name}`;
+}
+
+function round1(ms: number): number {
+  return Math.round(ms * 10) / 10;
+}

package/src/shared/__tests__/cold-start-timing.test.ts

@@ -3,11 +3,13 @@
  *
  * The warm-agent-surfaces baseline report derives its waterfall from these
  * emissions, so the invariants pinned here are load-bearing for a spend
- * decision: segments must partition the timeline (total == sum, no gaps),
- * the emitted line must keep its stable shape (`stigmer_timing` selector,
- * context spread, `segments` with per-segment `start_ms`/`duration_ms`),
- * and the process-boot timeline must emit exactly once — a worker restart
- * inside one process must never produce a bogus second timeline.
+ * decision: `mark` segments must partition the timeline (total == sum, no
+ * gaps), `span` segments may overlap and never move the mark cursor (so the
+ * two idioms compose on one recorder), the emitted line must keep its stable
+ * shape (`stigmer_timing` selector, context spread, `segments` with
+ * per-segment `start_ms`/`duration_ms`), and the process-boot timeline must
+ * emit exactly once — a worker restart inside one process must never produce
+ * a bogus second timeline.
  *
  * The boot singleton is module-level state, so those tests re-import the
  * module fresh via `vi.resetModules()` to get an unlatched instance.
@@ -101,6 +103,52 @@ describe("TimingRecorder", () => {
   });
 });
 
+describe("TimingRecorder.span", () => {
+  // Explicit bounds are absolute, like the origin; offsets come out relative.
+  it("records a segment with explicit bounds, relative to the origin", () => {
+    const origin = 1_000;
+    const recorder = new TimingRecorder(origin);
+    recorder.span("model_round", origin + 100, origin + 350.26);
+
+    expect(recorder.snapshot()).toEqual([{ name: "model_round", startMs: 100, durationMs: 250.3 }]);
+    expect(recorder.totalMs()).toBe(350.3);
+  });
+
+  it("lets spans overlap: the timeline is not partitioned and total is the latest end", () => {
+    const origin = 0;
+    const recorder = new TimingRecorder(origin);
+    recorder.span("model_round", 10, 500);
+    recorder.span("tool:read_file", 200, 300);
+    recorder.span("sub_agent:researcher", 250, 900);
+    recorder.span("model_round", 600, 700);
+
+    const segments = recorder.snapshot();
+    expect(segments.map((s) => s.name)).toEqual(["model_round", "tool:read_file", "sub_agent:researcher", "model_round"]);
+    // The tool sits inside the first round; the sub-agent crosses both rounds.
+    expect(segments[1]!.startMs).toBeGreaterThan(segments[0]!.startMs);
+    expect(segments[1]!.startMs + segments[1]!.durationMs).toBeLessThan(segments[0]!.startMs + segments[0]!.durationMs);
+    // Not the last span's end, and not the sum: the segment that ends latest.
+    expect(recorder.totalMs()).toBe(900);
+  });
+
+  it("never moves the mark cursor: a mark after a span still measures from the previous mark", () => {
+    const origin = performance.now();
+    const recorder = new TimingRecorder(origin);
+    recorder.span("model_round", origin + 100, origin + 250);
+    recorder.mark("after");
+
+    const [span, mark] = recorder.snapshot();
+    expect(span!.startMs).toBe(100);
+    // The mark closes from the ORIGIN (the previous mark cursor), not from
+    // the span's end: a span is a fact placed on the timeline, not progress
+    // of the sequential chain.
+    expect(mark!.startMs).toBe(0);
+    expect(mark!.durationMs).toBeLessThan(100);
+    // The span still ends latest, so it is the total.
+    expect(recorder.totalMs()).toBe(250);
+  });
+});
+
 describe("emitTimingLog", () => {
   it("emits one line with the stable shape the baseline report parses", () => {
     const spy = vi.spyOn(console, "log").mockImplementation(() => {});

package/src/shared/cold-start-timing.ts

@@ -31,7 +31,7 @@ export interface TimingSegment {
 }
 
 /**
- * Records a linear sequence of named segments.
+ * Records named segments on one timeline, two ways.
  *
  * `mark(name)` closes the span from the previous mark (or the origin) and
  * names it — so instrumentation reads as one line placed AFTER the work it
@@ -42,11 +42,22 @@ export interface TimingSegment {
  * const execution = await client.getExecution(executionId);
  * timing.mark("fetch_execution");
  * ```
+ *
+ * `span(name, startMs, endMs)` records a segment with explicit bounds, for a
+ * timeline whose phases OVERLAP (a tool call inside a model round, a
+ * sub-agent across two rounds) and so cannot be written as a chain of marks.
+ * A span never moves the mark cursor: a `mark` after a `span` still measures
+ * from the previous MARK, so the two idioms compose on one recorder without
+ * either changing what the other means. `totalMs()` is the latest end on the
+ * timeline, whichever way it was recorded — for a mark-only recorder that is
+ * the last mark, as it always was.
  */
 export class TimingRecorder {
   private readonly originMs: number;
   private readonly segments: TimingSegment[] = [];
   private lastMarkMs: number;
+  /** Absolute end of the segment that ends latest, by either idiom. */
+  private latestEndMs: number;
 
   /**
    * @param originMs absolute `performance.now()`-scale origin. Defaults to
@@ -57,6 +68,7 @@ export class TimingRecorder {
   constructor(originMs: number = performance.now()) {
     this.originMs = originMs;
     this.lastMarkMs = originMs;
+    this.latestEndMs = originMs;
   }
 
   /** Close the span since the previous mark (or origin) and name it. */
@@ -68,11 +80,25 @@ export class TimingRecorder {
       durationMs: round1(now - this.lastMarkMs),
     });
     this.lastMarkMs = now;
+    if (now > this.latestEndMs) this.latestEndMs = now;
+  }
+
+  /**
+   * Record a segment with explicit absolute bounds (`performance.now()`
+   * scale, like the origin). Leaves the mark cursor where it was.
+   */
+  span(name: string, startMs: number, endMs: number): void {
+    this.segments.push({
+      name,
+      startMs: round1(startMs - this.originMs),
+      durationMs: round1(endMs - startMs),
+    });
+    if (endMs > this.latestEndMs) this.latestEndMs = endMs;
   }
 
-  /** Total elapsed time from the origin to the latest mark (ms). */
+  /** Elapsed time from the origin to the latest end on the timeline (ms). */
   totalMs(): number {
-    return round1(this.lastMarkMs - this.originMs);
+    return round1(this.latestEndMs - this.originMs);
   }
 
   snapshot(): readonly TimingSegment[] {