@llm4ts/runner 2.22.1 → 2.23.0

package/src/Profile.ts

@@ -47,10 +47,22 @@ export const TurnStats = Schema.Struct({
   lastPrompt: Schema.optionalKey(Schema.Int),
   maxPrompt: Schema.optionalKey(Schema.Int),
   /** The last prompt is more than three times the first. */
-  promptGrowth: Schema.Boolean
+  promptGrowth: Schema.Boolean,
+  /** Tool calls per finished turn: each is a model round trip. */
+  avgSteps: Schema.Number,
+  /** The model's own time per round trip: a turn's time less its tools, over its steps + 1. */
+  avgStepMs: Ms
 })
 export type TurnStats = typeof TurnStats.Type
 
+/** A stretch of a story's time no `Timed` event covers, named by the events around it. */
+export const Gap = Schema.Struct({
+  ms: Ms,
+  after: Schema.String,
+  before: Schema.String
+})
+export type Gap = typeof Gap.Type
+
 export const StoryProfile = Schema.Struct({
   id: Schema.String,
   executor: Schema.optionalKey(Schema.String),
@@ -59,7 +71,15 @@ export const StoryProfile = Schema.Struct({
   /** From the plan being ready to this story starting: dependencies and free coder slots. */
   queuedMs: Schema.optionalKey(Ms),
   time: ProfileTime,
-  turns: TurnStats
+  turns: TurnStats,
+  /** Its largest untimed stretches, 30 seconds or more: where "unaccounted" sits. */
+  gaps: Schema.Array(Gap),
+  /**
+   * A coder turn the trace shows working (tool calls after its last finished
+   * call) but not ended: a run still going. Counted as model time so far.
+   */
+  openTurnMs: Ms,
+  openTurnTools: Schema.Int
 })
 export type StoryProfile = typeof StoryProfile.Type
 
@@ -102,7 +122,11 @@ export const ProfileReport = Schema.Struct({
   models: Schema.Array(ModelRow),
   gates: Schema.Array(GateRow),
   tools: Schema.Array(CountRow),
+  /** Coder tool time by kind of work: explore, edit, test, build, install, git, other. */
+  toolCategories: Schema.Array(CountRow),
   waits: Schema.Array(CountRow),
+  /** Timed on the run's events, in no story: the run's own planning, or a seat no story owns. */
+  outside: Schema.Array(CountRow),
   findings: Schema.Array(Finding)
 })
 export type ProfileReport = typeof ProfileReport.Type
@@ -128,6 +152,11 @@ interface Lane {
   /** The last turn boundary: any lane event but a tool call inside a turn. */
   last: number
   skipped: boolean
+  /** Every event on the lane, described without its content, for the gaps. */
+  readonly moments: Array<{ readonly at: number; readonly what: string }>
+  /** Since when a coder turn has been working without ending, and its tool calls. */
+  openSince: number | undefined
+  openTools: number
 }
 
 interface Lease {
@@ -166,6 +195,93 @@ const groupBy = <A>(items: ReadonlyArray<A>, key: (item: A) => string): Map<stri
 const laneOf = (event: FlowEvent): string | undefined =>
   "lane" in event && typeof event.lane === "string" ? event.lane : undefined
 
+/** An event as the gaps name it: its kind, never its content. */
+const describe = (event: FlowEvent): string => {
+  switch (event._tag) {
+    case "StageStarted":
+      return "a stage start"
+    case "StageCompleted":
+      return "a stage end"
+    case "StageFailed":
+      return "a stage failure"
+    case "ToolUse":
+      return "a coder tool call"
+    case "TokensUsed":
+      return "a token report"
+    case "Info":
+      return "a log note"
+    case "ReviewFindings":
+      return "a review round"
+    case "StoryJudged":
+      return "a judge verdict"
+    case "AssistantMessage":
+      return "an assistant message"
+    case "CapabilityUsed":
+    case "CapabilityDenied":
+    case "CapabilityUnenforceable":
+      return "a capability check"
+    case "ExecutorLeased":
+    case "ExecutorReleased":
+    case "ExecutorExcluded":
+    case "ExecutorResumed":
+    case "ExecutorHandedOver":
+      return "a roster change"
+    case "Timed":
+      switch (event.kind) {
+        case "model":
+          return `the end of a ${event.label} call`
+        case "tool":
+          return `the end of tool ${event.label}`
+        case "gate":
+          return `the end of gate ${event.label}`
+        case "wait":
+          return `the end of a wait for the ${event.label}`
+        default:
+          return `the end of ${event.label}`
+      }
+    default:
+      return "an event"
+  }
+}
+
+const gapThresholdMs = 30_000
+
+/** The parts of each step between two lane events that no timed interval covers, largest first. */
+const gapsOf = (lane: Lane): ReadonlyArray<Gap> => {
+  const covered = [
+    ...lane.timed.map((item) => [item.at - item.event.ms, item.at] as const),
+    // A turn still open is model time, not a gap.
+    ...(lane.openSince === undefined || lane.end !== undefined
+      ? []
+      : [[lane.openSince, Number.POSITIVE_INFINITY] as const])
+  ].sort((left, right) => left[0] - right[0])
+  const uncovered = (from: number, to: number): number => {
+    let free = 0
+    let cursor = from
+    for (const [start, end] of covered) {
+      if (end <= cursor || start >= to) {
+        continue
+      }
+      free += Math.max(0, start - cursor)
+      cursor = Math.max(cursor, end)
+    }
+    return free + Math.max(0, to - cursor)
+  }
+  const moments = [...lane.moments].sort((left, right) => left.at - right.at)
+  return moments
+    .slice(1)
+    .flatMap((next, index) => {
+      const previous = moments[index]
+      if (previous === undefined) {
+        return []
+      }
+      const ms = uncovered(previous.at, next.at)
+      return ms >= gapThresholdMs ? [{ ms, after: previous.what, before: next.what }] : []
+    })
+    .sort((left, right) => right.ms - left.ms)
+    .slice(0, 3)
+}
+
 /** The roster role a seat label stands for, to find its lease. */
 const leaseRole = (label: string): string => (label === "reasoning" ? "reviewer" : label)
 
@@ -200,7 +316,10 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
             prompts: [],
             estimatedCalls: [],
             last: at,
-            skipped: false
+            skipped: false,
+            moments: [{ at, what: describe(event) }],
+            openSince: undefined,
+            openTools: 0
           })
           continue
         }
@@ -219,8 +338,20 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
       case "Timed": {
         allTimed.push({ at, event })
         lane?.timed.push({ at, event })
+        if (lane !== undefined && event.kind === "model" && event.label === "coder") {
+          lane.openSince = undefined
+          lane.openTools = 0
+        }
         break
       }
+      case "ToolUse":
+        if (lane !== undefined) {
+          // A tool call opens a turn until a coder call ends: it began at
+          // the last turn boundary.
+          lane.openSince = lane.openSince ?? lane.last
+          lane.openTools += 1
+        }
+        break
       case "TokensUsed":
         if (lane !== undefined) {
           if (event.agent === "coder") {
@@ -241,6 +372,7 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
         break
     }
     if (lane !== undefined) {
+      lane.moments.push({ at, what: describe(event) })
       lane.executor = lane.executor ?? ("executor" in event ? event.executor : undefined)
       if (!withinTurn.has(event._tag)) {
         lane.last = at
@@ -324,7 +456,30 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
       ms: sum(items.map((item) => item.event.ms))
     }))
   const tools = counted("tool")
+  const toolCategories = [
+    ...groupBy(
+      allTimed.filter((item) => item.event.kind === "tool" && item.event.category !== undefined),
+      (item) => item.event.category ?? ""
+    ).entries()
+  ]
+    .map(([label, items]) => ({
+      label,
+      count: items.length,
+      ms: sum(items.map((item) => item.event.ms))
+    }))
+    .sort((left, right) => right.ms - left.ms)
   const waits = counted("wait")
+  const steps = stepStats(estimated ? [] : [...lanes.values()].flatMap(turnSteps))
+  const outside = [
+    ...groupBy(
+      allTimed.filter((item) => item.event.lane === undefined),
+      (item) => item.event.label
+    ).entries()
+  ].map(([label, items]) => ({
+    label,
+    count: items.length,
+    ms: sum(items.map((item) => item.event.ms))
+  }))
   const merges = [...counted("merge"), ...counted("git")]
 
   const totals = stories.reduce<ProfileTime>(
@@ -349,11 +504,49 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
     models,
     gates,
     tools,
+    toolCategories,
     waits,
-    findings: findingsOf({ stories, models, gates, tools, waits, merges, storyTime, estimated })
+    outside,
+    findings: findingsOf({
+      steps,
+      toolCategories,
+      stories,
+      models,
+      gates,
+      tools,
+      waits,
+      merges,
+      storyTime,
+      estimated
+    })
   }
 }
 
+const stepStats = (
+  turns: ReadonlyArray<{ readonly steps: number; readonly stepMs: number }>
+): { readonly avgSteps: number; readonly avgStepMs: number } => ({
+  avgSteps: average(turns.map((turn) => turn.steps)) ?? 0,
+  avgStepMs: average(turns.map((turn) => turn.stepMs)) ?? 0
+})
+
+/** Each finished coder turn's steps (its tool calls) and model time per step. */
+const turnSteps = (
+  lane: Lane
+): ReadonlyArray<{ readonly steps: number; readonly stepMs: number }> => {
+  const tools = lane.timed.filter((item) => item.event.kind === "tool")
+  return lane.timed
+    .filter((item) => item.event.kind === "model" && item.event.label === "coder")
+    .map((turn) => {
+      const start = turn.at - turn.event.ms
+      const inside = tools.filter((tool) => tool.at > start && tool.at <= turn.at)
+      const toolMs = sum(inside.map((tool) => tool.event.ms))
+      return {
+        steps: inside.length,
+        stepMs: Math.max(0, turn.event.ms - toolMs) / (inside.length + 1)
+      }
+    })
+}
+
 const storyProfile = (
   lane: Lane,
   lastAt: number,
@@ -367,7 +560,11 @@ const storyProfile = (
     .filter((item) => item.event.kind === "model" && item.event.label === "coder")
     .map((item) => item.event.ms)
   const tools = of("tool")
-  const model = estimated ? sum(lane.estimatedCalls) : Math.max(0, of("model") - tools)
+  // Only a story still running has a turn in progress; in a finished one an
+  // unclosed turn is a timing that never came, left to the gaps.
+  const open = !estimated && lane.end === undefined && lane.openSince !== undefined
+  const openTurnMs = open && lane.openSince !== undefined ? lastAt - lane.openSince : 0
+  const model = estimated ? sum(lane.estimatedCalls) : Math.max(0, of("model") + openTurnMs - tools)
   const gates = of("gate")
   const merge = of("merge") + of("git")
   const waiting = of("wait")
@@ -376,6 +573,9 @@ const storyProfile = (
   const lastPrompt = lane.prompts.at(-1)
   const maxPrompt = lane.prompts.length === 0 ? undefined : Math.max(...lane.prompts)
   return {
+    gaps: estimated ? [] : gapsOf(lane),
+    openTurnMs,
+    openTurnTools: open ? lane.openTools : 0,
     id: lane.id,
     ...(lane.executor === undefined ? {} : { executor: lane.executor }),
     status: lane.status,
@@ -396,6 +596,7 @@ const storyProfile = (
       ...(firstPrompt === undefined ? {} : { firstPrompt }),
       ...(lastPrompt === undefined ? {} : { lastPrompt }),
       ...(maxPrompt === undefined ? {} : { maxPrompt }),
+      ...stepStats(estimated ? [] : turnSteps(lane)),
       promptGrowth:
         firstPrompt !== undefined &&
         lastPrompt !== undefined &&
@@ -420,11 +621,16 @@ export const duration = (ms: number): string => {
   return `${Math.floor(minutes / 60)}h${String(minutes % 60).padStart(2, "0")}m`
 }
 
+const stepsText = (steps: number): string =>
+  `${Number.isInteger(steps) ? steps : steps.toFixed(1)} step${steps === 1 ? "" : "s"}`
+
 const share = (ms: number, of: number): string => (of <= 0 ? "" : `${Math.round((ms / of) * 100)}%`)
 
 const plural = (count: number, word: string): string => `${count} ${word}${count === 1 ? "" : "s"}`
 
 const findingsOf = (facts: {
+  readonly steps: { readonly avgSteps: number; readonly avgStepMs: number }
+  readonly toolCategories: ReadonlyArray<CountRow>
   readonly stories: ReadonlyArray<StoryProfile>
   readonly models: ReadonlyArray<ModelRow>
   readonly gates: ReadonlyArray<GateRow>
@@ -445,7 +651,11 @@ const findingsOf = (facts: {
       ms,
       text:
         role === "coder"
-          ? `coder turns: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}`
+          ? `coder turns: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}${
+              facts.steps.avgSteps > 0
+                ? `, ${stepsText(facts.steps.avgSteps)} per turn, ${duration(facts.steps.avgStepMs)} of model per step`
+                : ""
+            }`
           : `${role} calls: ${calls}, ${ofStories(ms)}, avg ${duration(ms / calls)}${first}`
     }
   })
@@ -457,10 +667,16 @@ const findingsOf = (facts: {
         row.failed === 0 ? "" : `, ${row.failed} failed`
       }`
     })),
-    ...facts.tools.map((row) => ({
-      ms: row.ms,
-      text: `coder tool \`${row.label}\`: ${plural(row.count, "call")}, ${ofStories(row.ms)}`
-    })),
+    // By kind of work when the trace has it (2.23), else by tool name.
+    ...(facts.toolCategories.length > 0
+      ? facts.toolCategories.map((row) => ({
+          ms: row.ms,
+          text: `coder tools, ${row.label}: ${plural(row.count, "call")}, ${ofStories(row.ms)}`
+        }))
+      : facts.tools.map((row) => ({
+          ms: row.ms,
+          text: `coder tool \`${row.label}\`: ${plural(row.count, "call")}, ${ofStories(row.ms)}`
+        }))),
     ...facts.waits.map((row) => ({
       ms: row.ms,
       text: `waiting for the ${row.label}: ${row.count}×, ${ofStories(row.ms)}`
@@ -470,6 +686,14 @@ const findingsOf = (facts: {
       text: `${row.label}: ${row.count}×, ${ofStories(row.ms)}`
     }))
   ]
+  for (const story of facts.stories) {
+    if (story.openTurnMs >= 1_000) {
+      candidates.push({
+        ms: story.openTurnMs,
+        text: `${story.id}: a coder turn still running for ${duration(story.openTurnMs)} (${plural(story.openTurnTools, "tool call")} so far)`
+      })
+    }
+  }
   const queued = sum(facts.stories.map((story) => story.queuedMs ?? 0))
   if (queued >= 1_000) {
     candidates.push({
@@ -519,7 +743,7 @@ export const renderProfile = (report: ProfileReport): string => {
     [report.estimated ? "model + tools + gates (estimated)" : "model", report.totals.model],
     ["coder tools", report.totals.tools],
     ["gates", report.totals.gates],
-    ["merge", report.totals.merge],
+    ["git + merge", report.totals.merge],
     ["waiting", report.totals.waiting],
     ["unaccounted", report.totals.unaccounted]
   ]
@@ -551,11 +775,13 @@ export const renderProfile = (report: ProfileReport): string => {
         "model",
         "tools",
         "gates",
-        "merge",
+        "git+merge",
         "wait",
         "other",
         "turns",
         "avg turn",
+        "steps/turn",
+        "model/step",
         "prompt ×"
       ],
       report.stories.map((story) => [
@@ -571,6 +797,8 @@ export const renderProfile = (report: ProfileReport): string => {
         duration(story.time.unaccounted),
         String(story.turns.count),
         duration(story.turns.avgMs),
+        story.turns.avgSteps === 0 ? "" : story.turns.avgSteps.toFixed(1),
+        story.turns.avgSteps === 0 ? "" : duration(story.turns.avgStepMs),
         story.turns.firstPrompt !== undefined &&
         story.turns.lastPrompt !== undefined &&
         story.turns.firstPrompt > 0
@@ -581,6 +809,27 @@ export const renderProfile = (report: ProfileReport): string => {
     ...(report.skipped.length === 0
       ? []
       : [`  skipped (merged on an earlier run): ${report.skipped.join(", ")}`]),
+    ...(report.stories.every((story) => story.gaps.length === 0)
+      ? []
+      : [
+          "",
+          "Unaccounted, largest gaps (no timing covers them)",
+          ...report.stories.flatMap((story) =>
+            story.gaps.map(
+              (gap) => `  ${story.id}: ${duration(gap.ms)} between ${gap.after} and ${gap.before}`
+            )
+          )
+        ]),
+    ...(report.outside.length === 0
+      ? []
+      : [
+          "",
+          "Timed outside any story (the run's own planning, or a seat no story owns)",
+          ...table(
+            ["label", "count", "total"],
+            report.outside.map((row) => [row.label, String(row.count), duration(row.ms)])
+          )
+        ]),
     ...(report.models.length === 0
       ? []
       : [
@@ -624,6 +873,21 @@ export const renderProfile = (report: ProfileReport): string => {
               String(row.failed)
             ])
           )
+        ]),
+    ...(report.toolCategories.length === 0
+      ? []
+      : [
+          "",
+          "Coder tools by kind",
+          ...table(
+            ["kind", "calls", "total", "avg"],
+            report.toolCategories.map((row) => [
+              row.label,
+              String(row.count),
+              duration(row.ms),
+              duration(row.ms / row.count)
+            ])
+          )
         ])
   ].join("\n")
 }