@llm4ts/runner 2.28.1 → 2.30.0

package/src/Doctor.ts

@@ -11,6 +11,7 @@ import {
   geminiBridgePort as bridgePort,
   piModelsConfig as parsePiModels
 } from "./PiModels.ts"
+import { otelConfig, otelDoctorLine, probeOtelEndpoint, type OtelConfig } from "./Otel.ts"
 
 const credentialKeys = [
   "OPENAI_API_KEY",
@@ -220,65 +221,75 @@ export const makeDoctorProgram = (
     nodePlainFileStore,
     process.cwd(),
     environment
-  )
-): Effect.Effect<string> =>
-  Effect.map(Effect.all([registry.healthCheckAll, nodeCheck]), ([statuses, node]) => {
-    const lines: Array<string> = []
-    lines.push("llm4ts doctor")
-    lines.push("")
-    lines.push("connectors:")
-    const sorted = Array.from(statuses.entries()).sort(([a], [b]) => a.value.localeCompare(b.value))
-    for (const [id, status] of sorted) {
-      lines.push(statusLine(id.value, status))
-    }
-    lines.push("")
-    lines.push("credentials:")
-    for (const key of credentialKeys) {
-      const value = environment[key]
-      lines.push(`  ${value === undefined || value.length === 0 ? "✖" : "✔"} ${key}`)
-    }
-    lines.push("")
-    lines.push("gates:")
-    lines.push(
-      ...wrap(
-        `${node._tag === "Mismatch" || node._tag === "NoNode" ? "✖" : "✔"} ${node.summary}`,
-        78,
-        "  "
-      )
-    )
-
-    // Connector prerequisites the environment must carry BEFORE a run: shown
-    // when the connector is selected or the machine has its CLI, because that
-    // is exactly when a missing one turns into a confusing mid-run failure.
-    const coder = selectedCoder(environment)
-    const geminiRelevant =
-      coder.startsWith("gemini") ||
-      Array.from(statuses.keys()).some(
-        (id) =>
-          id.value === ConnectorIds.GeminiCli.value || id.value === ConnectorIds.GeminiApi.value
+  ),
+  /** Whether the OTLP endpoint answers (ADR 0026); the network probe by default. */
+  probeOtel: (config: OtelConfig) => Effect.Effect<boolean | undefined> = probeOtelEndpoint
+): Effect.Effect<string> => {
+  const otel = otelConfig(environment)
+  return Effect.map(
+    Effect.all([registry.healthCheckAll, nodeCheck, probeOtel(otel)]),
+    ([statuses, node, otelAnswers]) => {
+      const lines: Array<string> = []
+      lines.push("llm4ts doctor")
+      lines.push("")
+      lines.push("connectors:")
+      const sorted = Array.from(statuses.entries()).sort(([a], [b]) =>
+        a.value.localeCompare(b.value)
       )
-    const bridge = geminiBridgePrerequisites(environment, readPiModelsJson())
-    if (geminiRelevant || bridge !== undefined) {
+      for (const [id, status] of sorted) {
+        lines.push(statusLine(id.value, status))
+      }
       lines.push("")
-      lines.push("prerequisites:")
-      const pushReport = (label: string, report: PrerequisiteReport): void => {
-        lines.push(`  ${report.satisfied ? "✔" : "?"} ${label}: ${report.summary}`)
-        if (report.hint !== undefined) {
-          lines.push(...wrap(report.hint, 78, "      "))
+      lines.push("credentials:")
+      for (const key of credentialKeys) {
+        const value = environment[key]
+        lines.push(`  ${value === undefined || value.length === 0 ? "✖" : "✔"} ${key}`)
+      }
+      lines.push("")
+      lines.push("gates:")
+      lines.push(
+        ...wrap(
+          `${node._tag === "Mismatch" || node._tag === "NoNode" ? "✖" : "✔"} ${node.summary}`,
+          78,
+          "  "
+        )
+      )
+
+      // Connector prerequisites the environment must carry BEFORE a run: shown
+      // when the connector is selected or the machine has its CLI, because that
+      // is exactly when a missing one turns into a confusing mid-run failure.
+      const coder = selectedCoder(environment)
+      const geminiRelevant =
+        coder.startsWith("gemini") ||
+        Array.from(statuses.keys()).some(
+          (id) =>
+            id.value === ConnectorIds.GeminiCli.value || id.value === ConnectorIds.GeminiApi.value
+        )
+      const bridge = geminiBridgePrerequisites(environment, readPiModelsJson())
+      if (geminiRelevant || bridge !== undefined) {
+        lines.push("")
+        lines.push("prerequisites:")
+        const pushReport = (label: string, report: PrerequisiteReport): void => {
+          lines.push(`  ${report.satisfied ? "✔" : "?"} ${label}: ${report.summary}`)
+          if (report.hint !== undefined) {
+            lines.push(...wrap(report.hint, 78, "      "))
+          }
+          for (const line of report.detail ?? []) {
+            lines.push(`      ${line}`)
+          }
         }
-        for (const line of report.detail ?? []) {
-          lines.push(`      ${line}`)
+        if (geminiRelevant) {
+          pushReport("gemini", geminiPrerequisites(environment))
+        }
+        if (bridge !== undefined) {
+          pushReport("pi-gemini-bridge", bridge)
         }
       }
-      if (geminiRelevant) {
-        pushReport("gemini", geminiPrerequisites(environment))
-      }
-      if (bridge !== undefined) {
-        pushReport("pi-gemini-bridge", bridge)
-      }
-    }
 
-    lines.push("")
-    lines.push(`coder: ${coder}`)
-    return `${lines.join("\n")}\n`
-  })
+      lines.push("")
+      lines.push(otelDoctorLine(otel, otelAnswers))
+      lines.push(`coder: ${coder}`)
+      return `${lines.join("\n")}\n`
+    }
+  )
+}

package/src/FlowArgs.ts

@@ -34,6 +34,8 @@ environment:
   LLM4TS_VERBOSITY        terminal verbosity: quiet|normal|verbose
   LLM4TS_UI               tree: the full-screen agent tree (ADR 0022; q returns to the classic view)
   LLM4TS_TRANSCRIPT       on: record each seat's input and output under .llm4ts/transcripts/
+  LLM4TS_OTEL             on: export spans over OTLP (a local Phoenix at localhost:6006 unless OTEL_* says otherwise)
+  LLM4TS_OTEL_CONTENT     on|full: prompts, replies and tool I/O travel too (off by default)
   OPENAI_API_KEY, ANTHROPIC_API_KEY, GEMINI_API_KEY
                           credentials for API providers
 

package/src/FlowRunner.ts

@@ -58,7 +58,18 @@ import {
 } from "@llm4ts/flow/FlowEvents"
 import { FlowContext, type ContextOptions, type FlowContextShape } from "@llm4ts/flow/FlowContext"
 import { makeFlowRecorder, type RunOutcome } from "@llm4ts/flow/FlowRecorder"
-import { timedJudgment, timedSeat, withTimedRole } from "@llm4ts/flow/Timing"
+import { timedJudgment, timedSeat, withTimedRole, type TimedSeatOptions } from "@llm4ts/flow/Timing"
+import { attr, otelContent, withKindSpan } from "@llm4ts/flow/Spans"
+import * as Tracer from "effect/Tracer"
+import { basename } from "node:path"
+import {
+  otelConfig,
+  otelWarning,
+  probeOtelEndpoint,
+  otelLayer,
+  otelSummary,
+  runnerVersion
+} from "./Otel.ts"
 import { transcriptSeat, type TranscriptSink } from "@llm4ts/flow/Transcript"
 import { nodeTranscriptFiles, nodeTranscriptSink, transcriptsWanted } from "./Transcripts.ts"
 import { idleAfterFrom } from "./AgentTree.ts"
@@ -362,6 +373,13 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
   const runEnvironment = options.environment ?? process.env
   const estimateOptions = estimatedUsageOptionsFromEnv(runEnvironment)
   const meterUsage = options.estimateUsage ?? !switchedOff(runEnvironment.LLM4TS_ESTIMATE_USAGE)
+  // Every seat's spans (ADR 0026): content only when asked, an estimate on a
+  // call the backend reports no usage for, and who served it when known.
+  const seatOptions = (executor?: Effect.Effect<string | undefined>): TimedSeatOptions => ({
+    content: otelContent(runEnvironment),
+    ...(meterUsage ? { estimate: estimateOptions } : {}),
+    ...(executor === undefined ? {} : { executor })
+  })
   const metered = (service: LlmServiceShape): Effect.Effect<LlmServiceShape> =>
     meterUsage
       ? Effect.map(makeEstimatedUsageMeter(service, estimateOptions), (meter) => meter.service)
@@ -490,18 +508,26 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
   ): FlowContextShape =>
     FlowContext.of({
       // Every call is timed on the lane's events, under its role.
-      reasoning: recorded(timedSeat(seats.reasoning, laneEvents, "reasoning"), lane, "reasoning"),
+      reasoning: recorded(
+        timedSeat(seats.reasoning, laneEvents, "reasoning", seatOptions()),
+        lane,
+        "reasoning"
+      ),
       coder: {
         ...seats.coder,
-        ...recorded(timedSeat(seats.coder, laneEvents, "coder"), lane, "coder")
+        ...recorded(timedSeat(seats.coder, laneEvents, "coder", seatOptions()), lane, "coder")
       },
       judgment: timedJudgment(seats.judgment, laneEvents),
-      judge: recorded(timedSeat(seats.judge, laneEvents, "judgment"), lane, "judgment"),
+      judge: recorded(
+        timedSeat(seats.judge, laneEvents, "judgment", seatOptions()),
+        lane,
+        "judgment"
+      ),
       git: makeGitTool(dependencies.process, workDir, laneEvents),
       hosting: makeGitHubTool(dependencies.process, workDir, laneEvents),
       events: laneEvents,
       reviewers: seats.reviewers.map((reviewer) =>
-        recorded(timedSeat(reviewer, laneEvents, "reviewer"), lane, "reviewer")
+        recorded(timedSeat(reviewer, laneEvents, "reviewer", seatOptions()), lane, "reviewer")
       ),
       coderCapabilities: seats.coder.capabilities,
       userPrompt: options.userPrompt,
@@ -671,7 +697,8 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
               label
             }),
             laneEvents,
-            role
+            role,
+            seatOptions()
           ),
           lane,
           role
@@ -709,7 +736,7 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
         return FlowContext.of({
           reasoning: view.forRole(rebind ? "planner" : "reviewer"),
           coder: recorded(
-            timedSeat(held.service, laneEvents, "coder"),
+            timedSeat(held.service, laneEvents, "coder", seatOptions(held.executor)),
             lane,
             "coder",
             held.executor
@@ -811,6 +838,9 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
   const environment = options.environment ?? process.env
   const verbosity = options.verbosity ?? "Normal"
   const persistRun = options.persistRun ?? true
+  // OpenTelemetry export (ADR 0026): decided from the environment, built once
+  // per run, flushed when the run's scope closes.
+  const otel = otelConfig(environment)
   const tracePath =
     options.tracePath ??
     (persistRun ? join(options.workDir, ".llm4ts", `trace-${startedAt}.jsonl`) : undefined)
@@ -867,6 +897,16 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
         )
       )
     }
+    const where = otelSummary(otel)
+    if (where !== undefined) {
+      yield* surface.log(palette.info(where))
+      // One warning, at the start, when nothing listens: the exporter itself
+      // drops batches quietly. The probe is bounded (1.5s) and runs once.
+      const warning = otelWarning(otel, yield* probeOtelEndpoint(otel))
+      if (warning !== undefined) {
+        yield* surface.log(palette.warn(warning))
+      }
+    }
   }
   const recorder =
     tracePath === undefined
@@ -925,7 +965,32 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
     tracePath === undefined
       ? bundle.context
       : { ...bundle.context, trace: { runId: bundle.runId, path: tracePath } }
-  return yield* body(context).pipe(
+  const runAttributes: Record<string, unknown> = {
+    [attr.run]: bundle.runId,
+    [attr.session]: bundle.runId,
+    ...(environment.LLM4TS_FLOW === undefined ? {} : { [attr.flow]: environment.LLM4TS_FLOW })
+  }
+  // The run is a CHAIN span every other span descends from or links to; its
+  // attributes ride on all of them. Effect's own function spans stay
+  // unsampled so an exported trace shows the flow, not the call stack.
+  const traced = withKindSpan(
+    "run",
+    { kind: "CHAIN", attributes: runAttributes },
+    Object.entries(runAttributes).reduce(
+      (acc, [key, value]) => Effect.annotateSpans(key, value)(acc),
+      body(context)
+    )
+  ).pipe(
+    Effect.provideService(Tracer.MinimumTraceLevel, otel.mode === "off" ? "All" : "None"),
+    Effect.provide(
+      otelLayer(
+        otel,
+        { serviceVersion: runnerVersion, project: basename(options.workDir) },
+        environment
+      )
+    )
+  )
+  return yield* traced.pipe(
     Effect.provideService(FlowContext, context),
     Effect.andThen((value) => Effect.as(enforceBudget, value)),
     Effect.ensuring(

package/src/Otel.ts

@@ -0,0 +1,198 @@
+// OpenTelemetry export (ADR 0026): Effect's own OTLP tracer and metrics
+// exporters over core's fetch client, so a run can show up in Phoenix,
+// Langfuse, SigNoz or any collector with no SDK and no new dependency.
+// The standard OTEL_* variables are the truth; `--otel` is the laptop case.
+import * as ConfigProvider from "effect/ConfigProvider"
+import * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as FetchHttpClient from "effect/http/FetchHttpClient"
+import * as OtlpExporter from "effect/observability/OtlpExporter"
+import * as OtlpMetrics from "effect/observability/OtlpMetrics"
+import * as OtlpSerialization from "effect/observability/OtlpSerialization"
+import * as OtlpTracer from "effect/observability/OtlpTracer"
+import { packageVersion } from "./Package.ts"
+
+export type OtelConfig =
+  | { readonly mode: "off" }
+  | { readonly mode: "phoenix"; readonly tracesUrl: string }
+  | { readonly mode: "env"; readonly endpoint: string }
+
+export const phoenixTracesUrl = "http://localhost:6006/v1/traces"
+
+const truthy = (value: string | undefined): boolean =>
+  /^(on|1|true|yes)$/iu.test(value?.trim() ?? "")
+const falsy = (value: string | undefined): boolean =>
+  /^(off|0|false|no)$/iu.test(value?.trim() ?? "")
+
+/**
+ * Off unless asked. An OTLP endpoint in the environment wins and is used as
+ * the standard variables say; `LLM4TS_OTEL=on` alone means a local Phoenix,
+ * traces only. `OTEL_SDK_DISABLED` or `LLM4TS_OTEL=off` switch everything off.
+ */
+export const otelConfig = (
+  environment: Readonly<Record<string, string | undefined>>
+): OtelConfig => {
+  if (truthy(environment.OTEL_SDK_DISABLED) || falsy(environment.LLM4TS_OTEL)) {
+    return { mode: "off" }
+  }
+  // Under the test runner the exporter's sleeps run on a TestClock nobody
+  // advances, so its final flush would wait forever: a developer with
+  // LLM4TS_OTEL=on in the shell must still get a green suite.
+  if (environment.VITEST !== undefined) {
+    return { mode: "off" }
+  }
+  const endpoint =
+    environment.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT?.trim() ||
+    environment.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()
+  if (endpoint !== undefined && endpoint.length > 0) {
+    return { mode: "env", endpoint }
+  }
+  return truthy(environment.LLM4TS_OTEL)
+    ? { mode: "phoenix", tracesUrl: phoenixTracesUrl }
+    : { mode: "off" }
+}
+
+export interface OtelResource {
+  readonly serviceVersion: string
+  /** Phoenix's project: the repository's basename, never a path. */
+  readonly project: string
+}
+
+/** The run end may wait this long for the last batch; the event drain waits the same. */
+export const flushTimeout = Duration.seconds(3)
+
+/** This runner's version, for `service.version`. */
+export const runnerVersion: string = packageVersion
+
+const resourceOf = (resource: OtelResource) => ({
+  serviceName: "llm4ts",
+  serviceVersion: resource.serviceVersion,
+  attributes: { "openinference.project.name": resource.project }
+})
+
+/**
+ * The exporter for `config`: nothing when off; traces only for Phoenix;
+ * traces and metrics from the standard variables for a generic endpoint.
+ * Logs are never exported. Export errors are the exporter's to swallow; the
+ * run never fails on them.
+ */
+export const otelLayer = (
+  config: OtelConfig,
+  resource: OtelResource,
+  /** What the standard variables are read from in env mode (the run's environment). */
+  environment: Readonly<Record<string, string | undefined>> = process.env
+): Layer.Layer<OtlpExporter.Flusher> => {
+  switch (config.mode) {
+    case "off":
+      // The exporter's no-op flusher: nothing to export, nothing to flush.
+      return OtlpExporter.layerFlusher
+    case "phoenix":
+      return OtlpTracer.layer({
+        url: config.tracesUrl,
+        resource: resourceOf(resource),
+        shutdownTimeout: flushTimeout
+      }).pipe(Layer.provide(OtlpSerialization.layerJson), Layer.provide(FetchHttpClient.layer))
+    case "env":
+      // The standard variables are read from the run's own environment, with
+      // the two exporter defaults an endpoint implies; the process env is
+      // never written.
+      return Layer.mergeAll(
+        OtlpTracer.layerFromConfig({ resource: resourceOf(resource) }),
+        OtlpMetrics.layerFromConfig({ resource: resourceOf(resource) })
+      ).pipe(
+        Layer.provide(OtlpSerialization.layerJson),
+        Layer.provide(FetchHttpClient.layer),
+        Layer.provide(
+          ConfigProvider.layer(
+            ConfigProvider.fromEnvRecord({
+              ...environment,
+              ...otelEnvironmentDefaults(config, environment)
+            })
+          )
+        )
+      )
+  }
+}
+
+/**
+ * `layerFromConfig` exports nothing unless `OTEL_TRACES_EXPORTER` names
+ * `otlp`; an endpoint in the environment is the operator's intent, so the
+ * layer reads its variables with that default (and the metrics one) added
+ * when they are unset.
+ */
+export const otelEnvironmentDefaults = (
+  config: OtelConfig,
+  environment: Readonly<Record<string, string | undefined>>
+): Readonly<Record<string, string>> =>
+  config.mode !== "env"
+    ? {}
+    : {
+        ...(environment.OTEL_TRACES_EXPORTER === undefined ? { OTEL_TRACES_EXPORTER: "otlp" } : {}),
+        ...(environment.OTEL_METRICS_EXPORTER === undefined
+          ? { OTEL_METRICS_EXPORTER: "otlp" }
+          : {})
+      }
+
+export const otelSummary = (config: OtelConfig): string | undefined => {
+  switch (config.mode) {
+    case "off":
+      return undefined
+    case "phoenix":
+      return `otel → Phoenix at ${config.tracesUrl} (traces)`
+    case "env":
+      return `otel → ${config.endpoint} (traces + metrics, OTEL_* variables)`
+  }
+}
+
+/**
+ * The one warning a run prints when export is on and nothing answers at the
+ * endpoint (spec §9): the exporter itself only logs at debug level and drops
+ * the batch. Nothing when off or when the endpoint answered.
+ */
+export const otelWarning = (
+  config: OtelConfig,
+  answers: boolean | undefined
+): string | undefined => {
+  if (config.mode === "off" || answers !== false) {
+    return undefined
+  }
+  const where = config.mode === "phoenix" ? config.tracesUrl : config.endpoint
+  return (
+    `⚠ otel: nothing answers at ${where} — spans will be dropped (the run is unaffected)` +
+    (config.mode === "phoenix"
+      ? "; start Phoenix with `docker run -p 6006:6006 arizephoenix/phoenix:latest`"
+      : "")
+  )
+}
+
+/** The doctor's line: `answers` is the probe's result, undefined when export is off. */
+export const otelDoctorLine = (config: OtelConfig, answers: boolean | undefined): string => {
+  switch (config.mode) {
+    case "off":
+      return "otel: off (LLM4TS_OTEL=on for a local Phoenix, or OTEL_EXPORTER_OTLP_ENDPOINT)"
+    case "phoenix":
+    case "env": {
+      const where = config.mode === "phoenix" ? `Phoenix at ${config.tracesUrl}` : config.endpoint
+      return `otel: ${where} — ${
+        answers === true
+          ? "endpoint answers"
+          : "endpoint does not answer (spans will be dropped, the run is unaffected)"
+      }`
+    }
+  }
+}
+
+/** Whether anything listens at the endpoint: any HTTP answer counts, a refused connection does not. */
+export const probeOtelEndpoint = (config: OtelConfig): Effect.Effect<boolean | undefined> =>
+  config.mode === "off"
+    ? Effect.succeed(undefined)
+    : Effect.tryPromise(() =>
+        fetch(config.mode === "phoenix" ? config.tracesUrl : config.endpoint, {
+          method: "OPTIONS",
+          signal: AbortSignal.timeout(1_500)
+        })
+      ).pipe(
+        Effect.as(true),
+        Effect.orElseSucceed(() => false)
+      )

package/src/Package.ts

@@ -1 +1,11 @@
+import { createRequire } from "node:module"
+import * as Schema from "effect/Schema"
+
+const Manifest = Schema.Struct({ version: Schema.String })
+
+const manifest = Schema.decodeUnknownSync(Manifest)(
+  createRequire(import.meta.url)("../package.json")
+)
+
 export const packageName = "@llm4ts/runner"
+export const packageVersion: string = manifest.version

package/src/Profile.ts

@@ -65,6 +65,15 @@ export const Gap = Schema.Struct({
 })
 export type Gap = typeof Gap.Type
 
+/** The coder's tool calls by kind of work (build, install and git count as other). */
+export const ToolCalls = Schema.Struct({
+  explore: Schema.Int,
+  edit: Schema.Int,
+  test: Schema.Int,
+  other: Schema.Int
+})
+export type ToolCalls = typeof ToolCalls.Type
+
 export const StoryProfile = Schema.Struct({
   id: Schema.String,
   executor: Schema.optionalKey(Schema.String),
@@ -81,7 +90,14 @@ export const StoryProfile = Schema.Struct({
    * call) but not ended: a run still going. Counted as model time so far.
    */
   openTurnMs: Ms,
-  openTurnTools: Schema.Int
+  openTurnTools: Schema.Int,
+  toolCalls: ToolCalls,
+  /** Task stages the coder ran on this lane: the tasks it gave itself, plus any revision. */
+  tasks: Schema.Int,
+  /** From the coder's first task (after setup and planning) to its first edit; absent when it never edited. */
+  firstEditMs: Schema.optionalKey(Ms),
+  /** Explore calls (ls, find, grep, read) made before that first edit. */
+  exploreBeforeEdit: Schema.Int
 })
 export type StoryProfile = typeof StoryProfile.Type
 
@@ -159,6 +175,12 @@ interface Lane {
   /** Since when a coder turn has been working without ending, and its tool calls. */
   openSince: number | undefined
   openTools: number
+  tasks: number
+  /** When the coder's first task began: setup and task planning come before it. */
+  firstTask: number | undefined
+  toolCalls: { explore: number; edit: number; test: number; other: number }
+  firstEdit: number | undefined
+  exploreBeforeEdit: number
 }
 
 interface Lease {
@@ -321,10 +343,23 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
             skipped: false,
             moments: [{ at, what: describe(event) }],
             openSince: undefined,
-            openTools: 0
+            openTools: 0,
+            tasks: 0,
+            firstTask: undefined,
+            toolCalls: { explore: 0, edit: 0, test: 0, other: 0 },
+            firstEdit: undefined,
+            exploreBeforeEdit: 0
           })
           continue
         }
+        if (
+          lane !== undefined &&
+          !storyStage.test(event.stage) &&
+          !event.stage.startsWith(`story ${lane.id}:`)
+        ) {
+          lane.tasks += 1
+          lane.firstTask = lane.firstTask ?? at
+        }
         break
       }
       case "StageCompleted":
@@ -340,6 +375,17 @@ export const profileOf = (inputs: ReadonlyArray<TreeInput>): ProfileReport => {
       case "Timed": {
         allTimed.push({ at, event })
         lane?.timed.push({ at, event })
+        if (lane !== undefined && event.kind === "tool" && event.category !== undefined) {
+          const kind =
+            event.category === "explore" || event.category === "edit" || event.category === "test"
+              ? event.category
+              : "other"
+          lane.toolCalls[kind] += 1
+          if (kind === "edit" && lane.firstEdit === undefined) {
+            lane.firstEdit = at - (lane.firstTask ?? lane.start)
+            lane.exploreBeforeEdit = lane.toolCalls.explore
+          }
+        }
         if (lane !== undefined && event.kind === "model" && event.label === "coder") {
           lane.openSince = undefined
           lane.openTools = 0
@@ -578,6 +624,10 @@ const storyProfile = (
     gaps: estimated ? [] : gapsOf(lane),
     openTurnMs,
     openTurnTools: open ? lane.openTools : 0,
+    toolCalls: { ...lane.toolCalls },
+    tasks: lane.tasks,
+    ...(lane.firstEdit === undefined ? {} : { firstEditMs: lane.firstEdit }),
+    exploreBeforeEdit: lane.exploreBeforeEdit,
     id: lane.id,
     ...(lane.executor === undefined ? {} : { executor: lane.executor }),
     status: lane.status,
@@ -683,6 +733,19 @@ const findingsOf = (facts: {
       })
     }
   }
+  for (const story of facts.stories) {
+    const wandered =
+      story.firstEditMs !== undefined &&
+      story.wallMs >= 60_000 &&
+      (story.exploreBeforeEdit >= 15 ||
+        (story.exploreBeforeEdit >= 5 && story.firstEditMs >= story.wallMs / 4))
+    if (wandered && story.firstEditMs !== undefined) {
+      candidates.push({
+        ms: story.firstEditMs,
+        text: `${story.id}: ${plural(story.exploreBeforeEdit, "explore call")} and ${duration(story.firstEditMs)} before the coder's first edit — it found its way instead of being told where to go`
+      })
+    }
+  }
   const queued = sum(facts.stories.map((story) => story.queuedMs ?? 0))
   if (queued >= 1_000) {
     candidates.push({
@@ -795,6 +858,41 @@ export const renderProfile = (report: ProfileReport): string => {
           : ""
       ])
     ),
+    ...(report.stories.every(
+      (story) =>
+        story.toolCalls.explore +
+          story.toolCalls.edit +
+          story.toolCalls.test +
+          story.toolCalls.other ===
+        0
+    )
+      ? []
+      : [
+          "",
+          "Coder work per story",
+          ...table(
+            [
+              "story",
+              "tasks",
+              "explore",
+              "edit",
+              "test",
+              "other",
+              "before 1st edit",
+              "1st edit at"
+            ],
+            report.stories.map((story) => [
+              story.id,
+              String(story.tasks),
+              String(story.toolCalls.explore),
+              String(story.toolCalls.edit),
+              String(story.toolCalls.test),
+              String(story.toolCalls.other),
+              story.firstEditMs === undefined ? "" : `${story.exploreBeforeEdit} explore`,
+              story.firstEditMs === undefined ? "never" : duration(story.firstEditMs)
+            ])
+          )
+        ]),
     ...(report.skipped.length === 0
       ? []
       : [`  skipped (merged on an earlier run): ${report.skipped.join(", ")}`]),