@llm4ts/flow 2.21.0 → 2.22.0

package/src/Activity.ts

@@ -1,9 +1,10 @@
+import * as Clock from "effect/Clock"
 import * as Effect from "effect/Effect"
 import * as Ref from "effect/Ref"
 import * as Stream from "effect/Stream"
 import type { LlmError } from "@llm4ts/core/Errors"
-import type { LlmChunk } from "@llm4ts/core/Models"
-import { ToolUse, UsageProgress, type FlowEventsShape } from "./FlowEvents.ts"
+import { LlmChunk } from "@llm4ts/core/Models"
+import { Timed, ToolUse, UsageProgress, type FlowEventsShape } from "./FlowEvents.ts"
 
 /**
  * Live agent activity.
@@ -122,6 +123,62 @@ const nextCall: Effect.Effect<string> = Effect.sync(() => {
  * tool call it carries. Wrap a connector stream with this before `collect` so
  * the run reports what the agent is doing while it is doing it.
  */
+interface OpenTool {
+  readonly id: string | undefined
+  readonly tool: string
+  readonly at: number
+}
+
+const toolIdOf = (chunk: LlmChunk): string | undefined => {
+  const id = chunk.metadata.tool_id ?? chunk.metadata.toolId
+  return id === undefined || id.length === 0 ? undefined : id
+}
+
+/**
+ * The `Timed{kind:"tool"}` a tool's end closes: the open call with
+ * the same id, or the oldest open one when the harness sends no ids. A
+ * harness that reports a tool only at its end (an older codex) gets its
+ * `ToolUse` published then, untimed, as before.
+ */
+const toolEnded = (
+  events: FlowEventsShape,
+  open: Ref.Ref<ReadonlyArray<OpenTool>>,
+  chunk: LlmChunk
+): Effect.Effect<void> =>
+  Effect.gen(function* () {
+    const id = toolIdOf(chunk)
+    const started = yield* Ref.modify(open, (calls) => {
+      const index = id === undefined ? 0 : calls.findIndex((call) => call.id === id)
+      const found = index < 0 ? undefined : calls[index]
+      return [
+        found,
+        found === undefined ? calls : [...calls.slice(0, index), ...calls.slice(index + 1)]
+      ] as const
+    })
+    if (started === undefined) {
+      const late = toolUseFrom(
+        LlmChunk.make({ delta: "", metadata: { ...chunk.metadata, event: "tool_use" } })
+      )
+      return late === undefined ? undefined : yield* events.publish(late)
+    }
+    const now = yield* Clock.currentTimeMillis
+    const failed = chunk.metadata.tool_failed === "true" || chunk.metadata.tool_status === "error"
+    yield* events.publish(
+      Timed.make({
+        kind: "tool",
+        label: started.tool,
+        ms: now - started.at,
+        ...(failed ? { failed: true } : {})
+      })
+    )
+  })
+
+/**
+ * Republishes the stream unchanged, publishing a `ToolUse` event for every
+ * tool call it carries, and a `Timed` one when the call ends. Wrap a
+ * connector stream with this before `collect` so the run reports what the
+ * agent is doing while it is doing it.
+ */
 export const withToolActivity = <R>(
   events: FlowEventsShape,
   stream: Stream.Stream<LlmChunk, LlmError, R>
@@ -130,22 +187,37 @@ export const withToolActivity = <R>(
   // display-only progress, so a long agent turn shows its tokens growing; a
   // call that reported any closes with `done` so the display drops it.
   Stream.unwrap(
-    Effect.map(Effect.all([nextCall, Ref.make(false)]), ([call, reported]) =>
-      Stream.tap(stream, (chunk) => {
-        const event = toolUseFrom(chunk)
-        const tool = event === undefined ? Effect.void : events.publish(event)
-        return chunk.usage === undefined
-          ? tool
-          : tool.pipe(
-              Effect.andThen(events.publish(UsageProgress.make({ call, usage: chunk.usage }))),
-              Effect.andThen(Ref.set(reported, true))
+    Effect.map(
+      Effect.all([nextCall, Ref.make(false), Ref.make<ReadonlyArray<OpenTool>>([])]),
+      ([call, reported, open]) =>
+        Stream.tap(stream, (chunk) => {
+          const event = toolUseFrom(chunk)
+          const tool =
+            event === undefined
+              ? chunk.metadata.event === "tool_result"
+                ? toolEnded(events, open, chunk)
+                : Effect.void
+              : Effect.andThen(
+                  events.publish(event),
+                  Effect.flatMap(Clock.currentTimeMillis, (at) =>
+                    Ref.update(open, (calls) => [
+                      ...calls,
+                      { id: toolIdOf(chunk), tool: event.tool, at }
+                    ])
+                  )
+                )
+          return chunk.usage === undefined
+            ? tool
+            : tool.pipe(
+                Effect.andThen(events.publish(UsageProgress.make({ call, usage: chunk.usage }))),
+                Effect.andThen(Ref.set(reported, true))
+              )
+        }).pipe(
+          Stream.ensuring(
+            Effect.flatMap(Ref.get(reported), (any) =>
+              any ? events.publish(UsageProgress.make({ call, done: true })) : Effect.void
             )
-      }).pipe(
-        Stream.ensuring(
-          Effect.flatMap(Ref.get(reported), (any) =>
-            any ? events.publish(UsageProgress.make({ call, done: true })) : Effect.void
           )
         )
-      )
     )
   )

package/src/FlowEvents.ts

@@ -181,6 +181,32 @@ export class BlackboardRun extends Schema.TaggedClass<BlackboardRun>()("Blackboa
   result: RunResult
 }) {}
 
+/**
+ * How long something took, published when it ends: a model call,
+ * a coder's tool, a gate command, git or a merge, a wait, a process start.
+ * `llm4ts profile` and the agent tree add these up; nothing else depends on
+ * them, and none carries content — only names, numbers and an exit code.
+ */
+export const TimedKind = Schema.Literals(["model", "tool", "gate", "git", "merge", "wait", "spawn"])
+export type TimedKind = typeof TimedKind.Type
+
+export class Timed extends Schema.TaggedClass<Timed>()("Timed", {
+  kind: TimedKind,
+  /** What was timed: the seat's role, the tool, the gate command as configured, the wait. */
+  label: Schema.String,
+  ms: Schema.Number,
+  /** A model call: when its first output came. */
+  firstMs: Schema.optionalKey(Schema.Number),
+  /** As the backend itself reported them (Gemini's stats): model API time and tool time. */
+  apiMs: Schema.optionalKey(Schema.Number),
+  toolMs: Schema.optionalKey(Schema.Number),
+  exitCode: Schema.optionalKey(Schema.Int),
+  /** It ended in a failure (the time was still spent). */
+  failed: Schema.optionalKey(Schema.Boolean),
+  lane: Schema.optionalKey(Schema.String),
+  executor: Schema.optionalKey(Schema.String)
+}) {}
+
 /** One rubric dimension of a story verdict: its score out of `max`. */
 export const JudgedDimension = Schema.Struct({
   id: Schema.String,
@@ -303,7 +329,8 @@ export const FlowEvent = Schema.Union([
   ExecutorExcluded,
   ExecutorResumed,
   ExecutorHandedOver,
-  StoryJudged
+  StoryJudged,
+  Timed
 ])
 export type FlowEvent = typeof FlowEvent.Type
 
@@ -395,6 +422,19 @@ const stamped = (
             ...(event.model === undefined ? {} : { model: event.model }),
             ...tags
           })
+    // The lane's executor is its coder's: it names a coder call or tool, not
+    // a reviewer's or judge's call made on another executor.
+    case "Timed":
+      return event.lane !== undefined
+        ? event
+        : Timed.make({
+            ...event,
+            lane,
+            ...(executor !== undefined &&
+            ((event.kind === "model" && event.label === "coder") || event.kind === "tool")
+              ? { executor }
+              : {})
+          })
     default:
       return event
   }

package/src/Review.ts

@@ -1,3 +1,4 @@
+import * as Clock from "effect/Clock"
 import * as Effect from "effect/Effect"
 import * as Schema from "effect/Schema"
 import type { LlmError } from "@llm4ts/core/Errors"
@@ -14,6 +15,7 @@ import {
   ReviewFinding,
   ReviewFindings,
   publishJudgmentObserved,
+  Timed,
   type FlowEventsShape
 } from "./FlowEvents.ts"
 import { Reviewer } from "./Reviewer.ts"
@@ -260,6 +262,7 @@ export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function
   if (executable === undefined) {
     return ReviewResult.make({ issues: [], summary: "" })
   }
+  const started = yield* Clock.currentTimeMillis
   const result = yield* guarded(
     Capabilities.Exec(executable),
     `lint: ${command.join(" ")}`,
@@ -273,6 +276,16 @@ export const lintCommand = Effect.fn("@llm4ts/flow/Review.lintCommand")(function
       )
     )
   )
+  // The command as configured and its exit code: never its output.
+  yield* events.publish(
+    Timed.make({
+      kind: "gate",
+      label: command.join(" "),
+      ms: (yield* Clock.currentTimeMillis) - started,
+      exitCode: result.exitCode,
+      ...(result.exitCode === 0 ? {} : { failed: true })
+    })
+  )
   if (result.exitCode === 0) {
     return ReviewResult.make({ issues: [], summary: "lint passed" })
   }

package/src/Roster.ts

@@ -20,6 +20,7 @@ import {
   ExecutorReleased,
   ExecutorResumed,
   Info,
+  Timed,
   type FlowEventsShape
 } from "./FlowEvents.ts"
 import { loadVersioned, saveVersioned, type PlainFileStoreShape } from "./Persistence.ts"
@@ -710,9 +711,24 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
     Effect.gen(function* () {
       const avoid = leaseOptions.avoid ?? []
       let announced = false
+      const asked = yield* Clock.currentTimeMillis
+      // A lease that had to wait says how long, for the story it is for.
+      const waited: Effect.Effect<void> = Effect.flatMap(Clock.currentTimeMillis, (now) =>
+        announced
+          ? events.publish(
+              Timed.make({
+                kind: "wait",
+                label: `roster ${role}`,
+                ms: now - asked,
+                ...(leaseOptions.label === undefined ? {} : { lane: leaseOptions.label })
+              })
+            )
+          : Effect.void
+      )
       while (true) {
         const spec = yield* attempt(role, leaseOptions)
         if (spec !== undefined) {
+          yield* waited
           return yield* leaseOf(spec, role, leaseOptions.label)
         }
         if (!(yield* canEverServe(role, avoid))) {
@@ -740,6 +756,7 @@ export const makeRoster = Effect.fn("@llm4ts/flow/Roster.make")(function* (
         yield* probeDue
         const recovered = yield* attempt(role, leaseOptions)
         if (recovered !== undefined) {
+          yield* waited
           return yield* leaseOf(recovered, role, leaseOptions.label)
         }
         const probing = yield* lock.withPermit(

package/src/Stories.ts

@@ -4,6 +4,7 @@
 // merges back only after its judge and perimeter checks pass, and the
 // target's gates run on the epic branch after every merge. Scheduling,
 // gating, resume and failure policy live here; seats come from `contextFor`.
+import * as Clock from "effect/Clock"
 import * as Effect from "effect/Effect"
 import * as Exit from "effect/Exit"
 import * as Queue from "effect/Queue"
@@ -28,7 +29,15 @@ import {
   StoryFailed,
   type FlowError
 } from "./FlowError.ts"
-import { Info, JudgedDimension, StoryJudged, withLane, type FlowEventsShape } from "./FlowEvents.ts"
+import {
+  Info,
+  JudgedDimension,
+  StoryJudged,
+  Timed,
+  withLane,
+  type FlowEventsShape
+} from "./FlowEvents.ts"
+import { timeEffect } from "./Timing.ts"
 import { statusPaths, type GitToolShape } from "./GitTool.ts"
 import {
   checkPerimeter,
@@ -464,7 +473,7 @@ export interface StoriesOptions {
    * installed dependencies, so the gates cannot run there without this.
    * Runs on every start and resume; a failure fails the story.
    */
-  readonly setup?: (workDir: string) => Effect.Effect<void, FlowError>
+  readonly setup?: (workDir: string, events?: FlowEventsShape) => Effect.Effect<void, FlowError>
   /**
    * When setup fails, give the story's coder ONE turn to make the worktree
    * ready (install, warm a cache, generate a client), then run setup again as
@@ -473,7 +482,14 @@ export interface StoriesOptions {
    */
   readonly setupAgent?: boolean
   /** The target's gates, run in a worktree per task and on the epic checkout after each merge. */
-  readonly gates: (workDir: string) => Effect.Effect<ReviewResult, FlowError>
+  /**
+   * The target's gates in a directory. `events` is the story's lane, so a
+   * gate's timing names its story; a gate may ignore it.
+   */
+  readonly gates: (
+    workDir: string,
+    events?: FlowEventsShape
+  ) => Effect.Effect<ReviewResult, FlowError>
   /** Story-level judge over the branch's diff against the epic branch; omit to skip. */
   readonly judge?: (
     story: Story,
@@ -624,24 +640,37 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
 
   /** Merge the story branch into the epic branch and re-gate the epic head, one story at a time. */
   const integrate = (story: Story, branch: string): Effect.Effect<void, FlowError> =>
-    mergeLock.withPermit(
-      Effect.gen(function* () {
-        yield* epicCheckoutClean(story)
-        const checkpoint = yield* context.git.checkpoint
-        yield* context.git.merge(branch, `${plan.epicId}: merge story ${story.id}`)
-        const gate = yield* options.gates(context.workDir)
-        if (!gate.isClean) {
-          // Never leave a red epic head for the next story to inherit.
-          yield* context.git.rollback(checkpoint)
-          return yield* failed(
-            story,
-            `epic gates failed after merging; merge undone:\n${issueLines(gate)}`
+    Effect.flatMap(Clock.currentTimeMillis, (requested) =>
+      mergeLock.withPermit(
+        Effect.gen(function* () {
+          const lane = laneOf(story)
+          // Merges are one at a time: how long this story queued for its turn.
+          const turn = yield* Clock.currentTimeMillis
+          yield* lane.publish(
+            Timed.make({ kind: "wait", label: "merge lock", ms: turn - requested })
           )
-        }
-        yield* laneOf(story).publish(
-          Info.make({ message: `story ${story.id}: merged into ${epicBranch}` })
-        )
-      })
+          yield* epicCheckoutClean(story)
+          const checkpoint = yield* context.git.checkpoint
+          yield* timeEffect(
+            lane,
+            context.git.merge(branch, `${plan.epicId}: merge story ${story.id}`),
+            (ms, failed) =>
+              Timed.make({ kind: "merge", label: "merge", ms, ...(failed ? { failed } : {}) })
+          )
+          const gate = yield* options.gates(context.workDir, lane)
+          if (!gate.isClean) {
+            // Never leave a red epic head for the next story to inherit.
+            yield* context.git.rollback(checkpoint)
+            return yield* failed(
+              story,
+              `epic gates failed after merging; merge undone:\n${issueLines(gate)}`
+            )
+          }
+          yield* laneOf(story).publish(
+            Info.make({ message: `story ${story.id}: merged into ${epicBranch}` })
+          )
+        })
+      )
     )
 
   /** The worktree and branch for a story, honouring the hash-guarded resume rules. */
@@ -794,7 +823,7 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
     // Catch up before setup: the epic may have changed the manifest too.
     yield* catchUp(story, git)
     if (options.setup !== undefined) {
-      const setup = options.setup(state.worktree)
+      const setup = options.setup(state.worktree, laneOf(story))
       yield* stage(
         laneOf(story),
         `story ${story.id}: setup`,
@@ -835,7 +864,7 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
     // The target's gates plus the perimeter: a stray path is a gate failure
     // the task review loop hands back to the coder before anything commits.
     const gates: Effect.Effect<ReviewResult, FlowError> = Effect.gen(function* () {
-      const target = yield* options.gates(state.worktree)
+      const target = yield* options.gates(state.worktree, laneOf(story))
       const changed = yield* perimeterNow(story, git)
       return combined(target, perimeterGate([...changed.sharedReadOnly, ...changed.outside], story))
     })

package/src/Timing.ts

@@ -0,0 +1,112 @@
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as Ref from "effect/Ref"
+import * as Stream from "effect/Stream"
+import type { LlmServiceShape } from "@llm4ts/core/LlmService"
+import type { LlmChunk } from "@llm4ts/core/Models"
+import { Timed, type FlowEventsShape } from "./FlowEvents.ts"
+
+/**
+ * Where a run's time goes. `timedSeat` decorates a seat so every
+ * call publishes a `Timed{kind:"model"}` when it ends — its wall time, when
+ * its first output came, and the API and tool time the backend reported, if
+ * it did. The events carry the seat's role, never the prompt or the reply.
+ */
+
+const reportedMs = (chunk: LlmChunk, key: string): number | undefined => {
+  const raw = chunk.metadata[key]
+  const value = raw === undefined ? Number.NaN : Number(raw)
+  return Number.isFinite(value) ? value : undefined
+}
+
+/** Times `effect`, publishing what `timed` makes of its duration whether it succeeds or fails. */
+export const timeEffect = <A, E, R>(
+  events: FlowEventsShape,
+  effect: Effect.Effect<A, E, R>,
+  timed: (ms: number, failed: boolean) => Timed
+): Effect.Effect<A, E, R> =>
+  Effect.flatMap(Clock.currentTimeMillis, (start) =>
+    Effect.onExit(effect, (exit) =>
+      Effect.flatMap(Clock.currentTimeMillis, (end) =>
+        events.publish(timed(end - start, Exit.isFailure(exit)))
+      )
+    )
+  )
+
+const timedStream = <E>(
+  events: FlowEventsShape,
+  label: string,
+  stream: Stream.Stream<LlmChunk, E>
+): Stream.Stream<LlmChunk, E> =>
+  Stream.unwrap(
+    Effect.gen(function* () {
+      const start = yield* Clock.currentTimeMillis
+      const first = yield* Ref.make<number | undefined>(undefined)
+      const reported = yield* Ref.make<{ apiMs?: number; toolMs?: number }>({})
+      const failed = yield* Ref.make(false)
+      return stream.pipe(
+        Stream.tap((chunk) =>
+          Effect.gen(function* () {
+            const now = yield* Clock.currentTimeMillis
+            yield* Ref.update(first, (at) => at ?? now - start)
+            const apiMs = reportedMs(chunk, "api_ms")
+            const toolMs = reportedMs(chunk, "tools_ms")
+            if (apiMs !== undefined || toolMs !== undefined) {
+              yield* Ref.update(reported, (current) => ({
+                ...current,
+                ...(apiMs === undefined ? {} : { apiMs }),
+                ...(toolMs === undefined ? {} : { toolMs })
+              }))
+            }
+          })
+        ),
+        Stream.tapError(() => Ref.set(failed, true)),
+        Stream.ensuring(
+          Effect.gen(function* () {
+            const end = yield* Clock.currentTimeMillis
+            const firstMs = yield* Ref.get(first)
+            yield* events.publish(
+              Timed.make({
+                kind: "model",
+                label,
+                ms: end - start,
+                ...(firstMs === undefined ? {} : { firstMs }),
+                ...(yield* Ref.get(reported)),
+                ...((yield* Ref.get(failed)) ? { failed: true } : {})
+              })
+            )
+          })
+        )
+      )
+    })
+  )
+
+export const timedSeat = (
+  service: LlmServiceShape,
+  events: FlowEventsShape,
+  label: string
+): LlmServiceShape => {
+  const model = (ms: number, failed: boolean): Timed =>
+    Timed.make({ kind: "model", label, ms, ...(failed ? { failed: true } : {}) })
+  const batched = service.scoreLabelSequence
+  return {
+    executeStream: (prompt) => timedStream(events, label, service.executeStream(prompt)),
+    executeStreamWithHistory: (messages) =>
+      timedStream(events, label, service.executeStreamWithHistory(messages)),
+    executeWithTools: (prompt, tools) =>
+      timeEffect(events, service.executeWithTools(prompt, tools), model),
+    executeStructured: (prompt, schema, jsonSchema) =>
+      timeEffect(events, service.executeStructured(prompt, schema, jsonSchema), model),
+    executeStructuredWithUsage: (prompt, schema, jsonSchema) =>
+      timeEffect(events, service.executeStructuredWithUsage(prompt, schema, jsonSchema), model),
+    scoreLabels: (prompt, labels) => timeEffect(events, service.scoreLabels(prompt, labels), model),
+    ...(batched === undefined
+      ? {}
+      : {
+          scoreLabelSequence: (prompt: string, labelSets: ReadonlyArray<ReadonlyArray<string>>) =>
+            timeEffect(events, batched(prompt, labelSets), model)
+        }),
+    isAvailable: service.isAvailable
+  }
+}