@cat-factory/executor-harness 1.114.0 → 1.118.0

package/src/inline.ts

@@ -18,14 +18,26 @@ import type { RunOptions } from './runner.js'
 // is injected.
 
 /**
- * Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the
- * inline `finishReason` the reviewer keys off. Only Claude Code reports it (`max_tokens` on
- * a `--output-format stream-json` result); Codex's thinner stream exposes none, so it reads
- * as `stop` — the same one-shot limitation the host-CLI runner has.
+ * Map the harness CLI's terminal stop reason (lifted onto the last call metric) to the inline
+ * `finishReason` the reviewer keys off, or `undefined` when the CLI reported NONE.
+ *
+ * Undefined rather than `stop`, which is what this returned for years on the strength of a
+ * comment claiming Claude Code reports the reason. It does not: its `stream-json` `assistant`
+ * envelopes carry the message-START snapshot, whose `stop_reason` is null, so every call metric
+ * a claude-code or codex run produces has a null reason and this answered `stop` for all of them.
+ * `stop` is a positive claim that the model finished of its own accord, and it is the exact claim
+ * a truncation check is trying to disprove — so the one caller keyed off it
+ * (`finishReason === 'length'`) could never fire, and every store that kept the row recorded a
+ * clean stop nobody observed.
+ *
+ * Kept as a MAPPING rather than deleted because the field remains reachable: a subagent turn is
+ * read from a completed JSONL transcript, which does carry `stop_reason`, and a future CLI build
+ * (or `--include-partial-messages`) would restore it on the parent stream too.
  */
-function deriveFinishReason(calls: HarnessCallMetric[] | undefined): 'stop' | 'length' {
+function deriveFinishReason(calls: HarnessCallMetric[] | undefined): 'stop' | 'length' | undefined {
   const last = calls?.[calls.length - 1]
-  const reason = last?.finishReason?.toLowerCase() ?? ''
+  const reason = last?.finishReason?.toLowerCase()
+  if (!reason) return undefined
   return reason === 'max_tokens' || reason === 'length' ? 'length' : 'stop'
 }
 
@@ -57,9 +69,10 @@ export async function handleInline(job: InlineJob, opts: RunOptions): Promise<In
       ...(opts.signal ? { signal: opts.signal } : {}),
       ...(opts.onActivity ? { onActivity: opts.onActivity } : {}),
     })
+    const finishReason = deriveFinishReason(outcome.callMetrics)
     return {
       text: outcome.summary,
-      finishReason: deriveFinishReason(outcome.callMetrics),
+      ...(finishReason ? { finishReason } : {}),
       ...(outcome.usage ? { usage: inlineUsage(outcome.usage, outcome.callMetrics) } : {}),
       ...(outcome.callMetrics ? { callMetrics: outcome.callMetrics } : {}),
     }

package/src/job.ts

@@ -1164,7 +1164,14 @@ export interface InlineJob extends HarnessAuthFields {
 /** The inline completion result: the reply text plus lifted token usage / per-call telemetry. */
 export interface InlineResult {
   text: string
-  /** `length` when the model hit its output cap (the reviewer rejects a truncated doc). */
+  /**
+   * `length` when the model hit its output cap (the reviewer rejects a truncated doc), `stop`
+   * when it finished of its own accord, ABSENT when the CLI reported no stop reason at all.
+   *
+   * Absent is the normal case today: neither subscription CLI exposes a per-call stop reason on
+   * its parent stream, and the three states must stay distinct because a reader that takes
+   * absent for `stop` is asserting the one thing a truncation check exists to disprove.
+   */
   finishReason?: 'stop' | 'length'
   /**
    * The job's token usage with the input side split into its three ORTHOGONAL classes:

package/src/pi.ts

@@ -642,6 +642,16 @@ export interface HarnessCallMetric {
    * never disagree about which phase billed a call.
    */
   phase?: string
+  /**
+   * This row is not a TURN: it stands for the job as a whole, carrying the spend the CLI reported
+   * in its terminal cumulative total and did not attribute to any turn it narrated (see
+   * {@link unaccountedUsageCall}). It has no bodies, because there was no request to capture.
+   *
+   * The backend files it with a NULL turn index for that reason, while still deriving its row id
+   * from {@link seq} so a replayed poll re-records instead of duplicating. Absent on every real
+   * turn. `CliInlineLanguageModel`'s step-level row is the same idea on the inline path.
+   */
+  standsForJob?: boolean
 }
 
 /**
@@ -657,9 +667,13 @@ export interface HarnessCallMetric {
  * A published call must be FINAL. The backend records it the moment the drain reaches it and
  * IGNORES the terminal repeat (first write wins, so its stored prompt delta stays valid against
  * the chain tip it was written against), which means a field mutated after publishing never
- * reaches the store. A producer whose calls can still change (the cumulative-usage fallback,
- * whose totals arrive with the CLI's terminal `result` event) publishes through
- * {@link createCallMetricPublisher} instead, which withholds exactly those.
+ * reaches the store.
+ *
+ * That is a rule about every producer, and it is why the cumulative-usage reconciliation files its
+ * shortfall as a NEW row here at the end of the run (`unaccountedUsageCall`) rather than growing the
+ * last captured turn. This used to be wrapped by a publisher that withheld the turn attribution
+ * could still rewrite, trading a turn of streaming lag for that mutability; with nothing mutated,
+ * the wrapper had no reason left to exist.
  */
 export function publishCallMetric(
   calls: HarnessCallMetric[],
@@ -670,54 +684,6 @@ export function publishCallMetric(
   onCallMetric?.(call)
 }
 
-/** Appends captured calls to a run's list, streaming each one as soon as it is final. */
-export interface CallMetricPublisher {
-  /** Append a captured call, streaming it now unless its tokens can still be rewritten. */
-  publish(call: HarnessCallMetric): void
-  /** Stream whatever is still withheld. Call once the run's totals are attributed. */
-  flush(): void
-}
-
-/**
- * A {@link publishCallMetric} wrapper for a producer whose per-call tokens may be filled in at
- * the END of the run: a CLI that reports only a cumulative total leaves every turn at zero, and
- * `attributeCumulativeUsage` pins the total onto the last call once the terminal `result` event
- * arrives.
- *
- * Since a published call must be final (the backend stores it on the drain and ignores the
- * terminal repeat), a call the CLI did NOT cost is appended to the list but WITHHELD from the
- * live stream — otherwise it records as a zero-token row and the attributed numbers never land.
- * The withholding window closes the moment any call IS costed: attribution can no longer fire, so
- * everything held is final and released at once, in capture order, and every later call streams
- * immediately whatever its tokens. {@link flush} covers the run that was never costed at all.
- */
-export function createCallMetricPublisher(
-  calls: HarnessCallMetric[],
-  onCallMetric?: (call: HarnessCallMetric) => void,
-): CallMetricPublisher {
-  const withheld: HarnessCallMetric[] = []
-  let anyCosted = false
-  const flush = (): void => {
-    for (const call of withheld) onCallMetric?.(call)
-    withheld.length = 0
-  }
-  return {
-    publish(call) {
-      const costed = call.inputTokens > 0 || call.outputTokens > 0
-      if (!costed && !anyCosted) {
-        publishCallMetric(calls, call)
-        withheld.push(call)
-        return
-      }
-      if (costed) anyCosted = true
-      // Released BEFORE this call so the live sequence stays in capture order.
-      flush()
-      publishCallMetric(calls, call, onCallMetric)
-    },
-    flush,
-  }
-}
-
 /** Pi's assistant summary plus {@link PiRunStats} describing what it did. */
 export interface PiRunOutcome {
   summary: string

package/src/usage-attribution.ts

@@ -0,0 +1,102 @@
+import { isObject, numberOf } from './claude-stream.js'
+import type { HarnessCallMetric } from './pi.js'
+
+// How a subscription CLI's TWO token channels are reconciled into the per-call rows the backend
+// stores: the per-turn usage the stream narrates, and the cumulative total the terminal `result`
+// event reports. They disagree routinely and in a specific direction, so the reconciliation is a
+// concern of its own rather than a helper beside the stream reader that happens to need it.
+//
+// Split out of `agent-runner.ts` when it hit its size budget.
+
+/**
+ * Read Claude Code's terminal cumulative usage.
+ *
+ * Counts every input bucket Anthropic bills: fresh input plus BOTH cache reads and cache writes
+ * (`cache_creation_input_tokens`), which are real consumed tokens and are the dominant share on a
+ * long agent run. Omitting them under-weights a token's true load in the usage-aware rotation
+ * window. `undefined` when the event carried no usage at all, so a caller can tell that from a
+ * genuine zero.
+ */
+export function claudeUsage(
+  raw: unknown,
+): { inputTokens: number; outputTokens: number } | undefined {
+  if (!isObject(raw)) return undefined
+  const input =
+    numberOf(raw.input_tokens) +
+    numberOf(raw.cache_read_input_tokens) +
+    numberOf(raw.cache_creation_input_tokens)
+  const output = numberOf(raw.output_tokens)
+  if (input === 0 && output === 0) return undefined
+  return { inputTokens: input, outputTokens: output }
+}
+
+/**
+ * The row standing for whatever the per-turn channel did NOT account for: the terminal cumulative
+ * usage minus the sum of the turns already costed, computed PER SIDE. `undefined` when the turns
+ * add up, so nothing is double counted.
+ *
+ * The per-side part is why this exists at all, and it replaced an all-or-nothing guard
+ * (`calls.some(c => c.inputTokens > 0 || c.outputTokens > 0)` ⇒ return) that only ever fired for a
+ * CLI reporting no per-turn usage at all. Claude Code reports plenty: its `assistant` envelopes
+ * carry the message-START usage snapshot, whose INPUT and cache counts are final and whose
+ * `output_tokens` is the 1-5 tokens produced when the message opened. So the guard saw costed
+ * turns, returned, and the run's whole output side stayed at that snapshot. Measured on a real
+ * board: a `coder` step recorded 198 output tokens across 34 calls against the 14,033 the terminal
+ * `result` event reported, an `initiative-analyst` 531 against 30,471. Input matched the terminal
+ * figure exactly, which is what made the shortfall invisible to a check that asked whether ANY
+ * tokens had been reported.
+ *
+ * **It is its OWN row rather than tokens added to the last captured call.** Growing a real turn by
+ * thousands of output tokens it did not produce makes a fabricated number indistinguishable from a
+ * measured one everywhere a per-call figure is read (`/api/v1/debug/*`, the observability panel, a
+ * step's per-call breakdown), and there is nothing on the row to mark it. The sibling rule on the
+ * inline path (`CliInlineLanguageModel.fileUnaccounted`) reached that conclusion first and files a
+ * step-level row; this is the same answer for the channel that has a call list. {@link
+ * HarnessCallMetric.standsForJob} is what keeps it from reading as a turn.
+ *
+ * **`calls` must be the PARENT loop's alone.** The terminal `result` event's cumulative covers the
+ * parent conversation only — a subagent's tokens live in its own transcript — so subtracting a
+ * subagent turn's tokens from it understates the shortfall, and pinning the remainder near one
+ * would bill a conversation for spend it never saw. Both were live in `ambientAuth` mode, where the
+ * CLI streams subagent turns onto the parent's stdout and no transcript watcher runs, so those
+ * turns are captured through the same publisher as the parent's.
+ *
+ * {@link claudeUsage} sums every billed input bucket, so the already-accounted input is the sum of
+ * all THREE per-call input classes, not `inputTokens` (fresh) alone. A residual input shortfall
+ * lands on `inputTokens` because nothing in the terminal event says which class it belonged to.
+ *
+ * Clamped at 0 per side: a CLI whose terminal figure is LOWER than its own per-turn sum has
+ * reported the two inconsistently, and negative spend is not a thing to record.
+ */
+export function unaccountedUsageCall(
+  parentCalls: readonly HarnessCallMetric[],
+  usage: { inputTokens: number; outputTokens: number } | undefined,
+): HarnessCallMetric | undefined {
+  if (!usage) return undefined
+  let accountedInput = 0
+  let accountedOutput = 0
+  for (const call of parentCalls) {
+    accountedInput += call.inputTokens + call.cacheReadTokens + call.cacheWriteTokens
+    accountedOutput += call.outputTokens
+  }
+  const inputTokens = Math.max(0, usage.inputTokens - accountedInput)
+  const outputTokens = Math.max(0, usage.outputTokens - accountedOutput)
+  if (!inputTokens && !outputTokens) return undefined
+  return {
+    // No `model`: the terminal event names none, and the recorder then files the row under the
+    // model the step DISPATCHED, which is the same answer without this claiming to have observed
+    // it. (Claude Code serves some turns with a different model, so a guess here misprices.)
+    promptText: '',
+    messageCount: 0,
+    responseText: '',
+    reasoningText: '',
+    inputTokens,
+    // Both 0 rather than a split of `inputTokens`: the terminal figure is one number and says
+    // nothing about which input class the remainder belonged to.
+    cacheReadTokens: 0,
+    cacheWriteTokens: 0,
+    outputTokens,
+    finishReason: null,
+    standsForJob: true,
+  }
+}