@cat-factory/app 0.232.2 → 0.233.0

package/app/stores/observability/toolCalls.ts

@@ -0,0 +1,173 @@
+import { ref } from 'vue'
+import type { RunToolCallFailures, RunToolCallTrajectory } from '~/types/execution'
+
+// The observability store's TOOL-CALL sink, extracted whole because it is one concern with two
+// reads and its own coherence rule between them.
+//
+// The rule: the two reads answer at DIFFERENT BOUNDS and must never be mistaken for each other.
+// `failures` is about the run — counts aggregated in SQL over every row it ever wrote. The
+// trajectory is a bounded PREFIX of the run, carrying every captured argument and result, which
+// is why it is loaded on demand rather than on open. Anything that counts, judges or headlines
+// reads the first; only the browse view reads the second, and it renders under a flag saying so.
+//
+// Keeping that pairing in one module is the point of the split: a caller reaching for whichever
+// list is nearest is exactly how a panel ends up reporting a long run's opening moves as
+// everything it did.
+
+/** What the sink needs from the API layer: the two reads, already bound to a workspace. */
+export interface ToolCallSinkDeps {
+  /**
+   * Whether a workspace is resolved yet.
+   *
+   * Checked BEFORE either read rather than left to the binding throwing, because these loads
+   * record their failures as "this sink did not answer" — a state the panel reports to an
+   * operator — and "no workspace selected yet" is not that. It is nobody having asked.
+   */
+  ready: () => boolean
+  fetchTrajectory: (executionId: string) => Promise<RunToolCallTrajectory & { executionId: string }>
+  fetchFailures: (executionId: string) => Promise<RunToolCallFailures & { executionId: string }>
+}
+
+/**
+ * What {@link ToolCallSinkState.toolCallsFor} answers for a run that was never loaded.
+ *
+ * A frozen shared value rather than a fresh object per call: this is read inside computeds, and a
+ * new identity on every evaluation re-triggers every one of them downstream.
+ */
+export const EMPTY_TRAJECTORY: RunToolCallTrajectory = Object.freeze({
+  toolCalls: Object.freeze([]) as never,
+  truncated: false,
+})
+
+/** Add or remove a key from a reactive `Set` ref, replacing it so the reactivity fires. */
+function withFlag(set: ReturnType<typeof ref<Set<string>>>, key: string, on: boolean) {
+  const next = new Set(set.value)
+  if (on) next.add(key)
+  else next.delete(key)
+  set.value = next
+}
+
+export function createToolCallSinkState(deps: ToolCallSinkDeps) {
+  /**
+   * Per-execution-id trajectory PREFIX (oldest first, the order the agent worked in) with the
+   * flag saying whether the run made more calls than it holds.
+   */
+  const toolCallsByExecution = ref<Record<string, RunToolCallTrajectory>>({})
+  /** Execution ids whose trajectory is currently loading. */
+  const toolCallsLoading = ref<Set<string>>(new Set())
+  /**
+   * Last trajectory-load error per execution id, or null. Recorded for the same reason the
+   * context load records its own: a swallowed failure would render as the "no tool calls
+   * recorded" empty state, which on this sink is a claim rather than a blank tab.
+   */
+  const toolCallErrors = ref<Record<string, string | null>>({})
+  /**
+   * Per-execution-id FAILING tool calls plus the run's exact counts.
+   *
+   * Separate from the trajectory because the numbers here are SQL aggregates over the whole run
+   * while that list is a bounded prefix of it. Counting failures off the prefix is how a panel
+   * ends up printing "nothing failed" over a run whose failures came after its opening moves.
+   */
+  const toolCallFailuresByExecution = ref<Record<string, RunToolCallFailures>>({})
+  /** Execution ids whose failure summary is currently loading. */
+  const toolCallFailuresLoading = ref<Set<string>>(new Set())
+  /**
+   * Last failure-summary load error per execution id, or null.
+   *
+   * The one error state the panel cannot afford to swallow: this read is what the pinned "what
+   * failed" section speaks from, so a failure here has to reach it as "this sink did not answer"
+   * rather than as the zero rows it would otherwise be indistinguishable from.
+   */
+  const toolCallFailureErrors = ref<Record<string, string | null>>({})
+
+  /**
+   * The loaded trajectory prefix, or the empty one.
+   *
+   * `truncated: false` on a run that was never loaded is not a claim that the run is short: a
+   * caller distinguishes the two through {@link hasToolCalls}, never by finding this empty.
+   */
+  function toolCallsFor(executionId: string): RunToolCallTrajectory {
+    return toolCallsByExecution.value[executionId] ?? EMPTY_TRAJECTORY
+  }
+  function isToolCallsLoading(executionId: string): boolean {
+    return toolCallsLoading.value.has(executionId)
+  }
+  /** Whether the trajectory has ever been loaded for this run (an empty answer still counts). */
+  function hasToolCalls(executionId: string): boolean {
+    return executionId in toolCallsByExecution.value
+  }
+
+  /** Load (or refresh) the tool-call trajectory for a run. */
+  async function loadToolCalls(executionId: string) {
+    if (!deps.ready()) return
+    withFlag(toolCallsLoading, executionId, true)
+    toolCallErrors.value = { ...toolCallErrors.value, [executionId]: null }
+    try {
+      const { toolCalls, truncated } = await deps.fetchTrajectory(executionId)
+      toolCallsByExecution.value = {
+        ...toolCallsByExecution.value,
+        [executionId]: { toolCalls, truncated },
+      }
+    } catch (err) {
+      toolCallErrors.value = {
+        ...toolCallErrors.value,
+        [executionId]: err instanceof Error ? err.message : 'Failed to load tool calls',
+      }
+    } finally {
+      withFlag(toolCallsLoading, executionId, false)
+    }
+  }
+
+  /** The loaded failure summary, or null when this run's has not answered (yet, or at all). */
+  function toolCallFailuresFor(executionId: string): RunToolCallFailures | null {
+    return toolCallFailuresByExecution.value[executionId] ?? null
+  }
+  function isToolCallFailuresLoading(executionId: string): boolean {
+    return toolCallFailuresLoading.value.has(executionId)
+  }
+
+  /**
+   * Load (or refresh) the run's failing tool calls and exact counts.
+   *
+   * Cleared on failure rather than left holding the previous answer: a stale summary beside a
+   * fresh error would let the panel keep asserting a failure count the backend just refused to
+   * confirm.
+   */
+  async function loadToolCallFailures(executionId: string) {
+    if (!deps.ready()) return
+    withFlag(toolCallFailuresLoading, executionId, true)
+    toolCallFailureErrors.value = { ...toolCallFailureErrors.value, [executionId]: null }
+    try {
+      const { total, failed, failures, failuresTruncated } = await deps.fetchFailures(executionId)
+      toolCallFailuresByExecution.value = {
+        ...toolCallFailuresByExecution.value,
+        [executionId]: { total, failed, failures, failuresTruncated },
+      }
+    } catch (err) {
+      const { [executionId]: _dropped, ...rest } = toolCallFailuresByExecution.value
+      toolCallFailuresByExecution.value = rest
+      toolCallFailureErrors.value = {
+        ...toolCallFailureErrors.value,
+        [executionId]: err instanceof Error ? err.message : 'Failed to load tool-call failures',
+      }
+    } finally {
+      withFlag(toolCallFailuresLoading, executionId, false)
+    }
+  }
+
+  return {
+    toolCallsByExecution,
+    toolCallErrors,
+    toolCallsFor,
+    hasToolCalls,
+    isToolCallsLoading,
+    loadToolCalls,
+    toolCallFailuresByExecution,
+    toolCallFailureErrors,
+    toolCallFailuresFor,
+    isToolCallFailuresLoading,
+    loadToolCallFailures,
+  }
+}
+
+export type ToolCallSinkState = ReturnType<typeof createToolCallSinkState>

package/app/stores/observability.ts

@@ -7,6 +7,7 @@ import type {
   LlmCallMetric,
 } from '~/types/execution'
 import { useWorkspaceStore } from '~/stores/workspace'
+import { createToolCallSinkState } from '~/stores/observability/toolCalls'
 
 /**
  * LLM observability state: the full per-call model activity for a run (prompts,
@@ -22,6 +23,17 @@ export const useObservabilityStore = defineStore('observability', () => {
   const api = useApi()
   const workspace = useWorkspaceStore()
 
+  /**
+   * The TOOL-CALL sink, extracted whole: two reads at two different bounds, plus the rule that
+   * keeps them apart (see `observability/toolCalls.ts`). The store owns the workspace binding and
+   * nothing else about it.
+   */
+  const toolCalls = createToolCallSinkState({
+    ready: () => !!workspace.workspaceId,
+    fetchTrajectory: (executionId) => api.getToolCalls(workspace.requireId(), executionId),
+    fetchFailures: (executionId) => api.getToolCallFailures(workspace.requireId(), executionId),
+  })
+
   /** Per-execution-id call list (newest first). */
   const callsByExecution = ref<Record<string, LlmCallMetric[]>>({})
   /** Per-execution-id provided-context snapshot list (newest first). */
@@ -223,5 +235,6 @@ export const useObservabilityStore = defineStore('observability', () => {
     searchQueriesFor,
     isSearchQueriesLoading,
     loadSearchQueries,
+    ...toolCalls,
   }
 })

package/app/types/execution.ts

@@ -24,6 +24,7 @@ export type {
   StepPhaseMetrics,
   LlmCallMetric,
   LlmCallActivity,
+  LlmCallOutcome,
   LlmExportInsight,
   LlmMetricsExport,
   PlatformObservability,
@@ -47,6 +48,11 @@ export type {
   ReportTotals,
   ReportsView,
   AgentSearchQuery,
+  AgentToolCall,
+  RunToolCallFailures,
+  RunToolCallTrajectory,
+  ToolCallBodiesState,
+  ToolCallOutcome,
   WebSearchAvailability,
   WebSearchProvider,
   PipelineStep,

package/app/utils/observability.spec.ts

@@ -1,6 +1,26 @@
 import { describe, it, expect } from 'vitest'
-import type { PipelineStep, StepPhaseMetrics } from '~/types/execution'
-import { foldRunPhaseMetrics, formatCost, sumCosts, totalInputTokens } from './observability'
+import type {
+  AgentFailure,
+  AgentToolCall,
+  LlmCallMetric,
+  PipelineStep,
+  RunToolCallFailures,
+  StepPhaseMetrics,
+} from '~/types/execution'
+import type { SinkAnswer } from './observability'
+import {
+  countCallOutcomes,
+  deriveRunFailureEvidence,
+  filterCallsByOutcome,
+  filterToolCallsByOutcome,
+  foldRunPhaseMetrics,
+  formatCost,
+  hasFailureEvidence,
+  noFailingCallReason,
+  sinkAnswer,
+  sumCosts,
+  totalInputTokens,
+} from './observability'
 
 describe('totalInputTokens', () => {
   it('sums all three input classes, so the headline matches Claude Code’s context gauge', () => {
@@ -128,3 +148,294 @@ describe('sumCosts', () => {
     expect(sumCosts([undefined])).toBeNull()
   })
 })
+
+describe('failing-call-first triage', () => {
+  const call = (over: Partial<LlmCallMetric> & Pick<LlmCallMetric, 'id'>): LlmCallMetric =>
+    ({
+      workspaceId: 'ws',
+      executionId: 'run',
+      agentKind: 'coder',
+      provider: 'anthropic',
+      model: 'm',
+      createdAt: 1,
+      streaming: false,
+      phase: 'agent',
+      turnIndex: null,
+      messageCount: 1,
+      toolCount: 0,
+      requestMaxTokens: null,
+      promptTokens: 0,
+      cacheReadTokens: 0,
+      cacheWriteTokens: 0,
+      completionTokens: 0,
+      totalTokens: 0,
+      finishReason: 'stop',
+      upstreamMs: 1,
+      overheadMs: 1,
+      totalMs: 2,
+      ok: true,
+      httpStatus: 200,
+      errorMessage: null,
+      promptText: '',
+      promptPrefixCount: 0,
+      promptHash: '',
+      responseText: '',
+      reasoningText: '',
+      ...over,
+    }) as LlmCallMetric
+
+  const tool = (over: Partial<AgentToolCall> & Pick<AgentToolCall, 'id'>): AgentToolCall => ({
+    workspaceId: 'ws',
+    executionId: 'run',
+    agentKind: 'coder',
+    jobId: 'job',
+    seq: 0,
+    tool: 'bash',
+    startedAt: 1,
+    endedAt: 2,
+    ok: true,
+    bodies: 'stored',
+    args: '',
+    result: '',
+    argsDropped: 0,
+    resultDropped: 0,
+    createdAt: 1,
+    ...over,
+  })
+
+  /** An untruncated failure read holding exactly these rows: the ordinary case. */
+  const toolFailures = (
+    failures: AgentToolCall[],
+    total = failures.length,
+  ): RunToolCallFailures => ({
+    total,
+    failed: failures.length,
+    failures,
+    failuresTruncated: false,
+  })
+
+  const failure = (): AgentFailure => ({
+    kind: 'agent',
+    message: 'the coder step failed',
+    detail: null,
+    hint: null,
+    occurredAt: 5,
+    lastSubtasks: null,
+  })
+
+  describe('countCallOutcomes', () => {
+    it('keeps a failed call and a TRUNCATED one in different buckets', () => {
+      // They need different fixes (transport/proxy/spend versus an output limit), so a filter
+      // that lumped them together would send an operator to the wrong conversation.
+      const counts = countCallOutcomes([
+        call({ id: 'a' }),
+        call({ id: 'b', ok: false, finishReason: null }),
+        call({ id: 'c', finishReason: 'length' }),
+        call({ id: 'd', finishReason: 'content_filter' }),
+      ])
+      expect(counts).toEqual({ all: 4, ok: 1, warning: 2, error: 1 })
+    })
+
+    it('counts a failed call as an error even when its finish reason looks like a warning', () => {
+      // `ok: false` wins: a call that failed AND reported `length` is a failure, not a
+      // truncation, and counting it in both buckets would make the chips sum past the total.
+      const counts = countCallOutcomes([call({ id: 'a', ok: false, finishReason: 'length' })])
+      expect(counts).toEqual({ all: 1, ok: 0, warning: 0, error: 1 })
+    })
+  })
+
+  describe('filterCallsByOutcome', () => {
+    const calls = [
+      call({ id: 'ok' }),
+      call({ id: 'warn', finishReason: 'length' }),
+      call({ id: 'err', ok: false }),
+    ]
+
+    it('narrows to one class and passes everything through on `all`', () => {
+      expect(filterCallsByOutcome(calls, 'error').map((c) => c.id)).toEqual(['err'])
+      expect(filterCallsByOutcome(calls, 'warning').map((c) => c.id)).toEqual(['warn'])
+      expect(filterCallsByOutcome(calls, 'all').map((c) => c.id)).toEqual(['ok', 'warn', 'err'])
+    })
+
+    it('returns a fresh array, never the caller’s own list', () => {
+      // The panel holds the store's array; a filter that aliased it on `all` would let a sort in
+      // the component reorder the store.
+      expect(filterCallsByOutcome(calls, 'all')).not.toBe(calls)
+    })
+  })
+
+  describe('filterToolCallsByOutcome', () => {
+    it('narrows a trajectory to the failing calls, keeping their order', () => {
+      const trajectory = [
+        tool({ id: '1' }),
+        tool({ id: '2', ok: false }),
+        tool({ id: '3' }),
+        tool({ id: '4', ok: false }),
+      ]
+      expect(filterToolCallsByOutcome(trajectory, 'error').map((c) => c.id)).toEqual(['2', '4'])
+      expect(filterToolCallsByOutcome(trajectory, 'ok').map((c) => c.id)).toEqual(['1', '3'])
+      expect(filterToolCallsByOutcome(trajectory, 'all')).toHaveLength(4)
+    })
+  })
+
+  describe('deriveRunFailureEvidence', () => {
+    /** Both sinks answered with the given rows. */
+    const answered = (rows: number): SinkAnswer => ({ status: 'answered', rows })
+    const evidenceFor = (input: {
+      failure?: AgentFailure | null
+      calls?: LlmCallMetric[]
+      toolFailures?: RunToolCallFailures | null
+    }) => {
+      const calls = input.calls ?? []
+      const toolFailures = input.toolFailures ?? null
+      return deriveRunFailureEvidence({
+        failure: input.failure ?? null,
+        calls,
+        callsAnswer: answered(calls.length),
+        toolFailures,
+        toolsAnswer: toolFailures ? answered(toolFailures.total) : { status: 'pending' },
+      })
+    }
+
+    it('picks the LAST failing row from each sink, respecting their opposite orders', () => {
+      // Calls arrive newest-first; the failing tool calls arrive oldest-first (trajectory order).
+      // Reading either the wrong way round still yields a failing call, just not the one nearest
+      // the failure, which is the only reason to pin one.
+      const evidence = evidenceFor({
+        failure: failure(),
+        calls: [
+          call({ id: 'newest-error', ok: false, createdAt: 30 }),
+          call({ id: 'older-error', ok: false, createdAt: 10 }),
+        ],
+        toolFailures: toolFailures([
+          tool({ id: 'oldest-fail', ok: false, startedAt: 10 }),
+          tool({ id: 'latest-fail', ok: false, startedAt: 30 }),
+        ]),
+      })
+      expect(evidence.lastErroredCall?.id).toBe('newest-error')
+      expect(evidence.lastFailedToolCall?.id).toBe('latest-fail')
+      expect(evidence.erroredCallCount).toBe(2)
+      expect(evidence.failedToolCallCount).toBe(2)
+    })
+
+    it('counts failing tool calls off the run AGGREGATE, never off the rows it holds', () => {
+      // The failure this split exists to prevent. The backend narrows and counts over the whole
+      // run; the rows come back bounded. A count taken from the list would under-report exactly
+      // the long runs worth opening the panel for, and would disagree with the debug overview's
+      // `toolCalls.totals.failures` on the same run.
+      const evidence = evidenceFor({
+        failure: failure(),
+        toolFailures: {
+          total: 5_000,
+          failed: 240,
+          failures: [tool({ id: 'held', ok: false })],
+          failuresTruncated: true,
+        },
+      })
+      expect(evidence.failedToolCallCount).toBe(240)
+      expect(evidence.failedToolCallsTruncated).toBe(true)
+      expect(evidence.lastFailedToolCall?.id).toBe('held')
+    })
+
+    it('reports a run whose model calls are all healthy but whose tools are not', () => {
+      // The whole failure class this surface exists for: the model call that requested the tool
+      // still reports `ok`, so every LLM rollup reads clean.
+      const evidence = evidenceFor({
+        failure: failure(),
+        calls: [call({ id: 'fine' })],
+        toolFailures: toolFailures([tool({ id: 'broke', ok: false })]),
+      })
+      expect(evidence.lastErroredCall).toBeNull()
+      expect(evidence.lastFailedToolCall?.id).toBe('broke')
+      expect(hasFailureEvidence(evidence)).toBe(true)
+    })
+
+    it('has nothing to pin for a run that neither failed nor recorded a failing call', () => {
+      expect(hasFailureEvidence(evidenceFor({ calls: [call({ id: 'a' })] }))).toBe(false)
+    })
+
+    it('pins the section for an UNREACHABLE sink even with no failure and no failing call', () => {
+      // "One of these reads did not come back" has to reach the operator before they read the
+      // rest of the page as whole. Staying silent is how the other numbers get believed.
+      const evidence = deriveRunFailureEvidence({
+        failure: null,
+        calls: [],
+        callsAnswer: { status: 'answered', rows: 0 },
+        toolFailures: null,
+        toolsAnswer: { status: 'unreachable' },
+      })
+      expect(hasFailureEvidence(evidence)).toBe(true)
+    })
+  })
+
+  describe('sinkAnswer', () => {
+    it('reports an in-flight read as pending whatever it is still holding', () => {
+      // A refresh over a previous answer (or a previous error) is not that answer: what the panel
+      // says next depends on what is coming back.
+      expect(sinkAnswer({ loading: true, error: 'boom', loaded: true, rows: 3 })).toEqual({
+        status: 'pending',
+      })
+    })
+
+    it('separates a failed read from an answer of zero rows', () => {
+      expect(sinkAnswer({ loading: false, error: 'boom', loaded: false, rows: 0 })).toEqual({
+        status: 'unreachable',
+      })
+      expect(sinkAnswer({ loading: false, error: null, loaded: true, rows: 0 })).toEqual({
+        status: 'answered',
+        rows: 0,
+      })
+    })
+
+    it('treats never-requested as pending, not as an answer', () => {
+      expect(sinkAnswer({ loading: false, error: null, loaded: false, rows: 0 })).toEqual({
+        status: 'pending',
+      })
+    })
+  })
+
+  describe('noFailingCallReason', () => {
+    const reasonFor = (calls: SinkAnswer, tools: SinkAnswer, failed = 0) =>
+      noFailingCallReason(
+        deriveRunFailureEvidence({
+          failure: failure(),
+          calls: [],
+          callsAnswer: calls,
+          toolFailures: { total: 0, failed, failures: [], failuresTruncated: false },
+          toolsAnswer: tools,
+        }),
+      )
+    const answered = (rows: number): SinkAnswer => ({ status: 'answered', rows })
+
+    it('says nothing when a failing call WAS found', () => {
+      expect(reasonFor(answered(1), answered(1), 1)).toBeNull()
+    })
+
+    it('distinguishes "both sinks answered and nothing failed" from "nothing was recorded"', () => {
+      // The distinction the whole helper exists for: an unwired sink, a capture opt-out and a
+      // container that died before reporting all produce zero failing rows, exactly like a run
+      // whose every call succeeded, and only one of those is a clean bill of health.
+      expect(reasonFor(answered(1), answered(1))).toBe('recorded-clean')
+      expect(reasonFor(answered(0), answered(0))).toBe('no-telemetry')
+    })
+
+    it('names WHICH sink is empty when only one holds rows', () => {
+      expect(reasonFor(answered(1), answered(0))).toBe('partial-calls-only')
+      expect(reasonFor(answered(0), answered(1))).toBe('partial-tools-only')
+    })
+
+    it('withholds every verdict while a sink has not answered yet', () => {
+      // A read still in flight is not a read that came back clean, and rendering it as one is a
+      // clean bill of health written before the evidence arrived.
+      expect(reasonFor({ status: 'pending' }, answered(1))).toBeNull()
+      expect(reasonFor(answered(1), { status: 'pending' })).toBeNull()
+    })
+
+    it('lets an UNREACHABLE sink outrank every statement about the run', () => {
+      // Each other reason is a claim about the run. This is the one case where the panel has no
+      // standing to make one, so it must win even against a sink that answered richly.
+      expect(reasonFor({ status: 'unreachable' }, answered(9))).toBe('sink-unreachable')
+      expect(reasonFor(answered(9), { status: 'unreachable' })).toBe('sink-unreachable')
+    })
+  })
+})