@frontera-sdk/cli 1.52.39 → 1.52.41

package/src/local/local-test-runner.ts

@@ -3,6 +3,13 @@ import { join, relative } from 'node:path'
 
 import { parse } from 'yaml'
 
+import { contentHash, loadBuild } from './local-applied'
+import { caseCouldNotRun, stopsTheRun, withDirectModel, type CaseNotRun } from './local-case-not-run'
+import { directModelFor } from './local-seed'
+import { allTargets } from './local-dev-sync'
+import type { LocalProject } from './local-project'
+import { checkAgainstSchema, describeProblems } from './local-schema-check'
+import { cliSchema, MAX_TURN_WAIT_SECONDS } from './local-schemas'
 import type { LocalCredentials } from './local-state'
 
 /**
@@ -25,11 +32,31 @@ import type { LocalCredentials } from './local-state'
 export type TestCheck =
   | { type: 'contains'; value: string; text?: string }
   | { type: 'regex'; value: string; flags?: string; text?: string }
-  | { type: 'tool_called' | 'tool_not_called'; tool: string; argsSubset?: Record<string, unknown>; mode?: 'exact' | 'ignore' | 'subset' | 'superset'; text?: string }
+  | {
+      type: 'tool_called' | 'tool_not_called'
+      tool: string
+      argsSubset?: Record<string, unknown>
+      mode?: 'exact' | 'ignore' | 'subset' | 'superset'
+      text?: string
+      /** The calls of every turn (`all`, the default) or only of the last one, whose reply the case grades. */
+      turn?: 'all' | 'last'
+    }
+
+/**
+ * One turn of a case's conversation: a person's message (sent), or an
+ * assistant's (kept for whoever reads the case). `waitSeconds` on a person's
+ * turn waits that long before sending it, so a case can keep a gap between
+ * turns ("turn 2 a minute after turn 1").
+ */
+export interface CaseTurn {
+  user?: string
+  assistant?: string
+  waitSeconds?: number
+}
 
 export interface TestCase {
   name: string
-  conversation: Array<Record<string, string>>
+  conversation: CaseTurn[]
   expected?: string
   /** Pass when the reply contains each of these, instead of matching exactly. */
   expectedContains?: string[]
@@ -39,10 +66,23 @@ export interface TestCase {
   judged?: string[]
   /** Human-certified on the platform. */
   gold?: boolean
+  /** Guards the Agent in change requests: run, against the last applied run, before a request applies (design 2026-10-18 §6). */
+  gate?: boolean
+  /** A turn that stops on a tool approval: approve or deny it and read on, or `expect` it (the case passes by waiting). */
+  decision?: CaseDecision
+  /** Run the tools that change something for real (a decision implies it); simulated otherwise. */
+  realActions?: boolean
+  /** The conversation's variables (`customer_id: CST-00060`), set before its first message. */
+  variables?: Record<string, string | number | boolean>
   /** A conversation with a simulated customer, run by the local runtime's evaluation runner. */
   scenario?: Record<string, unknown>
   /** What the scenario expects: its outcome, the actions that must (or must not) run. */
   expect?: Record<string, unknown>
+  /**
+   * Where the case breaks the evals format (`frontera local schema evals`), by
+   * file and path; such a case fails with these and is not sent to the Agent.
+   */
+  problems?: string[]
 }
 
 /** A tool the Agent called during the case, from the chat stream. */
@@ -51,6 +91,13 @@ export interface ToolCall {
   args: unknown
   /** It ran to completion. */
   ok: boolean
+  /**
+   * It waits on a person's approval (the turn parked on it, and nobody has
+   * decided it): the Agent called it, so `tool_called` counts it, with the
+   * arguments the approval shows. A call approved and run is `ok`; one denied
+   * is neither.
+   */
+  parked?: boolean
 }
 
 export interface CaseResult {
@@ -64,16 +111,72 @@ export interface CaseResult {
   /** The Agent's config id and the conversation the case ran in, so Studio can open it. */
   agentId?: string
   conversationId?: string
+  /** When the case was sent, so its traces can be found by their time (`local-case-evidence.ts`). */
+  startedAt?: string
+  /** How many tools the Agent called, for efficiency against a baseline. */
+  toolCalls?: number
+  /**
+   * The waits the case kept between turns (`waitSeconds`), in seconds, in
+   * order: said in the run's output, since such a case takes that long.
+   */
+  waited?: Array<{ turn: number; seconds: number }>
+  /**
+   * The model provider refused the first try for its limits (a 402 or 429
+   * about credits, rate or in-flight requests), so the case was run once more
+   * after a pause: what it said the first time.
+   */
+  retried?: string
+  /**
+   * The case couldn't run: the model provider refused it (credits, a rate
+   * limit, a key), the turn timed out, or the local runtime did not answer.
+   * It is not passed, and it is not a failure of the Agent: never newly
+   * failing, and never recorded as the case's last outcome.
+   */
+  notRun?: CaseNotRun
 }
 
 export interface CaseFile {
   agent: string
   file: string
   cases: TestCase[]
+  /**
+   * Why the file as a whole cannot run (it is not YAML, or has no `cases` list),
+   * said as one failed result rather than thrown, so one bad file fails only itself.
+   */
+  problem?: string
   /** The Agent's regression dataset in the workspace: `local test --regression` runs only these. */
   regression?: boolean
 }
 
+/**
+ * Only the cases marked `gate: true`, file by file; files left with none are
+ * dropped. Each case is the one the file gave, every field kept (`variables`,
+ * `realActions`, `decision`, `checks`): a gate run sends what any other run
+ * sends. A file that can't be read is kept: it may hold gate cases, and it
+ * fails with why rather than vanish from the run.
+ */
+export function gatedCaseFiles(files: CaseFile[]): CaseFile[] {
+  return files.map((f) => (f.problem ? f : { ...f, cases: f.cases.filter((c) => c.gate === true) })).filter((f) => f.cases.length || f.problem)
+}
+
+/**
+ * The case files a run takes, chosen the same way by every path that runs
+ * them (`frontera local test`, `local check`, MCP `test_run`, a change
+ * request's evals, Studio): one Agent's, one file, the regression datasets,
+ * the gate cases. The cases themselves are always as `readCaseFile` read
+ * them, so no path drops a field another sends.
+ */
+export function selectCaseFiles(root: string, scope: { agent?: string; file?: string; regression?: boolean; gate?: boolean } = {}): CaseFile[] {
+  const file = scope.file?.replace(/^\.\//, '')
+  const files = findCaseFiles(root, scope.agent)
+    .filter((f) => !file || f.file === file)
+    .filter((f) => !scope.regression || f.regression)
+  return scope.gate ? gatedCaseFiles(files) : files
+}
+
+/** How many results a run of these files gives: each case, or one for a file that can't be read. */
+export const caseCount = (files: CaseFile[]): number => files.reduce((n, f) => n + Math.max(f.cases.length, f.problem ? 1 : 0), 0)
+
 export function findCaseFiles(root: string, agent?: string): CaseFile[] {
   const agentsDir = join(root, 'agents')
   if (!existsSync(agentsDir)) return []
@@ -82,25 +185,62 @@ export function findCaseFiles(root: string, agent?: string): CaseFile[] {
     if (agent && slug !== agent) continue
     const evalsDir = join(agentsDir, slug, 'evals')
     if (!existsSync(evalsDir)) continue
-    for (const name of readdirSync(evalsDir)) {
-      if (!/\.ya?ml$/.test(name)) continue
+    for (const name of readdirSync(evalsDir).sort()) {
+      if (!/\.ya?ml$/.test(name) || name.startsWith('.')) continue
       const path = join(evalsDir, name)
-      const doc = parse(readFileSync(path, 'utf8')) as { cases?: TestCase[]; regression?: unknown } | null
-      out.push({ agent: slug, file: relative(root, path), cases: doc?.cases ?? [], ...(doc?.regression === true ? { regression: true } : {}) })
+      out.push(readCaseFile(slug, relative(root, path), readFileSync(path, 'utf8')))
     }
   }
   return out
 }
 
+/**
+ * One evals file, checked against its format before anything runs: a problem
+ * with the file as a whole is the file's `problem`; a case that breaks the
+ * format carries its `problems` (`agents/x/evals/y.yaml: cases.2.checks.0.value:
+ * must be text`) and fails with them instead of crashing the run.
+ */
+export function readCaseFile(agent: string, file: string, text: string): CaseFile {
+  let doc: unknown
+  try {
+    doc = parse(text)
+  } catch (err) {
+    return { agent, file, cases: [], problem: `${file}: the file is not YAML: ${(err as Error).message.split('\n')[0]}` }
+  }
+  const raw = (doc ?? {}) as { cases?: unknown; regression?: unknown }
+  const regression = raw.regression === true ? { regression: true } : {}
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc) || !Array.isArray(raw.cases)) {
+    return { agent, file, cases: [], problem: `${file}: the file needs a \`cases:\` list (\`frontera local schema evals\`)`, ...regression }
+  }
+  const problems = checkAgainstSchema(doc, cliSchema('evals')!)
+  const byCase = new Map<number, string[]>()
+  const fileWide: string[] = []
+  for (const p of problems) {
+    const m = /^cases\.(\d+)(\.|$)/.exec(p.path)
+    if (m) byCase.set(Number(m[1]), [...(byCase.get(Number(m[1])) ?? []), ...describeProblems([p], file)])
+    else fileWide.push(...describeProblems([p], file))
+  }
+  const cases = (raw.cases as unknown[]).map((c, i) => {
+    const o = (c && typeof c === 'object' && !Array.isArray(c) ? c : {}) as TestCase
+    const name = typeof o.name === 'string' && o.name ? o.name : `case ${i + 1}`
+    const found = [...fileWide, ...(byCase.get(i) ?? [])]
+    // The evaluation runner that runs a scenario gives it no variables yet: said, not silently dropped.
+    if (o.scenario && o.variables && Object.keys(o.variables).length) found.push(`${file}: cases.${i} (${name}): a scenario case can’t set \`variables\` yet (the evaluation runner gives a scenario none); give the variables to a case with \`conversation\` instead`)
+    return { ...o, name, ...(found.length ? { problems: found } : {}) }
+  })
+  return { agent, file, cases, ...regression }
+}
+
 function normalise(text: string): string {
   return text.replace(/\s+/g, ' ').trim()
 }
 
-export function judge(c: TestCase, reply: string, toolCalls: ToolCall[] = []): { passed: boolean; reason?: string } {
+/** `lastTurnCalls`: the last turn's calls, for a check scoped `turn: last`; every call when not given. */
+export function judge(c: TestCase, reply: string, toolCalls: ToolCall[] = [], lastTurnCalls?: ToolCall[]): { passed: boolean; reason?: string } {
   const answer = judgeAnswer(c, reply)
   if (!answer.passed) return answer
   for (const check of c.checks ?? []) {
-    const failed = runCheck(check, reply, toolCalls)
+    const failed = runCheck(check, reply, toolCalls, lastTurnCalls)
     if (failed) return { passed: false, reason: failed }
   }
   return { passed: true }
@@ -129,14 +269,23 @@ function argsMatch(check: Extract<TestCheck, { tool: string }>, call: ToolCall):
   const mode = check.mode ?? 'superset'
   if (mode === 'ignore' || !check.argsSubset) return true
   if (!isRecord(call.args)) return false
-  if (mode === 'superset') return containsAll(call.args, check.argsSubset)
-  if (mode === 'subset') return containsAll(check.argsSubset, call.args)
-  return containsAll(call.args, check.argsSubset) && Object.keys(call.args).length === Object.keys(check.argsSubset).length
+  // Every tool takes an `action` narration ("Running intake…") and strips it: not an argument the case means.
+  const { action: _narration, ...rest } = call.args
+  const args = 'action' in check.argsSubset ? call.args : rest
+  if (mode === 'superset') return containsAll(args, check.argsSubset)
+  if (mode === 'subset') return containsAll(check.argsSubset, args)
+  return containsAll(args, check.argsSubset) && Object.keys(args).length === Object.keys(check.argsSubset).length
 }
 
 /** Why a check failed, or null when it passed. */
-export function runCheck(check: TestCheck, reply: string, toolCalls: ToolCall[]): string | null {
+export function runCheck(check: TestCheck, reply: string, allCalls: ToolCall[], lastTurnCalls?: ToolCall[]): string | null {
   const label = check.text ? ` (${check.text})` : ''
+  const lastOnly = 'turn' in check && check.turn === 'last'
+  const toolCalls = lastOnly && lastTurnCalls ? lastTurnCalls : allCalls
+  // The format says text; a number or a missing value is refused, not thrown on.
+  if ((check.type === 'contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) {
+    return `the ${check.type} check's \`value\` must be text (it is ${check.value === undefined ? 'missing' : JSON.stringify(check.value)}); quote it in the file${label}`
+  }
   if (check.type === 'contains') return reply.toLowerCase().includes(check.value.toLowerCase()) ? null : `reply does not mention: ${check.value}${label}`
   if (check.type === 'regex') {
     if (check.value.length > 500 || NESTED_QUANTIFIER.test(check.value)) return `the pattern ${check.value} can't run safely${label}`
@@ -148,12 +297,14 @@ export function runCheck(check: TestCheck, reply: string, toolCalls: ToolCall[])
     }
     return pattern.test(reply.slice(0, 20_000)) ? null : `nothing in the reply matches /${check.value}/${label}`
   }
-  // A call that ran proves a tool was called; any attempt counts against one that must not be.
-  const candidates = check.type === 'tool_called' ? toolCalls.filter((c) => c.ok) : toolCalls
+  // A call that ran, or one waiting on a person's approval, proves a tool was called; any attempt counts against one that must not be.
+  const candidates = check.type === 'tool_called' ? toolCalls.filter((c) => c.ok || c.parked) : toolCalls
   const matched = candidates.some((c) => c.name === check.tool && argsMatch(check, c))
-  const made = toolCalls.length ? `calls made: ${[...new Set(toolCalls.map((c) => c.name))].join(', ')}` : 'no tools were called'
-  if (check.type === 'tool_called') return matched ? null : `no matching call to ${check.tool}; ${made}${label}`
-  return matched ? `${check.tool} was called, which the case does not allow${label}` : null
+  const said = (c: ToolCall) => `${c.name}${c.parked ? ' (waiting on approval)' : !c.ok ? ' (did not run)' : ''}`
+  const made = toolCalls.length ? `calls made: ${[...new Set(toolCalls.map(said))].join(', ')}` : 'no tools were called'
+  const where = lastOnly ? ' in the last turn' : ''
+  if (check.type === 'tool_called') return matched ? null : `no matching call to ${check.tool}${where}; ${made}${label}`
+  return matched ? `${check.tool} was called${where}, which the case does not allow${label}` : null
 }
 
 export interface Api {
@@ -161,7 +312,7 @@ export interface Api {
   creds: LocalCredentials
 }
 
-async function call<T>(api: Api, path: string, init: { method?: string; body?: unknown } = {}): Promise<T> {
+export async function call<T>(api: Api, path: string, init: { method?: string; body?: unknown } = {}): Promise<T> {
   const res = await fetch(new URL(path, api.serviceUrl), {
     method: init.method ?? (init.body === undefined ? 'GET' : 'POST'),
     headers: {
@@ -182,50 +333,264 @@ async function call<T>(api: Api, path: string, init: { method?: string; body?: u
 const TURN_TIMEOUT_MS = 180_000
 
 /** Send one person turn and collect the streamed reply's text and the tools it called. */
-async function sendTurn(api: Api, conversationId: string, agentName: string, text: string): Promise<{ reply: string; toolCalls: ToolCall[] }> {
+/** A turn that stopped on a decision a person makes: a tool waiting for approval, or a confirmation. */
+export interface PendingDecision {
+  /** The tool that waits, when the stream named it. */
+  tool: string
+  /** Present for a tool approval: what `POST /v1/chat/approvals/:approvalId` decides. */
+  approvalId?: string
+  /** The waiting call, when the stream gave its id. */
+  toolCallId?: string
+}
+
+/**
+ * The call a `tool-approval-request` parks, marked as waiting: the call the
+ * stream announced under its id, or, when the request is the first the
+ * stream says of it, one made from the request's own tool name and
+ * arguments (`toolName`, `input` or `args`). Null when it names no tool.
+ */
+export function parkCall(calls: Map<string, ToolCall>, event: { toolCallId?: string; toolName?: string; input?: unknown; args?: unknown; approvalId?: string }): (ToolCall & { id: string }) | null {
+  const id = event.toolCallId ?? (event.approvalId ? `approval:${event.approvalId}` : undefined)
+  if (!id) return null
+  let found = calls.get(id) as (ToolCall & { id?: string }) | undefined
+  if (!found) {
+    if (!event.toolName) return null
+    found = { id, name: event.toolName, args: event.input ?? event.args, ok: false }
+    calls.set(id, found)
+  } else if (found.args === undefined && (event.input ?? event.args) !== undefined) found.args = event.input ?? event.args
+  if (!found.ok) found.parked = true
+  return { ...found, id }
+}
+
+/** How a case answers a decision its turns stop on (`decision` in the evals format). */
+export type CaseDecision = 'approve' | 'deny' | 'expect'
+
+const chatHeaders = (api: Api) => ({
+  authorization: `Bearer ${api.creds.session}`,
+  'x-org-id': api.creds.organizationId,
+  'x-workspace-id': api.creds.workspaceId,
+  'content-type': 'application/json',
+})
+
+/**
+ * Read a turn's stream into the reply and the tools called. Stops at a
+ * decision a person makes (a tool approval, a confirmation): the stream stays
+ * open while the card waits, so reading on would hang the case until it timed
+ * out. Counts the events read, so a resumed stream starts after them.
+ */
+async function readTurn(body: ReadableStream<Uint8Array>, into: { reply: string; calls: Map<string, ToolCall>; events: number; seen: Set<string> }): Promise<PendingDecision | null> {
+  let buffer = ''
+  const decoder = new TextDecoder()
+  const reader = body.getReader()
+  try {
+    for (;;) {
+      const { value, done } = await reader.read()
+      if (done) return null
+      buffer += decoder.decode(value, { stream: true })
+      let newline: number
+      while ((newline = buffer.indexOf('\n')) >= 0) {
+        const line = buffer.slice(0, newline).trim()
+        buffer = buffer.slice(newline + 1)
+        if (!line.startsWith('data:')) continue
+        const payload = line.slice(5).trim()
+        if (payload === '[DONE]') continue
+        let event: { type?: string; delta?: string; errorText?: string; toolCallId?: string; toolName?: string; input?: unknown; args?: unknown; approvalId?: string; data?: Record<string, unknown> }
+        try {
+          event = JSON.parse(payload)
+        } catch {
+          continue
+        }
+        into.events += 1
+        if (event.type === 'text-delta' && typeof event.delta === 'string') into.reply += event.delta
+        if (event.type === 'tool-input-available' && event.toolCallId && event.toolName) into.calls.set(event.toolCallId, { name: event.toolName, args: event.input, ok: false })
+        if (event.type === 'tool-output-available' && event.toolCallId && into.calls.has(event.toolCallId)) Object.assign(into.calls.get(event.toolCallId)!, { ok: true, parked: false })
+        if ((event.type === 'tool-output-denied' || event.type === 'tool-output-error') && event.toolCallId && into.calls.has(event.toolCallId)) into.calls.get(event.toolCallId)!.parked = false
+        if (event.type === 'error') throw new Error(event.errorText ?? 'the turn failed')
+        // The turn is over: what follows is keepalives until the socket closes.
+        if (event.type === 'finish') return null
+        // Each decision once: a stream read on from an earlier point may say it again.
+        if (event.type === 'tool-approval-request' && !(event.approvalId && into.seen.has(event.approvalId))) {
+          if (event.approvalId) into.seen.add(event.approvalId)
+          const parked = parkCall(into.calls, event)
+          const tool = parked?.name || event.toolName || 'a tool'
+          return { tool, ...(typeof event.approvalId === 'string' ? { approvalId: event.approvalId } : {}), ...(parked?.id ? { toolCallId: parked.id } : {}) }
+        }
+        const card = event.type === 'data-person-confirmation' ? event.data : undefined
+        if (card && card.status === 'pending' && typeof card.id === 'string' && !into.seen.has(card.id)) {
+          into.seen.add(card.id)
+          return { tool: typeof card.summary === 'string' && card.summary ? `a confirmation (${card.summary})` : 'a confirmation' }
+        }
+      }
+    }
+  } finally {
+    // A stream parked on a decision is let go of, not drained.
+    await reader.cancel().catch(() => {})
+  }
+}
+
+/**
+ * One user turn: the reply, the tools called, and the decision it stopped on
+ * when it waits for a person. With `decide` (approve or deny) a tool approval
+ * is answered (`POST /v1/chat/approvals/:approvalId`) and the run's stream
+ * read on from where it stopped (`GET /v1/chat/:runId/stream?startIndex=N`),
+ * at most five times in a turn.
+ */
+export async function sendTurn(api: Api, conversationId: string, agentName: string, text: string, decide?: 'approve' | 'deny'): Promise<{ reply: string; toolCalls: ToolCall[]; pending?: PendingDecision }> {
+  const signal = AbortSignal.timeout(TURN_TIMEOUT_MS)
   const res = await fetch(new URL('/v1/chat', api.serviceUrl), {
     method: 'POST',
-    headers: {
-      authorization: `Bearer ${api.creds.session}`,
-      'x-org-id': api.creds.organizationId,
-      'x-workspace-id': api.creds.workspaceId,
-      'content-type': 'application/json',
-    },
+    headers: chatHeaders(api),
     body: JSON.stringify({
       conversationId,
       agentName,
       messages: [{ id: crypto.randomUUID(), role: 'user', parts: [{ type: 'text', text }] }],
     }),
     // A reply that never finishes (the stream's keepalives hold it open) fails the case, not the whole run.
-    signal: AbortSignal.timeout(TURN_TIMEOUT_MS),
+    signal,
   })
-  if (!res.ok || !res.body) throw new Error(`chat answered ${res.status}: ${(await res.text()).slice(0, 200)}`)
-  let reply = ''
-  let buffer = ''
-  const calls = new Map<string, ToolCall>()
-  const decoder = new TextDecoder()
-  for await (const chunk of res.body as unknown as AsyncIterable<Uint8Array>) {
-    buffer += decoder.decode(chunk, { stream: true })
-    let newline: number
-    while ((newline = buffer.indexOf('\n')) >= 0) {
-      const line = buffer.slice(0, newline).trim()
-      buffer = buffer.slice(newline + 1)
-      if (!line.startsWith('data:')) continue
-      const payload = line.slice(5).trim()
-      if (payload === '[DONE]') continue
-      try {
-        const event = JSON.parse(payload) as { type?: string; delta?: string; errorText?: string; toolCallId?: string; toolName?: string; input?: unknown }
-        if (event.type === 'text-delta' && typeof event.delta === 'string') reply += event.delta
-        if (event.type === 'tool-input-available' && event.toolCallId && event.toolName) calls.set(event.toolCallId, { name: event.toolName, args: event.input, ok: false })
-        if (event.type === 'tool-output-available' && event.toolCallId && calls.has(event.toolCallId)) calls.get(event.toolCallId)!.ok = true
-        if (event.type === 'error') throw new Error(event.errorText ?? 'the turn failed')
-      } catch (err) {
-        if (err instanceof SyntaxError) continue
-        throw err
-      }
-    }
+  if (!res.ok || !res.body) {
+    const body = await res.text()
+    // The conversation still waits on a decision from an earlier turn.
+    if (res.status === 409 && /decision_pending/.test(body)) throw new Error('the conversation waits for a decision from an earlier turn; give the case `decision: approve` or `deny`')
+    throw new Error(`chat answered ${res.status}: ${body.slice(0, 200)}`)
+  }
+  const runId = res.headers.get('x-workflow-run-id')
+  const into = { reply: '', calls: new Map<string, ToolCall>(), events: 0, seen: new Set<string>() }
+  let pending = await readTurn(res.body, into)
+  for (let n = 0; pending && decide && pending.approvalId && runId && n < 5; n++) {
+    const decided = await fetch(new URL(`/v1/chat/approvals/${encodeURIComponent(pending.approvalId)}`, api.serviceUrl), { method: 'POST', headers: chatHeaders(api), body: JSON.stringify({ decision: decide }), signal })
+    if (!decided.ok) throw new Error(`deciding ${pending.tool} answered ${decided.status}: ${(await decided.text()).slice(0, 200)}`)
+    // Decided: no longer waiting. Approved, it runs (and its output marks it done); denied, it never runs.
+    const waiting = pending.toolCallId ? into.calls.get(pending.toolCallId) : undefined
+    if (waiting) waiting.parked = false
+    const resumed = await fetch(new URL(`/v1/chat/${encodeURIComponent(runId)}/stream?startIndex=${into.events}`, api.serviceUrl), { headers: chatHeaders(api), signal })
+    if (!resumed.ok || !resumed.body) throw new Error(`reading on after ${pending.tool} answered ${resumed.status}`)
+    pending = await readTurn(resumed.body, into)
+  }
+  return { reply: into.reply, toolCalls: [...into.calls.values()], ...(pending ? { pending } : {}) }
+}
+
+/**
+ * Whether a case runs real actions: it says so, or it answers a decision, which
+ * only a real action raises (a simulated one answers an empty success).
+ */
+export function usesRealActions(c: Pick<TestCase, 'realActions' | 'decision'>): boolean {
+  return c.realActions === true || c.decision === 'approve' || c.decision === 'deny' || c.decision === 'expect'
+}
+
+/**
+ * Switch a test conversation to real actions before its first message, the
+ * call `agent_chat` with realActions makes. `agentId` is the Agent's own id,
+ * not its preview copy's. Sent only for such a case: the call without
+ * `simulateActions` means real actions.
+ */
+export async function useRealActions(api: Api, agentId: string, conversationId: string, slug: string, fetcher: typeof fetch = fetch): Promise<void> {
+  return setUpConversation(api, agentId, conversationId, slug, { realActions: true }, fetcher)
+}
+
+/**
+ * Set a test conversation up before its first message, in ONE call (the
+ * runtime refuses a second): real actions or simulated, and the case's
+ * variables. Leaving `simulateActions` out means real actions, so a case
+ * with variables and simulated actions says `simulateActions: true`. Nothing
+ * is sent for a simulated case without variables.
+ */
+export async function setUpConversation(
+  api: Api,
+  agentId: string,
+  conversationId: string,
+  slug: string,
+  setup: { realActions: boolean; variables?: Record<string, unknown> },
+  fetcher: typeof fetch = fetch,
+): Promise<void> {
+  const variables = setup.variables && Object.keys(setup.variables).length ? setup.variables : undefined
+  if (!setup.realActions && !variables) return
+  const res = await fetcher(new URL(`/v1/config/agents/${encodeURIComponent(agentId)}/try/conversations/${encodeURIComponent(conversationId)}`, api.serviceUrl), {
+    method: 'PUT',
+    headers: { authorization: `Bearer ${api.creds.session}`, 'x-org-id': api.creds.organizationId, 'x-workspace-id': api.creds.workspaceId, 'content-type': 'application/json' },
+    body: JSON.stringify({ ...(variables ? { variables } : {}), simulateActions: !setup.realActions }),
+    signal: AbortSignal.timeout(60_000),
+  })
+  if (res.ok) return
+  const text = (await res.text()).replace(/\s+/g, ' ').slice(0, 240)
+  const what = [setup.realActions ? 'real actions (it has `decision` or `realActions: true`)' : '', variables ? `the variables ${Object.keys(variables).join(', ')}` : ''].filter(Boolean).join(' and ')
+  if (res.status === 400) throw new Error(`this case sets up ${what}, which agents/${slug} refused here: ${text}`)
+  throw new Error(`the conversation could not be set up with ${what}: ${res.status} ${text}`)
+}
+
+/**
+ * Why an Agent's draft on the runtime is not what its files say, or null when
+ * it is: a file of it (agent.json, operating.json, bindings.json,
+ * lessons.yaml) changed since `local dev` applied it, never applied, or did not
+ * apply. Read from the build record (`.frontera/local/build.json`).
+ */
+export function agentFilesProblem(project: LocalProject, organizationId: string | null, slug: string): string | null {
+  const targets = allTargets(project.root).filter((t) => (t.kind === 'agent' || t.kind === 'operating' || t.kind === 'bindings' || t.kind === 'lessons') && t.slug === slug)
+  // An Agent the project has no files for (made in Studio only) is tested as the runtime has it.
+  if (!targets.length) return null
+  const build = loadBuild(project, organizationId)
+  const again = 'run `frontera local dev --once` (or keep `frontera local dev` running), then test again'
+  for (const t of targets) {
+    const file = (t as { file: string }).file
+    const entry = build[t.key]
+    if (!entry) return `${file} is not applied to the local runtime yet: ${again}`
+    if (entry.status === 'failed' || entry.status === 'blocked') return `${file} did not apply (${entry.status}${entry.error ? `: ${entry.error}` : ''}), so the Agent's draft is not what its files say: fix it, ${again}`
+    if (entry.hash !== contentHash(project, t)) return `${file} changed since \`local dev\` applied it, so the Agent's draft is not what its files say: ${again}`
+  }
+  return null
+}
+
+/** The Agent's config id and the name a conversation addresses: its draft's preview, or the live version. */
+export async function chatTarget(api: Api, slug: string, live: boolean): Promise<{ agentId: string; agentName: string }> {
+  const agent = await call<{ id: string }>(api, `/v1/config/agents/${encodeURIComponent(slug)}`)
+  if (live) return { agentId: agent.id, agentName: slug }
+  const preview = await call<{ previewAgentSlug: string }>(api, `/v1/config/agents/${agent.id}/test-session`, { body: {} })
+  return { agentId: agent.id, agentName: preview.previewAgentSlug }
+}
+
+/** Agent cases run at once by default: modest, so a provider account's limit on requests in flight holds. */
+export const DEFAULT_CASE_CONCURRENCY = 2
+/** How long a case waits before its one retry after the provider refused it for its limits. */
+export const PROVIDER_RETRY_DELAY_MS = 8_000
+
+/**
+ * Whether a failure is the model provider refusing for its limits rather than
+ * the Agent failing the case: a 402 or 429 about credits, rate, quota or the
+ * requests in flight ("This request would exceed your available credits given
+ * your current in-flight requests").
+ */
+export function isProviderLimit(message: string): boolean {
+  return /\b(402|429)\b/.test(message) && /credit|in-flight|in flight|rate.?limit|too many requests|quota|overloaded/i.test(message)
+    || /\b(rate limit(ed)?|too many requests)\b/i.test(message) && /provider|model|request/i.test(message)
+}
+
+/** `--concurrency` as given: a whole number from 1 to 16, else a usage error naming it. Undefined when not given. */
+export function caseConcurrency(raw: string | number | undefined): number | undefined {
+  if (raw === undefined || raw === '') return undefined
+  const n = Number(raw)
+  if (!Number.isInteger(n) || n < 1 || n > 16) throw new Error('--concurrency takes a whole number from 1 to 16, e.g. `--concurrency 1` to run Agent cases one at a time')
+  return n
+}
+
+/** Run `work` over `items`, at most `limit` at a time, keeping each result at its item's place. */
+async function pool<T, R>(items: readonly T[], limit: number, work: (item: T, index: number) => Promise<R>): Promise<R[]> {
+  const out: R[] = new Array(items.length)
+  let next = 0
+  const lane = async () => {
+    for (let i = next++; i < items.length; i = next++) out[i] = await work(items[i]!, i)
+  }
+  await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, lane))
+  return out
+}
+
+/** One at a time, in the order asked: cases that change records for real never run alongside each other. */
+function serial(): <R>(work: () => Promise<R>) => Promise<R> {
+  let tail: Promise<unknown> = Promise.resolve()
+  return <R>(work: () => Promise<R>) => {
+    const run = tail.then(work, work)
+    tail = run.catch(() => {})
+    return run
   }
-  return { reply, toolCalls: [...calls.values()] }
 }
 
 export async function runCases(args: {
@@ -234,59 +599,180 @@ export async function runCases(args: {
   files: CaseFile[]
   live: boolean
   onResult: (r: CaseResult) => void
+  /**
+   * Why an Agent's draft is not what its files say (`agentFilesProblem`), or
+   * null when it is: its cases fail with that instead of testing stale files.
+   * Not asked with `live`, which tests the published version.
+   */
+  staleDraft?: (slug: string) => string | null
+  /**
+   * How many conversation cases run at once (default `DEFAULT_CASE_CONCURRENCY`).
+   * Cases that change records for real (`realActions`, a `decision`) still run
+   * one at a time, so two never write the same records together.
+   */
+  concurrency?: number
+  /** The pause before a case's one retry after a provider limit (default `PROVIDER_RETRY_DELAY_MS`). */
+  retryDelayMs?: number
+  /** How a pause is waited out (tests pass a quick one). */
+  wait?: (ms: number) => Promise<void>
+  /** One conversation case, sent to the Agent (tests pass a stand-in). */
+  runOne?: (api: Api, target: CaseTarget, c: TestCase, opts?: ConversationCaseOptions) => Promise<CaseResult>
+  /** The Agent a file's cases talk to (`chatTarget`; tests pass a stand-in). */
+  target?: (api: Api, slug: string, live: boolean) => Promise<{ agentId: string; agentName: string }>
 }): Promise<CaseResult[]> {
   const api: Api = { serviceUrl: args.serviceUrl, creds: args.creds }
   const results: CaseResult[] = []
-  for (const all of args.files) {
+  const queued: Array<{ target: CaseTarget; c: TestCase }> = []
+  // A credit refusal names a model the runtime offers directly from another provider whose key it holds
+  // (the providers the runtime applied keys for; never a key), and how to make it the default.
+  const told = (r: CaseResult): CaseResult => (r.notRun?.kind === 'credits' ? { ...r, notRun: withDirectModel(r.notRun, directModelFor(args.creds.modelProviders, r.notRun.provider)) } : r)
+  for (const whole of args.files) {
+    if (whole.problem) {
+      const r: CaseResult = { agent: whole.agent, file: whole.file, name: '(file)', passed: false, reply: '', reason: whole.problem, ms: 0 }
+      results.push(r)
+      args.onResult(r)
+      continue
+    }
+    // A case that breaks the format fails with where, and is not sent to the Agent.
+    for (const c of whole.cases.filter((c) => c.problems?.length)) {
+      const r: CaseResult = { agent: whole.agent, file: whole.file, name: c.name, passed: false, reply: '', reason: c.problems!.join('; '), ms: 0 }
+      results.push(r)
+      args.onResult(r)
+    }
+    const all = { ...whole, cases: whole.cases.filter((c) => !c.problems?.length) }
+    if (!all.cases.length) continue
+    // A draft that is not what the files say fails every case with why, scenarios too.
+    const stale = args.live ? null : (args.staleDraft?.(all.agent) ?? null)
     // Scenarios run on the runtime's evaluation runner, the rest as conversations here.
-    const scenarios = all.cases.filter((c) => c.scenario)
-    const file = { ...all, cases: all.cases.filter((c) => !c.scenario) }
+    const scenarios = stale ? [] : all.cases.filter((c) => c.scenario)
+    const file = { ...all, cases: stale ? all.cases : all.cases.filter((c) => !c.scenario) }
     if (scenarios.length) {
-      for (const r of await runScenarios(api, all, scenarios, args.live)) {
+      for (const r of (await runScenarios(api, all, scenarios, args.live)).map(told)) {
         results.push(r)
         args.onResult(r)
       }
     }
     let agentName = file.agent
     let agentId: string | undefined
-    let setupError: string | undefined
-    try {
-      const agent = await call<{ id: string }>(api, `/v1/config/agents/${encodeURIComponent(file.agent)}`)
-      agentId = agent.id
-      if (!args.live) {
-        const preview = await call<{ previewAgentSlug: string }>(api, `/v1/config/agents/${agent.id}/test-session`, { body: {} })
-        agentName = preview.previewAgentSlug
-      }
-    } catch (err) {
-      // An Agent that is missing or whose draft cannot be tried fails its cases, not the run.
-      setupError = (err as Error).message
-    }
-    for (const c of file.cases) {
-      const started = Date.now()
-      let result: CaseResult
-      let conversationId: string | undefined
+    let setupError: string | undefined = stale ?? undefined
+    if (!setupError && file.cases.length) {
       try {
-        if (setupError) throw new Error(setupError)
-        const session = await call<{ id: string }>(api, '/v1/sessions', { body: { agentName, workspaceId: args.creds.workspaceId } })
-        conversationId = session.id
-        let reply = ''
-        const toolCalls: ToolCall[] = []
-        for (const turn of c.conversation ?? []) {
-          if (typeof turn.user !== 'string') continue
-          const answer = await sendTurn(api, session.id, agentName, turn.user)
-          reply = answer.reply
-          toolCalls.push(...answer.toolCalls)
-        }
-        const verdict = judge(c, reply, toolCalls)
-        result = { agent: file.agent, file: file.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, agentId, conversationId }
+        const target = await (args.target ?? chatTarget)(api, file.agent, args.live)
+        agentId = target.agentId
+        agentName = target.agentName
       } catch (err) {
-        result = { agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason: (err as Error).message, ms: Date.now() - started, agentId, conversationId }
+        // An Agent that is missing or whose draft cannot be tried fails its cases, not the run.
+        setupError = (err as Error).message
       }
-      results.push(result)
-      args.onResult(result)
     }
+    const target: CaseTarget = { agent: file.agent, file: file.file, agentName, ...(agentId ? { agentId } : {}), ...(setupError ? { setupError } : {}), workspaceId: args.creds.workspaceId }
+    for (const c of file.cases) queued.push({ target, c })
+  }
+  // The conversation cases, a few at a time; each retried once, after a pause, when the provider refused it for its limits.
+  const runOne = args.runOne ?? runConversationCase
+  const wait = args.wait ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)))
+  const delay = args.retryDelayMs ?? PROVIDER_RETRY_DELAY_MS
+  const writes = serial()
+  // A refusal every later case would meet too (the provider account's credit or key, a runtime that
+  // does not answer): the cases not sent yet are not sent, and say why, instead of each failing alike.
+  let stopped: { notRun: CaseNotRun; by: string } | null = null
+  const once = async ({ target, c }: { target: CaseTarget; c: TestCase }): Promise<CaseResult> => {
+    if (stopped) return { agent: target.agent, file: target.file, name: c.name, passed: false, reply: '', reason: `not run: ${stopped.notRun.what} at ${stopped.by}, so the cases after it were not sent`, ms: 0, notRun: { ...stopped.notRun, skipped: true } }
+    const attempt = () => (usesRealActions(c) ? writes(() => runOne(api, target, c, { wait })) : runOne(api, target, c, { wait }))
+    const first = await attempt()
+    if (first.passed || !first.reason || !isProviderLimit(first.reason)) return first
+    await wait(delay)
+    const second = await attempt()
+    const said = first.reason.replace(/\s+/g, ' ').slice(0, 200)
+    return { ...second, retried: said, ...(second.passed ? {} : { reason: `${second.reason ?? 'failed'} (run twice: the provider refused the first try for its limits: ${said})` }) }
+  }
+  const ran = await pool(queued, Math.max(1, Math.floor(args.concurrency ?? DEFAULT_CASE_CONCURRENCY)), async (q) => {
+    const r = told(await once(q))
+    if (!stopped && r.notRun && !r.notRun.skipped && stopsTheRun(r.notRun)) stopped = { notRun: r.notRun, by: `${r.agent} · ${r.name}` }
+    args.onResult(r)
+    return r
+  })
+  return [...results, ...ran]
+}
+
+/** Where a file's conversation cases go: the Agent, the name a conversation addresses, and why they can't run when they can't. */
+export interface CaseTarget {
+  agent: string
+  file: string
+  agentName: string
+  agentId?: string
+  setupError?: string
+  workspaceId: string
+}
+
+export interface ConversationCaseOptions {
+  /** How a turn's `waitSeconds` is waited out (tests pass a quick one). */
+  wait?: (ms: number) => Promise<void>
+}
+
+/** A turn's `waitSeconds` as the runner keeps it: a whole number of seconds from 0 to `MAX_TURN_WAIT_SECONDS`. */
+export function turnWaitSeconds(turn: CaseTurn): number {
+  const n = Number(turn.waitSeconds)
+  return Number.isFinite(n) && n > 0 ? Math.min(MAX_TURN_WAIT_SECONDS, Math.floor(n)) : 0
+}
+
+/** What a case's waits say in a run's output: `waited 60 s before turn 2`. */
+export function waitedWords(waited: CaseResult['waited']): string {
+  return waited?.length ? `waited ${waited.map((w) => `${w.seconds} s before turn ${w.turn}`).join(', ')}` : ''
+}
+
+/** One conversation case: a fresh conversation, its turns, its decision, and the verdict. */
+export async function runConversationCase(api: Api, target: CaseTarget, c: TestCase, opts: ConversationCaseOptions = {}): Promise<CaseResult> {
+  const started = Date.now()
+  const wait = opts.wait ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)))
+  const gaps: NonNullable<CaseResult['waited']> = []
+  const waitedField = () => (gaps.length ? { waited: gaps } : {})
+  const { agentId } = target
+  let conversationId: string | undefined
+  try {
+    if (target.setupError) throw new Error(target.setupError)
+    const session = await call<{ id: string }>(api, '/v1/sessions', { body: { agentName: target.agentName, workspaceId: target.workspaceId } })
+    conversationId = session.id
+    // Before the first message; never sent for a simulated case (without the flag it means real).
+    // One setup call: real actions and the case's variables together.
+    if (agentId) await setUpConversation(api, agentId, session.id, target.agent, { realActions: usesRealActions(c), ...(isRecord(c.variables) ? { variables: c.variables } : {}) })
+    let reply = ''
+    const toolCalls: ToolCall[] = []
+    let lastTurnCalls: ToolCall[] = []
+    let waited: PendingDecision | undefined
+    const decide = c.decision === 'approve' || c.decision === 'deny' ? c.decision : undefined
+    let userTurn = 0
+    for (const turn of c.conversation ?? []) {
+      if (typeof turn.user !== 'string') continue
+      userTurn++
+      // A gap the case keeps before this turn (`waitSeconds`), at most five minutes.
+      const seconds = turnWaitSeconds(turn)
+      if (seconds) {
+        await wait(seconds * 1000)
+        gaps.push({ turn: userTurn, seconds })
+      }
+      const answer = await sendTurn(api, session.id, target.agentName, turn.user, decide)
+      reply = answer.reply
+      toolCalls.push(...answer.toolCalls)
+      lastTurnCalls = answer.toolCalls
+      // A turn parked on a decision ends the case there: a later turn would be refused.
+      if (answer.pending) {
+        waited = answer.pending
+        break
+      }
+    }
+    // Waiting for a person is a failure unless the case expects it (`decision: expect`).
+    const verdict = waited && c.decision !== 'expect'
+      ? { passed: false, reason: waited.approvalId ? `turn waits on an approval for ${waited.tool}; decide it (\`decision: approve\` or \`deny\`) or mark the case \`decision: expect\`` : `turn waits on ${waited.tool}; mark the case \`decision: expect\`` }
+      : !waited && c.decision === 'expect'
+        ? { passed: false, reason: 'the case expects a decision, and no turn waited for one' }
+        : judge(c, reply, toolCalls, lastTurnCalls)
+    return { agent: target.agent, file: target.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, startedAt: new Date(started).toISOString(), agentId, conversationId, toolCalls: toolCalls.length, ...waitedField() }
+  } catch (err) {
+    const reason = (err as Error).message
+    const notRun = caseCouldNotRun(reason)
+    return { agent: target.agent, file: target.file, name: c.name, passed: false, reply: '', reason, ms: Date.now() - started, startedAt: new Date(started).toISOString(), agentId, conversationId, ...waitedField(), ...(notRun ? { notRun } : {}) }
   }
-  return results
 }
 
 const SCENARIO_POLL_MS = 3_000
@@ -309,7 +795,10 @@ export function scenarioDataset(file: CaseFile): string {
 export async function runScenarios(api: Api, file: CaseFile, cases: TestCase[], live: boolean, wait: (ms: number) => Promise<void> = (ms) => new Promise((r) => setTimeout(r, ms))): Promise<CaseResult[]> {
   const started = Date.now()
   const dataset = scenarioDataset(file)
-  const fail = (reason: string, agentId?: string): CaseResult[] => cases.map((c) => ({ agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason, ms: Date.now() - started, agentId }))
+  const fail = (reason: string, agentId?: string): CaseResult[] => {
+    const notRun = caseCouldNotRun(reason)
+    return cases.map((c) => ({ agent: file.agent, file: file.file, name: c.name, passed: false, reply: '', reason, ms: Date.now() - started, agentId, ...(notRun ? { notRun } : {}) }))
+  }
   let agentId: string | undefined
   try {
     const agent = await call<{ id: string }>(api, `/v1/config/agents/${encodeURIComponent(file.agent)}`)
@@ -341,7 +830,9 @@ export async function runScenarios(api: Api, file: CaseFile, cases: TestCase[],
       const reply = typeof at?.output === 'string' ? at.output : ''
       const passed = outcome === 'pass'
       const reason = passed ? undefined : !at ? 'the run has no result for this scenario' : at.error ? at.error : `the scenario ${OUTCOME_WORDS[outcome ?? ''] ?? 'did not finish'}`
-      return { agent: file.agent, file: file.file, name: c.name, passed, reply, ...(reason ? { reason } : {}), ms: Date.now() - started, agentId }
+      // An item the provider refused never ran: its error says so.
+      const notRun = !passed && at?.error ? caseCouldNotRun(at.error) : null
+      return { agent: file.agent, file: file.file, name: c.name, passed, reply, ...(reason ? { reason } : {}), ms: Date.now() - started, agentId, ...(notRun ? { notRun } : {}) }
     })
   } catch (err) {
     return fail((err as Error).message, agentId)