@frontera-sdk/cli 1.52.45 → 1.52.47

package/src/local/local-test-runner.ts

@@ -29,19 +29,36 @@ import type { LocalCredentials } from './local-state'
  */
 
 /** A check graded in code, as the platform grades an item's `expectations` (`expectation-grading.ts`). */
+/**
+ * Which turn a check reads (Forge improvement plan L3, T10): a person turn by
+ * its number (1 is the first), `last`, `any` (one turn passes) or `all`
+ * (every turn passes). A reply check reads the last reply unless it says; a
+ * tool check reads every turn's calls (`all`) unless it says.
+ */
+export type CheckTurn = number | 'last' | 'any' | 'all'
+
 export type TestCheck =
-  | { type: 'contains'; value: string; text?: string }
-  | { type: 'regex'; value: string; flags?: string; text?: string }
+  | { type: 'contains'; value: string; text?: string; turn?: CheckTurn }
+  /** The reply must not say this (any case): a phrase the Agent must never use. */
+  | { type: 'not_contains'; value: string; text?: string; turn?: CheckTurn }
+  | { type: 'regex'; value: string; flags?: string; text?: string; turn?: CheckTurn }
   | {
       type: 'tool_called' | 'tool_not_called'
       tool: string
+      /** Arguments the call carries; `{ absent: true }` as a value says that argument must not be there (L3). */
       argsSubset?: Record<string, unknown>
       mode?: 'exact' | 'ignore' | 'subset' | 'superset'
       text?: string
-      /** The calls of every turn (`all`, the default) or only of the last one, whose reply the case grades. */
-      turn?: 'all' | 'last'
+      /** The calls of every turn (`all`, the default, or `any`), only of the last one, or of turn n. */
+      turn?: CheckTurn
     }
 
+/** One person turn as the case ran it: the reply and the calls it made. */
+export interface TurnSeen {
+  reply: string
+  toolCalls: ToolCall[]
+}
+
 /**
  * One turn of a case's conversation: a person's message (sent), or an
  * assistant's (kept for whoever reads the case). `waitSeconds` on a person's
@@ -60,9 +77,15 @@ export interface TestCase {
   expected?: string
   /** Pass when the reply contains each of these, instead of matching exactly. */
   expectedContains?: string[]
+  /** Phrases the last reply must not contain, any case (L3). */
+  notContains?: string[]
   /** Checks on the reply and the tools called; every one must pass too. */
   checks?: TestCheck[]
-  /** Expectations only the platform's judge grades, in words: kept, not graded here. */
+  /**
+   * Expectations in words, graded by the expectation judge with the grading
+   * model, on the local runtime as on the platform (X1), after every other
+   * check passed.
+   */
   judged?: string[]
   /** Human-certified on the platform. */
   gold?: boolean
@@ -115,6 +138,8 @@ export interface CaseResult {
   startedAt?: string
   /** How many tools the Agent called, for efficiency against a baseline. */
   toolCalls?: number
+  /** The tools it called, by name, once each: what tells a failure on a local stand-in from the Agent's (X2). */
+  toolNames?: string[]
   /**
    * The waits the case kept between turns (`waitSeconds`), in seconds, in
    * order: said in the run's output, since such a case takes that long.
@@ -133,6 +158,21 @@ export interface CaseResult {
    * failing, and never recorded as the case's last outcome.
    */
   notRun?: CaseNotRun
+  /**
+   * The case's `judged` expectations, graded on the local runtime by the
+   * expectation judge with the grading model (`local model-default --for
+   * tests`), each with the judge's evidence. Absent when the case has none,
+   * or failed its other checks first.
+   */
+  judged?: Array<{ text: string; passed: boolean; evidence: string }>
+  /** Why its `judged` expectations weren't graded (no grading model, the runtime refused); the rest of the case was. */
+  judgedNotGraded?: string
+  /**
+   * `--repeat N`: how the case's runs went. A case that passes some runs and
+   * fails others is `flaky`; it passes when its pass rate is at least the
+   * run's threshold, and fails below it. Runs that couldn't run don't count.
+   */
+  repeat?: { runs: number; passes: number; passRate: number; flaky: boolean; notRun?: number }
 }
 
 export interface CaseFile {
@@ -236,11 +276,13 @@ function normalise(text: string): string {
 }
 
 /** `lastTurnCalls`: the last turn's calls, for a check scoped `turn: last`; every call when not given. */
-export function judge(c: TestCase, reply: string, toolCalls: ToolCall[] = [], lastTurnCalls?: ToolCall[]): { passed: boolean; reason?: string } {
+export function judge(c: TestCase, reply: string, toolCalls: ToolCall[] = [], lastTurnCalls?: ToolCall[], turns?: TurnSeen[]): { passed: boolean; reason?: string } {
   const answer = judgeAnswer(c, reply)
   if (!answer.passed) return answer
+  const said = (c.notContains ?? []).filter((s) => typeof s === 'string' && s && normalise(reply).toLowerCase().includes(normalise(s).toLowerCase()))
+  if (said.length) return { passed: false, reason: `reply says what the case doesn't allow: ${said.join(', ')}` }
   for (const check of c.checks ?? []) {
-    const failed = runCheck(check, reply, toolCalls, lastTurnCalls)
+    const failed = runCheck(check, reply, toolCalls, lastTurnCalls, turns)
     if (failed) return { passed: false, reason: failed }
   }
   return { passed: true }
@@ -251,8 +293,8 @@ function judgeAnswer(c: TestCase, reply: string): { passed: boolean; reason?: st
     const missing = c.expectedContains.filter((s) => !normalise(reply).toLowerCase().includes(normalise(s).toLowerCase()))
     return missing.length ? { passed: false, reason: `reply does not mention: ${missing.join(', ')}` } : { passed: true }
   }
-  // Checks alone are an expectation.
-  if (c.expected === undefined) return c.checks?.length ? { passed: true } : { passed: false, reason: 'the case has no `expected`, `expectedContains` or `checks`' }
+  // Checks alone are an expectation, and so are phrases it must not say.
+  if (c.expected === undefined) return c.checks?.length || c.notContains?.length ? { passed: true } : { passed: false, reason: 'the case has no `expected`, `expectedContains`, `notContains` or `checks`' }
   return normalise(reply) === normalise(c.expected) ? { passed: true } : { passed: false, reason: 'reply differs from `expected`' }
 }
 
@@ -263,7 +305,12 @@ const isRecord = (v: unknown): v is Record<string, unknown> => typeof v === 'obj
 
 const deepEqual = (a: unknown, b: unknown): boolean => JSON.stringify(a) === JSON.stringify(b) || (isRecord(a) && isRecord(b) && Object.keys(a).length === Object.keys(b).length && Object.keys(a).every((k) => deepEqual(a[k], b[k])))
 
-const containsAll = (haystack: Record<string, unknown>, needles: Record<string, unknown>) => Object.entries(needles).every(([k, v]) => k in haystack && deepEqual(haystack[k], v))
+/** `{ absent: true }` in a check's arguments: the call must not carry that argument (an empty value counts as not there). */
+const isAbsentMarker = (v: unknown) => isRecord(v) && v.absent === true && Object.keys(v).length === 1
+const isEmpty = (v: unknown) => v === undefined || v === null || v === ''
+
+const containsAll = (haystack: Record<string, unknown>, needles: Record<string, unknown>) =>
+  Object.entries(needles).every(([k, v]) => (isAbsentMarker(v) ? !(k in haystack) || isEmpty(haystack[k]) : k in haystack && deepEqual(haystack[k], v)))
 
 function argsMatch(check: Extract<TestCheck, { tool: string }>, call: ToolCall): boolean {
   const mode = check.mode ?? 'superset'
@@ -273,20 +320,45 @@ function argsMatch(check: Extract<TestCheck, { tool: string }>, call: ToolCall):
   const { action: _narration, ...rest } = call.args
   const args = 'action' in check.argsSubset ? call.args : rest
   if (mode === 'superset') return containsAll(args, check.argsSubset)
-  if (mode === 'subset') return containsAll(check.argsSubset, args)
-  return containsAll(args, check.argsSubset) && Object.keys(args).length === Object.keys(check.argsSubset).length
+  const present = Object.fromEntries(Object.entries(check.argsSubset).filter(([, v]) => !isAbsentMarker(v)))
+  const absent = Object.fromEntries(Object.entries(check.argsSubset).filter(([, v]) => isAbsentMarker(v)))
+  if (mode === 'subset') return containsAll(present, Object.fromEntries(Object.entries(args).filter(([k]) => !(k in absent)))) && containsAll(args, absent)
+  // Exact: the same arguments, and nothing besides; an absent-marked one may be there only empty.
+  return containsAll(args, check.argsSubset) && Object.keys(args).every((k) => k in check.argsSubset!)
+}
+
+/** A check's `turn` as a turn number (1-based), `last`, `any` or `all`; null when it is none of them. */
+function turnOf(turn: unknown): CheckTurn | null {
+  if (turn === undefined) return null
+  if (turn === 'last' || turn === 'any' || turn === 'all') return turn
+  return typeof turn === 'number' && Number.isInteger(turn) && turn >= 1 ? turn : null
 }
 
 /** Why a check failed, or null when it passed. */
-export function runCheck(check: TestCheck, reply: string, allCalls: ToolCall[], lastTurnCalls?: ToolCall[]): string | null {
+export function runCheck(check: TestCheck, reply: string, allCalls: ToolCall[], lastTurnCalls?: ToolCall[], turns?: TurnSeen[]): string | null {
   const label = check.text ? ` (${check.text})` : ''
-  const lastOnly = 'turn' in check && check.turn === 'last'
-  const toolCalls = lastOnly && lastTurnCalls ? lastTurnCalls : allCalls
+  const turn = turnOf((check as { turn?: unknown }).turn)
+  if ((check as { turn?: unknown }).turn !== undefined && turn === null) return `the check's \`turn\` is a turn number from 1, last, any or all (it is ${JSON.stringify((check as { turn?: unknown }).turn)})${label}`
+  // A turn the case never reached fails the check by name, rather than reading another turn.
+  if (typeof turn === 'number' && turns && turn > turns.length) return `the case has no turn ${turn} (it ran ${turns.length})${label}`
   // The format says text; a number or a missing value is refused, not thrown on.
-  if ((check.type === 'contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) {
+  if ((check.type === 'contains' || check.type === 'not_contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) {
     return `the ${check.type} check's \`value\` must be text (it is ${check.value === undefined ? 'missing' : JSON.stringify(check.value)}); quote it in the file${label}`
   }
+  // A reply check on several turns: any one passing, or every one (L3).
+  if ((check.type === 'contains' || check.type === 'not_contains' || check.type === 'regex') && (turn === 'any' || turn === 'all') && turns?.length) {
+    const failures = turns.map((t, i) => [i + 1, runCheck({ ...check, turn: 'last' } as TestCheck, t.reply, t.toolCalls, t.toolCalls)] as const)
+    if (turn === 'any') return failures.some(([, f]) => f === null) ? null : `no turn passes; the last: ${failures[failures.length - 1]![1]}`
+    const failed = failures.find(([, f]) => f !== null)
+    return failed ? `turn ${failed[0]}: ${failed[1]}` : null
+  }
+  if ((check.type === 'contains' || check.type === 'not_contains' || check.type === 'regex') && typeof turn === 'number' && turns) {
+    return runCheck({ ...check, turn: 'last' } as TestCheck, turns[turn - 1]!.reply, turns[turn - 1]!.toolCalls, turns[turn - 1]!.toolCalls)?.replace(/^/, `turn ${turn}: `) ?? null
+  }
+  const lastOnly = turn === 'last'
+  const toolCalls = typeof turn === 'number' && turns ? turns[turn - 1]!.toolCalls : lastOnly && lastTurnCalls ? lastTurnCalls : allCalls
   if (check.type === 'contains') return reply.toLowerCase().includes(check.value.toLowerCase()) ? null : `reply does not mention: ${check.value}${label}`
+  if (check.type === 'not_contains') return reply.toLowerCase().includes(check.value.toLowerCase()) ? `reply says ${check.value}, which the case doesn't allow${label}` : null
   if (check.type === 'regex') {
     if (check.value.length > 500 || NESTED_QUANTIFIER.test(check.value)) return `the pattern ${check.value} can't run safely${label}`
     let pattern: RegExp
@@ -302,7 +374,7 @@ export function runCheck(check: TestCheck, reply: string, allCalls: ToolCall[],
   const matched = candidates.some((c) => c.name === check.tool && argsMatch(check, c))
   const said = (c: ToolCall) => `${c.name}${c.parked ? ' (waiting on approval)' : !c.ok ? ' (did not run)' : ''}`
   const made = toolCalls.length ? `calls made: ${[...new Set(toolCalls.map(said))].join(', ')}` : 'no tools were called'
-  const where = lastOnly ? ' in the last turn' : ''
+  const where = lastOnly ? ' in the last turn' : typeof turn === 'number' ? ` in turn ${turn}` : ''
   if (check.type === 'tool_called') return matched ? null : `no matching call to ${check.tool}${where}; ${made}${label}`
   return matched ? `${check.tool} was called${where}, which the case does not allow${label}` : null
 }
@@ -619,10 +691,22 @@ export async function runCases(args: {
   runOne?: (api: Api, target: CaseTarget, c: TestCase, opts?: ConversationCaseOptions) => Promise<CaseResult>
   /** The Agent a file's cases talk to (`chatTarget`; tests pass a stand-in). */
   target?: (api: Api, slug: string, live: boolean) => Promise<{ agentId: string; agentName: string }>
+  /**
+   * `--repeat N`: each conversation case runs N times, and its result says how
+   * many passed (`combineRuns`). A scenario case runs once: the evaluation
+   * runner judges it as a whole.
+   */
+  repeat?: number
+  /** The pass rate a repeated, flaky case needs to pass (default `DEFAULT_PASS_RATE`). */
+  passRate?: number
+  /** How a case's `judged` expectations are graded (tests pass a stand-in). */
+  grade?: typeof gradeJudged
 }): Promise<CaseResult[]> {
   const api: Api = { serviceUrl: args.serviceUrl, creds: args.creds }
   const results: CaseResult[] = []
-  const queued: Array<{ target: CaseTarget; c: TestCase }> = []
+  const queued: Array<{ target: CaseTarget; c: TestCase; group: number }> = []
+  const repeat = Math.max(1, Math.min(10, Math.floor(args.repeat ?? 1)))
+  let groups = 0
   // A credit refusal names a model the runtime offers directly from another provider whose key it holds
   // (the providers the runtime applied keys for; never a key), and how to make it the default.
   const told = (r: CaseResult): CaseResult => (r.notRun?.kind === 'credits' ? { ...r, notRun: withDirectModel(r.notRun, directModelFor(args.creds.modelProviders, r.notRun.provider)) } : r)
@@ -666,7 +750,10 @@ export async function runCases(args: {
       }
     }
     const target: CaseTarget = { agent: file.agent, file: file.file, agentName, ...(agentId ? { agentId } : {}), ...(setupError ? { setupError } : {}), workspaceId: args.creds.workspaceId }
-    for (const c of file.cases) queued.push({ target, c })
+    for (const c of file.cases) {
+      const group = groups++
+      for (let n = 0; n < repeat; n++) queued.push({ target, c, group })
+    }
   }
   // The conversation cases, a few at a time; each retried once, after a pause, when the provider refused it for its limits.
   const runOne = args.runOne ?? runConversationCase
@@ -678,7 +765,8 @@ export async function runCases(args: {
   let stopped: { notRun: CaseNotRun; by: string } | null = null
   const once = async ({ target, c }: { target: CaseTarget; c: TestCase }): Promise<CaseResult> => {
     if (stopped) return { agent: target.agent, file: target.file, name: c.name, passed: false, reply: '', reason: `not run: ${stopped.notRun.what} at ${stopped.by}, so the cases after it were not sent`, ms: 0, notRun: { ...stopped.notRun, skipped: true } }
-    const attempt = () => (usesRealActions(c) ? writes(() => runOne(api, target, c, { wait })) : runOne(api, target, c, { wait }))
+    const options = { wait, ...(args.grade ? { grade: args.grade } : {}) }
+    const attempt = () => (usesRealActions(c) ? writes(() => runOne(api, target, c, options)) : runOne(api, target, c, options))
     const first = await attempt()
     if (first.passed || !first.reason || !isProviderLimit(first.reason)) return first
     await wait(delay)
@@ -686,13 +774,22 @@ export async function runCases(args: {
     const said = first.reason.replace(/\s+/g, ' ').slice(0, 200)
     return { ...second, retried: said, ...(second.passed ? {} : { reason: `${second.reason ?? 'failed'} (run twice: the provider refused the first try for its limits: ${said})` }) }
   }
-  const ran = await pool(queued, Math.max(1, Math.floor(args.concurrency ?? DEFAULT_CASE_CONCURRENCY)), async (q) => {
+  // A case's runs, as they come back: said as one result once the last of them is in.
+  const runsOf = new Map<number, CaseResult[]>()
+  const combined = new Map<number, CaseResult>()
+  await pool(queued, Math.max(1, Math.floor(args.concurrency ?? DEFAULT_CASE_CONCURRENCY)), async (q) => {
     const r = told(await once(q))
     if (!stopped && r.notRun && !r.notRun.skipped && stopsTheRun(r.notRun)) stopped = { notRun: r.notRun, by: `${r.agent} · ${r.name}` }
-    args.onResult(r)
+    const runs = [...(runsOf.get(q.group) ?? []), r]
+    runsOf.set(q.group, runs)
+    if (runs.length === repeat) {
+      const one = combineRuns(runs, args.passRate ?? DEFAULT_PASS_RATE)
+      combined.set(q.group, one)
+      args.onResult(one)
+    }
     return r
   })
-  return [...results, ...ran]
+  return [...results, ...[...combined.entries()].sort((a, b) => a[0] - b[0]).map(([, r]) => r)]
 }
 
 /** Where a file's conversation cases go: the Agent, the name a conversation addresses, and why they can't run when they can't. */
@@ -708,6 +805,97 @@ export interface CaseTarget {
 export interface ConversationCaseOptions {
   /** How a turn's `waitSeconds` is waited out (tests pass a quick one). */
   wait?: (ms: number) => Promise<void>
+  /** How a case's `judged` expectations are graded (`gradeJudged`; tests pass a stand-in). */
+  grade?: typeof gradeJudged
+}
+
+/**
+ * Grade a reply against expectations in words with the local runtime's
+ * expectation judge (`POST /v1/eval/expectations/grade`), the judge a
+ * platform evaluation run uses, on the grading model the runtime has
+ * (`frontera local model-default <model> --for tests`).
+ */
+export async function gradeJudged(api: Api, input: { expectations: string[]; conversation: string; reply: string; toolCalls: ToolCall[] }): Promise<{ results: Array<{ text: string; passed: boolean; evidence: string }> }> {
+  return call(api, '/v1/eval/expectations/grade', {
+    body: {
+      expectations: input.expectations,
+      conversation: input.conversation.slice(0, 200_000),
+      reply: input.reply.slice(0, 200_000),
+      toolCalls: input.toolCalls.slice(0, 200).map((c) => ({ name: c.name, ...(c.args === undefined ? {} : { args: c.args }), ok: c.ok })),
+    },
+  })
+}
+
+/** Why grading couldn't run, in words, with what fixes it. */
+export function judgedNotGradedWords(message: string): string {
+  if (/answered 404/.test(message)) return 'this runtime is older than judged grading: `frontera local start --update`, then test again'
+  if (/answered 409|not configured|no .*model|judge/i.test(message)) return `the runtime has no grading model it can use (${message.replace(/\s+/g, ' ').slice(0, 160)}): choose one with \`frontera local model-default <model> --for tests\``
+  return message.replace(/\s+/g, ' ').slice(0, 240)
+}
+
+/** `--repeat` as given: a whole number from 1 to 10. Undefined when not given. */
+export function caseRepeat(raw: string | number | undefined): number | undefined {
+  if (raw === undefined || raw === '') return undefined
+  const n = Number(raw)
+  if (!Number.isInteger(n) || n < 1 || n > 10) throw new Error('--repeat takes a whole number from 1 to 10, e.g. `--repeat 5` to run each case five times and report its pass rate')
+  return n
+}
+
+/** The pass rate a repeated case needs to pass, by default: 4 of 5 runs. */
+export const DEFAULT_PASS_RATE = 0.8
+
+/** `--pass-rate` as given: a share from 0 to 1, or a percentage (`80%`). Undefined when not given. */
+export function passRateThreshold(raw: string | number | undefined): number | undefined {
+  if (raw === undefined || raw === '') return undefined
+  const text = String(raw).trim()
+  const n = text.endsWith('%') ? Number(text.slice(0, -1)) / 100 : Number(text)
+  if (!Number.isFinite(n) || n <= 0 || n > 1) throw new Error('--pass-rate takes a share above 0 and up to 1 (`0.8`) or a percentage (`80%`): the share of runs a repeated case must pass')
+  return n
+}
+
+/**
+ * One case's repeated runs as one result (`--repeat N`): passed when every
+ * run that ran passed, or when it is flaky and its pass rate is at least
+ * `threshold`; the first failing run's reason and conversation otherwise, with
+ * how many runs passed. Runs that couldn't run aren't counted; when none ran,
+ * the case couldn't run.
+ */
+export function combineRuns(runs: CaseResult[], threshold = DEFAULT_PASS_RATE): CaseResult {
+  if (runs.length === 1) return runs[0]!
+  const ran = runs.filter((r) => !r.notRun)
+  if (!ran.length) return runs[0]!
+  const passes = ran.filter((r) => r.passed).length
+  const passRate = passes / ran.length
+  const flaky = passes > 0 && passes < ran.length
+  const passed = passes === ran.length || (flaky && passRate >= threshold)
+  const shown = ran.find((r) => !r.passed) ?? ran[ran.length - 1]!
+  const mean = (values: Array<number | undefined>) => {
+    const known = values.filter((v): v is number => typeof v === 'number')
+    return known.length ? Math.round(known.reduce((a, b) => a + b, 0) / known.length) : undefined
+  }
+  const counted = `passed ${passes} of ${ran.length} runs`
+  const toolCalls = mean(ran.map((r) => r.toolCalls))
+  return {
+    ...shown,
+    passed,
+    ms: mean(ran.map((r) => r.ms)) ?? shown.ms,
+    ...(toolCalls === undefined ? {} : { toolCalls }),
+    reason: passed ? (flaky ? `flaky: ${counted}, at or above the ${Math.round(threshold * 100)}% it needs; one failed: ${shown.reason ?? 'failed'}` : undefined) : `${shown.reason ?? 'failed'} (${counted}${flaky ? `, flaky, below the ${Math.round(threshold * 100)}% it needs` : ''})`,
+    repeat: { runs: ran.length, passes, passRate, flaky, ...(runs.length > ran.length ? { notRun: runs.length - ran.length } : {}) },
+  }
+}
+
+/** What a repeated case says after its name: `(passed 4 of 5 runs, flaky)`. */
+export function repeatWords(r: CaseResult): string {
+  if (!r.repeat) return ''
+  return ` (passed ${r.repeat.passes} of ${r.repeat.runs} runs${r.repeat.flaky ? ', flaky' : ''}${r.repeat.notRun ? `; ${r.repeat.notRun} couldn’t run` : ''})`
+}
+
+/** What a case's `judged` expectations came to, after its name: ` (judged: 2 of 2 met)`, or why they weren't graded. */
+export function judgedWords(r: CaseResult): string {
+  if (r.judgedNotGraded) return ` (judged expectations not graded: ${r.judgedNotGraded})`
+  if (!r.judged?.length) return ''
+  return ` (judged: ${r.judged.filter((j) => j.passed).length} of ${r.judged.length} met)`
 }
 
 /** A turn's `waitSeconds` as the runner keeps it: a whole number of seconds from 0 to `MAX_TURN_WAIT_SECONDS`. */
@@ -742,6 +930,10 @@ export async function runConversationCase(api: Api, target: CaseTarget, c: TestC
     let waited: PendingDecision | undefined
     const decide = c.decision === 'approve' || c.decision === 'deny' ? c.decision : undefined
     let userTurn = 0
+    // The conversation as the judge reads it, for the case's `judged` expectations.
+    const said: string[] = []
+    // Each person turn's reply and calls, for checks on a turn (L3).
+    const seen: TurnSeen[] = []
     for (const turn of c.conversation ?? []) {
       if (typeof turn.user !== 'string') continue
       userTurn++
@@ -752,6 +944,8 @@ export async function runConversationCase(api: Api, target: CaseTarget, c: TestC
         gaps.push({ turn: userTurn, seconds })
       }
       const answer = await sendTurn(api, session.id, target.agentName, turn.user, decide)
+      seen.push({ reply: answer.reply, toolCalls: answer.toolCalls })
+      said.push(`user: ${turn.user}`, `assistant: ${answer.reply}`)
       reply = answer.reply
       toolCalls.push(...answer.toolCalls)
       lastTurnCalls = answer.toolCalls
@@ -766,8 +960,21 @@ export async function runConversationCase(api: Api, target: CaseTarget, c: TestC
       ? { passed: false, reason: waited.approvalId ? `turn waits on an approval for ${waited.tool}; decide it (\`decision: approve\` or \`deny\`) or mark the case \`decision: expect\`` : `turn waits on ${waited.tool}; mark the case \`decision: expect\`` }
       : !waited && c.decision === 'expect'
         ? { passed: false, reason: 'the case expects a decision, and no turn waited for one' }
-        : judge(c, reply, toolCalls, lastTurnCalls)
-    return { agent: target.agent, file: target.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, startedAt: new Date(started).toISOString(), agentId, conversationId, toolCalls: toolCalls.length, ...waitedField() }
+        : judge(c, reply, toolCalls, lastTurnCalls, seen)
+    // The case's `judged` expectations, graded by the runtime's judge once everything else passed.
+    let judgedField: Pick<CaseResult, 'judged' | 'judgedNotGraded'> = {}
+    const judgedTexts = (c.judged ?? []).filter((j): j is string => typeof j === 'string' && j.trim() !== '')
+    if (verdict.passed && judgedTexts.length && !waited) {
+      try {
+        const graded = await (opts.grade ?? gradeJudged)(api, { expectations: judgedTexts, conversation: said.join('\n'), reply, toolCalls })
+        judgedField = { judged: graded.results.map((g) => ({ text: g.text, passed: g.passed, evidence: g.evidence })) }
+        const missed = judgedField.judged!.filter((g) => !g.passed)
+        if (missed.length) Object.assign(verdict, { passed: false, reason: `judged: ${missed.map((g) => `“${g.text}” not met (${g.evidence})`).join('; ')}` })
+      } catch (err) {
+        judgedField = { judgedNotGraded: judgedNotGradedWords((err as Error).message) }
+      }
+    }
+    return { agent: target.agent, file: target.file, name: c.name, passed: verdict.passed, reply, reason: verdict.reason, ms: Date.now() - started, startedAt: new Date(started).toISOString(), agentId, conversationId, toolCalls: toolCalls.length, ...(toolCalls.length ? { toolNames: [...new Set(toolCalls.map((t) => t.name))] } : {}), ...waitedField(), ...judgedField }
   } catch (err) {
     const reason = (err as Error).message
     const notRun = caseCouldNotRun(reason)

package/src/local/local-test-scope.ts

@@ -0,0 +1,174 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { join, normalize } from 'node:path'
+
+import { loadBuild } from './local-applied'
+import type { Unchanged } from './local-check-inherited'
+import type { LocalProject } from './local-project'
+import { hash, readLock } from './local-sync-plan'
+import { readBase } from './local-sync-snapshot'
+import { findCaseFiles, type CaseFile } from './local-test-runner'
+
+/**
+ * What a test run left out and why, and why each Agent's cases are in it
+ * (Forge improvement plan T9, X1). Before, a gate's count dropped from 13 to
+ * 5 with nothing said, and `local check` ran every Agent's cases, so a rename
+ * turned another Agent's flaky case into a blocker.
+ */
+
+/** Cases a run did not take, with why: `3 cases in agents/desk/evals/x.yaml: not marked gate: true`. */
+export interface SkippedCases {
+  reason: string
+  cases: number
+}
+
+/**
+ * The cases under the run's scope that it leaves out, by reason: a file with
+ * no cases, cases not marked `gate: true` in a gate run, files that aren't a
+ * regression dataset in a `--regression` run, and other Agents' cases when
+ * the run names Agents (`local check --agent`).
+ */
+export function skippedCases(root: string, scope: { agents?: readonly string[]; file?: string; regression?: boolean; gate?: boolean } = {}): SkippedCases[] {
+  const file = scope.file?.replace(/^\.\//, '')
+  const all = findCaseFiles(root).filter((f) => !file || f.file === file)
+  const out: SkippedCases[] = []
+  const add = (reason: string, cases: number) => out.push({ reason, cases })
+  const named = scope.agents?.length ? new Set(scope.agents) : null
+  if (named) {
+    const others = all.filter((f) => !named.has(f.agent))
+    const count = others.reduce((n, f) => n + f.cases.length, 0)
+    if (count) add(`other Agents' cases (${[...new Set(others.map((f) => f.agent))].sort().join(', ')}): not named with --agent`, count)
+  }
+  for (const f of all.filter((f) => !named || named.has(f.agent))) {
+    if (!f.cases.length && !f.problem) {
+      add(`${f.file} has no cases`, 0)
+      continue
+    }
+    if (scope.regression && !f.regression) {
+      add(`${f.file}: not a regression dataset (\`regression: true\`), and the run takes only those`, f.cases.length)
+      continue
+    }
+    const ungated = scope.gate ? f.cases.filter((c) => c.gate !== true).length : 0
+    if (ungated) add(`${f.file}: not marked \`gate: true\`, and the run takes only gate cases`, ungated)
+  }
+  return out
+}
+
+/** The skipped cases in one line: `5 skipped (3 agents/x/evals/a.yaml: not marked gate: true …; 2 …)`. */
+export function skippedWords(skipped: readonly SkippedCases[]): string {
+  const counted = skipped.filter((s) => s.cases > 0)
+  const total = counted.reduce((n, s) => n + s.cases, 0)
+  const empty = skipped.filter((s) => s.cases === 0)
+  return [total ? `${total} skipped (${counted.map((s) => (counted.length === 1 ? s.reason : `${s.cases} ${s.reason}`)).join('; ')})` : '', ...empty.map((s) => s.reason)].filter(Boolean).join('; ')
+}
+
+/** Why an Agent's cases are in a `local check`, and whether its failures fail it. */
+export interface AgentScope {
+  agent: string
+  cases: number
+  why: string
+  /**
+   * Its failures are inherited from the workspace: it is unchanged since the
+   * pull, or only its name changed. They pass unless `--strict`, as every
+   * other step's inherited failures do.
+   */
+  inherited: boolean
+}
+
+const parseJson = (text: string | null): unknown => {
+  if (text === null) return undefined
+  try {
+    return JSON.parse(text)
+  } catch {
+    return undefined
+  }
+}
+
+/** An agent.json without its name, for telling a rename from a change in what the Agent does. */
+function withoutName(doc: unknown): unknown {
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return doc
+  const { name: _name, ...rest } = doc as Record<string, unknown>
+  const config = rest.config
+  if (!config || typeof config !== 'object' || Array.isArray(config)) return rest
+  const { name: _configName, ...configRest } = config as Record<string, unknown>
+  return { ...rest, config: configRest }
+}
+
+const canonical = (value: unknown): string => JSON.stringify(value, (_k, v) => (v && typeof v === 'object' && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) : v))
+
+/**
+ * Whether the only change to an Agent since the pull is its name: every one
+ * of its files the pull wrote is as written, except agent.json, which differs
+ * from the pulled copy only in `config.name` (or a top-level `name`).
+ */
+export function nameOnlyChange(root: string, slug: string): boolean {
+  const entry = readLock(root)?.items[`agent/${slug}`]
+  if (!entry) return false
+  const agentJson = `agents/${slug}/agent.json`
+  let renamed = false
+  for (const [path, written] of Object.entries(entry.files ?? {})) {
+    const full = join(root, path)
+    const now = existsSync(full) ? readFileSync(full, 'utf8') : null
+    if (now !== null && hash(now) === written) continue
+    if (normalize(path) !== normalize(agentJson) || now === null) return false
+    const before = parseJson(readBase(root, path))
+    const after = parseJson(now)
+    if (before === undefined || after === undefined || canonical(withoutName(before)) !== canonical(withoutName(after))) return false
+    renamed = true
+  }
+  return renamed
+}
+
+/**
+ * Each Agent with cases in a check, and why it is in it. Named with
+ * `--agent`: those only. Otherwise every Agent; in a pulled project one
+ * unchanged since the pull, its cases included, or changed only in its name,
+ * has its failures inherited (they pass unless `--strict`).
+ */
+export function agentScope(root: string, files: readonly CaseFile[], unchanged: Unchanged, named: readonly string[] = []): AgentScope[] {
+  const pulled = !!readLock(root)
+  const bySlug = new Map<string, number>()
+  for (const f of files) bySlug.set(f.agent, (bySlug.get(f.agent) ?? 0) + f.cases.length)
+  return [...bySlug.entries()].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)).map(([agent, cases]) => {
+    if (named.includes(agent)) return { agent, cases, why: 'named with --agent', inherited: false }
+    if (!pulled) return { agent, cases, why: 'every Agent with cases is checked (`--agent <slug>` checks one)', inherited: false }
+    const caseFilesSame = files.filter((f) => f.agent === agent).every((f) => unchanged.files.has(normalize(f.file)))
+    if (!readLock(root)?.items[`agent/${agent}`]) return { agent, cases, why: 'added here, not pulled', inherited: false }
+    if (!caseFilesSame) return { agent, cases, why: 'its test cases changed since the pull', inherited: false }
+    if (unchanged.refs.has(`agent/${agent}`)) return { agent, cases, why: 'unchanged since the pull: its failures are inherited unless --strict', inherited: true }
+    if (nameOnlyChange(root, agent)) return { agent, cases, why: 'only its name changed since the pull, which is not a change in what it does: its failures are inherited unless --strict', inherited: true }
+    return { agent, cases, why: 'its files changed since the pull', inherited: false }
+  })
+}
+
+/**
+ * The Agents that run on another model here than their files name, as the
+ * apply said (`it uses <model> here`), one line each: a pass or a failure
+ * says less about the model the workspace runs.
+ */
+export function modelSubstitutions(project: LocalProject, organizationId: string | null, slugs: readonly string[]): string[] {
+  const build = loadBuild(project, organizationId)
+  const out: string[] = []
+  for (const slug of [...new Set(slugs)].sort()) {
+    const entry = build[`agent:${slug}`]
+    for (const a of entry?.adaptations ?? []) if (/\bmodel\b/i.test(a) && /\buses\b/i.test(a)) out.push(`Agent ${slug}: ${a.replace(/^Agent \S+ /, '')}`)
+  }
+  return out
+}
+
+/**
+ * The Agents the project's files make customer-facing: each with an
+ * `operating.json` (Forge improvement plan X4, T7; P5 A15). Behavior settings
+ * are what make an Agent serve customers, and the built-in Chat neither
+ * offers nor runs one; that is inferred, not chosen, until an Agent's
+ * audience is its own field (the plan's open question 5). `local check` says
+ * which, so a team's own Agent with an `operating.json` is noticed.
+ */
+export function customerFacingAgents(root: string): string[] {
+  const dir = join(root, 'agents')
+  if (!existsSync(dir)) return []
+  return readdirSync(dir).filter((slug) => existsSync(join(dir, slug, 'agent.json')) && existsSync(join(dir, slug, 'operating.json'))).sort()
+}
+
+export function customerFacingWords(slugs: readonly string[]): string {
+  return `${slugs.join(', ')} ${slugs.length === 1 ? 'serves' : 'serve'} customers, so the built-in Chat neither offers nor runs ${slugs.length === 1 ? 'it' : 'them'}: ${slugs.length === 1 ? 'it has' : 'each has'} behavior settings (operating.json). For an Agent your team uses in Chat, delete its operating.json; in the workspace, \`frontera agent behavior clear <agent>\` and publish`
+}

package/src/local/local-ungranted-reads.ts

@@ -0,0 +1,55 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { basename, join } from 'node:path'
+
+import { collectAppContract } from '../app-contract'
+import { readSnapshot } from './local-sync-snapshot'
+
+/**
+ * What the project's workflows and Apps read that the linked workspace isn't
+ * granted (Forge improvement plan X2, T12 step 3). The local runtime grants
+ * the workspace every object type, so a workflow or App reading a type the
+ * project adds works here. In the workspace it reads only its slice: a change
+ * request that creates the type grants it, and a Blueprint published by hand
+ * grants nothing. Said as a warning when the Blueprint applies, for a project
+ * linked to a workspace; nothing for one that isn't.
+ */
+export interface UngrantedRead {
+  objectType: string
+  /** `workflow/intake`, `app/desk`. */
+  readers: string[]
+}
+
+const TYPE_REF = /objectType\/([A-Z][A-Za-z0-9]*)/g
+
+function yamlFiles(dir: string): string[] {
+  return existsSync(dir) ? readdirSync(dir).filter((f) => /\.ya?ml$/.test(f)).map((f) => join(dir, f)) : []
+}
+
+export function ungrantedReads(root: string): UngrantedRead[] {
+  const slice = readSnapshot(root)?.names?.objectType
+  if (!slice) return []
+  const granted = new Set(Object.keys(slice))
+  const projectTypes = new Set(yamlFiles(join(root, 'blueprint', 'object-types')).map((p) => basename(p).replace(/\.ya?ml$/, '')))
+  const readers = new Map<string, Set<string>>()
+  const read = (type: string, by: string) => {
+    if (!projectTypes.has(type) || granted.has(type)) return
+    readers.set(type, (readers.get(type) ?? new Set()).add(by))
+  }
+  for (const path of yamlFiles(join(root, 'workflows'))) {
+    const name = basename(path).replace(/\.ya?ml$/, '')
+    for (const m of readFileSync(path, 'utf8').matchAll(TYPE_REF)) read(m[1]!, `workflow/${name}`)
+  }
+  const apps = join(root, 'apps')
+  if (existsSync(apps)) {
+    for (const slug of readdirSync(apps)) {
+      const dir = join(apps, slug)
+      if (!statSync(dir).isDirectory() || !existsSync(join(dir, 'frontera.config.json'))) continue
+      for (const type of Object.keys(collectAppContract(dir).objectTypes)) read(type, `app/${slug}`)
+    }
+  }
+  return [...readers].sort(([a], [b]) => a.localeCompare(b)).map(([objectType, by]) => ({ objectType, readers: [...by].sort() }))
+}
+
+export function ungrantedReadLines(found: UngrantedRead[]): string[] {
+  return found.map((f) => `  warning: ${f.readers.join(', ')} ${f.readers.length === 1 ? 'reads' : 'read'} ${f.objectType}, which the linked workspace isn't granted. A change request that creates ${f.objectType} grants it to the workspace; a Blueprint published by hand (\`frontera blueprint publish\`) grants nothing, and there ${f.readers.length === 1 ? 'it' : 'they'} can't read it`)
+}