@frontera-sdk/cli 1.52.39 → 1.52.41

package/src/mcp/tools/project-tools.ts

@@ -0,0 +1,205 @@
+import { existsSync, readFileSync } from 'node:fs'
+
+import { noticeLines, projectNotices } from '../../local/local-project-notices'
+import { solutionStatus, type ItemBuild } from '../../local/local-applied'
+import { applySummaryLine, inheritedRefsOf } from '../../local/local-apply-summary'
+import { keylessEmbeddingsStatus } from '../../local/local-org-settings'
+import { localOrganizationId } from '../../local/local-blueprint-outline'
+import { docsStatus } from '../../local/local-docs'
+import { readConfig } from '../../local/local-env'
+import { liveAppDeployments, workspaceGet } from '../../local/local-app-runtime'
+import { projectBuildRows, projectBuildSummary } from '../../local/local-build-summary'
+import { localSecretPresence } from '../../local/local-secrets'
+import { readBaseline } from '../../local/local-db-snapshot'
+import { stuckKnowledge } from '../../local/local-knowledge-ingestion'
+import { projectOrphans } from '../../local/local-orphans'
+import type { LocalProject } from '../../local/local-project'
+import { readLink } from '../../local/local-sync-link'
+import { logPath, readState, type ComponentName } from '../../local/local-state'
+import { runSelf } from '../mcp-cli'
+import { errorResult, intArg, jsonResult, stringArg, type McpTool } from '../mcp-protocol'
+import { runtimeLink, studioLink } from '../mcp-runtime'
+import { savedObjectSets } from './workflow-tools'
+import { listDevServers } from '../mcp-apps'
+
+/**
+ * Where the project stands: what the runtime made of each item (with why
+ * not, followed to the item that is actually broken), whether saved files
+ * are applied, the workspace link and the runtime's health. And the two
+ * things a person would type to move it: apply, and read a log.
+ */
+
+const COMPONENTS: ComponentName[] = ['service', 'jobs', 'runner', 'studio', 'postgres', 'storage']
+
+/** "Needs workflow telco-fault-triage" → `workflow/telco-fault-triage`. */
+function neededRef(reason: string | null): string | null {
+  const match = reason?.match(/\b(agent|workflow|function|skill|knowledge|plugin|chat-app)\s+([a-z0-9][a-z0-9._-]*)/i)
+  return match ? `${match[1]!.toLowerCase()}/${match[2]}` : null
+}
+
+/**
+ * A blocked item's reason chain: the items it waits on, in order, down to
+ * the one that failed or is missing, so the agent fixes the cause and not
+ * the symptom.
+ */
+export function reasonChain(ref: string, builds: Record<string, ItemBuild>, depth = 6): Array<{ ref: string; state: string; reason: string | null }> {
+  const chain: Array<{ ref: string; state: string; reason: string | null }> = []
+  const seen = new Set<string>()
+  let at: string | null = ref
+  while (at && !seen.has(at) && chain.length < depth) {
+    seen.add(at)
+    const build: ItemBuild | undefined = builds[at]
+    if (!build) {
+      chain.push({ ref: at, state: 'missing', reason: 'not in this project' })
+      break
+    }
+    chain.push({ ref: at, state: build.state, reason: build.reason })
+    at = build.state === 'blocked' ? neededRef(build.reason) : null
+  }
+  return chain
+}
+
+export function projectTools(project: LocalProject): McpTool[] {
+  return [
+    {
+      name: 'status',
+      description:
+        "Where the project stands on the local runtime: what each Agent, workflow, Function and Blueprint object type came to (built, adapted, stubbed, needs setup, blocked, failed, not applied) with the reason chain to the item actually at fault, files saved but not applied, the workspace link, the runtime's health and its documentation's version, the saved object sets with their ids (blueprint_query reads one), any knowledge document left pending or processing with no ingestion behind it (`knowledgeStuck`, each with the command that re-queues it), and each Agent, workflow, Function or knowledge base on the runtime with no file in the project (`notInProject`, each with its fix: apply archives an Agent or workflow whose file was deleted, and switches off such a Function). `build` counts exactly the rows `items` lists with all: true. Read it first after a change.",
+      inputSchema: { type: 'object', properties: { all: { type: 'boolean', description: 'List built items too, not only the ones with something to say (default false).' } }, additionalProperties: false },
+      annotations: { title: 'Project status', readOnlyHint: true, openWorldHint: false },
+      async run(input) {
+        const state = readState(project)
+        const link = runtimeLink(project)
+        const org = localOrganizationId(project.root)
+        // The runtime's secrets now: a "Needs a value" follows them, not the last apply.
+        const secrets = typeof link === 'string' ? null : await localSecretPresence(link.serviceUrl, link.creds)
+        // Each Custom App's deployment as the runtime holds it: a folder's record is not proof it is still there.
+        const liveApps = typeof link === 'string' ? null : await liveAppDeployments(project.root, workspaceGet(link.serviceUrl, link.creds)).catch(() => null)
+        // The rows the build line counts, object types and items never applied included, so the two agree.
+        const rows = projectBuildRows(project.root, project.id, null, null, secrets, liveApps) ?? []
+        const builds: Record<string, ItemBuild> = Object.fromEntries(rows.map(({ ref, ...b }) => [ref, b]))
+        const items = Object.entries(builds)
+          .filter(([, b]) => input.all === true || b.state !== 'built')
+          .map(([ref, b]) => ({
+            ref,
+            state: b.state,
+            ...(b.reason ? { reason: b.reason } : {}),
+            ...(b.adaptations.length ? { adaptations: b.adaptations } : {}),
+            ...(b.fix ? { fix: b.fix } : {}),
+            ...(b.state === 'blocked' ? { chain: reasonChain(ref, builds).slice(1) } : {}),
+          }))
+        // A Blueprint that applied with items the runtime refused is on the runtime: `builds` lists what it refused.
+        const unapplied = solutionStatus(project, org).filter((i) => i.state !== 'applied' && i.state !== 'partial')
+        const sync = readLink(project.root)
+        // The saved object sets with their ids: status lists `objectSet/<file>` as built, and an App or a query addresses a set by id.
+        const objectSets = typeof link === 'string' ? null : await savedObjectSets(link, project.root).catch(() => null)
+        return jsonResult({
+          runtime:
+            typeof link === 'string'
+              ? { running: false, why: link }
+              : {
+                  running: true,
+                  version: link.state.platformVersion,
+                  mode: link.state.mode,
+                  studio: link.studioUrl,
+                  components: link.state.components.map((c) => ({ name: c.name, status: c.status, ...(c.detail ? { detail: c.detail } : {}) })),
+                },
+          build: projectBuildSummary(project.root, project.id, null, secrets, liveApps)?.line ?? (state ? 'Nothing applied yet: run apply.' : 'The runtime has never started here.'),
+          items,
+          notApplied: unapplied.map((i) => ({ path: i.path, state: i.waiting ? 'waiting' : i.state, ...(i.detail ? { reason: i.detail } : {}), ...(i.code ? { code: i.code } : {}), ...(i.fix ? { fix: i.fix } : {}) })),
+          ...(objectSets?.length ? { objectSets: objectSets.map((o) => ({ name: o.name, id: o.id, ...(o.file ? { ref: `objectSet/${o.file}` } : {}) })) } : {}),
+          // apply helps only files changed since their last apply: one that failed as it is, or waits, says its own fix.
+          ...(unapplied.some((i) => (i.state === 'new' || i.state === 'changed') && !i.waiting) ? { next: 'apply puts the files changed since the last apply on the runtime.' } : {}),
+          // The App dev servers running against this runtime, with their ports (app_dev with stop, or `frontera local stop`, stops them).
+          ...(() => {
+            const dev = listDevServers(project)
+            return dev.length ? { appDevServers: dev.map((d) => ({ app: d.slug, port: d.port, url: d.url, startedBy: d.owner?.kind ?? 'hand' })) } : {}
+          })(),
+          // A knowledge document left unread with no ingestion behind it (a rollback, a lost job): its item reads built otherwise.
+          ...(await (async () => {
+            if (typeof link === 'string') return {}
+            const stuck = await stuckKnowledge(link.serviceUrl, link.creds, readBaseline(project, link.creds.organizationId)?.restoredAt).catch(() => null)
+            return stuck?.length ? { knowledgeStuck: stuck } : {}
+          })()),
+          // An Agent, workflow, Function or knowledge base on the runtime with no file in the project, with its fix.
+          ...(await (async () => {
+            if (typeof link === 'string') return {}
+            const orphans = await projectOrphans(project, link.serviceUrl, link.creds).catch(() => null)
+            return orphans?.length ? { notInProject: orphans.map((o) => ({ ref: o.ref, problem: o.problem, fix: o.fix, fileDeleted: o.deleted })) } : {}
+          })()),
+          workspace: sync ? { linked: true, workspace: sync.workspaceName ?? sync.workspace, platform: sync.platform } : { linked: false },
+          // Knowledge search on the keyless local embeddings: approximate, with what gives the platform's.
+          ...(state?.credentials?.modelProviders?.includes('local-embeddings') ? { embeddings: keylessEmbeddingsStatus() } : {}),
+          // What a pull couldn't bring and why, App code on the platform, older sample data or vendored SDK: each with its command.
+          ...(() => {
+            const all = projectNotices(project.root)
+            const notices = noticeLines(all)
+            // Each pulled Agent's notes from the pull (no behavior settings on the platform, a connection that stays there).
+            return { ...(notices.length ? { notices } : {}), ...(all.agentNotes?.length ? { agentPullNotes: all.agentNotes } : {}) }
+          })(),
+          // The documentation the docs tools serve, against the runtime: an old build is said, not served silently.
+          ...(state ? { docs: docsStatus(project, { mode: state.mode, platformVersion: state.platformVersion, platformSource: readConfig(project)?.platformSource }) } : {}),
+        })
+      },
+    },
+    {
+      name: 'apply',
+      description:
+        'Apply the project\'s saved files to the local runtime now (`frontera local dev --once`): only what changed since the last apply, unless reapply is true. Returns what was applied and what failed, with the runtime\'s errors, and what each applied item warned (result.warnings). `ok` is true when everything changed or added applied: items still as the last pull wrote them that fail are inheritedFailures (they fail as in the workspace), listed apart from problems, as `frontera local check` lists them.',
+      inputSchema: { type: 'object', properties: { reapply: { type: 'boolean', description: 'Apply every item again, changed or not (default false).' } }, additionalProperties: false },
+      annotations: { title: 'Apply to the local runtime', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const link = runtimeLink(project)
+        if (typeof link === 'string') return errorResult(link)
+        const result = await runSelf(['local', 'dev', '--once', ...(input.reapply === true ? ['--reapply'] : [])], { cwd: project.root })
+        const builds: Record<string, ItemBuild> = Object.fromEntries((projectBuildRows(project.root, project.id) ?? []).map(({ ref, ...b }) => [ref, b]))
+        const failing = Object.entries(builds)
+          .filter(([, b]) => b.state === 'failed' || b.state === 'blocked')
+          .map(([ref, b]) => ({ ref, state: b.state, reason: b.reason, ...(b.state === 'blocked' ? { chain: reasonChain(ref, builds).slice(1) } : {}) }))
+        // Split as the apply split them (and `local check`): items still as the pull wrote them fail as in the workspace,
+        // and don't make the apply fail. `problems` are what changed or was added here.
+        const inheritedRefs = inheritedRefsOf(result.data)
+        const problems = failing.filter((p) => !inheritedRefs.has(p.ref))
+        const inheritedFailures = failing.filter((p) => inheritedRefs.has(p.ref))
+        return {
+          ...(result.ok ? {} : { isError: true as const }),
+          content: [
+            {
+              type: 'text',
+              text: JSON.stringify(
+                { ok: result.ok, summary: applySummaryLine(result.ok, problems.length, inheritedFailures.length), result: result.data, build: projectBuildSummary(project.root, project.id)?.line ?? null, problems, ...(inheritedFailures.length ? { inheritedFailures } : {}), log: result.log, studio: studioLink(link, '/agents') },
+                null,
+                2,
+              ),
+            },
+          ],
+        }
+      },
+    },
+    {
+      name: 'logs',
+      description: "The end of a local runtime component's log (service by default), optionally only lines containing some text. Use it when a tool or apply reports a server error.",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          component: { type: 'string', enum: COMPONENTS, description: 'Which component (default service).' },
+          lines: { type: 'integer', minimum: 1, maximum: 1000, description: 'At most this many lines (default 120).' },
+          contains: { type: 'string', description: 'Only lines containing this text, case-insensitive.' },
+        },
+        additionalProperties: false,
+      },
+      annotations: { title: 'Runtime logs', readOnlyHint: true, openWorldHint: false },
+      async run(input) {
+        const component = (stringArg(input, 'component') ?? 'service') as ComponentName
+        if (!COMPONENTS.includes(component)) return errorResult(`unknown component "${component}"; use one of ${COMPONENTS.join(', ')}`)
+        const file = logPath(project, component)
+        if (!existsSync(file)) return errorResult(`No ${component} log yet: the runtime has not run here.`)
+        const needle = stringArg(input, 'contains')?.toLowerCase()
+        const all = readFileSync(file, 'utf8').split('\n').filter(Boolean)
+        const matching = needle ? all.filter((l) => l.toLowerCase().includes(needle)) : all
+        const lines = matching.slice(-intArg(input, 'lines', 120, 1, 1000))
+        return { content: [{ type: 'text', text: lines.length ? lines.join('\n') : `No ${component} log lines${needle ? ` contain "${needle}"` : ''}.` }] }
+      },
+    },
+  ]
+}

package/src/mcp/tools/test-tools.ts

@@ -0,0 +1,339 @@
+import { stringify } from 'yaml'
+
+import { caseTraces } from '../../local/local-case-evidence'
+import { notRunGroups, notRunWords } from '../../local/local-case-not-run'
+import type { LocalProject } from '../../local/local-project'
+import { compareWithLastRun, readLastRun, recordRun } from '../../local/local-test-baseline'
+import { appendTestCase, decisionOfApprovals, MIXED_DECISIONS_LINE, noteRedactedVariables, redactedVariablesLine, renderTestCase, type SavedDecision } from '../../local/local-test-cases'
+import { agentFilesProblem, caseCount, judge, runCases, selectCaseFiles, usesRealActions, waitedWords, type CaseFile, type CaseResult, type TestCase, type TestCheck, type ToolCall } from '../../local/local-test-runner'
+import { errorResult, jsonResult, stringArg, type McpTool } from '../mcp-protocol'
+import { requireRuntime, serviceCall, studioLink, type RuntimeLink } from '../mcp-runtime'
+import { caseReplyChecks, conversationRunsRealActions, conversationVariables, goneConversation, type McpMemory } from './agent-tools'
+
+/** The conversation's stored messages, or null when the runtime no longer has it. */
+async function storedMessages(link: RuntimeLink, conversationId: string): Promise<StoredMessage[] | null> {
+  try {
+    const messages = await serviceCall<unknown>(link, `/v1/sessions/${encodeURIComponent(conversationId)}/messages`)
+    return (Array.isArray(messages) ? messages : ((messages as { messages?: unknown[] })?.messages ?? [])) as StoredMessage[]
+  } catch (err) {
+    if (/answered 404/.test((err as Error).message)) return null
+    throw err
+  }
+}
+
+/**
+ * The project's saved Agent test cases, run the way `frontera local test`
+ * runs them, with the answer the coding agent needs: which cases its change
+ * made fail, compared with the last local run.
+ */
+
+function caseLine(link: RuntimeLink, r: CaseResult, files: CaseFile[] = [], replyCheck?: Awaited<ReturnType<typeof caseReplyChecks>>, trace?: { traceId: string; traceLink: string }) {
+  const c = files.find((f) => f.file === r.file)?.cases.find((x) => x.name === r.name)
+  return {
+    agent: r.agent,
+    file: r.file,
+    name: r.name,
+    passed: r.passed,
+    // A case that runs the tools that change something for real (realActions, or a decision) says so: its records changed.
+    ...(c && usesRealActions(c) ? { actions: 'real' } : {}),
+    ...(r.passed ? {} : { reason: r.reason, reply: r.reply.slice(0, 1_200) }),
+    // Never got a reply: the provider refused it, the turn timed out or the runtime did not answer. Not a regression.
+    ...(r.notRun ? { couldNotRun: { cause: r.notRun.cause, what: r.notRun.what, fix: r.notRun.fix, ...(r.notRun.skipped ? { sent: false } : {}) } } : {}),
+    // Run twice: the model provider refused the first try for its limits (credits, rate, requests in flight).
+    ...(r.retried ? { retried: `the provider refused the first try for its limits, so it ran again after a pause: ${r.retried}` } : {}),
+    // A case that keeps a gap between turns (`waitSeconds`) says so: it took that long on purpose.
+    ...(r.waited?.length ? { waited: waitedWords(r.waited) } : {}),
+    // A failing case whose reply the reply check rewrote or replaced: which check, what it found, both drafts.
+    ...(replyCheck ? { replyCheck } : {}),
+    ms: r.ms,
+    // The conversation, and its last turn's trace (trace_get reads it; trace_list with the conversation id lists every turn's).
+    ...(r.conversationId ? { conversationId: r.conversationId } : {}),
+    ...(trace ? { traceId: trace.traceId, traceLink: trace.traceLink } : {}),
+    ...(r.agentId && r.conversationId ? { link: studioLink(link, `/agents/${encodeURIComponent(r.agentId)}/c/${encodeURIComponent(r.conversationId)}`) } : {}),
+  }
+}
+
+const label = (r: CaseResult) => `${r.agent} · ${r.name}`
+
+interface StoredMessage {
+  role?: string
+  content?: unknown
+  parts?: Array<{ type?: string; text?: string; toolName?: string; input?: unknown; state?: string }>
+}
+
+function textOf(message: StoredMessage): string {
+  if (typeof message.content === 'string') return message.content
+  return (message.parts ?? [])
+    .filter((p) => p.type === 'text' && typeof p.text === 'string')
+    .map((p) => p.text)
+    .join('\n')
+    .trim()
+}
+
+/** The tools an assistant message called, from its stored parts (`tool-<name>` or `dynamic-tool`). */
+export function toolCallsOf(message: StoredMessage | undefined): ToolCall[] {
+  const calls: ToolCall[] = []
+  for (const part of message?.parts ?? []) {
+    const name = part.type === 'dynamic-tool' ? part.toolName : part.type?.startsWith('tool-') ? part.type.slice('tool-'.length) : undefined
+    if (!name) continue
+    // A call stored waiting on a person's approval counts as called, as the stream reads it.
+    calls.push({ name, args: part.input, ok: part.state === 'output-available', ...(part.state === 'approval-requested' ? { parked: true } : {}) })
+  }
+  return calls
+}
+
+/**
+ * The tools of a conversation, for grading a case against it: every turn's
+ * (`turn: all`, the default) and the last turn's alone (`turn: last`). The
+ * last turn's as `agent_chat` saw them stream when it did, else as stored.
+ */
+export function conversationCalls(messages: StoredMessage[], lastReply: StoredMessage | undefined, lastTurnStreamed?: ToolCall[]): { all: ToolCall[]; last: ToolCall[] } {
+  const at = lastReply ? messages.lastIndexOf(lastReply) : messages.length
+  // The assistant messages since the last person's message are the last turn's, however the runtime split them.
+  let start = at
+  while (start > 0 && messages[start - 1]?.role === 'assistant') start--
+  const earlier = messages.slice(0, start).filter((m) => m.role === 'assistant').flatMap((m) => toolCallsOf(m))
+  const last = lastTurnStreamed ?? messages.slice(start, at + 1).filter((m) => m.role === 'assistant').flatMap((m) => toolCallsOf(m))
+  return { all: [...earlier, ...last], last }
+}
+
+/** How the person decided the conversation's tool approvals, from the runtime's list of its cards; nothing when it can't say. */
+async function conversationDecision(link: RuntimeLink, conversationId: string): Promise<ReturnType<typeof decisionOfApprovals>> {
+  const cards = await serviceCall<unknown>(link, `/v1/chat/approvals?conversationId=${encodeURIComponent(conversationId)}`, { timeoutMs: 15_000 }).catch(() => [])
+  return decisionOfApprovals(Array.isArray(cards) ? cards : [])
+}
+
+const CHECK_TYPES = new Set(['contains', 'regex', 'tool_called', 'tool_not_called'])
+
+/** The checks as given, refused in words when one is not a check the runner grades. */
+export function readChecks(raw: unknown): TestCheck[] {
+  if (raw === undefined) return []
+  if (!Array.isArray(raw)) throw new Error('checks must be a list of checks')
+  return raw.map((c, i) => {
+    const check = (c && typeof c === 'object' ? c : {}) as Record<string, unknown>
+    const at = `checks[${i}]`
+    if (typeof check.type !== 'string' || !CHECK_TYPES.has(check.type)) throw new Error(`${at}.type must be contains, regex, tool_called or tool_not_called`)
+    if ((check.type === 'contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) throw new Error(`${at} (${check.type}) needs a value`)
+    if ((check.type === 'tool_called' || check.type === 'tool_not_called') && (typeof check.tool !== 'string' || !check.tool)) throw new Error(`${at} (${check.type}) needs a tool`)
+    if (check.argsSubset !== undefined && (typeof check.argsSubset !== 'object' || check.argsSubset === null || Array.isArray(check.argsSubset))) throw new Error(`${at}.argsSubset must be an object`)
+    if (check.mode !== undefined && !['exact', 'ignore', 'subset', 'superset'].includes(String(check.mode))) throw new Error(`${at}.mode must be exact, ignore, subset or superset`)
+    if (check.turn !== undefined && check.turn !== 'all' && check.turn !== 'last') throw new Error(`${at}.turn must be all or last`)
+    return check as unknown as TestCheck
+  })
+}
+
+/**
+ * A case with the coding agent's own expectations: the conversation up to
+ * the Agent's last reply, and what a good reply does. The reply itself is
+ * never the expectation here, so a case saved from a wrong answer does not
+ * make the wrong answer pass.
+ */
+export function expectationCase(args: {
+  name: string
+  savedAt: string
+  messages: StoredMessage[]
+  expectedContains: string[]
+  checks: TestCheck[]
+  /** The conversation ran actions for real: so does the case. */
+  realActions?: boolean
+  /** The variables the conversation started with: the case starts with them too. */
+  variables?: Record<string, string | number | boolean>
+  /** How the person decided the tool approvals it stopped on: the case answers them the same way. */
+  decision?: SavedDecision
+}): { testCase: TestCase & { savedFrom: string }; lastReply: StoredMessage | undefined } {
+  const turns = args.messages.filter((m) => (m.role === 'user' || m.role === 'assistant') && textOf(m))
+  const lastReply = [...args.messages].reverse().find((m) => m.role === 'assistant')
+  const cut = lastReply ? turns.lastIndexOf(lastReply) : -1
+  const inputs = cut >= 0 ? turns.slice(0, cut) : turns.filter((m) => m !== lastReply)
+  return {
+    testCase: {
+      name: args.name,
+      savedFrom: `frontera mcp ${args.savedAt}`,
+      conversation: inputs.map((m) => ({ [m.role!]: textOf(m) })),
+      ...(args.realActions ? { realActions: true } : {}),
+      ...(args.decision ? { decision: args.decision } : {}),
+      ...(args.variables ? { variables: args.variables } : {}),
+      ...(args.expectedContains.length ? { expectedContains: args.expectedContains } : {}),
+      ...(args.checks.length ? { checks: args.checks } : {}),
+    },
+    lastReply,
+  }
+}
+
+export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
+  return [
+    {
+      name: 'test_run',
+      description:
+        "Run the project's saved Agent test cases (agents/<slug>/evals/*.yaml) against the local runtime. Scope it to the Agent or file you changed. Returns each case's outcome and the cases that passed at the last run and fail now (newlyFailing): those are what your change broke. Cases the model provider refused (credits, a rate limit, a key), that timed out, or that the runtime did not answer are couldNotRun, with the cause and the fix: not a regression and not newly failing, but the run has not passed; tell the person the fix (keys and provider accounts are theirs). After a refusal every case would meet (no credit, a refused key), the cases not sent yet are not sent. A failing case whose reply the reply check rewrote or replaced carries replyCheck: which check fired, what it found, what the person got and both drafts. A case whose turns wait (`waitSeconds` on a user turn, up to 300 each) takes that long, and says so in `waited`. Each case names its conversationId and its last turn's traceId (trace_get reads it).",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          agent: { type: 'string', description: 'Only this Agent\'s cases (its slug).' },
+          file: { type: 'string', description: 'Only this case file, as a path from the project root, e.g. agents/desk/evals/regression.yaml.' },
+          live: { type: 'boolean', description: 'Test the published version instead of the draft (default false).' },
+          regression: { type: 'boolean', description: 'Only the regression datasets the workspace tests before publishing.' },
+          gate: { type: 'boolean', description: 'Only the cases marked gate: true: what a change request runs before it applies.' },
+          concurrency: { type: 'integer', minimum: 1, maximum: 16, description: 'How many cases run at once (default 2). Cases that change records for real still run one at a time. Lower it to 1 when the model provider refuses for requests in flight.' },
+        },
+        additionalProperties: false,
+      },
+      annotations: { title: 'Run test cases', readOnlyHint: false, openWorldHint: false },
+      async run(input) {
+        const link = requireRuntime(project)
+        const file = stringArg(input, 'file')?.replace(/^\.\//, '')
+        // Chosen as `local test` and a change request's evals choose them: each case with every field its file gives.
+        const files = selectCaseFiles(project.root, { agent: stringArg(input, 'agent'), file, regression: input.regression === true, gate: input.gate === true })
+          // A file with no cases at all is nothing to run; one that can't be read fails with why.
+          .filter((f) => f.cases.length || f.problem)
+        const total = caseCount(files)
+        if (total === 0) {
+          return errorResult(
+            file || input.agent
+              ? 'No test cases match. test_save_case turns a conversation into one; files live in agents/<slug>/evals/*.yaml.'
+              : 'No test cases in this project yet. Chat with an Agent (agent_chat), then save the conversation with test_save_case.',
+          )
+        }
+        const last = readLastRun(project)
+        // A draft that isn't what the Agent's files say fails its cases with what to run, as `local test` does.
+        const concurrency = typeof input.concurrency === 'number' && Number.isInteger(input.concurrency) && input.concurrency >= 1 && input.concurrency <= 16 ? input.concurrency : undefined
+        const results = await runCases({ serviceUrl: link.serviceUrl, creds: link.creds, files, live: input.live === true, concurrency, staleDraft: (slug) => agentFilesProblem(project, link.creds.organizationId, slug), onResult: () => {} })
+        const comparison = compareWithLastRun(results, last)
+        recordRun(project, results)
+        const failed = results.filter((r) => !r.passed && !r.notRun)
+        const passed = results.filter((r) => r.passed).length
+        const notRun = notRunGroups(results)
+        return jsonResult({
+          summary: `${passed} passed, ${failed.length} failed${comparison.newlyFailing.length ? `, ${comparison.newlyFailing.length} newly failing` : ''}${notRun.length ? `, ${notRunWords(notRun)}` : ''}`,
+          newlyFailing: comparison.newlyFailing.map(label),
+          fixed: comparison.fixed.map(label),
+          stillFailing: comparison.stillFailing.map(label),
+          ...(comparison.firstRun.length ? { firstRun: comparison.firstRun.length } : {}),
+          // Not the Agent: what kept these cases from running, and what fixes it.
+          ...(notRun.length ? { couldNotRun: notRun.map((g) => ({ what: `couldn’t run: ${g.what}`, fix: g.fix, cases: g.cases, ...(g.skipped ? { notSent: g.skipped } : {}) })) } : {}),
+          cases: await (async () => {
+            const traces = await caseTraces(project, results)
+            return Promise.all(results.map(async (r, i) => caseLine(link, r, files, r.passed || r.notRun ? null : await caseReplyChecks(link, r.agentId, r.conversationId).catch(() => null), traces.get(i))))
+          })(),
+        })
+      },
+    },
+    {
+      name: 'test_save_case',
+      description:
+        "Save a conversation as a regression test case in agents/<slug>/evals/regression.yaml. Say what a good reply does with expectedContains and checks (tool_called, tool_not_called, contains, regex): the case is graded against the conversation's last reply before it is saved. Without them the Agent's last reply becomes the exact expected answer, which locks in whatever it said. Saving the case for a bug you are about to fix? Pass expectFailingNow: true and expectations the current reply fails. Defaults to your last agent_chat conversation with that Agent.",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          agent: { type: 'string', description: 'The Agent\'s slug.' },
+          conversationId: { type: 'string', description: 'The conversation to save; your last agent_chat with this Agent when left out.' },
+          name: { type: 'string', maxLength: 120, description: 'The case\'s name (default conversation-<id>).' },
+          expectedContains: { type: 'array', items: { type: 'string', minLength: 1 }, description: 'Words or phrases a good reply contains, any case.' },
+          checks: {
+            type: 'array',
+            description: 'Checks on the reply and the tools the Agent called; every one must pass.',
+            items: {
+              type: 'object',
+              properties: {
+                type: { type: 'string', enum: ['contains', 'regex', 'tool_called', 'tool_not_called'] },
+                value: { type: 'string', description: 'contains, regex: the text or pattern.' },
+                flags: { type: 'string', description: 'regex: its flags (default i).' },
+                tool: { type: 'string', description: 'tool_called, tool_not_called: the tool\'s name as the trace shows it (a Plugin\'s tool by its capability name, mcp__…, never its bare name).' },
+                argsSubset: { type: 'object', description: 'tool_called: arguments the call must carry.' },
+                mode: { type: 'string', enum: ['exact', 'ignore', 'subset', 'superset'], description: 'How argsSubset is compared (default superset).' },
+                turn: { type: 'string', enum: ['all', 'last'], description: 'tool_called, tool_not_called: the calls of every turn (all, the default) or only of the last one.' },
+                text: { type: 'string', description: 'What the check means, in words, for its failure message.' },
+              },
+              required: ['type'],
+              additionalProperties: false,
+            },
+          },
+          expectFailingNow: {
+            type: 'boolean',
+            description: 'The expectations describe a fix not made yet: save the case only if the current reply fails them (a red test). Without it, a case that fails its own conversation is refused.',
+          },
+        },
+        required: ['agent'],
+        additionalProperties: false,
+      },
+      annotations: { title: 'Save a test case', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const link = requireRuntime(project)
+        const slug = stringArg(input, 'agent', { required: true })
+        const remembered = memory.lastConversation.get(slug)
+        const conversationId = stringArg(input, 'conversationId') ?? remembered?.conversationId
+        if (!conversationId) return errorResult(`No conversation with ${slug} yet in this session: chat with it first (agent_chat), or pass conversationId.`)
+        const expectedContains = Array.isArray(input.expectedContains) ? input.expectedContains.filter((v): v is string => typeof v === 'string' && v.trim() !== '') : []
+        const checks = readChecks(input.checks)
+        const expectFailingNow = input.expectFailingNow === true
+        const expectations = expectedContains.length > 0 || checks.length > 0
+        if (expectFailingNow && !expectations) return errorResult('expectFailingNow needs the expectations the fixed reply meets: pass expectedContains or checks.')
+        const stored = await storedMessages(link, conversationId)
+        // A conversation with a draft goes with the draft's test copy (an apply, a publish, an hour unused): what agent_chat saw of it still makes a case.
+        const seen = memory.conversations.get(conversationId)
+        if (seen && seen.agent !== slug) return errorResult(`Conversation ${conversationId} is with ${seen.agent}, not ${slug}: pass agent ${JSON.stringify(seen.agent)}.`)
+        const list = (stored ?? (seen?.transcript as StoredMessage[] | undefined) ?? []) as StoredMessage[]
+        if (stored === null && !seen) return errorResult(`${goneConversation(conversationId)} Then save the case right after that conversation, or pass the conversationId agent_chat gave you in this session.`)
+        if (!list.length) return errorResult(`Conversation ${conversationId} has no messages to save.`)
+        const savedAt = new Date().toISOString()
+        const name = stringArg(input, 'name') ?? `conversation-${conversationId.slice(0, 8)}`
+        // A conversation that ran actions for real (agent_chat realActions) makes a case that does too, or it would grade simulated runs.
+        const agentId = seen?.agentId ?? (remembered?.conversationId === conversationId ? remembered.agentId : '')
+        const { realActions } = seen?.live ? { realActions: false } : await conversationRunsRealActions(memory, link, agentId, conversationId)
+        // The variables it started with (a customer, a unit): without them a replay reads as nobody, and data-scoped reads see nothing.
+        const { variables, redacted } = seen?.live ? { variables: undefined, redacted: [] as string[] } : await conversationVariables(memory, link, agentId, conversationId)
+        const kept = variables ? { variables: `kept: ${Object.keys(variables).join(', ')} (the case starts with them, as the conversation did)` } : {}
+        // The tool approvals it stopped on, as the person decided them: without them a replay waits on the first card and fails.
+        const { decision, mixed } = seen?.live ? {} : await conversationDecision(link, conversationId)
+        const decided = {
+          ...(decision ? { decision: `${decision}: the case ${decision === 'approve' ? 'approves' : 'denies'} each tool approval it stops on, as the conversation did` } : {}),
+          ...(mixed ? { decisionNote: MIXED_DECISIONS_LINE } : {}),
+        }
+        // A sensitive variable's value is never recorded: the case names it, to be filled in by hand.
+        const fillIn = (path: string) => (redacted.length ? { redactedVariables: redactedVariablesLine(redacted, path) } : {})
+        if (!expectations) {
+          const rendered = renderTestCase({ conversationId, messages: list as never[], savedAt, ...(variables ? { variables } : {}), ...(decision ? { decision } : {}) })
+          const yaml = noteRedactedVariables(realActions ? rendered.replace(/^(  savedFrom: .*)$/m, '$1\n  realActions: true') : rendered, redacted)
+          const path = appendTestCase(project.root, slug, yaml)
+          return jsonResult({
+            saved: path,
+            case: `conversation-${conversationId.slice(0, 8)}`,
+            expectation: 'the exact last reply',
+            ...(realActions ? { actions: 'real: the case runs the tools that change something for real, as the conversation did' } : {}),
+            ...decided,
+            ...kept,
+            ...fillIn(path),
+            warning: 'This case passes only on the reply the Agent gave, right or wrong. Pass expectedContains or checks to say what a good reply does instead.',
+            next: `Run it with test_run { "file": "${path}" }.`,
+          })
+        }
+        const { testCase, lastReply } = expectationCase({ name, savedAt, messages: list, expectedContains, checks, realActions, ...(variables ? { variables } : {}), ...(decision ? { decision } : {}) })
+        if (!lastReply) return errorResult(`Conversation ${conversationId} has no reply from ${slug} to grade the expectations against.`)
+        if (!testCase.conversation.length) return errorResult(`Conversation ${conversationId} has no message from the person before the reply.`)
+        // The tools of every turn (`turn: all`) and of the last (`turn: last`): the last as agent_chat saw them stream, else as stored.
+        const fromMemory = remembered?.conversationId === conversationId ? remembered.lastTurn : undefined
+        const calls = conversationCalls(list, lastReply, fromMemory?.toolCalls)
+        const graded = judge(testCase, textOf(lastReply), calls.all, calls.last)
+        if (graded.passed && expectFailingNow) {
+          return errorResult(`The current reply already meets these expectations, so the case would not fail before your fix. Tighten them, or save it without expectFailingNow.`)
+        }
+        if (!graded.passed && !expectFailingNow) {
+          return errorResult(
+            `These expectations fail on the conversation they come from: ${graded.reason}. Correct them, or pass expectFailingNow: true to save a case that fails until your fix lands.`,
+          )
+        }
+        const path = appendTestCase(project.root, slug, noteRedactedVariables(stringify([testCase], { lineWidth: 0 }), redacted))
+        return jsonResult({
+          saved: path,
+          case: name,
+          now: graded.passed ? 'passes on this conversation' : `fails, as expected: ${graded.reason}`,
+          ...(realActions ? { actions: 'real: the case runs the tools that change something for real, as the conversation did' } : {}),
+          ...decided,
+          ...kept,
+          ...fillIn(path),
+          next: graded.passed ? `Run it with test_run { "file": "${path}" }.` : `Fix the Agent, apply, and run test_run { "file": "${path}" } until it passes.`,
+        })
+      },
+    },
+  ]
+}