@frontera-sdk/cli 1.52.45 → 1.52.46

package/src/mcp/tools/agent-tools.ts

@@ -2,11 +2,14 @@ import { readFileSync } from 'node:fs'
 import { join } from 'node:path'
 
 import { itemBuild } from '../../local/local-build-summary'
+import { restoreBaseline, saveBaseline } from '../../local/local-db-snapshot'
 import type { LocalProject } from '../../local/local-project'
 import { providerRefusal } from '../../local/local-provider-refusal'
+import { waitForDatabaseReady } from '../../local/local-runtime-health'
 import { plainVariables, redactedVariablesOfTimeline, variablesOfTimeline } from '../../local/local-test-cases'
 import { chatTarget, type ToolCall } from '../../local/local-test-runner'
 import { dropTurn, readTurn, reattachTurn, startTurn, TURN_IDLE_MS, type HeldTurn, type PendingDecision, type TurnRead } from '../mcp-chat'
+import { conversationTranscript, searchConversations } from '../mcp-conversations'
 import { errorResult, intArg, jsonResult, stringArg, type McpTool, type ToolResult } from '../mcp-protocol'
 import { requireRuntime, serviceCall, studioLink, type RuntimeLink } from '../mcp-runtime'
 import { awaitTurnTrace, conversationTraces, getTrace, listTraces, proceduresOf, readTrace, replyCheckOf, replyChecksOfTrace, type ReplyCheck } from '../mcp-traces'
@@ -33,7 +36,7 @@ export interface SeenMessage {
  */
 export interface McpMemory {
   lastConversation: Map<string, { conversationId: string; agentId: string; at: string; lastTurn?: { reply: string; toolCalls: ToolCall[] } }>
-  conversations: Map<string, { agent: string; agentId: string; agentName: string; live: boolean; transcript: SeenMessage[]; realActions?: boolean; realActionsNote?: string | null; variables?: Record<string, unknown> }>
+  conversations: Map<string, { agent: string; agentId: string; agentName: string; live: boolean; transcript: SeenMessage[]; realActions?: boolean; realActionsNote?: string | null; variables?: Record<string, unknown>; rollback?: { savedAt: string; rolledBack?: string } }>
   waiting: Map<string, { held: HeldTurn; pending: PendingDecision; since: string; before: string[] }>
 }
 
@@ -328,6 +331,14 @@ export async function caseReplyChecks(link: RuntimeLink, agentId: string | undef
   return lastReplyChecks(timeline, []) ?? lastReplyChecks([], await conversationTraces(link, conversationId).catch(() => []))
 }
 
+/**
+ * The headline for a turn whose writes were simulated (Forge improvement plan
+ * N8): the Agent's reply may read as if a record changed, and it did not.
+ */
+export function simulatedWritesLine(count: number): string {
+  return `${count} write${count === 1 ? '' : 's'} simulated, not applied: nothing changed, whatever the reply says (realActions: true in a new conversation runs them for real)`
+}
+
 export function agentTools(project: LocalProject, memory: McpMemory): McpTool[] {
   /**
    * The turn as the agent reads it: the reply so far, what the Agent waits
@@ -392,6 +403,8 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
       agent: slug,
       version: args.live ? 'live' : 'draft',
       conversationId,
+      // First, so it is never missed: a write the draft only simulated changed nothing (N8).
+      ...(!args.live && simulated.length ? { simulatedWrites: simulatedWritesLine(simulated.length) } : {}),
       ...(args.answered ? { answered: args.answered } : {}),
       reply: read.reply,
       ...(read.end.status === 'waiting' ? { pendingDecision: pendingDecisionView(read.end.pending, conversationId), runId: read.runId } : {}),
@@ -409,6 +422,8 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
                 : 'Functions and Workflows that only read run for real; ones that change something are not run (realActions: true in a new conversation runs them)',
             ...(args.realActionsNote ? { realActionsNote: args.realActionsNote } : {}),
             ...(simulated.length ? { notRun: simulated } : {}),
+            // L2: the records were saved when it started; agent_rollback puts them back.
+            ...(memory.conversations.get(args.conversationId)?.rollback && !memory.conversations.get(args.conversationId)!.rollback!.rolledBack ? { rollback: `records saved at ${memory.conversations.get(args.conversationId)!.rollback!.savedAt}: agent_rollback puts back what this conversation wrote` } : {}),
           }),
       ...(model ? { model } : {}),
       ms,
@@ -536,6 +551,10 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
             additionalProperties: { type: ['string', 'number', 'boolean'] },
             description: "The conversation's variables, by name ({ customer_id: 'CST-00060' }), as a channel would set them: what the instructions read as {{customer_id}} and what operating.json dataScopes narrow the Agent's reads by. Only when starting a conversation on the draft.",
           },
+          rollback: {
+            type: 'boolean',
+            description: "With realActions, when starting: save the local runtime's records first, so agent_rollback puts back what this conversation's actions changed once you have read them (default false). It replaces the runtime's saved baseline, as `frontera local snapshot save` does.",
+          },
         },
         required: ['agent', 'message'],
         additionalProperties: false,
@@ -553,6 +572,15 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
         const variables = input.variables && typeof input.variables === 'object' && !Array.isArray(input.variables) && Object.keys(input.variables).length ? (input.variables as Record<string, unknown>) : undefined
         if (variables && conversationId) return errorResult('variables are set when a conversation starts: leave out conversationId to start one with them.')
         if (variables && live) return errorResult('variables are for the draft here: the published version gets them from its channel.')
+        // L2: a real-action conversation whose writes are put back afterwards, by agent_rollback.
+        const rollback = input.rollback === true
+        if (rollback && (conversationId || !realActions)) return errorResult('rollback is chosen when a conversation starts with realActions: true; it saves the records first, and agent_rollback puts them back.')
+        let savedAt: string | null = null
+        if (rollback) {
+          const saved = await saveBaseline(project, link.creds.organizationId, { serviceAnswers: () => waitForDatabaseReady(link.serviceUrl) }, true)
+          if (!saved.ok) return errorResult(`The records could not be saved first, so the conversation was not started: ${saved.note}`)
+          savedAt = new Date().toISOString()
+        }
 
         let target: { agentId: string; agentName: string }
         let realActionsNote: string | null = null
@@ -591,7 +619,7 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
             }
           }
         }
-        const remembered = memory.conversations.get(conversationId) ?? { agent: slug, agentId: target.agentId, agentName: target.agentName, live, transcript: [], realActions: realActions && !realActionsNote, realActionsNote, ...(variables ? { variables } : {}) }
+        const remembered = memory.conversations.get(conversationId) ?? { agent: slug, agentId: target.agentId, agentName: target.agentName, live, transcript: [], realActions: realActions && !realActionsNote, realActionsNote, ...(variables ? { variables } : {}), ...(savedAt ? { rollback: { savedAt } } : {}) }
         memory.conversations.set(conversationId, remembered)
         remembered.transcript.push({ role: 'user', parts: [{ type: 'text', text: message }] })
         // The traces already there, so this turn's is the one that is new.
@@ -624,6 +652,29 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
         return finishTurn({ conversationId, agentId: target.agentId, held, read, since, before, started, live, realActions, realActionsNote })
       },
     },
+    {
+      name: 'agent_rollback',
+      description:
+        "Put the local runtime's records back as they were when a conversation started with rollback: true (Forge improvement plan L2): what its real actions wrote, and anything else written since, is undone. The conversation itself, its traces and run history stay to read. Do it once you have checked what the actions did (blueprint_query).",
+      inputSchema: {
+        type: 'object',
+        properties: { conversationId: { type: 'string', description: 'The conversation agent_chat started with rollback: true.' } },
+        required: ['conversationId'],
+        additionalProperties: false,
+      },
+      annotations: { title: 'Put back what a conversation wrote', readOnlyHint: false, destructiveHint: true, openWorldHint: false },
+      async run(input) {
+        const link = requireRuntime(project)
+        const conversationId = stringArg(input, 'conversationId', { required: true })
+        const known = memory.conversations.get(conversationId)
+        if (!known?.rollback) return errorResult('That conversation was not started with rollback: true in this session, so there is nothing saved for it. `frontera local snapshot restore` puts back the runtime\'s saved baseline, whenever it was taken.')
+        if (known.rollback.rolledBack) return errorResult(`Already put back, at ${known.rollback.rolledBack}.`)
+        const restored = await restoreBaseline(project, link.creds.organizationId, { serviceAnswers: () => waitForDatabaseReady(link.serviceUrl) }, { secrets: 'keep', history: 'keep' })
+        if (!restored.ok) return errorResult(`The records could not be put back: ${restored.note}`)
+        known.rollback.rolledBack = new Date().toISOString()
+        return jsonResult({ conversationId, putBack: `the records as they were at ${known.rollback.savedAt}`, note: restored.note, kept: 'the conversation, its traces and run history' })
+      },
+    },
     {
       name: 'agent_decide',
       description:
@@ -668,6 +719,47 @@ export function agentTools(project: LocalProject, memory: McpMemory): McpTool[]
         return jsonResult(readTrace(link, trace, { detail: input.detail !== false }))
       },
     },
+    {
+      name: 'conversation_get',
+      description: "Read one conversation on the local runtime whole: every turn's message and reply, the tools each turn called (with their input, clipped), the procedure the turn ran under and why, what the reply check did, and each turn's traceId. A test case's conversationId, agent_chat's, or one conversation_search found. Use it instead of reading the runtime's database or asking the person to paste a transcript.",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          conversationId: { type: 'string', description: 'The conversation (session) id.' },
+          traces: { type: 'boolean', description: 'Read each turn\'s trace for its procedure and reply check (default true).' },
+        },
+        required: ['conversationId'],
+        additionalProperties: false,
+      },
+      annotations: { title: 'Read a conversation', readOnlyHint: true, openWorldHint: false },
+      async run(input) {
+        const link = requireRuntime(project)
+        const id = stringArg(input, 'conversationId', { required: true })
+        const transcript = await conversationTranscript(link, id, { traces: input.traces !== false })
+        if (!transcript) return errorResult(`no conversation "${id}" on the local runtime; conversation_search finds them`)
+        return jsonResult(transcript)
+      },
+    },
+    {
+      name: 'conversation_search',
+      description: 'Find conversations on the local runtime, newest first: by words in them, by Agent, or only those test runs started. Returns each one\'s conversationId for conversation_get. The local runtime only: a workspace\'s conversations are not read from here.',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          query: { type: 'string', description: 'Words the conversation contains.' },
+          agent: { type: 'string', description: 'Only this Agent\'s (its slug).' },
+          tests: { type: 'boolean', description: 'Only conversations test runs started.' },
+          limit: { type: 'integer', minimum: 1, maximum: 100, description: 'At most this many (default 20).' },
+        },
+        additionalProperties: false,
+      },
+      annotations: { title: 'Find conversations', readOnlyHint: true, openWorldHint: false },
+      async run(input) {
+        const link = requireRuntime(project)
+        const hits = await searchConversations(link, { query: stringArg(input, 'query'), agent: stringArg(input, 'agent'), tests: input.tests === true, limit: intArg(input, 'limit', 20, 1, 100) })
+        return jsonResult({ conversations: hits })
+      },
+    },
     {
       name: 'trace_list',
       description: 'Recent traces on the local runtime, newest first: Agent turns, workflow runs and Function calls. Filter by conversation, name or errors.',

package/src/mcp/tools/project-tools.ts

@@ -3,7 +3,10 @@ import { existsSync, readFileSync } from 'node:fs'
 import { noticeLines, projectNotices } from '../../local/local-project-notices'
 import { solutionStatus, type ItemBuild } from '../../local/local-applied'
 import { applySummaryLine, inheritedRefsOf } from '../../local/local-apply-summary'
-import { keylessEmbeddingsStatus } from '../../local/local-org-settings'
+import { keylessEmbeddingsStatus, PLATFORM_DEFAULT_EMBEDDING } from '../../local/local-org-settings'
+import { embeddingKeyOptions, embeddingNextStep } from '../../local/local-embedding-setup'
+import { configuredProviders } from '../../local/local-keys'
+import { readSnapshot } from '../../local/local-sync-snapshot'
 import { localOrganizationId } from '../../local/local-blueprint-outline'
 import { docsStatus } from '../../local/local-docs'
 import { readConfig } from '../../local/local-env'
@@ -21,6 +24,9 @@ import { errorResult, intArg, jsonResult, stringArg, type McpTool } from '../mcp
 import { runtimeLink, studioLink } from '../mcp-runtime'
 import { savedObjectSets } from './workflow-tools'
 import { listDevServers } from '../mcp-apps'
+import { addPersonStep, checkProblem, CREDENTIALS, deliveryGroups, PERSON_STEPS_FILE, readPersonSteps, type StepCheck } from '../../local/local-person-steps'
+import { readLock, recentWriters, writerOf, writerWords } from '../../local/local-writers'
+import { FRONTERA_READ_ONLY_COMMANDS } from '../../forge/permission-rules'
 
 /**
  * Where the project stands: what the runtime made of each item (with why
@@ -64,7 +70,7 @@ export function projectTools(project: LocalProject): McpTool[] {
     {
       name: 'status',
       description:
-        "Where the project stands on the local runtime: what each Agent, workflow, Function and Blueprint object type came to (built, adapted, stubbed, needs setup, blocked, failed, not applied) with the reason chain to the item actually at fault, files saved but not applied, the workspace link, the runtime's health and its documentation's version, the saved object sets with their ids (blueprint_query reads one), any knowledge document left pending or processing with no ingestion behind it (`knowledgeStuck`, each with the command that re-queues it), and each Agent, workflow, Function or knowledge base on the runtime with no file in the project (`notInProject`, each with its fix: apply archives an Agent or workflow whose file was deleted, and switches off such a Function). `build` counts exactly the rows `items` lists with all: true. Read it first after a change.",
+        "Where the project stands on the local runtime: who last changed it or its files (`writers`, one per session, `self` for yours; `runtimeLock` while another run holds the runtime), what each Agent, workflow, Function and Blueprint object type came to (built, adapted, stubbed, needs setup, blocked, failed, not applied) with the reason chain to the item actually at fault, files saved but not applied, the workspace link, the runtime's health and its documentation's version, the saved object sets with their ids (blueprint_query reads one), any knowledge document left pending or processing with no ingestion behind it (`knowledgeStuck`, each with the command that re-queues it), and each Agent, workflow, Function or knowledge base on the runtime with no file in the project (`notInProject`, each with its fix: apply archives an Agent or workflow whose file was deleted, and switches off such a Function). `build` counts exactly the rows `items` lists with all: true. Read it first after a change.",
       inputSchema: { type: 'object', properties: { all: { type: 'boolean', description: 'List built items too, not only the ones with something to say (default false).' } }, additionalProperties: false },
       annotations: { title: 'Project status', readOnlyHint: true, openWorldHint: false },
       async run(input) {
@@ -93,7 +99,12 @@ export function projectTools(project: LocalProject): McpTool[] {
         const sync = readLink(project.root)
         // The saved object sets with their ids: status lists `objectSet/<file>` as built, and an App or a query addresses a set by id.
         const objectSets = typeof link === 'string' ? null : await savedObjectSets(link, project.root).catch(() => null)
+        // Who last changed the runtime or the files (one per session, `self` for this session), and the run holding the runtime (X9).
+        const me = writerOf()
+        const lock = readLock(project.stateDir)
         return jsonResult({
+          writers: recentWriters(project.stateDir).map((w) => ({ ...w, self: w.session !== null && w.session === me.session, said: writerWords(w, me) })),
+          ...(lock ? { runtimeLock: { command: lock.command, session: lock.session, harness: lock.harness, since: lock.since, self: lock.session !== null && lock.session === me.session } } : {}),
           runtime:
             typeof link === 'string'
               ? { running: false, why: link }
@@ -128,8 +139,13 @@ export function projectTools(project: LocalProject): McpTool[] {
             return orphans?.length ? { notInProject: orphans.map((o) => ({ ref: o.ref, problem: o.problem, fix: o.fix, fileDeleted: o.deleted })) } : {}
           })()),
           workspace: sync ? { linked: true, workspace: sync.workspaceName ?? sync.workspace, platform: sync.platform } : { linked: false },
+          // The steps prepared for the person and not yet run (person_step): the delivery card Forge shows.
+          ...(() => {
+            const pending = readPersonSteps(project.root).filter((p) => p.status === 'pending')
+            return pending.length ? { personSteps: pending.map((p) => ({ id: p.id, command: p.command, credential: p.credential, ...(p.delivery ? { delivery: p.delivery } : {}) })) } : {}
+          })(),
           // Knowledge search on the keyless local embeddings: approximate, with what gives the platform's.
-          ...(state?.credentials?.modelProviders?.includes('local-embeddings') ? { embeddings: keylessEmbeddingsStatus() } : {}),
+          ...(state?.credentials?.modelProviders?.includes('local-embeddings') ? { embeddings: { ...keylessEmbeddingsStatus(), next: embeddingNextStep(embeddingKeyOptions(process.env, configuredProviders(), readSnapshot(project.root)?.settings?.embedding ?? PLATFORM_DEFAULT_EMBEDDING), true) } } : {}),
           // What a pull couldn't bring and why, App code on the platform, older sample data or vendored SDK: each with its command.
           ...(() => {
             const all = projectNotices(project.root)
@@ -176,13 +192,129 @@ export function projectTools(project: LocalProject): McpTool[] {
         }
       },
     },
+    {
+      name: 'agent_publish',
+      description:
+        "Publish an Agent's draft on the local runtime (`frontera local publish`), so its public website keys, Chat Apps and channels there talk to it. The local runtime only: the workspace's Agent is published by a person, through a change request.",
+      inputSchema: { type: 'object', properties: { agent: { type: 'string', description: 'The Agent\'s slug, its folder under agents/.' }, notes: { type: 'string', maxLength: 500, description: 'What the version changes.' } }, required: ['agent'], additionalProperties: false },
+      annotations: { title: 'Publish an Agent on the local runtime', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const link = runtimeLink(project)
+        if (typeof link === 'string') return errorResult(link)
+        const agent = stringArg(input, 'agent', { required: true })
+        const result = await runSelf(['local', 'publish', agent, ...(stringArg(input, 'notes') ? ['--notes', stringArg(input, 'notes')!] : [])], { cwd: project.root })
+        return result.ok ? jsonResult({ published: agent, result: result.data, studio: studioLink(link, '/agents') }) : errorResult(result.log || `Agent ${agent} could not be published on the local runtime.`)
+      },
+    },
+    {
+      name: 'agent_new',
+      description:
+        "Write a new Agent's files into the project (`frontera local new agent`): the starter (agent.json, bindings.json, a case; operating.json only with customer: true), or with like a copy of another Agent's files under the new name. Writes files only, never over one that is there; apply then applies them.",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          slug: { type: 'string', description: 'The new Agent\'s slug, its folder under agents/: lower-case letters, digits and hyphens.' },
+          like: { type: 'string', description: 'Start from this Agent\'s files, by its slug.' },
+          name: { type: 'string', description: 'Its display name; defaults to the slug in words.' },
+          description: { type: 'string' },
+          model: { type: 'string', description: 'The model it runs on, by catalog slug.' },
+          customer: { type: 'boolean', description: 'It serves customers: write behavior settings (operating.json).' },
+        },
+        required: ['slug'],
+        additionalProperties: false,
+      },
+      annotations: { title: 'Start a new Agent', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const slug = stringArg(input, 'slug', { required: true })
+        const flags = (['like', 'name', 'description', 'model'] as const).flatMap((k) => (stringArg(input, k) ? [`--${k}`, stringArg(input, k)!] : []))
+        const result = await runSelf(['local', 'new', 'agent', slug, ...flags, ...(input.customer === true ? ['--customer'] : [])], { cwd: project.root })
+        return result.ok ? jsonResult(result.data) : errorResult(result.log || `agents/${slug}/ could not be written.`)
+      },
+    },
+    {
+      name: 'format_files',
+      description:
+        "Write the project's JSON and YAML files in one shape (`frontera local fmt`): two-space indents, a final newline, comments kept. check: true writes nothing and lists what would change. Names a file it can't read, with the line and column.",
+      inputSchema: { type: 'object', properties: { paths: { type: 'array', items: { type: 'string' }, description: 'Only these files or folders, from the project root.' }, check: { type: 'boolean' } }, additionalProperties: false },
+      annotations: { title: 'Format project files', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const paths = Array.isArray(input.paths) ? input.paths.filter((p): p is string => typeof p === 'string') : []
+        const result = await runSelf(['local', 'fmt', ...paths, ...(input.check === true ? ['--check'] : [])], { cwd: project.root })
+        return result.ok ? jsonResult(result.data) : errorResult(result.log || 'The files could not be formatted.')
+      },
+    },
+    {
+      name: 'data_load',
+      description:
+        "Load records through the project's governed Actions on the local runtime (`frontera local data load <file> --via-actions`): a JSON or YAML file of { action, records } groups, run in its order, each record submitted as the local submitter persona. The records are saved first and put back when one is refused; loading the same file again submits nothing twice. Returns each group's applied, waiting, not applied and refused counts.",
+      inputSchema: { type: 'object', properties: { file: { type: 'string', description: 'The file, by its path from the project root.' }, batch: { type: 'integer', minimum: 1, maximum: 16, description: 'Records submitted at once (default 4).' } }, required: ['file'], additionalProperties: false },
+      annotations: { title: 'Load records through governed Actions', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        const link = runtimeLink(project)
+        if (typeof link === 'string') return errorResult(link)
+        const file = stringArg(input, 'file', { required: true })
+        const batch = typeof input.batch === 'number' ? ['--batch', String(input.batch)] : []
+        const result = await runSelf(['local', 'data', 'load', file, '--via-actions', ...batch], { cwd: project.root, timeoutMs: 15 * 60_000 })
+        return result.ok ? jsonResult({ ok: true, result: result.data, log: result.log }) : { isError: true as const, content: [{ type: 'text', text: JSON.stringify({ ok: false, result: result.data, log: result.log }, null, 2) }] }
+      },
+    },
+    {
+      name: 'person_step',
+      description:
+        "Prepare a step only the person takes (publish, promote, apply a change request, decide or review an Action, grant, set a secret) for them to run, instead of saying what is ready or asking them to paste output back. Records the `frontera` command, why it is needed, what it changes and the credential it needs in .frontera/sync/person-steps.json; Forge shows the pending steps as one delivery card per `delivery`, in order, and Run in my terminal types the command into the person's shell without pressing Enter. Nothing runs it but the person. One command on one line, no chaining, never a secret (a value is piped in by the person with `--from -`). Prepare the whole delivery's steps once, in order, rather than one per message; the same command again in the same delivery updates its step.",
+      inputSchema: {
+        type: 'object',
+        properties: {
+          command: { type: 'string', description: 'One `frontera` command, as the person runs it: `frontera local sync request apply`.' },
+          why: { type: 'string', description: 'Why it is needed now, in a sentence.' },
+          changes: { type: 'string', description: 'What it changes, in the workspace’s words: "Publishes Agent sdr in Sales".' },
+          credential: { type: 'string', enum: [...CREDENTIALS], description: 'Who or what the step needs.' },
+          delivery: { type: 'string', description: 'The delivery it belongs to (a change request’s title): steps with the same one show as one card.' },
+          check: {
+            type: 'object',
+            description: 'Optional: the read that proves it done, so it checks itself. A read-only `frontera … --json` command, the list in its data (`in`, a property name), the row whose fields equal `find`, and the fields that equal `expect` once done: {command: "frontera action list --json", in: "actions", find: {apiName: "qualifyLead"}, expect: {available: true}}.',
+            properties: { command: { type: 'string' }, in: { type: 'string' }, find: { type: 'object' }, expect: { type: 'object' } },
+            required: ['command', 'find', 'expect'],
+            additionalProperties: false,
+          },
+        },
+        required: ['command', 'why', 'changes', 'credential'],
+        additionalProperties: false,
+      },
+      annotations: { title: 'Prepare a person-step', readOnlyHint: false, destructiveHint: false, openWorldHint: false },
+      async run(input) {
+        try {
+          const check = input.check && typeof input.check === 'object' ? (input.check as StepCheck) : undefined
+          const checkWrong = check ? checkProblem(check, FRONTERA_READ_ONLY_COMMANDS) : null
+          if (checkWrong) return errorResult(`Not recorded: the check: ${checkWrong}`)
+          const { step, updated } = addPersonStep(project.root, {
+            command: stringArg(input, 'command') ?? '',
+            why: stringArg(input, 'why') ?? '',
+            changes: stringArg(input, 'changes') ?? '',
+            credential: stringArg(input, 'credential') ?? '',
+            delivery: stringArg(input, 'delivery') ?? null,
+            ...(check ? { check } : {}),
+          })
+          const card = deliveryGroups(readPersonSteps(project.root)).find((g) => g.delivery === step.delivery)
+          return jsonResult({
+            recorded: updated ? 'updated' : 'added',
+            step,
+            onCard: card?.steps.map((s) => s.command) ?? [step.command],
+            file: PERSON_STEPS_FILE,
+            next: 'Tell the person the delivery card is ready in Forge (or `frontera local steps`). Do not run the step yourself; check for done afterwards with a read (status, list, get).',
+          })
+        } catch (err) {
+          return errorResult(`Not recorded: ${(err as Error).message}`)
+        }
+      },
+    },
     {
       name: 'logs',
-      description: "The end of a local runtime component's log (service by default), optionally only lines containing some text. Use it when a tool or apply reports a server error.",
+      description: "The end of a local runtime component's log (service by default), or of every component's (all), optionally only lines containing some text. Use it when a tool or apply reports a server error; all when you don't know which component it came from.",
       inputSchema: {
         type: 'object',
         properties: {
-          component: { type: 'string', enum: COMPONENTS, description: 'Which component (default service).' },
+          component: { type: 'string', enum: [...COMPONENTS, 'all'], description: 'Which component (default service), or all: every component\'s lines, each marked with its component, in the order they were written.' },
           lines: { type: 'integer', minimum: 1, maximum: 1000, description: 'At most this many lines (default 120).' },
           contains: { type: 'string', description: 'Only lines containing this text, case-insensitive.' },
         },
@@ -190,15 +322,19 @@ export function projectTools(project: LocalProject): McpTool[] {
       },
       annotations: { title: 'Runtime logs', readOnlyHint: true, openWorldHint: false },
       async run(input) {
-        const component = (stringArg(input, 'component') ?? 'service') as ComponentName
-        if (!COMPONENTS.includes(component)) return errorResult(`unknown component "${component}"; use one of ${COMPONENTS.join(', ')}`)
-        const file = logPath(project, component)
-        if (!existsSync(file)) return errorResult(`No ${component} log yet: the runtime has not run here.`)
+        const asked = stringArg(input, 'component') ?? 'service'
+        if (asked !== 'all' && !COMPONENTS.includes(asked as ComponentName)) return errorResult(`unknown component "${asked}"; use one of ${COMPONENTS.join(', ')}, or all`)
+        const components = asked === 'all' ? COMPONENTS : [asked as ComponentName]
         const needle = stringArg(input, 'contains')?.toLowerCase()
-        const all = readFileSync(file, 'utf8').split('\n').filter(Boolean)
-        const matching = needle ? all.filter((l) => l.toLowerCase().includes(needle)) : all
-        const lines = matching.slice(-intArg(input, 'lines', 120, 1, 1000))
-        return { content: [{ type: 'text', text: lines.length ? lines.join('\n') : `No ${component} log lines${needle ? ` contain "${needle}"` : ''}.` }] }
+        const present = components.filter((c) => existsSync(logPath(project, c)))
+        if (!present.length) return errorResult(`No ${asked === 'all' ? '' : `${asked} `}log yet: the runtime has not run here.`)
+        // Every component's lines, marked, kept in the order each was written; across files, by the time a line starts with.
+        const all = present.flatMap((c) => readFileSync(logPath(project, c), 'utf8').split('\n').filter(Boolean).map((line, i) => ({ c, line, i })))
+        const stamp = (line: string) => Date.parse(/^\[?(\d{4}-\d{2}-\d{2}[T ][\d:.]+Z?)/.exec(line)?.[1] ?? '') || null
+        const ordered = components.length > 1 ? [...all].sort((a, b) => (stamp(a.line) ?? 0) - (stamp(b.line) ?? 0) || (a.c === b.c ? a.i - b.i : 0)) : all
+        const matching = needle ? ordered.filter((l) => l.line.toLowerCase().includes(needle)) : ordered
+        const lines = matching.slice(-intArg(input, 'lines', 120, 1, 1000)).map((l) => (components.length > 1 ? `[${l.c}] ${l.line}` : l.line))
+        return { content: [{ type: 'text', text: lines.length ? lines.join('\n') : `No ${asked === 'all' ? '' : `${asked} `}log lines${needle ? ` contain "${needle}"` : ''}.` }] }
       },
     },
   ]

package/src/mcp/tools/test-tools.ts

@@ -5,9 +5,12 @@ import { notRunGroups, notRunWords } from '../../local/local-case-not-run'
 import type { LocalProject } from '../../local/local-project'
 import { compareWithLastRun, readLastRun, recordRun } from '../../local/local-test-baseline'
 import { appendTestCase, decisionOfApprovals, MIXED_DECISIONS_LINE, noteRedactedVariables, redactedVariablesLine, renderTestCase, type SavedDecision } from '../../local/local-test-cases'
+import { compareWithRun, comparisonWords, listTestRuns, readTestRun, runOnModel, saveTestRun, stuckWords } from '../../local/local-test-models'
 import { agentFilesProblem, caseCount, judge, runCases, selectCaseFiles, usesRealActions, waitedWords, type CaseFile, type CaseResult, type TestCase, type TestCheck, type ToolCall } from '../../local/local-test-runner'
 import { errorResult, jsonResult, stringArg, type McpTool } from '../mcp-protocol'
 import { requireRuntime, serviceCall, studioLink, type RuntimeLink } from '../mcp-runtime'
+import { conversationTranscript } from '../mcp-conversations'
+import { modelSubstitutions, skippedCases, skippedWords } from '../../local/local-test-scope'
 import { caseReplyChecks, conversationRunsRealActions, conversationVariables, goneConversation, type McpMemory } from './agent-tools'
 
 /** The conversation's stored messages, or null when the runtime no longer has it. */
@@ -43,6 +46,11 @@ function caseLine(link: RuntimeLink, r: CaseResult, files: CaseFile[] = [], repl
     ...(r.retried ? { retried: `the provider refused the first try for its limits, so it ran again after a pause: ${r.retried}` } : {}),
     // A case that keeps a gap between turns (`waitSeconds`) says so: it took that long on purpose.
     ...(r.waited?.length ? { waited: waitedWords(r.waited) } : {}),
+    // `repeat`: how many of its runs passed; one that passed some is flaky (X1).
+    ...(r.repeat ? { runs: r.repeat.runs, passes: r.repeat.passes, passRate: r.repeat.passRate, ...(r.repeat.flaky ? { flaky: true } : {}) } : {}),
+    // Its `judged` expectations, graded by the runtime's judge, or why they weren't.
+    ...(r.judged?.length ? { judged: r.judged } : {}),
+    ...(r.judgedNotGraded ? { judgedNotGraded: r.judgedNotGraded } : {}),
     // A failing case whose reply the reply check rewrote or replaced: which check, what it found, both drafts.
     ...(replyCheck ? { replyCheck } : {}),
     ms: r.ms,
@@ -103,7 +111,7 @@ async function conversationDecision(link: RuntimeLink, conversationId: string):
   return decisionOfApprovals(Array.isArray(cards) ? cards : [])
 }
 
-const CHECK_TYPES = new Set(['contains', 'regex', 'tool_called', 'tool_not_called'])
+const CHECK_TYPES = new Set(['contains', 'not_contains', 'regex', 'tool_called', 'tool_not_called'])
 
 /** The checks as given, refused in words when one is not a check the runner grades. */
 export function readChecks(raw: unknown): TestCheck[] {
@@ -112,12 +120,12 @@ export function readChecks(raw: unknown): TestCheck[] {
   return raw.map((c, i) => {
     const check = (c && typeof c === 'object' ? c : {}) as Record<string, unknown>
     const at = `checks[${i}]`
-    if (typeof check.type !== 'string' || !CHECK_TYPES.has(check.type)) throw new Error(`${at}.type must be contains, regex, tool_called or tool_not_called`)
-    if ((check.type === 'contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) throw new Error(`${at} (${check.type}) needs a value`)
+    if (typeof check.type !== 'string' || !CHECK_TYPES.has(check.type)) throw new Error(`${at}.type must be contains, not_contains, regex, tool_called or tool_not_called`)
+    if ((check.type === 'contains' || check.type === 'not_contains' || check.type === 'regex') && (typeof check.value !== 'string' || !check.value)) throw new Error(`${at} (${check.type}) needs a value`)
     if ((check.type === 'tool_called' || check.type === 'tool_not_called') && (typeof check.tool !== 'string' || !check.tool)) throw new Error(`${at} (${check.type}) needs a tool`)
     if (check.argsSubset !== undefined && (typeof check.argsSubset !== 'object' || check.argsSubset === null || Array.isArray(check.argsSubset))) throw new Error(`${at}.argsSubset must be an object`)
     if (check.mode !== undefined && !['exact', 'ignore', 'subset', 'superset'].includes(String(check.mode))) throw new Error(`${at}.mode must be exact, ignore, subset or superset`)
-    if (check.turn !== undefined && check.turn !== 'all' && check.turn !== 'last') throw new Error(`${at}.turn must be all or last`)
+    if (check.turn !== undefined && !['all', 'any', 'last'].includes(String(check.turn)) && !(typeof check.turn === 'number' && Number.isInteger(check.turn) && check.turn >= 1)) throw new Error(`${at}.turn must be a turn number from 1, last, any or all`)
     return check as unknown as TestCheck
   })
 }
@@ -165,7 +173,7 @@ export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
     {
       name: 'test_run',
       description:
-        "Run the project's saved Agent test cases (agents/<slug>/evals/*.yaml) against the local runtime. Scope it to the Agent or file you changed. Returns each case's outcome and the cases that passed at the last run and fail now (newlyFailing): those are what your change broke. Cases the model provider refused (credits, a rate limit, a key), that timed out, or that the runtime did not answer are couldNotRun, with the cause and the fix: not a regression and not newly failing, but the run has not passed; tell the person the fix (keys and provider accounts are theirs). After a refusal every case would meet (no credit, a refused key), the cases not sent yet are not sent. A failing case whose reply the reply check rewrote or replaced carries replyCheck: which check fired, what it found, what the person got and both drafts. A case whose turns wait (`waitSeconds` on a user turn, up to 300 each) takes that long, and says so in `waited`. Each case names its conversationId and its last turn's traceId (trace_get reads it).",
+        "Run the project's saved Agent test cases (agents/<slug>/evals/*.yaml) against the local runtime. Scope it to the Agent or file you changed. Returns each case's outcome and the cases that passed at the last run and fail now (newlyFailing): those are what your change broke. Cases the model provider refused (credits, a rate limit, a key), that timed out, or that the runtime did not answer are couldNotRun, with the cause and the fix: not a regression and not newly failing, but the run has not passed; tell the person the fix (keys and provider accounts are theirs). After a refusal every case would meet (no credit, a refused key), the cases not sent yet are not sent. A failing case whose reply the reply check rewrote or replaced carries replyCheck: which check fired, what it found, what the person got and both drafts. A case whose turns wait (`waitSeconds` on a user turn, up to 300 each) takes that long, and says so in `waited`. Each case names its conversationId and its last turn's traceId (trace_get reads it). A case's judged expectations are graded by the runtime's expectation judge on its grading model; judgedNotGraded says why when they couldn't be. To tell a flaky case from a regression, run once with repeat (e.g. 5): each case says how many runs passed, and one that passed some is flaky. skipped says what the scope left out and why; modelSubstitutions names Agents running on another model here.",
       inputSchema: {
         type: 'object',
         properties: {
@@ -175,6 +183,11 @@ export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
           regression: { type: 'boolean', description: 'Only the regression datasets the workspace tests before publishing.' },
           gate: { type: 'boolean', description: 'Only the cases marked gate: true: what a change request runs before it applies.' },
           concurrency: { type: 'integer', minimum: 1, maximum: 16, description: 'How many cases run at once (default 2). Cases that change records for real still run one at a time. Lower it to 1 when the model provider refuses for requests in flight.' },
+          transcripts: { type: 'boolean', description: 'Include each case\'s whole conversation: every turn\'s message, reply and tool calls, the procedure it ran under and what the reply check did (default false). Use it instead of reading the runtime\'s database.' },
+          repeat: { type: 'integer', minimum: 1, maximum: 10, description: 'Run each conversation case this many times (default 1) and report its pass rate. A case that passes some runs and fails others is flaky: it passes at passRate or above. Use it to tell a flaky case from a regression in one run, instead of running again.' },
+          passRate: { type: 'number', exclusiveMinimum: 0, maximum: 1, description: 'With repeat: the share of runs a flaky case must pass (default 0.8).' },
+          model: { type: 'string', description: 'Run the cases with each Agent on this catalog model instead of its own: its draft is put on it for the run and back after; the files never change. Compare models with baseline.' },
+          baseline: { type: 'string', description: 'Compare with a kept run: last, last-own (the newest on the Agents\' own models), last-<model>, or a run id. Every run is kept; the answer names its run.' },
         },
         additionalProperties: false,
       },
@@ -197,14 +210,47 @@ export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
         const last = readLastRun(project)
         // A draft that isn't what the Agent's files say fails its cases with what to run, as `local test` does.
         const concurrency = typeof input.concurrency === 'number' && Number.isInteger(input.concurrency) && input.concurrency >= 1 && input.concurrency <= 16 ? input.concurrency : undefined
-        const results = await runCases({ serviceUrl: link.serviceUrl, creds: link.creds, files, live: input.live === true, concurrency, staleDraft: (slug) => agentFilesProblem(project, link.creds.organizationId, slug), onResult: () => {} })
+        const repeat = typeof input.repeat === 'number' && Number.isInteger(input.repeat) && input.repeat >= 1 && input.repeat <= 10 ? input.repeat : undefined
+        const passRate = typeof input.passRate === 'number' && input.passRate > 0 && input.passRate <= 1 ? input.passRate : undefined
+        // L1: another model for the run, and a kept run to compare with.
+        const model = stringArg(input, 'model')?.trim() || null
+        if (model && input.live === true) return errorResult('model runs the drafts on another model; live tests the published versions as they are. Leave one out.')
+        const baselineRef = stringArg(input, 'baseline')?.trim()
+        const baseline = baselineRef ? readTestRun(project, baselineRef) : null
+        if (baselineRef && !baseline) return errorResult(`No kept test run ${baselineRef}: give last, last-own, last-<model> or one of ${listTestRuns(project).slice(-5).join(', ') || '(none yet: run once first)'}.`)
+        const runAll = () => runCases({ serviceUrl: link.serviceUrl, creds: link.creds, files, live: input.live === true, concurrency, ...(repeat ? { repeat } : {}), ...(passRate ? { passRate } : {}), staleDraft: (slug) => agentFilesProblem(project, link.creds.organizationId, slug), onResult: () => {} })
+        let onModel: { swapped: string[]; unchanged: string[]; stuck: string[] } | null = null
+        let results: Awaited<ReturnType<typeof runAll>>
+        if (model) {
+          try {
+            const ran = await runOnModel(project, link, files.map((f) => f.agent), model, () => runAll())
+            onModel = ran
+            results = ran.value
+          } catch (err) {
+            return errorResult((err as Error).message)
+          }
+        } else results = await runAll()
+        const kept = saveTestRun(project, results, model)
+        const against = baseline ? compareWithRun(results, baseline) : null
+        // What the run left out and why, and Agents on another model here than their files name (X1).
+        const skipped = skippedCases(project.root, { ...(stringArg(input, 'agent') ? { agents: [stringArg(input, 'agent')!] } : {}), ...(file ? { file } : {}), regression: input.regression === true, gate: input.gate === true }).filter((x) => !/other Agents/.test(x.reason))
+        const models = modelSubstitutions(project, link.creds.organizationId, results.map((r) => r.agent))
+        const flaky = results.filter((r) => r.repeat?.flaky)
         const comparison = compareWithLastRun(results, last)
-        recordRun(project, results)
+        // Another model's run is not the Agents' last run.
+        if (!model) recordRun(project, results)
         const failed = results.filter((r) => !r.passed && !r.notRun)
         const passed = results.filter((r) => r.passed).length
         const notRun = notRunGroups(results)
         return jsonResult({
-          summary: `${passed} passed, ${failed.length} failed${comparison.newlyFailing.length ? `, ${comparison.newlyFailing.length} newly failing` : ''}${notRun.length ? `, ${notRunWords(notRun)}` : ''}`,
+          summary: `${passed} passed${flaky.length ? ` (${flaky.filter((r) => r.passed).length} flaky)` : ''}, ${failed.length} failed${comparison.newlyFailing.length ? `, ${comparison.newlyFailing.length} newly failing` : ''}${notRun.length ? `, ${notRunWords(notRun)}` : ''}${repeat && repeat > 1 ? `; each case run ${repeat} times` : ''}${skippedWords(skipped) ? `; ${skippedWords(skipped)}` : ''}`,
+          run: kept.id,
+          ...(model && onModel ? { model, onModel: onModel.swapped, ownModel: onModel.unchanged, ...(onModel.stuck.length ? { stillOnModel: stuckWords(onModel.stuck, model) } : {}) } : {}),
+          ...(against && baseline ? { baseline: { run: baseline.id, model: baseline.model, said: comparisonWords(against, baseline), newlyFailing: against.newlyFailing.map(label), fixed: against.fixed.map(label), stillFailing: against.stillFailing.map(label) } } : {}),
+          ...(flaky.length ? { flaky: flaky.map(label) } : {}),
+          ...(skipped.length ? { skipped } : {}),
+          // The Agent ran on another model here: a pass or failure says less about the workspace's model.
+          ...(models.length ? { modelSubstitutions: models } : {}),
           newlyFailing: comparison.newlyFailing.map(label),
           fixed: comparison.fixed.map(label),
           stillFailing: comparison.stillFailing.map(label),
@@ -213,7 +259,12 @@ export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
           ...(notRun.length ? { couldNotRun: notRun.map((g) => ({ what: `couldn’t run: ${g.what}`, fix: g.fix, cases: g.cases, ...(g.skipped ? { notSent: g.skipped } : {}) })) } : {}),
           cases: await (async () => {
             const traces = await caseTraces(project, results)
-            return Promise.all(results.map(async (r, i) => caseLine(link, r, files, r.passed || r.notRun ? null : await caseReplyChecks(link, r.agentId, r.conversationId).catch(() => null), traces.get(i))))
+            return Promise.all(results.map(async (r, i) => {
+              const line = caseLine(link, r, files, r.passed || r.notRun ? null : await caseReplyChecks(link, r.agentId, r.conversationId).catch(() => null), traces.get(i))
+              if (input.transcripts !== true || !r.conversationId) return line
+              const transcript = await conversationTranscript(link, r.conversationId).catch(() => null)
+              return transcript ? { ...line, transcript: transcript.turns } : line
+            }))
           })(),
         })
       },
@@ -235,13 +286,13 @@ export function testTools(project: LocalProject, memory: McpMemory): McpTool[] {
             items: {
               type: 'object',
               properties: {
-                type: { type: 'string', enum: ['contains', 'regex', 'tool_called', 'tool_not_called'] },
-                value: { type: 'string', description: 'contains, regex: the text or pattern.' },
+                type: { type: 'string', enum: ['contains', 'not_contains', 'regex', 'tool_called', 'tool_not_called'] },
+                value: { type: 'string', description: 'contains, not_contains, regex: the text or pattern.' },
                 flags: { type: 'string', description: 'regex: its flags (default i).' },
                 tool: { type: 'string', description: 'tool_called, tool_not_called: the tool\'s name as the trace shows it (a Plugin\'s tool by its capability name, mcp__…, never its bare name).' },
-                argsSubset: { type: 'object', description: 'tool_called: arguments the call must carry.' },
+                argsSubset: { type: 'object', description: 'tool_called: arguments the call must carry; { absent: true } as a value says that argument must not be there.' },
                 mode: { type: 'string', enum: ['exact', 'ignore', 'subset', 'superset'], description: 'How argsSubset is compared (default superset).' },
-                turn: { type: 'string', enum: ['all', 'last'], description: 'tool_called, tool_not_called: the calls of every turn (all, the default) or only of the last one.' },
+                turn: { anyOf: [{ type: 'integer', minimum: 1 }, { type: 'string', enum: ['all', 'any', 'last'] }], description: 'Which turn the check reads: a reply check the last reply (default), turn n, any or all; a tool check every turn\'s calls (all, the default), the last turn\'s or turn n\'s.' },
                 text: { type: 'string', description: 'What the check means, in words, for its failure message.' },
               },
               required: ['type'],

package/src/output.ts

@@ -76,18 +76,29 @@ const defaultStreams: Streams = {
  */
 export function createOutput(
   mode: OutputMode,
-  opts: { quiet?: boolean; streams?: Streams } = {},
+  opts: { quiet?: boolean; streams?: Streams; envelope?: boolean } = {},
 ): Output {
   const streams = opts.streams ?? defaultStreams
   const quiet = opts.quiet ?? false
+  // `--envelope` (Forge improvement plan X10): one shape for every command,
+  // `{ ok, data, warnings }`, and a failure on stdout as `{ ok: false, error,
+  // warnings }`, so a caller reads one stream and one document whatever
+  // happened. Plain `--json` keeps the contract above.
+  const envelope = mode === 'json' && opts.envelope === true
+  const warnings: string[] = []
 
   return {
     render(data, toText) {
-      const text = mode === 'json' ? JSON.stringify(data, null, 2) : toText(data)
+      const text = envelope ? JSON.stringify({ ok: true, data, warnings }, null, 2) : mode === 'json' ? JSON.stringify(data, null, 2) : toText(data)
       streams.out(`${text}\n`)
     },
 
     note(message) {
+      // A warning the command said is the envelope's too, quiet or not.
+      if (envelope) for (const line of message.split('\n')) {
+        const m = /^\s*warning:\s*(.+)$/i.exec(line)
+        if (m) warnings.push(m[1]!)
+      }
       if (quiet) return
       streams.err(`${message}\n`)
     },
@@ -139,14 +150,18 @@ export function createOutput(
     },
 
     fail(err) {
-      const envelope = toEnvelope(err)
+      const failure = toEnvelope(err)
+      if (envelope) {
+        streams.out(`${JSON.stringify({ ok: false, error: failure, warnings }, null, 2)}\n`)
+        return exitCodeFor(failure.code, failure.details)
+      }
       if (mode === 'json') {
-        streams.err(`${JSON.stringify(envelope, null, 2)}\n`)
+        streams.err(`${JSON.stringify(failure, null, 2)}\n`)
       } else {
-        streams.err(`error [${envelope.code}]: ${envelope.message}\n`)
-        if (envelope.hint) streams.err(`  ${envelope.hint}\n`)
+        streams.err(`error [${failure.code}]: ${failure.message}\n`)
+        if (failure.hint) streams.err(`  ${failure.hint}\n`)
       }
-      return exitCodeFor(envelope.code, envelope.details)
+      return exitCodeFor(failure.code, failure.details)
     },
   }
 }