ai-agentmanager 1.12.0 → 1.14.0

package/public/style.css

@@ -968,8 +968,20 @@ body.remote .settings-links .local-only { display: none; }
 .built-body details > summary { cursor: pointer; padding: 4px 0; }
 .built-actions { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; }
 .built-exe { margin: 10px 0; padding: 8px 10px; border: 1px solid var(--border); border-radius: 8px; }
+.built-session, .built-panel { margin: 0 0 10px; padding: 8px 10px; border: 1px solid var(--border); border-radius: 8px; background: var(--panel, transparent); }
+.built-session .line1, .built-panel .line1 { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
+.built-panel { margin: 10px 0; }
+.built-panel p { margin: 6px 0 0; }
+.built-change { display: flex; gap: 6px; margin-top: 6px; align-items: flex-end; }
+.built-change textarea { flex: 1; min-width: 0; }
 .built-exe-row { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; margin-top: 6px; }
 .built-tests li { margin-bottom: 8px; } .built-tests li.pass b { color: var(--green); } .built-tests li.fail b { color: var(--red); }
 .built-tests pre, .built-runs div { white-space: pre-wrap; font-size: 12.5px; }
 .code-area { width: 100%; font: 12.5px/1.45 var(--mono); margin-top: 6px; }
 .built-runs { list-style: none; padding: 0; } .built-runs li { padding: 6px 0; border-bottom: 1px solid var(--border); }
+
+/* Approval rules (Settings) and agents waiting on the same command (inbox) */
+.rules-list { list-style: none; padding: 0; margin: 6px 0 0; display: grid; gap: 6px; }
+.rules-list li { display: flex; flex-wrap: wrap; align-items: center; gap: 6px; }
+.rules-list li button { margin-left: auto; }
+.need.group { border-style: dashed; }

package/src/agentkit/engines.js

@@ -58,6 +58,19 @@ export function approverServer({ url, token }) {
   return { command: process.execPath, args: isSea() ? ['approver'] : [AGENTKIT_BIN, 'approver'], env: { AGENTKIT_APPROVAL_URL: url, AGENTKIT_APPROVAL_TOKEN: token } };
 }
 
+/**
+ * Claude Code scopes file writes with Edit(<path>) rules (they cover Write too); a Write(<path>)
+ * rule is ignored. Each Write(<path>) also gets its Edit(<path>), so writing there is allowed.
+ */
+export function claudeRules(allow) {
+  const out = [...allow];
+  for (const p of allow) {
+    const m = /^Write\((.+)\)$/.exec(p);
+    if (m && !out.includes(`Edit(${m[1]})`)) out.push(`Edit(${m[1]})`);
+  }
+  return out;
+}
+
 /**
  * One turn. `session`: the engine session to continue (null starts one); returns
  * { text, session, costUsd }. `system` goes in as an extra system prompt where the engine has one,
@@ -77,7 +90,7 @@ export async function turn(agent, { prompt, system = '', session = null, mcpFile
     args.push('--setting-sources', 'project', '--strict-mcp-config');
     if (mcpFile) args.push('--mcp-config', mcpFile);
     // The approve list wins: an allow pattern that also covers an approval tool goes to the approver.
-    const allow = approver ? allowedOutright(agent) : agent.allow;
+    const allow = claudeRules(approver ? allowedOutright(agent) : agent.allow);
     if (allow.length) args.push('--allowedTools', ...allow);
     // With an approver, Claude asks it before anything not allowed outright (it says no to tools
     // off the approve list); without one, approval tools are simply not available.

package/src/builder.js

@@ -1,8 +1,10 @@
-// The agent builder: a task or workflow in words → an analysis and an agentkit agent
-// (docs/agentkit.md) with test cases, which the person reviews, tests, runs and downloads. Built
-// agents live in ~/.agentmanager/built/<name>/ as ordinary agentkit folders.
+// The agent builder: a task or workflow in words → an agentkit agent (docs/agentkit.md), developed
+// in an agent session that works in the agent's folder: it writes the analysis, spec, instructions
+// and tests, and makes the changes the person asks for. The person tests, runs, packages and
+// downloads it here. Built agents live in ~/.agentmanager/built/<name>/ as ordinary agentkit folders.
 import { spawn } from 'node:child_process';
 import fs from 'node:fs';
+import os from 'node:os';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import YAML from 'yaml';
@@ -19,79 +21,139 @@ const err = (status, message) => Object.assign(new Error(message), { status });
 const now = () => new Date().toISOString();
 const slug = (s) => String(s ?? '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40) || 'agent';
 
-export function designPrompt({ task, engine, attachments = [] }) {
+const DOCS = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', 'docs', 'agentkit.md');
+const ENGINES = ['claude', 'codex', 'muse'];
+
+/** The agent's name from the task: its first few words. */
+export function nameFrom(task) {
+  const words = String(task).toLowerCase().replace(/https?:\/\/\S+/g, ' ').split(/[^a-z0-9]+/).filter((w) => w && !STOP.has(w));
+  const n = slug(words.slice(0, 4).join('-')).slice(0, 32).replace(/-+$/, '');
+  return /^[a-z][a-z0-9-]{1,}$/.test(n) ? n : `agent-${n}`.replace(/-+$/, '').slice(0, 32);
+}
+const STOP = new Set(['a', 'an', 'the', 'to', 'of', 'for', 'and', 'or', 'in', 'on', 'by', 'with', 'from', 'that', 'this', 'it', 'is', 'are', 'be', 'i', 'we', 'my', 'our', 'want', 'need', 'agent', 'build', 'make', 'create', 'please', 'which', 'who']);
+
+/** The commands its session uses to check and try the agent. */
+export function agentkitCommands(dir) {
+  return { check: `${process.execPath} ${AGENTKIT} check`, run: `${process.execPath} ${AGENTKIT} run ${dir} --once` };
+}
+
+/** What its session may do without a permission prompt: check the agent, read the reference. */
+export function builderAllowedTools(dir) {
+  return [`Bash(${agentkitCommands(dir).check}:*)`, `Read(/${DOCS})`];
+}
+
+/**
+ * What a session that builds an agent is told (appended to its system prompt on every launch):
+ * the folder is the agent, the files it keeps, the format, and how to check and try it.
+ */
+export function builderPrompt({ dir, name, engine = 'claude' }) {
+  const cmd = agentkitCommands(dir);
   return [
-    'You design an autonomous agent that will run continuously on its own and do one job well. It runs through',
-    `${engine === 'claude' ? 'Claude Code' : engine} as its engine, uses tools through MCP servers, and is reached by people (chat) and other agents (A2A, MCP).`,
-    'From the task below, write the analysis and the agent. Be concrete; do not invent systems the task does not mention.',
+    `You are building an autonomous agent together with the person: an agentkit agent that runs on its own, continuously, and does one job well. This folder (${dir}) is the agent. Its name is "${name}"; keep it (it is the folder's name).`,
     '',
-    'Reply with one JSON object only:',
-    '{',
-    '  "analysis": {"problem": "<what the job is and why>", "inputs": ["<what it receives>"], "outputs": ["<what it produces>"], "risks": ["<what could go wrong, and what needs a person>"]},',
-    '  "name": "<lowercase-with-dashes, 2-40 chars>", "title": "<short>", "description": "<one line>",',
-    '  "goal": "<what is true when it has done its job, for each input; one or two sentences>",',
-    '  "instructions": "<Markdown: how to do the job, step by step; what to never do; when to ask>",',
-    '  "tools": [{"name": "<short id>", "why": "<what it needs it for>", "command": "<MCP server command, e.g. npx>", "args": ["…"], "env": {"VAR": "${VAR}"}}],',
-    '  "allow": ["<engine tool names it may use without asking, e.g. Read, Grep, mcp__<tool>__*>"],',
-    '  "approve": ["<tools that change things in the outside world and need a person\'s yes>"],',
-    '  "triggers": [{"every": "30m"} | {"cron": "0 9 * * 1-5"} | {"webhook": "/hooks/<name>"} | {"watch": "./inbox"}],',
-    '  "tests": [{"input": "<a realistic input>", "expect": "<what a good result does or contains>"}]',
-    '}',
+    'Keep these files current:',
+    '- agent.yaml: the spec (below).',
+    '- instructions.md: how to do the job, step by step; what it must never do; when it asks a person.',
+    '- tests.yaml: 3 to 8 test cases, a YAML list of {input, expect}: the normal case, an edge case, and a case it should refuse or escalate. The person runs them from the Built tab; each result is judged against `expect`.',
+    '- analysis.json: {"analysis": {"problem": "<the job and why>", "inputs": [...], "outputs": [...], "risks": ["<what could go wrong, what needs a person>"]}, "tools": [{"name": "<MCP server id>", "why": "<what for>"}]}. The person reads it in the Built tab; update it when the design changes.',
     '',
-    'Rules: 3 to 5 tests that cover the normal case, an edge case and a case it should refuse or escalate.',
-    'Only MCP servers that really exist and fit (leave "tools" empty when none is needed). Secrets only as "${VAR}".',
-    'Triggers only when the task implies running on its own (a schedule, events); chat-only agents have none.',
+    `agent.yaml has: name, title, description (one line), goal (what is true when it has done its job, for each input), instructions: instructions.md, engine: ${engine},`,
+    '  tools (MCP servers by id: {command, args, env}; only servers that really exist; secrets only as "${VAR}"),',
+    '  allow (engine tools it may use without asking, e.g. Read, Grep, "mcp__<id>__*"; to let it write files only under a folder: "Edit(./out/**)", which covers Write), approve (tools that change things in the outside world and need a person\'s yes),',
+    '  triggers (only when the task implies running on its own: {every: 30m}, {cron: "0 9 * * 1-5"}, {webhook: /hooks/<name>}, {watch: ./inbox}; a chat-only agent has none),',
+    '  loop: {plan: true, check: true, maxSteps: 4}, memory: true, interfaces: {http: {port: 0}, a2a: true, acp: true, mcp: true}.',
+    `The full reference: ${DOCS}.`,
     '',
-    '<task>',
-    task,
-    '</task>',
-    ...(attachments.length ? ['<attached_files>', ...attachments.map((a) => `--- ${a.name}\n${a.text}`), '</attached_files>'] : []),
+    'Write and edit these files with your file tools (Write, Edit), not with shell commands.',
+    `After every change, check it loads: ${cmd.check} ${dir}`,
+    `To try one input for real (it runs with its allowed tools; tools that need approval are refused): ${cmd.run} "<input>". Only with inputs that are safe to run for real.`,
+    'Be concrete; do not invent systems the task does not mention. When something only the person knows is missing (an account, a team name, a label set), ask.',
+    'The person tests, starts, packages and downloads it from the Built tab; you do not start it continuously or package it. When they ask for a change, make it in these files, check, and say in a sentence what changed.',
   ].join('\n');
 }
 
-export function judgePrompt({ goal, input, expect, result }) {
+/** The first message of a build session. */
+export function buildIntent({ task, name, engine }) {
+  return `Build the agent "${name}" (engine: ${engine}) for this task:\n\n${task}\n\nWrite analysis.json, agent.yaml, instructions.md and tests.yaml in this folder, check it, then tell me what it does and what I should decide or provide.`;
+}
+
+export function judgePrompt({ goal, input, expect, result, files = [] }) {
   return [
     "You check one test of an autonomous agent. Decide if the agent's result meets the expectation for the input.",
+    'The files it created or changed during the run are shown too: judge what it did from them, not only from what it says.',
     'Be strict about substance, lenient about wording. Reply with JSON only: {"pass": true|false, "why": "<one sentence>"}.',
     `<goal>\n${goal}\n</goal>`,
     `<input>\n${input}\n</input>`,
     `<expectation>\n${expect}\n</expectation>`,
     `<result>\n${result}\n</result>`,
+    ...(files.length ? ['<files_written>', ...files.map((f) => `--- ${f.path}${f.truncated ? ' (first part)' : ''}\n${f.text}`), '</files_written>'] : ['<files_written>\n(none)\n</files_written>']),
   ].join('\n');
 }
 
-/** agent.yaml text from a design. */
-export function yamlOf(d, engine) {
-  const tools = Object.fromEntries((d.tools ?? []).filter((t) => t?.name && t.command).map((t) => [slug(t.name), { command: t.command, ...(t.args?.length ? { args: t.args } : {}), ...(t.env && Object.keys(t.env).length ? { env: t.env } : {}) }]));
-  const doc = {
-    name: slug(d.name),
-    title: d.title || d.name,
-    description: d.description ?? '',
-    goal: d.goal,
-    instructions: 'instructions.md',
-    engine,
-    tools,
-    allow: d.allow ?? [],
-    approve: d.approve ?? [],
-    triggers: d.triggers ?? [],
-    loop: { plan: true, check: true, maxSteps: 4 },
-    memory: true,
-    interfaces: { http: { port: 0 }, a2a: true, acp: true, mcp: true },
+const SKIP = new Set(['.agentkit', '.git', 'node_modules', 'dist']);
+/** Files under `dir` (not its own state, git or build output) with when each last changed. */
+export function snapshot(dir, { max = 5000 } = {}) {
+  const seen = new Map();
+  const walk = (d) => {
+    let entries = [];
+    try {
+      entries = fs.readdirSync(d, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (seen.size >= max) return;
+      const p = path.join(d, e.name);
+      if (e.isDirectory()) {
+        if (!SKIP.has(e.name)) walk(p);
+      } else if (e.isFile()) {
+        const st = fs.statSync(p, { throwIfNoEntry: false });
+        if (st) seen.set(p, `${st.mtimeMs}:${st.size}`);
+      }
+    }
   };
-  return YAML.stringify(doc, { lineWidth: 0 });
+  walk(dir);
+  return seen;
+}
+
+/** The text files created or changed since `before` (a snapshot), for the judge; a few, each capped. */
+export function filesWritten(dir, before, { root = dir, limit = 6, bytes = 4000 } = {}) {
+  const out = [];
+  for (const [p, sig] of snapshot(dir)) {
+    if (before.get(p) === sig || out.length >= limit) continue;
+    const buf = fs.readFileSync(p);
+    if (buf.subarray(0, 2000).includes(0)) continue; // binary
+    out.push({ path: path.relative(root, p), text: buf.subarray(0, bytes).toString('utf8'), truncated: buf.length > bytes });
+  }
+  return out;
+}
+
+/**
+ * A throwaway copy of the agent's folder for one test run, so what a run writes (outputs, memory,
+ * changed fixtures) doesn't change the agent or the next test. It keeps the agent's memory, not its
+ * engine sessions, run logs, executables, dependencies or git data.
+ */
+export function testCopy(dir) {
+  const copy = fs.mkdtempSync(path.join(os.tmpdir(), `agentkit-test-${path.basename(dir)}-`));
+  const state = path.join(dir, '.agentkit');
+  const keep = (src) => src === dir || (path.dirname(src) === state ? path.basename(src) === 'memory.md' : src === state || !SKIP.has(path.basename(src)));
+  fs.cpSync(dir, copy, { recursive: true, filter: keep });
+  return copy;
 }
 
 export class Builder {
-  /** `oneShot(prompt)` answers a design or judge prompt (the daemon's writer engine). */
+  /** `oneShot(prompt)` answers a judge prompt (the daemon's writer engine). */
+  /** `sessions`: the daemon's agent sessions ({ create, get, tell, stop }); each built agent is developed in one. */
   /** `reportEnv()`: settings that make an agent started here report to this daemon. */
-  constructor({ oneShot, env = process.env, runTurn = engineTurn, pack = packageAgent, reportEnv = () => ({}) }) {
+  constructor({ oneShot, sessions, env = process.env, runTurn = engineTurn, pack = packageAgent, reportEnv = () => ({}) }) {
     this.reportEnv = reportEnv;
     this.oneShot = oneShot;
+    this.sessions = sessions;
     this.env = env;
     this.runTurn = runTurn;
     this.pack = pack;
     this.procs = new Map(); // name -> child process of `agentkit run`
-    this.pending = new Map(); // name -> what's in progress (designing, testing)
+    this.pending = new Map(); // name -> what's in progress (testing, packaging)
   }
 
   dir(name) {
@@ -128,59 +190,108 @@ export class Builder {
     const read = (f) => (fs.existsSync(path.join(dir, f)) ? fs.readFileSync(path.join(dir, f), 'utf8') : '');
     let spec = null;
     let specError = null;
-    try {
-      spec = loadAgent(dir, { env: this.env });
-    } catch (e) {
-      specError = e.message;
+    const yaml = read('agent.yaml');
+    if (yaml) {
+      try {
+        spec = loadAgent(dir, { env: this.env });
+      } catch (e) {
+        specError = e.message;
+      }
     }
     const p = this.procs.get(name);
     let runs = [];
     if (spec) runs = new Agent(spec, { env: this.env }).runs(20);
+    // Its session keeps analysis.json and tests.yaml current; read them as they are now.
+    let design = {};
+    try {
+      design = JSON.parse(read('analysis.json') || '{}');
+    } catch {}
+    let tests = m.tests ?? [];
+    try {
+      tests = (YAML.parse(read('tests.yaml') || '[]') ?? []).filter((t) => t?.input).map((t) => ({ input: String(t.input), expect: String(t.expect ?? '') }));
+    } catch {}
+    // Until its spec loads, the status follows the files: designing, then designed or needs fixing.
+    let status = m.status;
+    if (!yaml) status = 'designing';
+    else if (specError) status = 'needs-fixing';
+    else if (['designing', 'needs-fixing'].includes(status)) status = 'designed';
+    const s = m.session ? this.sessions?.get(m.session) : null;
+    // When its files last changed (its session edits them): results and executables from before are out of date.
+    const changedAt = new Date(Math.max(0, ...['agent.yaml', 'instructions.md', 'tests.yaml'].map((f) => fs.statSync(path.join(dir, f), { throwIfNoEntry: false })?.mtimeMs ?? 0))).toISOString();
     return {
       ...m,
       name,
+      status,
       title: spec?.title ?? name,
       description: spec?.description ?? '',
       dir,
-      yaml: read('agent.yaml'),
+      yaml,
       instructionsText: read('instructions.md'),
+      analysis: design.analysis ?? m.analysis ?? null,
+      tools: design.tools ?? m.tools ?? [],
+      tests,
       specError,
-      engine: spec?.engine ?? null,
+      engine: spec?.engine ?? m.engine ?? null,
       triggers: spec?.triggers.map((t) => t.label) ?? [],
       missingEnv: spec?.missingEnv ?? [],
       pending: this.pending.get(name) ?? null,
       running: p ? { pid: p.child.pid, url: p.url, since: p.since } : null,
+      session: s ? { id: s.id, name: s.name, status: s.status, running: s.runtime.running, busy: s.runtime.busy, attention: s.runtime.attention ?? s.runtime.waiting?.text ?? null } : null,
       runs,
       platforms: PLATFORMS.filter((x) => process.platform === 'darwin' || !x.startsWith('macos-')),
       host: HOST,
+      changedAt,
       // Executables built since the last edit that are still there.
       executables: Object.fromEntries(Object.entries(m.executables ?? {}).filter(([, x]) => fs.existsSync(path.join(dir, 'dist', x.file)))),
     };
   }
 
-  /** Design an agent from a task (one model call); writes the folder and returns it. */
-  async design({ task, engine = 'claude', attachments = [] }) {
+  /**
+   * Start building an agent for a task: its folder, and an agent session working in it that
+   * analyses the task and writes the agent (analysis, spec, instructions, tests). The person
+   * keeps developing it in that session.
+   */
+  design({ task, engine = 'claude', name, attachments, permissionMode, model }) {
     task = String(task ?? '').trim();
     if (!task) throw err(400, 'Describe the task or workflow');
-    if (!['claude', 'codex', 'muse'].includes(engine) && !/^acp:[a-z0-9-]+$/.test(engine)) throw err(400, 'engine: claude, codex, muse or acp:<name>');
-    const d = parseJsonReply(await this.oneShot(designPrompt({ task, engine, attachments })));
-    if (!d.goal || !d.name) throw err(502, 'The design was missing a name or goal; try again');
-    let name = slug(d.name);
-    for (let i = 2; fs.existsSync(path.join(root(), name)); i++) name = `${slug(d.name).slice(0, 36)}-${i}`;
-    const dir = path.join(root(), name);
+    if (!ENGINES.includes(engine) && !/^acp:[a-z0-9-]+$/.test(engine)) throw err(400, 'engine: claude, codex, muse or acp:<name>');
+    if (!this.sessions) throw err(503, 'Agent sessions are not available');
+    const want = name ? slug(name) : nameFrom(task);
+    if (!/^[a-z][a-z0-9-]{1,40}$/.test(want)) throw err(400, 'name: lowercase letters, digits and dashes, starting with a letter');
+    let n = want;
+    for (let i = 2; fs.existsSync(path.join(root(), n)); i++) n = `${want.slice(0, 36)}-${i}`;
+    const dir = path.join(root(), n);
     fs.mkdirSync(dir, { recursive: true });
-    fs.writeFileSync(path.join(dir, 'agent.yaml'), yamlOf({ ...d, name }, engine));
-    fs.writeFileSync(path.join(dir, 'instructions.md'), String(d.instructions ?? '').trim() + '\n');
-    const tests = (d.tests ?? []).filter((t) => t?.input).slice(0, 8).map((t) => ({ input: String(t.input), expect: String(t.expect ?? '') }));
-    fs.writeFileSync(path.join(dir, 'tests.yaml'), YAML.stringify(tests));
-    // Check it loads; a broken design is kept so the person can fix it.
-    let specError = null;
+    this.saveMeta(n, { task, engine, results: null, status: 'designing', createdAt: now() });
     try {
-      checkAgent(YAML.parse(fs.readFileSync(path.join(dir, 'agent.yaml'), 'utf8')), { dir, env: this.env });
+      const s = this.sessions.create({ name: n, dir, intent: buildIntent({ task, name: n, engine }), builds: { name: n, engine }, attachments, permissionMode, model });
+      this.saveMeta(n, { session: s.id });
     } catch (e) {
-      specError = e.message;
+      fs.rmSync(dir, { recursive: true, force: true });
+      throw e;
+    }
+    return this.view(n);
+  }
+
+  /**
+   * Ask its session for a change (typed in when it's idle; started if it's closed). An agent with
+   * no session yet (built before sessions, or its session deleted) gets a new one.
+   */
+  change(name, text) {
+    const dir = this.dir(name);
+    const m = this.meta(name);
+    if (!m) throw err(404, `No built agent "${name}"`);
+    text = String(text ?? '').trim();
+    const s = m.session ? this.sessions.get(m.session) : null;
+    if (s) {
+      if (!text) throw err(400, 'Say what to change');
+      this.sessions.tell(s.id, text);
+    } else {
+      const engine = this.view(name).engine ?? 'claude';
+      const intent = `This folder is the agent "${name}"${m.task ? `, built for this task:\n\n${m.task}\n\n` : '. '}Read agent.yaml, instructions.md and tests.yaml${fs.existsSync(path.join(dir, 'analysis.json')) ? ' and analysis.json' : ' and write analysis.json'}. ${text || 'Then tell me briefly what it does and wait for what to change.'}`;
+      const created = this.sessions.create({ name, dir, intent, builds: { name, engine } });
+      this.saveMeta(name, { session: created.id });
     }
-    this.saveMeta(name, { task, analysis: d.analysis ?? null, tools: (d.tools ?? []).map((t) => ({ name: t.name, why: t.why ?? '' })), tests, results: null, status: specError ? 'needs-fixing' : 'designed', createdAt: now() });
     return this.view(name);
   }
 
@@ -227,21 +338,32 @@ export class Builder {
     this.pending.set(name, 'testing');
     this.saveMeta(name, { status: 'testing' });
     try {
-      // Tests don't wait for people: approval tools are simply not available in them.
-      const agent = new Agent({ ...spec, triggers: [], report: null }, { env: this.env, engineTurn: this.runTurn, approvals: 'deny' });
+      // Each test runs in its own copy of the agent's folder (a workdir outside it is used in place),
+      // and the judge sees what the run wrote there, not only what it says it did. Tests don't wait
+      // for people: approval tools are simply not available in them.
       const results = [];
       for (const t of tests) {
-        const r = await agent.run(t.input, { source: 'test' });
-        let verdict = { pass: false, why: 'The run failed' };
-        if (r.status !== 'failed') {
-          try {
-            const v = parseJsonReply(await this.oneShot(judgePrompt({ goal: spec.goal, input: t.input, expect: t.expect, result: r.result })));
-            verdict = { pass: v.pass === true, why: String(v.why ?? '') };
-          } catch (e) {
-            verdict = { pass: false, why: `Could not judge: ${e.message}` };
+        const copy = testCopy(dir);
+        try {
+          const run = loadAgent(copy, { env: this.env });
+          const agent = new Agent({ ...run, triggers: [], report: null }, { env: this.env, engineTurn: this.runTurn, approvals: 'deny' });
+          const roots = [...new Set([copy, run.workdir])].filter((d, i, all) => !all.some((o, j) => j !== i && d.startsWith(o + path.sep)));
+          const before = roots.map((d) => snapshot(d));
+          const r = await agent.run(t.input, { source: 'test' });
+          const files = roots.flatMap((d, i) => filesWritten(d, before[i], { root: copy })).slice(0, 6);
+          let verdict = { pass: false, why: 'The run failed' };
+          if (r.status !== 'failed') {
+            try {
+              const v = parseJsonReply(await this.oneShot(judgePrompt({ goal: spec.goal, input: t.input, expect: t.expect, result: r.result, files })));
+              verdict = { pass: v.pass === true, why: String(v.why ?? '') };
+            } catch (e) {
+              verdict = { pass: false, why: `Could not judge: ${e.message}` };
+            }
           }
+          results.push({ input: t.input, expect: t.expect, result: r.result, status: r.status, costUsd: r.costUsd, files: files.map((f) => f.path), ...verdict });
+        } finally {
+          fs.rmSync(copy, { recursive: true, force: true });
         }
-        results.push({ input: t.input, expect: t.expect, result: r.result, status: r.status, costUsd: r.costUsd, ...verdict });
       }
       const passed = results.filter((r) => r.pass).length;
       this.saveMeta(name, { results: { at: now(), passed, total: results.length, list: results }, status: passed === results.length ? 'tested' : 'failing' });
@@ -326,6 +448,9 @@ export class Builder {
 
   remove(name) {
     this.stop(name);
+    // Its session works in the folder about to go; close it (the conversation is kept).
+    const sid = this.meta(name)?.session;
+    if (sid && this.sessions?.get(sid)) this.sessions.stop(sid);
     fs.rmSync(this.dir(name), { recursive: true, force: true });
     return { ok: true };
   }

package/src/cli.js

@@ -879,6 +879,8 @@ export async function main(argv) {
       await attach(ref, { kind: `shell:${n}` });
     });
 
+  const repoLabel = (k) => String(k).replace(/^dir:/, '').split('/').filter(Boolean).slice(-2).join('/');
+
   program
     .command('inbox')
     .description('Everything waiting on you: prompts, questions, stuck agents, overlapping changes')
@@ -888,6 +890,11 @@ export async function main(argv) {
       if (!items.length) return console.log(dim('Nothing needs you right now.'));
       const label = { prompt: red('needs input'), waiting: red('waiting'), question: cyan('question'), review: cyan('AI review'), stuck: yellow('stuck?'), conflict: yellow('overlap') };
       for (const it of items) {
+        if (it.kind === 'rule') {
+          console.log(`${cyan('always allow?')}  ${bold(repoLabel(it.repo))}  ${it.title} ${dim(`(allowed ${it.n} times)`)}`);
+          console.log(dim('    agentmgr rules'));
+          continue;
+        }
         console.log(`${label[it.kind]}  ${bold(it.agentName)} ${dim(it.agentId)}  ${it.title}`);
         if (it.kind === 'prompt') console.log(dim(`    ${(it.prompt?.choices ?? []).map((c) => `${c.key}. ${c.label}`).join('   ')}  → agentmgr attach ${it.agentId}`));
         if (it.kind === 'question') {
@@ -901,6 +908,32 @@ export async function main(argv) {
       }
     });
 
+  program
+    .command('rules')
+    .description('Approval rules: commands always allowed per repo, and the ones the inbox suggests')
+    .argument('[action]', 'allow <n> | no <n> (suggestions), rm <n> (allowed)')
+    .argument('[n]')
+    .action(async (action, n) => {
+      await daemon();
+      let r = await api('GET', '/rules');
+      const pick = (list) => {
+        const x = list[Number(n) - 1];
+        if (!x) throw new Error(`No rule ${n ?? ''}; see agentmgr rules`);
+        return { repo: x.repo, rule: x.rule };
+      };
+      if (action === 'allow') r = await api('POST', '/rules/accept', pick(r.suggestions));
+      else if (action === 'no') r = await api('POST', '/rules/dismiss', pick(r.suggestions));
+      else if (action === 'rm') r = await api('POST', '/rules/remove', pick(r.allowed));
+      else if (action) throw new Error('Use: agentmgr rules [allow <n> | no <n> | rm <n>]');
+      if (r.suggestions.length) {
+        console.log(bold('Suggested') + dim(`  (you allowed these ${r.suggestAfter}+ times; agentmgr rules allow <n> | no <n>)`));
+        r.suggestions.forEach((x, i) => console.log(`  ${i + 1}. ${x.label}  ${dim(`in ${repoLabel(x.repo)}, allowed ${x.n} times`)}`));
+      }
+      console.log(bold('Allowed') + dim('  (agents in the repo run these without asking; agentmgr rules rm <n>)'));
+      if (!r.allowed.length) console.log(dim('  none yet'));
+      r.allowed.forEach((x, i) => console.log(`  ${i + 1}. ${x.label}  ${dim(`in ${repoLabel(x.repo)}${x.uses ? `, answered ${x.uses} prompts` : ''}`)}`));
+    });
+
   program
     .command('answer')
     .description('Answer a question an agent asked in the inbox')