@arjunkhera/atlas 0.3.24 → 0.3.26

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "atlas",
-  "version": "0.3.24",
+  "version": "0.3.26",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, for any repository.",
   "author": {
     "name": "Arjun Khera"

package/door/cli.mjs

@@ -25,6 +25,7 @@ import { install, upgrade, doctor } from './lib/install.mjs';
 import { checkPaths, LIMITS } from './lib/ste.mjs';
 import { checkDesignFolder, designFolderOf, repoRootFrom } from './lib/design.mjs';
 import { writeDesignPage, readTracker } from './lib/design-build.mjs';
+import { writeViewPage, viewCapabilities } from './lib/view.mjs';
 import { loadPrivateTerms } from './lib/privacy.mjs';
 import { checkCommand as testsCheck, writeCommand as testsWrite } from './lib/tests.mjs';
 import { proofCommand as testsProof } from './lib/proof.mjs';
@@ -59,6 +60,10 @@ const HELP = `atlas — the door into a repo's Atlas files
       --out <file>                     the page; default docs/artifacts/<folder name>.html
       --draft                          build even when the check has findings, and mark the page a draft
 
+  atlas view build --product <id>      build the project view page of one product: the four views, read live from the tracker
+      --name <name>                    the product's name on the page; default the id with a capital
+      --out <file>                     the page; default docs/artifacts/<id>-view.html
+
   atlas tests check --root <area>      run the link check on the tests of an area
       --halves <list>                  static, lint, run; default all three
       --evidence <dir>                 where the evidence files are; default <area>/test/evidence
@@ -120,6 +125,7 @@ export const FLAGS = Object.freeze({
   check: { root: 'optional', repo: 'value', 'fail-on': 'value' },
   ste: { share: 'switch', terms: 'value' },
   design: { tracker: 'value', out: 'value', draft: 'switch', root: 'optional' },
+  view: { product: 'value', name: 'value', out: 'value' },
   tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch' },
   install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
   upgrade: { version: 'optional', yes: 'switch' },
@@ -296,6 +302,16 @@ function designCommand(chosen) {
   return count ? 1 : 0;
 }
 
+function viewCommand(chosen) {
+  if (chosen._[1] !== 'build' || chosen._.length > 2) throw new Error('use atlas view build --product <id>');
+  const product = chosen.product;
+  const out = resolve(chosen.out ?? join('docs', 'artifacts', `${product}-view.html`));
+  const page = writeViewPage(out, { packageRoot: PACKAGE_ROOT, product, name: chosen.name, version: ATLAS_VERSION });
+  line(`Wrote ${out}: the project view of ${page.name} (product ${page.product}). It holds no records; it reads them at open.`);
+  line(`Publish it with this capability, and no other: ${JSON.stringify(viewCapabilities())}`);
+  return 0;
+}
+
 async function testsCommand(chosen) {
   const root = rootOf(chosen);
   const what = chosen._[1];
@@ -343,6 +359,7 @@ export async function main(argv = process.argv.slice(2)) {
     case 'check': return checkCommand(chosen);
     case 'ste': return steCommand(chosen);
     case 'design': return designCommand(chosen);
+    case 'view': return viewCommand(chosen);
     case 'tests': return testsCommand(chosen);
     default: throw new Error(`there is no verb "${command}". Run "atlas help".`);
   }

package/door/lib/view.mjs

@@ -0,0 +1,46 @@
+// The project view page builder: `atlas view build`. It puts the view rules
+// (work/lib/view-rules.mjs) and one product id into the page template
+// (view/page.html), and writes one page. The page holds no record data: it
+// reads the tracker at open, through the viewer's Engram connector.
+// Design: docs/design-docs/project-view (tracker item 6dd1761a).
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+// The one connector grant the page needs, for the Artifact publish. Both
+// tools only read.
+export const VIEW_CONNECTOR = Object.freeze({ server: 'Engram', tools: Object.freeze(['entities_list', 'get']) });
+export const viewCapabilities = () => ({ mcp: { servers: [{ server: VIEW_CONNECTOR.server, tools: [...VIEW_CONNECTOR.tools] }] } });
+
+const PRODUCT_ID = /^[a-z0-9][a-z0-9-]{0,63}$/;
+const quoted = (value) => JSON.stringify(String(value)).slice(1, -1).replace(/</g, '\\u003c');
+const plain = (value) => String(value).replace(/[&<>"]/g, (ch) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' })[ch]);
+
+export function buildViewPage({ packageRoot, product, name, version }) {
+  if (!PRODUCT_ID.test(String(product ?? ''))) throw new Error('give a product id: atlas view build --product <id>, such as atlas');
+  const label = String(name ?? '').trim() || product.charAt(0).toUpperCase() + product.slice(1);
+  if (label.length > 80) throw new Error('--name is at most 80 characters');
+  const template = readFileSync(join(packageRoot, 'view', 'page.html'), 'utf8');
+  const rules = readFileSync(join(packageRoot, 'work', 'lib', 'view-rules.mjs'), 'utf8');
+  if (/<\/script/i.test(rules)) throw new Error('view-rules.mjs holds "</script", so it cannot go into a page');
+  const swaps = {
+    '/*@@VIEW_RULES@@*/': rules,
+    '@@PRODUCT@@': quoted(product),
+    '@@PRODUCT_NAME@@': quoted(label),
+    '@@ATLAS_VERSION@@': quoted(version ?? ''),
+    '@@TITLE@@': plain(`${label} project view`),
+  };
+  let html = template;
+  for (const [mark, value] of Object.entries(swaps)) {
+    if (!html.includes(mark)) throw new Error(`the page template has no ${mark}`);
+    html = html.split(mark).join(value);
+  }
+  if (/@@[A-Z_]+@@/.test(html)) throw new Error('the page template has a mark the builder does not fill');
+  return { html, product, name: label };
+}
+
+export function writeViewPage(out, options) {
+  const page = buildViewPage(options);
+  mkdirSync(dirname(out), { recursive: true });
+  writeFileSync(out, page.html);
+  return page;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arjunkhera/atlas",
-  "version": "0.3.24",
+  "version": "0.3.26",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, as a Claude Code plugin for any repository.",
   "type": "module",
   "license": "UNLICENSED",
@@ -21,6 +21,7 @@
     "work/",
     "shape/",
     "tests/",
+    "view/",
     "skills/",
     "agents/",
     ".claude-plugin/",

package/skills/lead/SKILL.md

@@ -102,6 +102,7 @@ set. A test keeps the two lists equal.
 | `atlas ste` | Before each publish; `--share` before a page goes to anyone else |
 | `atlas tests` | `write` puts the test kit in an area. `check` runs before a merge. `proof`, `verdict`, `named`, `covers` and `rerun` read the results of a run. `sweep` and `issues` run the daily sweep |
 | `atlas design` | `folder` to learn where designs live; `check` before you show a design; `build` to make its page |
+| `atlas view` | `build --product <id>` makes the project view page of one product, when the owner asks for it or after an Atlas release changes it. Publish it with only the grant the command prints |
 
 A person merges every file that `atlas tooling` writes.
 A person merges every file that `atlas tests write` writes.

package/skills/tests/SKILL.md

@@ -290,14 +290,39 @@ The adapter binds the library to the product. All its parts are optional:
 `standIns`, `warmUp`, `drivers`, `functions`, `loadModule`, `makers`, `signIns`,
 `data` and `actions`. The header of `test/atlas/contract.mjs` describes each one.
 
-A result is `pass`, `fail`, `not checked`, `not exercised`, `blocked` or
-`not here`. A test is red when any assertion is not `pass`. The one
+A result is `pass`, `fail`, `not checked`, `not exercised`, `blocked`,
+`unsure` or `not here`. A test is red when any assertion is not `pass`. The one
 exception is `not here`: the scenario does not run in that environment, it is
 not counted, and the test passes.
 
 After the product is ready, a lost connection, a server exit or a time-out is
 `fail`. `blocked` is for a guard refusal, a start-up fault and the first-call check.
 
+### A model in the test
+
+A model makes a test vary and costs money. Three parts of the kit handle it.
+Each one needs `budget.money` in `tests.yaml` (the cap for one run),
+`guards.model-hosts: [api.anthropic.com]`, and the key `MODEL_API_KEY` in
+`secrets`. These are guard parts, so the owner sets them.
+
+1. **`repeat: { runs: 3, pass: 3, why: "..." }`** in the front matter. The body
+   runs `runs` times, each time with a new run object (`run.round` is 1, 2, 3).
+   An assertion passes when it passed in `pass` runs. Use it only when a model
+   is part of the product. A steady subject that fails sometimes is flaky.
+2. **A judged assertion** names one judge of `tests.yaml` in backticks. Settle it
+   with `await run.judge(id, { evidence })`, never with `run.check`. Give the
+   judge the smallest evidence that answers the words. When the judge's runs
+   disagree, the result is `unsure`.
+3. **The agent way in** (`driver: agent`) runs `claude -p`. The agent holds
+   no built-in tools, only the MCP tools that the way in lists. `call({ prompt })` returns
+   `body.reply` and `body.steps`: each tool call, its input and its result.
+   Check the steps and the end state with `run.check`; judge the reply.
+
+A product that calls a model itself (a router) reads its own cost. Count it with
+`run.spend(usd)` and check the rest first with `run.budgetLeft()`.
+Model scenarios list `nightly` in `runs-in`, not `ci`. They run each night, not
+on each pull request (Q-C).
+
 ## Sort a red test
 
 Do not edit a red test before you know why it is red. There are four causes:

package/tests/contract.mjs

@@ -19,7 +19,14 @@
 //   4. Writes the evidence file (evidence.mjs): at the start, and after each
 //      assertion.
 //   5. A test fails (node:test red) when any assertion is not `pass`. That
-//      holds for `fail`, `not checked`, `not exercised` and `blocked`.
+//      holds for `fail`, `not checked`, `not exercised`, `blocked` and `unsure`.
+//   6. With `repeat: { runs, pass, why }` in the front matter, the body runs
+//      `runs` times for each way in, each time with a new run object. An
+//      assertion passes when it passed in at least `pass` runs. The evidence
+//      keeps the result of each run. A block stops the repeat at once.
+//   7. A judged assertion is settled by `await run.judge(id, { evidence })`, never
+//      by run.check. The judge of tests.yaml decides: pass, fail or `unsure`.
+//      Model calls count against `budget.money` of tests.yaml (model.mjs).
 //      When the scenario's `runs-in` excludes the environment, the result is
 //      `not here` and the test passes.
 //
@@ -48,13 +55,14 @@ import { dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { Blocked, GateFailed } from './errors.mjs';
 import { Redactor } from './redact.mjs';
-import { readTestsYaml, checkTestsYaml, compareGuardParts } from './tests-yaml.mjs';
+import { readTestsYaml, checkTestsYaml, compareGuardParts, MODEL_KEY } from './tests-yaml.mjs';
 import { readScenario, addFingerprints, combinations, pairs } from './scenario.mjs';
 import { Evidence, clip } from './evidence.mjs';
 import { makeFresh, makeRunId } from './fresh.mjs';
 import { startEnvironment, provideSecrets } from './environment.mjs';
 import { makeDriver } from './drivers/index.mjs';
 import { parseLimit, waitUntil } from './wait.mjs';
+import { Meter, askJudge, killModelCalls, parseMoney, requireModelHost } from './model.mjs';
 
 export { waitUntil };
 export const defineAdapter = (adapter) => adapter;
@@ -120,7 +128,8 @@ export function scenario(path, body, options = {}) {
   describe(`${sc.id}: ${sc.title}`, () => {
     const state = {
       redactor: new Redactor(), evidence: null, envPromise: null, env: null,
-      drivers: new Map(), personas: new Map(), cleanups: [], fresh: null,
+      drivers: new Map(), personas: new Map(), cleanups: [], fresh: null, provided: {},
+      meter: new Meter({ evidenceDir: ctx.evidenceDir, runId: RUN_ID, cap: parseMoney(tests.budget?.money), environment, runIdGiven: Boolean(process.env.ATLAS_RUN_ID || process.env.GITHUB_RUN_ID) }),
     };
     const evidence = () => {
       state.evidence ??= new Evidence({
@@ -132,11 +141,24 @@ export function scenario(path, body, options = {}) {
     const getEnv = () => {
       state.envPromise ??= (async () => {
         if (ctx.guardRefusal) throw new Blocked(ctx.guardRefusal);
+        // The model keys: of each judge that a judged assertion names, and of each way in that holds a model.
+        const judged = sc.assertions.filter((a) => a.kind === 'judged').flatMap((a) => a.names).filter((n) => Object.hasOwn(tests.judges ?? {}, n));
+        const modelKeys = [
+          ...judged.map((n) => tests.judges[n].key ?? MODEL_KEY),
+          // An agent way in uses the model key; any other way in that names a `key` (a product that calls a model) gets it too.
+          ...sc.ways.map((w) => tests['ways-in']?.[w]).filter((w) => w && (w.driver === 'agent' || w.key)).map((w) => w.key ?? MODEL_KEY),
+        ].filter(Boolean);
         const named = [...new Set([...sc.start, ...sc.steps, ...sc.assertions].flatMap((x) => x.names))]
           .filter((n) => Object.hasOwn(tests.secrets ?? {}, n));
         const provided = provideSecrets(named, { root, tests });
+        // A model key goes to the judge and the driver only, never into the product's environment
+        // (section 15, rule 1): the product is not the judge.
+        const models = [...new Set(modelKeys)].filter((n) => Object.hasOwn(tests.secrets ?? {}, n) && !named.includes(n));
+        const modelProvided = provideSecrets(models, { root, tests });
+        for (const value of Object.values(modelProvided)) state.redactor.add(value);
+        state.provided = { ...provided, ...modelProvided };
         const env = await startEnvironment({ root, tests, name: environment, runId: RUN_ID, redactor: state.redactor, adapter, provided });
-        evidence().record.provided_secrets = named;
+        evidence().record.provided_secrets = [...named, ...models];
         state.env = env;
         return env;
       })();
@@ -160,12 +182,60 @@ export function scenario(path, body, options = {}) {
         ev.setWay(way, { started: new Date().toISOString() });
         let run = null;
         let error = null;
-        try {
-          const env = await getEnv();
-          run = makeRun({ ctx, state, way, env, ev, adapter });
-          await withLimit(Promise.resolve().then(() => body(run, way)), limitMs, sc.fields.limit);
-        } catch (thrown) {
-          error = thrown;
+        const repeat = sc.repeat;
+        const rounds = new Map();
+        for (let round = 1; round <= (repeat?.runs ?? 1); round += 1) {
+          error = null;
+          if (repeat && round > 1) {
+            // A new run starts clean: no result, no driver and no persona of the run before.
+            run?.close();
+            await cleanUp(state, run, ev);
+            await closeDrivers(state);
+            state.personas.clear();
+            for (const a of ownEntries(sc, way, ev)) { a.result = 'not checked'; a.proof = null; }
+            ev.write();
+          }
+          try {
+            const env = await getEnv();
+            run = makeRun({ ctx, state, way, env, ev, adapter, round: repeat ? round : null });
+            await withLimit(Promise.resolve().then(() => body(run, way)), limitMs, sc.fields.limit);
+          } catch (thrown) {
+            error = thrown;
+            // A run that stopped (a time limit, say) may leave a model call going. Stop it before the next run.
+            killModelCalls();
+          }
+          if (!repeat) break;
+          const stopped = error && !(error instanceof GateFailed) ? state.redactor.text(String(error.message ?? error)) : null;
+          for (const a of ownEntries(sc, way, ev)) {
+            const list = rounds.get(a) ?? [];
+            const proof = a.proof ?? {};
+            list.push({
+              run: round, result: error instanceof Blocked && a.result === 'not checked' ? 'blocked' : a.result,
+              ...(proof.error ? { error: proof.error } : proof.why ? { why: proof.why } : {}),
+              ...(stopped ? { stopped } : {}),
+            });
+            rounds.set(a, list);
+          }
+          // A block is no product result: the repeat stops, and settle() marks the way blocked.
+          if (error instanceof Blocked) break;
+        }
+        if (repeat) {
+          const blocked = error instanceof Blocked;
+          for (const [a, list] of rounds) {
+            if (list.every((r) => r.result === 'not here')) continue;
+            const passed = list.filter((r) => r.result === 'pass').length;
+            const unsure = list.filter((r) => r.result === 'unsure').length;
+            const ok = passed >= repeat.pass;
+            // The evidence keeps every run, also when a block stopped the repeat. A blocked repeat is never a pass.
+            a.result = blocked ? 'blocked' : ok ? 'pass' : unsure && passed + unsure >= repeat.pass ? 'unsure' : 'fail';
+            a.proof = {
+              repeat: { runs: repeat.runs, pass: repeat.pass, passed }, runs: list,
+              ...(blocked ? { why: `the repeat stopped in run ${list.length}: ${state.redactor.text(error.message)}` } : ok ? {} : { error: `passed in ${passed} of ${list.length} runs; the rule is ${repeat.pass} of ${repeat.runs}` }),
+            };
+          }
+          ev.write();
+          // Each run is settled in its proof. A stop of a run (not a block) is in the proof of its run.
+          if (!blocked) error = null;
         }
         const outcome = settle({ sc, way, ev, error });
         // A failed run keeps a picture of each open page (a driver that has one), before the run closes.
@@ -179,12 +249,18 @@ export function scenario(path, body, options = {}) {
     }
 
     after(async () => {
-      for (const driver of state.drivers.values()) { try { await (await driver).close(); } catch { /* the stop goes on */ } }
+      killModelCalls();
+      await closeDrivers(state);
       try { await (await state.envPromise)?.stop(); } catch { /* it did not start */ }
     });
   });
 }
 
+async function closeDrivers(state) {
+  for (const driver of state.drivers.values()) { try { await (await driver).close(); } catch { /* the stop goes on */ } }
+  state.drivers.clear();
+}
+
 async function keepOnFail(state, label) {
   for (const pending of state.drivers.values()) {
     try { await (await pending).keepOnFail?.(`fail-${label}`); } catch { /* a missing picture is not the news */ }
@@ -202,6 +278,11 @@ async function cleanUp(state, run, ev) {
   for (const fn of list) { try { await fn(); } catch (error) { ev.note(`clean-up failed: ${error.message}`); } }
 }
 
+// The entries that a run through `way` settles: its own, and those of a way in that only a step names.
+function ownEntries(sc, way, ev) {
+  return ev.record.assertions.filter((a) => a.way === way || !sc.through.includes(a.way));
+}
+
 // Turns the end of a body into results, a verdict and a list of problems.
 function settle({ sc, way, ev, error }) {
   // The entries of this way in, and the entries of a way in that only a step names (the page).
@@ -251,7 +332,7 @@ function settle({ sc, way, ev, error }) {
 
 // ---------------------------------------------------------------- the run object
 
-function makeRun({ ctx, state, way, env, ev, adapter }) {
+function makeRun({ ctx, state, way, env, ev, adapter, round = null }) {
   const { sc, tests, root } = ctx;
   state.fresh ??= makeFresh(RUN_ID, tests.data, adapter.data);
   const cleanups = [];
@@ -271,7 +352,7 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
       // A driver is shared by the runs of a scenario (the page way in has no run of its own), so it reaches the
       // record and the attach of the run that is live now (state.live), not those of the run that made it.
       // attach: where a driver keeps a file (a screenshot) as evidence, in the folder of this scenario.
-      state.drivers.set(wayName, makeDriver({ way: wayName, spec, env, root, hosts: env.hosts, redactor: state.redactor, record: (call) => state.live?.record(call), adapter, runId: RUN_ID, attach: (name) => state.live.attach(name) }));
+      state.drivers.set(wayName, makeDriver({ way: wayName, spec, env, root, hosts: env.hosts, redactor: state.redactor, record: (call) => state.live?.record(call), adapter, runId: RUN_ID, attach: (name) => state.live.attach(name), tests, secrets: state.provided, meter: state.meter }));
     }
     return state.drivers.get(wayName);
   };
@@ -288,6 +369,7 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
   const settleOne = (full, fn, opts = {}, mode) => {
     open(`run.${mode}`);
     const { a, fp } = find(full);
+    if (a.kind === 'judged' && mode !== 'exercise') throw new Error(`${a.id} is a judged assertion: settle it with await run.judge(id, { evidence })`);
     // An assertion that reads a result of a step with its own way in (the page) belongs to that way.
     const target = opts.way ?? (a.ways.includes(way) ? way : a.ways[0]);
     if (!a.ways.includes(target)) throw new Error(`"${target}" is not a way in of ${a.id}: its ways are ${a.ways.join(', ')}`);
@@ -330,6 +412,8 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
 
   const run = {
     id: sc.id, title: sc.title, runId: RUN_ID, way, env, scenario: sc, tests, root, cleanups,
+    // The number of this run when the scenario has `repeat` (1, 2, ...); otherwise null.
+    round,
     redact: (text) => state.redactor.text(text),
     driver,
 
@@ -343,6 +427,33 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
     // The test could not run. Never use it for a product result.
     blocked: (why) => { throw new Blocked(why); },
 
+    // A judged assertion. The judge that its words name (in backticks) gets the
+    // words and the evidence, marked as data and redacted, never the test code.
+    // pass or fail when `agree` of its runs agree; otherwise `unsure`.
+    async judge(full, { evidence, way: wayName } = {}) {
+      open('run.judge');
+      const { a, fp } = find(full);
+      if (a.kind !== 'judged') throw new Error(`${a.id} is an exact assertion: settle it with run.check`);
+      const target = wayName ?? (a.ways.includes(way) ? way : a.ways[0]);
+      const calls = callsSince();
+      if (fp !== a.fp) { ev.setResult(a.fullId, target, 'fail', { error: `the fingerprint ${fp} is old: the words of ${a.id} now print ${a.fp}; convert the test again`, calls, settled_by: way }); return; }
+      const names = a.names.filter((n) => Object.hasOwn(tests.judges ?? {}, n));
+      if (names.length !== 1) throw new Blocked(`${a.id} must name exactly one judge of tests.yaml in backticks; it names ${names.length}`);
+      const judge = tests.judges[names[0]];
+      requireModelHost(tests, `the judge "${names[0]}"`);
+      const keyName = judge.key ?? MODEL_KEY;
+      const key = state.provided[keyName];
+      if (!key) throw new Blocked(`the judge "${names[0]}" needs the key ${keyName}: list it in secrets of tests.yaml, and give its value`);
+      const out = await askJudge({ name: names[0], judge, words: a.words, evidence, key, meter: state.meter, redactor: state.redactor });
+      open('run.judge');
+      ev.setResult(a.fullId, target, out.result, { judge: names[0], answers: out.answers, cost: out.cost, calls, settled_by: way, ...(out.result === 'pass' ? {} : { error: out.why }) });
+    },
+    // Money that a product's own model call spent (the router, say), read from its answer.
+    // It counts against budget.money with the judge and the agent calls.
+    spend: (usd, what = 'a product model call') => { open('run.spend'); return state.meter.add(usd, what); },
+    // What the run may still spend, in US$. A model call of the product checks it first.
+    budgetLeft: (what = 'a product model call') => state.meter.ensure(what),
+
     // The rows of a rule table, as objects keyed by the header. A test that
     // reads a table gets every row of the file at run time.
     table(name, { fingerprint } = {}) {

package/tests/drivers/agent.mjs

@@ -0,0 +1,109 @@
+// The agent way in (design section 7.3): a headless agent that holds only the
+// MCP tools that the way in names, run with `claude -p`.
+//
+//   ways-in:
+//     agent:
+//       driver: agent
+//       model: claude-sonnet-5-5   # the product's own model (section 17, "One limit on Haiku 5.5")
+//       key: MODEL_API_KEY         # a name in `secrets`
+//       tools-from: mcp            # an mcp-stdio way in: its command and its env start the server
+//       tools: [get_meal_week, save_meal_week]
+//
+// The four limits of section 7.3, and how the driver keeps each one:
+//   1. No built-in tools: --tools "" and --setting-sources "", and a new empty HOME.
+//   2. Only the named MCP tools: one server, from the way in of `tools-from`, with
+//      --strict-mcp-config, and --allowedTools lists only the named tools.
+//   3. Network to the local product and the model only. The agent has no tool that
+//      reaches a network. Its one server gets its addresses from the env of
+//      `tools-from`, and each one must be on guards.hosts, as for mcp-stdio. The
+//      model host must be on guards.model-hosts. A miss is `blocked`.
+//      The server starts with the model key taken out of its environment.
+//   4. The data it reads is test data: the server is the product of this run. An area with
+//      outside text (`outside-text: true`) gets no agent until Atlas has its outside-text reader.
+// What this does not limit: the server is product code, and it may reach what any way in of the
+// product may reach. The guards on its addresses are the same as for mcp-stdio.
+// The agent runs in a new empty folder (with --bare), so no CLAUDE.md of the repo reaches it.
+// A call of a tool that the way in does not list is refused by the command, and listed in body.offList.
+//
+// call({ prompt }, { persona }) returns { status, body, isError, raw }:
+//   body = { reply, steps: [{ tool, input, result, isError }], turns, cost }
+// `tool` is the bare tool name, without the mcp__<server>__ prefix.
+import { isAbsolute, resolve } from 'node:path';
+import { Blocked } from '../errors.mjs';
+import { requireLocalAddresses } from '../guards.mjs';
+import { clip } from '../evidence.mjs';
+import { modelEnv, requireModelHost, runClaude } from '../model.mjs';
+
+const SERVER = 'product';
+const TOOL_NAME = /^[A-Za-z0-9_-]+$/;
+
+export function createAgentDriver({ way, spec, toolsSpec, fill, tests, hosts, productRoot, key, meter, record, redactor, commandFor }) {
+  const tools = Array.isArray(spec.tools) ? spec.tools.map(String) : [];
+  if (!spec.model) throw new Blocked(`the way in "${way}" uses the agent driver and names no model`);
+  if (!tools.length) throw new Blocked(`the way in "${way}" names no tools: an agent holds only the MCP tools that the way in lists`);
+  for (const t of tools) if (!TOOL_NAME.test(t)) throw new Blocked(`the way in "${way}" names the tool "${t}", which is not a plain tool name`);
+  if (!toolsSpec) throw new Blocked(`the way in "${way}" needs tools-from: an mcp-stdio way in that starts the product's MCP server`);
+  if (toolsSpec.driver !== 'mcp-stdio') throw new Blocked(`tools-from of "${way}" names a way in with the driver "${toolsSpec.driver}"; it must be mcp-stdio`);
+  if (tests?.['outside-text'] === true) throw new Blocked(`the way in "${way}" is an agent in an area with outside text; no agent runs there until Atlas has its outside-text reader (section 15, rule 13)`);
+  requireModelHost(tests, `the way in "${way}"`);
+  const env = Object.fromEntries(Object.entries(toolsSpec.env ?? {}).map(([k, v]) => [k, fill(String(v))]));
+  // The server gets the env of the command (the allow-list) and its own env: check the whole of it.
+  requireLocalAddresses(env, hosts, `the MCP server of the way in "${way}"`, { ...modelEnv('/nonexistent', 'x'), ...env });
+  const command = commandFor(fill(toolsSpec.command));
+  if (!command) throw new Blocked(`tools-from of "${way}" has no command`);
+  const cwd = toolsSpec.cwd ? (isAbsolute(toolsSpec.cwd) ? toolsSpec.cwd : resolve(productRoot, toolsSpec.cwd)) : productRoot;
+
+  return {
+    name: 'agent',
+    way,
+    async call(request = {}, { persona = null } = {}) {
+      const prompt = String(request.prompt ?? '');
+      if (!prompt) throw new Error('the agent way in needs a prompt');
+      // `sh -c` runs the server's command as the mcp-stdio driver does; `env -u` takes the model key out first.
+      const config = { mcpServers: { [SERVER]: { type: 'stdio', command: 'env', args: ['-u', 'ANTHROPIC_API_KEY', 'sh', '-c', `cd ${JSON.stringify(cwd)} && exec ${command}`], env } } };
+      redactor.add(key);
+      const out = await runClaude({
+        prompt, key, meter, redactor, what: `the agent of "${way}"`, stream: true,
+        args: [
+          '--model', String(spec.model),
+          '--tools', '',
+          '--mcp-config', JSON.stringify(config),
+          '--allowedTools', ...tools.map((t) => `mcp__${SERVER}__${t}`),
+          ...(spec['system-prompt'] ? ['--append-system-prompt', String(spec['system-prompt'])] : []),
+        ],
+      });
+      const steps = stepsOf(out.events);
+      const offList = steps.filter((s) => !tools.includes(s.tool)).map((s) => s.tool);
+      const reply = String(out.envelope?.result ?? '');
+      const isError = out.exit !== 0 || Boolean(out.envelope?.is_error) || !out.envelope;
+      const body = { reply, steps, offList, turns: out.envelope?.num_turns ?? null, cost: out.cost, ...(isError ? { error: clip(out.stderr || out.stdout, redactor) } : {}) };
+      const safe = redactor.deep(body);
+      record?.({ way, persona, request: { prompt: redactor.text(prompt), tools }, response: { status: out.exit, body: clip(safe, redactor) } });
+      return { status: out.exit, body: safe, isError, raw: { envelope: redactor.deep(out.envelope) } };
+    },
+    async close() {},
+  };
+}
+
+// The path of tool calls in a stream-json transcript: each tool_use, joined to its tool_result.
+export function stepsOf(events) {
+  const steps = [];
+  const byId = new Map();
+  for (const e of events) {
+    const content = e?.message?.content;
+    if (!Array.isArray(content)) continue;
+    for (const block of content) {
+      if (e.type === 'assistant' && block.type === 'tool_use') {
+        const step = { tool: String(block.name ?? '').replace(new RegExp(`^mcp__${SERVER}__`), ''), input: block.input ?? {}, result: null, isError: null };
+        steps.push(step);
+        byId.set(block.id, step);
+      } else if (e.type === 'user' && block.type === 'tool_result') {
+        const step = byId.get(block.tool_use_id);
+        if (!step) continue;
+        step.isError = Boolean(block.is_error);
+        step.result = Array.isArray(block.content) ? block.content.map((c) => (c.type === 'text' ? c.text : c)).join('\n') : block.content;
+      }
+    }
+  }
+  return steps;
+}

package/tests/drivers/index.mjs

@@ -11,10 +11,12 @@ import { createHttpDriver } from './http.mjs';
 import { createMcpStdioDriver } from './mcp-stdio.mjs';
 import { createFunctionDriver } from './function.mjs';
 import { createBrowserDriver } from './browser.mjs';
+import { createAgentDriver } from './agent.mjs';
 
-export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, createBrowserDriver };
+export { createHttpDriver, createMcpStdioDriver, createFunctionDriver, createBrowserDriver, createAgentDriver };
 
-// ctx: { way, spec, env, root, hosts, redactor, record, adapter, runId, attach }
+// ctx: { way, spec, env, root, hosts, redactor, record, adapter, runId, attach, tests, secrets, meter }
+// `secrets` holds the provided values of the named secrets; `meter` counts model money (model.mjs).
 // `attach(fileName)` gives the path of a file to keep as evidence (a screenshot).
 // `adapter.playwright` is the Playwright module, for a test that has none installed.
 // In a fault run, ATLAS_PRODUCT_ROOT names a throwaway copy. A process that a
@@ -66,6 +68,17 @@ export async function makeDriver(ctx) {
       if (!target) throw new Blocked(`the way in "${way}" uses the function driver and the adapter gives no module: set adapter.functions`);
       return createFunctionDriver({ way, target, record, redactor });
     }
+    case 'agent': {
+      const keyName = spec.key ?? 'MODEL_API_KEY';
+      const key = ctx.secrets?.[keyName];
+      if (!Object.hasOwn(ctx.tests?.secrets ?? {}, keyName)) throw new Blocked(`the way in "${way}" needs the model key ${keyName} in secrets of tests.yaml`);
+      if (!key) throw new Blocked(`the way in "${way}" needs the key ${keyName}, and no value was given`);
+      const toolsSpec = ctx.tests?.['ways-in']?.[spec['tools-from']];
+      return createAgentDriver({
+        way, spec, toolsSpec, fill, tests: ctx.tests, hosts, productRoot, key, meter: ctx.meter, record, redactor,
+        commandFor: (command) => commandFor(command, root, productRoot),
+      });
+    }
     default:
       throw new Blocked(`the way in "${way}" uses the driver "${spec.driver}", which the kit does not ship; add it to adapter.drivers`);
   }

package/tests/evidence.mjs

@@ -43,7 +43,8 @@ function commitOf(root) {
 
 // The results of one assertion for one way in (design section 6.11). `flaky` is never set by a
 // test. `atlas tests verdict` and the proof table read it from two attempts (verdict.mjs).
-export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here', 'flaky']);
+// `unsure` comes from a judge whose runs disagree (model.mjs); it counts as a fail until a person looks.
+export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here', 'flaky', 'unsure']);
 
 const MAX_BODY = 8000;
 

package/tests/link-check.mjs

@@ -188,7 +188,7 @@ function checkBudget(scenario, tests, findings, rel) {
 }
 
 // The ids that tests name. Only a literal first argument can be read.
-const CALL = /\brun\.(check|gate|notExercised|waitFor)\s*\(/g;
+const CALL = /\brun\.(check|gate|judge|notExercised|waitFor)\s*\(/g;
 
 export function readRefs(file, code, rel, findings) {
   const refs = [];
@@ -281,6 +281,8 @@ function checkLinks(scenarios, root, testsDir, findings, rel) {
     if (!by.has(key)) by.set(key, []);
     by.get(key).push({ ...r, a });
     if (a.fp && r.fp !== a.fp) findings.push({ file: rel(r.file), line: r.line, code: 'old-fingerprint', message: `${r.full} has an old fingerprint; ${key} now prints #${a.fp}` });
+    else if (r.kind === 'judge' && a.kind !== 'judged') findings.push({ file: rel(r.file), line: r.line, code: 'judge-mismatch', message: `${key} is an exact assertion; the test must use run.check or run.gate, not run.judge` });
+    else if ((r.kind === 'check' || r.kind === 'gate') && a.kind === 'judged') findings.push({ file: rel(r.file), line: r.line, code: 'judge-mismatch', message: `${key} is a judged assertion; the test must use run.judge` });
     else if (r.kind === 'check' && a.gate) findings.push({ file: rel(r.file), line: r.line, code: 'gate-mismatch', message: `${key} is marked gate in the scenario; the test must use run.gate` });
     else if (r.kind === 'gate' && !a.gate) findings.push({ file: rel(r.file), line: r.line, code: 'gate-mismatch', message: `${key} is not marked gate in the scenario; the test must use run.check` });
   }
@@ -507,8 +509,8 @@ export function checkRun({ evidenceDir, runId = null, environment = null, scenar
       runOf.set(data.scenario, id);
     }
   }
-  const reached = new Set(['pass', 'fail', 'not exercised', 'not here', 'flaky']);
-  const ran = new Set(['pass', 'fail', 'not exercised', 'flaky']);
+  const reached = new Set(['pass', 'fail', 'not exercised', 'not here', 'flaky', 'unsure']);
+  const ran = new Set(['pass', 'fail', 'not exercised', 'flaky', 'unsure']);
   for (const s of scenarios) {
     if (!s.id) continue;
     const all = records.filter((r) => r.data.scenario === s.id);