@outerlayer/cli 0.28.3 → 0.29.0

package/dist/{chunk-ILU6664L.js → chunk-M4QY7YZ5.js}

@@ -1,31 +1,33 @@
 #!/usr/bin/env node
 import { createRequire } from 'node:module';
 import { readsJobState, READ_JOB_STATE_FORMATS, jobStateFormatOf } from './chunk-ZVB57WI6.js';
-import { DOCKERD_LOG, HELPER_PATH, HELPER_RECORD } from './chunk-OO6OADZN.js';
-import { senderPidfile } from './chunk-YTR435DO.js';
-import { installScrubContext } from './chunk-2BZTGJII.js';
-import { writeDockerClientConfig } from './chunk-RXFKXZJS.js';
-import { ADDED_USER, shellQuote, GH_CONFIG_YML, ghHostsYml, AssetError, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createLeaseChannel, createBuildDisk, machineConfig, resolveAgentUser, AgentUserError, runBuildSteps, LEASE_SECONDS, createContainerRuntime, stopContainer, cleanupContainer, outOfMemoryReason, containerSenderCommand, signalInContainer, syncInContainer, copyTranscriptsFromContainer, senderAliveScript, helperFirecrackerNote, HelperRefusal, isLiteralAddress, deleteBuildDisk, startCheckContainer, lifecycleSteps, lifecycleFailureReason, writeEvalInput, DeliveryFailure, EVAL_INPUT_NAME, copyTranscripts, senderSignalScript, containerJudgeEnvironment, copyJudgeTree } from './chunk-4SJ57DH7.js';
-import { waitForAgent, vmExecArgs } from './chunk-WL3QF3QF.js';
-import { socketPathProblem, vsockLinks, LEASE_LINK, linkVsockSockets } from './chunk-DIQN5SEL.js';
-import { describeCallFailure, resolveGoverningSource, refusalCode, removeStagedContext, stageGovernedContext, pinnedSourceOf, commandWithShim, logStamp, createRunnerLog, removeKeptContext, formatDuration, retrySend, formatClock, keptContextTree } from './chunk-JZJLKTZT.js';
-import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm, DEFAULT_CHECK_TIME_LIMIT_MINUTES } from './chunk-YYGXCRIW.js';
-import { readBranchHead, fetchCommit, diffBetween } from './chunk-TCBVV2JK.js';
+import { DOCKERD_LOG, HELPER_PATH, HELPER_RECORD } from './chunk-OD6WUNKZ.js';
+import { zodProblem } from './chunk-I3LTIAOR.js';
+import { senderPidfile } from './chunk-PP7C7JQO.js';
+import { installScrubContext } from './chunk-GJJDPZS2.js';
+import { writeDockerClientConfig } from './chunk-KBIJZAH4.js';
+import { ADDED_USER, shellQuote, GH_CONFIG_YML, ghHostsYml, AssetError, obtainImage, ensureRootfs, gitRemoteFor, startServicesFor, createLeaseChannel, createBuildDisk, machineConfig, resolveAgentUser, AgentUserError, runBuildSteps, LEASE_SECONDS, createContainerRuntime, stopContainer, cleanupContainer, outOfMemoryReason, containerSenderCommand, signalInContainer, syncInContainer, copyTranscriptsFromContainer, senderAliveScript, helperFirecrackerNote, HelperRefusal, isLiteralAddress, deleteBuildDisk, startCheckContainer, lifecycleSteps, lifecycleFailureReason, writeEvalInput, DeliveryFailure, EVAL_INPUT_NAME, copyTranscripts, senderSignalScript, containerJudgeEnvironment, copyJudgeTree } from './chunk-MO2NI6W6.js';
+import { waitForAgent, vmExecArgs } from './chunk-77YT7YC4.js';
+import { socketPathProblem, vsockLinks, LEASE_LINK, linkVsockSockets } from './chunk-YSH4KCUJ.js';
+import { describeCallFailure, resolveGoverningSource, refusalCode, removeStagedContext, stageGovernedContext, pinnedSourceOf, commandWithShim, logStamp, createRunnerLog, removeKeptContext, formatDuration, retrySend, formatClock, keptContextTree } from './chunk-SN3C57RV.js';
+import { isBuiltinContainer, isBuiltinProcess, isBuiltinVm, DEFAULT_CHECK_TIME_LIMIT_MINUTES } from './chunk-NAHTO4XM.js';
+import { readBranchHead, fetchCommit, diffBetween } from './chunk-V47KJGNE.js';
 import { isSameProcess, startTimeOf } from './chunk-JBJTC3UU.js';
-import { HASH_ARGV, parseProgramHash, JUDGE_PROGRAM, writeJobFile, readJobFile, childEnvironment, judgeCommand, JUDGE_MAX_TURNS, MAX_REPORT_BYTES, JUDGE_DIR, COMMAND_DIR } from './chunk-MFI55TNI.js';
+import { HASH_ARGV, parseProgramHash, JUDGE_PROGRAM, writeJobFile, readJobFile, childEnvironment, withAnswerProtocol, judgeCommand, MAX_REPORT_BYTES, JUDGE_MAX_TURNS, JUDGE_DIR, COMMAND_DIR, MAX_ITEMS_BYTES } from './chunk-BFA2LNJG.js';
 import { parseSize } from './chunk-YJLHUKYS.js';
 import { CLAUDE_CODE_VERSION, FIRECRACKER_VERSION } from './chunk-SJZL5F3F.js';
-import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-X7SFJZIG.js';
-import { planArtifactUpload } from './chunk-BUP5N3X6.js';
-import { scrubText } from './chunk-UHU7H57K.js';
+import { WorkCommandError, postJson, isSignatureRefusal, getJson } from './chunk-6AG2XW5S.js';
+import { planArtifactUpload } from './chunk-6XQPCQBW.js';
+import { scrubText } from './chunk-LKN4J37L.js';
 import { readCloudConfig } from './chunk-PJDQPF5N.js';
-import { signerFor } from './chunk-R7TMTFJP.js';
-import { RUNNER_FILE_MODE, runnerDir, buildDir, vsockPath } from './chunk-DWVAG73B.js';
-import { JOB_MOUNT, claudeCredential, repositoriesOf, primaryOf, buildEnvironment, CHECKOUT_DIR, parseIsolation, proxyEnvironment, withProxyWarningHidden, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, withHooksPath, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT, repositoryPath, lifecycleWorkdir, currentClaudeCredential, removedClaudeCredentialValues, checkEnvironment, EVAL_INPUT_DIR } from './chunk-N5EEEKT6.js';
-import { resolvePackRoot } from './chunk-JGML2VUN.js';
+import { signerFor } from './chunk-SHVE5JZO.js';
+import { RUNNER_FILE_MODE, runnerDir, buildDir, vsockPath } from './chunk-PAHZQQB3.js';
+import { JOB_MOUNT, claudeCredential, repositoriesOf, primaryOf, buildEnvironment, CHECKOUT_DIR, parseIsolation, proxyEnvironment, withProxyWarningHidden, containerLocalAddress, destinationEnvironment, CLAUDE_CREDENTIAL_VARIABLES, builtInEnvironment, withHooksPath, startBuildServices, BUILT_IN_NAMES, AGENT_CREDENTIAL_MISSING, sharedUserEnvironment, headerVariables, authorEnvironment, withTunnelRecord, commitIdentityRecord, RELAY_PORT, repositoryPath, lifecycleWorkdir, CHECK_ITEMS_FILE, currentClaudeCredential, removedClaudeCredentialValues, checkEnvironment, EVAL_INPUT_DIR } from './chunk-C7F6U2YZ.js';
+import { resolvePackRoot } from './chunk-NX2G6PH6.js';
 import { MAINTAINED_SKILLS } from './chunk-CDDED64E.js';
 import { resolveVersion } from './chunk-RA3O54FC.js';
-import { parsePolicy } from './chunk-RKOTJQVG.js';
+import { parsePolicy } from './chunk-DXI6A6JN.js';
+import { CheckItemsSchema, CHECK_ITEM_REASON_MAX_LENGTH } from './chunk-JL6VZHAJ.js';
 import { homeFromEnv } from './chunk-VNVZDWO3.js';
 import { fdName } from './chunk-XD4ZCMA4.js';
 import { RUNNER_WORDED_REASONS, UNKNOWN_TRACKING_REASON } from './chunk-MESOOLF4.js';
@@ -821,6 +823,12 @@ function microVmJudgeEnvironment(input) {
   const needsVm = () => {
     if ((state.firecracker !== void 0 || state.jailed !== void 0) && !isRunning(runtime, state)) throw new Error("the build's microVM is gone");
   };
+  const readJudgeFile = async (name, maxBytes) => {
+    needsVm();
+    const file = `${GUEST_JUDGE_DIR}/${name}`;
+    const read = await asRoot(runtime, state, ["/bin/sh", "-c", `[ -f ${file} ] && [ ! -L ${file} ] && head -c ${maxBytes + 1} ${file}`]);
+    return read.status === 0 && Buffer.byteLength(read.stdout) <= maxBytes ? read.stdout : void 0;
+  };
   return {
     stop: async () => {
       needsVm();
@@ -865,12 +873,8 @@ function microVmJudgeEnvironment(input) {
       appendFileSync(opts.logPath, ran.stdout + ran.stderr);
       return ran;
     },
-    readReport: async () => {
-      needsVm();
-      const file = `${GUEST_JUDGE_DIR}/report.html`;
-      const read = await asRoot(runtime, state, ["/bin/sh", "-c", `[ -f ${file} ] && [ ! -L ${file} ] && head -c ${MAX_REPORT_BYTES + 1} ${file}`]);
-      return read.status === 0 && Buffer.byteLength(read.stdout) <= MAX_REPORT_BYTES ? read.stdout : void 0;
-    }
+    readReport: () => readJudgeFile("report.html", MAX_REPORT_BYTES),
+    readItems: () => readJudgeFile("items.json", MAX_ITEMS_BYTES)
   };
 }
 async function startCheckMicroVm(input) {
@@ -1538,10 +1542,144 @@ async function provisionProcess(input) {
 function cleanupProcess(runtime, state) {
   return closeHostBuild(runtime, state);
 }
+var ARTIFACT_PROOF_REASON = "Its declared artifact proves it; the judge does not judge artifacts.";
+var NO_CITING_TEST_REASON = "No test cites this criterion, and it declares no artifact proof.";
+var NO_VERDICT_REASON = "The judge gave no result for this criterion.";
+function criterionInputs(input) {
+  const paths = [...new Set((input.citations[input.criterion.id] ?? []).map((citation) => citation.path))].sort();
+  const hash = createHash("sha256");
+  const part = (text2) => {
+    hash.update(`${Buffer.byteLength(text2)}:`);
+    hash.update(text2);
+  };
+  part("judge-criterion-v1");
+  part(input.brief);
+  part(input.criterion.id);
+  part(input.criterion.text);
+  part(input.criterion.proof ?? "");
+  for (const path of paths) {
+    part(path);
+    part(input.testFiles.get(path) ?? "");
+  }
+  return hash.digest("hex");
+}
+var HTTPS_LINK = /^https:\/\/\S+$/;
+function refsFor(id, citations) {
+  const refs = (citations[id] ?? []).slice(0, 20).map((citation) => ({
+    path: citation.path,
+    line: citation.line,
+    // Only an https link becomes an href on a page; any other is left off rather than refusing the check.
+    ...HTTPS_LINK.test(citation.href ?? "") ? { href: citation.href } : {}
+  }));
+  return refs.length === 0 ? void 0 : refs;
+}
+function planCriteria(input) {
+  const previousById = new Map(input.previous.map((item) => [item.id, item]));
+  const settled = [];
+  const toJudge = [];
+  for (const criterion of input.criteria) {
+    const inputs = input.inputs.get(criterion.id);
+    const base = { id: criterion.id, criterion: criterion.id, ...inputs === void 0 ? {} : { inputs } };
+    const refs = refsFor(criterion.id, input.citations);
+    if (criterion.proof !== null) {
+      settled.push({ ...base, result: "skipped", reason: ARTIFACT_PROOF_REASON });
+      continue;
+    }
+    if (refs === void 0) {
+      settled.push({ ...base, result: "fail", reason: NO_CITING_TEST_REASON });
+      continue;
+    }
+    const kept = previousById.get(criterion.id);
+    if (kept !== void 0 && inputs !== void 0 && kept.inputs === inputs && (kept.result === "pass" || kept.result === "fail")) {
+      settled.push({ ...base, result: kept.result, ...kept.reason === void 0 ? {} : { reason: kept.reason }, refs });
+      continue;
+    }
+    toJudge.push(criterion);
+  }
+  return { settled, toJudge };
+}
+var MAX_ANSWER_ENTRIES = 500;
+function readAnswer(raw) {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return { ok: false, problem: "expected an object" };
+  const { result, items } = raw;
+  if (result !== void 0 && result !== "pass" && result !== "fail") return { ok: false, problem: 'result: must be "pass" or "fail"' };
+  if (items !== void 0 && !Array.isArray(items)) return { ok: false, problem: "items: must be an array" };
+  const entries = items ?? [];
+  if (entries.length > MAX_ANSWER_ENTRIES) return { ok: false, problem: `items: at most ${MAX_ANSWER_ENTRIES} entries` };
+  const read = [];
+  for (const [index, entry] of entries.entries()) {
+    const { id, result: itemResult, reason } = typeof entry === "object" && entry !== null ? entry : {};
+    if (typeof id !== "string" || id === "" || id.length > 200) return { ok: false, problem: `items.${index}.id: must be text of 1 to 200 characters` };
+    if (itemResult !== "pass" && itemResult !== "fail") return { ok: false, problem: `items.${index}.result: must be "pass" or "fail"` };
+    if (reason !== void 0 && typeof reason !== "string") return { ok: false, problem: `items.${index}.reason: must be text` };
+    read.push({ id, result: itemResult, ...reason === void 0 ? {} : { reason } });
+  }
+  return { ok: true, answer: { ...result === void 0 ? {} : { result }, items: read } };
+}
+function parseJudgeAnswer(text2) {
+  if (text2 === void 0 || text2.trim() === "") return void 0;
+  let raw;
+  try {
+    raw = JSON.parse(text2);
+  } catch {
+    return { ok: false, reason: "the judge's items.json is not JSON" };
+  }
+  const read = readAnswer(raw);
+  return read.ok ? read : { ok: false, reason: `the judge's items.json does not match its shape: ${read.problem}` };
+}
+function oneLineReason(text2) {
+  if (text2 === void 0) return void 0;
+  const line = text2.replace(/\s+/g, " ").trim();
+  if (line === "") return void 0;
+  return line.length <= CHECK_ITEM_REASON_MAX_LENGTH ? line : `${line.slice(0, CHECK_ITEM_REASON_MAX_LENGTH - 1)}\u2026`;
+}
+function judgedItems(input) {
+  return input.toJudge.map((criterion) => {
+    const said = input.results.get(criterion.id);
+    const inputs = input.inputs.get(criterion.id);
+    const refs = refsFor(criterion.id, input.citations);
+    const reason = said === void 0 ? NO_VERDICT_REASON : oneLineReason(said.reason);
+    return {
+      id: criterion.id,
+      criterion: criterion.id,
+      result: said?.result ?? "fail",
+      ...reason === void 0 ? {} : { reason },
+      ...refs === void 0 ? {} : { refs },
+      ...inputs === void 0 || said === void 0 ? {} : { inputs }
+    };
+  });
+}
+function inCriteriaOrder(items, criteria) {
+  const order = new Map(criteria.map((criterion, index) => [criterion.id, index]));
+  return [...items].sort((a, b) => (order.get(a.id) ?? Number.MAX_SAFE_INTEGER) - (order.get(b.id) ?? Number.MAX_SAFE_INTEGER));
+}
+function itemsSentence(items, noun) {
+  const failing = items.filter((item) => item.result === "fail").map((item) => item.id);
+  if (failing.length === 0) return void 0;
+  return `${failing.length} of ${items.length} ${items.length === 1 ? noun.one : noun.many} not satisfied: ${failing.join(", ")}`;
+}
+
+// src/runner/host-checks.ts
 var MAX_VALIDATOR_FILES = 20;
 var VALIDATORS_DIR = ".outerlayer/validators";
 var POLICY_FILE = ".outerlayer/policy.yaml";
 var LIMIT_KEY_TAIL = " (runner.checks.timeLimitMinutes)";
+var MAX_CHECK_ITEMS_BYTES = 1024 * 1024;
+function commandItems(read, scrub) {
+  if (read.status !== 0 || read.stdout.trim() === "") return { ok: true, items: void 0 };
+  if (Buffer.byteLength(read.stdout) > MAX_CHECK_ITEMS_BYTES) return { ok: false, reason: `its check items file is longer than ${MAX_CHECK_ITEMS_BYTES} bytes` };
+  let raw;
+  try {
+    raw = JSON.parse(read.stdout);
+  } catch {
+    return { ok: false, reason: "its check items file (OUTERLAYER_CHECK_ITEMS) is not JSON" };
+  }
+  const parsed = CheckItemsSchema.safeParse(raw);
+  if (!parsed.success) {
+    return { ok: false, reason: `its check items file does not match the check item shape: ${zodProblem(parsed.error.issues)}` };
+  }
+  return { ok: true, items: parsed.data.map((item) => item.reason === void 0 ? item : { ...item, reason: scrub(item.reason) }) };
+}
 function isJudgeCheck(check) {
   return "judge" in check;
 }
@@ -1595,7 +1733,7 @@ function checksIn(root, log, tag) {
   const hostChecks = loaded.customs.filter((def) => def.runWhere === "host" && def.level !== "off" && def.declaresEmit !== null);
   const checks = [];
   for (const def of hostChecks) {
-    if (def.judge !== void 0) checks.push({ name: def.declaresEmit, judge: { brief: def.judge.brief, text: readBrief(root, def.judge.brief) } });
+    if (def.judge !== void 0) checks.push({ name: def.declaresEmit, judge: { brief: def.judge.brief, text: readBrief(root, def.judge.brief), subjects: def.judge.subjects } });
     else if (def.command !== void 0) checks.push({ name: def.declaresEmit, command: def.command });
   }
   return checks;
@@ -1665,7 +1803,11 @@ function makeRecorder(input) {
         "/v1/emitted-results",
         config.apiKey,
         config.appId,
-        { schemaVersion: 1, emit: { clientEmitId, name, result, ...body === void 0 ? {} : { body }, ...opts.link === void 0 ? {} : { link: opts.link }, emittedAt: now().toISOString(), item: { number: Number(job.item) } } },
+        {
+          schemaVersion: 1,
+          // A field left undefined is not sent: the request is JSON.
+          emit: { clientEmitId, name, result, body, link: opts.link, items: opts.items, emittedAt: now().toISOString(), item: { number: Number(job.item) } }
+        },
         deps.fetchImpl,
         void 0,
         signer
@@ -1715,7 +1857,7 @@ async function runHostChecks(input) {
   log.line(`${tag} checks started (${checks.map((check) => check.name).join(", ")})`);
   const startedMs = now().getTime();
   const recorder = makeRecorder({ config, job, deps });
-  const record = (check, result, sentence, keyTail = "") => recorder(check.name, result, sentence, { keyTail });
+  const record = (check, result, sentence, keyTail = "", items) => recorder(check.name, result, sentence, { keyTail, items });
   const failAll = async (reason, keyTail = "") => {
     log.error(`${tag} checks failed: ${reason}${keyTail}`);
     for (const check of checks) await record(check, "fail", reason, keyTail);
@@ -1884,7 +2026,9 @@ async function runHostChecks(input) {
         continue;
       }
       const checkStartedMs = now().getTime();
-      const running = ready.exec(["/bin/sh", "-c", check.command], CHECKOUT_DIR, { logPath: join(jobDir, "checks", `${check.name}.log`) });
+      const logPath = join(jobDir, "checks", `${check.name}.log`);
+      await ready.exec(["rm", "-f", CHECK_ITEMS_FILE], CHECKOUT_DIR, { logPath });
+      const running = ready.exec(["/bin/sh", "-c", check.command], CHECKOUT_DIR, { logPath });
       running.catch(() => void 0);
       const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
       let outcome;
@@ -1904,14 +2048,25 @@ async function runHostChecks(input) {
         restartTimer.cancel();
         continue;
       }
-      if (outcome.status === 0) {
+      const read = await ready.exec(["head", "-c", String(MAX_CHECK_ITEMS_BYTES + 1), CHECK_ITEMS_FILE], CHECKOUT_DIR);
+      const items = commandItems(read, (text2) => scrubReason(text2, config) ?? text2);
+      if (!items.ok) {
+        await record(check, "fail", items.reason);
+        continue;
+      }
+      const failing = items.items === void 0 ? void 0 : itemsSentence(items.items, { one: "check item", many: "check items" });
+      if (outcome.status === 0 && failing === void 0) {
         log.line(`${tag} check ${check.name} ran (${formatDuration(now().getTime() - checkStartedMs)})`);
-        await record(check, "pass");
+        await record(check, "pass", void 0, "", items.items);
+        continue;
+      }
+      if (outcome.status === 0) {
+        await record(check, "fail", failing, "", items.items);
         continue;
       }
       const last = lastLine(outcome.stderr) ?? lastLine(outcome.stdout);
       const scrubbed = last === void 0 ? void 0 : scrubReason(last, config);
-      await record(check, "fail", `exited ${outcome.status}${scrubbed === void 0 ? " with no output" : `: ${cutReasonLine(scrubbed, 140)}`}`);
+      await record(check, "fail", `exited ${outcome.status}${scrubbed === void 0 ? " with no output" : `: ${cutReasonLine(scrubbed, 140)}`}`, "", items.items);
     }
   } catch (err) {
     log.error(`${tag} checks stopped: ${describeCallFailure(err)}`);
@@ -1934,9 +2089,6 @@ async function runHostChecks(input) {
 }
 
 // src/runner/judge-verdict.ts
-function mechanicalFailures(criteria, citations) {
-  return criteria.filter((criterion) => criterion.proof === null && (citations[criterion.id]?.length ?? 0) === 0).map((criterion) => criterion.id);
-}
 function firstLine2(text2) {
   return text2.trim().split(/\r\n|\r|\n/)[0]?.trim() ?? "";
 }
@@ -1952,26 +2104,68 @@ function parseOutput(stdout) {
 }
 var RESULT_LINE = /^result:\s*(pass|fail)\s*$/i;
 var UNMET_LINE = /^unmet:\s*(\S+)\s*$/i;
-function decideVerdict(input) {
-  const { run, criteria } = input;
-  if (run.kind === "timeout") return { result: "fail", reason: `the judge ran out of time after ${input.minutes}`, limitKey: true };
+function readJudgeReply(run, minutes) {
+  if (run.kind === "timeout") return { kind: "none", reason: `the judge ran out of time after ${minutes}`, limitKey: true };
   const output = parseOutput(run.stdout);
   if (output === void 0) {
     const said = firstLine2(run.stderr);
-    return { result: "fail", reason: `the judge exited ${run.status} and printed no result${said === "" ? "" : `: ${said}`}` };
+    return { kind: "none", reason: `the judge exited ${run.status} and printed no result${said === "" ? "" : `: ${said}`}` };
   }
-  if (output.subtype === "error_max_turns") return { result: "fail", reason: `the judge reached the turn limit of ${JUDGE_MAX_TURNS} turns before it gave a result` };
+  if (output.subtype === "error_max_turns") return { kind: "none", reason: `the judge reached the turn limit of ${JUDGE_MAX_TURNS} turns before it gave a result` };
   const lines = output.result.split(/\r\n|\r|\n/).map((line) => line.trim()).filter((line) => line !== "");
   const verdict = RESULT_LINE.exec(lines.at(-1) ?? "")?.[1]?.toLowerCase();
-  if (verdict === void 0) return { result: "fail", reason: "the judge gave no `result:` line" };
-  const named = new Set(lines.flatMap((line) => UNMET_LINE.exec(line)?.[1] ?? []));
-  const mechanical = new Set(mechanicalFailures(criteria, input.citations));
-  const unmet = criteria.map((criterion) => criterion.id).filter((id) => named.has(id) || mechanical.has(id));
-  if (verdict === "pass" && unmet.length === 0) return { result: "pass" };
-  const ruled = unmet.filter((id) => mechanical.has(id));
-  const rule = ruled.length === 0 ? "" : `: the mechanical rule found no proof annotation and no citing test for ${ruled.join(", ")}`;
-  if (unmet.length === 0) return { result: "fail", reason: "the judge failed the change and named no criterion" };
-  return { result: "fail", reason: `${unmet.length} of ${criteria.length} criteria not satisfied, first ${unmet[0]}${rule}` };
+  if (verdict === void 0) return { kind: "none", reason: "the judge gave no `result:` line" };
+  return { kind: "answered", verdict, unmet: new Set(lines.flatMap((line) => UNMET_LINE.exec(line)?.[1] ?? [])) };
+}
+
+// src/runner/judge-report.ts
+function escape(text2) {
+  return text2.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+var RESULT_LABEL = { pass: "satisfied", fail: "not satisfied", skipped: "not judged" };
+function refsCell(item) {
+  return (item.refs ?? []).map((ref) => {
+    const label = escape(ref.line === void 0 ? ref.path : `${ref.path}:${ref.line}`);
+    return ref.href === void 0 ? `<code>${label}</code>` : `<a href="${escape(ref.href)}"><code>${label}</code></a>`;
+  }).join("<br>");
+}
+function renderJudgeReport(input) {
+  const count = (result) => input.items.filter((item) => item.result === result).length;
+  const summary = `${count("pass")} satisfied, ${count("fail")} not satisfied, ${count("skipped")} not judged`;
+  const rows = input.items.map(
+    (item) => `<tr class="${item.result}"><td><code>${escape(item.id)}</code></td><td>${RESULT_LABEL[item.result]}</td><td>${escape(item.reason ?? "")}</td><td>${refsCell(item)}</td></tr>`
+  ).join("\n");
+  return `<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>${escape(input.check)}</title>
+<style>
+:root { color-scheme: light dark; --fail: #b42318; --pass: #067647; --muted: #667085; }
+body { font: 14px/1.5 system-ui, sans-serif; margin: 24px; max-width: 1100px; }
+table { border-collapse: collapse; width: 100%; }
+th, td { text-align: left; vertical-align: top; padding: 6px 10px; border-bottom: 1px solid color-mix(in srgb, currentColor 15%, transparent); }
+tr.fail td:nth-child(2) { color: var(--fail); font-weight: 600; }
+tr.pass td:nth-child(2) { color: var(--pass); }
+tr.skipped td:nth-child(2) { color: var(--muted); }
+.meta { color: var(--muted); }
+</style>
+${input.extraHead ?? ""}
+</head>
+<body>
+<h1>${escape(input.check)}: ${input.result}</h1>
+<p>${escape(summary)}.${input.reason === void 0 ? "" : ` ${escape(input.reason)}.`}</p>
+<p class="meta">Judged at commit <code>${escape(input.commit)}</code>.</p>
+<table>
+<thead><tr><th>Item</th><th>Result</th><th>Reason</th><th>Tests</th></tr></thead>
+<tbody>
+${rows}
+</tbody>
+</table>
+</body>
+</html>
+`;
 }
 
 // src/runner/judge.ts
@@ -2029,6 +2223,14 @@ function criteriaOf(workItem) {
   }
   return items.length === 0 ? void 0 : items;
 }
+function hostItemsOf(workItem, name) {
+  const checks = Object(workItem).checks;
+  if (!Array.isArray(checks)) return [];
+  const check = checks.find((entry) => Object(entry).name === name);
+  if (check?.source !== "host") return [];
+  const parsed = CheckItemsSchema.safeParse(check.items);
+  return parsed.success ? parsed.data : [];
+}
 function environmentFor(input) {
   const { config, job, jobDir, deps } = input;
   if (deps.judgeEnvironment !== void 0) return deps.judgeEnvironment;
@@ -2105,7 +2307,8 @@ async function runJudge(input) {
     if (environment === void 0) return await fail(check, `${NO_CLAUDE}: this build ran outside a container or microVM`);
     const expected = (job.container ?? job.microvm)?.judgeProgram;
     if (expected === void 0) return await fail(check, NO_CLAUDE);
-    if (check.judge.text === void 0) return await fail(check, `the brief ${check.judge.brief} is not in the definitions repository`);
+    const brief = check.judge.text;
+    if (brief === void 0) return await fail(check, `the brief ${check.judge.brief} is not in the definitions repository`);
     if (tokens === void 0) return await fail(check, "the build's repository tokens are no longer held, so the branch cannot be fetched");
     const credential = claudeCredential(runtime.hostEnv);
     if (credential === void 0) return await fail(check, "the host holds no Claude credential for the judge");
@@ -2125,17 +2328,34 @@ async function runJudge(input) {
     if (read.kind === "unreadable") return await fail(check, `the branch ${branch} could not be read: ${read.reason}`);
     const head = read.sha;
     let criteria;
-    try {
-      criteria = criteriaOf(await readWorkItem(config, job.item, deps.fetchImpl));
-    } catch (err) {
-      return await fail(check, `the item's criteria could not be read: ${describeCallFailure(err)}`);
+    let previous = [];
+    if (check.judge.subjects === "criteria") {
+      let workItem;
+      try {
+        workItem = await readWorkItem(config, job.item, deps.fetchImpl);
+      } catch (err) {
+        return await fail(check, `the item's criteria could not be read: ${describeCallFailure(err)}`);
+      }
+      criteria = criteriaOf(workItem);
+      if (criteria === void 0) return await fail(check, "no criteria are recorded for the item");
+      previous = hostItemsOf(workItem, check.name);
     }
-    if (criteria === void 0) return await fail(check, "no criteria are recorded for the item");
     const branchDir = join(scratch, "branch");
     const fetched = await (deps.fetchCommit ?? fetchCommit)({ remote, dir: branchDir, commit: head, token: readToken });
     if (!fetched.ok) return await fail(check, `the build's tests could not be fetched: ${fetched.reason}`);
     const finder = await loadCitationFinder();
-    const citations = finder.buildCitations({ ids: criteria.map((criterion) => criterion.id), files: testFilesIn(branchDir, finder.TEST_FILE), repository: repo, sha: head });
+    const list = criteria ?? [];
+    const testFiles = testFilesIn(branchDir, finder.TEST_FILE);
+    const citations = finder.buildCitations({ ids: list.map((criterion) => criterion.id), files: testFiles, repository: repo, sha: head });
+    const testFileText = new Map(testFiles.map((file) => [file.path, file.contents]));
+    const inputs = new Map(list.map((criterion) => [criterion.id, criterionInputs({ brief, criterion, citations, testFiles: testFileText })]));
+    const plan = planCriteria({ criteria: list, citations, inputs, previous });
+    if (plan.toJudge.length < list.length) {
+      log.line(`${tag} judge: ${plan.toJudge.length} of ${list.length} criteria to judge, ${list.length - plan.toJudge.length} settled without the judge`);
+    }
+    if (criteria !== void 0 && plan.toJudge.length === 0) {
+      return await finish(check, { criteria, items: plan.settled, citations, head, branch, finder, judgeReport: void 0 });
+    }
     let diff = "";
     const startCommit = (job.container ?? job.microvm)?.startCommit;
     if (startCommit !== void 0) {
@@ -2147,19 +2367,26 @@ async function runJudge(input) {
 [the diff was cut at ${MAX_DIFF_BYTES} bytes]
 ` : bytes.toString("utf8");
     }
-    await environment.prepare({
-      "criteria.json": `${JSON.stringify({ criteria }, null, 2)}
-`,
-      "citations.json": `${JSON.stringify(citations, null, 2)}
+    await environment.prepare(
+      {
+        ...criteria === void 0 ? {} : {
+          // Only the criteria the judge is asked about. The others are settled.
+          "criteria.json": `${JSON.stringify({ criteria: plan.toJudge }, null, 2)}
 `,
-      "diff.patch": diff
-    }, branchDir);
+          "citations.json": `${JSON.stringify(citations, null, 2)}
+`
+        },
+        "diff.patch": diff
+      },
+      branchDir
+    );
     const limitMinutes = config.checks?.timeLimitMinutes ?? DEFAULT_CHECK_TIME_LIMIT_MINUTES;
     const minutes = `${limitMinutes} ${limitMinutes === 1 ? "minute" : "minutes"}`;
     const clear = async () => {
       await environment.clear?.().catch((err) => log.error(`${tag} what the judge left in its folder could not be removed: ${describeCallFailure(err)}`));
     };
-    const running = environment.run(judgeCommand({ brief: check.judge.text, credential }), { logPath: join(jobDir, "judge.log") });
+    const prompt = withAnswerProtocol(brief, check.judge.subjects);
+    const running = environment.run(judgeCommand({ brief: prompt, credential }), { logPath: join(jobDir, "judge.log") });
     running.catch(() => void 0);
     const timer = timeoutAfter(limitMinutes * 6e4, deps.sleep);
     let outcome;
@@ -2172,27 +2399,86 @@ async function runJudge(input) {
       timer.cancel();
     }
     let report;
-    const readReport2 = async () => {
+    let answerText;
+    const readOutputs = async () => {
       try {
         report = await environment.readReport();
       } catch (err) {
         log.error(`${tag} the judge's report could not be read: ${describeCallFailure(err)}`);
       }
+      try {
+        answerText = await environment.readItems();
+      } catch (err) {
+        log.error(`${tag} the judge's items could not be read: ${describeCallFailure(err)}`);
+      }
     };
     if (outcome === "timeout") {
-      await readReport2();
+      await readOutputs();
       await environment.stop().catch((err) => log.error(`${tag} the judge could not be stopped after its time ran out: ${describeCallFailure(err)}`));
+    } else {
+      await readOutputs();
     }
-    const decision = decideVerdict({
-      run: outcome === "timeout" ? { kind: "timeout" } : { kind: "exited", status: outcome.status, stdout: outcome.stdout, stderr: outcome.stderr },
-      criteria,
-      citations,
-      minutes
-    });
-    if (outcome !== "timeout") await readReport2();
     await clear();
-    const link = report === void 0 ? void 0 : await uploadReport(check, report, { branch, head });
-    await record(check.name, decision.result, decision.reason, { keyTail: decision.limitKey === true ? LIMIT_KEY_TAIL2 : "", link });
+    const run = outcome === "timeout" ? { kind: "timeout" } : { kind: "exited", status: outcome.status, stdout: outcome.stdout, stderr: outcome.stderr };
+    const reply = readJudgeReply(run, minutes);
+    const answer = parseJudgeAnswer(answerText);
+    if (criteria === void 0) return await finishWhole(check, { reply, answer, report, head, branch });
+    const judged = (results2) => [
+      ...plan.settled,
+      ...judgedItems({ toJudge: plan.toJudge, results: results2, citations, inputs })
+    ];
+    if (answer?.ok === true) {
+      const results2 = new Map(answer.answer.items.map((entry) => [entry.id, entry]));
+      return await finish(check, { criteria, items: judged(results2), citations, head, branch, finder, judgeReport: void 0 });
+    }
+    if (reply.kind === "none") {
+      return await finish(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: reply.reason, limitKey: reply.limitKey === true });
+    }
+    if (answer !== void 0) {
+      return await finish(check, { criteria, items: judged(/* @__PURE__ */ new Map()), citations, head, branch, finder, judgeReport: report, reason: answer.reason });
+    }
+    const failsAll = reply.verdict === "fail" && reply.unmet.size === 0;
+    const results = new Map(plan.toJudge.map((criterion) => [criterion.id, { result: failsAll || reply.unmet.has(criterion.id) ? "fail" : "pass" }]));
+    return await finish(check, { criteria, items: judged(results), citations, head, branch, finder, judgeReport: report });
+  }
+  async function finish(check, at) {
+    const items = inCriteriaOrder(at.items, at.criteria);
+    const failing = itemsSentence(items, { one: "criterion", many: "criteria" });
+    const result = failing === void 0 ? "pass" : "fail";
+    const reason = at.reason ?? failing;
+    const report = at.judgeReport !== void 0 && at.judgeReport !== "" ? at.judgeReport : renderJudgeReport({ check: check.name, result, reason, items, commit: at.head, extraHead: at.finder.renderBlock(at.citations) });
+    const link = await uploadReport(check, report, { branch: at.branch, head: at.head });
+    await record(check.name, result, reason, { keyTail: at.limitKey === true ? LIMIT_KEY_TAIL2 : "", link, items: sendable(items) });
+  }
+  async function finishWhole(check, at) {
+    const where = { head: at.head, branch: at.branch };
+    if (at.answer?.ok === true) {
+      const items = at.answer.answer.items.map((entry) => {
+        const reason2 = oneLineReason(entry.reason);
+        return { id: entry.id, result: entry.result, ...reason2 === void 0 ? {} : { reason: reason2 } };
+      });
+      const failing = itemsSentence(items, { one: "check item", many: "check items" });
+      const said = at.answer.answer.result ?? (at.reply.kind === "answered" ? at.reply.verdict : void 0);
+      const result = said === "pass" && failing === void 0 ? "pass" : "fail";
+      const reason = failing ?? (said === "fail" ? "the judge failed the change" : said === void 0 ? at.reply.kind === "none" ? at.reply.reason : "the judge gave no result" : void 0);
+      const report = renderJudgeReport({ check: check.name, result, reason, items, commit: at.head });
+      return await recordWhole(check, { ...where, result, reason, items, report });
+    }
+    if (at.reply.kind === "none") return await recordWhole(check, { ...where, result: "fail", reason: at.reply.reason, report: at.report, limitKey: at.reply.limitKey === true });
+    if (at.answer !== void 0) return await recordWhole(check, { ...where, result: "fail", reason: at.answer.reason, report: at.report });
+    const failed = at.reply.verdict === "fail";
+    return await recordWhole(check, { ...where, result: at.reply.verdict, reason: failed ? "the judge failed the change" : void 0, report: at.report });
+  }
+  async function recordWhole(check, at) {
+    const link = at.report === void 0 ? void 0 : await uploadReport(check, at.report, { branch: at.branch, head: at.head });
+    await record(check.name, at.result, at.reason, { keyTail: at.limitKey === true ? LIMIT_KEY_TAIL2 : "", link, items: sendable(at.items) });
+  }
+  function sendable(items) {
+    if (items === void 0) return void 0;
+    const parsed = CheckItemsSchema.safeParse(items);
+    if (parsed.success) return parsed.data;
+    log.error(`${tag} the judge's check items were left off the result: ${parsed.error.issues.map((issue) => issue.message).join("; ")}`);
+    return void 0;
   }
   async function uploadReport(check, report, at) {
     const bytes = Buffer.from(report, "utf8");