efaimo 0.1.0 → 0.1.1

package/dist/rules/injection.js

@@ -93,7 +93,14 @@ export function scanTextForInjection(text, opts) {
                 // Deliberately info: these are shallow heuristics an attacker evades
                 // trivially. Surfacing them must never move a grade or fail CI, or a
                 // clean report would read as a security pass (it is not one).
+                //
+                // `graded: false` is what makes the first half of that true. It used
+                // to be a comment only, while gradeFindings charged every info finding
+                // a point, so ten heuristic hits quietly cost a whole letter -- the
+                // exact "clean report reads as a security verdict" failure this
+                // comment was written to prevent, running in reverse.
                 severity: "info",
+                graded: false,
                 title: "possible injection pattern (heuristic)",
                 message: `${opts.where}: ${p.label}`,
                 target: opts.where,

package/dist/rules/mcp/index.d.ts

@@ -1,4 +1,18 @@
 import type { McpRule } from "../../core/types.js";
+/**
+ * Does this tool name read as destructive?
+ *
+ * This was a single `\b(delete|remove|...)\b` regex, and `_` is a word
+ * character, so `\bdelete\b` has no boundary inside `delete_file` - nor inside
+ * `deleteFile`. Every conventional MCP tool name escaped it: delete_file,
+ * deleteFile, drop_table, removeUser, kill_process, wipe_db all returned
+ * false, while only a bare `delete` or `delete-file` matched. E123 is
+ * documented in docs/RULES.md as catching "a destructive-looking tool lacking
+ * destructiveHint" and it could not fire on essentially any real server.
+ *
+ * Split on separators AND on camelCase boundaries, then test each segment.
+ */
+export declare function readsDestructive(name: string): boolean;
 export declare const MCP_RULES: McpRule[];
 /** E101-E118 are 2026-07-28 readiness rules: reported as a migration diff, not graded. */
 export declare function isReadinessRuleId(id: string): boolean;

package/dist/rules/mcp/index.js

@@ -1,7 +1,8 @@
 import { scanTextForInjection } from "../injection.js";
 import { looksLikeInitGate } from "../../clients/rawprobe.js";
 import { formatTokens, truncate } from "../../util/misc.js";
-const RC = "2026-07-28";
+import { formatWindowShare } from "../../weigh/window.js";
+const SPEC = "2026-07-28";
 function isOutcome(x) {
     return !!x && typeof x === "object" && "ok" in x;
 }
@@ -25,7 +26,7 @@ const REPO_MATCH_RULES = [
         category: "sampling",
         severity: "warn",
         title: "uses deprecated Sampling",
-        message: (c) => `source references sampling/createMessage (${matchStr(c)}); Sampling is deprecated in ${RC} (SEP-2577)`,
+        message: (c) => `source references sampling/createMessage (${matchStr(c)}); Sampling is deprecated in ${SPEC} (SEP-2577)`,
         fixHint: "integrate directly with an LLM provider API instead of MCP Sampling",
     },
     {
@@ -33,7 +34,7 @@ const REPO_MATCH_RULES = [
         category: "roots",
         severity: "warn",
         title: "uses deprecated Roots",
-        message: (c) => `source references roots/list (${matchStr(c)}); Roots is deprecated in ${RC} (SEP-2577)`,
+        message: (c) => `source references roots/list (${matchStr(c)}); Roots is deprecated in ${SPEC} (SEP-2577)`,
         fixHint: "pass directories or files via tool parameters, resource URIs, or server configuration",
     },
     {
@@ -41,7 +42,7 @@ const REPO_MATCH_RULES = [
         category: "sse-resume",
         severity: "info",
         title: "relies on removed SSE resumability",
-        message: (c) => `source references Last-Event-ID/resumability (${matchStr(c)}); stream resumability is removed in ${RC}`,
+        message: (c) => `source references Last-Event-ID/resumability (${matchStr(c)}); stream resumability is removed in ${SPEC}`,
         fixHint: "persist cross-call state behind server-minted handles passed as tool arguments",
     },
     {
@@ -49,7 +50,7 @@ const REPO_MATCH_RULES = [
         category: "elicitation",
         severity: "warn",
         title: "uses legacy elicitation",
-        message: (c) => `source references elicitation/create (${matchStr(c)}); replaced in ${RC} by MRTR results (resultType "input_required" with inputRequests)`,
+        message: (c) => `source references elicitation/create (${matchStr(c)}); replaced in ${SPEC} by MRTR results (resultType "input_required" with inputRequests)`,
         fixHint: "return input_required results and correlate retries via requestState (SEP-2322)",
     },
     {
@@ -57,21 +58,21 @@ const REPO_MATCH_RULES = [
         category: "session-state",
         severity: "info",
         title: "possible in-process session state",
-        message: (c) => `source shows session-state patterns (${matchStr(c)}); ${RC} statelessness expects server-minted handles passed via tool arguments`,
+        message: (c) => `source shows session-state patterns (${matchStr(c)}); ${SPEC} statelessness expects server-minted handles passed via tool arguments`,
     },
     {
         id: "E114",
         category: "ping",
         severity: "info",
         title: "uses removed ping",
-        message: (c) => `source references the ping utility (${matchStr(c)}); ping is removed in ${RC}`,
+        message: (c) => `source references the ping utility (${matchStr(c)}); ping is removed in ${SPEC}`,
     },
     {
         id: "E115",
         category: "subscribe",
         severity: "info",
         title: "uses replaced resource subscriptions",
-        message: (c) => `source references resources/subscribe (${matchStr(c)}); replaced by subscriptions/listen in ${RC}`,
+        message: (c) => `source references resources/subscribe (${matchStr(c)}); replaced by subscriptions/listen in ${SPEC}`,
     },
 ];
 const repoMatchRules = REPO_MATCH_RULES.map((spec) => ({
@@ -105,10 +106,14 @@ const e101 = {
                 ruleId: "E101",
                 severity: "warn",
                 title: "legacy SDK generation",
-                message: `depends on ${s.package}${s.range ? `@${s.range}` : ""} (pre-${RC} line)`,
+                message: `depends on ${s.package}${s.range ? `@${s.range}` : ""} (pre-${SPEC} line)`,
                 detail: s.language === "ts"
-                    ? `the ${RC} revision ships as the new @modelcontextprotocol/server package (2.x beta since 2026-07)`
-                    : `the ${RC} revision ships as mcp 2.x (pre-releases on PyPI since 2026-06)`,
+                    ? // No maturity label here on purpose. This said "2.x beta since
+                        // 2026-07" and stayed saying it after both packages cut 2.0.0 on
+                        // 2026-07-27, which told people to migrate onto a beta that was
+                        // not one. The package name and major line are the durable facts.
+                        `the ${SPEC} revision ships as the new @modelcontextprotocol/server package (2.x)`
+                    : `the ${SPEC} revision ships as mcp 2.x on PyPI`,
                 fixHint: "upgrade to the 2.x SDK line to get stateless transport and MRTR support",
             });
         }
@@ -126,7 +131,7 @@ const e104 = {
                     ruleId: "E104",
                     severity: "warn",
                     title: "uses deprecated MCP Logging",
-                    message: `server declares the logging capability; MCP Logging is deprecated in ${RC} (SEP-2577) and logging/setLevel is removed`,
+                    message: `server declares the logging capability; MCP Logging is deprecated in ${SPEC} (SEP-2577) and logging/setLevel is removed`,
                     fixHint: "log to stderr (stdio) or use OpenTelemetry; per-request level arrives via _meta io.modelcontextprotocol/logLevel",
                 },
             ];
@@ -135,7 +140,7 @@ const e104 = {
             ruleId: "E104",
             severity: "warn",
             title: "uses deprecated MCP Logging",
-            message: `source references MCP logging APIs (${matchStr(count)}); deprecated in ${RC} (SEP-2577)`,
+            message: `source references MCP logging APIs (${matchStr(count)}); deprecated in ${SPEC} (SEP-2577)`,
             detail,
             fixHint: "log to stderr (stdio) or use OpenTelemetry",
         }));
@@ -156,7 +161,7 @@ const e105 = {
                     severity: "warn",
                     title: "requires the removed initialize handshake",
                     message: `the server rejected a bare stateless tools/list as not initialized (${bare.errorMessage ?? `code ${bare.errorCode}`})`,
-                    detail: `${RC} removes initialize and sessions; RC servers answer bare requests carrying version info in _meta. Whether an answering server is RC-conformant is judged separately (E107 resultType, E118 cache fields, E106 server/discover).`,
+                    detail: `${SPEC} removes initialize and sessions; stateless servers answer bare requests carrying version info in _meta. Whether an answering server is fully ${SPEC}-conformant is judged separately (E107 resultType, E118 cache fields, E106 server/discover).`,
                     fixHint: "upgrade to a 2.x SDK, or accept requests without a prior initialize",
                 },
             ];
@@ -189,7 +194,7 @@ const e106 = {
                 severity: "warn",
                 title: "server/discover not implemented",
                 message: `server/discover is not implemented (${d.errorMessage ?? "method not found"})`,
-                detail: `${RC} servers MUST implement server/discover (SEP-2575) to advertise versions, capabilities, and identity; clients also use it as the back-compat probe.`,
+                detail: `${SPEC} servers MUST implement server/discover (SEP-2575) to advertise versions, capabilities, and identity; clients also use it as the back-compat probe.`,
                 fixHint: "the 2.x SDKs implement server/discover for you",
             },
         ];
@@ -207,8 +212,8 @@ const e107 = {
                 ruleId: "E107",
                 severity: "info",
                 title: "results missing resultType",
-                message: `results do not carry the resultType field required in ${RC} ("complete" | "input_required")`,
-                detail: "the RC requires resultType on every result (SEP-2322); on list results the value must be \"complete\". RC clients treat missing resultType from earlier-protocol servers as \"complete\", so this is informational until you upgrade.",
+                message: `results do not carry the resultType field required in ${SPEC} ("complete" | "input_required")`,
+                detail: `${SPEC} requires resultType on every result (SEP-2322); on list results the value must be "complete". ${SPEC} clients treat missing resultType from earlier-protocol servers as "complete", so this is informational until you upgrade.`,
             },
         ];
     },
@@ -256,7 +261,7 @@ const e112 = {
                 ruleId: "E112",
                 severity: "warn",
                 title: "nondeterministic tools/list order",
-                message: "tool order differed across two fresh connections; deterministic ordering enables prompt-cache hits in RC hosts",
+                message: "tool order differed across two fresh connections; deterministic ordering enables prompt-cache hits in stateless hosts",
                 fixHint: "sort tools stably before returning them",
             },
         ];
@@ -322,14 +327,26 @@ const e118 = {
     title: "missing cache fields (ttlMs, cacheScope)",
     surface: "mcp",
     check(ctx) {
-        if (ctx.probes?.cacheFieldsPresent !== false)
+        // Two measured surfaces, not one. server/discover only became cacheable in
+        // the spec that published on 2026-07-28 (DiscoverResult extends
+        // CacheableResult); under the locked RC it extended plain Result, so a
+        // server that migrated against the RC can pass tools/list and still miss
+        // this. Undefined means the surface was never measured, which is not a
+        // finding: E106 already owns "server/discover did not answer".
+        const surfaces = [];
+        if (ctx.probes?.cacheFieldsPresent === false)
+            surfaces.push("tools/list");
+        if (ctx.probes?.discoverCacheFieldsPresent === false)
+            surfaces.push("server/discover");
+        if (!surfaces.length)
             return [];
+        const subject = surfaces.length > 1 ? `${surfaces.join(" and ")} results omit` : `${surfaces[0]} result omits`;
         return [
             {
                 ruleId: "E118",
                 severity: "warn",
                 title: "missing cache fields (ttlMs, cacheScope)",
-                message: `tools/list result omits ttlMs and/or cacheScope, which ${RC} requires on list and resource-read results (SEP-2549, CacheableResult)`,
+                message: `${subject} ttlMs and/or cacheScope, which ${SPEC} requires on list and resource-read results (SEP-2549, CacheableResult)`,
                 detail: "required on tools/list, prompts/list, resources/list, resources/read, resources/templates/list, and server/discover; cacheScope is \"public\" or \"private\"",
                 fixHint: "return ttlMs and cacheScope on these results so clients can cache and stop polling; the 2.x SDKs add them for you",
             },
@@ -421,7 +438,34 @@ const e122 = {
         return capList(findings, 5).shown;
     },
 };
-const DESTRUCTIVE_NAME_RE = /\b(delete|remove|drop|purge|destroy|overwrite|truncate|erase|wipe|revoke|reset|format|deploy|kill)\b/i;
+// "format" was in this list and is dropped. It only ever meant "format a
+// disk" in the author's head, but `format_date`, `format_output` and
+// `format_json` are among the most common tool names there are, and once the
+// matcher below started working the word turned into pure noise. Nobody could
+// have found that while the rule was firing on nothing.
+const DESTRUCTIVE_WORDS = new Set([
+    "delete", "remove", "drop", "purge", "destroy", "overwrite",
+    "truncate", "erase", "wipe", "revoke", "reset", "deploy", "kill",
+]);
+/**
+ * Does this tool name read as destructive?
+ *
+ * This was a single `\b(delete|remove|...)\b` regex, and `_` is a word
+ * character, so `\bdelete\b` has no boundary inside `delete_file` - nor inside
+ * `deleteFile`. Every conventional MCP tool name escaped it: delete_file,
+ * deleteFile, drop_table, removeUser, kill_process, wipe_db all returned
+ * false, while only a bare `delete` or `delete-file` matched. E123 is
+ * documented in docs/RULES.md as catching "a destructive-looking tool lacking
+ * destructiveHint" and it could not fire on essentially any real server.
+ *
+ * Split on separators AND on camelCase boundaries, then test each segment.
+ */
+export function readsDestructive(name) {
+    return name
+        .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+        .split(/[^A-Za-z0-9]+/)
+        .some((seg) => DESTRUCTIVE_WORDS.has(seg.toLowerCase()));
+}
 const e123 = {
     id: "E123",
     title: "missing tool annotations",
@@ -441,7 +485,7 @@ const e123 = {
                 fixHint: "annotations let hosts gate confirmations and parallelize safely",
             });
         }
-        const risky = tools.filter((t) => DESTRUCTIVE_NAME_RE.test(t.name) && !(t.annotations && "destructiveHint" in t.annotations));
+        const risky = tools.filter((t) => readsDestructive(t.name) && !(t.annotations && "destructiveHint" in t.annotations));
         for (const t of capList(risky, 5).shown) {
             findings.push({
                 ruleId: "E123",
@@ -586,7 +630,7 @@ const e128 = {
                     ruleId: "E128",
                     severity: "warn",
                     title: "total context tax",
-                    message: `tool definitions alone are ~${formatTokens(total)} tokens (~${((total / 200000) * 100).toFixed(1)}% of a 200k window, estimated) before any work happens`,
+                    message: `tool definitions alone are ~${formatTokens(total)} tokens (${formatWindowShare(total)}, estimated) before any work happens`,
                     fixHint: "trim descriptions/schemas, split the server, or rely on deferred tool loading",
                 },
             ];
@@ -597,7 +641,7 @@ const e128 = {
                     ruleId: "E128",
                     severity: "warn",
                     title: "total context tax",
-                    message: `tool definitions are ~${formatTokens(total)} tokens (~${((total / 200000) * 100).toFixed(1)}% of a 200k window)`,
+                    message: `tool definitions are ~${formatTokens(total)} tokens (${formatWindowShare(total)})`,
                     fixHint: "run \`efaimo weigh\` for the per-tool breakdown",
                 },
             ];

package/dist/skills/parse.js

@@ -1,7 +1,17 @@
 import fs from "node:fs";
 import path from "node:path";
 import YAML from "yaml";
-import { walkFiles } from "../util/misc.js";
+import { walkFiles, readTextSafe } from "../util/misc.js";
+/**
+ * Cap on a single SKILL.md. Referenced files were already capped at 512KB by
+ * readTextSafe while the body itself went through an uncapped readFileSync:
+ * the safer path had the limit and the main one did not. A 35MB SKILL.md
+ * tokenized to 7.2 million tokens at 367MB resident, and auditing skills you
+ * have NOT installed yet is the stated use case, so this is untrusted input
+ * by construction. 2MB is far above any real skill (the largest in the graded
+ * corpus is a few tens of KB) and far below a memory problem.
+ */
+const MAX_SKILL_BYTES = 2 * 1024 * 1024;
 const MD_LINK_RE = /\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
 // Only backticked strings that look like a real intra-skill path: a directory
 // separator, no glob star. A bare filename in prose (`package.json`, `*.py`) is
@@ -21,7 +31,15 @@ export function parseSkillFile(file) {
     };
     let raw;
     try {
-        raw = fs.readFileSync(file, "utf8");
+        // Capped, like the referenced files this same module already caps at 512KB
+        // via readTextSafe. The body went through plain readFileSync with no limit
+        // while the SAFER path had a cap, and the stated use case is auditing
+        // skills BEFORE you install them, which is untrusted input by definition.
+        // A 35MB SKILL.md tokenized to 7.2M tokens and 367MB resident.
+        const capped = readTextSafe(file, MAX_SKILL_BYTES);
+        if (capped === undefined)
+            throw new Error("unreadable");
+        raw = capped;
     }
     catch (e) {
         return { ...base, parseError: `cannot read file: ${e.message}` };
@@ -79,7 +97,25 @@ export function parseSkillFile(file) {
                 continue;
             seen.add(cleaned);
             const resolved = path.resolve(dir, cleaned);
-            base.referencedPaths.push({ raw: cleaned, resolved, exists: fs.existsSync(resolved), source });
+            // Does this reference stay inside the skill? Absolute paths, URLs and
+            // globs are already rejected above; `../` was not, and weigh.ts then
+            // READ every referenced file to count its tokens. A skill fetched from
+            // anywhere could point at ../../../.ssh/id_rsa and get back an
+            // existence-and-size oracle, published if --md goes to a CI job summary.
+            // S106 already knew this was wrong and warned about it - but only after
+            // the read had happened, and `weigh` does not run S106 at all.
+            //
+            // Recorded here rather than filtered, so S106 can still report it and
+            // the reader still learns the reference exists. weigh skips the read.
+            const rel = path.relative(dir, resolved);
+            const escapes = rel.startsWith("..") || path.isAbsolute(rel);
+            base.referencedPaths.push({
+                raw: cleaned,
+                resolved,
+                exists: fs.existsSync(resolved),
+                escapes,
+                source,
+            });
         }
     }
     for (const f of walkFiles(dir, { maxDepth: 4 })) {

package/dist/targets/clientConfigs.d.ts

@@ -7,6 +7,13 @@ export interface ClientServerEntry {
 }
 export interface ClientConfigResult {
     entries: ClientServerEntry[];
+    /**
+     * Config files that came from the CURRENT DIRECTORY rather than the user
+     * config. Every command in these was written by whoever wrote the
+     * repository you are standing in, and --client spawns them. The CLI warns
+     * about these by name before it executes anything.
+     */
+    fromCwd: string[];
     /** Config files that were found and parsed. */
     sources: string[];
     /** Paths that were checked but absent. */

package/dist/targets/clientConfigs.js

@@ -27,7 +27,7 @@ export function loadClientServers(client, cwd = process.cwd()) {
         default:
             throw new Error(`unknown client "${client}" (supported: ${SUPPORTED_CLIENTS.join(", ")})`);
     }
-    const result = { entries: [], sources: [], missing: [] };
+    const result = { entries: [], sources: [], missing: [], fromCwd: [] };
     const seen = new Set();
     for (const { file, pick } of candidates) {
         if (!fs.existsSync(file)) {
@@ -48,6 +48,9 @@ export function loadClientServers(client, cwd = process.cwd()) {
             continue;
         }
         result.sources.push(file);
+        // Project-scoped config is the untrusted kind: it ships with a repo.
+        if (file.startsWith(cwd))
+            result.fromCwd.push(file);
         for (const [name, valueRaw] of Object.entries(servers)) {
             if (seen.has(name))
                 continue;

package/dist/testing/anthropicRunner.js

@@ -13,6 +13,8 @@ export function anthropicRunner(apiKey) {
             },
             body: JSON.stringify({
                 model: req.model,
+                // 0 for the judge, 1 for the subject. See RunnerRequest.deterministic.
+                temperature: req.deterministic ? 0 : 1,
                 max_tokens: 1024,
                 ...(req.system ? { system: req.system } : {}),
                 messages: [{ role: "user", content: req.user }],

package/dist/testing/harness.d.ts

@@ -18,10 +18,24 @@ export interface RunnerRequest {
     system?: string;
     user: string;
     model: string;
+    /**
+     * Sample deterministically. Set for the JUDGE, never for the subject.
+     * Neither runner set a temperature, so both defaulted to 1.0 and a "strict
+     * grader" asked for one word was being sampled: the same answer could be
+     * graded PASS on one trial and FAIL on the next, and that variance lands in
+     * the measurement as if it came from the skill. The subject arm keeps
+     * temperature 1 on purpose, because the thing being measured is how the
+     * model behaves normally.
+     */
+    deterministic?: boolean;
 }
 export type Runner = (req: RunnerRequest) => Promise<string>;
 export interface ArmResult {
     trials: number;
+    /** Trials the judge actually answered PASS or FAIL on. */
+    scored: number;
+    /** Trials whose judge verdict was neither; excluded from passRate. */
+    unparseable: number;
     passes: number;
     passRate: number;
 }
@@ -33,6 +47,13 @@ export interface TestReport {
     withoutSkill: ArmResult;
     /** withSkill.passRate - withoutSkill.passRate, in percentage points. */
     deltaPoints: number;
+    /** Two-sided Fisher exact p on the 2x2 of scored trials. */
+    p: number;
+    /** 95% interval on the delta, in percentage points (Newcombe). */
+    ci: {
+        lo: number;
+        hi: number;
+    };
     verdict: "helps" | "hurts" | "no measurable effect" | "inconclusive";
     notes: string[];
 }

package/dist/testing/harness.js

@@ -1,3 +1,4 @@
+import { fisherExactTwoSided, deltaInterval } from "./stats.js";
 import fs from "node:fs";
 import path from "node:path";
 import YAML from "yaml";
@@ -24,7 +25,10 @@ export function parseScenario(file) {
     const skill = set.skills[0];
     if (!skill)
         throw new Error(`${file}: no SKILL.md found at '${skillRel}'`);
-    const trials = Math.min(50, Math.max(1, Math.round(num(doc.trials) ?? 5)));
+    // Default 20, not 5. At 5 per arm a total separation (5/5 vs 0/5) is the ONLY
+    // outcome that reaches p < 0.05, so every partial result is uninterpretable
+    // and the old default guaranteed a verdict nobody could stand behind.
+    const trials = Math.min(50, Math.max(1, Math.round(num(doc.trials) ?? 20)));
     return {
         name,
         skillPath,
@@ -47,42 +51,87 @@ export function armSystems(scenario) {
     const withSkill = `You have access to the following skill. Use it if relevant.\n\n<skill name="${scenario.skillName}">\n${scenario.skillBody}\n</skill>`;
     return { withoutSkill: undefined, withSkill };
 }
+// PASS, FAIL, or neither. "Neither" used to be counted as FAIL: the old test
+// was `/pass/i.test(out) && !/fail/i.test(out)`, so a refusal, an API
+// error string, or a hedge like "passes on X but fails on Y" all became a
+// failed trial. Those are not evidence about the skill, and quietly scoring
+// them against whichever arm produced them biases the result.
 async function judgeOne(runner, scenario, answer) {
     const out = await runner({
         system: JUDGE_SYSTEM,
         user: `TASK:\n${scenario.task}\n\nRUBRIC:\n${scenario.judge}\n\nASSISTANT ANSWER:\n${answer}\n\nVerdict (PASS or FAIL):`,
         model: scenario.model,
+        deterministic: true,
     });
-    return /\bpass\b/i.test(out) && !/\bfail\b/i.test(out);
+    const saysPass = /\bpass(ed|es)?\b/i.test(out);
+    const saysFail = /\bfail(ed|s|ure)?\b/i.test(out);
+    if (saysPass && !saysFail)
+        return "pass";
+    if (saysFail && !saysPass)
+        return "fail";
+    return "unparseable";
 }
 async function runArm(runner, scenario, system) {
     let passes = 0;
+    let unparseable = 0;
     for (let i = 0; i < scenario.trials; i++) {
         const answer = await runner({ system, user: scenario.task, model: scenario.model });
-        if (await judgeOne(runner, scenario, answer))
+        const v = await judgeOne(runner, scenario, answer);
+        if (v === "pass")
             passes++;
+        else if (v === "unparseable")
+            unparseable++;
     }
-    return { trials: scenario.trials, passes, passRate: (passes / scenario.trials) * 100 };
+    // Scored trials exclude the ones the judge did not answer, so the rate is a
+    // rate over evidence rather than over attempts.
+    const scored = scenario.trials - unparseable;
+    return {
+        trials: scenario.trials,
+        scored,
+        unparseable,
+        passes,
+        passRate: scored > 0 ? (passes / scored) * 100 : 0,
+    };
 }
 export async function runScenario(scenario, runner) {
     const systems = armSystems(scenario);
     const withoutSkill = await runArm(runner, scenario, systems.withoutSkill);
     const withSkill = await runArm(runner, scenario, systems.withSkill);
     const deltaPoints = Math.round((withSkill.passRate - withoutSkill.passRate) * 10) / 10;
+    // The verdict is a significance test, not a threshold on the gap.
+    //
+    // It used to be `>= +15 points helps, <= -15 hurts`, with 5 trials per arm
+    // as the default. At that size 5/5 against 4/5 is +20 points and a two-sided
+    // Fisher p of 1.0000: the least significant result obtainable, reported as a
+    // finding, and `hurts` set exit code 1. The +-15 band sits inside the noise
+    // floor at every trial count below roughly 25 per arm, so the threshold was
+    // measuring sample size more than it was measuring the skill.
+    const p = fisherExactTwoSided(withSkill.passes, withSkill.scored, withoutSkill.passes, withoutSkill.scored);
+    const ci = deltaInterval(withSkill.passes, withSkill.scored, withoutSkill.passes, withoutSkill.scored);
     const notes = [
-        `${scenario.trials} trials per arm; probabilistic. Treat small deltas as noise and raise trials for confidence.`,
+        `${scenario.trials} trials per arm. Two-sided Fisher exact p = ${p < 0.0001 ? "<0.0001" : p.toFixed(4)}; ` +
+            `95% interval on the delta ${ci.lo >= 0 ? "+" : ""}${ci.lo} to ${ci.hi >= 0 ? "+" : ""}${ci.hi} points.`,
     ];
+    const unscored = withSkill.unparseable + withoutSkill.unparseable;
+    if (unscored) {
+        notes.push(`${unscored} trial(s) produced a judge verdict that was neither PASS nor FAIL and are excluded from the rates, not counted as failures.`);
+    }
     let verdict;
-    if (scenario.trials < 5) {
+    if (withSkill.scored === 0 || withoutSkill.scored === 0) {
         verdict = "inconclusive";
-        notes.push("fewer than 5 trials: not enough signal to conclude.");
+        notes.push("an arm produced no scoreable trial; there is nothing to compare.");
+    }
+    else if (p >= 0.05) {
+        verdict = "no measurable effect";
+        if (Math.abs(deltaPoints) >= 15) {
+            notes.push(`the ${deltaPoints >= 0 ? "+" : ""}${deltaPoints} point gap is not significant at this sample size (p = ${p.toFixed(4)}). ` +
+                `Raise trials rather than reading the gap.`);
+        }
     }
-    else if (deltaPoints >= 15)
+    else if (deltaPoints > 0)
         verdict = "helps";
-    else if (deltaPoints <= -15)
-        verdict = "hurts";
     else
-        verdict = "no measurable effect";
+        verdict = "hurts";
     return {
         scenario: scenario.name,
         skill: scenario.skillName,
@@ -90,6 +139,8 @@ export async function runScenario(scenario, runner) {
         withSkill,
         withoutSkill,
         deltaPoints,
+        p,
+        ci,
         verdict,
         notes,
     };

package/dist/testing/openaiRunner.js

@@ -14,7 +14,14 @@ export function openaiRunner(apiKey) {
                 "content-type": "application/json",
                 authorization: `Bearer ${apiKey}`,
             },
-            body: JSON.stringify({ model: req.model, messages, max_completion_tokens: 1024 }),
+            // temperature: 0 for the judge, 1 for the subject. See
+            // RunnerRequest.deterministic in harness.ts.
+            body: JSON.stringify({
+                model: req.model,
+                temperature: req.deterministic ? 0 : 1,
+                messages,
+                max_completion_tokens: 1024,
+            }),
             redirect: "error",
             signal: AbortSignal.timeout(60000),
         });

package/dist/testing/stats.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Two-sided Fisher exact test on
+ *   [withPass, withFail]
+ *   [withoutPass, withoutFail]
+ *
+ * Sums the probability of every table at least as extreme as the observed one,
+ * which is the standard two-sided construction and needs no normal
+ * approximation.
+ */
+export declare function fisherExactTwoSided(withPass: number, withTrials: number, withoutPass: number, withoutTrials: number): number;
+/**
+ * Wilson score interval for one proportion. Chosen over the normal-approximation
+ * interval because that one produces a zero-width interval at 0/n and n/n,
+ * which is exactly where these runs land (8/8 and 0/8 both occur in the two
+ * committed example runs).
+ */
+export declare function wilson(passes: number, trials: number, z?: number): {
+    lo: number;
+    hi: number;
+};
+/**
+ * Interval on the DIFFERENCE of two proportions, in percentage points, by the
+ * Newcombe method: combine each arm's Wilson interval. Reported so a reader can
+ * see how wide the uncertainty actually is instead of inferring it from a
+ * point estimate.
+ */
+export declare function deltaInterval(withPass: number, withTrials: number, withoutPass: number, withoutTrials: number): {
+    lo: number;
+    hi: number;
+};
+/** The smallest per-arm trial count at which a total separation (n/n vs 0/n) reaches p < 0.05. */
+export declare const MIN_TRIALS_FOR_SIGNIFICANCE = 4;