efaimo 0.1.2 → 0.2.0

package/CHANGELOG.md

@@ -4,6 +4,67 @@ All notable changes to efaimo are documented here. Format follows
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this project uses
 [Semantic Versioning](https://semver.org/).
 
+## [0.2.0] - 2026-09-02
+
+### Added
+
+- **`efaimo find`**: does anything actually surface these tools? A host can
+  mark tools `defer_loading: true`, which keeps their definitions out of the
+  context window until a search returns them, and Anthropic recommends that
+  once definitions pass ~10k tokens. Under deferral a tool nothing surfaces
+  costs no context and provides no capability. Two numbers, offline and
+  deterministic, no API key:
+  - `distinct`, the headline: how many tools own a term no other tool in the
+    catalog has. A tool with none cannot be matched by any query that does not
+    also match a competitor, which follows from the index rather than from a
+    model of how anyone searches. Measured on `@playwright/mcp` 0.0.78: 22 of
+    24, with `browser_close` and `browser_navigate` owning nothing.
+  - `probe`, secondary and labelled: a simulated BM25 search over the four
+    fields the tool search documentation names as searchable. It reads 100% on
+    both the official reference server and playwright, so the output says on
+    the page that it is a floor test and not a ranking.
+  - New ungraded rule family E141-E145, `--min-distinct` as the CI gate,
+    `--top` to change the simulated result window, `--json` / `--md`.
+    Reasoning in ADR-030; method and its limits in docs/METHODOLOGY.md.
+- `rulesVersion` in every JSON envelope, because the tool version does not
+  identify a ruleset: a patch release can change what a rule fires on, and a
+  published grade is only reproducible next to the rules that produced it. The
+  rule inventory is pinned by a test (ADR-032).
+- `--judge-model` and scenario key `judge_model` for `efaimo test`, so the
+  model under test no longer has to grade its own answers. Subject and judge
+  are routed independently and may be different providers.
+- `--no-timestamp` on `weigh`, `check` and `find`.
+
+### Fixed
+
+- **`efaimo test --live` could not run on its own default model.** The runner
+  sent `temperature` on every request, and Claude removed sampling parameters
+  from the 4.7 line onward, so `claude-sonnet-5` rejected the first call with a
+  400 that is correctly not retried. Sampling parameters now go only to models
+  that still accept them, the request body is a pure function with a test
+  against it, and where the judge cannot be pinned the report says so instead
+  of implying a determinism it does not have. ADR-031.
+- `efaimo test` reports a Fisher exact p on the **unparseable** counts as well
+  as on the passes, and calls a run `inconclusive` when the excluded trials are
+  skewed across the arms: two pass rates computed over different populations
+  cannot be subtracted.
+- `--anthropic` names the model it measured against, in the output and in the
+  JSON. Claude model lines do not share a tokenizer, so a count without its
+  model cannot be reproduced. This is the rule `--window` already followed for
+  the context-window denominator.
+- `weigh --out` no longer writes `generatedAt` into the baseline. A baseline
+  exists to be compared against, and a field that differs on every write is the
+  one field a comparison must ignore.
+
+### Changed
+
+- docs/RULES.md now cites Anthropic's tool search documentation for E125's
+  lower threshold, which previously carried no source at all (the MCP quality
+  table has never had a source column). The thresholds themselves are
+  unchanged, and the citation covers the 30 boundary only: Anthropic writes
+  that tool selection "degrades once you exceed 30-50 available tools", which
+  says nothing about the 60 boundary, and the table says so.
+
 ## [0.1.2] - 2026-08-03
 
 The 0.1.2 code backlog: edge-case fixes surfaced by the five-agent review, none
@@ -197,7 +258,8 @@ First release.
   Documented rule set (`docs/RULES.md`), token methodology (`docs/METHODOLOGY.md`),
   and integration guide (`docs/INTEGRATIONS.md`).
 
-[unreleased]: https://github.com/efaimo-ai/efaimo/compare/v0.1.2...HEAD
+[unreleased]: https://github.com/efaimo-ai/efaimo/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/efaimo-ai/efaimo/releases/tag/v0.2.0
 [0.1.2]: https://github.com/efaimo-ai/efaimo/releases/tag/v0.1.2
 [0.1.1]: https://github.com/efaimo-ai/efaimo/releases/tag/v0.1.1
 [0.1.0]: https://github.com/efaimo-ai/efaimo/releases/tag/v0.1.0

package/README.md

@@ -176,6 +176,56 @@ grade A (95)   quality: 0 errors  1 warning  0 info
 
 </details>
 
+### Can a search even single your tools out?
+
+A host can mark tools `defer_loading: true`, which keeps their definitions out of
+the context window until a search returns them; Anthropic recommends turning that
+on once tool definitions pass ~10k tokens. Under deferral, a tool nothing
+surfaces costs no context and provides no capability, and neither `weigh` nor
+`check` looks at that.
+
+```bash
+npx efaimo find "npx -y my-mcp-server"
+```
+
+Two numbers come back, and they are different kinds of claim.
+
+**`distinct`** is a property of your catalog: how many tools own a term no other
+tool has. A tool that owns none cannot be matched by any query that does not also
+match a competitor. That follows from the index, not from a model of how anyone
+searches, and it is the number `--min-distinct` gates on in CI.
+
+```text
+distinct  22/24 (91.7%)   tools that own a word no other tool has
+probe     24/24 (100%)    returned by a simulated search for their own description
+
+  words  tool                     vocabulary no other tool has
+      0  browser_close            none: every word also on browser_click, browser_drag, browser_drop
+      0  browser_navigate         none: every word also on browser_tabs, browser_navigate_back, browser_network_requests
+      1  browser_evaluate         evaluate
+      1  browser_hover            hover
+      2  browser_resize           window, resize
+```
+
+*(That is `@playwright/mcp` 0.0.78, a well-maintained server, verbatim. Two of
+its twenty-four tools have no word of their own, and two more own only the word
+in their own name, which nobody describing a task would type.)*
+
+Four public servers measured this way, with the commands that reproduce each
+row, are in [the findability report](./research/findability/REPORT.md). The
+weakest was Notion's official server at 16 of 24, where `API-get-user` and
+`API-get-users` are different tools and nothing in their vocabulary separates
+them.
+
+**`probe`** is a simulated BM25 search over the four fields tool search matches
+on, and it is labelled a simulation because it is one. It reads 100% on both the
+official reference server and playwright, so it is a floor test that catches an
+empty description or a literal duplicate, not a ranking. `find` says that on
+every run rather than letting a saturated number look like a good score.
+
+There is no findability grade, on purpose:
+[docs/METHODOLOGY.md](./docs/METHODOLOGY.md) has the method, its limits, and why.
+
 ## From an agent
 
 `efaimo mcp` runs efaimo as a small, read-only MCP server, so an agent can lint or
@@ -252,8 +302,16 @@ And a mitigation you cannot measure is a mitigation you are trusting on faith:
 `weigh` reports the definitions your server actually ships, which is the input
 every mitigation operates on.
 
-If deferred loading covers your whole setup, you may not need this tool. Run
-`weigh` once and find out; that answer is worth more than our opinion.
+**And there is a fourth reason we did not have when this section was written.** Deferring a tool does not remove the question, it replaces
+it: a deferred tool that no search returns costs nothing and does nothing, so
+the thing worth measuring becomes whether each tool can be picked out of the
+catalog at all. `efaimo find` measures that, and on four public servers it
+found two to eight tools apiece that own no word their neighbours lack.
+Deferral is what makes findability matter, not what makes it moot.
+
+If deferred loading covers your whole setup, run `weigh` and `find` once and
+see what it costs you and what it hides; those answers are worth more than our
+opinion.
 
 ## Install
 
@@ -296,6 +354,7 @@ More recipes (pre-commit, GitLab, editor audit, programmatic use):
 | **S101 to S106** | skills: frontmatter and trigger quality, trigger collisions, context budget, reference integrity, injection hygiene |
 | **E101 to E118** | MCP 2026-07-28 readiness: deprecated primitives, statelessness, `server/discover`, `resultType`, cache fields, transport |
 | **E121 to E128, E130** | MCP quality: description quality, annotations, schema hygiene, tool-count and token-cost budgets |
+| **E141 to E146** | MCP findability: exclusive vocabulary, whether a search returns the tool, indistinguishable pairs, names that carry no domain word |
 
 `E000` is not a finding about your server: it means one of efaimo's own rules
 threw and was skipped, so the report is incomplete. It is never scored.
@@ -303,15 +362,18 @@ threw and was skipped, so the report is incomplete. It is never scored.
 Quality (E12x-E13x) and skill (S) findings set the letter grade; readiness
 findings (E101-E118) are reported as an ungraded migration diff, which
 `--strict-readiness` can turn into a non-zero exit
-without touching the grade.
+without touching the grade. Findability findings (E14x) belong to `efaimo find`
+and never touch a grade either.
 Every finding carries a stable id you can suppress or link. See
 [docs/RULES.md](./docs/RULES.md).
 
 ## Roadmap
 
-- Harden `efaimo test`: a separately chosen judge model, confidence intervals on
-  the delta, multi-turn tool-use trials, and judge calibration, so the
-  experimental harness earns unqualified trust.
+- Harden `efaimo test`: a length-matched placebo arm (the control currently gets
+  no system prompt at all, which confounds the skill's content with having any
+  system prompt), multi-turn tool-use trials, and judge calibration, so the
+  experimental harness earns unqualified trust. A separately chosen judge model
+  and confidence intervals on the delta are done.
 - Readiness rules for the two 2026-07-28 changes that have none yet: the
   `Mcp-Method` / `Mcp-Name` request headers (SEP-2243), and Tasks moving out of
   core into the `io.modelcontextprotocol/tasks` extension (SEP-2663).

package/SKILL.md

@@ -1,9 +1,9 @@
 ---
 name: efaimo
-description: Audit MCP servers and Agent Skills for quality and context-window cost. Use when the user wants to check an MCP server's token cost, lint a skill, verify 2026-07-28 MCP spec readiness, or add an efaimo badge or CI gate.
+description: Audit MCP servers and Agent Skills for quality and context-window cost. Use when the user wants to check an MCP server's token cost, lint a skill, verify 2026-07-28 MCP spec readiness, find out whether a search can single out a server's tools under deferred tool loading, or add an efaimo badge or CI gate.
 license: Apache-2.0
 metadata:
-  version: "0.1.2"
+  version: "0.2.0"
   homepage: "https://efaimo.ai"
 ---
 
@@ -49,6 +49,11 @@ and this file is where agents take orders.
 - **Lint a skill**
   `npx efaimo check --skill ./skills/` validates frontmatter, trigger quality,
   context budget, file references, and injection patterns.
+- **Check whether a search can single a tool out**
+  `npx efaimo find "npx -y <server>"` reports how many tools own a term no
+  other tool in the catalog has. A tool that owns none cannot be matched by any
+  query that does not also match a competitor, which matters once a host defers
+  tool loading. `--min-distinct 100` is the CI gate.
 - **Gate a pull request on context cost**
   Save a baseline with `npx efaimo weigh "<server>" --out base.json`, then in CI
   run `npx efaimo weigh "<server>" --diff base.json --allow-increase 10` to fail
@@ -56,11 +61,13 @@ and this file is where agents take orders.
 
 ## Reading the output
 
-Findings carry a stable rule id (E1xx MCP readiness, E12x-E13x MCP quality, S1xx
-skills) and a severity. The letter grade covers quality and skill findings only;
-2026-07-28 readiness items appear separately as an ungraded migration diff. Token
-numbers are o200k estimates unless `--anthropic` is used; see the methodology doc
-for the full method.
+Findings carry a stable rule id (E1xx MCP readiness, E12x-E13x MCP quality,
+E14x findability, S1xx skills) and a severity. The letter grade
+covers quality and skill findings only; 2026-07-28 readiness items appear
+separately as an ungraded migration diff, and findability is a
+separate command with no grade at all. Token numbers are o200k estimates unless
+`--anthropic` is used, which names the model it measured against; see the
+methodology doc for the full method.
 
 ## Notes
 

package/dist/cli.js

@@ -13,12 +13,14 @@ import { findSkills } from "./skills/parse.js";
 import { diffServerWeigh } from "./weigh/diff.js";
 import { DEFAULT_CONTEXT_WINDOW, formatWindowShare, setContextWindow } from "./weigh/window.js";
 import { checkMcpRepoOnly, checkMcpTarget, checkSkillSet } from "./check/check.js";
-import { renderCheckPretty, renderDiffPretty, renderScenarioPlan, renderServerWeighPretty, renderSkillSetPretty, renderSkillWeighPretty, renderTestReportPretty, setColor, } from "./reporters/pretty.js";
+import { analyzeFind, DEFAULT_TOP_K } from "./find/find.js";
+import { runFindRules } from "./core/engine.js";
+import { renderCheckPretty, renderDiffPretty, renderFindFindingsPretty, renderFindPretty, renderScenarioPlan, renderServerWeighPretty, renderSkillSetPretty, renderSkillWeighPretty, renderTestReportPretty, setColor, } from "./reporters/pretty.js";
 import { parseScenario, runScenario } from "./testing/harness.js";
-import { anthropicRunner } from "./testing/anthropicRunner.js";
+import { acceptsSamplingParams, anthropicRunner } from "./testing/anthropicRunner.js";
 import { openaiRunner, providerForModel } from "./testing/openaiRunner.js";
 import { toJsonEnvelope } from "./reporters/json.js";
-import { renderCheckMarkdown, renderDiffMarkdown, renderSkillSetMarkdown, renderWeighMarkdown } from "./reporters/markdown.js";
+import { renderCheckMarkdown, renderDiffMarkdown, renderFindMarkdown, renderSkillSetMarkdown, renderWeighMarkdown } from "./reporters/markdown.js";
 import { gradeBadgeSpec, makeBadgeSvg, toShieldsEndpoint, weighBadgeSpec } from "./reporters/badge.js";
 import { loadDotEnv } from "./util/dotenv.js";
 // Load a local .env before any command reads a key. Shell env always wins.
@@ -36,6 +38,7 @@ examples:
   $ npx efaimo weigh --client claude-code
   $ npx efaimo check --mcp "npx -y my-mcp-server"      # incl. 2026-07-28 readiness
   $ npx efaimo check --skill ./skills/
+  $ npx efaimo find "npx -y my-mcp-server"             # would a search surface these tools?
   $ npx efaimo weigh "npx -y my-server" --out base.json && npx efaimo weigh "npx -y my-server" --diff base.json
 `);
 function collectPairs(sep) {
@@ -104,6 +107,13 @@ function anthropicKeyFor(opts) {
         fail("--anthropic needs ANTHROPIC_API_KEY in the environment or a .env file");
     return { apiKey, model: typeof opts.anthropic === "string" ? opts.anthropic : undefined };
 }
+// `--no-timestamp` is commander's negated form, so `opts.timestamp` is true
+// unless the flag is passed. Kept in one helper because three commands take it
+// and the polarity is easy to invert by accident.
+function envelopeOpts(opts) {
+    return { timestamp: opts.timestamp !== false };
+}
+const NO_TIMESTAMP_HELP = "omit generatedAt from JSON output (for committed or diffed artifacts)";
 program
     .command("weigh")
     .description("measure the context cost of MCP tool definitions or Agent Skills")
@@ -116,6 +126,7 @@ program
     .option("--timeout <seconds>", "connect timeout in seconds", "45")
     .option("--json", "print JSON")
     .option("--md", "print Markdown")
+    .option("--no-timestamp", NO_TIMESTAMP_HELP)
     .option("--out <file>", "write the JSON result to a file (usable as a --diff baseline)")
     .option("--diff <baseline>", "compare against a baseline JSON from --out")
     .option("--max-tokens <n>", "exit 1 if the primary total exceeds n tokens")
@@ -212,14 +223,18 @@ program
     const single = targets.length === 1 && results.length === 1 ? results[0] : undefined;
     const budgetTotal = results.reduce((s, r) => s + (r.kind === "mcp" ? r.totals.claudeStyle : r.totals.metadata + r.totals.body), 0);
     if (opts.out && single) {
-        fs.writeFileSync(opts.out, toJsonEnvelope("weigh", single));
+        // Never stamped. A baseline exists to be compared against, so a field
+        // that differs on every write is the one field a comparison must ignore,
+        // and a committed baseline that diffs on nothing but the clock trains
+        // reviewers to skim exactly the file they should read.
+        fs.writeFileSync(opts.out, toJsonEnvelope("weigh", single, { timestamp: false }));
         console.error(pc.dim(`baseline written: ${opts.out}`));
     }
     else if (opts.out) {
         console.error(pc.yellow("--out skipped: it writes a single baseline, but multiple targets were requested"));
     }
     if (opts.json) {
-        console.log(toJsonEnvelope("weigh", single ?? results));
+        console.log(toJsonEnvelope("weigh", single ?? results, envelopeOpts(opts)));
     }
     else if (opts.md) {
         console.log(results.map(renderWeighMarkdown).join("\n\n---\n\n"));
@@ -286,6 +301,7 @@ program
     .option("--timeout <seconds>", "connect timeout in seconds", "45")
     .option("--json", "print JSON")
     .option("--md", "print Markdown")
+    .option("--no-timestamp", NO_TIMESTAMP_HELP)
     .option("--badge [file]", "write a grade badge SVG + shields endpoint JSON")
     .option("--anthropic [model]", "use exact Claude token counts where relevant")
     .option("--window <tokens>", "context window the share is reported against", String(DEFAULT_CONTEXT_WINDOW))
@@ -334,7 +350,7 @@ program
     }
     if (skillSet) {
         if (opts.json)
-            console.log(toJsonEnvelope("check", skillSet));
+            console.log(toJsonEnvelope("check", skillSet, envelopeOpts(opts)));
         else if (opts.md)
             console.log(renderSkillSetMarkdown(skillSet));
         else
@@ -350,7 +366,7 @@ program
     if (!report)
         return;
     if (opts.json)
-        console.log(toJsonEnvelope("check", report));
+        console.log(toJsonEnvelope("check", report, envelopeOpts(opts)));
     else if (opts.md)
         console.log(renderCheckMarkdown(report));
     else
@@ -405,41 +421,165 @@ program
         process.exitCode = 1;
     }
 });
+program
+    .command("find")
+    .description("would a search surface these tools? (findability under deferred tool loading)")
+    .argument("[target]", "stdio command | http(s) URL")
+    .option("--stdio", "treat target as a stdio command string")
+    .option("--header <header>", 'HTTP header "Key: Value" (repeatable)', collectPairs(":"))
+    .option("--env <pair>", "KEY=VALUE for stdio servers (repeatable)", collectPairs("="))
+    .option("--timeout <seconds>", "connect timeout in seconds", "45")
+    .option("--top <n>", `result window to simulate (Anthropic's documented default is ${DEFAULT_TOP_K}; the probe is vacuous when tools <= top, so gate on --min-distinct rather than on this)`, String(DEFAULT_TOP_K))
+    .option("--min-distinct <pct>", "exit 1 if fewer than pct percent of tools own a word no other tool has")
+    .option("--json", "print JSON")
+    .option("--md", "print Markdown")
+    .option("--no-timestamp", NO_TIMESTAMP_HELP)
+    .option("--no-color", "disable colors")
+    .addHelpText("after", "\nA tool marked defer_loading is kept out of the context window until a search finds it, so a tool\n" +
+    "nothing surfaces costs nothing and does nothing. Two numbers come back. `distinct` is a property of\n" +
+    "the catalog: a tool that owns no word the others lack cannot be matched by any query that does not\n" +
+    "also match a competitor. `probe` is a simulated BM25 search, offline and deterministic, and it\n" +
+    "saturates on well-formed catalogs, so it is a floor test rather than a ranking.\n" +
+    "No API key, no network beyond connecting to the server. docs/METHODOLOGY.md has the method.")
+    .action(async (targetArg, opts) => {
+    colorSetup(opts);
+    if (!targetArg)
+        fail("nothing to search: pass an MCP server (stdio command or URL)");
+    const timeoutMs = seconds(opts.timeout) * 1000;
+    const topK = parseNumberOpt(opts.top, "--top", { min: 1 }) ?? DEFAULT_TOP_K;
+    // A window is a count of results, so a fraction of one is not a stricter
+    // setting, it is a nonsensical one: `--top 2.5` printed "result window 2.5"
+    // while `rank <= 2.5` behaved as 2, so the number in the report and the
+    // number in the comparison disagreed.
+    if (!Number.isInteger(topK))
+        fail(`--top must be a whole number of results, got "${opts.top}"`);
+    const minDistinct = parseNumberOpt(opts.minDistinct, "--min-distinct", { min: 0 });
+    if (minDistinct !== undefined && minDistinct > 100)
+        fail(`--min-distinct is a percentage, got "${opts.minDistinct}"`);
+    const target = resolveTarget(targetArg, { forceStdio: opts.stdio, env: opts.env, headers: opts.header });
+    if (target.kind === "skillset" || target.kind === "repo") {
+        fail(`"${target.label}" is a ${target.kind === "skillset" ? "skill path" : "source directory"}; find needs a live MCP server (stdio command or URL). ` +
+            `Skills have no searchable tool catalog; their trigger-overlap check is S103 in \`efaimo check --skill\`.`);
+    }
+    console.error(pc.dim(`connecting to ${target.label} ...`));
+    const intro = await introspectServer(target, { timeoutMs });
+    // Weighed only to answer one question: is this catalog big enough that a
+    // host would defer it. No API key is involved; the count is the local
+    // o200k estimate the rest of the tool uses.
+    const weigh = await weighServer(intro);
+    const result = analyzeFind(target.label, intro.tools, {
+        topK,
+        definitionTokens: weigh.totals.claudeStyle,
+    });
+    const findings = runFindRules({ find: result });
+    if (opts.json) {
+        console.log(toJsonEnvelope("find", { ...result, findings }, envelopeOpts(opts)));
+    }
+    else if (opts.md) {
+        console.log(renderFindMarkdown(result, findings));
+    }
+    else {
+        console.log(renderFindPretty(result));
+        console.log(renderFindFindingsPretty(result, findings));
+    }
+    // Gated on `distinct`, not on the probe. The probe is 100% by construction
+    // whenever the catalog fits inside the result window, so a gate on it
+    // would pass without examining anything for every small server. Exclusive
+    // vocabulary can fail at two tools and up.
+    //
+    // At ONE tool it cannot: every term is trivially exclusive and the figure
+    // is 100% for any tool whatsoever. Exit 2 rather than 0, because "this
+    // gate cannot be evaluated here" is a different fact from "this gate
+    // passed", and a CI run that silently passes on a catalog nothing was
+    // measured against is the failure this whole command was built around.
+    if (minDistinct !== undefined && result.distinctVacuous) {
+        fail(`--min-distinct cannot be evaluated on a ${result.toolCount}-tool catalog: with nothing to be distinct from, ` +
+            `every term is trivially exclusive and the figure is 100% for any tool. Drop the gate for this server.`);
+    }
+    if (minDistinct !== undefined && result.distinct.pct < minDistinct) {
+        console.error(pc.red(`distinct ${result.distinct.pct}% is below --min-distinct ${minDistinct}% ` +
+            `(${result.distinct.total - result.distinct.count} of ${result.distinct.total} tools own no word the others lack)`));
+        process.exitCode = 1;
+    }
+});
 program
     .command("test")
     .description("does a skill actually improve task completion? (experimental A/B outcome harness)")
     .argument("<scenario>", "a scenario YAML file (see examples/scenario.example.yaml)")
     .option("--live", "run for real (spends tokens; Claude models need ANTHROPIC_API_KEY, GPT models need OPENAI_API_KEY)")
     .option("--model <model>", "override the scenario's model (e.g. gpt-4o-mini, claude-sonnet-5)")
+    .option("--judge-model <model>", "grade with a different model than the one under test (removes self-preference)")
     .option("--json", "print JSON")
     .option("--no-color", "disable colors")
-    .addHelpText("after", "\nWithout --live this validates the scenario and prints the plan, making no API calls.")
+    .addHelpText("after", "\nWithout --live this validates the scenario and prints the plan, making no API calls.\n" +
+    "The judge defaults to the model under test, which means a model grades its own answers; set\n" +
+    "judge_model in the scenario or --judge-model here to separate them. Subject and judge may be\n" +
+    "different providers, in which case both keys are required.")
     .action(async (file, opts) => {
     colorSetup(opts);
     const parsed = parseScenario(file);
-    const scenario = opts.model ? { ...parsed, model: opts.model } : parsed;
+    // --model moves the judge too WHEN the judge was never pinned separately.
+    // Without this, overriding the subject on a scenario that has no
+    // `judge_model` would silently leave the judge on the scenario's original
+    // model: a cross-model judge nobody asked for, in a run whose whole point
+    // is that the comparison is controlled.
+    const judgeFollowsSubject = !parsed.judgeModelExplicit;
+    const model = opts.model ?? parsed.model;
+    const judgeModel = opts.judgeModel ?? (judgeFollowsSubject ? model : parsed.judgeModel);
+    const scenario = { ...parsed, model, judgeModel };
     if (!opts.live) {
         console.log(renderScenarioPlan(scenario));
         return;
     }
-    const provider = providerForModel(scenario.model);
-    if (provider === "unknown") {
-        fail(`model "${scenario.model}" is not supported: efaimo test runs Claude (claude-*) and OpenAI (gpt-*, o*) models`);
+    // Subject and judge are routed independently, so a Claude subject can be
+    // graded by a GPT judge (or the reverse). Every distinct provider in play
+    // needs its own key, and the error says which model asked for it.
+    const runners = new Map();
+    for (const [role, m] of [["model", scenario.model], ["judge model", scenario.judgeModel]]) {
+        const provider = providerForModel(m);
+        if (provider === "unknown") {
+            fail(`${role} "${m}" is not supported: efaimo test runs Claude (claude-*) and OpenAI (gpt-*, o*) models`);
+        }
+        if (runners.has(provider))
+            continue;
+        const envVar = provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY";
+        const apiKey = process.env[envVar];
+        if (!apiKey)
+            fail(`${role} "${m}" needs ${envVar} in the environment or a .env file`);
+        if (dotEnvKeys.has(envVar))
+            console.error(pc.dim(`using ${envVar} from .env`));
+        runners.set(provider, provider === "openai" ? openaiRunner(apiKey) : anthropicRunner(apiKey));
     }
-    const envVar = provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY";
-    const apiKey = process.env[envVar];
-    if (!apiKey)
-        fail(`model "${scenario.model}" needs ${envVar} in the environment or a .env file`);
-    if (dotEnvKeys.has(envVar))
-        console.error(pc.dim(`using ${envVar} from .env`));
-    const runner = provider === "openai" ? openaiRunner(apiKey) : anthropicRunner(apiKey);
-    console.error(pc.dim(`running ~${scenario.trials * 4} ${provider} API calls against ${scenario.model} ...`));
-    const report = await runScenario(scenario, runner);
+    const runner = (req) => {
+        const r = runners.get(providerForModel(req.model));
+        // Only the two validated models can reach here today. A future arm that
+        // introduces a third would otherwise die as "undefined is not a
+        // function" with no idea which model it was.
+        if (!r)
+            throw new Error(`no runner for model "${req.model}"; efaimo test runs Claude and OpenAI models`);
+        return r(req);
+    };
+    // Say what the measurement cannot pin down, in the report, every time.
+    const extraNotes = [];
+    if (scenario.judgeModel === scenario.model) {
+        extraNotes.push(`the judge is the same model as the subject, so part of any measured effect is a model preferring its own output. Set judge_model (or --judge-model) to a different model to remove that.`);
+    }
+    if (providerForModel(scenario.judgeModel) === "anthropic" && !acceptsSamplingParams(scenario.judgeModel)) {
+        extraNotes.push(`judge sampling is not pinned: ${scenario.judgeModel} does not accept temperature, so the judge is sampled at its default and its own variance is part of this measurement.`);
+    }
+    const models = scenario.judgeModel === scenario.model ? scenario.model : `${scenario.model} judged by ${scenario.judgeModel}`;
+    console.error(pc.dim(`running ~${scenario.trials * 4} API calls against ${models} ...`));
+    const report = await runScenario(scenario, runner, { extraNotes });
     if (opts.json)
         console.log(toJsonEnvelope("test", report));
     else
         console.log(renderTestReportPretty(report));
-    if (report.verdict === "hurts")
+    // `inconclusive` fails too. It means an arm produced no scoreable trial, or
+    // the judge refused on one arm far more than the other, and in both cases
+    // the measurement did not happen. Exiting 0 there would report a green for
+    // precisely the runs where the instrument broke down, which is this
+    // project's documented dominant failure mode wearing a p-value.
+    if (report.verdict === "hurts" || report.verdict === "inconclusive")
         process.exitCode = 1;
 });
 program

package/dist/core/engine.d.ts

@@ -1,6 +1,7 @@
-import type { Finding, McpRuleContext } from "./types.js";
+import type { Finding, FindRuleContext, McpRuleContext } from "./types.js";
 /** Run a list of rules, isolating each: a broken rule must never break the audit. */
 export declare function runRules<Ctx>(rules: readonly {
     check(ctx: Ctx): Finding[];
 }[], ctx: Ctx): Finding[];
 export declare function runMcpRules(ctx: McpRuleContext): Finding[];
+export declare function runFindRules(ctx: FindRuleContext): Finding[];

package/dist/core/engine.js

@@ -1,4 +1,5 @@
 import { MCP_RULES } from "../rules/mcp/index.js";
+import { FIND_RULES } from "../rules/find/index.js";
 /** Run a list of rules, isolating each: a broken rule must never break the audit. */
 export function runRules(rules, ctx) {
     const findings = [];
@@ -33,3 +34,6 @@ export function runRules(rules, ctx) {
 export function runMcpRules(ctx) {
     return runRules(MCP_RULES, ctx);
 }
+export function runFindRules(ctx) {
+    return runRules(FIND_RULES, ctx);
+}