efaimo 0.1.2 → 0.2.0

package/dist/rules/find/index.js

@@ -0,0 +1,262 @@
+/**
+ * Findability rules, E141-E146.
+ *
+ * A separate family with its own command on purpose. They are not readiness
+ * (which asks whether a 2026-07-28 client can talk to this server) and not
+ * quality (which asks what a model experiences once a tool is in front of it).
+ * They ask a third thing: under deferred loading, does anything ever put this
+ * tool in front of the model at all.
+ *
+ * E141 is the one that is not a heuristic. The others read a simulation and
+ * are worth less; they are kept because the simulation is the only thing here
+ * that models the real mechanism, and because the cases they do catch (a tool
+ * with no description, two tools with the same one) are worth catching.
+ *
+ * None of them feeds `check`'s grade, and `find` produces no grade of its own.
+ * ADR-030 has the reasoning.
+ */
+function cap(items, n) {
+    return { shown: items.slice(0, n), more: Math.max(0, items.length - n) };
+}
+/**
+ * A target-supplied value going into a rule message or detail.
+ *
+ * The reporters strip control characters but keep newlines, because a detail
+ * block is legitimately multi-line. That makes a newline inside a tool name a
+ * way for the audited server to add its own line to the auditor's output. The
+ * values these rules interpolate - names, terms - are all single-line by
+ * nature, so they are flattened here, at the point where a multi-line region
+ * is built out of them.
+ */
+function one(s) {
+    return s.replace(/[\r\n]+/g, " ");
+}
+const list = (xs) => xs.map(one).join(", ");
+const e141 = {
+    id: "E141",
+    title: "tool owns no exclusive vocabulary",
+    surface: "find",
+    check({ find }) {
+        const shared = find.perTool.filter((t) => t.ownTermCount === 0);
+        const { shown, more } = cap(shared, 5);
+        const findings = shown.map((t) => ({
+            ruleId: "E141",
+            severity: "warn",
+            title: "tool owns no exclusive vocabulary",
+            // Stated as what it is: an existence claim about queries, not a
+            // prediction about one search. Every term this tool has appears on some
+            // other tool, so there is no query that matches it alone. Whether it
+            // still wins a shared query depends on term frequencies, which is a much
+            // weaker thing to rely on than owning a word.
+            message: `tool "${one(t.name)}" has no term that no other tool has, so no search can return it without also returning a competitor`,
+            target: t.name,
+            // "also appears on", not "covered by": no single tool on this list
+            // necessarily carries the whole vocabulary. They are the tools with the
+            // most overlap, which is where to look first when writing the fix.
+            detail: t.sharedWith.length ? `every word it has also appears on: ${list(t.sharedWith)} (most overlap first)` : undefined,
+            fixHint: "put something in the description that only this tool does: the object it acts on, the format it returns, the case it is for",
+        }));
+        if (more) {
+            findings.push({
+                ruleId: "E141",
+                severity: "warn",
+                title: "tool owns no exclusive vocabulary",
+                message: `...and ${more} more tools that share every one of their terms with another tool`,
+            });
+        }
+        return findings;
+    },
+};
+const e142 = {
+    id: "E142",
+    title: "a search for the tool's own description does not return it",
+    surface: "find",
+    check({ find }) {
+        const findings = [];
+        const unsearchable = find.perTool.filter((t) => t.rank === undefined);
+        for (const t of cap(unsearchable, 5).shown) {
+            findings.push({
+                ruleId: "E142",
+                severity: "warn",
+                title: "a search for the tool's own description does not return it",
+                message: `tool "${one(t.name)}" offers no words to search for: its description is empty or made only of function words, so nothing but the exact name reaches it`,
+                target: t.name,
+                fixHint: "write a description containing the words someone would use to describe the task this tool does",
+            });
+        }
+        const missed = find.perTool.filter((t) => t.rank !== undefined && !t.reachable);
+        for (const t of cap(missed, 5).shown) {
+            findings.push({
+                ruleId: "E142",
+                severity: "warn",
+                title: "a search for the tool's own description does not return it",
+                message: `tool "${one(t.name)}" ranks ${t.rank} of ${find.toolCount} for a search built from its own description (${list(t.query)}), outside a result window of ${find.method.topK}`,
+                target: t.name,
+                detail: t.outrankedBy.length ? `outranked by: ${list(t.outrankedBy)}` : undefined,
+                fixHint: "make the description carry a term the other tools do not share",
+            });
+        }
+        const more = cap(unsearchable, 5).more + cap(missed, 5).more;
+        if (more) {
+            findings.push({
+                ruleId: "E142",
+                severity: "warn",
+                title: "a search for the tool's own description does not return it",
+                message: `...and ${more} more tools a search for their own description does not return`,
+            });
+        }
+        return findings;
+    },
+};
+const e143 = {
+    id: "E143",
+    title: "tools are indistinguishable to a search",
+    surface: "find",
+    check({ find }) {
+        // Report each pair once. A ties with B is the same fact as B ties with A,
+        // and printing both halves doubles the finding count, which would double
+        // the apparent size of the problem.
+        const seen = new Set();
+        const findings = [];
+        for (const t of find.perTool) {
+            for (const other of t.tiedWith) {
+                const key = t.name < other ? `${one(t.name)} ${one(other)}` : `${one(other)} ${one(t.name)}`;
+                if (seen.has(key))
+                    continue;
+                seen.add(key);
+                if (findings.length >= 5)
+                    continue;
+                findings.push({
+                    ruleId: "E143",
+                    severity: "warn",
+                    title: "tools are indistinguishable to a search",
+                    message: `tools "${one(t.name)}" and "${one(other)}" score identically for a search built from "${one(t.name)}"'s description (${list(t.query)}); only the alphabetical tie-break separates them`,
+                    target: t.name,
+                    fixHint: "give each description a term the other does not have, or merge the tools",
+                });
+            }
+        }
+        // The true pair count comes from tiedWithCount, not from the names above.
+        // `tiedWith` is capped at three per tool for display, so counting the
+        // pairs it yields reported 30 for a catalog whose real answer was 66: a
+        // count of what survived a display cap, printed as a count of the problem.
+        // Each pair is counted from both ends, hence the halving.
+        const totalPairs = Math.round(find.perTool.reduce((n, t) => n + t.tiedWithCount, 0) / 2);
+        if (totalPairs > findings.length) {
+            findings.push({
+                ruleId: "E143",
+                severity: "warn",
+                title: "tools are indistinguishable to a search",
+                message: `...and ${totalPairs - findings.length} more indistinguishable pairs (${totalPairs} in total)`,
+            });
+        }
+        return findings;
+    },
+};
+const e146 = {
+    id: "E146",
+    title: "the only word a tool owns is its own name",
+    surface: "find",
+    check({ find }) {
+        // Names ARE searchable, so this tool is findable in principle and E141
+        // correctly does not fire. It is not findable by anyone describing a task,
+        // which is the situation deferred loading creates, so the two cases are
+        // reported separately rather than merged into one number.
+        const nameOnly = find.perTool.filter((t) => t.ownTermCount > 0 && t.ownOutsideNameCount === 0);
+        const { shown, more } = cap(nameOnly, 5);
+        const findings = shown.map((t) => ({
+            ruleId: "E146",
+            severity: "info",
+            title: "the only word a tool owns is its own name",
+            message: t.ownTermCount === 1
+                ? `tool "${one(t.name)}" owns one term and it comes from its own name; nothing in its description is unique, so only someone who already knows the name can single it out`
+                : `tool "${one(t.name)}" owns ${t.ownTermCount} terms and every one of them comes from its own name; nothing in its description is unique, so only someone who already knows the name can single it out`,
+            target: t.name,
+            fixHint: "put a word in the description that no other tool's description uses",
+        }));
+        if (more) {
+            findings.push({
+                ruleId: "E146",
+                severity: "info",
+                title: "the only word a tool owns is its own name",
+                message: `...and ${more} more tools whose only exclusive words are their own name`,
+            });
+        }
+        return findings;
+    },
+};
+const e144 = {
+    id: "E144",
+    title: "tool name carries no domain term",
+    surface: "find",
+    check({ find }) {
+        // All-generic AND no name token of its own.
+        //
+        // "Generic" alone was too weak: the fixture server's `add` is a perfectly
+        // descriptive name for an arithmetic tool, and the rule told its author to
+        // "put the subject in the name" with no usable suggestion. What separates
+        // `add` from `run` is not the word list, it is whether the word belongs to
+        // this tool alone in this catalog. `ownTermCount > ownOutsideNameCount`
+        // means at least one exclusive term came from the name, which is exactly
+        // the case where a broad pattern search does reach it.
+        const generic = find.perTool.filter((t) => t.genericName && t.ownTermCount === t.ownOutsideNameCount);
+        const { shown, more } = cap(generic, 5);
+        const findings = shown.map((t) => ({
+            ruleId: "E144",
+            severity: "info",
+            title: "tool name carries no domain term",
+            message: `tool "${one(t.name)}" is named only with generic words, so a broad pattern search (Claude writes things like ".*weather.*") can reach it only through its description`,
+            target: t.name,
+            fixHint: "put the subject in the name: what does it act on",
+        }));
+        if (more) {
+            findings.push({
+                ruleId: "E144",
+                severity: "info",
+                title: "tool name carries no domain term",
+                message: `...and ${more} more tools named only with generic words`,
+            });
+        }
+        return findings;
+    },
+};
+const e145 = {
+    id: "E145",
+    title: "deferred catalog with tools a search cannot single out",
+    surface: "find",
+    check({ find }) {
+        if (!find.deferRecommended)
+            return [];
+        // The union, not the sum. These two sets overlap heavily - a tool with no
+        // exclusive vocabulary is often also one the probe misses - and adding
+        // them read as twice as many problems as there are. On a 15-tool server
+        // this line once claimed 24.
+        const affected = new Set();
+        for (const t of find.perTool) {
+            if (t.ownTermCount === 0 || !t.reachable)
+                affected.add(t.name);
+        }
+        if (!affected.size)
+            return [];
+        return [
+            {
+                ruleId: "E145",
+                severity: "warn",
+                title: "deferred catalog with tools a search cannot single out",
+                // The reason comes from the analysis, not from a repeat of one
+                // threshold. Anthropic lists several conditions; hardcoding the token
+                // one here printed "~3500 tokens, past the ~10.0k" on a catalog that
+                // qualified on tool count instead.
+                message: `tool search is recommended for this catalog (${find.deferBecause.map(one).join("; ")}), ` +
+                    `and ${affected.size} of ${find.toolCount} tool${affected.size === 1 ? " cannot" : "s cannot"} be singled out by a search. ` +
+                    `Once this catalog is deferred, a tool nothing surfaces costs nothing and does nothing.`,
+                fixHint: "fix the descriptions above, or split the server so fewer tools compete in one catalog",
+            },
+        ];
+    },
+};
+export const FIND_RULES = [e141, e142, e143, e144, e145, e146];
+/** E141-E146 are findability rules: reported by `efaimo find`, never part of a `check` grade. */
+export function isFindabilityRuleId(id) {
+    return /^E14\d$/.test(id);
+}

package/dist/rules/version.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Which ruleset produced a report.
+ *
+ * The tool version does not answer this. A patch release can change what a
+ * rule fires on without changing any documented behaviour (0.1.2 did exactly
+ * that: E123 started matching `delete_file`, and every grade that rule touched
+ * moved), so "efaimo 0.1.2 said A(95)" is only reproducible if the reader can
+ * also tell which rules were running. Every JSON envelope now carries this.
+ *
+ * Values:
+ *   "1"  every ruleset through efaimo 0.1.2. Reports from those versions carry
+ *        no `rulesVersion` field at all; the absence IS the value.
+ *   "2"  the first ruleset that declares itself. Adds the findability family
+ *        (E141-E145), which no earlier version had.
+ *
+ * Bump it when a rule is added, removed, renumbered, or changed in what it
+ * fires on. `test/meta.test.ts` pins the rule inventory, so an added or
+ * removed rule cannot land without someone looking at this file. It cannot
+ * detect a changed threshold inside an existing rule: that one is human
+ * discipline, and this comment is where the discipline is written down.
+ *
+ * Deliberately not derived from package.json. Tying it to the release number
+ * would make it move on every release whether or not a rule moved, which is
+ * the thing it exists to distinguish.
+ */
+export declare const RULES_VERSION = "2";

package/dist/rules/version.js

@@ -0,0 +1,26 @@
+/**
+ * Which ruleset produced a report.
+ *
+ * The tool version does not answer this. A patch release can change what a
+ * rule fires on without changing any documented behaviour (0.1.2 did exactly
+ * that: E123 started matching `delete_file`, and every grade that rule touched
+ * moved), so "efaimo 0.1.2 said A(95)" is only reproducible if the reader can
+ * also tell which rules were running. Every JSON envelope now carries this.
+ *
+ * Values:
+ *   "1"  every ruleset through efaimo 0.1.2. Reports from those versions carry
+ *        no `rulesVersion` field at all; the absence IS the value.
+ *   "2"  the first ruleset that declares itself. Adds the findability family
+ *        (E141-E145), which no earlier version had.
+ *
+ * Bump it when a rule is added, removed, renumbered, or changed in what it
+ * fires on. `test/meta.test.ts` pins the rule inventory, so an added or
+ * removed rule cannot land without someone looking at this file. It cannot
+ * detect a changed threshold inside an existing rule: that one is human
+ * discipline, and this comment is where the discipline is written down.
+ *
+ * Deliberately not derived from package.json. Tying it to the release number
+ * would make it move on every release whether or not a rule moved, which is
+ * the thing it exists to distinguish.
+ */
+export const RULES_VERSION = "2";

package/dist/testing/anthropicRunner.d.ts

@@ -1,4 +1,34 @@
 import type { Runner } from "./harness.js";
+/**
+ * Which Claude models still accept `temperature` / `top_p` / `top_k`.
+ *
+ * Sampling parameters were removed from the Claude 4.7 line onward and now
+ * return a 400 on Fable 5, Opus 5, Opus 4.8, Opus 4.7 and Sonnet 5. This
+ * runner used to send `temperature` on every request, and the scenario default
+ * model is `claude-sonnet-5`, so `efaimo test --live` sent a rejected
+ * parameter on its first call; `isTransient()` correctly does not retry a 400,
+ * so the whole scenario threw before a single trial completed.
+ *
+ * Nothing in the suite could see it. The runner is injected precisely so the
+ * A/B logic can be tested without spending tokens, which means every test in
+ * this repo exercises a fake and none of them ever built this request body.
+ * The fix is therefore paired with a test that asserts on the body itself.
+ *
+ * The list is an ALLOWLIST of models that accept the parameter, and it is
+ * closed: it names model lines that already shipped, so it cannot go stale in
+ * the direction that breaks a run. A model released after this was written is
+ * unknown to it and gets no `temperature`, which costs a little determinism on
+ * the judge and cannot cost a 400. The opposite shape - a denylist of models
+ * that reject it - would fail every future release by default.
+ */
+export declare function acceptsSamplingParams(model: string): boolean;
+/** The request body, split out so a test can assert on it without a network call. */
+export declare function anthropicRequestBody(req: {
+    model: string;
+    system?: string;
+    user: string;
+    deterministic?: boolean;
+}): Record<string, unknown>;
 /**
  * The live runner. Used only by `efaimo test --live`, because every call spends
  * tokens on the caller's ANTHROPIC_API_KEY.

package/dist/testing/anthropicRunner.js

@@ -1,3 +1,45 @@
+/**
+ * Which Claude models still accept `temperature` / `top_p` / `top_k`.
+ *
+ * Sampling parameters were removed from the Claude 4.7 line onward and now
+ * return a 400 on Fable 5, Opus 5, Opus 4.8, Opus 4.7 and Sonnet 5. This
+ * runner used to send `temperature` on every request, and the scenario default
+ * model is `claude-sonnet-5`, so `efaimo test --live` sent a rejected
+ * parameter on its first call; `isTransient()` correctly does not retry a 400,
+ * so the whole scenario threw before a single trial completed.
+ *
+ * Nothing in the suite could see it. The runner is injected precisely so the
+ * A/B logic can be tested without spending tokens, which means every test in
+ * this repo exercises a fake and none of them ever built this request body.
+ * The fix is therefore paired with a test that asserts on the body itself.
+ *
+ * The list is an ALLOWLIST of models that accept the parameter, and it is
+ * closed: it names model lines that already shipped, so it cannot go stale in
+ * the direction that breaks a run. A model released after this was written is
+ * unknown to it and gets no `temperature`, which costs a little determinism on
+ * the judge and cannot cost a 400. The opposite shape - a denylist of models
+ * that reject it - would fail every future release by default.
+ */
+export function acceptsSamplingParams(model) {
+    return /^claude-(3|(sonnet|haiku|opus)-4-5|(opus|sonnet)-4-6)\b/i.test(model);
+}
+/** The request body, split out so a test can assert on it without a network call. */
+export function anthropicRequestBody(req) {
+    const body = {
+        model: req.model,
+        max_tokens: 1024,
+        ...(req.system ? { system: req.system } : {}),
+        messages: [{ role: "user", content: req.user }],
+    };
+    // 0 for the judge, 1 for the subject (see RunnerRequest.deterministic), but
+    // only where the model still has the parameter. Where it does not, the
+    // subject arm is unaffected (1 was the default anyway) and the judge simply
+    // is not pinned; `runScenario` is told so it can say it in the report,
+    // because an unpinned judge is variance that lands in the measurement.
+    if (acceptsSamplingParams(req.model))
+        body.temperature = req.deterministic ? 0 : 1;
+    return body;
+}
 /**
  * The live runner. Used only by `efaimo test --live`, because every call spends
  * tokens on the caller's ANTHROPIC_API_KEY.
@@ -11,14 +53,7 @@ export function anthropicRunner(apiKey) {
                 "x-api-key": apiKey,
                 "anthropic-version": "2023-06-01",
             },
-            body: JSON.stringify({
-                model: req.model,
-                // 0 for the judge, 1 for the subject. See RunnerRequest.deterministic.
-                temperature: req.deterministic ? 0 : 1,
-                max_tokens: 1024,
-                ...(req.system ? { system: req.system } : {}),
-                messages: [{ role: "user", content: req.user }],
-            }),
+            body: JSON.stringify(anthropicRequestBody(req)),
             redirect: "error",
             signal: AbortSignal.timeout(60000),
         });

package/dist/testing/harness.d.ts

@@ -10,6 +10,24 @@ export interface Scenario {
     skillName: string;
     skillBody: string;
     model: string;
+    /**
+     * The model that grades the attempts. Defaults to `model`, which is what it
+     * always was, and that default is a known confound: a model grading its own
+     * output prefers it, so an effect measured that way is partly a measurement
+     * of self-preference. Set it to a different model (`judge_model:` in the
+     * scenario, or `--judge-model`) and the report prints both.
+     */
+    judgeModel: string;
+    /**
+     * Whether `judge_model:` was written in the scenario.
+     *
+     * `--model` moves the judge with the subject only when the judge was never
+     * pinned. Without this flag the two cases are indistinguishable: a scenario
+     * that deliberately pins `judge_model` to the same id as `model` looked
+     * identical to one that omitted the key, so `--model gpt-4o-mini` would have
+     * moved a deliberately pinned judge.
+     */
+    judgeModelExplicit: boolean;
     trials: number;
     task: string;
     judge: string;
@@ -43,12 +61,23 @@ export interface TestReport {
     scenario: string;
     skill: string;
     model: string;
+    judgeModel: string;
     withSkill: ArmResult;
     withoutSkill: ArmResult;
     /** withSkill.passRate - withoutSkill.passRate, in percentage points. */
     deltaPoints: number;
     /** Two-sided Fisher exact p on the 2x2 of scored trials. */
     p: number;
+    /**
+     * Two-sided Fisher exact p on the 2x2 of UNPARSEABLE trials.
+     *
+     * Dropping a trial the judge did not answer is only unbiased if the dropping
+     * is unrelated to which arm produced it. If one arm loses materially more
+     * trials than the other, the surviving denominators are not comparable and
+     * the delta is measuring the judge as much as the skill. Low p here means
+     * the exclusions are skewed.
+     */
+    unparseableSkewP: number;
     /** 95% interval on the delta, in percentage points (Newcombe). */
     ci: {
         lo: number;
@@ -64,4 +93,6 @@ export declare function armSystems(scenario: Scenario): {
     withSkill: string;
 };
 export declare function withRetry(runner: Runner, tries?: number, baseDelayMs?: number): Runner;
-export declare function runScenario(scenario: Scenario, runner: Runner): Promise<TestReport>;
+export declare function runScenario(scenario: Scenario, runner: Runner, opts?: {
+    extraNotes?: string[];
+}): Promise<TestReport>;

package/dist/testing/harness.js

@@ -29,12 +29,16 @@ export function parseScenario(file) {
     // outcome that reaches p < 0.05, so every partial result is uninterpretable
     // and the old default guaranteed a verdict nobody could stand behind.
     const trials = Math.min(50, Math.max(1, Math.round(num(doc.trials) ?? 20)));
+    const model = str(doc.model) ?? "claude-sonnet-5";
+    const judgeModel = str(doc.judge_model);
     return {
         name,
         skillPath,
         skillName: skill.name ?? path.basename(skill.dir),
         skillBody: skill.body.trim(),
-        model: str(doc.model) ?? "claude-sonnet-5",
+        model,
+        judgeModel: judgeModel ?? model,
+        judgeModelExplicit: judgeModel !== undefined,
         trials,
         task,
         judge,
@@ -60,7 +64,7 @@ async function judgeOne(runner, scenario, answer) {
     const out = await runner({
         system: JUDGE_SYSTEM,
         user: `TASK:\n${scenario.task}\n\nRUBRIC:\n${scenario.judge}\n\nASSISTANT ANSWER:\n${answer}\n\nVerdict (PASS or FAIL):`,
-        model: scenario.model,
+        model: scenario.judgeModel,
         deterministic: true,
     });
     const saysPass = /\bpass(ed|es)?\b/i.test(out);
@@ -119,7 +123,7 @@ async function runArm(runner, scenario, system) {
         passRate: scored > 0 ? (passes / scored) * 100 : 0,
     };
 }
-export async function runScenario(scenario, runner) {
+export async function runScenario(scenario, runner, opts = {}) {
     const systems = armSystems(scenario);
     // Wrap once so both the subject answers and the judge calls survive a
     // transient blip instead of throwing away the whole (paid-for) scenario.
@@ -150,11 +154,28 @@ export async function runScenario(scenario, runner) {
     if (unscored) {
         notes.push(`${unscored} trial(s) produced a judge verdict that was neither PASS nor FAIL and are excluded from the rates, not counted as failures.`);
     }
+    // Excluding a trial the judge did not answer is the right call, and it is
+    // unbiased only while the exclusions fall on both arms alike. They need not:
+    // a skill that makes answers longer, or hedged, or formatted differently can
+    // make the judge hedge too, and then one arm quietly loses more trials than
+    // the other and the two surviving pass rates are rates over different things.
+    // Tested with the same statistic and the same threshold used for the effect
+    // itself, so there is one standard of evidence in this report, not two.
+    const unparseableSkewP = fisherExactTwoSided(withSkill.unparseable, withSkill.trials, withoutSkill.unparseable, withoutSkill.trials);
+    const exclusionsAreSkewed = unscored > 0 && unparseableSkewP < 0.05;
     let verdict;
     if (withSkill.scored === 0 || withoutSkill.scored === 0) {
         verdict = "inconclusive";
         notes.push("an arm produced no scoreable trial; there is nothing to compare.");
     }
+    else if (exclusionsAreSkewed) {
+        verdict = "inconclusive";
+        notes.push(`the judge failed to answer on ${withSkill.unparseable} of ${withSkill.trials} with-skill trials and ` +
+            `${withoutSkill.unparseable} of ${withoutSkill.trials} without-skill trials, a difference this size at ` +
+            `p = ${unparseableSkewP < 0.0001 ? "<0.0001" : unparseableSkewP.toFixed(4)}. The excluded trials are not ` +
+            `spread evenly across the arms, so the two pass rates are computed over different populations and the ` +
+            `delta cannot be attributed to the skill. Tighten the rubric so the judge always answers, then re-run.`);
+    }
     else if (p >= 0.05) {
         verdict = "no measurable effect";
         if (Math.abs(deltaPoints) >= 15) {
@@ -166,14 +187,18 @@ export async function runScenario(scenario, runner) {
         verdict = "helps";
     else
         verdict = "hurts";
+    if (opts.extraNotes?.length)
+        notes.push(...opts.extraNotes);
     return {
         scenario: scenario.name,
         skill: scenario.skillName,
         model: scenario.model,
+        judgeModel: scenario.judgeModel,
         withSkill,
         withoutSkill,
         deltaPoints,
         p,
+        unparseableSkewP,
         ci,
         verdict,
         notes,

package/dist/testing/stats.d.ts

@@ -28,3 +28,19 @@ export declare function deltaInterval(withPass: number, withTrials: number, with
     lo: number;
     hi: number;
 };
+/**
+ * The smallest difference in pass rate this many trials per arm could ever
+ * call significant, in percentage points.
+ *
+ * A plan that cannot detect the effect it is looking for produces a green that
+ * means nothing, and the only place to notice that is before spending the
+ * tokens. So the dry run prints this: at 8 trials per arm nothing under about
+ * 50 points is reachable, which is worth knowing before paying for 32 API
+ * calls to find out.
+ *
+ * Computed by exhaustive search over every 2x2 the plan can produce, because
+ * the number of tables is small (trials is capped at 50, so at most 2,601
+ * Fisher evaluations) and a closed form for the exact test is not worth
+ * deriving. Returns Infinity when no outcome at this size reaches alpha.
+ */
+export declare function minimumDetectableDelta(trials: number, alpha?: number): number;

package/dist/testing/stats.js

@@ -87,3 +87,29 @@ export function deltaInterval(withPass, withTrials, withoutPass, withoutTrials)
     const hi = d + Math.sqrt((w.hi - p1) ** 2 + (p2 - o.lo) ** 2);
     return { lo: Math.round(Math.max(-1, lo) * 1000) / 10, hi: Math.round(Math.min(1, hi) * 1000) / 10 };
 }
+/**
+ * The smallest difference in pass rate this many trials per arm could ever
+ * call significant, in percentage points.
+ *
+ * A plan that cannot detect the effect it is looking for produces a green that
+ * means nothing, and the only place to notice that is before spending the
+ * tokens. So the dry run prints this: at 8 trials per arm nothing under about
+ * 50 points is reachable, which is worth knowing before paying for 32 API
+ * calls to find out.
+ *
+ * Computed by exhaustive search over every 2x2 the plan can produce, because
+ * the number of tables is small (trials is capped at 50, so at most 2,601
+ * Fisher evaluations) and a closed form for the exact test is not worth
+ * deriving. Returns Infinity when no outcome at this size reaches alpha.
+ */
+export function minimumDetectableDelta(trials, alpha = 0.05) {
+    let best = Infinity;
+    for (let a = 0; a <= trials; a++) {
+        for (let b = 0; b < a; b++) {
+            if (fisherExactTwoSided(a, trials, b, trials) < alpha) {
+                best = Math.min(best, ((a - b) / trials) * 100);
+            }
+        }
+    }
+    return best;
+}

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Kept in sync with package.json by test/meta.test.ts. */
-export declare const VERSION = "0.1.2";
+export declare const VERSION = "0.2.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 /** Kept in sync with package.json by test/meta.test.ts. */
-export const VERSION = "0.1.2";
+export const VERSION = "0.2.0";

package/dist/weigh/anthropicExact.d.ts

@@ -4,8 +4,19 @@ import type { ToolDef } from "../core/types.js";
  * /v1/messages/count_tokens with and without the tools array (the endpoint
  * accepts `tools`; verified against the official API reference 2026-07-16).
  * Returns undefined on any failure; callers treat this as optional precision.
+ *
+ * Returns the model alongside the count, and the model is not optional in the
+ * result. Claude model lines do not share a tokenizer, so "the exact count" is
+ * exact for one model and an approximation for the next, and a number printed
+ * without the model it was measured on cannot be reproduced or compared. This
+ * is the same rule `src/weigh/window.ts` already applies to the context-window
+ * denominator: never print a figure whose basis is an assumption without
+ * naming the assumption.
  */
 export declare function countClaudeToolTokens(tools: ToolDef[], opts: {
     apiKey: string;
     model?: string;
-}): Promise<number | undefined>;
+}): Promise<{
+    tokens: number;
+    model: string;
+} | undefined>;

package/dist/weigh/anthropicExact.js

@@ -5,6 +5,14 @@ const DEFAULT_MODEL = "claude-sonnet-5";
  * /v1/messages/count_tokens with and without the tools array (the endpoint
  * accepts `tools`; verified against the official API reference 2026-07-16).
  * Returns undefined on any failure; callers treat this as optional precision.
+ *
+ * Returns the model alongside the count, and the model is not optional in the
+ * result. Claude model lines do not share a tokenizer, so "the exact count" is
+ * exact for one model and an approximation for the next, and a number printed
+ * without the model it was measured on cannot be reproduced or compared. This
+ * is the same rule `src/weigh/window.ts` already applies to the context-window
+ * denominator: never print a figure whose basis is an assumption without
+ * naming the assumption.
  */
 export async function countClaudeToolTokens(tools, opts) {
     const model = opts.model ?? DEFAULT_MODEL;
@@ -22,7 +30,7 @@ export async function countClaudeToolTokens(tools, opts) {
         if (withTools === undefined || withoutTools === undefined)
             return undefined;
         const delta = withTools - withoutTools;
-        return delta >= 0 ? delta : undefined;
+        return delta >= 0 ? { tokens: delta, model } : undefined;
     }
     catch {
         return undefined;