efaimo 0.1.1 → 0.2.0

package/dist/rules/skill/index.js

@@ -16,8 +16,10 @@ const s101 = {
         const push = (severity, message, fixHint) => findings.push({ ruleId: "S101", severity, title: "frontmatter does not match the spec", message, target: label, fixHint });
         if (skill.parseError) {
             push("error", `${skill.file}: ${skill.parseError}`, "SKILL.md must start with a --- YAML frontmatter block");
-            if (!skill.frontmatterRaw)
-                return findings;
+            // No early return on empty frontmatter. Returning here after one error let
+            // a skill with NO frontmatter grade higher than one with BROKEN frontmatter
+            // (the required-field checks below never ran), so total absence outscored a
+            // typo by two errors. Absent fields ARE missing fields; report them too.
         }
         const name = skill.frontmatter.name;
         if (typeof name !== "string" || !name.trim()) {
@@ -254,7 +256,7 @@ const s106 = {
                     fixHint: "agents will fail mid-task when they try to open it",
                 });
             }
-            else if (ref.raw.includes("..")) {
+            else if (ref.escapes) {
                 findings.push({
                     ruleId: "S106",
                     severity: "warn",

package/dist/rules/version.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Which ruleset produced a report.
+ *
+ * The tool version does not answer this. A patch release can change what a
+ * rule fires on without changing any documented behaviour (0.1.2 did exactly
+ * that: E123 started matching `delete_file`, and every grade that rule touched
+ * moved), so "efaimo 0.1.2 said A(95)" is only reproducible if the reader can
+ * also tell which rules were running. Every JSON envelope now carries this.
+ *
+ * Values:
+ *   "1"  every ruleset through efaimo 0.1.2. Reports from those versions carry
+ *        no `rulesVersion` field at all; the absence IS the value.
+ *   "2"  the first ruleset that declares itself. Adds the findability family
+ *        (E141-E145), which no earlier version had.
+ *
+ * Bump it when a rule is added, removed, renumbered, or changed in what it
+ * fires on. `test/meta.test.ts` pins the rule inventory, so an added or
+ * removed rule cannot land without someone looking at this file. It cannot
+ * detect a changed threshold inside an existing rule: that one is human
+ * discipline, and this comment is where the discipline is written down.
+ *
+ * Deliberately not derived from package.json. Tying it to the release number
+ * would make it move on every release whether or not a rule moved, which is
+ * the thing it exists to distinguish.
+ */
+export declare const RULES_VERSION = "2";

package/dist/rules/version.js

@@ -0,0 +1,26 @@
+/**
+ * Which ruleset produced a report.
+ *
+ * The tool version does not answer this. A patch release can change what a
+ * rule fires on without changing any documented behaviour (0.1.2 did exactly
+ * that: E123 started matching `delete_file`, and every grade that rule touched
+ * moved), so "efaimo 0.1.2 said A(95)" is only reproducible if the reader can
+ * also tell which rules were running. Every JSON envelope now carries this.
+ *
+ * Values:
+ *   "1"  every ruleset through efaimo 0.1.2. Reports from those versions carry
+ *        no `rulesVersion` field at all; the absence IS the value.
+ *   "2"  the first ruleset that declares itself. Adds the findability family
+ *        (E141-E145), which no earlier version had.
+ *
+ * Bump it when a rule is added, removed, renumbered, or changed in what it
+ * fires on. `test/meta.test.ts` pins the rule inventory, so an added or
+ * removed rule cannot land without someone looking at this file. It cannot
+ * detect a changed threshold inside an existing rule: that one is human
+ * discipline, and this comment is where the discipline is written down.
+ *
+ * Deliberately not derived from package.json. Tying it to the release number
+ * would make it move on every release whether or not a rule moved, which is
+ * the thing it exists to distinguish.
+ */
+export const RULES_VERSION = "2";

package/dist/testing/anthropicRunner.d.ts

@@ -1,4 +1,34 @@
 import type { Runner } from "./harness.js";
+/**
+ * Which Claude models still accept `temperature` / `top_p` / `top_k`.
+ *
+ * Sampling parameters were removed from the Claude 4.7 line onward and now
+ * return a 400 on Fable 5, Opus 5, Opus 4.8, Opus 4.7 and Sonnet 5. This
+ * runner used to send `temperature` on every request, and the scenario default
+ * model is `claude-sonnet-5`, so `efaimo test --live` sent a rejected
+ * parameter on its first call; `isTransient()` correctly does not retry a 400,
+ * so the whole scenario threw before a single trial completed.
+ *
+ * Nothing in the suite could see it. The runner is injected precisely so the
+ * A/B logic can be tested without spending tokens, which means every test in
+ * this repo exercises a fake and none of them ever built this request body.
+ * The fix is therefore paired with a test that asserts on the body itself.
+ *
+ * The list is an ALLOWLIST of models that accept the parameter, and it is
+ * closed: it names model lines that already shipped, so it cannot go stale in
+ * the direction that breaks a run. A model released after this was written is
+ * unknown to it and gets no `temperature`, which costs a little determinism on
+ * the judge and cannot cost a 400. The opposite shape - a denylist of models
+ * that reject it - would fail every future release by default.
+ */
+export declare function acceptsSamplingParams(model: string): boolean;
+/** The request body, split out so a test can assert on it without a network call. */
+export declare function anthropicRequestBody(req: {
+    model: string;
+    system?: string;
+    user: string;
+    deterministic?: boolean;
+}): Record<string, unknown>;
 /**
  * The live runner. Used only by `efaimo test --live`, because every call spends
  * tokens on the caller's ANTHROPIC_API_KEY.

package/dist/testing/anthropicRunner.js

@@ -1,3 +1,45 @@
+/**
+ * Which Claude models still accept `temperature` / `top_p` / `top_k`.
+ *
+ * Sampling parameters were removed from the Claude 4.7 line onward and now
+ * return a 400 on Fable 5, Opus 5, Opus 4.8, Opus 4.7 and Sonnet 5. This
+ * runner used to send `temperature` on every request, and the scenario default
+ * model is `claude-sonnet-5`, so `efaimo test --live` sent a rejected
+ * parameter on its first call; `isTransient()` correctly does not retry a 400,
+ * so the whole scenario threw before a single trial completed.
+ *
+ * Nothing in the suite could see it. The runner is injected precisely so the
+ * A/B logic can be tested without spending tokens, which means every test in
+ * this repo exercises a fake and none of them ever built this request body.
+ * The fix is therefore paired with a test that asserts on the body itself.
+ *
+ * The list is an ALLOWLIST of models that accept the parameter, and it is
+ * closed: it names model lines that already shipped, so it cannot go stale in
+ * the direction that breaks a run. A model released after this was written is
+ * unknown to it and gets no `temperature`, which costs a little determinism on
+ * the judge and cannot cost a 400. The opposite shape - a denylist of models
+ * that reject it - would fail every future release by default.
+ */
+export function acceptsSamplingParams(model) {
+    return /^claude-(3|(sonnet|haiku|opus)-4-5|(opus|sonnet)-4-6)\b/i.test(model);
+}
+/** The request body, split out so a test can assert on it without a network call. */
+export function anthropicRequestBody(req) {
+    const body = {
+        model: req.model,
+        max_tokens: 1024,
+        ...(req.system ? { system: req.system } : {}),
+        messages: [{ role: "user", content: req.user }],
+    };
+    // 0 for the judge, 1 for the subject (see RunnerRequest.deterministic), but
+    // only where the model still has the parameter. Where it does not, the
+    // subject arm is unaffected (1 was the default anyway) and the judge simply
+    // is not pinned; `runScenario` is told so it can say it in the report,
+    // because an unpinned judge is variance that lands in the measurement.
+    if (acceptsSamplingParams(req.model))
+        body.temperature = req.deterministic ? 0 : 1;
+    return body;
+}
 /**
  * The live runner. Used only by `efaimo test --live`, because every call spends
  * tokens on the caller's ANTHROPIC_API_KEY.
@@ -11,14 +53,7 @@ export function anthropicRunner(apiKey) {
                 "x-api-key": apiKey,
                 "anthropic-version": "2023-06-01",
             },
-            body: JSON.stringify({
-                model: req.model,
-                // 0 for the judge, 1 for the subject. See RunnerRequest.deterministic.
-                temperature: req.deterministic ? 0 : 1,
-                max_tokens: 1024,
-                ...(req.system ? { system: req.system } : {}),
-                messages: [{ role: "user", content: req.user }],
-            }),
+            body: JSON.stringify(anthropicRequestBody(req)),
             redirect: "error",
             signal: AbortSignal.timeout(60000),
         });

package/dist/testing/harness.d.ts

@@ -10,6 +10,24 @@ export interface Scenario {
     skillName: string;
     skillBody: string;
     model: string;
+    /**
+     * The model that grades the attempts. Defaults to `model`, which is what it
+     * always was, and that default is a known confound: a model grading its own
+     * output prefers it, so an effect measured that way is partly a measurement
+     * of self-preference. Set it to a different model (`judge_model:` in the
+     * scenario, or `--judge-model`) and the report prints both.
+     */
+    judgeModel: string;
+    /**
+     * Whether `judge_model:` was written in the scenario.
+     *
+     * `--model` moves the judge with the subject only when the judge was never
+     * pinned. Without this flag the two cases are indistinguishable: a scenario
+     * that deliberately pins `judge_model` to the same id as `model` looked
+     * identical to one that omitted the key, so `--model gpt-4o-mini` would have
+     * moved a deliberately pinned judge.
+     */
+    judgeModelExplicit: boolean;
     trials: number;
     task: string;
     judge: string;
@@ -43,12 +61,23 @@ export interface TestReport {
     scenario: string;
     skill: string;
     model: string;
+    judgeModel: string;
     withSkill: ArmResult;
     withoutSkill: ArmResult;
     /** withSkill.passRate - withoutSkill.passRate, in percentage points. */
     deltaPoints: number;
     /** Two-sided Fisher exact p on the 2x2 of scored trials. */
     p: number;
+    /**
+     * Two-sided Fisher exact p on the 2x2 of UNPARSEABLE trials.
+     *
+     * Dropping a trial the judge did not answer is only unbiased if the dropping
+     * is unrelated to which arm produced it. If one arm loses materially more
+     * trials than the other, the surviving denominators are not comparable and
+     * the delta is measuring the judge as much as the skill. Low p here means
+     * the exclusions are skewed.
+     */
+    unparseableSkewP: number;
     /** 95% interval on the delta, in percentage points (Newcombe). */
     ci: {
         lo: number;
@@ -63,4 +92,7 @@ export declare function armSystems(scenario: Scenario): {
     withoutSkill: string | undefined;
     withSkill: string;
 };
-export declare function runScenario(scenario: Scenario, runner: Runner): Promise<TestReport>;
+export declare function withRetry(runner: Runner, tries?: number, baseDelayMs?: number): Runner;
+export declare function runScenario(scenario: Scenario, runner: Runner, opts?: {
+    extraNotes?: string[];
+}): Promise<TestReport>;

package/dist/testing/harness.js

@@ -29,12 +29,16 @@ export function parseScenario(file) {
     // outcome that reaches p < 0.05, so every partial result is uninterpretable
     // and the old default guaranteed a verdict nobody could stand behind.
     const trials = Math.min(50, Math.max(1, Math.round(num(doc.trials) ?? 20)));
+    const model = str(doc.model) ?? "claude-sonnet-5";
+    const judgeModel = str(doc.judge_model);
     return {
         name,
         skillPath,
         skillName: skill.name ?? path.basename(skill.dir),
         skillBody: skill.body.trim(),
-        model: str(doc.model) ?? "claude-sonnet-5",
+        model,
+        judgeModel: judgeModel ?? model,
+        judgeModelExplicit: judgeModel !== undefined,
         trials,
         task,
         judge,
@@ -60,7 +64,7 @@ async function judgeOne(runner, scenario, answer) {
     const out = await runner({
         system: JUDGE_SYSTEM,
         user: `TASK:\n${scenario.task}\n\nRUBRIC:\n${scenario.judge}\n\nASSISTANT ANSWER:\n${answer}\n\nVerdict (PASS or FAIL):`,
-        model: scenario.model,
+        model: scenario.judgeModel,
         deterministic: true,
     });
     const saysPass = /\bpass(ed|es)?\b/i.test(out);
@@ -71,6 +75,32 @@ async function judgeOne(runner, scenario, answer) {
         return "fail";
     return "unparseable";
 }
+// A live run makes trials * 2 arms * 2 (answer + judge) API calls. One
+// transient 429 or 5xx among them used to throw and discard every trial already
+// paid for. Retry the transient failures with backoff; a non-429 4xx (bad key,
+// unknown model) is a real error and is not retried.
+function isTransient(err) {
+    const m = String(err?.message ?? err);
+    return (/\b(429|500|502|503|504)\b/.test(m) ||
+        /timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|fetch failed|network/i.test(m));
+}
+export function withRetry(runner, tries = 4, baseDelayMs = 500) {
+    return async (req) => {
+        let lastErr;
+        for (let attempt = 0; attempt < tries; attempt++) {
+            try {
+                return await runner(req);
+            }
+            catch (e) {
+                lastErr = e;
+                if (attempt === tries - 1 || !isTransient(e))
+                    throw e;
+                await new Promise((r) => setTimeout(r, baseDelayMs * 2 ** attempt));
+            }
+        }
+        throw lastErr;
+    };
+}
 async function runArm(runner, scenario, system) {
     let passes = 0;
     let unparseable = 0;
@@ -93,10 +123,13 @@ async function runArm(runner, scenario, system) {
         passRate: scored > 0 ? (passes / scored) * 100 : 0,
     };
 }
-export async function runScenario(scenario, runner) {
+export async function runScenario(scenario, runner, opts = {}) {
     const systems = armSystems(scenario);
-    const withoutSkill = await runArm(runner, scenario, systems.withoutSkill);
-    const withSkill = await runArm(runner, scenario, systems.withSkill);
+    // Wrap once so both the subject answers and the judge calls survive a
+    // transient blip instead of throwing away the whole (paid-for) scenario.
+    const r = withRetry(runner);
+    const withoutSkill = await runArm(r, scenario, systems.withoutSkill);
+    const withSkill = await runArm(r, scenario, systems.withSkill);
     const deltaPoints = Math.round((withSkill.passRate - withoutSkill.passRate) * 10) / 10;
     // The verdict is a significance test, not a threshold on the gap.
     //
@@ -108,19 +141,41 @@ export async function runScenario(scenario, runner) {
     // measuring sample size more than it was measuring the skill.
     const p = fisherExactTwoSided(withSkill.passes, withSkill.scored, withoutSkill.passes, withoutSkill.scored);
     const ci = deltaInterval(withSkill.passes, withSkill.scored, withoutSkill.passes, withoutSkill.scored);
+    // With no scoreable trial in an arm the pass rate and its interval are
+    // 0/0 = NaN; say so plainly instead of printing "NaN to NaN points".
+    const bothScored = withSkill.scored > 0 && withoutSkill.scored > 0;
     const notes = [
-        `${scenario.trials} trials per arm. Two-sided Fisher exact p = ${p < 0.0001 ? "<0.0001" : p.toFixed(4)}; ` +
-            `95% interval on the delta ${ci.lo >= 0 ? "+" : ""}${ci.lo} to ${ci.hi >= 0 ? "+" : ""}${ci.hi} points.`,
+        bothScored
+            ? `${scenario.trials} trials per arm. Two-sided Fisher exact p = ${p < 0.0001 ? "<0.0001" : p.toFixed(4)}; ` +
+                `95% interval on the delta ${ci.lo >= 0 ? "+" : ""}${ci.lo} to ${ci.hi >= 0 ? "+" : ""}${ci.hi} points.`
+            : `${scenario.trials} trials per arm. An arm produced no scoreable trial, so no p-value or delta interval can be computed.`,
     ];
     const unscored = withSkill.unparseable + withoutSkill.unparseable;
     if (unscored) {
         notes.push(`${unscored} trial(s) produced a judge verdict that was neither PASS nor FAIL and are excluded from the rates, not counted as failures.`);
     }
+    // Excluding a trial the judge did not answer is the right call, and it is
+    // unbiased only while the exclusions fall on both arms alike. They need not:
+    // a skill that makes answers longer, or hedged, or formatted differently can
+    // make the judge hedge too, and then one arm quietly loses more trials than
+    // the other and the two surviving pass rates are rates over different things.
+    // Tested with the same statistic and the same threshold used for the effect
+    // itself, so there is one standard of evidence in this report, not two.
+    const unparseableSkewP = fisherExactTwoSided(withSkill.unparseable, withSkill.trials, withoutSkill.unparseable, withoutSkill.trials);
+    const exclusionsAreSkewed = unscored > 0 && unparseableSkewP < 0.05;
     let verdict;
     if (withSkill.scored === 0 || withoutSkill.scored === 0) {
         verdict = "inconclusive";
         notes.push("an arm produced no scoreable trial; there is nothing to compare.");
     }
+    else if (exclusionsAreSkewed) {
+        verdict = "inconclusive";
+        notes.push(`the judge failed to answer on ${withSkill.unparseable} of ${withSkill.trials} with-skill trials and ` +
+            `${withoutSkill.unparseable} of ${withoutSkill.trials} without-skill trials, a difference this size at ` +
+            `p = ${unparseableSkewP < 0.0001 ? "<0.0001" : unparseableSkewP.toFixed(4)}. The excluded trials are not ` +
+            `spread evenly across the arms, so the two pass rates are computed over different populations and the ` +
+            `delta cannot be attributed to the skill. Tighten the rubric so the judge always answers, then re-run.`);
+    }
     else if (p >= 0.05) {
         verdict = "no measurable effect";
         if (Math.abs(deltaPoints) >= 15) {
@@ -132,14 +187,18 @@ export async function runScenario(scenario, runner) {
         verdict = "helps";
     else
         verdict = "hurts";
+    if (opts.extraNotes?.length)
+        notes.push(...opts.extraNotes);
     return {
         scenario: scenario.name,
         skill: scenario.skillName,
         model: scenario.model,
+        judgeModel: scenario.judgeModel,
         withSkill,
         withoutSkill,
         deltaPoints,
         p,
+        unparseableSkewP,
         ci,
         verdict,
         notes,

package/dist/testing/openaiRunner.js

@@ -8,20 +8,27 @@ export function openaiRunner(apiKey) {
         if (req.system)
             messages.push({ role: "system", content: req.system });
         messages.push({ role: "user", content: req.user });
+        // o-series reasoning models (o1, o3, o4, ...) reject any non-default
+        // temperature with a 400, and a chat-sized 1024 cap starves them: their
+        // hidden reasoning tokens count against max_completion_tokens, so a small
+        // cap can return empty content. So omit temperature and give them headroom.
+        const isReasoning = /^o[0-9]/i.test(req.model);
+        const body = {
+            model: req.model,
+            messages,
+            max_completion_tokens: isReasoning ? 8192 : 1024,
+        };
+        // temperature: 0 for the judge, 1 for the subject (see
+        // RunnerRequest.deterministic), but only where the model accepts it.
+        if (!isReasoning)
+            body.temperature = req.deterministic ? 0 : 1;
         const res = await fetch("https://api.openai.com/v1/chat/completions", {
             method: "POST",
             headers: {
                 "content-type": "application/json",
                 authorization: `Bearer ${apiKey}`,
             },
-            // temperature: 0 for the judge, 1 for the subject. See
-            // RunnerRequest.deterministic in harness.ts.
-            body: JSON.stringify({
-                model: req.model,
-                temperature: req.deterministic ? 0 : 1,
-                messages,
-                max_completion_tokens: 1024,
-            }),
+            body: JSON.stringify(body),
             redirect: "error",
             signal: AbortSignal.timeout(60000),
         });

package/dist/testing/stats.d.ts

@@ -28,5 +28,19 @@ export declare function deltaInterval(withPass: number, withTrials: number, with
     lo: number;
     hi: number;
 };
-/** The smallest per-arm trial count at which a total separation (n/n vs 0/n) reaches p < 0.05. */
-export declare const MIN_TRIALS_FOR_SIGNIFICANCE = 4;
+/**
+ * The smallest difference in pass rate this many trials per arm could ever
+ * call significant, in percentage points.
+ *
+ * A plan that cannot detect the effect it is looking for produces a green that
+ * means nothing, and the only place to notice that is before spending the
+ * tokens. So the dry run prints this: at 8 trials per arm nothing under about
+ * 50 points is reachable, which is worth knowing before paying for 32 API
+ * calls to find out.
+ *
+ * Computed by exhaustive search over every 2x2 the plan can produce, because
+ * the number of tables is small (trials is capped at 50, so at most 2,601
+ * Fisher evaluations) and a closed form for the exact test is not worth
+ * deriving. Returns Infinity when no outcome at this size reaches alpha.
+ */
+export declare function minimumDetectableDelta(trials: number, alpha?: number): number;

package/dist/testing/stats.js

@@ -87,5 +87,29 @@ export function deltaInterval(withPass, withTrials, withoutPass, withoutTrials)
     const hi = d + Math.sqrt((w.hi - p1) ** 2 + (p2 - o.lo) ** 2);
     return { lo: Math.round(Math.max(-1, lo) * 1000) / 10, hi: Math.round(Math.min(1, hi) * 1000) / 10 };
 }
-/** The smallest per-arm trial count at which a total separation (n/n vs 0/n) reaches p < 0.05. */
-export const MIN_TRIALS_FOR_SIGNIFICANCE = 4;
+/**
+ * The smallest difference in pass rate this many trials per arm could ever
+ * call significant, in percentage points.
+ *
+ * A plan that cannot detect the effect it is looking for produces a green that
+ * means nothing, and the only place to notice that is before spending the
+ * tokens. So the dry run prints this: at 8 trials per arm nothing under about
+ * 50 points is reachable, which is worth knowing before paying for 32 API
+ * calls to find out.
+ *
+ * Computed by exhaustive search over every 2x2 the plan can produce, because
+ * the number of tables is small (trials is capped at 50, so at most 2,601
+ * Fisher evaluations) and a closed form for the exact test is not worth
+ * deriving. Returns Infinity when no outcome at this size reaches alpha.
+ */
+export function minimumDetectableDelta(trials, alpha = 0.05) {
+    let best = Infinity;
+    for (let a = 0; a <= trials; a++) {
+        for (let b = 0; b < a; b++) {
+            if (fisherExactTwoSided(a, trials, b, trials) < alpha) {
+                best = Math.min(best, ((a - b) / trials) * 100);
+            }
+        }
+    }
+    return best;
+}

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Kept in sync with package.json by test/meta.test.ts. */
-export declare const VERSION = "0.1.1";
+export declare const VERSION = "0.2.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 /** Kept in sync with package.json by test/meta.test.ts. */
-export const VERSION = "0.1.1";
+export const VERSION = "0.2.0";

package/dist/weigh/anthropicExact.d.ts

@@ -4,8 +4,19 @@ import type { ToolDef } from "../core/types.js";
  * /v1/messages/count_tokens with and without the tools array (the endpoint
  * accepts `tools`; verified against the official API reference 2026-07-16).
  * Returns undefined on any failure; callers treat this as optional precision.
+ *
+ * Returns the model alongside the count, and the model is not optional in the
+ * result. Claude model lines do not share a tokenizer, so "the exact count" is
+ * exact for one model and an approximation for the next, and a number printed
+ * without the model it was measured on cannot be reproduced or compared. This
+ * is the same rule `src/weigh/window.ts` already applies to the context-window
+ * denominator: never print a figure whose basis is an assumption without
+ * naming the assumption.
  */
 export declare function countClaudeToolTokens(tools: ToolDef[], opts: {
     apiKey: string;
     model?: string;
-}): Promise<number | undefined>;
+}): Promise<{
+    tokens: number;
+    model: string;
+} | undefined>;

package/dist/weigh/anthropicExact.js

@@ -5,6 +5,14 @@ const DEFAULT_MODEL = "claude-sonnet-5";
  * /v1/messages/count_tokens with and without the tools array (the endpoint
  * accepts `tools`; verified against the official API reference 2026-07-16).
  * Returns undefined on any failure; callers treat this as optional precision.
+ *
+ * Returns the model alongside the count, and the model is not optional in the
+ * result. Claude model lines do not share a tokenizer, so "the exact count" is
+ * exact for one model and an approximation for the next, and a number printed
+ * without the model it was measured on cannot be reproduced or compared. This
+ * is the same rule `src/weigh/window.ts` already applies to the context-window
+ * denominator: never print a figure whose basis is an assumption without
+ * naming the assumption.
  */
 export async function countClaudeToolTokens(tools, opts) {
     const model = opts.model ?? DEFAULT_MODEL;
@@ -22,7 +30,7 @@ export async function countClaudeToolTokens(tools, opts) {
         if (withTools === undefined || withoutTools === undefined)
             return undefined;
         const delta = withTools - withoutTools;
-        return delta >= 0 ? delta : undefined;
+        return delta >= 0 ? { tokens: delta, model } : undefined;
     }
     catch {
         return undefined;

package/dist/weigh/weigh.js

@@ -33,13 +33,19 @@ export async function weighServer(intro, opts = {}) {
         "token counts are estimates using the o200k_base tokenizer; hosts add fixed framing text on top (see docs/METHODOLOGY.md)",
     ];
     let anthropicExactTotal;
+    let anthropicExactModel;
     if (opts.anthropicApiKey && intro.tools.length > 0) {
-        anthropicExactTotal = await countClaudeToolTokens(intro.tools, {
+        const exact = await countClaudeToolTokens(intro.tools, {
             apiKey: opts.anthropicApiKey,
             model: opts.anthropicModel,
         });
-        if (anthropicExactTotal !== undefined) {
-            notes.push("anthropic-exact measured via /v1/messages/count_tokens (tools delta)");
+        if (exact !== undefined) {
+            anthropicExactTotal = exact.tokens;
+            anthropicExactModel = exact.model;
+            // Name the model. Claude model lines do not share a tokenizer, so this
+            // figure is exact for `exact.model` and an estimate for anything else,
+            // and a reader comparing two runs has to be able to see which is which.
+            notes.push(`anthropic-exact measured via /v1/messages/count_tokens (tools delta) against ${exact.model}`);
         }
         else {
             notes.push("anthropic-exact measurement failed; showing o200k estimates only");
@@ -56,6 +62,7 @@ export async function weighServer(intro, opts = {}) {
         framingTokens,
         instructionsTokens: count(intro.instructions ?? ""),
         anthropicExactTotal,
+        anthropicExactModel,
         notes,
     };
 }

package/docs/INTEGRATIONS.md

@@ -36,6 +36,18 @@ Ratchet MCP tool-definition cost with the budget gate (commit `base.json` once w
 - run: npx efaimo weigh "npx -y your-server" --diff base.json --allow-increase 10
 ```
 
+Keep every tool findable once a host defers your catalog. `--min-distinct` fails the run
+when a tool owns no word the other tools lack, which means no search can return
+it without also returning a competitor:
+
+```yaml
+- run: npx efaimo find "npx -y your-server" --min-distinct 100
+```
+
+Start it at the number you have today rather than at 100 and raise it as you
+fix tools; the command prints the line to copy. It refuses to run on a one-tool
+catalog, where the figure is 100% by construction, rather than passing.
+
 Post the grade on the PR with `--md` (pipe into any comment action):
 
 ```yaml
@@ -108,7 +120,9 @@ the current SDK.
 
 ## Programmatic use
 
-Every capability is also a typed function:
+Every capability is also a typed function. `analyzeFind`, `runFindRules` and the
+two `find` renderers are exported alongside the rest; the rule
+tables themselves stay private, like the other two families.
 
 ```ts
 import { weighServer, introspectServer, checkSkillSet } from "efaimo";