efaimo 0.1.2 → 0.2.0

package/dist/core/types.d.ts

@@ -155,6 +155,13 @@ export interface ServerWeighResult {
     instructionsTokens: number;
     /** Exact count via Anthropic count_tokens API, when a key was provided. */
     anthropicExactTotal?: number;
+    /**
+     * The model `anthropicExactTotal` was measured against. Always set when the
+     * total is: Claude model lines do not share a tokenizer, so the figure is
+     * exact for this model and approximate for any other, and a count printed
+     * without its model cannot be reproduced.
+     */
+    anthropicExactModel?: string;
     notes: string[];
 }
 export interface SkillWeighEntry {
@@ -178,6 +185,111 @@ export interface SkillSetWeighResult {
     notes: string[];
 }
 export type WeighResult = ServerWeighResult | SkillSetWeighResult;
+/** One tool's result in the findability report (`efaimo find`). */
+export interface ToolFindEntry {
+    name: string;
+    /**
+     * Terms this tool has that no other tool in the catalog has.
+     *
+     * The primary signal, and the only one here that is not a model of anything:
+     * if this is empty, every word the tool contains also appears on some other
+     * tool, so no query can match it without also matching a competitor. That is
+     * a property of the catalog, provable by inspection, with no assumption
+     * about how anyone searches. Capped for display.
+     */
+    ownTerms: string[];
+    /** How many exclusive terms there are in total, before the display cap. */
+    ownTermCount: number;
+    /**
+     * How many of them survive if the searcher does not know the tool's name.
+     *
+     * Names are searchable, so a tool whose only exclusive word is its own name
+     * is findable in principle. It is not findable by anyone describing a task,
+     * which is the situation deferred loading creates. E146 is this number
+     * reaching zero while `ownTermCount` does not.
+     */
+    ownOutsideNameCount: number;
+    /** When ownTermCount is 0: tools whose vocabulary covers this one's (capped). */
+    sharedWith: string[];
+    /**
+     * The terms this tool's own description offers a searcher, most distinctive
+     * first. Never includes the tool's name: the point of the probe is that the
+     * person searching does not know it.
+     */
+    query: string[];
+    /** 1-based rank for that query. Undefined when the description offers no searchable word. */
+    rank?: number;
+    /** BM25 score for its own query, kept so a reader can check the arithmetic. */
+    score: number;
+    /** Tools that scored higher for this tool's own query (capped at 3). */
+    outrankedBy: string[];
+    /** Tools whose score is indistinguishable from this one's, so only the tie-break separates them. Capped for display. */
+    tiedWith: string[];
+    /** How many there are in total, before the display cap. */
+    tiedWithCount: number;
+    /** Rank falls inside the simulated result window. The per-tool form of `probe`. */
+    reachable: boolean;
+    /** Every token of the name is a generic word (E144). */
+    genericName: boolean;
+}
+export interface FindResult {
+    kind: "find";
+    label: string;
+    toolCount: number;
+    /** The whole method, carried with the numbers so a report is self-describing. */
+    method: {
+        tokenizer: string;
+        bm25: {
+            k1: number;
+            b: number;
+        };
+        queryTerms: number;
+        topK: number;
+    };
+    perTool: ToolFindEntry[];
+    /**
+     * The headline: how many tools own at least one word no other tool has.
+     * A measured proportion, deliberately not a letter grade (ADR-030).
+     */
+    distinct: {
+        count: number;
+        total: number;
+        pct: number;
+    };
+    /**
+     * Secondary: how many tools a simulated BM25 search for their own
+     * description returns inside the result window.
+     *
+     * Named `probe` in the JSON, in the output and in the docs, all three. It
+     * was `reach` in one of them and `probe` in the others for a day, which is
+     * how someone scripting a gate on the number they saw printed ends up
+     * looking for a field that does not exist.
+     *
+     * Kept, and kept second, because it is the only number here that models the
+     * actual mechanism, and because it saturates: 100% on every real server
+     * measured. Treat it as a floor test that catches catastrophe, not as a
+     * ranking. See docs/METHODOLOGY.md.
+     */
+    probe: {
+        returned: number;
+        total: number;
+        pct: number;
+    };
+    /** toolCount <= topK, which makes the probe figure 100% by construction. */
+    windowCoversCatalog: boolean;
+    /**
+     * toolCount < 2, which makes the DISTINCT figure 100% by construction: with
+     * one tool every term it has is trivially exclusive. `--min-distinct` cannot
+     * be evaluated in that state and refuses rather than passing.
+     */
+    distinctVacuous: boolean;
+    definitionTokens?: number;
+    /** Anthropic's guidance recommends tool search for a catalog this shape. */
+    deferRecommended: boolean;
+    /** Which of the checkable conditions fired, in words. Empty when none did. */
+    deferBecause: string[];
+    notes: string[];
+}
 export interface SkillInfo {
     dir: string;
     file: string;
@@ -248,6 +360,9 @@ export interface SkillRuleContext {
     set: SkillSet;
     weigh?: SkillSetWeighResult;
 }
+export interface FindRuleContext {
+    find: FindResult;
+}
 export interface McpRule {
     id: string;
     title: string;
@@ -260,3 +375,9 @@ export interface SkillRule {
     surface: "skill";
     check(ctx: SkillRuleContext): Finding[];
 }
+export interface FindRule {
+    id: string;
+    title: string;
+    surface: "find";
+    check(ctx: FindRuleContext): Finding[];
+}

package/dist/find/bm25.d.ts

@@ -0,0 +1,77 @@
+/**
+ * A small, exact BM25 index.
+ *
+ * `efaimo find` needs to answer "would a search over this catalog surface this
+ * tool", and the built-in tool search is server-side: its parameters are not
+ * published, so nothing here can be a reproduction. It is the textbook
+ * Robertson/Sparck-Jones BM25 with its conventional parameters, run offline on
+ * the four fields Anthropic documents as searchable, and both the formula and
+ * the parameters are printed in the output. A reader can disagree with the
+ * model and still check the arithmetic.
+ *
+ * Two properties this file exists to guarantee:
+ *
+ * - **Determinism.** Ties break on the document id by code point, never by
+ *   `localeCompare`, whose collation depends on the host's ICU data. A tool
+ *   that reports other people's nondeterminism (E112) must not have any of its
+ *   own, and "it ranked differently on CI" is exactly the bug that would be
+ *   impossible to reproduce.
+ * - **No silent empties.** `score()` on an empty query returns an empty array
+ *   rather than the whole catalog at score 0. Callers treat that as a finding,
+ *   not as a pass.
+ */
+export interface Bm25Doc {
+    id: string;
+    terms: string[];
+}
+export interface Bm25Hit {
+    id: string;
+    score: number;
+    /**
+     * The document's position in the corpus.
+     *
+     * Present because `id` is not a key. MCP does not forbid two tools sharing a
+     * name, and a caller that located its own row with `findIndex(h => h.id ===
+     * name)` got the better-scoring twin's rank, score and tie set: a tool that
+     * really sat eighth reported rank 1, and the headline moved in the flattering
+     * direction. Match on this instead.
+     */
+    index: number;
+}
+export interface Bm25Params {
+    k1: number;
+    b: number;
+}
+/** The conventional defaults. Printed in every report so the method travels with the number. */
+export declare const DEFAULT_BM25: Bm25Params;
+export declare class Bm25Index {
+    readonly params: Bm25Params;
+    readonly size: number;
+    private readonly ids;
+    private readonly tf;
+    private readonly len;
+    private readonly df;
+    private readonly avgdl;
+    constructor(docs: readonly Bm25Doc[], params?: Bm25Params);
+    /** BM25 IDF. Always positive, so a term present in every document still contributes. */
+    idf(term: string): number;
+    /** Document frequency: how many tools contain this term at all. */
+    documentFrequency(term: string): number;
+    termFrequency(index: number, term: string): number;
+    /**
+     * Score every document against a query, highest first.
+     *
+     * Duplicate query terms are collapsed: a query is a set of keywords here,
+     * not a passage, and counting a repeated term twice would silently weight it.
+     */
+    score(query: readonly string[]): Bm25Hit[];
+}
+/**
+ * Are two scores the same as far as a ranker is concerned?
+ *
+ * Relative, because BM25 scores are unbounded: an absolute epsilon that is
+ * right at score 2 is meaningless at score 40. Two tools this close are
+ * separated only by the tie-break, which means the search picks between them
+ * on the alphabet rather than on relevance.
+ */
+export declare function scoresTie(a: number, bScore: number, rel?: number): boolean;

package/dist/find/bm25.js

@@ -0,0 +1,107 @@
+/**
+ * A small, exact BM25 index.
+ *
+ * `efaimo find` needs to answer "would a search over this catalog surface this
+ * tool", and the built-in tool search is server-side: its parameters are not
+ * published, so nothing here can be a reproduction. It is the textbook
+ * Robertson/Sparck-Jones BM25 with its conventional parameters, run offline on
+ * the four fields Anthropic documents as searchable, and both the formula and
+ * the parameters are printed in the output. A reader can disagree with the
+ * model and still check the arithmetic.
+ *
+ * Two properties this file exists to guarantee:
+ *
+ * - **Determinism.** Ties break on the document id by code point, never by
+ *   `localeCompare`, whose collation depends on the host's ICU data. A tool
+ *   that reports other people's nondeterminism (E112) must not have any of its
+ *   own, and "it ranked differently on CI" is exactly the bug that would be
+ *   impossible to reproduce.
+ * - **No silent empties.** `score()` on an empty query returns an empty array
+ *   rather than the whole catalog at score 0. Callers treat that as a finding,
+ *   not as a pass.
+ */
+/** The conventional defaults. Printed in every report so the method travels with the number. */
+export const DEFAULT_BM25 = { k1: 1.2, b: 0.75 };
+export class Bm25Index {
+    params;
+    size;
+    ids;
+    tf;
+    len;
+    df = new Map();
+    avgdl;
+    constructor(docs, params = DEFAULT_BM25) {
+        this.params = params;
+        this.size = docs.length;
+        this.ids = docs.map((d) => d.id);
+        this.tf = docs.map((d) => {
+            const m = new Map();
+            for (const t of d.terms)
+                m.set(t, (m.get(t) ?? 0) + 1);
+            return m;
+        });
+        this.len = docs.map((d) => d.terms.length);
+        for (const m of this.tf) {
+            for (const t of m.keys())
+                this.df.set(t, (this.df.get(t) ?? 0) + 1);
+        }
+        const total = this.len.reduce((a, b) => a + b, 0);
+        // A catalog of zero-length documents would divide by zero in the length
+        // normalization. It cannot happen with real tools (a name always tokenizes
+        // to something) but the guard costs nothing and a NaN score would rank
+        // unpredictably rather than fail.
+        this.avgdl = docs.length ? Math.max(1, total / docs.length) : 1;
+    }
+    /** BM25 IDF. Always positive, so a term present in every document still contributes. */
+    idf(term) {
+        const n = this.df.get(term) ?? 0;
+        return Math.log(1 + (this.size - n + 0.5) / (n + 0.5));
+    }
+    /** Document frequency: how many tools contain this term at all. */
+    documentFrequency(term) {
+        return this.df.get(term) ?? 0;
+    }
+    termFrequency(index, term) {
+        return this.tf[index]?.get(term) ?? 0;
+    }
+    /**
+     * Score every document against a query, highest first.
+     *
+     * Duplicate query terms are collapsed: a query is a set of keywords here,
+     * not a passage, and counting a repeated term twice would silently weight it.
+     */
+    score(query) {
+        const terms = [...new Set(query)];
+        if (!terms.length)
+            return [];
+        const { k1, b } = this.params;
+        const hits = [];
+        for (let i = 0; i < this.size; i++) {
+            let s = 0;
+            const norm = k1 * (1 - b + (b * this.len[i]) / this.avgdl);
+            for (const t of terms) {
+                const f = this.tf[i].get(t);
+                if (!f)
+                    continue;
+                s += this.idf(t) * ((f * (k1 + 1)) / (f + norm));
+            }
+            hits.push({ id: this.ids[i], score: s, index: i });
+        }
+        // Ties break on the id by code point, then on position, so two documents
+        // with the same name and the same score still have a fixed order.
+        hits.sort((x, y) => (y.score - x.score) || (x.id < y.id ? -1 : x.id > y.id ? 1 : 0) || (x.index - y.index));
+        return hits;
+    }
+}
+/**
+ * Are two scores the same as far as a ranker is concerned?
+ *
+ * Relative, because BM25 scores are unbounded: an absolute epsilon that is
+ * right at score 2 is meaningless at score 40. Two tools this close are
+ * separated only by the tie-break, which means the search picks between them
+ * on the alphabet rather than on relevance.
+ */
+export function scoresTie(a, bScore, rel = 1e-9) {
+    const scale = Math.max(Math.abs(a), Math.abs(bScore), 1);
+    return Math.abs(a - bScore) <= rel * scale;
+}

package/dist/find/find.d.ts

@@ -0,0 +1,120 @@
+import type { FindResult, ToolDef } from "../core/types.js";
+import { Bm25Index } from "./bm25.js";
+/**
+ * Findability: can a search over this catalog pick this tool out at all?
+ *
+ * Why this exists. Since 2026-01 a host can mark tools `defer_loading: true`,
+ * which keeps them out of the system-prompt prefix entirely; the model then
+ * finds them by searching the catalog, and Anthropic recommends turning that
+ * on once tool definitions pass ~10k tokens. Under deferral a tool no search
+ * surfaces costs zero tokens and provides zero capability, and no existing
+ * check looks at that: `weigh` measures what a tool costs, `check` measures
+ * what a model experiences once the tool is already in front of it.
+ *
+ * Two measurements, and they are not the same kind of thing.
+ *
+ * 1. EXCLUSIVE VOCABULARY, the headline. Which terms does a tool have that no
+ *    other tool in the catalog has? A tool with none cannot be matched by any
+ *    query that does not also match a competitor, because every word it
+ *    contains is somewhere else too. That is a fact about the catalog, checked
+ *    by inspection, with no model of how anyone searches, no parameters, and
+ *    nothing to disagree with except the tokenizer.
+ *
+ *    It also discriminates, which the alternative did not. Measured on
+ *    `@playwright/mcp` 0.0.78: 22 of 24 tools own a word, and `browser_close`
+ *    and `browser_navigate` own none.
+ *
+ * 2. THE SEARCH PROBE, secondary. Simulated BM25 over the four fields tool
+ *    search documents as searchable, each tool queried with the top terms of
+ *    its own description. This is a model, and a generous one: the query comes
+ *    from the tool's own words. It reads 100% on both the official reference
+ *    server and playwright, so it is a floor test that catches catastrophe -
+ *    an empty description, a literal duplicate - and not a ranking. The report
+ *    says so on every run rather than letting a saturated number look like a
+ *    good score.
+ *
+ * Neither is a grade. See ADR-030 for why this ships as proportions.
+ */
+/** How many of a tool's own terms make up its self-query. */
+export declare const DEFAULT_QUERY_TERMS = 4;
+/**
+ * How many results a search returns. Anthropic documents a default of 5 (the
+ * model may raise it), so 5 is what a tool has to land inside to be seen
+ * without the model deciding to look harder.
+ */
+export declare const DEFAULT_TOP_K = 5;
+/**
+ * When Anthropic's own guidance says to turn tool search on.
+ *
+ * The doc lists five conditions joined by "any of the following apply", and
+ * two of them can be checked from a tool list: "You have 10 or more tools
+ * available" and "Your tool definitions consume more than 10k tokens". The
+ * other three are about the operator's situation, not the catalog.
+ *
+ * Both are checked. An earlier version implemented only the token clause,
+ * which told a reader that a 24-tool, 3.5k-token server was "probably loaded
+ * up front today" when Anthropic's first condition already applied to it, and
+ * left E145 structurally unable to fire on the most common shape there is: a
+ * catalog with plenty of tools and modest descriptions.
+ */
+export declare const DEFER_RECOMMENDED_TOKENS = 10000;
+export declare const DEFER_RECOMMENDED_TOOLS = 10;
+/**
+ * The four fields tool search matches on: name, description, argument names,
+ * argument descriptions. Kept in one function so the index and the docs can
+ * never disagree about what is searched.
+ *
+ * `title` is deliberately NOT among them, for two reasons that point the same
+ * way. Anthropic's list of searchable fields is these four and does not
+ * include it. And MCP `title` is a human-readable restatement of the name:
+ * every one of the reference server's thirteen tools sets one, and every one
+ * is the name title-cased ("Echo Tool", "Get Annotated Message Tool"). An
+ * earlier version indexed and PROBED with it, which quietly put the tool's own
+ * name back into its own query through the back door and undid the whole
+ * reason the name is excluded. Two tools with identical descriptions and
+ * name-derived titles scored a clean rank 1 each instead of tying.
+ *
+ * Indexing a field the real search does not read would also make `distinct`
+ * optimistic: a tool could "own" a word that exists only in its title, where
+ * no query can reach it.
+ */
+export declare function indexedTerms(tool: ToolDef): string[];
+/**
+ * The terms a person would plausibly type to reach this tool.
+ *
+ * Drawn from the description, and deliberately NOT from the name, the title,
+ * or the argument names.
+ *
+ * Not the name, because the person searching does not know it. That is the
+ * whole situation being modelled: the tool is deferred, the user describes a
+ * task, and the model turns that description into a query. An earlier version
+ * of this drew query terms from the name too, and the result was a metric that
+ * could barely fail - a unique name token has maximal idf, so it entered every
+ * query and pulled its own tool to the top. Two tools with the same
+ * description and different names both scored a clean rank 1, which is exactly
+ * backwards: identical descriptions are the case where a searcher cannot tell
+ * them apart. The name stays in the INDEX, so a name whose words echo the
+ * description still helps the tool rank.
+ *
+ * Not the title either, and that one is the same bug wearing a different hat:
+ * MCP `title` is usually the name title-cased, so reading it here put the name
+ * back into its own query after all the trouble taken to keep it out. See
+ * `indexedTerms`, which drops it from the index for the same reason.
+ *
+ * Not argument names, because a user describes a task ("post a message to
+ * slack"), not a parameter list. They are indexed, so a search can still reach
+ * them; they are just never what the tool is probed with.
+ *
+ * Ranked by tf*idf inside this catalog, so the terms chosen are the ones that
+ * separate this tool from its siblings, with stopwords excluded because in a
+ * small catalog a function word repeated in one description can outrank the
+ * noun that identifies the tool.
+ */
+export declare function taskQuery(tool: ToolDef, index: Bm25Index, limit?: number): string[];
+export interface FindOptions {
+    topK?: number;
+    queryTerms?: number;
+    /** Claude-style tool-definition tokens, used only to say whether deferral is likely. */
+    definitionTokens?: number;
+}
+export declare function analyzeFind(label: string, tools: readonly ToolDef[], opts?: FindOptions): FindResult;