@bitbaum/ai-kit 1.2.0 → 1.3.0

package/README.md

@@ -333,6 +333,67 @@ It does not summarise, re-rank with an LLM, crawl, render JavaScript, or cache.
 The first two are the model's job and belong upstream where the app's prompt
 lives; the last three are a different product with a different cost profile.
 
+### What can this model do? — observe, never assert
+
+```ts
+import {
+  planToolAttempt,
+  classifyToolAttempt,
+  claimableVerdict,
+  currentVerdict,
+  makeRecord,
+  scopeKey,
+} from "@bitbaum/ai-kit/capability";
+
+const observed = currentVerdict(await store.get(key)); // "native" | "text" | "none" | "unobserved"
+const plan = planToolAttempt({ observed }); // what to put on the wire
+
+const res = await fetch(endpoint, {
+  /* … tools attached when plan.sendTools … */
+});
+const seen = classifyToolAttempt({ status: res.status, parsed, bodyText, textProtocolFound });
+if (seen.record) await store.put(makeRecord({ ...key, verdict: seen.verdict, via: "live" }));
+```
+
+This replaces the line every app writes and every app gets wrong:
+
+```ts
+const TOOL_CAPABLE_PROVIDERS = ["groq", "openrouter"]; // wrong tomorrow
+```
+
+That list is wrong the moment a user brings a model nobody has heard of, which
+is every day. It is also wrong in the other direction: it cannot express that
+**five of nine** free models probed here answer tools only in prose, so a
+native-only client loses most of its chain while believing it is fine.
+
+**The first real call is the probe.** Send the tools, read what comes back,
+write down what it proved. Every model a user brings classifies itself on its
+first message, at no extra cost and with no release from us. Nothing here
+spends a separate request, which matters most when the key is the user's.
+
+**A positive is cheap; a negative is expensive and sticky.** One `tool_calls`
+response proves capability outright. A 400 proves nothing *unless the vendor
+says it is about tools* — a context-length overflow, a content filter or a bad
+parameter must be recorded as nothing at all, because writing one down as "no
+tools" cripples a capable model until the record expires and nothing in the
+product explains why. Hence `record: false`: the right response to an
+uninformative failure is to learn nothing, not to guess.
+
+**"Never asked" is its own answer, read two ways.** `planToolAttempt` is
+optimistic about it, because asking is the only way to learn. `claimableVerdict`
+is pessimistic about it, because announcing a capability a model has never
+demonstrated is a promise it may not keep, and the user meets that as a broken
+feature rather than a missing one. A single boolean cannot hold both, which is
+the whole reason this module exists.
+
+Records expire, and negatives expire sooner than positives: a model that gained
+tool support and is still marked incapable is invisibly crippled, while one that
+lost it says so loudly on the next call. Observations are keyed by a **hash** of
+the credential, never the credential, because capability genuinely differs per
+key and observations must not leak across them.
+
+Storage stays yours — a table, a KV, a file. This owns the shape and the rules.
+
 ---
 
 ## What it deliberately does not ship

package/dist/capability/classify.d.ts

@@ -0,0 +1,52 @@
+/**
+ * Reading a real response for what it proves about capability.
+ *
+ * The whole design rests on one asymmetry:
+ *
+ *   A POSITIVE is cheap. One response carrying `tool_calls` proves, beyond
+ *   argument, that this model on this provider with this key can call tools.
+ *   Write it down immediately.
+ *
+ *   A NEGATIVE is expensive and sticky. If a 400 gets recorded as "this model
+ *   has no tools", that model is crippled until the record expires — and the
+ *   user sees a capable model behaving like a toy with nothing explaining why.
+ *   So a negative requires the vendor to SAY it is about tools. A 400 for a
+ *   context-length overflow, a malformed parameter, a content filter or a
+ *   billing problem proves nothing about tools and must be recorded as
+ *   nothing at all.
+ *
+ * That asymmetry is why `record: false` exists. Most failures are
+ * uninformative, and the correct response to an uninformative failure is to
+ * learn nothing, not to guess.
+ */
+import type { Classification } from "./types.js";
+/**
+ * Does this error body explicitly say the model cannot do tools?
+ *
+ * Conservative on purpose — see the header. A false positive here is a model
+ * permanently downgraded for a reason nobody can see; a false negative just
+ * means we ask again next time, which costs one request.
+ */
+export declare function saysToolsUnsupported(body: string): boolean;
+export type ToolAttempt = {
+    /** HTTP status. 0 or undefined for a transport failure. */
+    status?: number;
+    /** Parsed JSON body, when there was one. */
+    parsed?: unknown;
+    /** Raw body text. Used only for error classification. */
+    bodyText?: string;
+    /**
+     * Did the caller's own text-protocol parser find a usable tool call in the
+     * assistant's prose? Only the app knows its envelope, so it answers this.
+     * Absent means "not checked", which is not the same as "no".
+     */
+    textProtocolFound?: boolean;
+};
+/**
+ * What does this attempt prove?
+ *
+ * Call it after EVERY request that carried tool definitions. Real traffic then
+ * classifies every model a user brings, on its first message, at no extra cost
+ * — which is the property that makes this scale to models nobody has heard of.
+ */
+export declare function classifyToolAttempt(attempt: ToolAttempt): Classification;

package/dist/capability/classify.js

@@ -0,0 +1,96 @@
+/**
+ * Phrases that mean "this model does not do tools", conservatively.
+ *
+ * Every entry names tools or functions explicitly. Deliberately absent:
+ * "invalid request", "bad parameter", "unsupported" on its own — each of those
+ * appears in vendor 400s for a dozen unrelated reasons, and a match on one of
+ * them would silently disable a working model. When in doubt the answer is to
+ * record nothing; an unobserved model gets asked again on the next message,
+ * whereas a wrongly-negative one does not.
+ */
+const TOOLS_UNSUPPORTED_PATTERNS = [
+    /tool[\s_-]?(use|call|calls|calling)\s+(is\s+)?(not|un)[\s_-]?support/i,
+    /does\s+not\s+support\s+tool/i,
+    /doesn'?t\s+support\s+tool/i,
+    /no\s+support\s+for\s+tool/i,
+    /function[\s_-]?call(ing)?\s+(is\s+)?(not|un)[\s_-]?support/i,
+    /does\s+not\s+support\s+function/i,
+    /doesn'?t\s+support\s+function/i,
+    /model\s+.{0,60}?\s+does\s+not\s+support\s+(the\s+)?(`?tools`?|`?functions`?)/i,
+    /unsupported\s+parameter:?\s*'?"?tools?"?'?/i,
+    /unknown\s+(field|parameter):?\s*'?"?tools?"?'?/i,
+    /`?tools`?\s+is\s+not\s+(a\s+)?(valid|supported|allowed)/i,
+];
+/**
+ * Does this error body explicitly say the model cannot do tools?
+ *
+ * Conservative on purpose — see the header. A false positive here is a model
+ * permanently downgraded for a reason nobody can see; a false negative just
+ * means we ask again next time, which costs one request.
+ */
+export function saysToolsUnsupported(body) {
+    if (!body)
+        return false;
+    return TOOLS_UNSUPPORTED_PATTERNS.some((re) => re.test(body));
+}
+/**
+ * What does this attempt prove?
+ *
+ * Call it after EVERY request that carried tool definitions. Real traffic then
+ * classifies every model a user brings, on its first message, at no extra cost
+ * — which is the property that makes this scale to models nobody has heard of.
+ */
+export function classifyToolAttempt(attempt) {
+    const { status, parsed, bodyText = "", textProtocolFound } = attempt;
+    // ── transport failures prove nothing ────────────────────────────────────
+    if (!status) {
+        return { verdict: "unobserved", record: false, evidence: "no response" };
+    }
+    // ── the vendor refused, and we must be careful about why ────────────────
+    if (status >= 400) {
+        if (saysToolsUnsupported(bodyText)) {
+            return {
+                verdict: "none",
+                record: true,
+                evidence: `${status}: the vendor says this model does not support tools`,
+            };
+        }
+        // Everything else — 429, 401, 500, a 400 about context length or a bad
+        // parameter — says nothing about tools. Learning nothing is correct.
+        return {
+            verdict: "unobserved",
+            record: false,
+            evidence: `${status}: not a statement about tool support`,
+        };
+    }
+    // ── a success: did it actually call a tool? ─────────────────────────────
+    const body = (parsed ?? {});
+    const choice = body.choices?.[0];
+    const toolCalls = choice?.message?.tool_calls;
+    if (Array.isArray(toolCalls) && toolCalls.length > 0) {
+        return { verdict: "native", record: true, evidence: "returned tool_calls" };
+    }
+    if (choice?.finish_reason === "tool_calls") {
+        return { verdict: "native", record: true, evidence: "finish_reason was tool_calls" };
+    }
+    // The app's own envelope parser found a call in the prose. That is the text
+    // protocol, and it is a real capability — five of nine free models probed in
+    // this fleet answer only this way.
+    if (textProtocolFound === true) {
+        return {
+            verdict: "text",
+            record: true,
+            evidence: "no tool_calls, but a tool call was parsed from the text",
+        };
+    }
+    // A successful answer with no tool call is the ambiguous case, and the
+    // ambiguity is real: the model may be incapable, or it may simply have
+    // decided no tool was needed — which is the correct behaviour for most
+    // messages. Treating this as evidence of incapacity would mark almost every
+    // model `none` within a few turns of ordinary chat.
+    return {
+        verdict: "unobserved",
+        record: false,
+        evidence: "answered without calling a tool, which is not evidence either way",
+    };
+}

package/dist/capability/decide.d.ts

@@ -0,0 +1,75 @@
+import type { CapabilityKind, CapabilityRecord, ToolVerdict } from "./types.js";
+/**
+ * How long an observation stands before it must be re-earned.
+ *
+ * Models change under their own names — a vendor updates the weights behind an
+ * alias, an org enables a feature, a local user swaps a quantization. A record
+ * with no expiry is a hardcoded list again, just one we wrote ourselves.
+ *
+ * Negatives expire sooner than positives: a model that gained tool support and
+ * is still marked `none` is invisibly crippled, while a model that lost it
+ * announces itself loudly on the next call.
+ */
+export declare const DEFAULT_TTL_MS: {
+    readonly native: number;
+    readonly text: number;
+    readonly none: number;
+    readonly unobserved: 0;
+};
+export declare function isStale(record: Pick<CapabilityRecord, "verdict" | "observedAt">, now?: Date, ttl?: Partial<Record<ToolVerdict, number>>): boolean;
+/** The verdict a record still supports, or `unobserved` once it has expired. */
+export declare function currentVerdict(record: CapabilityRecord | null | undefined, now?: Date, ttl?: Partial<Record<ToolVerdict, number>>): ToolVerdict;
+export type ToolPlan = {
+    /** Put tool definitions on the request? */
+    sendTools: boolean;
+    /** Parse the prose for a tool envelope as well? */
+    expectTextProtocol: boolean;
+    /** True when this request is also the thing that will teach us. */
+    isLearning: boolean;
+    reason: string;
+};
+/**
+ * What to send. Optimistic about the unknown, because asking is how we learn
+ * and a declared prior is only a guess about where to start.
+ */
+export declare function planToolAttempt(input: {
+    observed: ToolVerdict;
+    /** What a registry or the vendor's docs claim. A prior, never a fact. */
+    declared?: ToolVerdict;
+}): ToolPlan;
+/**
+ * What we may TELL the user, and what the prompt may claim.
+ *
+ * Pessimistic about the unknown. `unobserved` returns `none` here on purpose:
+ * until a model has demonstrated a capability, an assistant that announces it
+ * is writing a cheque the model may not honour, and the user discovers that as
+ * a broken promise rather than as a missing feature.
+ */
+export declare function claimableVerdict(observed: ToolVerdict): Exclude<ToolVerdict, "unobserved">;
+/**
+ * A stable, non-reversible handle for the credential an observation was made
+ * through. Capability differs per key, so observations must not leak across
+ * keys — and the key itself must never be stored to achieve that.
+ */
+export declare function scopeKey(secret: string | undefined | null): string;
+/** Build a record from a classification. Keeps `observedAt` in one place. */
+export declare function makeRecord(input: {
+    provider: string;
+    model: string;
+    scope: string;
+    capability: CapabilityKind;
+    verdict: ToolVerdict;
+    via: CapabilityRecord["via"];
+    evidence?: string;
+    now?: Date;
+}): CapabilityRecord;
+/**
+ * Should a new observation overwrite the stored one?
+ *
+ * Strength beats age, and `live` beats `declared`, so a real call always
+ * overrules a registry guess. Between two observations of equal provenance the
+ * newer wins — including a `none` replacing a `native`, because a model really
+ * can lose a capability and refusing to believe that is how a chain keeps
+ * calling something that no longer works.
+ */
+export declare function shouldReplace(existing: CapabilityRecord | null | undefined, incoming: CapabilityRecord): boolean;

package/dist/capability/decide.js

@@ -0,0 +1,143 @@
+/**
+ * What to send this time, and what to tell the user we can do.
+ *
+ * Two different questions, deliberately separated:
+ *
+ *   `planToolAttempt` decides what to PUT ON THE WIRE. It is optimistic about
+ *   an unobserved model, because the only way to learn is to ask, and the cost
+ *   of asking is one request that may ignore the tools.
+ *
+ *   `claimableVerdict` decides what to SAY. It is pessimistic about an
+ *   unobserved model, because promising a capability we have never seen is how
+ *   an assistant comes to announce an action it cannot perform.
+ *
+ * Those two pulling in opposite directions is the whole point. A single
+ * "supportsTools" boolean cannot express it, and every app that has tried has
+ * either refused to learn or lied to its users.
+ */
+import { createHash } from "node:crypto";
+/**
+ * How long an observation stands before it must be re-earned.
+ *
+ * Models change under their own names — a vendor updates the weights behind an
+ * alias, an org enables a feature, a local user swaps a quantization. A record
+ * with no expiry is a hardcoded list again, just one we wrote ourselves.
+ *
+ * Negatives expire sooner than positives: a model that gained tool support and
+ * is still marked `none` is invisibly crippled, while a model that lost it
+ * announces itself loudly on the next call.
+ */
+export const DEFAULT_TTL_MS = {
+    native: 30 * 24 * 60 * 60 * 1000,
+    text: 30 * 24 * 60 * 60 * 1000,
+    none: 7 * 24 * 60 * 60 * 1000,
+    unobserved: 0,
+};
+export function isStale(record, now = new Date(), ttl = {}) {
+    const limit = ttl[record.verdict] ?? DEFAULT_TTL_MS[record.verdict];
+    if (!limit)
+        return true;
+    const age = now.getTime() - new Date(record.observedAt).getTime();
+    return !Number.isFinite(age) || age > limit;
+}
+/** The verdict a record still supports, or `unobserved` once it has expired. */
+export function currentVerdict(record, now = new Date(), ttl = {}) {
+    if (!record)
+        return "unobserved";
+    return isStale(record, now, ttl) ? "unobserved" : record.verdict;
+}
+/**
+ * What to send. Optimistic about the unknown, because asking is how we learn
+ * and a declared prior is only a guess about where to start.
+ */
+export function planToolAttempt(input) {
+    const { observed, declared } = input;
+    if (observed === "native") {
+        return {
+            sendTools: true,
+            expectTextProtocol: false,
+            isLearning: false,
+            reason: "observed to return tool_calls",
+        };
+    }
+    if (observed === "text") {
+        return {
+            sendTools: false,
+            expectTextProtocol: true,
+            isLearning: false,
+            reason: "observed to answer tools only in prose",
+        };
+    }
+    if (observed === "none") {
+        return {
+            sendTools: false,
+            expectTextProtocol: false,
+            isLearning: false,
+            reason: "the vendor said this model does not support tools",
+        };
+    }
+    // Unobserved. Ask — and accept EITHER answer, because a model that ignores
+    // the definitions and writes the envelope in prose is capable, just not
+    // natively, and a native-only client silently loses most of a free chain.
+    return {
+        sendTools: declared !== "none",
+        expectTextProtocol: true,
+        isLearning: true,
+        reason: declared === "none"
+            ? "never observed, and the registry says no — asking in prose only"
+            : "never observed — this request is also the probe",
+    };
+}
+/**
+ * What we may TELL the user, and what the prompt may claim.
+ *
+ * Pessimistic about the unknown. `unobserved` returns `none` here on purpose:
+ * until a model has demonstrated a capability, an assistant that announces it
+ * is writing a cheque the model may not honour, and the user discovers that as
+ * a broken promise rather than as a missing feature.
+ */
+export function claimableVerdict(observed) {
+    return observed === "unobserved" ? "none" : observed;
+}
+/**
+ * A stable, non-reversible handle for the credential an observation was made
+ * through. Capability differs per key, so observations must not leak across
+ * keys — and the key itself must never be stored to achieve that.
+ */
+export function scopeKey(secret) {
+    if (!secret)
+        return "anonymous";
+    return createHash("sha256").update(secret).digest("hex").slice(0, 16);
+}
+/** Build a record from a classification. Keeps `observedAt` in one place. */
+export function makeRecord(input) {
+    return {
+        provider: input.provider,
+        model: input.model,
+        scope: input.scope,
+        capability: input.capability,
+        verdict: input.verdict,
+        via: input.via,
+        observedAt: (input.now ?? new Date()).toISOString(),
+        ...(input.evidence ? { evidence: input.evidence } : {}),
+    };
+}
+/**
+ * Should a new observation overwrite the stored one?
+ *
+ * Strength beats age, and `live` beats `declared`, so a real call always
+ * overrules a registry guess. Between two observations of equal provenance the
+ * newer wins — including a `none` replacing a `native`, because a model really
+ * can lose a capability and refusing to believe that is how a chain keeps
+ * calling something that no longer works.
+ */
+export function shouldReplace(existing, incoming) {
+    if (!existing)
+        return true;
+    const rank = { declared: 0, probe: 1, live: 2 };
+    if (rank[incoming.via] > rank[existing.via])
+        return true;
+    if (rank[incoming.via] < rank[existing.via])
+        return false;
+    return new Date(incoming.observedAt).getTime() >= new Date(existing.observedAt).getTime();
+}

package/dist/capability/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * ai-kit/capability — what a model can do, observed rather than declared.
+ *
+ * Replaces the list every app writes and every app gets wrong:
+ *
+ *   const TOOL_CAPABLE_PROVIDERS = ['groq', 'openrouter'];
+ *
+ * That line is wrong the moment a user brings a model nobody on the team has
+ * heard of, which is every day. It is also wrong in the other direction: it
+ * cannot express that five of nine free models answer tools only in prose, so
+ * a native-only client silently loses most of its chain while believing it is
+ * fine.
+ *
+ * The replacement is not a better list. It is three rules:
+ *
+ *   1. The first real call IS the probe. Send tools, read what comes back,
+ *      write down what it proved. Every model a user brings classifies itself
+ *      on its first message, at no extra cost and with no release from us.
+ *   2. A positive is cheap and a negative is expensive. One `tool_calls`
+ *      proves capability. A 400 proves nothing unless the vendor SAYS it is
+ *      about tools — otherwise a context-length overflow permanently cripples
+ *      a capable model and nothing explains why.
+ *   3. "Never asked" is its own answer. Optimistic on the wire, because asking
+ *      is how we learn; pessimistic in the prompt and the UI, because
+ *      announcing an unproven capability is a promise the model may not keep.
+ *
+ * Storage stays with the app — a table, a KV, a file. This package owns the
+ * shape and the rules, which is the part everyone gets wrong; it does not own
+ * where rows live, which is the part where apps legitimately differ.
+ */
+export type { ToolVerdict, CapabilityKind, Provenance, CapabilityRecord, CapabilityStore, Classification, } from "./types.js";
+export { classifyToolAttempt, saysToolsUnsupported, type ToolAttempt } from "./classify.js";
+export { planToolAttempt, claimableVerdict, currentVerdict, isStale, makeRecord, scopeKey, shouldReplace, DEFAULT_TTL_MS, type ToolPlan, } from "./decide.js";

package/dist/capability/index.js

@@ -0,0 +1,2 @@
+export { classifyToolAttempt, saysToolsUnsupported } from "./classify.js";
+export { planToolAttempt, claimableVerdict, currentVerdict, isStale, makeRecord, scopeKey, shouldReplace, DEFAULT_TTL_MS, } from "./decide.js";

package/dist/capability/types.d.ts

@@ -0,0 +1,81 @@
+/**
+ * What a model can actually do, as OBSERVED rather than as claimed.
+ *
+ * The problem this exists for: capability is not a property of a model name.
+ * It is a property of a model, on a provider, through a particular deployment,
+ * reached with a particular credential. A quantized local build drops tool
+ * support the upstream weights have. A proxy strips `tool_calls`. An org's key
+ * has vision disabled. A vendor updates a model in place behind an alias. None
+ * of that is knowable from a name, and every one of it is knowable by asking
+ * once.
+ *
+ * So this module holds no list of models. It holds the shape of an observation,
+ * the rules for turning a real call into one, and the decision of what to send
+ * next time. The list every app is tempted to write — "these providers support
+ * tools" — is the thing being replaced: it was wrong the day a user brought a
+ * model nobody had heard of, which is every day.
+ */
+/**
+ * How a model answers a request carrying tool definitions.
+ *
+ * Four values, and the fourth is the one that matters. `unobserved` is not a
+ * synonym for `none`: it means nobody has ever asked, and a system that treats
+ * it as `none` silently disables tools for every model it has not met yet,
+ * while a system that treats it as `native` promises a capability it cannot
+ * demonstrate. It has to stay its own answer all the way to the user.
+ */
+export type ToolVerdict = "native" | "text" | "none" | "unobserved";
+/** What a single capability question is asked about. */
+export type CapabilityKind = "tools" | "vision";
+/** How we came to believe something, ordered weakest to strongest. */
+export type Provenance = 
+/** The vendor's docs or a hand-maintained registry. A prior, never a fact. */
+"declared"
+/** A deliberate probe request made to answer this question. */
+ | "probe"
+/** Real traffic the user asked for, which answered it for free. */
+ | "live";
+export type CapabilityRecord = {
+    provider: string;
+    model: string;
+    /**
+     * Which credential this was observed through — a HASH, never the key.
+     * Capability differs per key (an org with vision disabled, a proxy that
+     * strips tool calls), so an observation made with one credential is not
+     * evidence about another.
+     */
+    scope: string;
+    capability: CapabilityKind;
+    verdict: ToolVerdict;
+    /** ISO 8601. Used for staleness; models change under their own names. */
+    observedAt: string;
+    via: Provenance;
+    /** Short, human-readable reason. Goes in logs and in the UI. */
+    evidence?: string;
+};
+/**
+ * Storage is the app's problem — a table, a KV, a file. This package owns the
+ * shape and the rules, because those are what every app gets wrong; it does not
+ * own where the rows live, because that is the one part where apps legitimately
+ * differ.
+ */
+export type CapabilityStore = {
+    get(key: {
+        provider: string;
+        model: string;
+        scope: string;
+        capability: CapabilityKind;
+    }): Promise<CapabilityRecord | null>;
+    put(record: CapabilityRecord): Promise<void>;
+};
+/** The outcome of reading one real response for what it says about capability. */
+export type Classification = {
+    verdict: ToolVerdict;
+    /**
+     * Whether this is worth WRITING DOWN. A response can be uninformative —
+     * a 429, a 500, a timeout, a 400 about something other than tools — and
+     * recording those is how a model gets wrongly marked incapable forever.
+     */
+    record: boolean;
+    evidence: string;
+};

package/dist/capability/types.js

@@ -0,0 +1,18 @@
+/**
+ * What a model can actually do, as OBSERVED rather than as claimed.
+ *
+ * The problem this exists for: capability is not a property of a model name.
+ * It is a property of a model, on a provider, through a particular deployment,
+ * reached with a particular credential. A quantized local build drops tool
+ * support the upstream weights have. A proxy strips `tool_calls`. An org's key
+ * has vision disabled. A vendor updates a model in place behind an alias. None
+ * of that is knowable from a name, and every one of it is knowable by asking
+ * once.
+ *
+ * So this module holds no list of models. It holds the shape of an observation,
+ * the rules for turning a real call into one, and the decision of what to send
+ * next time. The list every app is tempted to write — "these providers support
+ * tools" — is the thing being replaced: it was wrong the day a user brought a
+ * model nobody had heard of, which is every day.
+ */
+export {};

package/package.json

@@ -1,7 +1,6 @@
 {
   "name": "@bitbaum/ai-kit",
-  "version": "1.2.0",
-  "description": "One install for the AI layer of an app: which model to call, what to do when the vendor retires it, how to walk the fallback chain and know when none of it worked, how to read the three kinds of 429, a fair daily budget across users, headless AI form filling \u2014 and now the model registry (one SSOT for every callable id, with the paid/free boundary as a field) and the grounding harness (facts, contract, deterministic fabrication check).",
+  "version": "1.3.0",
   "license": "MIT",
   "author": "Mao Nakamoto",
   "homepage": "https://github.com/bitbaum/ai-kit#readme",
@@ -67,6 +66,10 @@
       "types": "./dist/web/index.d.ts",
       "default": "./dist/web/index.js"
     },
+    "./capability": {
+      "types": "./dist/capability/index.d.ts",
+      "default": "./dist/capability/index.js"
+    },
     "./grounding": {
       "types": "./dist/grounding/index.d.ts",
       "require": "./dist-cjs/grounding/index.js",

package/src/capability/classify.ts

@@ -0,0 +1,149 @@
+/**
+ * Reading a real response for what it proves about capability.
+ *
+ * The whole design rests on one asymmetry:
+ *
+ *   A POSITIVE is cheap. One response carrying `tool_calls` proves, beyond
+ *   argument, that this model on this provider with this key can call tools.
+ *   Write it down immediately.
+ *
+ *   A NEGATIVE is expensive and sticky. If a 400 gets recorded as "this model
+ *   has no tools", that model is crippled until the record expires — and the
+ *   user sees a capable model behaving like a toy with nothing explaining why.
+ *   So a negative requires the vendor to SAY it is about tools. A 400 for a
+ *   context-length overflow, a malformed parameter, a content filter or a
+ *   billing problem proves nothing about tools and must be recorded as
+ *   nothing at all.
+ *
+ * That asymmetry is why `record: false` exists. Most failures are
+ * uninformative, and the correct response to an uninformative failure is to
+ * learn nothing, not to guess.
+ */
+import type { Classification } from "./types.js";
+
+/**
+ * Phrases that mean "this model does not do tools", conservatively.
+ *
+ * Every entry names tools or functions explicitly. Deliberately absent:
+ * "invalid request", "bad parameter", "unsupported" on its own — each of those
+ * appears in vendor 400s for a dozen unrelated reasons, and a match on one of
+ * them would silently disable a working model. When in doubt the answer is to
+ * record nothing; an unobserved model gets asked again on the next message,
+ * whereas a wrongly-negative one does not.
+ */
+const TOOLS_UNSUPPORTED_PATTERNS: RegExp[] = [
+  /tool[\s_-]?(use|call|calls|calling)\s+(is\s+)?(not|un)[\s_-]?support/i,
+  /does\s+not\s+support\s+tool/i,
+  /doesn'?t\s+support\s+tool/i,
+  /no\s+support\s+for\s+tool/i,
+  /function[\s_-]?call(ing)?\s+(is\s+)?(not|un)[\s_-]?support/i,
+  /does\s+not\s+support\s+function/i,
+  /doesn'?t\s+support\s+function/i,
+  /model\s+.{0,60}?\s+does\s+not\s+support\s+(the\s+)?(`?tools`?|`?functions`?)/i,
+  /unsupported\s+parameter:?\s*'?"?tools?"?'?/i,
+  /unknown\s+(field|parameter):?\s*'?"?tools?"?'?/i,
+  /`?tools`?\s+is\s+not\s+(a\s+)?(valid|supported|allowed)/i,
+];
+
+/**
+ * Does this error body explicitly say the model cannot do tools?
+ *
+ * Conservative on purpose — see the header. A false positive here is a model
+ * permanently downgraded for a reason nobody can see; a false negative just
+ * means we ask again next time, which costs one request.
+ */
+export function saysToolsUnsupported(body: string): boolean {
+  if (!body) return false;
+  return TOOLS_UNSUPPORTED_PATTERNS.some((re) => re.test(body));
+}
+
+/** Shape of an OpenAI-compatible chat completion, as far as we care. */
+type ChatBody = {
+  choices?: Array<{
+    finish_reason?: unknown;
+    message?: { content?: unknown; tool_calls?: unknown };
+  }>;
+};
+
+export type ToolAttempt = {
+  /** HTTP status. 0 or undefined for a transport failure. */
+  status?: number;
+  /** Parsed JSON body, when there was one. */
+  parsed?: unknown;
+  /** Raw body text. Used only for error classification. */
+  bodyText?: string;
+  /**
+   * Did the caller's own text-protocol parser find a usable tool call in the
+   * assistant's prose? Only the app knows its envelope, so it answers this.
+   * Absent means "not checked", which is not the same as "no".
+   */
+  textProtocolFound?: boolean;
+};
+
+/**
+ * What does this attempt prove?
+ *
+ * Call it after EVERY request that carried tool definitions. Real traffic then
+ * classifies every model a user brings, on its first message, at no extra cost
+ * — which is the property that makes this scale to models nobody has heard of.
+ */
+export function classifyToolAttempt(attempt: ToolAttempt): Classification {
+  const { status, parsed, bodyText = "", textProtocolFound } = attempt;
+
+  // ── transport failures prove nothing ────────────────────────────────────
+  if (!status) {
+    return { verdict: "unobserved", record: false, evidence: "no response" };
+  }
+
+  // ── the vendor refused, and we must be careful about why ────────────────
+  if (status >= 400) {
+    if (saysToolsUnsupported(bodyText)) {
+      return {
+        verdict: "none",
+        record: true,
+        evidence: `${status}: the vendor says this model does not support tools`,
+      };
+    }
+    // Everything else — 429, 401, 500, a 400 about context length or a bad
+    // parameter — says nothing about tools. Learning nothing is correct.
+    return {
+      verdict: "unobserved",
+      record: false,
+      evidence: `${status}: not a statement about tool support`,
+    };
+  }
+
+  // ── a success: did it actually call a tool? ─────────────────────────────
+  const body = (parsed ?? {}) as ChatBody;
+  const choice = body.choices?.[0];
+  const toolCalls = choice?.message?.tool_calls;
+
+  if (Array.isArray(toolCalls) && toolCalls.length > 0) {
+    return { verdict: "native", record: true, evidence: "returned tool_calls" };
+  }
+  if (choice?.finish_reason === "tool_calls") {
+    return { verdict: "native", record: true, evidence: "finish_reason was tool_calls" };
+  }
+
+  // The app's own envelope parser found a call in the prose. That is the text
+  // protocol, and it is a real capability — five of nine free models probed in
+  // this fleet answer only this way.
+  if (textProtocolFound === true) {
+    return {
+      verdict: "text",
+      record: true,
+      evidence: "no tool_calls, but a tool call was parsed from the text",
+    };
+  }
+
+  // A successful answer with no tool call is the ambiguous case, and the
+  // ambiguity is real: the model may be incapable, or it may simply have
+  // decided no tool was needed — which is the correct behaviour for most
+  // messages. Treating this as evidence of incapacity would mark almost every
+  // model `none` within a few turns of ordinary chat.
+  return {
+    verdict: "unobserved",
+    record: false,
+    evidence: "answered without calling a tool, which is not evidence either way",
+  };
+}

package/src/capability/decide.ts

@@ -0,0 +1,183 @@
+/**
+ * What to send this time, and what to tell the user we can do.
+ *
+ * Two different questions, deliberately separated:
+ *
+ *   `planToolAttempt` decides what to PUT ON THE WIRE. It is optimistic about
+ *   an unobserved model, because the only way to learn is to ask, and the cost
+ *   of asking is one request that may ignore the tools.
+ *
+ *   `claimableVerdict` decides what to SAY. It is pessimistic about an
+ *   unobserved model, because promising a capability we have never seen is how
+ *   an assistant comes to announce an action it cannot perform.
+ *
+ * Those two pulling in opposite directions is the whole point. A single
+ * "supportsTools" boolean cannot express it, and every app that has tried has
+ * either refused to learn or lied to its users.
+ */
+import { createHash } from "node:crypto";
+import type { CapabilityKind, CapabilityRecord, ToolVerdict } from "./types.js";
+
+/**
+ * How long an observation stands before it must be re-earned.
+ *
+ * Models change under their own names — a vendor updates the weights behind an
+ * alias, an org enables a feature, a local user swaps a quantization. A record
+ * with no expiry is a hardcoded list again, just one we wrote ourselves.
+ *
+ * Negatives expire sooner than positives: a model that gained tool support and
+ * is still marked `none` is invisibly crippled, while a model that lost it
+ * announces itself loudly on the next call.
+ */
+export const DEFAULT_TTL_MS = {
+  native: 30 * 24 * 60 * 60 * 1000,
+  text: 30 * 24 * 60 * 60 * 1000,
+  none: 7 * 24 * 60 * 60 * 1000,
+  unobserved: 0,
+} as const;
+
+export function isStale(
+  record: Pick<CapabilityRecord, "verdict" | "observedAt">,
+  now: Date = new Date(),
+  ttl: Partial<Record<ToolVerdict, number>> = {},
+): boolean {
+  const limit = ttl[record.verdict] ?? DEFAULT_TTL_MS[record.verdict];
+  if (!limit) return true;
+  const age = now.getTime() - new Date(record.observedAt).getTime();
+  return !Number.isFinite(age) || age > limit;
+}
+
+/** The verdict a record still supports, or `unobserved` once it has expired. */
+export function currentVerdict(
+  record: CapabilityRecord | null | undefined,
+  now: Date = new Date(),
+  ttl: Partial<Record<ToolVerdict, number>> = {},
+): ToolVerdict {
+  if (!record) return "unobserved";
+  return isStale(record, now, ttl) ? "unobserved" : record.verdict;
+}
+
+export type ToolPlan = {
+  /** Put tool definitions on the request? */
+  sendTools: boolean;
+  /** Parse the prose for a tool envelope as well? */
+  expectTextProtocol: boolean;
+  /** True when this request is also the thing that will teach us. */
+  isLearning: boolean;
+  reason: string;
+};
+
+/**
+ * What to send. Optimistic about the unknown, because asking is how we learn
+ * and a declared prior is only a guess about where to start.
+ */
+export function planToolAttempt(input: {
+  observed: ToolVerdict;
+  /** What a registry or the vendor's docs claim. A prior, never a fact. */
+  declared?: ToolVerdict;
+}): ToolPlan {
+  const { observed, declared } = input;
+
+  if (observed === "native") {
+    return {
+      sendTools: true,
+      expectTextProtocol: false,
+      isLearning: false,
+      reason: "observed to return tool_calls",
+    };
+  }
+  if (observed === "text") {
+    return {
+      sendTools: false,
+      expectTextProtocol: true,
+      isLearning: false,
+      reason: "observed to answer tools only in prose",
+    };
+  }
+  if (observed === "none") {
+    return {
+      sendTools: false,
+      expectTextProtocol: false,
+      isLearning: false,
+      reason: "the vendor said this model does not support tools",
+    };
+  }
+
+  // Unobserved. Ask — and accept EITHER answer, because a model that ignores
+  // the definitions and writes the envelope in prose is capable, just not
+  // natively, and a native-only client silently loses most of a free chain.
+  return {
+    sendTools: declared !== "none",
+    expectTextProtocol: true,
+    isLearning: true,
+    reason:
+      declared === "none"
+        ? "never observed, and the registry says no — asking in prose only"
+        : "never observed — this request is also the probe",
+  };
+}
+
+/**
+ * What we may TELL the user, and what the prompt may claim.
+ *
+ * Pessimistic about the unknown. `unobserved` returns `none` here on purpose:
+ * until a model has demonstrated a capability, an assistant that announces it
+ * is writing a cheque the model may not honour, and the user discovers that as
+ * a broken promise rather than as a missing feature.
+ */
+export function claimableVerdict(observed: ToolVerdict): Exclude<ToolVerdict, "unobserved"> {
+  return observed === "unobserved" ? "none" : observed;
+}
+
+/**
+ * A stable, non-reversible handle for the credential an observation was made
+ * through. Capability differs per key, so observations must not leak across
+ * keys — and the key itself must never be stored to achieve that.
+ */
+export function scopeKey(secret: string | undefined | null): string {
+  if (!secret) return "anonymous";
+  return createHash("sha256").update(secret).digest("hex").slice(0, 16);
+}
+
+/** Build a record from a classification. Keeps `observedAt` in one place. */
+export function makeRecord(input: {
+  provider: string;
+  model: string;
+  scope: string;
+  capability: CapabilityKind;
+  verdict: ToolVerdict;
+  via: CapabilityRecord["via"];
+  evidence?: string;
+  now?: Date;
+}): CapabilityRecord {
+  return {
+    provider: input.provider,
+    model: input.model,
+    scope: input.scope,
+    capability: input.capability,
+    verdict: input.verdict,
+    via: input.via,
+    observedAt: (input.now ?? new Date()).toISOString(),
+    ...(input.evidence ? { evidence: input.evidence } : {}),
+  };
+}
+
+/**
+ * Should a new observation overwrite the stored one?
+ *
+ * Strength beats age, and `live` beats `declared`, so a real call always
+ * overrules a registry guess. Between two observations of equal provenance the
+ * newer wins — including a `none` replacing a `native`, because a model really
+ * can lose a capability and refusing to believe that is how a chain keeps
+ * calling something that no longer works.
+ */
+export function shouldReplace(
+  existing: CapabilityRecord | null | undefined,
+  incoming: CapabilityRecord,
+): boolean {
+  if (!existing) return true;
+  const rank = { declared: 0, probe: 1, live: 2 } as const;
+  if (rank[incoming.via] > rank[existing.via]) return true;
+  if (rank[incoming.via] < rank[existing.via]) return false;
+  return new Date(incoming.observedAt).getTime() >= new Date(existing.observedAt).getTime();
+}

package/src/capability/index.ts

@@ -0,0 +1,52 @@
+/**
+ * ai-kit/capability — what a model can do, observed rather than declared.
+ *
+ * Replaces the list every app writes and every app gets wrong:
+ *
+ *   const TOOL_CAPABLE_PROVIDERS = ['groq', 'openrouter'];
+ *
+ * That line is wrong the moment a user brings a model nobody on the team has
+ * heard of, which is every day. It is also wrong in the other direction: it
+ * cannot express that five of nine free models answer tools only in prose, so
+ * a native-only client silently loses most of its chain while believing it is
+ * fine.
+ *
+ * The replacement is not a better list. It is three rules:
+ *
+ *   1. The first real call IS the probe. Send tools, read what comes back,
+ *      write down what it proved. Every model a user brings classifies itself
+ *      on its first message, at no extra cost and with no release from us.
+ *   2. A positive is cheap and a negative is expensive. One `tool_calls`
+ *      proves capability. A 400 proves nothing unless the vendor SAYS it is
+ *      about tools — otherwise a context-length overflow permanently cripples
+ *      a capable model and nothing explains why.
+ *   3. "Never asked" is its own answer. Optimistic on the wire, because asking
+ *      is how we learn; pessimistic in the prompt and the UI, because
+ *      announcing an unproven capability is a promise the model may not keep.
+ *
+ * Storage stays with the app — a table, a KV, a file. This package owns the
+ * shape and the rules, which is the part everyone gets wrong; it does not own
+ * where rows live, which is the part where apps legitimately differ.
+ */
+export type {
+  ToolVerdict,
+  CapabilityKind,
+  Provenance,
+  CapabilityRecord,
+  CapabilityStore,
+  Classification,
+} from "./types.js";
+
+export { classifyToolAttempt, saysToolsUnsupported, type ToolAttempt } from "./classify.js";
+
+export {
+  planToolAttempt,
+  claimableVerdict,
+  currentVerdict,
+  isStale,
+  makeRecord,
+  scopeKey,
+  shouldReplace,
+  DEFAULT_TTL_MS,
+  type ToolPlan,
+} from "./decide.js";

package/src/capability/types.ts

@@ -0,0 +1,87 @@
+/**
+ * What a model can actually do, as OBSERVED rather than as claimed.
+ *
+ * The problem this exists for: capability is not a property of a model name.
+ * It is a property of a model, on a provider, through a particular deployment,
+ * reached with a particular credential. A quantized local build drops tool
+ * support the upstream weights have. A proxy strips `tool_calls`. An org's key
+ * has vision disabled. A vendor updates a model in place behind an alias. None
+ * of that is knowable from a name, and every one of it is knowable by asking
+ * once.
+ *
+ * So this module holds no list of models. It holds the shape of an observation,
+ * the rules for turning a real call into one, and the decision of what to send
+ * next time. The list every app is tempted to write — "these providers support
+ * tools" — is the thing being replaced: it was wrong the day a user brought a
+ * model nobody had heard of, which is every day.
+ */
+
+/**
+ * How a model answers a request carrying tool definitions.
+ *
+ * Four values, and the fourth is the one that matters. `unobserved` is not a
+ * synonym for `none`: it means nobody has ever asked, and a system that treats
+ * it as `none` silently disables tools for every model it has not met yet,
+ * while a system that treats it as `native` promises a capability it cannot
+ * demonstrate. It has to stay its own answer all the way to the user.
+ */
+export type ToolVerdict = "native" | "text" | "none" | "unobserved";
+
+/** What a single capability question is asked about. */
+export type CapabilityKind = "tools" | "vision";
+
+/** How we came to believe something, ordered weakest to strongest. */
+export type Provenance =
+  /** The vendor's docs or a hand-maintained registry. A prior, never a fact. */
+  | "declared"
+  /** A deliberate probe request made to answer this question. */
+  | "probe"
+  /** Real traffic the user asked for, which answered it for free. */
+  | "live";
+
+export type CapabilityRecord = {
+  provider: string;
+  model: string;
+  /**
+   * Which credential this was observed through — a HASH, never the key.
+   * Capability differs per key (an org with vision disabled, a proxy that
+   * strips tool calls), so an observation made with one credential is not
+   * evidence about another.
+   */
+  scope: string;
+  capability: CapabilityKind;
+  verdict: ToolVerdict;
+  /** ISO 8601. Used for staleness; models change under their own names. */
+  observedAt: string;
+  via: Provenance;
+  /** Short, human-readable reason. Goes in logs and in the UI. */
+  evidence?: string;
+};
+
+/**
+ * Storage is the app's problem — a table, a KV, a file. This package owns the
+ * shape and the rules, because those are what every app gets wrong; it does not
+ * own where the rows live, because that is the one part where apps legitimately
+ * differ.
+ */
+export type CapabilityStore = {
+  get(key: {
+    provider: string;
+    model: string;
+    scope: string;
+    capability: CapabilityKind;
+  }): Promise<CapabilityRecord | null>;
+  put(record: CapabilityRecord): Promise<void>;
+};
+
+/** The outcome of reading one real response for what it says about capability. */
+export type Classification = {
+  verdict: ToolVerdict;
+  /**
+   * Whether this is worth WRITING DOWN. A response can be uninformative —
+   * a 429, a 500, a timeout, a 400 about something other than tools — and
+   * recording those is how a model gets wrongly marked incapable forever.
+   */
+  record: boolean;
+  evidence: string;
+};