@bitbaum/ai-kit 1.2.0 → 1.4.0

package/src/capability/index.ts

@@ -0,0 +1,52 @@
+/**
+ * ai-kit/capability — what a model can do, observed rather than declared.
+ *
+ * Replaces the list every app writes and every app gets wrong:
+ *
+ *   const TOOL_CAPABLE_PROVIDERS = ['groq', 'openrouter'];
+ *
+ * That line is wrong the moment a user brings a model nobody on the team has
+ * heard of, which is every day. It is also wrong in the other direction: it
+ * cannot express that five of nine free models answer tools only in prose, so
+ * a native-only client silently loses most of its chain while believing it is
+ * fine.
+ *
+ * The replacement is not a better list. It is three rules:
+ *
+ *   1. The first real call IS the probe. Send tools, read what comes back,
+ *      write down what it proved. Every model a user brings classifies itself
+ *      on its first message, at no extra cost and with no release from us.
+ *   2. A positive is cheap and a negative is expensive. One `tool_calls`
+ *      proves capability. A 400 proves nothing unless the vendor SAYS it is
+ *      about tools — otherwise a context-length overflow permanently cripples
+ *      a capable model and nothing explains why.
+ *   3. "Never asked" is its own answer. Optimistic on the wire, because asking
+ *      is how we learn; pessimistic in the prompt and the UI, because
+ *      announcing an unproven capability is a promise the model may not keep.
+ *
+ * Storage stays with the app — a table, a KV, a file. This package owns the
+ * shape and the rules, which is the part everyone gets wrong; it does not own
+ * where rows live, which is the part where apps legitimately differ.
+ */
+export type {
+  ToolVerdict,
+  CapabilityKind,
+  Provenance,
+  CapabilityRecord,
+  CapabilityStore,
+  Classification,
+} from "./types.js";
+
+export { classifyToolAttempt, saysToolsUnsupported, type ToolAttempt } from "./classify.js";
+
+export {
+  planToolAttempt,
+  claimableVerdict,
+  currentVerdict,
+  isStale,
+  makeRecord,
+  scopeKey,
+  shouldReplace,
+  DEFAULT_TTL_MS,
+  type ToolPlan,
+} from "./decide.js";

package/src/capability/types.ts

@@ -0,0 +1,87 @@
+/**
+ * What a model can actually do, as OBSERVED rather than as claimed.
+ *
+ * The problem this exists for: capability is not a property of a model name.
+ * It is a property of a model, on a provider, through a particular deployment,
+ * reached with a particular credential. A quantized local build drops tool
+ * support the upstream weights have. A proxy strips `tool_calls`. An org's key
+ * has vision disabled. A vendor updates a model in place behind an alias. None
+ * of that is knowable from a name, and every one of it is knowable by asking
+ * once.
+ *
+ * So this module holds no list of models. It holds the shape of an observation,
+ * the rules for turning a real call into one, and the decision of what to send
+ * next time. The list every app is tempted to write — "these providers support
+ * tools" — is the thing being replaced: it was wrong the day a user brought a
+ * model nobody had heard of, which is every day.
+ */
+
+/**
+ * How a model answers a request carrying tool definitions.
+ *
+ * Four values, and the fourth is the one that matters. `unobserved` is not a
+ * synonym for `none`: it means nobody has ever asked, and a system that treats
+ * it as `none` silently disables tools for every model it has not met yet,
+ * while a system that treats it as `native` promises a capability it cannot
+ * demonstrate. It has to stay its own answer all the way to the user.
+ */
+export type ToolVerdict = "native" | "text" | "none" | "unobserved";
+
+/** What a single capability question is asked about. */
+export type CapabilityKind = "tools" | "vision";
+
+/** How we came to believe something, ordered weakest to strongest. */
+export type Provenance =
+  /** The vendor's docs or a hand-maintained registry. A prior, never a fact. */
+  | "declared"
+  /** A deliberate probe request made to answer this question. */
+  | "probe"
+  /** Real traffic the user asked for, which answered it for free. */
+  | "live";
+
+export type CapabilityRecord = {
+  provider: string;
+  model: string;
+  /**
+   * Which credential this was observed through — a HASH, never the key.
+   * Capability differs per key (an org with vision disabled, a proxy that
+   * strips tool calls), so an observation made with one credential is not
+   * evidence about another.
+   */
+  scope: string;
+  capability: CapabilityKind;
+  verdict: ToolVerdict;
+  /** ISO 8601. Used for staleness; models change under their own names. */
+  observedAt: string;
+  via: Provenance;
+  /** Short, human-readable reason. Goes in logs and in the UI. */
+  evidence?: string;
+};
+
+/**
+ * Storage is the app's problem — a table, a KV, a file. This package owns the
+ * shape and the rules, because those are what every app gets wrong; it does not
+ * own where the rows live, because that is the one part where apps legitimately
+ * differ.
+ */
+export type CapabilityStore = {
+  get(key: {
+    provider: string;
+    model: string;
+    scope: string;
+    capability: CapabilityKind;
+  }): Promise<CapabilityRecord | null>;
+  put(record: CapabilityRecord): Promise<void>;
+};
+
+/** The outcome of reading one real response for what it says about capability. */
+export type Classification = {
+  verdict: ToolVerdict;
+  /**
+   * Whether this is worth WRITING DOWN. A response can be uninformative —
+   * a 429, a 500, a timeout, a 400 about something other than tools — and
+   * recording those is how a model gets wrongly marked incapable forever.
+   */
+  record: boolean;
+  evidence: string;
+};

package/src/complete.ts

@@ -61,6 +61,7 @@ import { ChainExhaustedError, type ChainAttemptFailure } from "./attempt.js";
 import type { HealthTracker } from "./health.js";
 import { classifyRateLimit, retryAfterSeconds, type RateLimitKind } from "./limits.js";
 import { readQuota, readingFromRefusal, type QuotaReading } from "./meter.js";
+import { parseTextToolCalls, stripToolCallLines, toolNamesFrom } from "./tool-protocol.js";
 
 /** One message in the OpenAI chat-completions shape every provider here speaks. */
 export interface ChatMessage {
@@ -76,9 +77,11 @@ export interface ChatMessage {
  * actually answer on.
  *
  * Both exist in the default chain: of nine free models probed live, four
- * answered with native `tool_calls` and five only in text. Callers get the
- * native shape here; parsing the text protocol is the caller's business,
- * because its convention differs per app.
+ * answered with native `tool_calls` and five only in text — and three of the
+ * seven models shipped in the chain today are in that second group. Since 1.4.0
+ * both are read here, so a caller no longer has to know which half of the chain
+ * answered. See `toolProtocol` on CompleteOptions, and tool-protocol.ts for why
+ * a native-only read is a fabrication path rather than a missing feature.
  */
 export interface ToolCall {
   id: string;
@@ -145,6 +148,23 @@ export interface CompleteOptions {
   temperature?: number;
   /** Tool definitions in the OpenAI shape; passed through untouched. */
   tools?: unknown[];
+  /**
+   * Which tool-call protocols to READ from the reply. Default `"both"`.
+   *
+   * `"both"` also parses the `TOOL:` / `ARGS:` line protocol out of ordinary
+   * content and strips those lines from `text`. This matters more than it
+   * sounds: three of the seven models in the default chain cannot emit a native
+   * tool call at all, and a native-only read hands their narration back as a
+   * finished answer. The turn then reports a lookup that never happened.
+   *
+   * Parsing is skipped entirely when no `tools` are supplied — a model does not
+   * narrate a call it was never offered — so this is inert for the callers who
+   * do not use tools, which today is all of them.
+   *
+   * Set `"native"` only if you parse the text protocol yourself and would
+   * otherwise execute each call twice.
+   */
+  toolProtocol?: "both" | "native";
   /** Extra body fields for a vendor-specific parameter. Merged last, so it can override. */
   extraBody?: Record<string, unknown>;
   /**
@@ -432,11 +452,31 @@ async function callLink(
 
   const choice = (parsed as { choices?: Array<{ message?: Record<string, unknown> }> })
     ?.choices?.[0];
-  const content = firstText(choice?.message);
-  const toolCalls = toolCallsFrom(choice?.message);
+  const rawContent = firstText(choice?.message);
+  const native = toolCallsFrom(choice?.message);
+
+  // Read the line protocol out of the prose as well, unless the caller opted
+  // out or offered no tools. Native wins on a tie: a model that emits BOTH the
+  // real call and a prose echo of it must not run the tool twice — that wastes
+  // a round trip and can double-propose an action.
+  const wantsText = (options.toolProtocol ?? "both") === "both" && (options.tools?.length ?? 0) > 0;
+  const fromText = wantsText
+    ? parseTextToolCalls(rawContent, toolNamesFrom(options.tools)).filter((c) => {
+        const key = `${c.name}:${c.args}`;
+        return !native.some((n) => `${n.name}:${n.args}` === key);
+      })
+    : [];
+
+  const toolCalls = [...native, ...fromText];
+  // Strip the protocol lines only when they were actually read as calls.
+  // Removing them without parsing would delete the evidence and leave a shorter
+  // hallucination behind, which is worse than either extreme.
+  const content = fromText.length > 0 ? stripToolCallLines(rawContent) : rawContent;
 
   // A 200 that carries neither text nor a tool call is an outage wearing a
   // success code — see the header. Demote, so the chain gets its chance.
+  // Note the ordering: a reply that was ONLY a narrated call is now empty text
+  // WITH tool calls, which is a valid turn, not an outage.
   if (content.trim() === "" && toolCalls.length === 0) {
     throw new LinkFailure(
       link,

package/src/index.ts

@@ -128,6 +128,15 @@ export {
   answersRemaining,
 } from "./meter.js";
 
+export {
+  type ParsedToolCall,
+  TEXT_TOOL_PROTOCOL_HINT,
+  parseTextToolCalls,
+  stripToolCallLines,
+  safeJsonObject,
+  toolNamesFrom,
+} from "./tool-protocol.js";
+
 export {
   type PoolId,
   type RungId,

package/src/tool-protocol.ts

@@ -0,0 +1,223 @@
+/**
+ * The other way models ask for a tool: by writing it in the reply.
+ *
+ * ── WHY A NATIVE-ONLY CLIENT LOSES MOST OF THIS PACKAGE'S OWN CHAIN ──────────
+ *
+ * `chain.ts` carries a probe table from nine free models called with a real
+ * tool. Four answered with native `tool_calls`. Five answered only in TEXT.
+ * Of the seven models in the shipped default chain today, three are in that
+ * second group:
+ *
+ *   google/gemma-4-26b-a4b-it:free      text
+ *   cohere/north-mini-code:free         text
+ *   openrouter/free                     text
+ *
+ * A client that reads only `message.tool_calls` gets ZERO tool calls from those
+ * three — and, far worse, gets the model's narration of the call as ordinary
+ * content. So a turn that should have looked up a record instead returns a
+ * confident sentence describing the lookup it did not perform, and every layer
+ * downstream treats it as an answer. That is not a missing feature; it is a
+ * fabrication path, and it is open on nearly half the chain this package ships.
+ *
+ * ── WHY LINE-BASED AND NOT NESTED JSON ───────────────────────────────────────
+ *
+ * The format is chosen for the WEAKEST model expected to run it, not the
+ * strongest. An 8B model reliably reproduces two flat lines:
+ *
+ *     TOOL: search_people
+ *     ARGS: {"query": "Elena"}
+ *
+ * The same model routinely breaks nested-JSON escaping. Every leniency in the
+ * parser below is a shape a small model actually emitted in production — bolded
+ * keys because it was writing markdown, a fenced ARGS block, the parentheses
+ * copied from the example, a missing ARGS line for a no-argument tool. Rejecting
+ * any of them would fail the turn over formatting, which is the exact failure
+ * this protocol exists to avoid.
+ */
+
+/** The tool-call shape, matching `complete`'s. `args` stays a raw JSON string. */
+export interface ParsedToolCall {
+  id: string;
+  name: string;
+  args: string;
+}
+
+/**
+ * The convention, as prompt text.
+ *
+ * Exported so apps stop writing their own wording and drifting from the parser
+ * that has to read it back. A prompt and its parser are one contract; keeping
+ * them in separate repos is how they diverge.
+ *
+ * Note what it does NOT say: it never tells the model to use this INSTEAD of
+ * native calling. A model with native support should use it, and this is the
+ * fallback for one that cannot — offering both costs a few tokens and covers
+ * both halves of the chain.
+ */
+export const TEXT_TOOL_PROTOCOL_HINT = [
+  "If you cannot emit a native tool call, call a tool by writing these two lines in your reply, exactly like this:",
+  "",
+  "TOOL: tool_name",
+  'ARGS: {"argument": "value"}',
+  "",
+  "Write nothing else in a reply that calls tools — you will be given the results and asked again.",
+].join("\n");
+
+/** Matches a TOOL: line, tolerating list bullets, markdown bold, and `=`. */
+const TOOL_LINE = /^\s*(?:[-*>]\s*)?(?:\*\*)?TOOL(?:\*\*)?\s*[:=]\s*(.+?)\s*$/i;
+/** Matches an ARGS: line, same tolerances. */
+const ARGS_LINE = /^\s*(?:[-*>]\s*)?(?:\*\*)?ARGS(?:\*\*)?\s*[:=]\s*(.*)$/i;
+/** Either key, for the strip pass. */
+const EITHER_LINE = /^\s*(?:[-*>]\s*)?(?:\*\*)?(?:TOOL|ARGS)(?:\*\*)?\s*[:=]/i;
+
+/** How far past a TOOL: line to look for its ARGS:, and how far to join a multi-line object. */
+const ARGS_SEARCH_LINES = 4;
+const ARGS_JOIN_LINES = 8;
+
+/**
+ * Parse a JSON object, tolerating fences and trailing prose. Null if hopeless.
+ *
+ * `null` and `{}` must stay distinguishable. Returning `{}` for "nothing
+ * parseable here" satisfies a caller's `?? {}` fallback and silently discards
+ * arguments the model put on the NEXT line — which is exactly what a model does
+ * when it opens a ```json fence after `ARGS:`. That cost a debugging cycle once
+ * already; it is a test here now.
+ */
+export function safeJsonObject(raw: string): Record<string, unknown> | null {
+  const cleaned = raw
+    .trim()
+    .replace(/^```(?:json)?/i, "")
+    .replace(/```$/, "")
+    .trim();
+  if (!cleaned) return null;
+  if (cleaned === "{}") return {};
+
+  const start = cleaned.indexOf("{");
+  if (start === -1) return null;
+
+  // Walk to the matching brace so trailing commentary does not break the parse.
+  // String-aware: a brace inside a quoted value must not change the depth, or
+  // an argument like {"q": "a } b"} truncates at the wrong place.
+  let depth = 0;
+  let inString = false;
+  let escaped = false;
+  for (let i = start; i < cleaned.length; i++) {
+    const ch = cleaned[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (ch === "\\") {
+      escaped = true;
+      continue;
+    }
+    if (ch === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (inString) continue;
+    if (ch === "{") depth++;
+    else if (ch === "}") {
+      depth--;
+      if (depth === 0) {
+        try {
+          const parsed: unknown = JSON.parse(cleaned.slice(start, i + 1));
+          return parsed && typeof parsed === "object" && !Array.isArray(parsed)
+            ? (parsed as Record<string, unknown>)
+            : null;
+        } catch {
+          return null;
+        }
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * Extract text-protocol calls from a reply.
+ *
+ * `validNames` is the closed set the model was offered. A line naming anything
+ * else is left alone: without that check, a model writing the words "TOOL:
+ * whatever" in prose would manufacture a call to a tool that does not exist,
+ * and the executor would report a failure the model never asked for.
+ */
+export function parseTextToolCalls(text: string, validNames: string[]): ParsedToolCall[] {
+  if (!text || validNames.length === 0) return [];
+  const calls: ParsedToolCall[] = [];
+  const valid = new Set(validNames);
+  const lines = text.split(/\r?\n/);
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i] ?? "";
+    const m = TOOL_LINE.exec(line);
+    const rawName = m?.[1];
+    if (!rawName) continue;
+
+    // Tolerate `name(...)`, backticks, and trailing punctuation copied from prose.
+    const name = rawName
+      .replace(/[`*]/g, "")
+      .replace(/\(.*$/, "")
+      .replace(/[.,;]$/, "")
+      .trim();
+    if (!valid.has(name)) continue;
+
+    let args = "{}";
+    for (let j = i + 1; j < Math.min(i + ARGS_SEARCH_LINES, lines.length); j++) {
+      const next = lines[j] ?? "";
+      const a = ARGS_LINE.exec(next);
+      if (a) {
+        // The object may continue past this line: a model opening a ```json
+        // fence puts the brace on the NEXT line, and a pretty-printed object
+        // spans several. Join forward and let the brace matcher find the end.
+        const rest = lines.slice(j, Math.min(j + ARGS_JOIN_LINES, lines.length)).join("\n");
+        const parsed =
+          safeJsonObject(a[1] ?? "") ?? safeJsonObject(rest.replace(/^[^:=]*[:=]/, ""));
+        if (parsed) args = JSON.stringify(parsed);
+        break;
+      }
+      // A new TOOL line means this call simply had no arguments.
+      if (TOOL_LINE.test(next)) break;
+    }
+
+    calls.push({ id: `text_${calls.length}_${name}`, name, args });
+  }
+  return calls;
+}
+
+/**
+ * Remove protocol lines from prose.
+ *
+ * A narrated call must never reach the user as though it were an answer, which
+ * is the whole failure this module exists to close. Stripping also means a reply
+ * that was ONLY a tool call ends up empty — and an empty reply carrying tool
+ * calls is a valid turn, while an empty reply carrying none is an outage. The
+ * caller has to keep those apart.
+ */
+export function stripToolCallLines(text: string): string {
+  return text
+    .split(/\r?\n/)
+    .filter((l) => !EITHER_LINE.test(l))
+    .join("\n")
+    .replace(/```(?:json)?\s*```/g, "")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+/**
+ * The tool names inside an OpenAI-shaped `tools` array.
+ *
+ * Derived from what the caller already passed rather than asked for separately:
+ * two lists that must agree are one list that cannot disagree, and the failure
+ * of the split version is silent — a name missing from the second list makes
+ * the parser ignore a real call.
+ */
+export function toolNamesFrom(tools: unknown[] | undefined): string[] {
+  if (!Array.isArray(tools)) return [];
+  const names: string[] = [];
+  for (const entry of tools) {
+    const fn = (entry as { function?: { name?: unknown } } | null)?.function;
+    if (fn && typeof fn.name === "string" && fn.name) names.push(fn.name);
+  }
+  return names;
+}