@cubicecho/agent-core 2.4.0 → 2.6.0

package/dist/side-task.js

@@ -1,8 +1,9 @@
 import OpenAI from "openai";
-import { capabilitiesFor, negotiate } from "./capabilities.js";
-import { endpointKey, getClient } from "./client.js";
+import { capabilitiesFor, modelCapabilitiesFor, negotiate, } from "./capabilities.js";
+import { endpointId, getClient } from "./client.js";
 import { errorMessage } from "./errors.js";
 import { isTransient } from "./retry.js";
+import { relaxTools, sanitizeTools } from "./schema-compat.js";
 /**
  * One-shot calls that support a run without being one: picking tools, naming a session,
  * summarising a transcript, proposing follow-ups. They share a shape — small prompt, short
@@ -25,7 +26,7 @@ const NO_THINKING = { chat_template_kwargs: { enable_thinking: false } };
  * The models that turned out not to take the hints, by endpoint and model.
  *
  * Keyed rather than global for the reason the client cache is keyed, and on the same
- * `endpointKey` it is: a refusal is a fact about what is on the other end, not about this
+ * endpoint it is, by `endpointId`: a refusal is a fact about what is on the other end, not about this
  * process. A llama.cpp box and a cloud API are both reachable from one consumer over its
  * lifetime, and the first one's refusal must not stop the second from ever being asked.
  *
@@ -38,7 +39,10 @@ const NO_THINKING = { chat_template_kwargs: { enable_thinking: false } };
  * the same (endpoint, model) pair, where a run on that model can read it too.
  */
 const noHints = new Set();
-const hintKey = (config, model) => JSON.stringify([endpointKey(config), model]);
+/** An (endpoint, model) pair as `noHints` holds it: `[endpointId, model]`, stringified. */
+export const hintKey = (endpoint, model) => JSON.stringify([endpoint, model]);
+/** The pairs that refused the hints, the live set, for `exportCapabilities` and `importCapabilities`. */
+export const refusedHints = () => noHints;
 /** Test seam, alongside `resetClients` and `resetAll`: forget which models refused the hints. */
 export const resetHints = () => noHints.clear();
 /**
@@ -86,48 +90,73 @@ const stripThinking = (text) => text
  * @param user The input it applies to.
  * @param options Reply ceiling, temperature, cancellation, notices.
  */
-export async function ask(config, model, system, user, { maxTokens = 512, temperature = 0.3, signal, onNotice } = {}) {
-    const send = (hints, refused) => getClient(config).chat.completions.create({
-        model,
-        // The reasoning models want the ceiling spelled the other way, and they are exactly the
-        // models a side task most wants to stop deliberating.
-        ...(refused && !refused.legacyTokenLimit
-            ? { max_completion_tokens: maxTokens }
-            : { max_tokens: maxTokens }),
-        // One that will only run at the temperature it was built with is sent none: a side task
-        // wants the same answer twice, and 1.0 from that model is as close as it gets.
-        ...(refused && !refused.chosenTemperature ? {} : { temperature }),
-        messages: [
-            { role: "system", content: system },
-            { role: "user", content: user },
-        ],
-        ...(hints ? NO_THINKING : {}),
-        ...(hints && refused?.reasoningEffort !== false ? { reasoning_effort: "none" } : {}),
-    }, { signal });
+export function ask(config, model, system, user, options = {}) {
+    return complete(config, model, system, user, options);
+}
+/**
+ * The request `ask` and `askJson` share, with `format` deciding the extra body fields from what
+ * the model and the endpoint have refused, rebuilt on every re-send.
+ */
+async function complete(config, model, system, user, { maxTokens = 512, temperature = 0.3, signal, onNotice }, format) {
+    // Whether the last request carried an effort, which `negotiate` decides and not this function.
+    let sentEffort = false;
+    const send = (hints, effort, supports, refused) => {
+        sentEffort = effort && refused?.reasoningEffort !== false;
+        return getClient(config).chat.completions.create({
+            model,
+            // The reasoning models want the ceiling spelled the other way, and they are exactly the
+            // models a side task most wants to stop deliberating.
+            ...(refused && !refused.legacyTokenLimit
+                ? { max_completion_tokens: maxTokens }
+                : { max_tokens: maxTokens }),
+            // One that will only run at the temperature it was built with is sent none: a side task
+            // wants the same answer twice, and 1.0 from that model is as close as it gets.
+            ...(refused && !refused.chosenTemperature ? {} : { temperature }),
+            messages: [
+                { role: "system", content: system },
+                { role: "user", content: user },
+            ],
+            ...(hints ? NO_THINKING : {}),
+            // Not gated on `hints`: a model that refuses `chat_template_kwargs` may still read the
+            // effort, and the two latches would otherwise contradict each other.
+            ...(sentEffort ? { reasoning_effort: "none" } : {}),
+            ...(format && refused ? format(supports, refused) : {}),
+        }, { signal });
+    };
     // The endpoint's own object, not one of this module's: what a model refuses is the same fact
     // whether a run or a side task found it out, and the point of latching it is that only one of
-    // them has to pay for it. Nothing here sends tools or `stream_options`, so the two
-    // endpoint-level flags are not in play — the model's three are the whole of what this meets.
+    // them has to pay for it. Nothing here sends tools or `stream_options`; the grammar flag is in
+    // play only for `askJson`, whose schema a llama.cpp server compiles the way it does a tool's.
     const supports = capabilitiesFor(config.baseUrl, config.apiKey);
-    const attempt = (hints) => negotiate(supports, (_supports, _produced, refused) => send(hints, refused), {
+    const attempt = (hints, effort) => negotiate(supports, (latched, _produced, refused) => send(hints, effort, latched, refused), {
         model,
         onNotice,
     });
-    const key = hintKey(config, model);
+    const key = hintKey(endpointId(config), model);
     const hints = !noHints.has(key);
     let response;
     try {
-        response = await attempt(hints);
+        response = await attempt(hints, true);
     }
     catch (error) {
         // Whatever is left after `negotiate` has answered everything it knows: on this path that is
         // the hints it does not, which is `chat_template_kwargs` and an effort the model has but
-        // does not offer as `none`.
-        if (!hints || !rejectedTheRequest(error))
+        // does not offer as `none`. Or a 400 about something else entirely, which is why the
+        // notice says what was tried rather than what was wrong.
+        const effort = sentEffort;
+        if (!(hints || effort) || !rejectedTheRequest(error))
             throw error;
-        onNotice?.(`${model} rejected the no-thinking hints; retrying without them`);
-        noHints.add(key);
-        response = await attempt(false);
+        onNotice?.(`${model} rejected a request carrying the no-thinking hints; retrying without them`);
+        response = await attempt(false, false);
+        // Latched on the finding, not the hypothesis: a context overflow is a 400 `negotiate` does
+        // not recognise too, and it fails the retry the same way, leaving the next call to try the
+        // hints again. When both went out the refusal cannot say which, so the one `negotiate`
+        // cannot latch is blamed; if it was the effort after all, the next call is left with only
+        // the effort to drop and latches that instead.
+        if (hints)
+            noHints.add(key);
+        else
+            modelCapabilitiesFor(supports, model).reasoningEffort = false;
     }
     const message = response.choices[0]?.message;
     const answer = stripThinking(message?.content ?? "").trim();
@@ -138,6 +167,46 @@ export async function ask(config, model, system, user, { maxTokens = 512, temper
     const reasoning = message?.reasoning_content;
     return typeof reasoning === "string" ? stripThinking(reasoning).trim() : "";
 }
+/**
+ * A side task whose answer is JSON matching a schema, parsed. Undefined when no JSON came back.
+ * Throws like any request.
+ *
+ * Sends `response_format` with the schema, which a llama.cpp server compiles into a grammar and
+ * vLLM, LM Studio, Ollama and OpenAI each hold the reply to, so a small model that wraps JSON in
+ * prose cannot. The schema is normalised as a tool's parameters are, and relaxed where the endpoint
+ * could not build a grammar, since the same converter reads both. A model that refuses the field
+ * latches it off and is asked in words: the schema rides on the system prompt either way, and
+ * the reply goes through `parseJson`, which finds the JSON in whatever came back.
+ *
+ * @param config Where to send it and how long to wait.
+ * @param model The model to ask.
+ * @param system The instruction. The schema is appended to it.
+ * @param user The input it applies to.
+ * @param schema The JSON Schema of the answer. Its root is held to an object, as a tool's is.
+ * @param options A side task's options, plus the schema's `name` and whether it is `strict`.
+ */
+export async function askJson(config, model, system, user, schema, { name = "answer", strict = true, ...options } = {}) {
+    const tool = (parameters) => ({
+        type: "function",
+        function: { name, parameters },
+    });
+    const [sanitized] = sanitizeTools([tool(schema)]);
+    const shapeOf = (definition) => definition?.type === "function" ? (definition.function.parameters ?? {}) : {};
+    const instruction = `${system}\n\nReply with JSON alone, matching this JSON Schema:\n${JSON.stringify(shapeOf(sanitized))}`;
+    const reply = await complete(config, model, instruction, user, options, (supports, refused) => refused.structuredOutput
+        ? {
+            response_format: {
+                type: "json_schema",
+                json_schema: {
+                    name,
+                    strict,
+                    schema: shapeOf(supports.strictSchemas ? sanitized : relaxTools([sanitized])[0]),
+                },
+            },
+        }
+        : {});
+    return parseJson(reply);
+}
 /**
  * A side task is never worth failing the work it supports. Callers that can carry on without
  * an answer use this and get `undefined` instead of an exception.

package/dist/snapshot.d.ts

@@ -0,0 +1,57 @@
+/**
+ * What endpoints and models refused, carried across a restart.
+ *
+ * Every latch here dies with the process, so each restart of a consumer spends one refused request
+ * per endpoint and model learning the same facts again, with a notice each time — and on a local
+ * reasoning model the first side task after boot is sent the hints, refused and sent again. The
+ * package cannot know where a consumer keeps state, so it hands over a blob and takes one back.
+ */
+/** The version `importCapabilities` accepts. Raised when what is latched changes shape. */
+export declare const CAPABILITY_SNAPSHOT_VERSION = 1;
+/** What one model on an endpoint refused. `true` is not refused, as on `ModelCapabilities`. */
+export interface ModelSnapshot {
+    reasoningEffort: boolean;
+    legacyTokenLimit: boolean;
+    chosenTemperature: boolean;
+    refusedFields: string[];
+    structuredOutput: boolean;
+    /** Takes the no-thinking hints `ask` sends. */
+    thinkingHints: boolean;
+}
+/** What one endpoint refused, and under it what each of its models did. */
+export interface EndpointSnapshot {
+    strictSchemas: boolean;
+    usageInStream: boolean;
+    models: Record<string, ModelSnapshot>;
+}
+/** Every latched refusal in the process, JSON-safe. See `exportCapabilities`. */
+export interface CapabilitySnapshot {
+    version: number;
+    /** When it was taken, as an ISO string, for the consumer to judge how stale is too stale. */
+    savedAt: string;
+    /** By `endpointId`: the endpoint's URL and key hashed together, so no key is in the blob. */
+    endpoints: Record<string, EndpointSnapshot>;
+}
+/**
+ * Every refusal this process has latched, as a JSON-safe blob to store and hand back on boot.
+ *
+ * Covers what `negotiate` latches on endpoints and models and the models `ask` found refusing the
+ * no-thinking hints. Only what was actually refused is in it, so a snapshot of a process that met
+ * no refusals has no endpoints. Endpoints are named by digest rather than URL and key, since the
+ * blob is meant to be written somewhere and a key must not be written with it.
+ */
+export declare function exportCapabilities(): CapabilitySnapshot;
+/**
+ * Latches what a stored snapshot says was refused, on top of whatever this process has learned.
+ *
+ * Refusals only ever latch off, so importing merges rather than replaces: a flag already off stays
+ * off whatever the snapshot says, and one the snapshot has off is turned off. A snapshot of another
+ * version, or anything that is not one, is ignored — a stale shape costs the refused requests it
+ * would have saved, which is what a restart cost before. How old is too old is the consumer's call,
+ * made on `savedAt` before importing, since a server behind a URL can be upgraded between boots.
+ *
+ * @param snapshot What `exportCapabilities` returned, as stored. Read defensively: a field of the
+ * wrong type is skipped rather than trusted.
+ * @returns Whether the snapshot was of this version and applied.
+ */
+export declare function importCapabilities(snapshot: unknown): boolean;

package/dist/snapshot.js

@@ -0,0 +1,123 @@
+import { capabilitiesById, knownCapabilities, modelCapabilitiesFor } from "./capabilities.js";
+import { hintKey, refusedHints } from "./side-task.js";
+/**
+ * What endpoints and models refused, carried across a restart.
+ *
+ * Every latch here dies with the process, so each restart of a consumer spends one refused request
+ * per endpoint and model learning the same facts again, with a notice each time — and on a local
+ * reasoning model the first side task after boot is sent the hints, refused and sent again. The
+ * package cannot know where a consumer keeps state, so it hands over a blob and takes one back.
+ */
+/** The version `importCapabilities` accepts. Raised when what is latched changes shape. */
+export const CAPABILITY_SNAPSHOT_VERSION = 1;
+const optimisticModel = () => ({
+    reasoningEffort: true,
+    legacyTokenLimit: true,
+    chosenTemperature: true,
+    refusedFields: [],
+    structuredOutput: true,
+    thinkingHints: true,
+});
+const refusedAnything = (model) => !model.reasoningEffort ||
+    !model.legacyTokenLimit ||
+    !model.chosenTemperature ||
+    !model.thinkingHints ||
+    !model.structuredOutput ||
+    model.refusedFields.length > 0;
+/**
+ * Every refusal this process has latched, as a JSON-safe blob to store and hand back on boot.
+ *
+ * Covers what `negotiate` latches on endpoints and models and the models `ask` found refusing the
+ * no-thinking hints. Only what was actually refused is in it, so a snapshot of a process that met
+ * no refusals has no endpoints. Endpoints are named by digest rather than URL and key, since the
+ * blob is meant to be written somewhere and a key must not be written with it.
+ */
+export function exportCapabilities() {
+    const endpoints = {};
+    const entry = (id) => {
+        endpoints[id] ??= { strictSchemas: true, usageInStream: true, models: {} };
+        return endpoints[id];
+    };
+    for (const [id, supports] of knownCapabilities()) {
+        const models = {};
+        for (const [name, refused] of supports.models) {
+            const model = {
+                ...optimisticModel(),
+                reasoningEffort: refused.reasoningEffort,
+                legacyTokenLimit: refused.legacyTokenLimit,
+                chosenTemperature: refused.chosenTemperature,
+                refusedFields: [...refused.refusedFields].sort(),
+                structuredOutput: refused.structuredOutput,
+            };
+            if (refusedAnything(model))
+                models[name] = model;
+        }
+        if (!supports.strictSchemas || !supports.usageInStream || Object.keys(models).length) {
+            Object.assign(entry(id), {
+                strictSchemas: supports.strictSchemas,
+                usageInStream: supports.usageInStream,
+                models,
+            });
+        }
+    }
+    for (const key of refusedHints()) {
+        const [id, name] = JSON.parse(key);
+        const models = entry(id).models;
+        models[name] ??= optimisticModel();
+        models[name].thinkingHints = false;
+    }
+    return { version: CAPABILITY_SNAPSHOT_VERSION, savedAt: new Date().toISOString(), endpoints };
+}
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/**
+ * Latches what a stored snapshot says was refused, on top of whatever this process has learned.
+ *
+ * Refusals only ever latch off, so importing merges rather than replaces: a flag already off stays
+ * off whatever the snapshot says, and one the snapshot has off is turned off. A snapshot of another
+ * version, or anything that is not one, is ignored — a stale shape costs the refused requests it
+ * would have saved, which is what a restart cost before. How old is too old is the consumer's call,
+ * made on `savedAt` before importing, since a server behind a URL can be upgraded between boots.
+ *
+ * @param snapshot What `exportCapabilities` returned, as stored. Read defensively: a field of the
+ * wrong type is skipped rather than trusted.
+ * @returns Whether the snapshot was of this version and applied.
+ */
+export function importCapabilities(snapshot) {
+    if (!isRecord(snapshot) || snapshot.version !== CAPABILITY_SNAPSHOT_VERSION)
+        return false;
+    if (!isRecord(snapshot.endpoints))
+        return false;
+    for (const [id, endpoint] of Object.entries(snapshot.endpoints)) {
+        if (!isRecord(endpoint))
+            continue;
+        const supports = capabilitiesById(id);
+        if (endpoint.strictSchemas === false)
+            supports.strictSchemas = false;
+        if (endpoint.usageInStream === false)
+            supports.usageInStream = false;
+        if (!isRecord(endpoint.models))
+            continue;
+        for (const [name, model] of Object.entries(endpoint.models)) {
+            if (!isRecord(model))
+                continue;
+            const refused = modelCapabilitiesFor(supports, name);
+            if (model.reasoningEffort === false)
+                refused.reasoningEffort = false;
+            if (model.legacyTokenLimit === false)
+                refused.legacyTokenLimit = false;
+            if (model.chosenTemperature === false)
+                refused.chosenTemperature = false;
+            if (model.structuredOutput === false)
+                refused.structuredOutput = false;
+            if (Array.isArray(model.refusedFields)) {
+                for (const field of model.refusedFields) {
+                    if (typeof field === "string")
+                        refused.refusedFields.add(field);
+                }
+            }
+            if (model.thinkingHints === false)
+                refusedHints().add(hintKey(id, name));
+        }
+    }
+    return true;
+}

package/dist/stream.d.ts

@@ -14,6 +14,14 @@ export interface TurnUsage {
     prompt: number;
     completion: number;
     total: number;
+    /**
+     * How much of `prompt` came from the endpoint's prompt cache — a part of it, not in addition.
+     *
+     * The only way a caller can tell whether the prefix it is careful to keep still is actually
+     * being reused: a prefix that stops hitting the cache otherwise shows up as a bill and nothing
+     * else. Zero is also what a server that does not report it sends.
+     */
+    cached: number;
 }
 /** One streamed turn, put back together into the shape a loop and a transcript work with. */
 export interface Turn {

package/dist/stream.js

@@ -50,7 +50,7 @@ export async function streamTurn(client, body, { signal, idleMs, produced, onThi
         const stream = await client.chat.completions.create(body, { signal: linked });
         const content = [];
         const calls = new Map();
-        const usage = { prompt: 0, completion: 0, total: 0 };
+        const usage = { prompt: 0, completion: 0, total: 0, cached: 0 };
         let finishReason = "";
         for await (const chunk of stream) {
             // Rearmed on every chunk, latched below on only some: a priming chunk is the endpoint
@@ -64,6 +64,9 @@ export async function streamTurn(client, body, { signal, idleMs, produced, onThi
                 usage.prompt = chunk.usage.prompt_tokens ?? 0;
                 usage.completion = chunk.usage.completion_tokens ?? 0;
                 usage.total = chunk.usage.total_tokens ?? 0;
+                const reported = chunk.usage;
+                usage.cached =
+                    reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens ?? 0;
             }
             // One choice, because that is what an agent loop asks for. A body with `n` above one
             // keeps only the first; nothing here is built to reassemble several at once.

package/dist/tool-calls.d.ts

@@ -0,0 +1,67 @@
+import type OpenAI from "openai";
+/**
+ * Reading what a model meant by a tool call when it did not write one cleanly.
+ *
+ * Local models get tool calls wrong in two ways a hosted one rarely does. The arguments come back
+ * almost-JSON — single quotes, Python's `True`, a trailing comma, a string holding the JSON
+ * rather than the JSON — or cut off at the ceiling. And the call comes back not as a call at all
+ * but as text, in the model's own template, because the server's tool-call parser was written for
+ * a different one. Both were being handled in every consumer, differently, or not at all.
+ */
+/** A tool call as the loop handles it, recovered or streamed. */
+export type ToolCall = OpenAI.ChatCompletionMessageFunctionToolCall;
+/**
+ * Tool arguments that could not be read as an object, with why.
+ *
+ * `truncated` is a call cut off at the reply ceiling, which no repair can finish and the fix for
+ * is a larger `maxTokens`; `malformed` is one the model wrote wrongly, which it can be told about
+ * and try again.
+ */
+export declare class ToolArgumentsError extends Error {
+    /** Whether the model ran out of room or wrote something unreadable. */
+    readonly kind: "truncated" | "malformed";
+    /**
+     * @param kind Why the arguments could not be read.
+     * @param message What the model is handed back as the tool's result.
+     */
+    constructor(kind: "truncated" | "malformed", message: string);
+}
+/**
+ * A tool call's arguments as the object the tool is handed. Empty is no arguments.
+ *
+ * Lenient where the model's meaning is plain and strict where it is not: an object already
+ * parsed passes through, JSON inside a string is opened, and the almost-JSON local models write —
+ * single quotes, `True`, bare keys, a trailing comma — is repaired. What is still not an object
+ * throws a `ToolArgumentsError`, and the loop hands its message back to the model as the tool's
+ * result so it can try again.
+ *
+ * @param raw The arguments as the model sent them: usually the streamed string, sometimes an
+ * object a server parsed already. Null, absent or blank is no arguments.
+ * @param options `finishReason`, the turn's. A turn that stopped at `"length"` makes a failure
+ * `truncated`, since a call cut off at the ceiling reads exactly like a malformed one.
+ */
+export declare function parseToolArguments(raw: unknown, { finishReason }?: {
+    finishReason?: string | null;
+}): Record<string, unknown>;
+/**
+ * Tool calls a model wrote into its reply as text, taken out of it and made into calls.
+ *
+ * A server whose tool-call parser does not match the model's chat template streams the call as
+ * content, and the run ends on what reads like a finished answer. The templates' own markers are
+ * looked for — `<tool_call>` (Hermes, Qwen, Qwen3-Coder's markup included), `[TOOL_CALLS]`
+ * (Mistral, both spellings) and `<|python_tag|>` (Llama 3) — and, failing those, a reply that is
+ * nothing but a JSON call, or holds one fenced one, provided every name in it is in `names`. Only
+ * text after the last `</think>` is searched, since a model deliberating about a call is not making
+ * one.
+ *
+ * @param content The turn's text.
+ * @param options `names`, the tools that exist. Without them only the templates' markers count.
+ * @returns The text with the calls taken out, and the calls, numbered `call_recovered_0` onward.
+ * No calls leaves the text as it was.
+ */
+export declare function recoverToolCalls(content: string, { names }?: {
+    names?: Iterable<string>;
+}): {
+    content: string;
+    toolCalls: ToolCall[];
+};