@cubicecho/agent-core 2.11.0 → 2.13.0

package/dist/index.js

@@ -10,14 +10,16 @@
  * that differs between one server and the next.
  */
 export { buildBody, preselect, preview, resolveApiKey, runAgentLoop, } from "./agent-loop.js";
+export { calibrate, charsPerTokenFor, resetCalibration } from "./calibration.js";
 export { capabilitiesFor, modelCapabilitiesFor, negotiate, resetCapabilities, } from "./capabilities.js";
 export { configureClients, contextLimitFor, endpointId, endpointKey, FIRST_TOKEN_FACTOR, firstTokenMs, getClient, listModels, NO_KEY, resetClients, servedWindow, timeoutMs, } from "./client.js";
-export { COMPACT_AT, compactTranscript, KEEP_RATIO, planCompaction, pruneToolResults, SUMMARY_LEAD, SUMMARY_PROMPT, summariser, summaryInput, } from "./compaction.js";
+export { applyCompaction, COMPACT_AT, compactTranscript, KEEP_RATIO, planCompaction, pruneToolResults, requestIndex, runCompaction, SUMMARY_LEAD, SUMMARY_PROMPT, summariser, summaryInput, } from "./compaction.js";
+export { continueTurn, isContinuable, } from "./continuation.js";
 export { errorMessage } from "./errors.js";
-export { configureEvents, emit, endRun, fold, history, resetEvents, watch, } from "./events.js";
-export { assembleContext, configureHooks, gather, HOOK_CONTEXT_TOKENS, HOOK_EVENTS, HOOK_PREFACE, INJECT_EVENTS, notify, resetHooks, turnIndex, turnMessages, UNTRUSTED_PREFACE, untrusted, withContext, } from "./hooks.js";
+export { configureEvents, emit, endRun, fold, history, resetEvents, runMetrics, watch, } from "./events.js";
+export { assembleContext, configureHooks, consult, gather, HOOK_CONTEXT_TOKENS, HOOK_EVENTS, HOOK_PREFACE, INJECT_EVENTS, notify, resetHooks, turnIndex, turnMessages, UNTRUSTED_PREFACE, untrusted, withContext, } from "./hooks.js";
 export { resetAll } from "./reset.js";
-export { backoffMs, ContextOverflow, compact, EndpointSilent, isModelLoading, isOverflow, isTransient, LOADING_POLL_MS, LOADING_TIMEOUT_MS, messageTokens, requestTokens, SMALLEST_LIKELY_WINDOW, sleep, } from "./retry.js";
+export { backoffMs, CHARS_PER_TOKEN, ContextOverflow, compact, EndpointSilent, isModelLoading, isOverflow, isTransient, LOADING_POLL_MS, LOADING_TIMEOUT_MS, messageTokens, requestChars, requestTokens, SMALLEST_LIKELY_WINDOW, sleep, toolsChars, } from "./retry.js";
 export { runTurn } from "./run-turn.js";
 export { isGrammarError, relaxTools, sanitizeTools } from "./schema-compat.js";
 export { ask, askJson, clean, listLines, parseJson, resetHints, tryAsk, } from "./side-task.js";
@@ -26,4 +28,4 @@ export { streamTurn, } from "./stream.js";
 export { ALL_FENCES, DEFAULT_FENCES, FenceSplitter, stripThinking, THINK_FENCE, } from "./thinking.js";
 export { estimateTokens } from "./tokens.js";
 export { parseToolArguments, recoverToolCalls, ToolArgumentsError, } from "./tool-calls.js";
-export { carryOver, catalogList, catalogPrompt, expandNames, inCatalog, LOAD_TOOLS, LOAD_TOOLS_DEFINITION, loadedTools, loadResult, MAX_CARRIED, MAX_PER_LOAD, PRESELECT_SCHEMA, PRESELECT_SYSTEM, preselectInput, preselection, preselectSystem, requestedNames, } from "./tool-loading.js";
+export { carryOver, catalogList, catalogPrompt, expandNames, inCatalog, LOAD_TOOLS, LOAD_TOOLS_DEFINITION, loadedTools, loadResult, MAX_CARRIED, MAX_PER_LOAD, orderTools, PRESELECT_SCHEMA, PRESELECT_SYSTEM, preselectInput, preselection, preselectSystem, requestedNames, } from "./tool-loading.js";

package/dist/reset.d.ts

@@ -1,12 +1,13 @@
 /**
  * Forgets everything this package remembers between calls.
  *
- * Five modules here keep state for the life of the process, each for a good reason and each
+ * Six modules here keep state for the life of the process, each for a good reason and each
  * with its own seam: the pooled clients and their model listings, the endpoints that turned
- * out not to take `stream_options` or a grammar, the models that refused the no-thinking
- * hints, the event bus, and the hooks' configured budget and preface. `resetClients`,
- * `resetCapabilities`, `resetHints`, `resetEvents` and `resetHooks` stay exported, because a
- * test that means to clear one thing should say so.
+ * out not to take `stream_options` or a grammar, each model's measured characters per token,
+ * the models that refused the no-thinking hints, the event bus, and the hooks' configured
+ * budget and preface. `resetClients`, `resetCapabilities`, `resetCalibration`, `resetHints`,
+ * `resetEvents` and `resetHooks` stay exported, because a test that means to clear one thing
+ * should say so.
  *
  * This is for the other case, which is every teardown. What they hold is *latched
  * refusals* — a fact one test taught the process about an endpoint, still true as far as the
@@ -15,7 +16,7 @@
  * one that reads the latch fails only when it happens to run second. `tests/side-task-hints.test.ts`
  * was written that way and only passed because every case had been handed a hostname of its own.
  *
- * It is also the seam that does not need finding again. A sixth module with a cache is a sixth
+ * It is also the seam that does not need finding again. A seventh module with a cache is a seventh
  * line here, rather than an edit to the teardown of three consumers who will not all notice.
  */
 export declare function resetAll(): void;

package/dist/reset.js

@@ -1,3 +1,4 @@
+import { resetCalibration } from "./calibration.js";
 import { resetCapabilities } from "./capabilities.js";
 import { resetClients } from "./client.js";
 import { resetEvents } from "./events.js";
@@ -6,12 +7,13 @@ import { resetHints } from "./side-task.js";
 /**
  * Forgets everything this package remembers between calls.
  *
- * Five modules here keep state for the life of the process, each for a good reason and each
+ * Six modules here keep state for the life of the process, each for a good reason and each
  * with its own seam: the pooled clients and their model listings, the endpoints that turned
- * out not to take `stream_options` or a grammar, the models that refused the no-thinking
- * hints, the event bus, and the hooks' configured budget and preface. `resetClients`,
- * `resetCapabilities`, `resetHints`, `resetEvents` and `resetHooks` stay exported, because a
- * test that means to clear one thing should say so.
+ * out not to take `stream_options` or a grammar, each model's measured characters per token,
+ * the models that refused the no-thinking hints, the event bus, and the hooks' configured
+ * budget and preface. `resetClients`, `resetCapabilities`, `resetCalibration`, `resetHints`,
+ * `resetEvents` and `resetHooks` stay exported, because a test that means to clear one thing
+ * should say so.
  *
  * This is for the other case, which is every teardown. What they hold is *latched
  * refusals* — a fact one test taught the process about an endpoint, still true as far as the
@@ -20,12 +22,13 @@ import { resetHints } from "./side-task.js";
  * one that reads the latch fails only when it happens to run second. `tests/side-task-hints.test.ts`
  * was written that way and only passed because every case had been handed a hostname of its own.
  *
- * It is also the seam that does not need finding again. A sixth module with a cache is a sixth
+ * It is also the seam that does not need finding again. A seventh module with a cache is a seventh
  * line here, rather than an edit to the teardown of three consumers who will not all notice.
  */
 export function resetAll() {
     resetClients();
     resetCapabilities();
+    resetCalibration();
     resetHints();
     resetEvents();
     resetHooks();

package/dist/retry.d.ts

@@ -29,10 +29,41 @@ export declare class ContextOverflow extends Error {
  * @param tokens The count to render.
  */
 export declare const compact: (tokens: number) => string;
+/**
+ * The divisor behind `estimateTokens`, applied here to a character count rather than a string.
+ *
+ * The fallback, not the rule: once a turn has come back with a reported prompt count, `runTurn`
+ * divides by what that endpoint's model was measured at instead. See `charsPerTokenFor`.
+ */
+export declare const CHARS_PER_TOKEN = 4;
+/** What the two token estimates below take besides what they measure. */
+export interface TokenEstimateOptions {
+    /**
+     * The divisor, `CHARS_PER_TOKEN` unless given — `charsPerTokenFor` for a model whose reported
+     * usage has calibrated it. A value that is not a number above zero is ignored.
+     */
+    charsPerToken?: number;
+}
+/**
+ * How many characters a tool array is worth, measured once per array.
+ *
+ * @param tools The tool definitions as they will be sent. An empty array is worth nothing.
+ */
+export declare function toolsChars(tools: OpenAI.ChatCompletionTool[]): number;
+/**
+ * How many characters a request is worth: the walk `requestTokens` divides, without the division.
+ *
+ * What calibration reads a reported prompt count against, since a ratio is only as good as the
+ * character count it was taken over agreeing with the one it is later applied to.
+ *
+ * @param body The request as it was sent, tools included.
+ */
+export declare function requestChars(body: OpenAI.ChatCompletionCreateParamsStreaming): number;
 /**
  * What this request will cost the window, in tokens, near enough.
  *
- * See `estimateTokens` for why it is characters over four and which way it is wrong on purpose.
+ * See `estimateTokens` for why it is characters over four and which way it is wrong on purpose,
+ * and `charsPerTokenFor` for the divisor a model's own reported usage has measured instead.
  *
  * Summed by walking the body rather than by serialising it. `JSON.stringify` on the messages
  * built the entire transcript into a string on every call and threw it away having read nothing
@@ -45,8 +76,9 @@ export declare const compact: (tokens: number) => string;
  * over four.
  *
  * @param body The request as it will be sent, tools included.
+ * @param options The divisor, `CHARS_PER_TOKEN` when none is given.
  */
-export declare const requestTokens: (body: OpenAI.ChatCompletionCreateParamsStreaming) => number;
+export declare const requestTokens: (body: OpenAI.ChatCompletionCreateParamsStreaming, { charsPerToken }?: TokenEstimateOptions) => number;
 /**
  * One message's estimated tokens, by the same count `requestTokens` sums for a whole request.
  *
@@ -54,8 +86,9 @@ export declare const requestTokens: (body: OpenAI.ChatCompletionCreateParamsStre
  * tail — where `estimateTokens` on the text alone would leave out the calls and the envelope.
  *
  * @param message The message as it will be sent.
+ * @param options The divisor, `CHARS_PER_TOKEN` when none is given.
  */
-export declare const messageTokens: (message: OpenAI.ChatCompletionMessageParam) => number;
+export declare const messageTokens: (message: OpenAI.ChatCompletionMessageParam, { charsPerToken }?: TokenEstimateOptions) => number;
 /**
  * Whether a refusal means the request was too big, rather than merely refused.
  *

package/dist/retry.js

@@ -1,5 +1,4 @@
 import OpenAI from "openai";
-import { estimateTokens } from "./tokens.js";
 /**
  * Everything about a request failing that is not about what the request said.
  *
@@ -63,8 +62,15 @@ const REFUSAL_PART = 32;
 const REASONING_KEY = 23;
 /** The same for `"reasoning":"",`, OpenRouter's spelling of it. */
 const REASONING_ALT_KEY = 15;
-/** The divisor behind `estimateTokens`, applied here to a character count rather than a string. */
-const CHARS_PER_TOKEN = 4;
+/**
+ * The divisor behind `estimateTokens`, applied here to a character count rather than a string.
+ *
+ * The fallback, not the rule: once a turn has come back with a reported prompt count, `runTurn`
+ * divides by what that endpoint's model was measured at instead. See `charsPerTokenFor`.
+ */
+export const CHARS_PER_TOKEN = 4;
+/** The divisor an option asked for, or the fallback when it asked for nothing usable. */
+const divisor = (charsPerToken) => charsPerToken !== undefined && charsPerToken > 0 ? charsPerToken : CHARS_PER_TOKEN;
 /** How many characters one message is worth: its keys, and its content in whichever shape. */
 function messageChars(message) {
     let chars = message.role.length + ENVELOPE;
@@ -114,22 +120,47 @@ function messageChars(message) {
  * the array-level memoisation `tests/retry.test.ts` pins, which deliberately holds a mutated array
  * to its first reading. Sizing happens once per turn either way, so the miss costs one walk of the
  * schemas rather than a walk per attempt.
+ *
+ * Characters rather than tokens, so a calibrated divisor applies to the schemas without walking
+ * them again.
+ */
+const toolLengths = new WeakMap();
+/**
+ * How many characters a tool array is worth, measured once per array.
+ *
+ * @param tools The tool definitions as they will be sent. An empty array is worth nothing.
  */
-const toolTokens = new WeakMap();
-function toolsCost(tools) {
-    const hit = toolTokens.get(tools);
+export function toolsChars(tools) {
+    if (!tools.length)
+        return 0;
+    const hit = toolLengths.get(tools);
     if (hit !== undefined)
         return hit;
     // Schemas are arbitrarily shaped, so this one really is a serialisation — but it happens once
     // per tool array rather than once per turn.
-    const cost = estimateTokens(JSON.stringify(tools));
-    toolTokens.set(tools, cost);
-    return cost;
+    const length = JSON.stringify(tools).length;
+    toolLengths.set(tools, length);
+    return length;
+}
+/**
+ * How many characters a request is worth: the walk `requestTokens` divides, without the division.
+ *
+ * What calibration reads a reported prompt count against, since a ratio is only as good as the
+ * character count it was taken over agreeing with the one it is later applied to.
+ *
+ * @param body The request as it was sent, tools included.
+ */
+export function requestChars(body) {
+    let chars = 0;
+    for (const message of body.messages)
+        chars += messageChars(message);
+    return chars;
 }
 /**
  * What this request will cost the window, in tokens, near enough.
  *
- * See `estimateTokens` for why it is characters over four and which way it is wrong on purpose.
+ * See `estimateTokens` for why it is characters over four and which way it is wrong on purpose,
+ * and `charsPerTokenFor` for the divisor a model's own reported usage has measured instead.
  *
  * Summed by walking the body rather than by serialising it. `JSON.stringify` on the messages
  * built the entire transcript into a string on every call and threw it away having read nothing
@@ -142,14 +173,15 @@ function toolsCost(tools) {
  * over four.
  *
  * @param body The request as it will be sent, tools included.
+ * @param options The divisor, `CHARS_PER_TOKEN` when none is given.
  */
-export const requestTokens = (body) => {
+export const requestTokens = (body, { charsPerToken } = {}) => {
     // Characters first and the division once at the end, rather than a rounded count per message:
     // `Math.ceil` on every one of a few hundred messages is a few hundred tokens of pure rounding.
-    let chars = 0;
-    for (const message of body.messages)
-        chars += messageChars(message);
-    return Math.ceil(chars / CHARS_PER_TOKEN) + (body.tools?.length ? toolsCost(body.tools) : 0);
+    // The tools are divided on their own, as they were when their tokens were cached, so the
+    // uncalibrated count is the one it always was.
+    const per = divisor(charsPerToken);
+    return Math.ceil(requestChars(body) / per) + Math.ceil(toolsChars(body.tools ?? []) / per);
 };
 /**
  * One message's estimated tokens, by the same count `requestTokens` sums for a whole request.
@@ -158,8 +190,9 @@ export const requestTokens = (body) => {
  * tail — where `estimateTokens` on the text alone would leave out the calls and the envelope.
  *
  * @param message The message as it will be sent.
+ * @param options The divisor, `CHARS_PER_TOKEN` when none is given.
  */
-export const messageTokens = (message) => Math.ceil(messageChars(message) / CHARS_PER_TOKEN);
+export const messageTokens = (message, { charsPerToken } = {}) => Math.ceil(messageChars(message) / divisor(charsPerToken));
 /**
  * Servers refuse an over-long request in their own words; these are the ones worth reading as
  * that rather than as a broken request. Matched loosely — every one of them is some

package/dist/run-turn.d.ts

@@ -22,6 +22,10 @@ import { type StreamTurnOptions, type Turn } from "./stream.ts";
  * attempt would say everything twice. That is what `produced` is, one box per attempt — set by
  * a chunk that carried something rather than by a chunk arriving, so the empty opening chunk
  * most servers send does not cost the retry.
+ *
+ * What the turn cost in attempts comes back on its usage — `wallMs`, `retries`, `timeouts` — and
+ * what its prompt was reported at calibrates the ratio the next request to that model is sized
+ * with. See `calibrate`.
  */
 /** A retry is not the same event as a downgrade, but a watcher wants to be told about both. */
 export interface RunTurnOptions extends Omit<StreamTurnOptions, "produced"> {

package/dist/run-turn.js

@@ -1,6 +1,7 @@
+import { calibrate, charsPerTokenFor } from "./calibration.js";
 import { negotiate } from "./capabilities.js";
 import { errorMessage } from "./errors.js";
-import { backoffMs, ContextOverflow, compact, isModelLoading, isOverflow, isTransient, LOADING_POLL_MS, LOADING_TIMEOUT_MS, requestTokens, SMALLEST_LIKELY_WINDOW, sleep, } from "./retry.js";
+import { backoffMs, ContextOverflow, compact, EndpointSilent, isModelLoading, isOverflow, isTransient, LOADING_POLL_MS, LOADING_TIMEOUT_MS, requestTokens, SMALLEST_LIKELY_WINDOW, sleep, } from "./retry.js";
 import { streamTurn } from "./stream.js";
 /**
  * `request` is a callback rather than a body because the body has to be rebuilt from whatever
@@ -21,11 +22,16 @@ export async function runTurn(client, supports, request, { maxRetries = 0, onNot
     // does not change between attempts — so the first body is the one worth measuring, and
     // measuring the rest would only spend the walk again to reach the same answer.
     let sized = false;
+    // The body the answer was given to, for the calibration once it is in hand.
+    let sent;
     const measured = (capabilities, forModel) => {
         const body = request(capabilities, forModel);
+        sent = body;
         if (!sized && contextLimit >= SMALLEST_LIKELY_WINDOW) {
             sized = true;
-            const needed = requestTokens(body);
+            const needed = requestTokens(body, {
+                charsPerToken: charsPerTokenFor(supports, body.model),
+            });
             // The endpoint refuses on the prompt plus the reply — llama.cpp sizes the slot with
             // `n_predict` in, OpenAI with the ceiling — so a prompt that fits the window but not the
             // window less the ceiling was let through here to be refused one round trip later, which
@@ -45,10 +51,17 @@ export async function runTurn(client, supports, request, { maxRetries = 0, onNot
     // When the server first said it was loading. Unset until then, and never reset: a model that
     // loads, fails and loads again has had its allowance.
     let loadingSince;
+    const started = Date.now();
+    let retries = 0;
+    let timeouts = 0;
     for (let attempt = 0;; attempt++) {
         const produced = { any: false };
         try {
-            return await negotiate(supports, (capabilities, box, forModel) => streamTurn(client, measured(capabilities, forModel), { ...stream, produced: box }), { produced, onNotice, model, droppable });
+            const turn = await negotiate(supports, (capabilities, box, forModel) => streamTurn(client, measured(capabilities, forModel), { ...stream, produced: box }), { produced, onNotice, model, droppable });
+            if (sent)
+                calibrate(supports, sent, turn.usage.prompt);
+            Object.assign(turn.usage, { wallMs: Date.now() - started, retries, timeouts });
+            return turn;
         }
         catch (error) {
             // The abort is read before the classification, not after. A run stopped by its operator
@@ -85,6 +98,9 @@ export async function runTurn(client, supports, request, { maxRetries = 0, onNot
             }
             if (attempt >= maxRetries || !isTransient(error))
                 throw error;
+            retries++;
+            if (error instanceof EndpointSilent)
+                timeouts++;
             const wait = backoffMs(attempt);
             // Reported in whatever unit reads as a number: the first backoff is under a second, and
             // "retrying in 0s" is what rounding it to seconds says.

package/dist/snapshot.d.ts

@@ -15,6 +15,11 @@ export interface ModelSnapshot {
     chosenTemperature: boolean;
     refusedFields: string[];
     structuredOutput: boolean;
+    /**
+     * Continues a trailing assistant message. Absent in a snapshot taken before it was latched, which
+     * reads as not refused.
+     */
+    assistantPrefill: boolean;
     /** Takes the no-thinking hints `ask` sends. */
     thinkingHints: boolean;
 }

package/dist/snapshot.js

@@ -16,6 +16,7 @@ const optimisticModel = () => ({
     chosenTemperature: true,
     refusedFields: [],
     structuredOutput: true,
+    assistantPrefill: true,
     thinkingHints: true,
 });
 const refusedAnything = (model) => !model.reasoningEffort ||
@@ -23,6 +24,7 @@ const refusedAnything = (model) => !model.reasoningEffort ||
     !model.chosenTemperature ||
     !model.thinkingHints ||
     !model.structuredOutput ||
+    !model.assistantPrefill ||
     model.refusedFields.length > 0;
 /**
  * Every refusal this process has latched, as a JSON-safe blob to store and hand back on boot.
@@ -48,6 +50,7 @@ export function exportCapabilities() {
                 chosenTemperature: refused.chosenTemperature,
                 refusedFields: [...refused.refusedFields].sort(),
                 structuredOutput: refused.structuredOutput,
+                assistantPrefill: refused.assistantPrefill,
             };
             if (refusedAnything(model))
                 models[name] = model;
@@ -109,6 +112,8 @@ export function importCapabilities(snapshot) {
                 refused.chosenTemperature = false;
             if (model.structuredOutput === false)
                 refused.structuredOutput = false;
+            if (model.assistantPrefill === false)
+                refused.assistantPrefill = false;
             if (Array.isArray(model.refusedFields)) {
                 for (const field of model.refusedFields) {
                     if (typeof field === "string")

package/dist/stream.d.ts

@@ -10,7 +10,16 @@ import { type Fence } from "./thinking.ts";
  * hangs the turn until somebody presses stop. `EndpointSilent` and `timeoutMs` were exported for
  * this loop long before the loop itself was.
  */
-/** What a turn cost. Zero throughout means the server did not say. */
+/**
+ * What a turn cost, and how it went. Zero throughout the four counts means the server did not say.
+ *
+ * The four counts are always there, as they always were. Everything after them is absent rather
+ * than zero when nothing reported or measured it, because a zero cache hit and a server that says
+ * nothing about its cache are different findings, and a consumer drawing one as the other is
+ * explaining a slow turn with a number nobody sent. Which layer fills each is said on the field:
+ * `streamTurn` reads what the endpoint sends, `runTurn` measures the attempts, and
+ * `runAgentLoop` compares one request with the one before it.
+ */
 export interface TurnUsage {
     prompt: number;
     completion: number;
@@ -20,9 +29,72 @@ export interface TurnUsage {
      *
      * The only way a caller can tell whether the prefix it is careful to keep still is actually
      * being reused: a prefix that stops hitting the cache otherwise shows up as a bill and nothing
-     * else. Zero is also what a server that does not report it sends.
+     * else. Zero is also what a server that does not report it sends; `uncached` is the field whose
+     * presence says the count was reported. llama.cpp's `timings.cache_n` stands in for it where the
+     * usage block has none.
      */
     cached: number;
+    /**
+     * `prompt` less `cached`: the tokens actually prefilled, and the number a cache miss moves.
+     * Present only where both a cache count and the prompt were reported, so its absence is how a
+     * zero `cached` reads as unknown.
+     */
+    uncached?: number;
+    /** How much of `completion` was reasoning, where the usage block breaks it out. Never estimated. */
+    reasoningTokens?: number;
+    /** Prefill time, from llama.cpp's `timings`. On a local box it dominates once the cache misses. */
+    promptMs?: number;
+    /** Prefill speed, from the same `timings`. */
+    promptTokensPerSecond?: number;
+    /** Decode time, from the same `timings`. */
+    predictedMs?: number;
+    /** Decode speed, from the same `timings`. */
+    tokensPerSecond?: number;
+    /**
+     * Tokens a draft model or MTP head proposed, from the same `timings`. Against `draftAccepted`,
+     * whether speculative decoding is paying for itself.
+     */
+    draftTotal?: number;
+    /** How many of `draftTotal` the target model kept. */
+    draftAccepted?: number;
+    /**
+     * From sending the request to the first chunk that carried anything, measured by `streamTurn`.
+     * The latency a watcher feels, and absent on a turn that said nothing at all.
+     */
+    firstTokenMs?: number;
+    /** The whole turn as `runTurn` saw it, queue, retries, downgrades and loading wait included. */
+    wallMs?: number;
+    /** How many times `runTurn` sent a lost request again. A downgrade is not a retry. */
+    retries?: number;
+    /** How many of those were the endpoint going silent, rather than refusing or dropping it. */
+    timeouts?: number;
+    /** How many requests `continueTurn` joined onto the first. Absent on a turn not continued. */
+    continuations?: number;
+    /**
+     * What this request should have reused of the one before it in the loop: that request's prompt
+     * and its reply, which is what a cache holds once the reply is generated. Filled by
+     * `runAgentLoop` from the second request on, where the previous prompt was reported.
+     */
+    cacheExpected?: number;
+    /**
+     * `cached` came in under nine tenths of the previous request's prompt — the part of
+     * `cacheExpected` no re-rendering of the reply can disturb. Present only where both sides were
+     * reported, so a server that says nothing about its cache is never accused of losing it.
+     */
+    cacheBroken?: boolean;
+    /**
+     * What the loop changed that would explain `cacheBroken`, the earliest in the prompt first,
+     * since a change there is the one that costs the rest. `none-known` is the server's doing — an
+     * eviction, another client on the slot — or a change the loop cannot see. Only on a broken turn.
+     */
+    cacheBreakReason?: "tools-changed" | "system-changed" | "history-rewritten" | "none-known";
+    /** How many tools the request declared, filled by `runAgentLoop`. */
+    toolsDeclared?: number;
+    /**
+     * The declared tool block's estimated tokens. A chat template renders it ahead of the system
+     * prompt, so this is what every change to the tool list costs the cache.
+     */
+    toolSchemaTokens?: number;
 }
 /** One streamed turn, put back together into the shape a loop and a transcript work with. */
 export interface Turn {

package/dist/stream.js

@@ -1,5 +1,18 @@
 import { EndpointSilent } from "./retry.js";
 import { DEFAULT_FENCES, FenceSplitter } from "./thinking.js";
+/**
+ * The fields of `timings` a turn reports, under the names `TurnUsage` gives them. A value that is
+ * not a finite number is left out rather than guessed at.
+ */
+const TIMINGS = [
+    ["prompt_ms", "promptMs"],
+    ["prompt_per_second", "promptTokensPerSecond"],
+    ["predicted_ms", "predictedMs"],
+    ["predicted_per_second", "tokensPerSecond"],
+    ["draft_n", "draftTotal"],
+    ["draft_n_accepted", "draftAccepted"],
+];
+const isCount = (value) => typeof value === "number" && Number.isFinite(value);
 /** The largest delay a timer takes, which is as close to none as the SDK's timeout option goes. */
 const NO_SDK_TIMEOUT = 2 ** 31 - 1;
 /**
@@ -40,7 +53,8 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     };
     try {
         rearm(false);
-        return await collect();
+        const started = Date.now();
+        return await collect(started);
     }
     catch (error) {
         // The caller's own stop has to stay distinguishable from ours: one is a run that was called
@@ -57,7 +71,7 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     finally {
         clearTimeout(idle);
     }
-    async function collect() {
+    async function collect(started) {
         // The SDK's own timer runs until the headers arrive, which for a stream is the end of
         // prefill, and it was set from the idle number. Where a watchdog is armed it covers that
         // wait already, with the allowance meant for it, so the SDK is told to leave it alone.
@@ -78,6 +92,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // its place in the order they arrived.
         const calls = [];
         const usage = { prompt: 0, completion: 0, total: 0, cached: 0 };
+        // The usage block's own cache count wins over llama.cpp's, which is the same number in the
+        // servers that send both; kept apart so the order the two arrive in does not decide.
+        let cacheReport;
+        let cacheTimings;
         let finishReason = "";
         for await (const chunk of stream) {
             // Rearmed on every chunk, latched below on only some: a priming chunk is the endpoint
@@ -93,8 +111,22 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
                 usage.completion = chunk.usage.completion_tokens ?? 0;
                 usage.total = chunk.usage.total_tokens ?? 0;
                 const reported = chunk.usage;
-                usage.cached =
-                    reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens ?? 0;
+                const hit = reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens;
+                if (isCount(hit))
+                    cacheReport = hit;
+                const reasoned = reported.completion_tokens_details?.reasoning_tokens;
+                if (isCount(reasoned))
+                    usage.reasoningTokens = reasoned;
+            }
+            const { timings } = chunk;
+            if (timings) {
+                for (const [from, to] of TIMINGS) {
+                    const value = timings[from];
+                    if (isCount(value))
+                        usage[to] = value;
+                }
+                if (isCount(timings.cache_n))
+                    cacheTimings = timings.cache_n;
             }
             // One choice, because that is what an agent loop asks for. A body with `n` above one
             // keeps only the first; nothing here is built to reassemble several at once.
@@ -116,8 +148,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
             // has accumulated, and losing a retry is the safer half of that trade. Set before the
             // callbacks, so a watcher that throws mid-token cannot be told the same token twice.
             const carried = Boolean(thinking || delta.content || delta.tool_calls?.length);
-            if (carried && !talking)
+            if (carried && !talking) {
                 rearm(true);
+                usage.firstTokenMs = Date.now() - started;
+            }
             if (produced && carried)
                 produced.any = true;
             if (thinking) {
@@ -174,6 +208,12 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // cut off at the ceiling mid-scratchpad, it has no answer, and promoting the deliberation to
         // one is how a truncated turn gets recorded as output.
         report(splitter.finish());
+        const cached = cacheReport ?? cacheTimings;
+        if (cached !== undefined) {
+            usage.cached = cached;
+            if (usage.prompt > 0)
+                usage.uncached = Math.max(0, usage.prompt - cached);
+        }
         const minted = new Set();
         return {
             content: splitter.output,

package/dist/tool-loading.d.ts

@@ -56,16 +56,41 @@ export declare function catalogPrompt(catalog: CatalogServer[], loaded?: Readonl
 /**
  * A tool array with newly loaded definitions appended, in the order they were loaded.
  *
- * Never re-sorted and never rebuilt from a set. A template renders the tool array into the
- * prompt near its head, and a load that moved an earlier definition moved everything after it,
- * so the cache was lost from there on every load; appended, the definitions already sent stay
- * a prefix of the new array.
+ * Appended and never rebuilt from a set, so what a load adds is decided by the load and not by
+ * the shape of whatever collection the definitions came out of. Where the appended array ends up
+ * in the request is `orderTools`' business, which the loop applies after this.
  *
  * @param previous What the last request declared, `load_tools` included. Not written to.
  * @param matched The definitions to add. Ones whose name is already declared, here or earlier in
  * this list, are skipped rather than moved.
  */
 export declare function loadedTools(previous: readonly OpenAI.ChatCompletionTool[], matched: readonly OpenAI.ChatCompletionTool[]): OpenAI.ChatCompletionTool[];
+/**
+ * How a tool array is ordered before it is sent: `true` by name, `false` as the caller built it,
+ * or a comparator over the two names.
+ */
+export type ToolOrder = boolean | ((a: string, b: string) => number);
+/**
+ * The tool array in a stable order, so the same set of tools renders the same way twice.
+ *
+ * A chat template renders the declared tools ahead of the system prompt, which makes the tool
+ * array the first thing a prompt cache has to match — and an array assembled from a map, from
+ * database rows, or from the order servers happened to connect in changes between processes and
+ * between reconnects. Every such change costs the cache for the whole transcript rather than for
+ * the tools alone, and nothing about the request the model sees is different. Ordering by name
+ * makes the array a property of the set instead of of how it was built, at the price of a load
+ * inserting rather than appending. Definitions come back by identity, so `sanitizeTools` still
+ * finds each one in its cache.
+ *
+ * `false` is for a caller that means its order: the model reads the array top to bottom, and a
+ * host may be putting what it wants reached for first at the front.
+ *
+ * @param tools The definitions to order. Not written to.
+ * @param order `true` for name order, `false` to leave it alone, or a comparator over the names.
+ * A tool that is not a function orders as the empty name.
+ * @returns `tools` itself when it is already in that order, so the common case copies nothing.
+ */
+export declare function orderTools(tools: OpenAI.ChatCompletionTool[], order?: ToolOrder): OpenAI.ChatCompletionTool[];
 /**
  * The most a single `load_tools` call may pull in.
  *