@cubicecho/agent-core 2.10.0 → 2.12.0

package/dist/stream.d.ts

@@ -10,7 +10,16 @@ import { type Fence } from "./thinking.ts";
  * hangs the turn until somebody presses stop. `EndpointSilent` and `timeoutMs` were exported for
  * this loop long before the loop itself was.
  */
-/** What a turn cost. Zero throughout means the server did not say. */
+/**
+ * What a turn cost, and how it went. Zero throughout the four counts means the server did not say.
+ *
+ * The four counts are always there, as they always were. Everything after them is absent rather
+ * than zero when nothing reported or measured it, because a zero cache hit and a server that says
+ * nothing about its cache are different findings, and a consumer drawing one as the other is
+ * explaining a slow turn with a number nobody sent. Which layer fills each is said on the field:
+ * `streamTurn` reads what the endpoint sends, `runTurn` measures the attempts, and
+ * `runAgentLoop` compares one request with the one before it.
+ */
 export interface TurnUsage {
     prompt: number;
     completion: number;
@@ -20,9 +29,72 @@ export interface TurnUsage {
      *
      * The only way a caller can tell whether the prefix it is careful to keep still is actually
      * being reused: a prefix that stops hitting the cache otherwise shows up as a bill and nothing
-     * else. Zero is also what a server that does not report it sends.
+     * else. Zero is also what a server that does not report it sends; `uncached` is the field whose
+     * presence says the count was reported. llama.cpp's `timings.cache_n` stands in for it where the
+     * usage block has none.
      */
     cached: number;
+    /**
+     * `prompt` less `cached`: the tokens actually prefilled, and the number a cache miss moves.
+     * Present only where both a cache count and the prompt were reported, so its absence is how a
+     * zero `cached` reads as unknown.
+     */
+    uncached?: number;
+    /** How much of `completion` was reasoning, where the usage block breaks it out. Never estimated. */
+    reasoningTokens?: number;
+    /** Prefill time, from llama.cpp's `timings`. On a local box it dominates once the cache misses. */
+    promptMs?: number;
+    /** Prefill speed, from the same `timings`. */
+    promptTokensPerSecond?: number;
+    /** Decode time, from the same `timings`. */
+    predictedMs?: number;
+    /** Decode speed, from the same `timings`. */
+    tokensPerSecond?: number;
+    /**
+     * Tokens a draft model or MTP head proposed, from the same `timings`. Against `draftAccepted`,
+     * whether speculative decoding is paying for itself.
+     */
+    draftTotal?: number;
+    /** How many of `draftTotal` the target model kept. */
+    draftAccepted?: number;
+    /**
+     * From sending the request to the first chunk that carried anything, measured by `streamTurn`.
+     * The latency a watcher feels, and absent on a turn that said nothing at all.
+     */
+    firstTokenMs?: number;
+    /** The whole turn as `runTurn` saw it, queue, retries, downgrades and loading wait included. */
+    wallMs?: number;
+    /** How many times `runTurn` sent a lost request again. A downgrade is not a retry. */
+    retries?: number;
+    /** How many of those were the endpoint going silent, rather than refusing or dropping it. */
+    timeouts?: number;
+    /** How many requests `continueTurn` joined onto the first. Absent on a turn not continued. */
+    continuations?: number;
+    /**
+     * What this request should have reused of the one before it in the loop: that request's prompt
+     * and its reply, which is what a cache holds once the reply is generated. Filled by
+     * `runAgentLoop` from the second request on, where the previous prompt was reported.
+     */
+    cacheExpected?: number;
+    /**
+     * `cached` came in under nine tenths of the previous request's prompt — the part of
+     * `cacheExpected` no re-rendering of the reply can disturb. Present only where both sides were
+     * reported, so a server that says nothing about its cache is never accused of losing it.
+     */
+    cacheBroken?: boolean;
+    /**
+     * What the loop changed that would explain `cacheBroken`, the earliest in the prompt first,
+     * since a change there is the one that costs the rest. `none-known` is the server's doing — an
+     * eviction, another client on the slot — or a change the loop cannot see. Only on a broken turn.
+     */
+    cacheBreakReason?: "tools-changed" | "system-changed" | "history-rewritten" | "none-known";
+    /** How many tools the request declared, filled by `runAgentLoop`. */
+    toolsDeclared?: number;
+    /**
+     * The declared tool block's estimated tokens. A chat template renders it ahead of the system
+     * prompt, so this is what every change to the tool list costs the cache.
+     */
+    toolSchemaTokens?: number;
 }
 /** One streamed turn, put back together into the shape a loop and a transcript work with. */
 export interface Turn {

package/dist/stream.js

@@ -1,5 +1,18 @@
 import { EndpointSilent } from "./retry.js";
 import { DEFAULT_FENCES, FenceSplitter } from "./thinking.js";
+/**
+ * The fields of `timings` a turn reports, under the names `TurnUsage` gives them. A value that is
+ * not a finite number is left out rather than guessed at.
+ */
+const TIMINGS = [
+    ["prompt_ms", "promptMs"],
+    ["prompt_per_second", "promptTokensPerSecond"],
+    ["predicted_ms", "predictedMs"],
+    ["predicted_per_second", "tokensPerSecond"],
+    ["draft_n", "draftTotal"],
+    ["draft_n_accepted", "draftAccepted"],
+];
+const isCount = (value) => typeof value === "number" && Number.isFinite(value);
 /** The largest delay a timer takes, which is as close to none as the SDK's timeout option goes. */
 const NO_SDK_TIMEOUT = 2 ** 31 - 1;
 /**
@@ -40,7 +53,8 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     };
     try {
         rearm(false);
-        return await collect();
+        const started = Date.now();
+        return await collect(started);
     }
     catch (error) {
         // The caller's own stop has to stay distinguishable from ours: one is a run that was called
@@ -57,7 +71,7 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     finally {
         clearTimeout(idle);
     }
-    async function collect() {
+    async function collect(started) {
         // The SDK's own timer runs until the headers arrive, which for a stream is the end of
         // prefill, and it was set from the idle number. Where a watchdog is armed it covers that
         // wait already, with the allowance meant for it, so the SDK is told to leave it alone.
@@ -78,6 +92,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // its place in the order they arrived.
         const calls = [];
         const usage = { prompt: 0, completion: 0, total: 0, cached: 0 };
+        // The usage block's own cache count wins over llama.cpp's, which is the same number in the
+        // servers that send both; kept apart so the order the two arrive in does not decide.
+        let cacheReport;
+        let cacheTimings;
         let finishReason = "";
         for await (const chunk of stream) {
             // Rearmed on every chunk, latched below on only some: a priming chunk is the endpoint
@@ -93,8 +111,22 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
                 usage.completion = chunk.usage.completion_tokens ?? 0;
                 usage.total = chunk.usage.total_tokens ?? 0;
                 const reported = chunk.usage;
-                usage.cached =
-                    reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens ?? 0;
+                const hit = reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens;
+                if (isCount(hit))
+                    cacheReport = hit;
+                const reasoned = reported.completion_tokens_details?.reasoning_tokens;
+                if (isCount(reasoned))
+                    usage.reasoningTokens = reasoned;
+            }
+            const { timings } = chunk;
+            if (timings) {
+                for (const [from, to] of TIMINGS) {
+                    const value = timings[from];
+                    if (isCount(value))
+                        usage[to] = value;
+                }
+                if (isCount(timings.cache_n))
+                    cacheTimings = timings.cache_n;
             }
             // One choice, because that is what an agent loop asks for. A body with `n` above one
             // keeps only the first; nothing here is built to reassemble several at once.
@@ -116,8 +148,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
             // has accumulated, and losing a retry is the safer half of that trade. Set before the
             // callbacks, so a watcher that throws mid-token cannot be told the same token twice.
             const carried = Boolean(thinking || delta.content || delta.tool_calls?.length);
-            if (carried && !talking)
+            if (carried && !talking) {
                 rearm(true);
+                usage.firstTokenMs = Date.now() - started;
+            }
             if (produced && carried)
                 produced.any = true;
             if (thinking) {
@@ -174,6 +208,12 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // cut off at the ceiling mid-scratchpad, it has no answer, and promoting the deliberation to
         // one is how a truncated turn gets recorded as output.
         report(splitter.finish());
+        const cached = cacheReport ?? cacheTimings;
+        if (cached !== undefined) {
+            usage.cached = cached;
+            if (usage.prompt > 0)
+                usage.uncached = Math.max(0, usage.prompt - cached);
+        }
         const minted = new Set();
         return {
             content: splitter.output,

package/llms.txt

@@ -24,6 +24,14 @@ The loop above a turn: send, run the tools the model asked for, send again, unti
 - `ToolCallOutcome` (type) — What one tool call did, in the order the model asked.
 - `ToolCallRequest` (type) — One call the model made, as `dispatch` is handed it.
 
+### calibration
+
+How many characters a token is worth on one model, learned from what its endpoint reports.
+
+- `calibrate` — Takes one reading from a request that was answered, so the next one to this model is sized by it.
+- `charsPerTokenFor` — The characters per token to size a request to this model with, `CHARS_PER_TOKEN` until a turn has reported one.
+- `resetCalibration` — Forgets every reading, so the next request is sized at `CHARS_PER_TOKEN` again.
+
 ### capabilities
 
 What an endpoint turned out not to support, and answering it when it says so.
@@ -86,6 +94,14 @@ What this package needs to know about a caller's configuration.
 - `RetryPolicy` (type) — How many times a lost or refused request is worth sending again.
 - `ToolPolicy` (type) — How tools reach the model, and how long it may keep calling them.
 
+### continuation
+
+Picking up an answer the token ceiling cut off, instead of keeping half of it.
+
+- `ContinueTurnOptions` (type) — What `continueTurn` takes besides what `runTurn` does.
+- `continueTurn` — Carries on an answer the token ceiling cut off, by sending the transcript again with the answer so far as a trailing assistant message, and joins the pieces into one turn.
+- `isContinuable` — Whether a turn is one a continuation can finish: cut off at the ceiling, with an answer begun and no tool call in it.
+
 ### errors
 
 - `errorMessage` — What went wrong, as a string.
@@ -103,8 +119,12 @@ What a run is doing, while it is doing it.
 - `RunEvent` (type) — One thing that happened in a run, as a watcher receives it.
 - `RunEventInput` (type) — What `emit` is given: the run and the sequence are the bus's to assign.
 - `RunEventKind` (type) — Which kind of thing happened, and what `text`, `name`, `ok` and `usage` carry for it.
+- `RunMetrics` (type) — A run summed and derived from its events: what it cost, where the time went, and why.
+- `RunMetricsOptions` (type) — What `runMetrics` takes besides the events.
 - `RunUsage` (type) — What a run has spent, counted from the start of the run rather than for the turn that carried it: a client draws the latest one it has seen and needs no arithmetic of its own, and one lost to the backlog cap costs nothing because the next supersedes it.
 - `resetEvents` — Test seam: forget every run, so one test's events cannot be read by the next.
+- `runMetrics` — A run's totals, timings and cache findings, derived from the events it emitted.
+- `TurnReport` (type) — One turn's usage and measurements as a `usage` event carries them.
 - `watch` — Everything that has happened on a run, then everything that happens next, until it ends.
 
 ### hooks
@@ -113,6 +133,7 @@ Lifecycle hooks, from the host's side: what a session looks like to them, where
 
 - `assembleContext` — Builds the context a set of outcomes adds and the notes that go with it.
 - `configureHooks` — Changes what hooks are held to, for a process whose windows are not the size these defaults were chosen for, or whose host wants its own name above the context.
+- `consult` — Runs an event's hooks and waits for their say, for a host that will hold off when one of them vetoes.
 - `Gathered` (type) — The context a set of outcomes adds to a request, and a note for each hook worth mentioning.
 - `gather` — Runs the hooks ahead of a request and builds what they add to it.
 - `HOOK_CONTEXT_TOKENS` — The most context all of a request's hooks add between them by default, in estimated tokens.
@@ -143,6 +164,7 @@ Lifecycle hooks, from the host's side: what a session looks like to them, where
 Everything about a request failing that is not about what the request said.
 
 - `backoffMs` — Exponential, with jitter so several tasks failing at once do not return in lockstep.
+- `CHARS_PER_TOKEN` — The divisor behind `estimateTokens`, applied here to a character count rather than a string.
 - `ContextOverflow` — The request was bigger than the model will read.
 - `compact` — 1234 → "1.2k".
 - `EndpointSilent` — The endpoint stopped answering mid-request.
@@ -152,9 +174,12 @@ Everything about a request failing that is not about what the request said.
 - `LOADING_POLL_MS` — How long to wait between asking a loading server again.
 - `LOADING_TIMEOUT_MS` — How long `runTurn` waits for a model to load unless told otherwise.
 - `messageTokens` — One message's estimated tokens, by the same count `requestTokens` sums for a whole request.
+- `requestChars` — How many characters a request is worth: the walk `requestTokens` divides, without the division.
 - `requestTokens` — What this request will cost the window, in tokens, near enough.
 - `SMALLEST_LIKELY_WINDOW` — The smallest window worth believing in, and the floor under `runTurn`'s guard.
 - `sleep` — A delay an abort cuts short, rejecting rather than resolving early.
+- `TokenEstimateOptions` (type) — What the two token estimates below take besides what they measure.
+- `toolsChars` — How many characters a tool array is worth, measured once per array.
 
 ### run-turn
 
@@ -182,6 +207,7 @@ One-shot calls that support a run without being one: picking tools, naming a ses
 - `listLines` — A list-shaped reply, one item per line, cleaned of the bullets and quotes models decorate them with.
 - `parseJson` — Models are asked for JSON and often answer with prose around it, or a fenced block.
 - `resetHints` — Test seam, alongside `resetClients` and `resetAll`: forget which models refused the hints.
+- `SideTaskInput` (type) — The input a side task applies its instruction to: text, or the content parts a vision model reads.
 - `SideTaskOptions` (type) — What a side task may be given.
 - `tryAsk` — A side task is never worth failing the work it supports.
 
@@ -204,7 +230,7 @@ Reading one streamed turn back into a message.
 - `StreamTurnOptions` (type) — What `streamTurn` takes besides the request body.
 - `streamTurn` — Runs one turn as a stream, reporting tokens as they arrive and assembling them back into a message.
 - `Turn` (type) — One streamed turn, put back together into the shape a loop and a transcript work with.
-- `TurnUsage` (type) — What a turn cost.
+- `TurnUsage` (type) — What a turn cost, and how it went.
 
 ### thinking
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cubicecho/agent-core",
-  "version": "2.10.0",
+  "version": "2.12.0",
   "description": "The endpoint-agnostic half of an OpenAI-compatible agent loop: tool-schema compatibility, on-demand tool loading, one-shot side tasks, run events, and a pooled client.",
   "keywords": [
     "openai",