@cubicecho/agent-core 2.11.0 → 2.12.0

package/dist/stream.js

@@ -1,5 +1,18 @@
 import { EndpointSilent } from "./retry.js";
 import { DEFAULT_FENCES, FenceSplitter } from "./thinking.js";
+/**
+ * The fields of `timings` a turn reports, under the names `TurnUsage` gives them. A value that is
+ * not a finite number is left out rather than guessed at.
+ */
+const TIMINGS = [
+    ["prompt_ms", "promptMs"],
+    ["prompt_per_second", "promptTokensPerSecond"],
+    ["predicted_ms", "predictedMs"],
+    ["predicted_per_second", "tokensPerSecond"],
+    ["draft_n", "draftTotal"],
+    ["draft_n_accepted", "draftAccepted"],
+];
+const isCount = (value) => typeof value === "number" && Number.isFinite(value);
 /** The largest delay a timer takes, which is as close to none as the SDK's timeout option goes. */
 const NO_SDK_TIMEOUT = 2 ** 31 - 1;
 /**
@@ -40,7 +53,8 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     };
     try {
         rearm(false);
-        return await collect();
+        const started = Date.now();
+        return await collect(started);
     }
     catch (error) {
         // The caller's own stop has to stay distinguishable from ours: one is a run that was called
@@ -57,7 +71,7 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
     finally {
         clearTimeout(idle);
     }
-    async function collect() {
+    async function collect(started) {
         // The SDK's own timer runs until the headers arrive, which for a stream is the end of
         // prefill, and it was set from the idle number. Where a watchdog is armed it covers that
         // wait already, with the allowance meant for it, so the SDK is told to leave it alone.
@@ -78,6 +92,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // its place in the order they arrived.
         const calls = [];
         const usage = { prompt: 0, completion: 0, total: 0, cached: 0 };
+        // The usage block's own cache count wins over llama.cpp's, which is the same number in the
+        // servers that send both; kept apart so the order the two arrive in does not decide.
+        let cacheReport;
+        let cacheTimings;
         let finishReason = "";
         for await (const chunk of stream) {
             // Rearmed on every chunk, latched below on only some: a priming chunk is the endpoint
@@ -93,8 +111,22 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
                 usage.completion = chunk.usage.completion_tokens ?? 0;
                 usage.total = chunk.usage.total_tokens ?? 0;
                 const reported = chunk.usage;
-                usage.cached =
-                    reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens ?? 0;
+                const hit = reported.prompt_tokens_details?.cached_tokens ?? reported.prompt_cache_hit_tokens;
+                if (isCount(hit))
+                    cacheReport = hit;
+                const reasoned = reported.completion_tokens_details?.reasoning_tokens;
+                if (isCount(reasoned))
+                    usage.reasoningTokens = reasoned;
+            }
+            const { timings } = chunk;
+            if (timings) {
+                for (const [from, to] of TIMINGS) {
+                    const value = timings[from];
+                    if (isCount(value))
+                        usage[to] = value;
+                }
+                if (isCount(timings.cache_n))
+                    cacheTimings = timings.cache_n;
             }
             // One choice, because that is what an agent loop asks for. A body with `n` above one
             // keeps only the first; nothing here is built to reassemble several at once.
@@ -116,8 +148,10 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
             // has accumulated, and losing a retry is the safer half of that trade. Set before the
             // callbacks, so a watcher that throws mid-token cannot be told the same token twice.
             const carried = Boolean(thinking || delta.content || delta.tool_calls?.length);
-            if (carried && !talking)
+            if (carried && !talking) {
                 rearm(true);
+                usage.firstTokenMs = Date.now() - started;
+            }
             if (produced && carried)
                 produced.any = true;
             if (thinking) {
@@ -174,6 +208,12 @@ export async function streamTurn(client, body, { signal, idleMs, firstChunkMs, p
         // cut off at the ceiling mid-scratchpad, it has no answer, and promoting the deliberation to
         // one is how a truncated turn gets recorded as output.
         report(splitter.finish());
+        const cached = cacheReport ?? cacheTimings;
+        if (cached !== undefined) {
+            usage.cached = cached;
+            if (usage.prompt > 0)
+                usage.uncached = Math.max(0, usage.prompt - cached);
+        }
         const minted = new Set();
         return {
             content: splitter.output,

package/llms.txt

@@ -24,6 +24,14 @@ The loop above a turn: send, run the tools the model asked for, send again, unti
 - `ToolCallOutcome` (type) — What one tool call did, in the order the model asked.
 - `ToolCallRequest` (type) — One call the model made, as `dispatch` is handed it.
 
+### calibration
+
+How many characters a token is worth on one model, learned from what its endpoint reports.
+
+- `calibrate` — Takes one reading from a request that was answered, so the next one to this model is sized by it.
+- `charsPerTokenFor` — The characters per token to size a request to this model with, `CHARS_PER_TOKEN` until a turn has reported one.
+- `resetCalibration` — Forgets every reading, so the next request is sized at `CHARS_PER_TOKEN` again.
+
 ### capabilities
 
 What an endpoint turned out not to support, and answering it when it says so.
@@ -86,6 +94,14 @@ What this package needs to know about a caller's configuration.
 - `RetryPolicy` (type) — How many times a lost or refused request is worth sending again.
 - `ToolPolicy` (type) — How tools reach the model, and how long it may keep calling them.
 
+### continuation
+
+Picking up an answer the token ceiling cut off, instead of keeping half of it.
+
+- `ContinueTurnOptions` (type) — What `continueTurn` takes besides what `runTurn` does.
+- `continueTurn` — Carries on an answer the token ceiling cut off, by sending the transcript again with the answer so far as a trailing assistant message, and joins the pieces into one turn.
+- `isContinuable` — Whether a turn is one a continuation can finish: cut off at the ceiling, with an answer begun and no tool call in it.
+
 ### errors
 
 - `errorMessage` — What went wrong, as a string.
@@ -103,8 +119,12 @@ What a run is doing, while it is doing it.
 - `RunEvent` (type) — One thing that happened in a run, as a watcher receives it.
 - `RunEventInput` (type) — What `emit` is given: the run and the sequence are the bus's to assign.
 - `RunEventKind` (type) — Which kind of thing happened, and what `text`, `name`, `ok` and `usage` carry for it.
+- `RunMetrics` (type) — A run summed and derived from its events: what it cost, where the time went, and why.
+- `RunMetricsOptions` (type) — What `runMetrics` takes besides the events.
 - `RunUsage` (type) — What a run has spent, counted from the start of the run rather than for the turn that carried it: a client draws the latest one it has seen and needs no arithmetic of its own, and one lost to the backlog cap costs nothing because the next supersedes it.
 - `resetEvents` — Test seam: forget every run, so one test's events cannot be read by the next.
+- `runMetrics` — A run's totals, timings and cache findings, derived from the events it emitted.
+- `TurnReport` (type) — One turn's usage and measurements as a `usage` event carries them.
 - `watch` — Everything that has happened on a run, then everything that happens next, until it ends.
 
 ### hooks
@@ -113,6 +133,7 @@ Lifecycle hooks, from the host's side: what a session looks like to them, where
 
 - `assembleContext` — Builds the context a set of outcomes adds and the notes that go with it.
 - `configureHooks` — Changes what hooks are held to, for a process whose windows are not the size these defaults were chosen for, or whose host wants its own name above the context.
+- `consult` — Runs an event's hooks and waits for their say, for a host that will hold off when one of them vetoes.
 - `Gathered` (type) — The context a set of outcomes adds to a request, and a note for each hook worth mentioning.
 - `gather` — Runs the hooks ahead of a request and builds what they add to it.
 - `HOOK_CONTEXT_TOKENS` — The most context all of a request's hooks add between them by default, in estimated tokens.
@@ -143,6 +164,7 @@ Lifecycle hooks, from the host's side: what a session looks like to them, where
 Everything about a request failing that is not about what the request said.
 
 - `backoffMs` — Exponential, with jitter so several tasks failing at once do not return in lockstep.
+- `CHARS_PER_TOKEN` — The divisor behind `estimateTokens`, applied here to a character count rather than a string.
 - `ContextOverflow` — The request was bigger than the model will read.
 - `compact` — 1234 → "1.2k".
 - `EndpointSilent` — The endpoint stopped answering mid-request.
@@ -152,9 +174,12 @@ Everything about a request failing that is not about what the request said.
 - `LOADING_POLL_MS` — How long to wait between asking a loading server again.
 - `LOADING_TIMEOUT_MS` — How long `runTurn` waits for a model to load unless told otherwise.
 - `messageTokens` — One message's estimated tokens, by the same count `requestTokens` sums for a whole request.
+- `requestChars` — How many characters a request is worth: the walk `requestTokens` divides, without the division.
 - `requestTokens` — What this request will cost the window, in tokens, near enough.
 - `SMALLEST_LIKELY_WINDOW` — The smallest window worth believing in, and the floor under `runTurn`'s guard.
 - `sleep` — A delay an abort cuts short, rejecting rather than resolving early.
+- `TokenEstimateOptions` (type) — What the two token estimates below take besides what they measure.
+- `toolsChars` — How many characters a tool array is worth, measured once per array.
 
 ### run-turn
 
@@ -205,7 +230,7 @@ Reading one streamed turn back into a message.
 - `StreamTurnOptions` (type) — What `streamTurn` takes besides the request body.
 - `streamTurn` — Runs one turn as a stream, reporting tokens as they arrive and assembling them back into a message.
 - `Turn` (type) — One streamed turn, put back together into the shape a loop and a transcript work with.
-- `TurnUsage` (type) — What a turn cost.
+- `TurnUsage` (type) — What a turn cost, and how it went.
 
 ### thinking
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cubicecho/agent-core",
-  "version": "2.11.0",
+  "version": "2.12.0",
   "description": "The endpoint-agnostic half of an OpenAI-compatible agent loop: tool-schema compatibility, on-demand tool loading, one-shot side tasks, run events, and a pooled client.",
   "keywords": [
     "openai",