@cubicecho/agent-core 2.22.0 → 2.24.0

package/dist/calibration.js

@@ -1,4 +1,6 @@
-import { CHARS_PER_TOKEN, requestChars, toolsChars } from "./retry.js";
+import { requestChars, toolsChars } from "./retry.js";
+import { scoped } from "./scope.js";
+import { CHARS_PER_TOKEN } from "./tokens.js";
 /**
  * How many characters a token is worth on one model, learned from what its endpoint reports.
  *
@@ -27,7 +29,7 @@ const PLAUSIBLE = { least: 1, most: 8 };
  * `resetCapabilities` takes its readings with it. Replaced rather than cleared by `resetCalibration`,
  * since a `WeakMap` has no `clear`.
  */
-let readings = new WeakMap();
+const readings = scoped(() => new WeakMap());
 /**
  * The characters per token to size a request to this model with, `CHARS_PER_TOKEN` until a turn
  * has reported one.
@@ -43,7 +45,7 @@ let readings = new WeakMap();
  * @param model The name the endpoint knows the model as, as it goes in the body.
  */
 export function charsPerTokenFor(supports, model) {
-    const known = readings.get(supports)?.get(model);
+    const known = readings().get(supports)?.get(model);
     return known?.length ? Math.max(...known) : CHARS_PER_TOKEN;
 }
 /** Whether any message carries a part whose tokens its characters do not count. */
@@ -69,10 +71,10 @@ export function calibrate(supports, body, promptTokens) {
     if (ratio < PLAUSIBLE.least || ratio > PLAUSIBLE.most) {
         return charsPerTokenFor(supports, body.model);
     }
-    let models = readings.get(supports);
+    let models = readings().get(supports);
     if (!models) {
         models = new Map();
-        readings.set(supports, models);
+        readings().set(supports, models);
     }
     const known = models.get(body.model) ?? [];
     known.push(ratio);
@@ -83,5 +85,5 @@ export function calibrate(supports, body, promptTokens) {
 }
 /** Forgets every reading, so the next request is sized at `CHARS_PER_TOKEN` again. */
 export function resetCalibration() {
-    readings = new WeakMap();
+    readings.reset();
 }

package/dist/capabilities.d.ts

@@ -118,6 +118,20 @@ export interface ModelCapabilities {
      */
     assistantPrefill: boolean;
 }
+/** The name of one of a model's on/off flags: every `boolean` on `ModelCapabilities`. */
+export type ModelFlag = {
+    [Key in keyof ModelCapabilities]-?: ModelCapabilities[Key] extends boolean ? Key : never;
+}[keyof ModelCapabilities];
+/**
+ * A model's on/off flags as it starts out, every one optimistic.
+ *
+ * A record over every boolean on `ModelCapabilities`, so a flag added there does not compile until
+ * it is listed here — and this is the list a snapshot is exported, imported and judged empty by,
+ * where each of those used to name the flags again and one could be missed.
+ */
+export declare const OPTIMISTIC_MODEL: Readonly<Record<ModelFlag, true>>;
+/** The names of those flags, for a caller that reads or latches each in turn. */
+export declare const MODEL_FLAGS: ModelFlag[];
 /**
  * What this endpoint is known not to support. The same object every time, so what `negotiate`
  * latches off stays off.
@@ -210,6 +224,28 @@ export declare const EFFORT_LADDER: readonly ["none", "minimal", "low", "medium"
  * @param asked What the config asks for. `"off"` and absent mean no effort.
  */
 export declare function effortFor(refused: ModelCapabilities | undefined, asked: string | undefined): string;
+/**
+ * The reply ceiling and the temperature as this model takes them, to spread into a request body.
+ *
+ * The reasoning models want the ceiling spelled the other way, and one that will only run at the
+ * temperature it was built with is sent none. Both are tested `=== false`, so a caller that named
+ * no model sends what one that has refused nothing is sent; see `ModelCapabilities.legacyTokenLimit`.
+ *
+ * @param refused What the model has refused, as `negotiate` hands it over. Absent is a model that
+ * has refused nothing.
+ * @param maxTokens The ceiling. Zero or less sends none and leaves it to the server: a zero sent
+ * as `max_tokens: 0` asks for an empty reply.
+ * @param temperature What to sample at, where the model takes one that was picked for it.
+ */
+export declare const ceilingAndTemperature: (refused: ModelCapabilities | undefined, maxTokens: number, temperature: number) => {
+    max_completion_tokens: number;
+    temperature?: number | undefined;
+} | {
+    max_tokens: number;
+    temperature?: number | undefined;
+} | {
+    temperature?: number | undefined;
+};
 /** What `negotiate` takes besides the request. All optional. */
 export interface NegotiateOptions {
     /**

package/dist/capabilities.js

@@ -1,6 +1,23 @@
 import { endpointId } from "./client.js";
 import { errorMessage } from "./errors.js";
 import { isGrammarError } from "./schema-compat.js";
+import { scoped } from "./scope.js";
+/**
+ * A model's on/off flags as it starts out, every one optimistic.
+ *
+ * A record over every boolean on `ModelCapabilities`, so a flag added there does not compile until
+ * it is listed here — and this is the list a snapshot is exported, imported and judged empty by,
+ * where each of those used to name the flags again and one could be missed.
+ */
+export const OPTIMISTIC_MODEL = {
+    reasoningEffort: true,
+    legacyTokenLimit: true,
+    chosenTemperature: true,
+    structuredOutput: true,
+    assistantPrefill: true,
+};
+/** The names of those flags, for a caller that reads or latches each in turn. */
+export const MODEL_FLAGS = Object.keys(OPTIMISTIC_MODEL);
 /**
  * What each endpoint cannot do, remembered for the life of the process.
  *
@@ -19,7 +36,7 @@ import { isGrammarError } from "./schema-compat.js";
  * behind it, including the `models` map underneath, which is the level where two keys through
  * one host are most likely to differ at all.
  */
-const capabilities = new Map();
+const capabilities = scoped(() => new Map());
 /**
  * What this endpoint is known not to support. The same object every time, so what `negotiate`
  * latches off stays off.
@@ -50,30 +67,22 @@ export function capabilitiesFor(baseUrl, apiKey) {
 export function modelCapabilitiesFor(supports, model) {
     let known = supports.models.get(model);
     if (!known) {
-        known = {
-            reasoningEffort: true,
-            legacyTokenLimit: true,
-            chosenTemperature: true,
-            refusedFields: new Set(),
-            structuredOutput: true,
-            refusedEfforts: new Set(),
-            assistantPrefill: true,
-        };
+        known = { ...OPTIMISTIC_MODEL, refusedFields: new Set(), refusedEfforts: new Set() };
         supports.models.set(model, known);
     }
     return known;
 }
 /** Every endpoint's capabilities by `endpointId`, the live objects, for `exportCapabilities`. */
-export const knownCapabilities = () => capabilities;
+export const knownCapabilities = () => capabilities();
 /**
  * The capabilities of the endpoint with this `endpointId`, created optimistic if unseen — the way
  * `importCapabilities` reaches an endpoint it has only a digest for.
  */
 export function capabilitiesById(id) {
-    let known = capabilities.get(id);
+    let known = capabilities().get(id);
     if (!known) {
         known = { strictSchemas: true, usageInStream: true, models: new Map(), since: Date.now() };
-        capabilities.set(id, known);
+        capabilities().set(id, known);
     }
     return known;
 }
@@ -92,11 +101,11 @@ export function capabilitiesById(id) {
  */
 export function resetCapabilities(endpoint) {
     if (!endpoint) {
-        const held = capabilities.size > 0;
-        capabilities.clear();
+        const held = capabilities().size > 0;
+        capabilities().clear();
         return held;
     }
-    return capabilities.delete(endpointId(endpoint));
+    return capabilities().delete(endpointId(endpoint));
 }
 /**
  * Forgets every endpoint whose entry is older than this, so the next request finds out again.
@@ -117,10 +126,10 @@ export function resetCapabilities(endpoint) {
  */
 export function expireCapabilities(maxAgeMs, now = Date.now()) {
     let dropped = 0;
-    for (const [id, known] of capabilities) {
+    for (const [id, known] of capabilities()) {
         if (now - known.since < maxAgeMs)
             continue;
-        capabilities.delete(id);
+        capabilities().delete(id);
         dropped++;
     }
     return dropped;
@@ -260,6 +269,27 @@ export function effortFor(refused, asked) {
         return asked;
     return nextEffort(refused, asked) ?? asked;
 }
+/**
+ * The reply ceiling and the temperature as this model takes them, to spread into a request body.
+ *
+ * The reasoning models want the ceiling spelled the other way, and one that will only run at the
+ * temperature it was built with is sent none. Both are tested `=== false`, so a caller that named
+ * no model sends what one that has refused nothing is sent; see `ModelCapabilities.legacyTokenLimit`.
+ *
+ * @param refused What the model has refused, as `negotiate` hands it over. Absent is a model that
+ * has refused nothing.
+ * @param maxTokens The ceiling. Zero or less sends none and leaves it to the server: a zero sent
+ * as `max_tokens: 0` asks for an empty reply.
+ * @param temperature What to sample at, where the model takes one that was picked for it.
+ */
+export const ceilingAndTemperature = (refused, maxTokens, temperature) => ({
+    ...(maxTokens > 0
+        ? refused?.legacyTokenLimit === false
+            ? { max_completion_tokens: maxTokens }
+            : { max_tokens: maxTokens }
+        : {}),
+    ...(refused?.chosenTemperature === false ? {} : { temperature }),
+});
 /**
  * How to answer a refused effort *value*, or `undefined` when there is nothing new to learn or
  * nowhere left to go — in which case the refusal is the caller's, as it was before this existed.
@@ -346,9 +376,58 @@ function unknownFields(detail) {
         .split(/[.[]/)[0])
         .filter(Boolean);
 }
-/** Whether a refusal names a droppable field this model has not already refused. */
-const refusesDroppable = (detail, droppable, refused) => droppable.size > 0 &&
-    unknownFields(detail).some((field) => droppable.has(field) && !refused.refusedFields.has(field));
+/** The answer to a refusal that is the endpoint's: a flag still set, and a wording that clears it. */
+const endpointAnswer = (flag, matches, notice) => ({ detail, supports }) => {
+    if (!supports[flag] || !matches(detail))
+        return undefined;
+    supports[flag] = false;
+    return notice;
+};
+/** The same for a refusal that is the model's, which only a request that named one can meet. */
+const modelAnswer = (flag, matches, notice) => ({ detail, named }) => {
+    if (!named?.refused[flag] || !matches(detail))
+        return undefined;
+    named.refused[flag] = false;
+    return notice(named.name);
+};
+/** An effort refused by value: latched, so `effortFor` sends the next rung down. */
+const effortStepAnswer = ({ detail, named }) => {
+    const stepped = named && planEffortStep(detail, named.refused);
+    if (!named || !stepped)
+        return undefined;
+    named.refused.refusedEfforts.add(stepped.value);
+    if (stepped.supported)
+        named.refused.supportedEfforts = stepped.supported;
+    return `${named.name} does not reason at ${stepped.value}; retrying at ${stepped.next}`;
+};
+/** Fields the endpoint has never heard of, where they are droppable and not refused already. */
+const droppedFieldsAnswer = ({ detail, named, droppable }) => {
+    if (!named || droppable.size === 0)
+        return undefined;
+    const fields = unknownFields(detail).filter((field) => droppable.has(field) && !named.refused.refusedFields.has(field));
+    if (!fields.length)
+        return undefined;
+    for (const field of fields)
+        named.refused.refusedFields.add(field);
+    return `${named.name} does not take ${fields.join(", ")}; retrying without ${fields.length > 1 ? "them" : "it"}`;
+};
+/**
+ * Every refusal `negotiate` answers, in the order it asks. The first to answer is the only one.
+ *
+ * The order is behaviour rather than layout: one error text can satisfy two of these — a message
+ * about `reasoning_effort` is a refused field to one and a refused value to the next — and the
+ * earlier row is the reading that wins.
+ */
+const ANSWERS = [
+    endpointAnswer("strictSchemas", isGrammarError, "server could not build a grammar; retrying without pattern/format"),
+    endpointAnswer("usageInStream", (detail) => REJECTS_USAGE.test(detail), "server rejected stream_options; token counts unavailable"),
+    modelAnswer("reasoningEffort", rejectsEffort, (name) => `${name} does not take a reasoning effort; retrying without one`),
+    effortStepAnswer,
+    modelAnswer("legacyTokenLimit", wantsCompletionLimit, (name) => `${name} wants max_completion_tokens; retrying with the limit spelled that way`),
+    modelAnswer("chosenTemperature", refusesChosenTemperature, (name) => `${name} takes only its own temperature; retrying without ours`),
+    modelAnswer("structuredOutput", rejectsResponseFormat, (name) => `${name} does not take response_format; asking for JSON in words instead`),
+    droppedFieldsAnswer,
+];
 /**
  * Sends a request, re-sending it each time the answer is this endpoint refusing something the
  * request can do without. Returns once the endpoint has answered, or throws if the refusal is
@@ -385,16 +464,13 @@ const refusesDroppable = (detail, droppable, refused) => droppable.size > 0 &&
  */
 export async function negotiate(supports, send, { produced = { any: false }, onNotice, model: name, droppable } = {}) {
     const optional = new Set(droppable);
-    // The name and what it has refused, bound together because the notices below need both. They
-    // exist or are absent as one — the second is resolved from the first — but that is a fact
-    // about two locals, and narrowing one of those tells TypeScript nothing about the other.
     const named = name === undefined ? undefined : { name, refused: modelCapabilitiesFor(supports, name) };
     const model = named?.refused;
     for (;;) {
         // What this attempt was built with. `capabilitiesFor` and `modelCapabilitiesFor` hand one
         // object per endpoint and per model to everyone on them, so a run starting alongside this
-        // one may latch a flag off while this call is in flight — and the branches below are guarded
-        // on the flag still being set.
+        // one may latch a flag off while this call is in flight — and every answer in `ANSWERS` is
+        // guarded on its flag still being set.
         const sent = flagsOf(supports, model);
         try {
             return await send(supports, produced, model);
@@ -402,47 +478,22 @@ export async function negotiate(supports, send, { produced = { any: false }, onN
         catch (error) {
             if (produced.any)
                 throw error;
-            const detail = errorMessage(error);
-            const stepped = named && planEffortStep(detail, named.refused);
-            if (supports.strictSchemas && isGrammarError(detail)) {
-                supports.strictSchemas = false;
-                onNotice?.("server could not build a grammar; retrying without pattern/format");
-            }
-            else if (supports.usageInStream && REJECTS_USAGE.test(detail)) {
-                supports.usageInStream = false;
-                onNotice?.("server rejected stream_options; token counts unavailable");
-            }
-            else if (named?.refused.reasoningEffort && rejectsEffort(detail)) {
-                named.refused.reasoningEffort = false;
-                onNotice?.(`${named.name} does not take a reasoning effort; retrying without one`);
+            const refusal = {
+                detail: errorMessage(error),
+                supports,
+                named,
+                droppable: optional,
+            };
+            let notice;
+            for (const answer of ANSWERS) {
+                notice = answer(refusal);
+                if (notice !== undefined)
+                    break;
             }
-            else if (named && stepped) {
-                named.refused.refusedEfforts.add(stepped.value);
-                if (stepped.supported)
-                    named.refused.supportedEfforts = stepped.supported;
-                onNotice?.(`${named.name} does not reason at ${stepped.value}; retrying at ${stepped.next}`);
-            }
-            else if (named?.refused.legacyTokenLimit && wantsCompletionLimit(detail)) {
-                named.refused.legacyTokenLimit = false;
-                onNotice?.(`${named.name} wants max_completion_tokens; retrying with the limit spelled that way`);
-            }
-            else if (named?.refused.chosenTemperature && refusesChosenTemperature(detail)) {
-                named.refused.chosenTemperature = false;
-                onNotice?.(`${named.name} takes only its own temperature; retrying without ours`);
-            }
-            else if (named?.refused.structuredOutput && rejectsResponseFormat(detail)) {
-                named.refused.structuredOutput = false;
-                onNotice?.(`${named.name} does not take response_format; asking for JSON in words instead`);
-            }
-            else if (named && refusesDroppable(detail, optional, named.refused)) {
-                const fields = unknownFields(detail).filter((field) => optional.has(field) && !named.refused.refusedFields.has(field));
-                for (const field of fields)
-                    named.refused.refusedFields.add(field);
-                onNotice?.(`${named.name} does not take ${fields.join(", ")}; retrying without ${fields.length > 1 ? "them" : "it"}`);
-            }
-            else if (flagsOf(supports, model).every((flag, index) => flag === sent[index])) {
+            if (notice !== undefined)
+                onNotice?.(notice);
+            else if (flagsOf(supports, model).every((flag, index) => flag === sent[index]))
                 throw error;
-            }
             // Otherwise the refusal was answered by whoever got there first, and this attempt was
             // built before the answer existed. Two runs opening on a fresh llama.cpp box both get the
             // grammar error; the first latches it off and re-sends, and the second used to find the

package/dist/client.d.ts

@@ -27,10 +27,11 @@ export interface ClientPoolOptions {
  * Changes what the client pool is held to, for a process whose endpoints are not shaped like the
  * deployments these defaults were chosen for.
  *
- * Module-level for the same reason `configureEvents` is: the pool is one module-level thing, and
- * its size is a deployment's setting, said once at startup. A multi-tenant host keying on a key
- * per user raises `maxClients` so its tenants stop evicting each other's connection pools; a dev
- * box shortens `listingMissMs` so a model it has just pulled is seen sooner.
+ * One setting per runtime for the same reason `configureEvents` is: the pool is one thing per
+ * runtime, and its size is a deployment's setting, said once at startup. A multi-tenant host
+ * keying on a key per user raises `maxClients` so its tenants stop evicting each other's
+ * connection pools, or gives each tenant a `createRuntime` of its own; a dev box shortens
+ * `listingMissMs` so a model it has just pulled is seen sooner.
  *
  * A `maxClients` below the pool's current size evicts down to it at once, least recently asked
  * for first, as the next `getClient` would have. A shorter `listingMissMs` applies to misses