@abot-ai/runtime 1.8.1 → 2.0.0

package/README.md

@@ -68,7 +68,7 @@ You need:
 
 - Node.js `^20.19.0` or `>=22.12.0`; Node.js 21 is not supported
 - npm
-- An OpenAI or Ollama model connection
+- An OpenAI, Ollama or llama.cpp model connection
 
 Create a directory for your ABot installation:
 
@@ -89,6 +89,10 @@ choosing tools, and optionally enabling semantic memory.
 - **Ollama:** install and start Ollama separately, download your chosen model,
   and provide an address reachable from the Runtime.
 
+- **llama.cpp:** run llama-server separately and connect its model alias and
+  reachable origin. Thinking-capable models can use a token budget. See
+  [llama.cpp setup](docs/llamacpp.md).
+
 ABot includes the provider adapters. You supply the model and any required
 credentials. Configuration and runtime state are stored in your project.
 

package/dist/scripts/add-runtime-model.js

@@ -1,4 +1,6 @@
 import { mergeRuntimeModelAddition, parseRuntimeModelProfileId, } from "./runtime-model-addition.js";
+import { supportsProviderAddress } from "./runtime-setup-provider-options.js";
+import { parseSetupCommandOptions, requireSetupCommandProviderOptions } from "./runtime-setup-command-options.js";
 import { constants } from "node:fs";
 import { access, mkdir } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
@@ -17,6 +19,8 @@ function requireValue(argv, index, flag) {
     return value;
 }
 function parseArgs(argv, commandMode) {
+    const shared = parseSetupCommandOptions(argv);
+    argv = shared.argv;
     let rootDir = process.cwd();
     let provider;
     let model;
@@ -91,10 +95,11 @@ function parseArgs(argv, commandMode) {
         throw new Error("--model is required");
     if (!profile)
         throw new Error("--profile is required");
-    if (baseUrl && provider !== "ollama") {
-        throw new Error("--base-url is supported only with --provider ollama");
+    if (baseUrl && !supportsProviderAddress(provider)) {
+        throw new Error("--base-url is supported only with --provider ollama or llamacpp");
     }
-    return { rootDir, provider, model, profile, baseUrl, setDefault };
+    requireSetupCommandProviderOptions(provider, shared);
+    return { ...shared, rootDir, provider, model, profile, baseUrl, setDefault };
 }
 function printHelp(commandMode) {
     console.log([
@@ -165,14 +170,14 @@ export async function runAddRuntimeModel(argv = process.argv.slice(2), runOption
     const merged = mergeRuntimeModelAddition(runtimeConfig, {
         profileId: options.profile,
         providerId: options.provider,
-        provider: buildProviderConfig(options.provider, options.baseUrl),
+        provider: buildProviderConfig(options.provider, options.baseUrl, options.apiKeyEnv),
         profile: { configRef: `./models/${options.profile}.config.json` },
         baseUrl: options.baseUrl,
     });
     const nextRunnerConfig = options.setDefault
         ? selectDefaultProfile(runnerConfig, options.profile)
         : runnerConfig;
-    const modelConfig = buildModelConfig(modelTemplate, options.provider, options.model);
+    const modelConfig = buildModelConfig(modelTemplate, options.provider, options.model, options.contextWindowTokens, options.modelOptions);
     await mkdir(dirname(modelConfigPath), { recursive: true });
     await writeJsonObject(modelConfigPath, modelConfig, { exclusive: true });
     await writeJsonObject(runtimeConfigPath, merged.config);
@@ -188,6 +193,7 @@ export async function runAddRuntimeModel(argv = process.argv.slice(2), runOption
         ...(options.provider === "openai"
             ? ["credential: set OPENAI_API_KEY in .env"]
             : []),
+        ...(options.apiKeyEnv ? ["credential: set " + options.apiKeyEnv + " in .env"] : []),
         getRestartInstruction(commandMode),
     ].join("\n"));
 }

package/dist/scripts/init-runtime.js

@@ -1,3 +1,5 @@
+import { supportsProviderAddress } from "./runtime-setup-provider-options.js";
+import { parseSetupCommandOptions, requireSetupCommandProviderOptions } from "./runtime-setup-command-options.js";
 import { constants } from "node:fs";
 import { access, copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
 import { dirname, join, relative, resolve } from "node:path";
@@ -22,6 +24,8 @@ function requireValue(argv, index, flag) {
     return value;
 }
 function parseArgs(argv, commandMode) {
+    const shared = parseSetupCommandOptions(argv);
+    argv = shared.argv;
     let rootDir = process.cwd();
     let provider;
     let model;
@@ -88,10 +92,11 @@ function parseArgs(argv, commandMode) {
     if (!model) {
         throw new Error("--model is required");
     }
-    if (baseUrl && provider !== "ollama") {
-        throw new Error("--base-url is supported only with --provider ollama");
+    if (baseUrl && !supportsProviderAddress(provider)) {
+        throw new Error("--base-url is supported only with --provider ollama or llamacpp");
     }
-    return { rootDir, provider, model, baseUrl, force };
+    requireSetupCommandProviderOptions(provider, shared);
+    return { ...shared, rootDir, provider, model, baseUrl, force };
 }
 function printHelp(commandMode) {
     console.log([
@@ -108,7 +113,7 @@ function printHelp(commandMode) {
         "- .runtime/prod and .runtime/dev",
         "",
         "Both --provider and --model are required; the runtime has no built-in model choice.",
-        "--base-url configures the Ollama origin when Ollama runs outside the runtime host.",
+        "--base-url configures the Ollama or llama.cpp origin reachable from the Runtime.",
     ].join("\n"));
 }
 async function exists(path) {
@@ -147,12 +152,12 @@ async function ensureEnvLine(envPath) {
     await writeFile(envPath, next, "utf-8");
     return true;
 }
-async function configureRuntimeProvider(configPath, provider, baseUrl) {
+async function configureRuntimeProvider(configPath, provider, baseUrl, apiKeyEnv) {
     const config = await readJsonObject(configPath);
     const models = isRecord(config.models) ? config.models : {};
     const { defaults: _ignoredDefaults, ...modelSettings } = models;
     const providers = {
-        [provider]: buildProviderConfig(provider, baseUrl),
+        [provider]: buildProviderConfig(provider, baseUrl, apiKeyEnv),
     };
     config.models = {
         ...modelSettings,
@@ -165,9 +170,9 @@ async function configureRuntimeProvider(configPath, provider, baseUrl) {
     };
     await writeJsonObject(configPath, config);
 }
-async function configureModelProfile(configPath, provider, model) {
+async function configureModelProfile(configPath, provider, model, options) {
     const config = await readJsonObject(configPath);
-    await writeJsonObject(configPath, buildModelConfig(config, provider, model));
+    await writeJsonObject(configPath, buildModelConfig(config, provider, model, options.contextWindowTokens, options.modelOptions));
 }
 async function configureRunner(configPath) {
     const config = await readJsonObject(configPath);
@@ -221,13 +226,13 @@ export async function runInitRuntime(argv = process.argv.slice(2), runOptions =
         }),
     })));
     if (configStatus !== "kept") {
-        await configureRuntimeProvider(configPath, options.provider, options.baseUrl);
+        await configureRuntimeProvider(configPath, options.provider, options.baseUrl, options.apiKeyEnv);
     }
     if (requestRunnerConfigStatus !== "kept") {
         await configureRunner(requestRunnerConfigPath);
     }
     if (modelConfigStatus !== "kept") {
-        await configureModelProfile(modelConfigPath, options.provider, options.model);
+        await configureModelProfile(modelConfigPath, options.provider, options.model, options);
     }
     const runtimeDirectories = await initializeConfiguredRuntimeDirectories(options.rootDir, configPath);
     console.log([

package/dist/scripts/runtime-model-addition.js

@@ -1,3 +1,4 @@
+import { supportsProviderAddress } from "./runtime-setup-provider-options.js";
 import { isRecord } from "./runtime-setup-files.js";
 export function parseRuntimeModelProfileId(value) {
     const profile = value.trim();
@@ -28,11 +29,15 @@ export function mergeRuntimeModelAddition(runtimeConfig, input) {
             existingProvider.type !== input.provider.type) {
             throw new Error(`provider id is already configured with a different adapter: ${input.providerId}; no files were changed`);
         }
-        if (input.provider.type === "ollama" &&
+        if (supportsProviderAddress(String(input.provider.type)) &&
             input.baseUrl &&
             existingProvider.baseUrl !== input.baseUrl) {
             throw new Error(`provider ${input.providerId} already uses a different base URL; no files were changed`);
         }
+        const changesLlamaCredentialBinding = input.provider.type === "llamacpp" &&
+            input.provider.apiKeyEnv !== undefined && input.provider.apiKeyEnv !== existingProvider.apiKeyEnv;
+        if (changesLlamaCredentialBinding)
+            throw new Error("provider " + input.providerId + " already uses a different credential binding; no files were changed");
     }
     return {
         providerAdded: existingProvider === undefined,

package/dist/scripts/runtime-setup-command-options.d.ts

@@ -0,0 +1,11 @@
+import { type SetupModelOptions } from "./runtime-setup-model-options.js";
+export type SetupCommandOptions = Readonly<{
+    contextWindowTokens?: number;
+    apiKeyEnv?: string;
+    modelOptions: SetupModelOptions;
+}>;
+/** Reads shared setup flags before the existing command-specific argument parser. */
+export declare function parseSetupCommandOptions(args: string[]): SetupCommandOptions & {
+    argv: string[];
+};
+export declare function requireSetupCommandProviderOptions(provider: string, options: SetupCommandOptions): void;

package/dist/scripts/runtime-setup-command-options.js

@@ -0,0 +1,41 @@
+import { parseContextWindowTokens } from "./runtime-setup-files.js";
+import { parseProviderCredentialEnvironment } from "./runtime-setup-provider-options.js";
+import { parseCliReasoningBudget, parseSetupModelOptions } from "./runtime-setup-model-options.js";
+/** Reads shared setup flags before the existing command-specific argument parser. */
+export function parseSetupCommandOptions(args) {
+    const argv = [];
+    const options = {};
+    let contextWindowTokens;
+    let apiKeyEnv;
+    const flags = ["--supports-thinking", "--reasoning-budget", "--context-window", "--api-key-env"];
+    for (let index = 0; index < args.length; index += 1) {
+        const arg = args[index];
+        const flag = arg.split("=")[0];
+        if (!flags.includes(flag)) {
+            argv.push(arg);
+            continue;
+        }
+        const hasInlineValue = arg.startsWith(flag + "=");
+        const value = hasInlineValue ? arg.slice(flag.length + 1) : args[++index];
+        if (!value || value.startsWith("--"))
+            throw new Error(flag + " requires a value");
+        if (flag === "--supports-thinking") {
+            if (value !== "true" && value !== "false")
+                throw new Error("--supports-thinking must be true or false");
+            options.supportsThinking = value === "true";
+        }
+        if (flag === "--reasoning-budget")
+            options.reasoningBudget = parseCliReasoningBudget(value);
+        if (flag === "--context-window")
+            contextWindowTokens = parseContextWindowTokens(Number(value));
+        if (flag === "--api-key-env")
+            apiKeyEnv = parseProviderCredentialEnvironment(value);
+    }
+    return { argv, contextWindowTokens, apiKeyEnv, modelOptions: parseSetupModelOptions(options) };
+}
+export function requireSetupCommandProviderOptions(provider, options) {
+    if (options.apiKeyEnv && provider !== "llamacpp")
+        throw new Error("--api-key-env is supported only with --provider llamacpp");
+    if (options.modelOptions.reasoningBudget && provider !== "llamacpp")
+        throw new Error("--reasoning-budget is supported only with --provider llamacpp");
+}

package/dist/scripts/runtime-setup-files.d.ts

@@ -1,3 +1,4 @@
+import { type SetupModelOptions } from "./runtime-setup-model-options.js";
 export type Provider = string;
 export type JsonObject = Record<string, unknown>;
 export declare const DEFAULT_ROOT_RESPONSE_METHODOLOGY_FILES: readonly string[];
@@ -9,7 +10,7 @@ export declare function readJsonObject(path: string): Promise<JsonObject>;
 export declare function writeJsonObject(path: string, value: JsonObject, options?: Readonly<{
     exclusive?: boolean;
 }>): Promise<void>;
-export declare function buildProviderConfig(provider: Provider, baseUrl?: string): JsonObject;
-export declare function buildModelConfig(template: JsonObject, provider: Provider, model: string, contextWindowTokens?: number): JsonObject;
+export declare function buildProviderConfig(provider: Provider, baseUrl?: string, apiKeyEnv?: string): JsonObject;
+export declare function buildModelConfig(template: JsonObject, provider: Provider, model: string, contextWindowTokens?: number, modelOptions?: SetupModelOptions): JsonObject;
 export declare function parseContextWindowTokens(value: unknown): number | undefined;
 export declare function readDefaultContextWindowTokens(): Promise<number>;

package/dist/scripts/runtime-setup-files.js

@@ -1,6 +1,8 @@
 import { existsSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import { dirname, join, resolve } from "node:path";
+import { defaultProviderAddress, parseProviderCredentialEnvironment } from "./runtime-setup-provider-options.js";
+import { buildSetupModelReasoning } from "./runtime-setup-model-options.js";
 export const DEFAULT_ROOT_RESPONSE_METHODOLOGY_FILES = Object.freeze([
     "methodologies/response-ux.md",
     "methodologies/memory-informed-response.md",
@@ -61,7 +63,7 @@ export async function writeJsonObject(path, value, options = {}) {
         ...(options.exclusive ? { flag: "wx" } : {}),
     });
 }
-export function buildProviderConfig(provider, baseUrl) {
+export function buildProviderConfig(provider, baseUrl, apiKeyEnv) {
     if (provider === "openai") {
         return {
             type: "openai",
@@ -74,12 +76,49 @@ export function buildProviderConfig(provider, baseUrl) {
             baseUrl: baseUrl ?? "http://127.0.0.1:11434",
         };
     }
+    if (provider === "llamacpp") {
+        const credential = parseProviderCredentialEnvironment(apiKeyEnv);
+        return {
+            type: "llamacpp",
+            baseUrl: baseUrl ?? defaultProviderAddress(provider),
+            ...(credential ? { apiKeyEnv: credential } : {}),
+        };
+    }
     return { type: provider };
 }
-export function buildModelConfig(template, provider, model, contextWindowTokens) {
+export function buildModelConfig(template, provider, model, contextWindowTokens, modelOptions = {}) {
     const context = isRecord(template.context) ? template.context : {};
-    return {
+    const tokenEstimation = isRecord(context.tokenEstimation)
+        ? context.tokenEstimation
+        : {};
+    const calibration = isRecord(template.calibration) ? template.calibration : {};
+    const compaction = isRecord(calibration["context.compact"])
+        ? calibration["context.compact"]
+        : {};
+    const generation = isRecord(compaction.generation) ? compaction.generation : {};
+    const rawPayload = isRecord(calibration["toolPayload.raw"])
+        ? calibration["toolPayload.raw"]
+        : {};
+    const shouldApplyOllamaSetupCalibration = provider === "ollama";
+    const shouldApplyOpenAiStepTimeouts = provider === "openai";
+    const modelConfig = {
         ...template,
+        ...buildSetupModelReasoning(provider, isRecord(template.generation) ? template.generation : {}, modelOptions),
+        ...(shouldApplyOllamaSetupCalibration
+            ? {
+                calibration: {
+                    ...calibration,
+                    "context.compact": {
+                        ...compaction,
+                        generation: {
+                            temperature: 0,
+                            reasoningEffort: "none",
+                            ...generation,
+                        },
+                    },
+                },
+            }
+            : {}),
         label: model,
         provider,
         model,
@@ -95,11 +134,31 @@ export function buildModelConfig(template, provider, model, contextWindowTokens)
             : {}),
         context: {
             ...context,
+            ...(shouldApplyOllamaSetupCalibration
+                ? {
+                    tokenEstimation: {
+                        asciiCharactersPerToken: 3,
+                        nonAsciiBytesPerToken: 2,
+                        messageOverheadTokens: 6,
+                        ...tokenEstimation,
+                    },
+                }
+                : {}),
             formatTokenAccounting: {
-                mode: provider === "ollama" ? "none" : "estimate",
+                mode: shouldApplyOllamaSetupCalibration ? "none" : "estimate",
             },
         },
     };
+    if (!shouldApplyOpenAiStepTimeouts)
+        return modelConfig;
+    return {
+        ...modelConfig,
+        calibration: {
+            ...calibration,
+            "context.compact": { timeoutMs: 300_000, ...compaction },
+            "toolPayload.raw": { timeoutMs: 300_000, ...rawPayload },
+        },
+    };
 }
 export function parseContextWindowTokens(value) {
     if (value === undefined)

package/dist/scripts/runtime-setup-model-options.d.ts

@@ -0,0 +1,8 @@
+import { type ModelReasoningBudget } from "../src/shared/model-reasoning.js";
+export type SetupModelOptions = Readonly<{
+    supportsThinking?: boolean;
+    reasoningBudget?: ModelReasoningBudget;
+}>;
+export declare function parseSetupModelOptions(body: Record<string, unknown>): SetupModelOptions;
+export declare function parseCliReasoningBudget(value: string): ModelReasoningBudget;
+export declare function buildSetupModelReasoning(provider: string, generation: Record<string, unknown>, options: SetupModelOptions): Record<string, unknown>;

package/dist/scripts/runtime-setup-model-options.js

@@ -0,0 +1,40 @@
+import { parseModelReasoningBudget, } from "../src/shared/model-reasoning.js";
+export function parseSetupModelOptions(body) {
+    const supportsThinking = body.supportsThinking;
+    if (supportsThinking !== undefined && typeof supportsThinking !== "boolean")
+        throw new Error("Supports reasoning must be true or false.");
+    const reasoningBudget = parseModelReasoningBudget(body.reasoningBudget);
+    if (reasoningBudget !== undefined && supportsThinking !== true)
+        throw new Error("Enable reasoning support before configuring its budget.");
+    return {
+        ...(supportsThinking !== undefined ? { supportsThinking } : {}),
+        ...(reasoningBudget !== undefined ? { reasoningBudget } : {}),
+    };
+}
+export function parseCliReasoningBudget(value) {
+    const hasReasoningBudgetInput = value.trim().length > 0;
+    if (!hasReasoningBudgetInput)
+        throw new Error("A reasoning budget is required.");
+    if (value === "server-default")
+        return { mode: "server-default" };
+    const budget = parseModelReasoningBudget({ mode: "tokens", tokens: Number(value) });
+    if (!budget)
+        throw new Error("A reasoning budget is required.");
+    return budget;
+}
+export function buildSetupModelReasoning(provider, generation, options) {
+    const validated = parseSetupModelOptions(options);
+    if (validated.reasoningBudget && provider !== "llamacpp")
+        throw new Error("A token reasoning budget requires a llama.cpp provider.");
+    if (provider !== "llamacpp")
+        return validated.supportsThinking === undefined
+            ? {}
+            : { supportsThinking: validated.supportsThinking };
+    return {
+        supportsThinking: validated.supportsThinking === true,
+        capabilities: { inputModalities: ["text"] },
+        ...(validated.reasoningBudget
+            ? { generation: { ...generation, reasoningBudget: validated.reasoningBudget } }
+            : {}),
+    };
+}

package/dist/scripts/runtime-setup-presentation.js

@@ -3,11 +3,11 @@ function commandForMode(mode, sourceCommand, packageCommand) {
 }
 export function getInitUsage(mode) {
     const command = commandForMode(mode, "npm run init --", "npx abot init");
-    return `Usage: ${command} --provider <provider-adapter> --model <model-id> [--base-url <provider-origin>] [--force] [--root <dir>]`;
+    return `Usage: ${command} --provider <provider-adapter> --model <model-id> [--base-url <provider-origin>] [--context-window <tokens>] [--supports-thinking <true|false>] [--reasoning-budget <tokens|server-default>] [--api-key-env <name>] [--force] [--root <dir>]`;
 }
 export function getAddModelUsage(mode) {
     const command = commandForMode(mode, "npm run add-model --", "npx abot add-model");
-    return `Usage: ${command} --profile <profile-id> --provider <provider-adapter> --model <model-id> [--base-url <provider-origin>] [--default] [--root <dir>]`;
+    return `Usage: ${command} --profile <profile-id> --provider <provider-adapter> --model <model-id> [--base-url <provider-origin>] [--context-window <tokens>] [--supports-thinking <true|false>] [--reasoning-budget <tokens|server-default>] [--api-key-env <name>] [--default] [--root <dir>]`;
 }
 export function getMissingInitializationInstruction(mode) {
     return commandForMode(mode, "run npm run init first", "run npx abot init first");

package/dist/scripts/runtime-setup-provider-options.d.ts

@@ -0,0 +1,4 @@
+export declare function supportsProviderAddress(provider: string): boolean;
+export declare function defaultProviderAddress(provider: string): string | undefined;
+export declare function supportsProviderCredential(provider: string): boolean;
+export declare function parseProviderCredentialEnvironment(value: unknown): string | undefined;

package/dist/scripts/runtime-setup-provider-options.js

@@ -0,0 +1,20 @@
+export function supportsProviderAddress(provider) {
+    return provider === "ollama" || provider === "llamacpp";
+}
+export function defaultProviderAddress(provider) {
+    if (provider === "ollama")
+        return "http://127.0.0.1:11434";
+    if (provider === "llamacpp")
+        return "http://127.0.0.1:8080";
+    return undefined;
+}
+export function supportsProviderCredential(provider) {
+    return provider === "openai" || provider === "llamacpp";
+}
+export function parseProviderCredentialEnvironment(value) {
+    if (value === undefined)
+        return undefined;
+    if (typeof value === "string" && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(value))
+        return value;
+    throw new Error("API key environment name must be a valid environment variable name.");
+}

package/dist/src/model-gateway/client/chat.js

@@ -1,6 +1,7 @@
 import { traceDebug } from "../../runtime/observability/debug-logger.js";
 import { permitsModelContentTrace } from "../observability/content-trace-policy.js";
 import { buildModelInvocationMetrics } from "./invocation-metrics.js";
+import { fetchModelGatewayResponse, ModelGatewayInvocationError, } from "./invocation-error.js";
 import { buildChatGatewayRequestBody, resolveMetricMessages, resolveRequestMessages, } from "./message-projection.js";
 import { resolveClientOptions } from "./options.js";
 import { ChatStreamAccumulator, classifyEmptyResponse, InactivityBoundStreamReader, } from "./stream-consumer.js";
@@ -24,14 +25,16 @@ export async function invokeModelGatewayWithOptions(params, options = {}) {
         textLength: typeof params.text === "string" ? params.text.length : 0,
         messageCount: requestMessages.length,
     });
-    const response = await fetch(url + "/chat", {
+    const serializedRequestBody = JSON.stringify(requestBody);
+    const response = await fetchModelGatewayResponse(url + "/chat", {
         method: "POST",
         headers: {
             "Content-Type": "application/json",
         },
         signal: params.abortSignal,
-        body: JSON.stringify(requestBody),
-    });
+        body: serializedRequestBody,
+    }, params.abortSignal);
+    params.abortSignal.throwIfAborted();
     traceDebug("model-gateway.client", "chat.response.received", {
         requestId,
         status: response.status,
@@ -39,6 +42,7 @@ export async function invokeModelGatewayWithOptions(params, options = {}) {
     });
     if (!response.ok) {
         const errorText = await response.text().catch(() => "");
+        params.abortSignal.throwIfAborted();
         traceDebug("model-gateway.client", "chat.response.error", {
             requestId,
             status: response.status,
@@ -47,13 +51,13 @@ export async function invokeModelGatewayWithOptions(params, options = {}) {
                 ? { bodyPreview: errorText.slice(0, 400) }
                 : { contentOmitted: true }),
         });
-        throw new Error(`bridge_request_failed:${response.status}:${permitsModelContentTrace(params.modelStep, params.messages) ? errorText || "empty error body" : "content omitted"}`);
+        throw new ModelGatewayInvocationError("http", `bridge_request_failed:${response.status}:${permitsModelContentTrace(params.modelStep, params.messages) ? errorText || "empty error body" : "content omitted"}`);
     }
     if (!response.body) {
         traceDebug("model-gateway.client", "chat.response.no_body", {
             requestId,
         });
-        throw new Error("no stream");
+        throw new ModelGatewayInvocationError("missing_stream", "no stream");
     }
     const streamReader = new InactivityBoundStreamReader(response.body.getReader(), params.abortSignal, clientOptions.getStreamInactivityTimeoutMs);
     const stream = new ChatStreamAccumulator(params);

package/dist/src/model-gateway/client/contracts.d.ts

@@ -1,3 +1,4 @@
+import type { ModelReasoningPreference } from "../../shared/model-reasoning.js";
 import type { AgentMode, ModelReasoningLevel, ModelStep } from "../../shared/types.js";
 import type { StreamCallbacks } from "../client-stream-parser.js";
 import type { ModelGatewayAttachment, ModelGatewayPolicyConfig, ModelPreference, ModelTokenUsage } from "../types.js";
@@ -19,6 +20,7 @@ export type InvokeModelGatewayParams = StreamCallbacks & {
     modelOverride?: string;
     reasoningOverride?: ModelReasoningLevel;
     modelPreference?: ModelPreference;
+    reasoningPreference?: ModelReasoningPreference;
     modelPolicy?: ModelGatewayPolicyConfig;
     abortSignal: AbortSignal;
     debugRequestId?: string;
@@ -32,6 +34,7 @@ export type InvokeRawModelGatewayParams = {
     modelOverride?: string;
     reasoningOverride?: ModelReasoningLevel;
     modelPreference?: ModelPreference;
+    reasoningPreference?: ModelReasoningPreference;
     modelPolicy?: ModelGatewayPolicyConfig;
     abortSignal: AbortSignal;
     debugRequestId?: string;

package/dist/src/model-gateway/client/input-token-count.js

@@ -1,5 +1,6 @@
 import { createHash } from "node:crypto";
 import { traceDebug } from "../../runtime/observability/debug-logger.js";
+import { InputTokenCountDiagnostics } from "../observability/input-token-count-diagnostics.js";
 import { toolMediaSafeMessage } from "../observability/tool-media-privacy.js";
 import { buildChatGatewayRequestBody, resolveRequestMessages, } from "./message-projection.js";
 import { resolveClientOptions } from "./options.js";
@@ -61,6 +62,10 @@ function updateInputTokenCountCache(cache, cacheKey, result) {
         cache.delete(oldestKey);
     }
 }
+// A local llama.cpp server can change its template/tokenizer without changing the request.
+function canCacheProviderInputTokenCount(provider) {
+    return provider !== "llamacpp";
+}
 export async function countModelGatewayInputTokensWithOptions(params, options = {}, cache) {
     const clientOptions = resolveClientOptions(options);
     const provider = params.provider.trim();
@@ -71,7 +76,8 @@ export async function countModelGatewayInputTokensWithOptions(params, options =
     const modelPolicy = params.modelPolicy ?? clientOptions.modelPolicy;
     const requestBody = buildChatGatewayRequestBody(params, requestMessages, modelPolicy);
     const cacheKey = createInputTokenCountCacheKey(provider, requestBody);
-    const cached = cache?.get(cacheKey);
+    const eligibleCache = canCacheProviderInputTokenCount(provider) ? cache : undefined;
+    const cached = eligibleCache?.get(cacheKey);
     if (cached) {
         return cached;
     }
@@ -84,21 +90,35 @@ export async function countModelGatewayInputTokensWithOptions(params, options =
         modelStep: params.modelStep,
         messageCount: requestMessages.length,
     });
-    const response = await fetch(`${clientOptions.baseUrl}/input-tokens`, {
-        method: "POST",
-        headers: {
-            "Content-Type": "application/json",
-        },
-        signal: params.abortSignal,
-        body: JSON.stringify(requestBody),
-    });
-    if (!response.ok) {
-        const errorText = await response.text().catch(() => "");
-        throw new Error(`bridge_input_token_count_failed:${response.status}:${toolMediaSafeMessage(errorText || "empty error body", requestMessages)}`);
+    const diagnostics = new InputTokenCountDiagnostics("model-gateway.client", requestBody);
+    diagnostics.setProvider(provider);
+    let statusCode;
+    let result;
+    try {
+        diagnostics.setPhase("bridge_fetch");
+        const response = await fetch(`${clientOptions.baseUrl}/input-tokens`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+            },
+            signal: params.abortSignal,
+            body: JSON.stringify(requestBody),
+        });
+        statusCode = response.status;
+        diagnostics.setPhase("bridge_response");
+        if (!response.ok) {
+            const errorText = await response.text().catch(() => "");
+            throw new Error(`bridge_input_token_count_failed:${response.status}:${toolMediaSafeMessage(errorText || "empty error body", requestMessages)}`);
+        }
+        diagnostics.setPhase("bridge_parse");
+        result = parseInputTokenCountResult(await response.json().catch(() => undefined));
+        if (result.provider !== provider) {
+            throw new Error("model_gateway_input_token_count_provider_mismatch");
+        }
     }
-    const result = parseInputTokenCountResult(await response.json().catch(() => undefined));
-    if (result.provider !== provider) {
-        throw new Error("model_gateway_input_token_count_provider_mismatch");
+    catch (error) {
+        diagnostics.failed({ kind: "exception", error, statusCode, aborted: params.abortSignal.aborted });
+        throw error;
     }
     traceDebug("model-gateway.client", "input_tokens.request.completed", {
         requestId,
@@ -108,6 +128,6 @@ export async function countModelGatewayInputTokensWithOptions(params, options =
         inputTokens: result.inputTokens,
         durationMs: Date.now() - startedAt,
     });
-    updateInputTokenCountCache(cache, cacheKey, result);
+    updateInputTokenCountCache(eligibleCache, cacheKey, result);
     return result;
 }

package/dist/src/model-gateway/client/invocation-error.d.ts

@@ -0,0 +1,9 @@
+export type ModelGatewayInvocationFailureKind = "network" | "http" | "stream" | "inactivity" | "missing_stream";
+/** A failed transport attempt, distinct from local validation or callbacks. */
+export declare class ModelGatewayInvocationError extends Error {
+    readonly kind: ModelGatewayInvocationFailureKind;
+    constructor(kind: ModelGatewayInvocationFailureKind, message: string, options?: ErrorOptions);
+}
+export declare function wrapModelGatewayTransportError(kind: "network" | "stream", cause: unknown): ModelGatewayInvocationError;
+/** Only fetch rejection is classified; request construction stays with callers. */
+export declare function fetchModelGatewayResponse(url: string, init: RequestInit, abortSignal: AbortSignal): Promise<Response>;

package/dist/src/model-gateway/client/invocation-error.js

@@ -0,0 +1,30 @@
+/** A failed transport attempt, distinct from local validation or callbacks. */
+export class ModelGatewayInvocationError extends Error {
+    kind;
+    constructor(kind, message, options) {
+        super(message, options);
+        this.kind = kind;
+        this.name = "ModelGatewayInvocationError";
+    }
+}
+export function wrapModelGatewayTransportError(kind, cause) {
+    return new ModelGatewayInvocationError(kind, readTransportFailureMessage(kind, cause), { cause });
+}
+function readTransportFailureMessage(kind, cause) {
+    if (cause instanceof Error)
+        return cause.message;
+    if (typeof cause === "string")
+        return cause;
+    return `model_gateway_${kind}_failed`;
+}
+/** Only fetch rejection is classified; request construction stays with callers. */
+export async function fetchModelGatewayResponse(url, init, abortSignal) {
+    try {
+        return await fetch(url, init);
+    }
+    catch (error) {
+        if (abortSignal.aborted)
+            throw error;
+        throw wrapModelGatewayTransportError("network", error);
+    }
+}