@atomic-chat/core 0.5.1 → 0.13.2

package/dist/runtime/tensorrt-llm/adapter.d.ts

@@ -0,0 +1,250 @@
+import { type GenerationDefaults } from '../managed-text/generation-defaults.js';
+import type { ModelFamilySupport } from '../../contracts/index.js';
+import { reasoningIntoContent } from '../managed-text/index.js';
+import type { ManagedEngineLaunch, ManagedExitClassification, ManagedLaunchContext, ManagedRoute, ManagedTextAdapter, ManagedTextCapabilities } from '../managed-text/index.js';
+export declare const TENSORRT_LLM_MIN_CONTEXT_LENGTH = 512;
+export declare const TENSORRT_LLM_MAX_CONTEXT_LENGTH = 1048576;
+export declare const TENSORRT_LLM_DEFAULT_CONTEXT_LENGTH = 8192;
+export declare const TENSORRT_LLM_MIN_MAX_OUTPUT_TOKENS = 1;
+export declare const TENSORRT_LLM_MAX_MAX_OUTPUT_TOKENS = 1048576;
+export declare const TENSORRT_LLM_DEFAULT_MAX_OUTPUT_TOKENS = 4096;
+/** Same bounds `trtllm-serve`'s own `--kv_cache_free_gpu_memory_fraction` documents as sane: never
+ *  the whole of free memory (nothing left for the engine's own workspace) and never a sliver. */
+export declare const TENSORRT_LLM_MIN_KV_CACHE_FREE_FRACTION = 0.1;
+export declare const TENSORRT_LLM_MAX_KV_CACHE_FREE_FRACTION = 0.95;
+export declare const TENSORRT_LLM_MIN_LOAD_TIMEOUT_SECONDS = 1;
+export declare const TENSORRT_LLM_MAX_LOAD_TIMEOUT_SECONDS = 3600;
+/** The adapter's own validated settings shape (spec `tensorrt-llm-runtime`, "Выбор карты и настройки
+ *  провайдера"). The JSON settings schema this validates against on the wire is task 2.14's. */
+export interface TensorrtLlmSettings {
+    /** `GPU-<uuid>`/`MIG-<uuid>`, or `null` to let the load pick the card with the most free memory. */
+    gpu_id: string | null;
+    context_length: number;
+    max_output_tokens: number;
+    kv_cache_free_gpu_memory_fraction: number;
+    /** `--max_batch_size` (`TENSORRT_LLM_MAX_BATCH_SIZE` by default). */
+    max_batch_size: number;
+    /** `kv_cache_config.max_tokens`; `null` sizes it as `context_length × max_batch_size`. */
+    kv_cache_max_tokens: number | null;
+    cuda_graphs: 'auto' | 'on' | 'off';
+    kv_cache_dtype: 'auto' | 'fp8';
+    /** Seconds. `null` leaves `readinessTimeoutMs`'s own weight-based estimate in force. */
+    load_timeout_seconds: number | null;
+    /** `kv_cache_config.enable_block_reuse` (the engine's own default: on). */
+    enable_prefix_caching: boolean;
+    /** `false` writes `disable_overlap_scheduler: true` (the engine's own default: overlap on). */
+    overlap_scheduler: boolean;
+    /** `scheduler_config.capacity_scheduler_policy`, written only when not the engine's default. */
+    capacity_scheduler_policy: 'guaranteed_no_evict' | 'max_utilization';
+    /** The LLM API `dtype` of unquantized weights; `auto` writes nothing. */
+    dtype: 'auto' | 'float16' | 'bfloat16' | 'float32';
+    /** Sampling defaults the session gateway writes into a request that sets none. */
+    generation: GenerationDefaults;
+}
+/** Throws `AtomicCoreError('INVALID_ARGUMENT', ...)` before any container exists (adapter contract). */
+export declare function validateTensorrtLlmSettings(raw: unknown): TensorrtLlmSettings;
+/** `trtllm-serve serve`'s own default `--port`. */
+export declare const TENSORRT_LLM_CONTAINER_PORT = 8000;
+/**
+ * The LLM API option file a launch passes to `trtllm-serve serve --extra_llm_api_options`
+ * (`commands/serve.py` in v1.2.1: `--config`/`--extra_llm_api_options`, a YAML file whose keys
+ * overwrite the LLM API arguments), written when either of two keys applies:
+ *
+ * - `guided_decoding_backend: xgrammar`, for a family that declares structured output
+ *   (`llmapi/llm_args.py`'s `guided_decoding_backend: Optional[Literal["xgrammar", "llguidance"]]`,
+ *   default `None`). Without it, `openai_protocol.py` still turns `response_format` into
+ *   guided-decoding parameters, but no backend exists to enforce them. `xgrammar` is the backend both
+ *   the PyTorch and TensorRT backends implement.
+ * - `kv_cache_config: {max_tokens: N}`, on a unified-memory card (`tensorrtLlmUnifiedKvMaxTokens`,
+ *   `kv-cache.ts`).
+ *   `llm_args.py` `KvCacheConfig.max_tokens` (1.2.1, line 1636): "If both `max_tokens` and
+ *   `free_gpu_memory_fraction` are specified, memory corresponding to the minimum will be used."
+ *   The fraction the argv passes survives the YAML: `serve.py`'s `get_llm_args` builds
+ *   `KvCacheConfig(free_gpu_memory_fraction=...)` (line 125), and `update_llm_args_with_extra_dict`
+ *   (`llm_args.py` line 3312) deep-merges a YAML `kv_cache_config` over it
+ *   (`model_dump(exclude_unset=True) | yaml`), so the fraction stays the upper bound.
+ */
+export declare const TENSORRT_LLM_API_OPTIONS_FILE = "llm-api-options.yaml";
+export declare const TENSORRT_LLM_GUIDED_DECODING_OPTIONS = "guided_decoding_backend: xgrammar\n";
+/** Below this much card memory `cuda_graphs: auto` leaves CUDA graphs off: they took about 2 GB on
+ *  an 8 GB card in the Windows live acceptance (Qwen3.5-2B, "Memory used outside torch … 2.12 GiB").
+ *  A card that reports no size of its own is a unified-memory one (GB10): it keeps them. */
+export declare const TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES: number;
+/**
+ * `--max_batch_size`: how many sequences the engine serves at once. `trtllm-serve` defaults to 2048, a
+ * server's number; a desktop runs a chat, an agent turn and a few API calls at most. For a model with
+ * recurrent layers (Qwen3.5, hybrid Mamba) the engine reserves the recurrent state for every one of
+ * those sequences up front: on an 8 GB card Qwen3.5-2B failed with "The V2 Mamba GPU cache quota is
+ * too small … need at least 20696801280 bytes" (Windows live acceptance, 1.3.0rc29). Attention-only
+ * models only lose queueing beyond this many concurrent requests. One by default (owner's decision,
+ * change `add-vllm-runtime`): the KV cache is then sized for one full context, which is what fits a
+ * desktop card; a person who runs agents in parallel raises it.
+ */
+export declare const TENSORRT_LLM_MAX_BATCH_SIZE = 1;
+export declare const TENSORRT_LLM_MAX_MAX_BATCH_SIZE = 256;
+export declare const TENSORRT_LLM_MAX_KV_CACHE_MAX_TOKENS = 16777216;
+/** Builds `trtllm-serve serve <model-dir> ...` (task 2.13; flags sourced from `serve.py`, see the
+ *  file header). `serve` is named explicitly rather than relying on the CLI's default-command
+ *  resolution (`DefaultGroup.resolve_command`, `commands/serve.py`) — that quirk keeps `trtllm-serve
+ *  MODEL` working when a user types it by hand, but a generated argv should not depend on undocumented
+ *  fallback dispatch. The tool/reasoning parser flags are appended only when the pinned descriptor's
+ *  `model_families` entry for this model's architecture names them (spec "Возможности модели
+ *  объявляются, а не угадываются", design D9); the descriptor's own regex already validated those
+ *  names, so they are passed through verbatim.
+ *
+ *  `--max_num_tokens` is set to `context_length`, not `max_output_tokens` (findings-2.13-r1.md item
+ *  1's ruling): on the pytorch backend this flag caps the *prompt* alone
+ *  (`llmapi/llm.py`'s `_check_arguments`, file header), so pointing it at the output-length setting
+ *  was wrong from the start — it silently shrank the usable prompt window to whatever the output
+ *  setting happened to be. `max_output_tokens` is enforced per request instead, by
+ *  `tensorrtLlmRewriteRequestBody` through the session gateway (below; see
+ *  `docs/decisions/2026-09-28-tensorrt-llm-output-cap-enforced-by-the-session-gateway.md`), because
+ *  this engine release has no server-side flag that caps a reply's length at all. */
+export declare function buildTensorrtLlmLaunch(context: ManagedLaunchContext<TensorrtLlmSettings>): ManagedEngineLaunch;
+/**
+ * Placeholder coefficients (design D8's own open question: "Точные... коэффициенты таймаута
+ * загрузки — измеряются в live-тесте core", i.e. live test 2.19 replaces these). Until then they
+ * only have to clear the spec's own acceptance scenario ("Долгий первый старт": a 30 GB model
+ * starting in three minutes must not time out) with real headroom, not cut it close.
+ *
+ * Raised from 60 s + 6 s/GiB (2026-10-06): on an NVIDIA RTX Spark (Windows on Arm, WSL) Qwen3.5-2B
+ * spent 62 s in executor profiling alone and was still sizing its KV cache, with no error, when the
+ * 128 s that formula gave ran out. A healthy slow start must not fail; a stuck one still ends.
+ */
+export declare const TENSORRT_LLM_READINESS_BASE_MS = 240000;
+/** Budgeted per GiB of checkpoint weights, before the margin multiplier — to be measured in live test 2.19. */
+export declare const TENSORRT_LLM_READINESS_PER_GIB_MS = 12000;
+/** Applied to the whole base-plus-per-GiB estimate — to be measured in live test 2.19. */
+export declare const TENSORRT_LLM_READINESS_MARGIN = 1.5;
+/** Base plus time-per-GiB-of-weights, with margin (spec "Этапы и таймаут загрузки", design D8) —
+ *  unless `settings.load_timeout_seconds` is set, which wins outright (findings-2.13-r1.md item 4:
+ *  round 1 validated this setting but never read it back, so it was accepted and then silently
+ *  ignored). The lifecycle's own `resolveReadinessTimeoutMs` (`../managed-text/load-policy.ts`)
+ *  still applies a *second*, load-request-level override on top of whatever this returns — that
+ *  path exists for a caller that computes its own override outside the settings object entirely; a
+ *  `tensorrt-llm` load never populates it, so this function's return value is what actually wins. */
+export declare function tensorrtLlmReadinessTimeoutMs(weightBytes: number, settings: TensorrtLlmSettings): number;
+/**
+ * Whether a `trtllm-serve` that answered `/health` can hold one sequence of the context it was
+ * launched with: the KV-cache manager logs how long a sequence it can keep (the last line wins — the
+ * engine logs one for its memory-profiling pass and one for the final allocation). Shorter than the
+ * context length is an out-of-memory refusal with both numbers — Ministral-3b bf16 on an RTX 4070
+ * Laptop started with room for 224 tokens of a 4096-token context and answered every request with an
+ * empty, failed stream. No such line (another engine build) is not a refusal.
+ */
+export declare function classifyTensorrtLlmReady(log: string, settings: TensorrtLlmSettings): ManagedExitClassification | null;
+/**
+ * The engine's own words for the request it failed mid-stream: the last `RequestError` in the log
+ * tail. `trtllm-serve` raises it inside the streaming generator after `200 OK` and only logs it, so
+ * the client otherwise saw an empty answer (Ministral-3b on an 8 GB card: "KV_CACHE_MANAGER requires
+ * 67 KV cache blocks to complete the request, which exceeds its GPU-primary capacity of 7 blocks").
+ */
+export declare function describeTensorrtLlmStreamFailure(logTail: string): string | null;
+export declare function classifyTensorrtLlmExit(log: string, exitCode: number | null): ManagedExitClassification;
+/** What a loaded model can do (spec "Возможности модели объявляются, а не угадываются", design D9):
+ *  tool calling and reasoning only when the pinned descriptor's `model_families` entry for this
+ *  model's architecture names the parser the launch was actually started with; structured output
+ *  only when that same entry declares it; vision, embeddings and the Responses API are unsupported
+ *  in this slice regardless of family. `settings` plays no part in this — capabilities are read
+ *  entirely off the descriptor, never guessed the way the llama.cpp provider does — but the
+ *  parameter stays because `ManagedTextAdapter.capabilities` always hands both through. */
+export declare function tensorrtLlmCapabilities(context: {
+    settings: TensorrtLlmSettings;
+    family: ModelFamilySupport | null;
+}): ManagedTextCapabilities;
+/** The OpenAI method+path routes this adapter's engine actually serves (`openai_server.py`
+ *  registers exactly these three for text; task 2.14 refuses any other public route for a
+ *  `tensorrt-llm` model — spec: "Публичный сервер MUST отвечать понятной ошибкой на маршрут,
+ *  который провайдер не объявил"). Also `ManagedTextAdapter.routes` below: the session gateway
+ *  404s a path not listed here at all, and 405s a listed path used with a method not listed for
+ *  it (findings-2.13-r3.md item 1) — either way before it ever reaches the container. This closes
+ *  off `trtllm-serve`'s own undeclared routes — its `/health` (probed directly by the lifecycle,
+ *  never through the gateway), its administrative routes this adapter never wanted reachable at
+ *  all (`/update_weights`, `/release_memory`, `/resume_memory`, `/kv_cache_events`,
+ *  `/steady_clock_offset`), `/v1/responses` (real in `openai_server.py`, unsupported in this
+ *  slice) — and a wrong method on a route that is otherwise real, e.g. `GET
+ *  /v1/chat/completions` or `POST /v1/models` (findings-2.13-r2.md item 3, keyed on method+path
+ *  since findings-2.13-r3.md item 1). No route here declares `HEAD`; a `HEAD /v1/models` request
+ *  gets the same `405` as any other undeclared method for a declared path — this engine's `GET
+ *  /v1/models` has no dedicated `HEAD` handler to serve it from. */
+export declare const TENSORRT_LLM_ROUTES: readonly ManagedRoute[];
+/** The subset of `TENSORRT_LLM_ROUTES` `tensorrtLlmRewriteRequestBody` may rewrite: both POST
+ *  routes, never `GET /v1/models` (no request body to rewrite in the first place). */
+export declare const TENSORRT_LLM_REWRITABLE_ROUTES: readonly ManagedRoute[];
+/** An OpenAI-compatible error body: `{"error": {...}}`, the shape every OpenAI-compatible client
+ *  already expects (matches `server/public/errors.ts`'s `structureBackendErrorBody`'s envelope
+ *  shape for the llama.cpp/MLX providers, so a caller does not need a second error shape to
+ *  handle). */
+export interface TensorrtLlmOpenAIError {
+    error: {
+        message: string;
+        type: string;
+        param: string | null;
+        code: string;
+    };
+}
+/**
+ * Maps `trtllm-serve` 1.2.1's context-overflow errors — the pytorch backend's two real, reachable
+ * shapes, see the file header — to an OpenAI-compatible `context_length_exceeded` envelope with
+ * both numbers. `null` for any other `400` (a different validation failure, e.g. a bad sampling
+ * parameter) or any other status — 2.14 falls back to its own generic error wrapping for those, the
+ * same way `server/public/errors.ts` already does for llama.cpp/MLX. This is a pure text mapping;
+ * wiring it into the public `/v1/*` route handler is task 2.14's.
+ */
+export declare function mapTensorrtLlmContextLengthError(status: number, body: string): TensorrtLlmOpenAIError | null;
+/**
+ * Reasoning parsers that assume every reply starts inside the reasoning, because the chat template
+ * opens `<think>` in the prompt when thinking is on. `qwen3_5` (Qwen3.5, 1.3.0rc29) is one: with
+ * thinking off the template closes an empty `<think></think>` in the prompt instead, the model
+ * answers with no tags at all, and the parser still files the whole answer under
+ * `reasoning_content`, leaving `content` empty (Windows live acceptance, Qwen3.5-2B). The parser is
+ * fixed per container; only the request says whether thinking was on.
+ */
+export declare const TENSORRT_LLM_REASONING_AT_START_PARSERS: ReadonlySet<string>;
+/** Moves a no-thinking reply filed as reasoning back into `content` (the shared rule, `request-rules.ts`). */
+export declare const tensorrtLlmReasoningIntoContent: typeof reasoningIntoContent;
+/** The response rewrite for a chat request with thinking off on a reasoning-at-start parser. */
+export declare function tensorrtLlmRewriteResponseFor(route: string, requestBody: unknown, family: ModelFamilySupport | null): ((json: Record<string, unknown>) => Record<string, unknown>) | null;
+/**
+ * Enforces `settings.max_output_tokens` per request, since `trtllm-serve` 1.2.1 has no server-side
+ * flag that does it (`buildTensorrtLlmLaunch`'s doc comment; ADR
+ * `docs/decisions/2026-09-28-tensorrt-llm-output-cap-enforced-by-the-session-gateway.md`). Only
+ * `TENSORRT_LLM_REWRITABLE_ROUTES` are touched; every other route's body returns unchanged, as does
+ * a body that is not a plain JSON object (an array, a primitive, `null`) — the gateway's own JSON
+ * parse already rejected anything that is not valid JSON at all before this ever runs.
+ *
+ * `/v1/completions` only ever had `max_tokens`: a present value is clamped to
+ * `min(value, settings.max_output_tokens)`; absent, the setting is written outright.
+ *
+ * `/v1/chat/completions` writes exactly ONE of `max_tokens`/`max_completion_tokens` — never both —
+ * because TRT-LLM 1.2.1's `ChatCompletionRequest` (`openai_protocol.py`, `extra="forbid"`) has a
+ * single Python field, `max_completion_tokens`, whose `validation_alias` is `max_tokens`; sending
+ * both as separate top-level keys makes pydantic reject the whole request with `400
+ * extra_forbidden` (findings-2.13-r2.md item 1, reproduced by the reviewer with `pydantic` against
+ * this exact model — round 1's own ruling did exactly this and broke every chat request). If the
+ * client sent `max_completion_tokens`, the rewritten body keeps that key; else if it sent
+ * `max_tokens`, that key; else `max_tokens` (this route's own default when nothing was sent at
+ * all). If the client sent both, the lower of the two (each still capped) wins, under the
+ * `max_completion_tokens` key.
+ *
+ * A present value that is not a positive integer (a string, `0`, a negative number — `null` counts
+ * as not sent, matching how OpenAI clients omit a field) throws rather than silently substituting
+ * the setting, so the client sees why its request was refused instead of one that silently used a
+ * different limit than it asked for. The gateway (`../managed-text/gateway.ts`) turns that throw
+ * into an OpenAI-shaped `400` using this function's own `Error.message`.
+ *
+ * `n` (multiple choices) is untouched: the cap is a per-choice output limit — the same as
+ * `max_output_tokens` is documented to mean — not a budget shared across `n` completions.
+ *
+ * With the session's `capabilities` (the lifecycle always passes them; task 2.14 fix round 1), a
+ * request asking for what the model cannot do — tool calls without a parser, JSON output without
+ * structured-output support — is refused first (`refuseUnsupported`), before anything is rewritten.
+ *
+ * On both routes (`CompletionRequest` and `ChatCompletionRequest` share one `ResponseFormat` in
+ * 1.2.1), an OpenAI `json_schema` wrapper in `response_format` is unwrapped to the bare schema the
+ * engine reads (`unwrapJsonSchemaFormat`).
+ */
+export declare function tensorrtLlmRewriteRequestBody(route: string, body: unknown, settings: TensorrtLlmSettings, capabilities?: ManagedTextCapabilities): unknown;
+/** The `tensorrt-llm` engine's `ManagedTextAdapter` (task 2.13). Registered against the pinned
+ *  descriptor's `adapter_id`/`adapter_contract_version` by whatever wires the registry up (task 2.14). */
+export declare const tensorrtLlmAdapter: ManagedTextAdapter<TensorrtLlmSettings>;