@atomic-chat/core 0.13.2 → 0.17.0

package/dist/runtime/sglang/memory.d.ts

@@ -0,0 +1,64 @@
+/**
+ * SGLang's memory, decided by core (change `add-sglang-runtime`, task 2.3, design D4; spec
+ * `sglang-runtime`, "Память SGLang задаёт core"). SGLang sizes its budget from the memory it finds
+ * free at start-up, once its own CUDA context exists: it keeps `--mem-fraction-static` of it for the
+ * weights and the KV cache, and the rest is its room for activations and CUDA graphs
+ * (`mem_cache/kv_cache_configurator.py::_profile_available_bytes`). Its own default share is computed
+ * from the card's *total* memory, so on a card a desktop already holds part of, that room comes out
+ * smaller than SGLang meant. Core instead:
+ *
+ * - gives SGLang (unless the share setting fixes it) `f = (F − R) / F`, clamped to [0.5, 0.95], where
+ *   `F` is what the card has free right before the container is created and `R` the room SGLang needs
+ *   (2 GiB with CUDA graphs, 1.25 GiB without): SGLang then keeps about `R` whatever other processes
+ *   took. On a card with the host's memory the static share holds at most half of the host's memory:
+ *   `(min(F, half the host) − R) / F`. Within its share SGLang sizes the KV cache itself — hybrid
+ *   models' linear-attention state included — and core passes `--max-total-tokens` only from the
+ *   setting.
+ * - checks, before any container, that the weights, an estimate of the KV cache (context × parallel
+ *   requests × the model's per-token KV bytes over its attention layers, or the KV token limit) and
+ *   SGLang's own overhead fit the free memory — the same rule `POST /models/sglang/check` applies. It
+ *   is an estimate for the verdict and the refusal before a container; it sizes nothing SGLang runs
+ *   with.
+ *
+ * The reserve, the overhead (~2 GiB: CUDA context, activations, graphs) and the margin are first
+ * estimates; the live run of task 4.1 records the numbers (`summary.json`, `rulings/core.md`).
+ */
+import type { GpuFacts } from '../../contracts/index.js';
+import { type CheckpointMemoryInputs, type HostMemory, type JsonObject, type KvReserveBasis, type ManagedCheckEngine, type MemoryNeed } from '../managed-models/index.js';
+import type { SglangLaunchPlan } from './adapter.js';
+import type { SglangSettings } from './settings.js';
+/** SGLang's room for activations and CUDA graphs captured up to the parallel requests (design D4, D5). */
+export declare const SGLANG_RUNTIME_RESERVE_BYTES: number;
+/** The same room with CUDA graphs off. */
+export declare const SGLANG_RUNTIME_RESERVE_NO_GRAPHS_BYTES: number;
+/** What SGLang holds beyond weights and KV cache: CUDA context, activations, graphs. */
+export declare const SGLANG_ENGINE_OVERHEAD_BYTES: number;
+/** Counted as needed in the memory check: what other processes may still take meanwhile. */
+export declare const SGLANG_FREE_MEMORY_MARGIN_BYTES: number;
+export declare const SGLANG_MIN_MEM_FRACTION_STATIC = 0.5;
+export declare const SGLANG_MAX_MEM_FRACTION_STATIC = 0.95;
+/**
+ * The KV cache the memory check counts: the KV token limit when set, else context × parallel
+ * requests, × the bytes per token over the model's attention layers — and whether the model's shape
+ * sized it. Without a shape: a quarter of the weights, at least 1 GiB.
+ */
+export declare function sglangKvCacheBytes(checkpoint: {
+    configJson: JsonObject;
+    hfQuantConfigJson: JsonObject | null;
+    weightBytesTotal?: number;
+}, settings: SglangSettings, gpu: GpuFacts): {
+    bytes: number;
+    basis: KvReserveBasis;
+};
+/**
+ * SGLang's memory rule on `gpu`: weights + KV cache estimate + its own overhead + a margin for what
+ * other processes may take meanwhile; on a card with the host's memory, whatever of `MemAvailable`
+ * lies beyond half the host's memory counts as needed too.
+ */
+export declare function sglangMemoryNeed(gpu: GpuFacts, checkpoint: CheckpointMemoryInputs, settings: SglangSettings, host: HostMemory): MemoryNeed;
+/** `--mem-fraction-static` from the card as it stands now (see the file banner). */
+export declare function sglangMemFractionStatic(settings: SglangSettings, gpu: GpuFacts, host: HostMemory): number;
+/** The launch plan `beforeCreate` hands the adapter, from the card re-probed right before the container. */
+export declare function sglangLaunchPlan(settings: SglangSettings, checkpoint: CheckpointMemoryInputs, gpu: GpuFacts, host: HostMemory): SglangLaunchPlan;
+/** The check skeleton's hooks for SGLang: its memory rule, and no checkpoint quirks of its own. */
+export declare function sglangCheckEngine(settings: SglangSettings): ManagedCheckEngine;

package/dist/runtime/sglang/memory.js

@@ -0,0 +1,90 @@
+import { checkpointIsMultimodal, freeMemoryBytes, isUnifiedMemory, kvCacheQuantAlgo, readKvCacheShape, supportsFp8KvCache, totalMemoryBytes, } from '../managed-models/index.js';
+const GiB = 1024 ** 3;
+const MiB = 1024 ** 2;
+/** SGLang's room for activations and CUDA graphs captured up to the parallel requests (design D4, D5). */
+export const SGLANG_RUNTIME_RESERVE_BYTES = 2 * GiB;
+/** The same room with CUDA graphs off. */
+export const SGLANG_RUNTIME_RESERVE_NO_GRAPHS_BYTES = 1.25 * GiB;
+/** What SGLang holds beyond weights and KV cache: CUDA context, activations, graphs. */
+export const SGLANG_ENGINE_OVERHEAD_BYTES = 2 * GiB;
+/** Counted as needed in the memory check: what other processes may still take meanwhile. */
+export const SGLANG_FREE_MEMORY_MARGIN_BYTES = 512 * MiB;
+export const SGLANG_MIN_MEM_FRACTION_STATIC = 0.5;
+export const SGLANG_MAX_MEM_FRACTION_STATIC = 0.95;
+/** Without the model's KV shape: a quarter of the weights, at least 1 GiB (vLLM's rule). */
+const FALLBACK_KV_WEIGHT_SHARE = 0.25;
+const FALLBACK_KV_MIN_BYTES = GiB;
+/**
+ * The KV cache the memory check counts: the KV token limit when set, else context × parallel
+ * requests, × the bytes per token over the model's attention layers — and whether the model's shape
+ * sized it. Without a shape: a quarter of the weights, at least 1 GiB.
+ */
+export function sglangKvCacheBytes(checkpoint, settings, gpu) {
+    const shape = readKvCacheShape(checkpoint.configJson);
+    if (shape === undefined) {
+        const weights = checkpoint.weightBytesTotal ?? 0;
+        return {
+            bytes: Math.max(FALLBACK_KV_MIN_BYTES, Math.ceil(weights * FALLBACK_KV_WEIGHT_SHARE)),
+            basis: 'weight_fraction',
+        };
+    }
+    const fp8 = (settings.kv_cache_dtype === 'fp8' && supportsFp8KvCache(gpu.compute_capability)) ||
+        kvCacheQuantAlgo(checkpoint.configJson, checkpoint.hfQuantConfigJson)?.toUpperCase() === 'FP8';
+    const perToken = 2 * shape.numHiddenLayers * shape.numKeyValueHeads * shape.headDim * (fp8 ? 1 : 2);
+    const tokens = settings.max_total_tokens ?? settings.context_length * settings.max_running_requests;
+    return { bytes: perToken * tokens, basis: 'config' };
+}
+/**
+ * SGLang's memory rule on `gpu`: weights + KV cache estimate + its own overhead + a margin for what
+ * other processes may take meanwhile; on a card with the host's memory, whatever of `MemAvailable`
+ * lies beyond half the host's memory counts as needed too.
+ */
+export function sglangMemoryNeed(gpu, checkpoint, settings, host) {
+    const unified = isUnifiedMemory(gpu);
+    const kv = sglangKvCacheBytes(checkpoint, settings, gpu);
+    const systemReserve = unified
+        ? Math.max(0, freeMemoryBytes(gpu, host) - totalMemoryBytes(gpu, host) / 2)
+        : 0;
+    const neededBytes = checkpoint.weightBytesTotal +
+        kv.bytes +
+        SGLANG_ENGINE_OVERHEAD_BYTES +
+        SGLANG_FREE_MEMORY_MARGIN_BYTES +
+        systemReserve;
+    return {
+        neededBytes,
+        kvReserveBasis: kv.basis,
+        details: `weight_bytes=${checkpoint.weightBytesTotal} kv_cache_bytes=${kv.bytes} kv_reserve_basis=${kv.basis} ` +
+            `engine_overhead_bytes=${SGLANG_ENGINE_OVERHEAD_BYTES} free_margin_bytes=${SGLANG_FREE_MEMORY_MARGIN_BYTES}` +
+            (unified ? ` system_reserve_bytes=${systemReserve}` : '') +
+            ` needed_bytes=${neededBytes}`,
+    };
+}
+/** `--mem-fraction-static` from the card as it stands now (see the file banner). */
+export function sglangMemFractionStatic(settings, gpu, host) {
+    const free = freeMemoryBytes(gpu, host);
+    if (free <= 0)
+        return SGLANG_MIN_MEM_FRACTION_STATIC;
+    const reserve = settings.cuda_graphs === 'off' ? SGLANG_RUNTIME_RESERVE_NO_GRAPHS_BYTES : SGLANG_RUNTIME_RESERVE_BYTES;
+    // On the host's memory, half of it stays the system's (the same rule as TensorRT-LLM's and vLLM's);
+    // SGLang still measures all of the available memory as free, so the share is of that.
+    const budget = isUnifiedMemory(gpu) ? Math.min(free, totalMemoryBytes(gpu, host) / 2) : free;
+    const share = (budget - reserve) / free;
+    const clamped = Math.min(SGLANG_MAX_MEM_FRACTION_STATIC, Math.max(SGLANG_MIN_MEM_FRACTION_STATIC, share));
+    return Math.floor(clamped * 10_000) / 10_000;
+}
+/** The launch plan `beforeCreate` hands the adapter, from the card re-probed right before the container. */
+export function sglangLaunchPlan(settings, checkpoint, gpu, host) {
+    return {
+        memFractionStatic: settings.mem_fraction_static ?? sglangMemFractionStatic(settings, gpu, host),
+        multimodal: checkpointIsMultimodal(checkpoint.configJson),
+    };
+}
+/** The check skeleton's hooks for SGLang: its memory rule, and no checkpoint quirks of its own. */
+export function sglangCheckEngine(settings) {
+    return {
+        engineId: 'sglang',
+        checkpointProblems: () => null,
+        memoryNeed: (gpu, checkpoint, host) => sglangMemoryNeed(gpu, checkpoint, settings, host),
+    };
+}
+//# sourceMappingURL=memory.js.map

package/dist/runtime/sglang/memory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory.js","sourceRoot":"","sources":["../../../src/runtime/sglang/memory.ts"],"names":[],"mappings":"AA0BA,OAAO,EACL,sBAAsB,EACtB,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,GAOjB,MAAM,4BAA4B,CAAA;AAInC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,CAAA;AACrB,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,CAAA;AAErB,0GAA0G;AAC1G,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,GAAG,GAAG,CAAA;AACnD,0CAA0C;AAC1C,MAAM,CAAC,MAAM,sCAAsC,GAAG,IAAI,GAAG,GAAG,CAAA;AAChE,wFAAwF;AACxF,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,GAAG,GAAG,CAAA;AACnD,4FAA4F;AAC5F,MAAM,CAAC,MAAM,+BAA+B,GAAG,GAAG,GAAG,GAAG,CAAA;AACxD,MAAM,CAAC,MAAM,8BAA8B,GAAG,GAAG,CAAA;AACjD,MAAM,CAAC,MAAM,8BAA8B,GAAG,IAAI,CAAA;AAClD,4FAA4F;AAC5F,MAAM,wBAAwB,GAAG,IAAI,CAAA;AACrC,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAuG,EACvG,QAAwB,EACxB,GAAa;IAEb,MAAM,KAAK,GAAG,gBAAgB,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;IACrD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,UAAU,CAAC,gBAAgB,IAAI,CAAC,CAAA;QAChD,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,wBAAwB,CAAC,CAAC;YACrF,KAAK,EAAE,iBAAiB;SACzB,CAAA;IACH,CAAC;IACD,MAAM,GAAG,GACP,CAAC,QAAQ,CAAC,cAAc,KAAK,KAAK,IAAI,kBAAkB,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;QACjF,gBAAgB,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,iBAAiB,CAAC,EAAE,WAAW,EAAE,KAAK,KAAK,CAAA;IAChG,MAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACnG,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC,cAAc,GAAG,QAAQ,CAAC,oBAAoB,CAAA;IACnG,OAAO,EAAE,KAAK,EAAE,QAAQ,GAAG,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAA;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,GAAa,EACb,UAAkC,EAClC,QAAwB,EACxB,IAAgB;IAEhB,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAA;IACpC,MAAM,EAAE,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAA;IACxD,MAAM,aAAa,GAAG,OAAO;QAC3B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC,CAAA;IACL,MAAM,WAAW,GACf,UAAU,CAAC,gBAAgB;QAC3B,EAAE,CAAC,KAAK;QACR,4BAA4B;QAC5B,+BAA+B;QAC/B,aAAa,CAAA;IACf,OAAO;QACL,WAAW;QACX,cAAc,EAAE,EAAE,CAAC,KAAK;QACxB,OAAO,EACL,gBAAgB,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,KAAK,qBAAqB,EAAE,CAAC,KAAK,GAAG;YACtG,yBAAyB,4BAA4B,sBAAsB,+BAA+B,EAAE;YAC5G,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,iBAAiB,WAAW,EAAE;KACjC,CAAA;AACH,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,uBAAuB,CAAC,QAAwB,EAAE,GAAa,EAAE,IAAgB;IAC/F,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IACvC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,8BAA8B,CAAA;IACpD,MAAM,OAAO,GACX,QAAQ,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,sCAAsC,CAAC,CAAC,CAAC,4BAA4B,CAAA;IACxG,oGAAoG;IACpG,sFAAsF;IACtF,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC5F,MAAM,KAAK,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,GAAG,IAAI,CAAA;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,IAAI,CAAC,GAAG,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC,CAAA;IACzG,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,MAAM,CAAA;AAC9C,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,gBAAgB,CAC9B,QAAwB,EACxB,UAAkC,EAClC,GAAa,EACb,IAAgB;IAEhB,OAAO;QACL,iBAAiB,EAAE,QAAQ,CAAC,mBAAmB,IAAI,uBAAuB,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;QAC/F,UAAU,EAAE,sBAAsB,CAAC,UAAU,CAAC,UAAU,CAAC;KAC1D,CAAA;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,iBAAiB,CAAC,QAAwB;IACxD,OAAO;QACL,QAAQ,EAAE,QAAQ;QAClB,kBAAkB,EAAE,GAAG,EAAE,CAAC,IAAI;QAC9B,UAAU,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,CAAC;KACzF,CAAA;AACH,CAAC"}

package/dist/runtime/sglang/settings.d.ts

@@ -0,0 +1,49 @@
+/**
+ * The `sglang` provider's settings (change `add-sglang-runtime`, task 2.1, design D6; spec
+ * `sglang-runtime`, "Выбор карты и настройки провайдера sglang"): stored values, overlaid by a load's
+ * own overrides, reduced to this provider's keys and validated before anything is asked of the
+ * machine — `INVALID_ARGUMENT` for anything the schema (`src/settings/schema/sglang.json`) would
+ * refuse. A value left at its default reaches the engine as nothing (SGLang's own), and what the
+ * engine is never told by a setting — `--trust-remote-code`, request logging, an API key — has no key
+ * here.
+ */
+import { type GenerationDefaults } from '../managed-text/generation-defaults.js';
+export interface SglangSettings {
+    /** `GPU-<uuid>`/`MIG-<uuid>`, or `null` to let the load pick the card with the most free memory. */
+    gpu_id: string | null;
+    context_length: number;
+    max_output_tokens: number;
+    /** `--max-running-requests`. */
+    max_running_requests: number;
+    /**
+     * `--mem-fraction-static`: the share of the card's free memory SGLang keeps for weights and KV
+     * cache; `null` (auto) lets core compute it from the card's free memory right before the container.
+     */
+    mem_fraction_static: number | null;
+    /** `--max-total-tokens`; `null` lets SGLang size the KV cache from its share. */
+    max_total_tokens: number | null;
+    /** `auto`: `--cuda-graph-max-bs <max_running_requests>`; `on`: SGLang's own; `off`: `--disable-cuda-graph`. */
+    cuda_graphs: 'auto' | 'on' | 'off';
+    kv_cache_dtype: 'auto' | 'fp8';
+    /** Seconds. `null` leaves the adapter's own weight-based estimate in force. */
+    load_timeout_seconds: number | null;
+    /** `false` passes `--disable-radix-cache`; SGLang's own default is on. */
+    enable_prefix_caching: boolean;
+    /** `--chunked-prefill-size`; `null` leaves SGLang's own. */
+    chunked_prefill_size: number | null;
+    /** `--dtype`; `auto` passes nothing. */
+    dtype: 'auto' | 'float16' | 'bfloat16' | 'float32';
+    /** `--random-seed`; `null` leaves SGLang's own. */
+    seed: number | null;
+    /** Sampling defaults for a request that sets none; a key is present only when the person set it. */
+    generation: GenerationDefaults;
+}
+/** Validates already-merged values; anything absent takes the schema's default. */
+export declare function validateSglangSettings(raw: Record<string, unknown>): SglangSettings;
+/**
+ * Stored values, overlaid by a load's own overrides (the control API's `overrides`), reduced to this
+ * provider's keys and validated. Anything else in either object — another provider's key, a stray
+ * field a client sent — is ignored rather than passed to the engine. `''` and `0` are the stored
+ * spelling of "not set" (`''` for a generation default, whose `0` can be a real value).
+ */
+export declare function sglangSettings(stored: Record<string, unknown>, overrides?: Record<string, unknown>): SglangSettings;

package/dist/runtime/sglang/settings.js

@@ -0,0 +1,118 @@
+/**
+ * The `sglang` provider's settings (change `add-sglang-runtime`, task 2.1, design D6; spec
+ * `sglang-runtime`, "Выбор карты и настройки провайдера sglang"): stored values, overlaid by a load's
+ * own overrides, reduced to this provider's keys and validated before anything is asked of the
+ * machine — `INVALID_ARGUMENT` for anything the schema (`src/settings/schema/sglang.json`) would
+ * refuse. A value left at its default reaches the engine as nothing (SGLang's own), and what the
+ * engine is never told by a setting — `--trust-remote-code`, request logging, an API key — has no key
+ * here.
+ */
+import { GENERATION_DEFAULT_KEYS, readGenerationDefaults, } from '../managed-text/generation-defaults.js';
+import { settingValues } from '../managed-text/setting-values.js';
+const KEYS = [
+    'gpu_id',
+    'context_length',
+    'max_output_tokens',
+    'max_running_requests',
+    'mem_fraction_static',
+    'max_total_tokens',
+    'cuda_graphs',
+    'kv_cache_dtype',
+    'load_timeout_seconds',
+    'enable_prefix_caching',
+    'chunked_prefill_size',
+    'dtype',
+    'seed',
+    ...Object.values(GENERATION_DEFAULT_KEYS),
+];
+const DEFAULTS = {
+    gpu_id: null,
+    context_length: 8192,
+    max_output_tokens: 4096,
+    // One request at a time, as every managed engine (owner's decision, change add-vllm-runtime).
+    max_running_requests: 1,
+    mem_fraction_static: null,
+    max_total_tokens: null,
+    cuda_graphs: 'auto',
+    kv_cache_dtype: 'auto',
+    load_timeout_seconds: null,
+    enable_prefix_caching: true,
+    chunked_prefill_size: null,
+    dtype: 'auto',
+    seed: null,
+    generation: {},
+};
+const GPU_UUID = /^(GPU|MIG)-[0-9A-Za-z-]+$/;
+const { invalid, integer, decimal, yesNo, oneOf } = settingValues('sglang');
+/** Validates already-merged values; anything absent takes the schema's default. */
+export function validateSglangSettings(raw) {
+    const settings = { ...DEFAULTS };
+    if (raw['gpu_id'] !== undefined && raw['gpu_id'] !== null) {
+        if (typeof raw['gpu_id'] !== 'string' || !GPU_UUID.test(raw['gpu_id'])) {
+            invalid('gpu_id', raw['gpu_id'], 'must be an NVIDIA GPU or MIG UUID');
+        }
+        settings.gpu_id = raw['gpu_id'];
+    }
+    if (raw['context_length'] !== undefined)
+        settings.context_length = integer('context_length', raw['context_length'], 512, 1_048_576);
+    if (raw['max_output_tokens'] !== undefined)
+        settings.max_output_tokens = integer('max_output_tokens', raw['max_output_tokens'], 1, 1_048_576);
+    if (raw['max_running_requests'] !== undefined)
+        settings.max_running_requests = integer('max_running_requests', raw['max_running_requests'], 1, 256);
+    // Below 1: the rest of SGLang's free memory is its room for activations and CUDA graphs.
+    if (raw['mem_fraction_static'] !== undefined && raw['mem_fraction_static'] !== null)
+        settings.mem_fraction_static = decimal('mem_fraction_static', raw['mem_fraction_static'], (n) => n > 0 && n < 1, 'above 0 and below 1');
+    if (raw['max_total_tokens'] !== undefined && raw['max_total_tokens'] !== null)
+        settings.max_total_tokens = integer('max_total_tokens', raw['max_total_tokens'], 1, 16_777_216);
+    if (raw['cuda_graphs'] !== undefined)
+        settings.cuda_graphs = oneOf('cuda_graphs', raw['cuda_graphs'], ['auto', 'on', 'off']);
+    if (raw['kv_cache_dtype'] !== undefined)
+        settings.kv_cache_dtype = oneOf('kv_cache_dtype', raw['kv_cache_dtype'], ['auto', 'fp8']);
+    if (raw['load_timeout_seconds'] !== undefined && raw['load_timeout_seconds'] !== null)
+        settings.load_timeout_seconds = integer('load_timeout_seconds', raw['load_timeout_seconds'], 1, 3600);
+    if (raw['enable_prefix_caching'] !== undefined)
+        settings.enable_prefix_caching = yesNo('enable_prefix_caching', raw['enable_prefix_caching']);
+    if (raw['chunked_prefill_size'] !== undefined && raw['chunked_prefill_size'] !== null)
+        settings.chunked_prefill_size = integer('chunked_prefill_size', raw['chunked_prefill_size'], 1, 1_048_576);
+    if (raw['dtype'] !== undefined)
+        settings.dtype = oneOf('dtype', raw['dtype'], ['auto', 'float16', 'bfloat16', 'float32']);
+    if (raw['seed'] !== undefined && raw['seed'] !== null)
+        settings.seed = integer('seed', raw['seed'], 1, 2_147_483_647);
+    settings.generation = readGenerationDefaults('sglang', raw);
+    if (settings.max_output_tokens >= settings.context_length) {
+        invalid('max_output_tokens', settings.max_output_tokens, `must be less than the context length (${settings.context_length})`);
+    }
+    return settings;
+}
+/**
+ * Stored values, overlaid by a load's own overrides (the control API's `overrides`), reduced to this
+ * provider's keys and validated. Anything else in either object — another provider's key, a stray
+ * field a client sent — is ignored rather than passed to the engine. `''` and `0` are the stored
+ * spelling of "not set" (`''` for a generation default, whose `0` can be a real value).
+ */
+export function sglangSettings(stored, overrides = {}) {
+    const merged = {};
+    for (const key of KEYS) {
+        const value = key in overrides ? overrides[key] : stored[key];
+        if (value !== undefined)
+            merged[key] = value;
+    }
+    if (typeof merged['gpu_id'] === 'string' && merged['gpu_id'].trim() === '')
+        merged['gpu_id'] = null;
+    for (const key of [
+        'load_timeout_seconds',
+        'mem_fraction_static',
+        'max_total_tokens',
+        'chunked_prefill_size',
+        'seed',
+    ]) {
+        if (merged[key] === 0)
+            merged[key] = null;
+    }
+    for (const key of Object.values(GENERATION_DEFAULT_KEYS)) {
+        if (typeof merged[key] === 'string' && merged[key].trim() === '')
+            merged[key] = null;
+    }
+    return validateSglangSettings(merged);
+}
+//# sourceMappingURL=settings.js.map

package/dist/runtime/sglang/settings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settings.js","sourceRoot":"","sources":["../../../src/runtime/sglang/settings.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EACL,uBAAuB,EACvB,sBAAsB,GAEvB,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,mCAAmC,CAAA;AAiCjE,MAAM,IAAI,GAAG;IACX,QAAQ;IACR,gBAAgB;IAChB,mBAAmB;IACnB,sBAAsB;IACtB,qBAAqB;IACrB,kBAAkB;IAClB,aAAa;IACb,gBAAgB;IAChB,sBAAsB;IACtB,uBAAuB;IACvB,sBAAsB;IACtB,OAAO;IACP,MAAM;IACN,GAAG,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC;CACjC,CAAA;AAEV,MAAM,QAAQ,GAAmB;IAC/B,MAAM,EAAE,IAAI;IACZ,cAAc,EAAE,IAAI;IACpB,iBAAiB,EAAE,IAAI;IACvB,8FAA8F;IAC9F,oBAAoB,EAAE,CAAC;IACvB,mBAAmB,EAAE,IAAI;IACzB,gBAAgB,EAAE,IAAI;IACtB,WAAW,EAAE,MAAM;IACnB,cAAc,EAAE,MAAM;IACtB,oBAAoB,EAAE,IAAI;IAC1B,qBAAqB,EAAE,IAAI;IAC3B,oBAAoB,EAAE,IAAI;IAC1B,KAAK,EAAE,MAAM;IACb,IAAI,EAAE,IAAI;IACV,UAAU,EAAE,EAAE;CACf,CAAA;AAED,MAAM,QAAQ,GAAG,2BAA2B,CAAA;AAE5C,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;AAE3E,mFAAmF;AACnF,MAAM,UAAU,sBAAsB,CAAC,GAA4B;IACjE,MAAM,QAAQ,GAAmB,EAAE,GAAG,QAAQ,EAAE,CAAA;IAChD,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,CAAC,QAAQ,CAAC,KAAK,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACvE,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,EAAE,mCAAmC,CAAC,CAAA;QACvE,CAAC;QACD,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAW,CAAA;IAC3C,CAAC;IACD,IAAI,GAAG,CAAC,gBAAgB,CAAC,KAAK,SAAS;QACrC,QAAQ,CAAC,cAAc,GAAG,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,CAAA;IAC5F,IAAI,GAAG,CAAC,mBAAmB,CAAC,KAAK,SAAS;QACxC,QAAQ,CAAC,iBAAiB,GAAG,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,mBAAmB,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IACnG,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,SAAS;QAC3C,QAAQ,CAAC,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAA;IACtG,yFAAyF;IACzF,IAAI,GAAG,CAAC,qBAAqB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,qBAAqB,CAAC,KAAK,IAAI;QACjF,QAAQ,CAAC,mBAAmB,GAAG,OAAO,CACpC,qBAAqB,EACrB,GAAG,CAAC,qBAAqB,CAAC,EAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EACrB,qBAAqB,CACtB,CAAA;IACH,IAAI,GAAG,CAAC,kBAAkB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,kBAAkB,CAAC,KAAK,IAAI;QAC3E,QAAQ,CAAC,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,kBAAkB,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAA;IACjG,IAAI,GAAG,CAAC,aAAa,CAAC,KAAK,SAAS;QAClC,QAAQ,CAAC,WAAW,GAAG,KAAK,CAAC,aAAa,EAAE,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IACxF,IAAI,GAAG,CAAC,gBAAgB,CAAC,KAAK,SAAS;QACrC,QAAQ,CAAC,cAAc,GAAG,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAA;IAC3F,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,IAAI;QACnF,QAAQ,CAAC,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAA;IACvG,IAAI,GAAG,CAAC,uBAAuB,CAAC,KAAK,SAAS;QAC5C,QAAQ,CAAC,qBAAqB,GAAG,KAAK,CAAC,uBAAuB,EAAE,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAA;IAC/F,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,sBAAsB,CAAC,KAAK,IAAI;QACnF,QAAQ,CAAC,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,EAAE,GAAG,CAAC,sBAAsB,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAA;IAC5G,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,SAAS;QAC5B,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC,CAAA;IAC3F,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI;QACnD,QAAQ,CAAC,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;IAChE,QAAQ,CAAC,UAAU,GAAG,sBAAsB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAC3D,IAAI,QAAQ,CAAC,iBAAiB,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC1D,OAAO,CACL,mBAAmB,EACnB,QAAQ,CAAC,iBAAiB,EAC1B,yCAAyC,QAAQ,CAAC,cAAc,GAAG,CACpE,CAAA;IACH,CAAC;IACD,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,MAA+B,EAC/B,YAAqC,EAAE;IAEvC,MAAM,MAAM,GAA4B,EAAE,CAAA;IAC1C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7D,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;IAC9C,CAAC;IACD,IAAI,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAA;IACnG,KAAK,MAAM,GAAG,IAAI;QAChB,sBAAsB;QACtB,qBAAqB;QACrB,kBAAkB;QAClB,sBAAsB;QACtB,MAAM;KACP,EAAE,CAAC;QACF,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAA;IAC3C,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,uBAAuB,CAAC,EAAE,CAAC;QACzD,IAAI,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAA;IACtF,CAAC;IACD,OAAO,sBAAsB,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC"}

package/dist/runtime/shared/external-sessions.js

@@ -25,6 +25,7 @@ const PROVIDERS = new Set([
     'foundation-models',
     'tensorrt-llm',
     'vllm',
+    'sglang',
 ]);
 export class ExternalSessions {
     options;

package/dist/runtime/shared/external-sessions.js.map

@@ -1 +1 @@
-{"version":3,"file":"external-sessions.js","sourceRoot":"","sources":["../../../src/runtime/shared/external-sessions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAG1D,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAC7C,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAuB7C,MAAM,SAAS,GAAwB,IAAI,GAAG,CAAC;IAC7C,UAAU;IACV,mBAAmB;IACnB,cAAc;IACd,KAAK;IACL,mBAAmB;IACnB,cAAc;IACd,MAAM;CACP,CAAC,CAAA;AASF,MAAM,OAAO,gBAAgB;IAOE;IANZ,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAA;IACxC,UAAU,GAAG,IAAI,GAAG,EAGlC,CAAA;IAEH,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;IAAG,CAAC;IAEzD,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;IACzC,CAAC;IAEO,IAAI;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChD,IAAI,YAAY,CAAC,SAAS,IAAI,GAAG;gBAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAChE,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAA;IACzB,CAAC;IAEO,IAAI,CAAC,KAAa,EAAE,MAAkC;QAC5D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,OAAM;QACtC,2DAA2D;QAC3D,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBAC1B,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;IAChF,CAAC;IAED,sGAAsG;IACtG,OAAO,CAAC,KAAa,EAAE,UAAkB,EAAE,QAAiB;QAC1D,IAAI,CAAC,KAAK;YACR,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,iDAAiD,CAAC,CAAA;QAClG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;YACrD,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,2CAA2C,CAAC,CAAA;QAC5F,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,OAAO,IAAI,UAAU,GAAG,OAAO,CAAC,UAAU;YAC5C,MAAM,IAAI,eAAe,CACvB,sBAAsB,EACtB,0BAA0B,KAAK,gBAAgB,UAAU,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAChG,CAAA;QACH,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE;YACrB,UAAU;YACV,QAAQ,EAAE,MAAM;YAChB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,uBAAuB,CAAC;SACxE,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;QACvG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;IAChD,CAAC;IAED,8FAA8F;IAC9F,SAAS,CAAC,KAAa,EAAE,UAAkB;QACzC,IAAI,CAAC,IAAI,EAAE,CAAA;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,KAAK,UAAU;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC1E,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,uBAAuB,CAAC,CAAA;QAChF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IACxB,CAAC;IAED,UAAU,CAAC,KAAa,EAAE,UAAmB;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC;YAAE,OAAO,KAAK,CAAA;QAC7F,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAA;QAChC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,qDAAqD;IACrD,IAAI;QACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5G,CAAC;IAED,IAAI,CACF,QAAgB,EAChB,OAAqC;QAErC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAChF,CAAC;IAED,yFAAyF;IACzF,kBAAkB,CAChB,KAAa,EACb,QAAgB,EAChB,OAAe,EACf,OAAe;QAEf,MAAM,SAAS,GAAG,UAAU,EAAE,CAAA;QAC9B,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,EAAE;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC;oBAAE,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAClF,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,uBAAuB,CAAC,CAAA;YACxD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;YACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC7B,KAAK;gBACL,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;oBACnB,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,OAAO,CAAC,OAAO,CAAC,CAAA;gBAClB,CAAC;aACF,CAAC,CAAA;YACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,iCAAiC,EAAE;gBACnD,UAAU,EAAE,SAAS;gBACrB,KAAK;gBACL,QAAQ;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO;aACR,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,gGAAgG;IAChG,iBAAiB,CAAC,KAAa,EAAE,SAAiB,EAAE,OAAgB;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC9C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO,KAAK,CAAA;QACrD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACjC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAA4B,CAAA;QACpD,OAAO,CAAC,OAAO,CAAC;YACd,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI;YACpB,GAAG,CAAC,OAAO,CAAC,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,yBAAyB,CAAC,CAAA;IACjG,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAA;QAClD,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QACtB,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC1D,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,oCAAoC,CAAC,CAAA;QACtG,IAAI,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE;YAC3D,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,uBAAuB,CAAC,CAAA;QACzF,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,KAAK;YAClF,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,sBAAsB,CAAC,CAAA;QACxF,OAAO;YACL,QAAQ,EAAE,QAA2B;YACrC,QAAQ,EAAE,CAAC,CAAC,UAAU,CAAC;YACvB,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;YAC7D,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC,KAAK,IAAI;YACxC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;SACpD,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
+{"version":3,"file":"external-sessions.js","sourceRoot":"","sources":["../../../src/runtime/shared/external-sessions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAG1D,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAC7C,MAAM,CAAC,MAAM,uBAAuB,GAAG,MAAM,CAAA;AAuB7C,MAAM,SAAS,GAAwB,IAAI,GAAG,CAAC;IAC7C,UAAU;IACV,mBAAmB;IACnB,cAAc;IACd,KAAK;IACL,mBAAmB;IACnB,cAAc;IACd,MAAM;IACN,QAAQ;CACT,CAAC,CAAA;AASF,MAAM,OAAO,gBAAgB;IAOE;IANZ,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAA;IACxC,UAAU,GAAG,IAAI,GAAG,EAGlC,CAAA;IAEH,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;IAAG,CAAC;IAEzD,GAAG;QACT,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;IACzC,CAAC;IAEO,IAAI;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,KAAK,MAAM,CAAC,KAAK,EAAE,YAAY,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChD,IAAI,YAAY,CAAC,SAAS,IAAI,GAAG;gBAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAA;QAChE,CAAC;QACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAA;IACzB,CAAC;IAEO,IAAI,CAAC,KAAa,EAAE,MAAkC;QAC5D,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,OAAM;QACtC,2DAA2D;QAC3D,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBAC1B,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAA;YACtD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;IAChF,CAAC;IAED,sGAAsG;IACtG,OAAO,CAAC,KAAa,EAAE,UAAkB,EAAE,QAAiB;QAC1D,IAAI,CAAC,KAAK;YACR,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,iDAAiD,CAAC,CAAA;QAClG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;YACrD,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,2CAA2C,CAAC,CAAA;QAC5F,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,OAAO,IAAI,UAAU,GAAG,OAAO,CAAC,UAAU;YAC5C,MAAM,IAAI,eAAe,CACvB,sBAAsB,EACtB,0BAA0B,KAAK,gBAAgB,UAAU,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAChG,CAAA;QACH,MAAM,MAAM,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAA;QACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE;YACrB,UAAU;YACV,QAAQ,EAAE,MAAM;YAChB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,uBAAuB,CAAC;SACxE,CAAC,CAAA;QACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;QACvG,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAA;IAChD,CAAC;IAED,8FAA8F;IAC9F,SAAS,CAAC,KAAa,EAAE,UAAkB;QACzC,IAAI,CAAC,IAAI,EAAE,CAAA;QACX,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,KAAK,UAAU;YAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC1E,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,uBAAuB,CAAC,CAAA;QAChF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;IACxB,CAAC;IAED,UAAU,CAAC,KAAa,EAAE,UAAmB;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,OAAO,CAAC,UAAU,KAAK,UAAU,CAAC;YAAE,OAAO,KAAK,CAAA;QAC7F,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,cAAc,CAAC,CAAA;QAChC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,qDAAqD;IACrD,IAAI;QACF,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5G,CAAC;IAED,IAAI,CACF,QAAgB,EAChB,OAAqC;QAErC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;IAChF,CAAC;IAED,yFAAyF;IACzF,kBAAkB,CAChB,KAAa,EACb,QAAgB,EAChB,OAAe,EACf,OAAe;QAEf,MAAM,SAAS,GAAG,UAAU,EAAE,CAAA;QAC9B,OAAO,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,EAAE;YACjD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC;oBAAE,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;YAClF,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,uBAAuB,CAAC,CAAA;YACxD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAA;YACf,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE;gBAC7B,KAAK;gBACL,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;oBACnB,YAAY,CAAC,KAAK,CAAC,CAAA;oBACnB,OAAO,CAAC,OAAO,CAAC,CAAA;gBAClB,CAAC;aACF,CAAC,CAAA;YACF,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,iCAAiC,EAAE;gBACnD,UAAU,EAAE,SAAS;gBACrB,KAAK;gBACL,QAAQ;gBACR,QAAQ,EAAE,OAAO;gBACjB,OAAO;aACR,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;IACJ,CAAC;IAED,gGAAgG;IAChG,iBAAiB,CAAC,KAAa,EAAE,SAAiB,EAAE,OAAgB;QAClE,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QAC9C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,KAAK;YAAE,OAAO,KAAK,CAAA;QACrD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QACjC,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAA4B,CAAA;QACpD,OAAO,CAAC,OAAO,CAAC;YACd,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI;YACpB,GAAG,CAAC,OAAO,CAAC,CAAC,aAAa,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpE,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAED,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,yBAAyB,CAAC,CAAA;IACjG,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAA4B,CAAA;QAClD,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,CAAA;QAC9B,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QACtB,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC1D,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,oCAAoC,CAAC,CAAA;QACtG,IAAI,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE;YAC3D,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,uBAAuB,CAAC,CAAA;QACzF,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,GAAG,KAAK;YAClF,MAAM,IAAI,eAAe,CAAC,kBAAkB,EAAE,YAAY,KAAK,sBAAsB,CAAC,CAAA;QACxF,OAAO;YACL,QAAQ,EAAE,QAA2B;YACrC,QAAQ,EAAE,CAAC,CAAC,UAAU,CAAC;YACvB,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE;YAC7D,YAAY,EAAE,CAAC,CAAC,cAAc,CAAC,KAAK,IAAI;YACxC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;SACpD,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/dist/runtime/shared/ports.d.ts

@@ -23,6 +23,16 @@ export interface PortOptions {
 }
 /** A random port in [3000, 4000) that no session uses and that binds. */
 export declare function randomFreePort(usedPorts: Iterable<number>, opts?: PortOptions): Promise<number>;
+/** How many ports a start tries when each one is taken before the engine can bind it. */
+export declare const PORT_TAKEN_ATTEMPTS = 3;
+/** Whether an engine that exited while starting did so because its port was already taken. */
+export declare function exitedOnTakenPort(details: string | undefined): boolean;
+/**
+ * `start` again, on a port it picks anew, while it fails because its port was taken (the probe is
+ * racy by nature: another process can bind the port between the probe and the engine's own bind).
+ * `portTaken` reads the failure; anything else, or the last attempt's failure, is thrown as it is.
+ */
+export declare function retryOnTakenPort<T>(start: () => Promise<T>, portTaken: (error: unknown) => boolean, onRetry?: (attempt: number) => void, attempts?: number): Promise<T>;
 /** Secret the extension has always used for session keys (`index.ts:531`). */
 export declare const DEFAULT_API_SECRET = "JustAskNow";
 /**

package/dist/runtime/shared/ports.js

@@ -44,6 +44,36 @@ export async function randomFreePort(usedPorts, opts = {}) {
     }
     throw new Error(PORT_EXHAUSTED_MESSAGE);
 }
+/**
+ * What `llama-server` prints when its port was taken between `randomFreePort`'s probe and its own
+ * bind ("couldn't bind HTTP server socket, hostname: …, port: …"), and what a Node server says.
+ */
+const PORT_TAKEN = /couldn't bind HTTP server socket|address already in use|EADDRINUSE/i;
+/** How many ports a start tries when each one is taken before the engine can bind it. */
+export const PORT_TAKEN_ATTEMPTS = 3;
+/** Whether an engine that exited while starting did so because its port was already taken. */
+export function exitedOnTakenPort(details) {
+    return details !== undefined && PORT_TAKEN.test(details);
+}
+/**
+ * `start` again, on a port it picks anew, while it fails because its port was taken (the probe is
+ * racy by nature: another process can bind the port between the probe and the engine's own bind).
+ * `portTaken` reads the failure; anything else, or the last attempt's failure, is thrown as it is.
+ */
+export async function retryOnTakenPort(start, portTaken, onRetry = () => { }, attempts = PORT_TAKEN_ATTEMPTS) {
+    const attempt = async (n) => {
+        try {
+            return await start();
+        }
+        catch (error) {
+            if (n >= attempts || !portTaken(error))
+                throw error;
+            onRetry(n);
+            return attempt(n + 1);
+        }
+    };
+    return attempt(1);
+}
 /** Secret the extension has always used for session keys (`index.ts:531`). */
 export const DEFAULT_API_SECRET = 'JustAskNow';
 /**

package/dist/runtime/shared/ports.js.map

@@ -1 +1 @@
-{"version":3,"file":"ports.js","sourceRoot":"","sources":["../../../src/runtime/shared/ports.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAEvC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA;AAClC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA,CAAC,YAAY;AAC/C,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAA;AACtC,MAAM,CAAC,MAAM,sBAAsB,GAAG,wDAAwD,CAAA;AAE9F;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CAAC;IAC9D,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IACvG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IACvG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IACtG,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;CAC1F,CAAC,CAAA;AAEF,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3E,CAAC,CAAC,CAAA;AACJ,CAAC;AAUD,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAA2B,EAAE,OAAoB,EAAE;IACtF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,cAAc,CAAA;IACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,cAAc,CAAA;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAA;IACnD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,eAAe,CAAA;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7D,IAAI,MAAM,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IAC1C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;AACzC,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,MAAM,kBAAkB,GAAG,YAAY,CAAA;AAE9C;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAAqB,EACrB,SAAiB,kBAAkB;IAEnC,OAAO,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;AAClF,CAAC"}
+{"version":3,"file":"ports.js","sourceRoot":"","sources":["../../../src/runtime/shared/ports.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,aAAa,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAEvC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA;AAClC,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,CAAA,CAAC,YAAY;AAC/C,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,CAAA;AACtC,MAAM,CAAC,MAAM,sBAAsB,GAAG,wDAAwD,CAAA;AAE9F;;;;GAIG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAwB,IAAI,GAAG,CAAC;IAC9D,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IACvG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IACvG,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;IACtG,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;CAC1F,CAAC,CAAA;AAEF,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QAC1C,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3E,CAAC,CAAC,CAAA;AACJ,CAAC;AAUD,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,SAA2B,EAAE,OAAoB,EAAE;IACtF,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAA;IAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,cAAc,CAAA;IACtC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,cAAc,CAAA;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAA;IACnD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAA;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,eAAe,CAAA;IACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QAC7B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7D,IAAI,MAAM,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;IAC1C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAA;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,GAAG,qEAAqE,CAAA;AAExF,yFAAyF;AACzF,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AAEpC,8FAA8F;AAC9F,MAAM,UAAU,iBAAiB,CAAC,OAA2B;IAC3D,OAAO,OAAO,KAAK,SAAS,IAAI,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AAC1D,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAAuB,EACvB,SAAsC,EACtC,UAAqC,GAAG,EAAE,GAAE,CAAC,EAC7C,QAAQ,GAAG,mBAAmB;IAE9B,MAAM,OAAO,GAAG,KAAK,EAAE,CAAS,EAAc,EAAE;QAC9C,IAAI,CAAC;YACH,OAAO,MAAM,KAAK,EAAE,CAAA;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAA;YACnD,OAAO,CAAC,CAAC,CAAC,CAAA;YACV,OAAO,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACvB,CAAC;IACH,CAAC,CAAA;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,CAAA;AACnB,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,MAAM,kBAAkB,GAAG,YAAY,CAAA;AAE9C;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,IAAqB,EACrB,SAAiB,kBAAkB;IAEnC,OAAO,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;AAClF,CAAC"}

package/dist/runtime/tensorrt-llm/adapter.d.ts

@@ -66,10 +66,7 @@ export declare const TENSORRT_LLM_CONTAINER_PORT = 8000;
  */
 export declare const TENSORRT_LLM_API_OPTIONS_FILE = "llm-api-options.yaml";
 export declare const TENSORRT_LLM_GUIDED_DECODING_OPTIONS = "guided_decoding_backend: xgrammar\n";
-/** Below this much card memory `cuda_graphs: auto` leaves CUDA graphs off: they took about 2 GB on
- *  an 8 GB card in the Windows live acceptance (Qwen3.5-2B, "Memory used outside torch … 2.12 GiB").
- *  A card that reports no size of its own is a unified-memory one (GB10): it keeps them. */
-export declare const TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES: number;
+export { TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES } from './cuda-graphs.js';
 /**
  * `--max_batch_size`: how many sequences the engine serves at once. `trtllm-serve` defaults to 2048, a
  * server's number; a desktop runs a chat, an agent turn and a few API calls at most. For a model with

package/dist/runtime/tensorrt-llm/adapter.js

@@ -121,7 +121,8 @@
 import { AtomicCoreError } from '../../contracts/index.js';
 import { readGenerationDefaults, withGenerationDefaults, } from '../managed-text/generation-defaults.js';
 import { MANAGED_TEXT_ADAPTER_CONTRACT_VERSION, ManagedRequestRefusal, asksForStructuredOutput, asksForTools, reasoningIntoContent, thinkingRequested, } from '../managed-text/index.js';
-import { TENSORRT_LLM_DEFAULT_KV_CACHE_FREE_FRACTION, tensorrtLlmUnifiedKvMaxTokens } from './kv-cache.js';
+import { tensorrtLlmCudaGraphsOn } from './cuda-graphs.js';
+import { TENSORRT_LLM_DEFAULT_KV_CACHE_FREE_FRACTION, tensorrtLlmKvTokens, tensorrtLlmUnifiedKvMaxTokens, tensorrtLlmWholeBlockTokens, } from './kv-cache.js';
 const GiB = 1024 ** 3;
 // ---------------------------------------------------------------------------------------------
 // Settings
@@ -291,10 +292,7 @@ function llmApiOptions(guided, kvMaxTokens, kvFp8 = false, cudaGraphsOff = false
         lines.push(`dtype: ${settings.dtype}`);
     return lines.length === 0 ? null : `${lines.join('\n')}\n`;
 }
-/** Below this much card memory `cuda_graphs: auto` leaves CUDA graphs off: they took about 2 GB on
- *  an 8 GB card in the Windows live acceptance (Qwen3.5-2B, "Memory used outside torch … 2.12 GiB").
- *  A card that reports no size of its own is a unified-memory one (GB10): it keeps them. */
-export const TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES = 12 * 1024 ** 3;
+export { TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES } from './cuda-graphs.js';
 /** FP8 KV cache needs compute capability 8.9 (Ada) or newer. */
 function supportsFp8Kv(computeCapability) {
     if (computeCapability === null || computeCapability === undefined)
@@ -320,6 +318,14 @@ export const TENSORRT_LLM_MAX_KV_CACHE_MAX_TOKENS = 16_777_216;
 /** The container binds every interface; only the host-side publication (design D1/D11) is
  *  loopback-restricted, by the executor, not by the engine's own bind address. */
 const CONTAINER_BIND_HOST = '0.0.0.0';
+/** `ManagedLaunchContext.plan` as this engine's own plan, or undefined when there is none. */
+function launchPlanOf(plan) {
+    return typeof plan === 'object' &&
+        plan !== null &&
+        typeof plan.cudaGraphs === 'boolean'
+        ? plan
+        : undefined;
+}
 /** Builds `trtllm-serve serve <model-dir> ...` (task 2.13; flags sourced from `serve.py`, see the
  *  file header). `serve` is named explicitly rather than relying on the CLI's default-command
  *  resolution (`DefaultGroup.resolve_command`, `commands/serve.py`) — that quirk keeps `trtllm-serve
@@ -369,14 +375,18 @@ export function buildTensorrtLlmLaunch(context) {
     // Windows live acceptance with Qwen3.5-2B). Room for every sequence the batch admits at full context;
     // the engine takes the smaller of this and the free-memory fraction, so attention-only models keep
     // the memory they had. A unified-memory card keeps its own, tighter bound.
-    const vram = context.gpuTotalVramBytes ?? null;
-    const cudaGraphsOff = settings.cuda_graphs === 'off' ||
-        (settings.cuda_graphs === 'auto' && vram !== null && vram < TENSORRT_LLM_CUDA_GRAPHS_MIN_VRAM_BYTES);
+    // CUDA graphs: the plan `beforeCreate` made on the card as it stands now (`tensorrtLlmLaunchPlan`:
+    // `auto` keeps them only where they fit beside the model); without one, the card's size alone.
+    const cudaGraphsOff = !(launchPlanOf(context.plan)?.cudaGraphs ??
+        tensorrtLlmCudaGraphsOn(settings.cuda_graphs, context.gpuTotalVramBytes ?? null));
     const kvFp8 = settings.kv_cache_dtype === 'fp8' && supportsFp8Kv(context.gpuComputeCapability);
-    const options = llmApiOptions(family?.structured_output === true, settings.kv_cache_max_tokens ??
-        (context.unifiedMemory
+    // In whole blocks: the engine rounds `max_tokens` down to them, and a sequence one block short of
+    // the context is a load the readiness check refuses (`kv-cache.ts`).
+    const options = llmApiOptions(family?.structured_output === true, settings.kv_cache_max_tokens !== null
+        ? tensorrtLlmWholeBlockTokens(settings.kv_cache_max_tokens)
+        : context.unifiedMemory
             ? tensorrtLlmUnifiedKvMaxTokens(settings.context_length)
-            : settings.context_length * settings.max_batch_size), kvFp8, cudaGraphsOff, settings);
+            : tensorrtLlmKvTokens(settings.context_length, settings.max_batch_size), kvFp8, cudaGraphsOff, settings);
     if (options !== null) {
         argv.push('--extra_llm_api_options', `${generationFilesPath}/${TENSORRT_LLM_API_OPTIONS_FILE}`);
     }
@@ -977,6 +987,13 @@ export const tensorrtLlmAdapter = {
     id: 'tensorrt-llm',
     contractVersion: MANAGED_TEXT_ADAPTER_CONTRACT_VERSION,
     readiness: { path: '/health', expectedStatus: 200 },
+    // `/health` answers before anything has been generated, and a hybrid model's cache manager logs no
+    // `max sequence length=` line for `classifyReady` to read: one token proves the engine can serve.
+    generationProbe: {
+        modelsPath: '/v1/models',
+        path: '/v1/chat/completions',
+        body: { messages: [{ role: 'user', content: 'Hi' }], max_tokens: 1, stream: false },
+    },
     routes: TENSORRT_LLM_ROUTES,
     rewritableRoutes: TENSORRT_LLM_REWRITABLE_ROUTES,
     stageMarkers: STAGE_MARKERS,