@bendyline/gezel-inference 1.0.7 → 1.0.9

package/dist/providers/ds4/residency.d.ts

@@ -4,6 +4,10 @@ export { DS4_FULL_RESIDENCY_HEADROOM_BYTES };
  * Conservative floor for the full-residency working set. Exclusivity is an
  * explicit device lease at process startup, not a consequence of this number:
  * the automatic model budget can exceed 96 GiB on larger machines.
+ *
+ * Only for a launch nobody measured: an entry with `fullResidencyMemory` is
+ * reserved at that working set ({@link planDs4MeasuredLaunch}). Flooring a
+ * 45 GiB Qwen3.8 launch here put it over a 96 GiB Mac's 80 GiB budget.
  */
 export declare const DS4_FULL_RESIDENCY_RESERVATION_BYTES: number;
 /**
@@ -73,12 +77,27 @@ export interface Ds4ExpertCachePlanOptions {
     catalogCacheBytes?: number;
     catalogResidentBytes?: number;
     totalRamBytes?: number;
+    /**
+     * A measured streamed footprint at the launch window, sidecars included
+     * ({@link ds4StreamingFixedBytes}). With {@link budgetBytes} it replaces the
+     * catalog subtraction and the generic 24 GiB headroom.
+     */
+    measuredFixedBytes?: number;
+    /** This device's local-model budget, the ceiling the capacity broker admits under. */
+    budgetBytes?: number;
+    /** Smallest cache the entry says is worth streaming with. */
+    minCacheBytes?: number;
 }
 export interface Ds4ExpertCachePlan {
     cacheGb: number;
     requestedGb: number;
     clamped: boolean;
     safe: boolean;
+    /**
+     * What bounded the cache: the device's local-model budget beside a measured
+     * footprint, or the generic {@link DS4_FULL_RESIDENCY_HEADROOM_BYTES}.
+     */
+    limit: 'model-budget' | 'headroom';
 }
 /**
  * Pick a bounded routed-expert cache for streaming mode. Catalog guidance is
@@ -86,8 +105,66 @@ export interface Ds4ExpertCachePlan {
  * the old RAM-tier heuristic which incorrectly gave both models 64 GiB on a
  * 128 GiB Mac. Manual values are still accepted, but clamped so fixed model
  * state + the cache cannot consume the runtime/OS reserve.
+ *
+ * The 24 GiB headroom guards a footprint guessed as `residentBytes −
+ * cacheExpertsBytes`, which omits graph and staging memory ds4 also holds.
+ * An entry with a measured streamed footprint has no such gap, so its cache
+ * may use whatever this device's local-model budget leaves beside it. A fixed
+ * 24 GiB refused Qwen3.8 on 32 GiB Macs whose budget held it with room to spare.
  */
 export declare function planDs4ExpertCache(opts: Ds4ExpertCachePlanOptions): Ds4ExpertCachePlan;
+/** A measured ds4 memory line from the catalog: bytes at a window, plus a per-token slope. */
+export interface Ds4MeasuredLine {
+    bytes: number;
+    ctxTokens: number;
+    bytesPerToken: number;
+}
+/** Evaluate a measured line at a window, never below its context-free part. */
+export declare function ds4MeasuredBytesAt(line: Ds4MeasuredLine, ctxTokens: number): number;
+/** The memory fields of a catalog `ds4` block that price a launch. */
+export interface Ds4MemorySource {
+    cacheExpertsBytes?: number | undefined;
+    fullResidencyMemory?: {
+        workingSetBytes: number;
+        ctxTokens: number;
+        bytesPerToken: number;
+    } | undefined;
+    streamingMemory?: {
+        fixedBytes: number;
+        ctxTokens: number;
+        bytesPerToken: number;
+        minCacheBytes?: number;
+    } | undefined;
+}
+/** Measured streamed footprint without the expert cache, at a window; undefined when unmeasured. */
+export declare function ds4StreamingFixedBytes(source: Ds4MemorySource | undefined, ctxTokens: number): number | undefined;
+export interface Ds4MeasuredLaunchOptions {
+    source: Ds4MemorySource | undefined;
+    numCtx: number;
+    ssdStreaming: boolean;
+    totalRamBytes: number;
+    /** Local-model budget, needed to size a streamed launch's cache. */
+    budgetBytes?: number | undefined;
+    /** Resident memory beside the model: a vision encoder, a DSpark companion. */
+    sidecarBytes?: number;
+    configuredCacheGb?: number | undefined;
+}
+export interface Ds4MeasuredLaunch {
+    /** What the launch holds, sidecars included. The broker reserves exactly this. */
+    workingSetBytes: number;
+    /** The part of {@link workingSetBytes} the window does not move. */
+    contextFreeBytes: number;
+    bytesPerToken: number;
+    /** Streamed launches only. */
+    cache?: Ds4ExpertCachePlan;
+}
+/**
+ * Price a launch from the entry's measurement for the chosen mode. Undefined
+ * when that mode is unmeasured (or a streamed launch has no budget to size
+ * against), so callers keep their catalog-`residentBytes` path, including the
+ * 96 GiB full-residency floor that stands in for a missing measurement.
+ */
+export declare function planDs4MeasuredLaunch(opts: Ds4MeasuredLaunchOptions): Ds4MeasuredLaunch | undefined;
 /** Effective capacity-broker reservation for the chosen DS4 residency mode. */
 export declare function ds4ResidentBytesForMode(catalogResidentBytes: number, ssdStreaming: boolean, ssdStreamingSupported?: boolean): number;
 /** Allowance for ds4's compute and staging buffers beyond weights and KV rows. */

package/dist/providers/ds4/residency.js

@@ -6,6 +6,10 @@ export { DS4_FULL_RESIDENCY_HEADROOM_BYTES };
  * Conservative floor for the full-residency working set. Exclusivity is an
  * explicit device lease at process startup, not a consequence of this number:
  * the automatic model budget can exceed 96 GiB on larger machines.
+ *
+ * Only for a launch nobody measured: an entry with `fullResidencyMemory` is
+ * reserved at that working set ({@link planDs4MeasuredLaunch}). Flooring a
+ * 45 GiB Qwen3.8 launch here put it over a 96 GiB Mac's 80 GiB budget.
  */
 export const DS4_FULL_RESIDENCY_RESERVATION_BYTES = 96 * GB;
 /**
@@ -80,20 +84,97 @@ export function shouldUseDs4SsdStreaming(opts = {}) {
  * the old RAM-tier heuristic which incorrectly gave both models 64 GiB on a
  * 128 GiB Mac. Manual values are still accepted, but clamped so fixed model
  * state + the cache cannot consume the runtime/OS reserve.
+ *
+ * The 24 GiB headroom guards a footprint guessed as `residentBytes −
+ * cacheExpertsBytes`, which omits graph and staging memory ds4 also holds.
+ * An entry with a measured streamed footprint has no such gap, so its cache
+ * may use whatever this device's local-model budget leaves beside it. A fixed
+ * 24 GiB refused Qwen3.8 on 32 GiB Macs whose budget held it with room to spare.
  */
 export function planDs4ExpertCache(opts) {
     const totalRamBytes = opts.totalRamBytes ?? totalMemoryBytes();
     const catalogCacheBytes = opts.catalogCacheBytes ?? 0;
-    const fixedResidentBytes = Math.max(0, (opts.catalogResidentBytes ?? 0) - catalogCacheBytes);
     const fallbackGb = totalRamBytes >= 120 * GB ? 64 : totalRamBytes >= 88 * GB ? 48 : 32;
     const requestedGb = opts.configuredGb ?? (catalogCacheBytes > 0 ? catalogCacheBytes / GB : fallbackGb);
-    const safeCacheBytes = totalRamBytes - fixedResidentBytes - DS4_FULL_RESIDENCY_HEADROOM_BYTES;
+    const modelBudget = opts.measuredFixedBytes !== undefined && opts.budgetBytes !== undefined;
+    const safeCacheBytes = modelBudget
+        ? opts.budgetBytes - opts.measuredFixedBytes
+        : totalRamBytes -
+            Math.max(0, (opts.catalogResidentBytes ?? 0) - catalogCacheBytes) -
+            DS4_FULL_RESIDENCY_HEADROOM_BYTES;
     const safeCacheGb = Math.floor(safeCacheBytes / GB);
-    if (safeCacheGb < 1) {
-        return { cacheGb: 1, requestedGb, clamped: true, safe: false };
+    const minCacheGb = Math.max(1, Math.ceil((opts.minCacheBytes ?? 0) / GB));
+    const limit = modelBudget ? 'model-budget' : 'headroom';
+    if (safeCacheGb < minCacheGb) {
+        return { cacheGb: minCacheGb, requestedGb, clamped: true, safe: false, limit };
+    }
+    const cacheGb = Math.max(minCacheGb, Math.min(requestedGb, safeCacheGb));
+    return { cacheGb, requestedGb, clamped: cacheGb !== requestedGb, safe: true, limit };
+}
+/** Evaluate a measured line at a window, never below its context-free part. */
+export function ds4MeasuredBytesAt(line, ctxTokens) {
+    const contextFree = Math.max(0, line.bytes - line.bytesPerToken * line.ctxTokens);
+    return Math.round(contextFree + line.bytesPerToken * Math.max(0, ctxTokens));
+}
+function fullResidencyLine(source) {
+    const m = source?.fullResidencyMemory;
+    return m
+        ? { bytes: m.workingSetBytes, ctxTokens: m.ctxTokens, bytesPerToken: m.bytesPerToken }
+        : undefined;
+}
+function streamingLine(source) {
+    const m = source?.streamingMemory;
+    return m
+        ? { bytes: m.fixedBytes, ctxTokens: m.ctxTokens, bytesPerToken: m.bytesPerToken }
+        : undefined;
+}
+/** Measured streamed footprint without the expert cache, at a window; undefined when unmeasured. */
+export function ds4StreamingFixedBytes(source, ctxTokens) {
+    const line = streamingLine(source);
+    return line ? ds4MeasuredBytesAt(line, ctxTokens) : undefined;
+}
+/**
+ * Price a launch from the entry's measurement for the chosen mode. Undefined
+ * when that mode is unmeasured (or a streamed launch has no budget to size
+ * against), so callers keep their catalog-`residentBytes` path, including the
+ * 96 GiB full-residency floor that stands in for a missing measurement.
+ */
+export function planDs4MeasuredLaunch(opts) {
+    const sidecarBytes = opts.sidecarBytes ?? 0;
+    if (!opts.ssdStreaming) {
+        const line = fullResidencyLine(opts.source);
+        if (!line)
+            return undefined;
+        const contextFreeBytes = ds4MeasuredBytesAt(line, 0) + sidecarBytes;
+        return {
+            workingSetBytes: contextFreeBytes + line.bytesPerToken * Math.max(0, opts.numCtx),
+            contextFreeBytes,
+            bytesPerToken: line.bytesPerToken,
+        };
     }
-    const cacheGb = Math.max(1, Math.min(requestedGb, safeCacheGb));
-    return { cacheGb, requestedGb, clamped: cacheGb !== requestedGb, safe: true };
+    const line = streamingLine(opts.source);
+    if (!line || opts.budgetBytes === undefined)
+        return undefined;
+    const fixedBytes = ds4MeasuredBytesAt(line, opts.numCtx) + sidecarBytes;
+    const cache = planDs4ExpertCache({
+        ...(opts.configuredCacheGb !== undefined ? { configuredGb: opts.configuredCacheGb } : {}),
+        ...(opts.source?.cacheExpertsBytes !== undefined
+            ? { catalogCacheBytes: opts.source.cacheExpertsBytes }
+            : {}),
+        totalRamBytes: opts.totalRamBytes,
+        measuredFixedBytes: fixedBytes,
+        budgetBytes: opts.budgetBytes,
+        ...(opts.source?.streamingMemory?.minCacheBytes !== undefined
+            ? { minCacheBytes: opts.source.streamingMemory.minCacheBytes }
+            : {}),
+    });
+    const cacheBytes = cache.cacheGb * GB;
+    return {
+        workingSetBytes: fixedBytes + cacheBytes,
+        contextFreeBytes: ds4MeasuredBytesAt(line, 0) + sidecarBytes + cacheBytes,
+        bytesPerToken: line.bytesPerToken,
+        cache,
+    };
 }
 /** Effective capacity-broker reservation for the chosen DS4 residency mode. */
 export function ds4ResidentBytesForMode(catalogResidentBytes, ssdStreaming, ssdStreamingSupported = true) {

package/dist/providers/image/sd-cpp.js

@@ -273,13 +273,8 @@ export class StableDiffusionCppProvider {
         const height = input.height ?? defaultDimension;
         const resolvedLoras = await this.resolveLoras(modelId, input.loras ?? []);
         let loras = resolvedLoras.loras;
-        // A speed LoRA's recipe is what makes its few steps valid, so it wins
-        // over a style's.
-        const render = loras.find((l) => l.purpose === 'speed')?.render ?? loras.find((l) => l.render)?.render;
-        const steps = input.steps ?? render?.steps ?? (await this.defaultSampleSteps(modelId));
         const seed = input.seed ?? Math.floor(Math.random() * 2_147_483_647);
         const started = Date.now();
-        log.info(`[sd-cpp] generate model=${modelId ?? 'default'} steps=${steps} explicitSteps=${input.steps !== undefined} size=${width}x${height}${loras.length > 0 ? ` loras=${loras.map((l) => l.id).join(',')}` : ''}`);
         // Model selection: sd-server serves one model per process. When a
         // request targets a different installed model than the one currently
         // loaded, point resolveLaunch at it (via the shared launchState) and
@@ -290,8 +285,9 @@ export class StableDiffusionCppProvider {
             this.launchState.modelId = modelId;
             await this.supervisor.stop();
         }
-        // A render the caller abandons keeps running inside sd-server, which
-        // cannot interrupt one it has started, so cancelling stops the engine.
+        // A render the caller abandons keeps running inside sd-server, whose
+        // `/sdapi` endpoints cannot interrupt one they have started, so
+        // cancelling stops the engine.
         const request = new AbortController();
         const stopRender = () => {
             request.abort();
@@ -333,6 +329,15 @@ export class StableDiffusionCppProvider {
             }
             if (signal?.aborted)
                 throw new ImageGenerationCancelledError();
+            const settingsNotes = [...resolvedLoras.notes];
+            if (loras.length > 0) {
+                loras = await this.lorasTheEngineHas(baseUrl, loras, request.signal, settingsNotes);
+            }
+            // Read only after the engine's list: a fast mode's 4 steps and CFG 1
+            // on the bare model it was dropped from render mush.
+            const render = loraRenderOverrides(loras);
+            const steps = input.steps ?? render?.steps ?? (await this.defaultSampleSteps(modelId));
+            log.info(`[sd-cpp] generate model=${modelId ?? 'default'} steps=${steps} explicitSteps=${input.steps !== undefined} size=${width}x${height}${loras.length > 0 ? ` loras=${loras.map((l) => l.id).join(',')}` : ''}`);
             // Drive sd-server's A1111-compatible endpoints (`/sdapi/v1/txt2img`
             // and `/sdapi/v1/img2img`). Unlike the OpenAI `/v1/images/*` shape
             // — which reads only prompt/n/size and takes steps/seed/cfg from the
@@ -341,10 +346,6 @@ export class StableDiffusionCppProvider {
             // actually take effect. `cfg_scale` is set per model (distilled /
             // Flux-flow models need CFG 1; others inherit the server default).
             const cfg = render?.cfg ?? (modelId ? MODEL_CFG_DEFAULTS.get(modelId) : undefined);
-            const settingsNotes = [...resolvedLoras.notes];
-            if (loras.length > 0) {
-                loras = await this.lorasTheEngineHas(baseUrl, loras, request.signal, settingsNotes);
-            }
             let inputImages = input.inputImages ?? [];
             // Capability gate: models whose architecture doesn't honor init
             // latents on the pinned sd-server fall back to txt2img instead of
@@ -996,6 +997,13 @@ export function withoutEngineDirectives(text) {
     } while (out !== previous);
     return out;
 }
+/**
+ * The render settings the applied LoRAs set. A speed LoRA's recipe is what
+ * makes its few steps valid, so it wins over a style's.
+ */
+function loraRenderOverrides(applied) {
+    return (applied.find((l) => l.purpose === 'speed')?.render ?? applied.find((l) => l.render)?.render);
+}
 /**
  * Read an installed LoRA's manifest, or nothing when it is malformed. The
  * filename must be a bare `.safetensors` name: the engine resolves it inside

package/dist/providers/llama-cpp/backend-health.d.ts

@@ -1,7 +1,8 @@
 /**
  * Whether llama-server is getting the hardware this machine has, as the
  * engine process sees it: the CUDA build discovery passed over for the
- * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), a GPU build that started without its
+ * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), the ROCm build it passed over for this
+ * machine's AMD GPUs (`GEZEL_LLAMA_ROCM_BLOCKER`), a GPU build that started without its
  * GPU (gpu-startup-watch.ts), and Linux's out-of-memory killer stopping the
  * engine. Each is invisible otherwise — the engine keeps answering, slower,
  * or a chat fails with a crash that reads like any other.
@@ -24,7 +25,7 @@ export declare function noteHostOutOfMemory(input: {
     model?: string | undefined;
     whileStarting: boolean;
 }, now?: Date): void;
-declare const BACKENDS: readonly ["cuda", "vulkan", "metal", "cpu"];
+declare const BACKENDS: readonly ["cuda", "rocm", "vulkan", "metal", "cpu"];
 /**
  * The backend this process's on-device llama-server runs, for catalog
  * backend exclusions: the build discovery resolved, else what the probe

package/dist/providers/llama-cpp/backend-health.js

@@ -1,7 +1,8 @@
 /**
  * Whether llama-server is getting the hardware this machine has, as the
  * engine process sees it: the CUDA build discovery passed over for the
- * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), a GPU build that started without its
+ * driver (`GEZEL_LLAMA_CUDA_BLOCKER`), the ROCm build it passed over for this
+ * machine's AMD GPUs (`GEZEL_LLAMA_ROCM_BLOCKER`), a GPU build that started without its
  * GPU (gpu-startup-watch.ts), and Linux's out-of-memory killer stopping the
  * engine. Each is invisible otherwise — the engine keeps answering, slower,
  * or a chat fails with a crash that reads like any other.
@@ -9,7 +10,7 @@
  * Process-wide like the backend env it sits beside: every llama provider in
  * this process launches the same build.
  */
-import { missingMachineRuntime, readLlamaCudaBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
+import { missingMachineRuntime, readLlamaCudaBlocker, readRocmBlocker, vcRedistributableUrl, } from '@bendyline/gezel/native';
 /**
  * Reading an engine's import tables takes tens of milliseconds of synchronous
  * file reads, and the answer only changes when someone installs the runtime.
@@ -61,7 +62,7 @@ export function noteHostOutOfMemory(input, now = new Date()) {
         whileStarting: input.whileStarting,
     };
 }
-const BACKENDS = ['cuda', 'vulkan', 'metal', 'cpu'];
+const BACKENDS = ['cuda', 'rocm', 'vulkan', 'metal', 'cpu'];
 /**
  * The backend this process's on-device llama-server runs, for catalog
  * backend exclusions: the build discovery resolved, else what the probe
@@ -85,6 +86,7 @@ export function llamaBackendHealthView() {
     const runningRaw = process.env.GEZEL_LLAMA_SERVER_BACKEND;
     const running = BACKENDS.find((backend) => backend === runningRaw);
     const cudaBlocker = readLlamaCudaBlocker();
+    const rocmBlocker = readRocmBlocker();
     // A verdict about a build the process has since moved off (a quarantine
     // fallback, a lifted quarantine) no longer describes what runs.
     const unused = gpuUnused && gpuUnused.backend === running ? gpuUnused : null;
@@ -92,6 +94,7 @@ export function llamaBackendHealthView() {
     return {
         ...(running ? { running } : {}),
         ...(cudaBlocker ? { cudaBlocker } : {}),
+        ...(rocmBlocker ? { rocmBlocker } : {}),
         ...(unused ? { gpuUnused: unused } : {}),
         ...(missingRuntime ? { missingRuntime } : {}),
         ...(outOfMemory ? { outOfMemory } : {}),

package/dist/providers/llama-cpp/backend-quarantine.js

@@ -11,7 +11,7 @@
  * next engine launch instead of the next app start.
  */
 import { LLAMA_QUARANTINE_STRIKES, clearLlamaQuarantine, createLlamaBuildCheck, isBinaryQuarantined, llamaVariantBinaryPath, publishLlamaCudaBlocker, readLlamaQuarantine, resolveLlamaFallbackBinary, resolveLlamaVariantBinary, } from '@bendyline/gezel/native';
-const BACKENDS = ['cuda', 'vulkan', 'metal', 'cpu'];
+const BACKENDS = ['cuda', 'rocm', 'vulkan', 'metal', 'cpu'];
 export function isLlamaBackend(value) {
     return BACKENDS.some((backend) => backend === value);
 }