@bendyline/gezel-inference 1.0.2 → 1.0.4

package/dist/providers/image/local-factory.js

@@ -0,0 +1,270 @@
+/**
+ * The local image provider: stable-diffusion.cpp's `sd-server` on this
+ * machine. Shared by the full service (behind its cloud and remote routing)
+ * and by hosted inference, which serves only this.
+ *
+ * Selection rules (first-match):
+ *   1. `GEZEL_MOCK_PROVIDER=1` → `MockImageProvider` (tests, CI).
+ *   2. `GEZEL_SD_SERVER_URL` present → `StableDiffusionCppProvider`
+ *      pointed at that URL, no supervisor ("user runs their own
+ *      server" mode — useful during development).
+ *   3. `GEZEL_SD_SERVER_BIN` present → `StableDiffusionCppProvider`
+ *      backed by a `NativeEngineSupervisor` that spawns the binary
+ *      on-demand at an ephemeral port.
+ *   4. Fallback: `StableDiffusionCppProvider` with the default
+ *      base URL (127.0.0.1:9081) and no supervisor. `health()` reports
+ *      `not-configured` so the UI can say the engine is not installed.
+ */
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { imageMediaPool, sdServerLedgerClaim, sdServerRequirement, } from '../../models/image-fit.js';
+import { findModelRoot, listOverlayModelIds, modelStorageRoots, verifyReadOnlyModelPayload, } from '../../models/storage-roots.js';
+import { localSdCppWeightFiles } from '../../models/weight-facts.js';
+import { pressureIdleGraceMs } from '../native/capacity-broker.js';
+import { measuredCapacityBudget } from '../native/measured-budget.js';
+import { pickFreePort } from '../native/port.js';
+import { NativeEngineSupervisor } from '../native/supervisor.js';
+import { patientFetch } from '../patient-fetch.js';
+import { MockImageProvider } from './mock.js';
+import { MODEL_SAMPLE_STEP_DEFAULTS, StableDiffusionCppProvider, sdServerLoadProfile, } from './sd-cpp.js';
+/**
+ * Nothing to generate with: the person has to download a model first. Its
+ * own type so a route can answer with the fix, not a bare 500, which reads
+ * to an outside app as "something went wrong inside Gezel".
+ */
+export class NoLocalImageModelError extends Error {
+    code = 'no_local_image_model';
+    constructor() {
+        super('No image model is available locally. Download one from Settings → Image generation before generating.');
+        this.name = 'NoLocalImageModelError';
+    }
+}
+/**
+ * How sd-server is admitted to the device memory ledger: by bytes, like any
+ * other engine, never exclusively. The ledger already refuses co-residency
+ * where the byte or GPU budget cannot hold both, and `detectGpuPolicy` states
+ * the rule: automatic heuristics must not serialize models that fit.
+ * Exclusive admission did exactly that. On a DGX Spark with gemma4-31b
+ * resident (95 GB of a 128 GB machine), a 4-second SDXL render waited up to
+ * 89 s for the chat engine to go idle, then was evicted as soon as the next
+ * chat turn asked for memory, so every render paid a fresh load.
+ */
+export function sdServerCapacity(home) {
+    return { home };
+}
+export async function createLocalImageProvider(opts) {
+    const env = opts.env ?? process.env;
+    const storageRoots = modelStorageRoots({ home: opts.home, engine: 'sd-cpp', env });
+    const modelsRoot = storageRoots.writableRoot;
+    const defaultLocalModelId = opts.defaultModelId;
+    if (env.GEZEL_MOCK_PROVIDER === '1') {
+        return new MockImageProvider();
+    }
+    if (env.GEZEL_SD_SERVER_URL) {
+        return new StableDiffusionCppProvider({
+            baseUrl: env.GEZEL_SD_SERVER_URL,
+            modelsRoot,
+            storageRoots,
+            fetchImpl: patientFetch(),
+            ...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
+        });
+    }
+    if (env.GEZEL_SD_SERVER_BIN) {
+        const binary = env.GEZEL_SD_SERVER_BIN;
+        // Track the most-recently-resolved launch so the provider can point
+        // fetch() at the right port without the supervisor exposing state.
+        let cachedPort;
+        // Shared with the provider: which installed model to launch
+        // sd-server with. sd-server is single-model per process, so the
+        // provider sets this to the requested model and stops the server
+        // when it changes; resolveLaunch reads it on the next start.
+        const launchState = { modelId: undefined };
+        let launchedModel;
+        const supervisor = new NativeEngineSupervisor({
+            capacity: {
+                ...sdServerCapacity(opts.home),
+                // Price the launched model from its own weight files — the same
+                // pricing the download page used — and claim what the render
+                // actually holds on the card (see sdServerLedgerClaim).
+                requirement: async () => {
+                    if (!launchedModel)
+                        return undefined;
+                    const budget = await measuredCapacityBudget();
+                    const requirement = sdServerRequirement({
+                        modelId: launchedModel.id,
+                        weightsKind: launchedModel.weightsKind,
+                        files: await localSdCppWeightFiles(launchedModel),
+                        pool: imageMediaPool(budget.kind),
+                    });
+                    return sdServerLedgerClaim(requirement, budget.fastBytes);
+                },
+            },
+            logPrefix: '[sd-server]',
+            // sd-server has no `/health`. It answers the root with an HTML file
+            // (with --serve-html-path) or a 404, and starts listening only once
+            // it has read the model's metadata — the weights themselves load
+            // inside the first request. Any HTTP response is the signal.
+            readinessPath: '/',
+            readyOnAnyResponse: true,
+            // Cold start can take a while: metadata for a 49 GB model, backend
+            // init, first kernel compile. Give the readiness probe generous
+            // headroom so first launch doesn't trip the restart-budget guard.
+            startupTimeoutMs: 5 * 60 * 1000,
+            // An idle image engine gives its memory back when a chat model is
+            // waiting for it, the same way the chat engines yield to a render.
+            ...(opts.arbiter
+                ? {
+                    memoryPressure: () => opts.arbiter.getMemoryPressureStatus(),
+                    pressureIdleTimeoutMs: pressureIdleGraceMs(),
+                }
+                : {}),
+            resolveLaunch: async () => {
+                const port = cachedPort ?? (await pickFreePort());
+                cachedPort = port;
+                // Prefer the model the provider asked for, then the configured
+                // default, then the first installed — covering a generate with no
+                // explicit model, and a requested (or configured) model having
+                // since been deleted.
+                const requestedId = launchState.modelId ?? defaultLocalModelId;
+                const model = (requestedId ? await findInstalledModel(storageRoots, requestedId) : undefined) ??
+                    (await findFirstInstalledModel(storageRoots));
+                if (!model)
+                    throw new NoLocalImageModelError();
+                launchedModel = model;
+                return {
+                    command: binary,
+                    args: buildSdServerArgs(model, port),
+                    baseUrl: `http://127.0.0.1:${port}`,
+                };
+            },
+        });
+        return new StableDiffusionCppProvider({
+            // The baseUrl here is only the fallback when the supervisor is
+            // not yet resolved; generate() overrides from `ensureRunning()`.
+            baseUrl: 'http://127.0.0.1:9081',
+            modelsRoot,
+            storageRoots,
+            supervisor,
+            launchState,
+            fetchImpl: patientFetch(),
+            ...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
+            ...(opts.arbiter ? { arbiter: opts.arbiter } : {}),
+        });
+    }
+    return new StableDiffusionCppProvider({
+        baseUrl: 'http://127.0.0.1:9081',
+        modelsRoot,
+        storageRoots,
+        ...(defaultLocalModelId ? { defaultModelId: defaultLocalModelId } : {}),
+        // Nothing was wired up. Flag the provider so `health()` can return
+        // `not-configured` (distinct UI guidance) rather than making the user
+        // wait for a generic "unreachable" probe to fail.
+        configured: false,
+    });
+}
+/**
+ * Read one installed model's resolved launch info from its on-disk
+ * `manifest.json`, or undefined if absent / malformed / incomplete.
+ */
+async function resolveInstalledModel(modelsRoot, id, storageRoots) {
+    try {
+        const itemDir = join(modelsRoot, id);
+        const raw = await readFile(join(itemDir, 'manifest.json'), 'utf8');
+        const parsed = JSON.parse(raw);
+        if (!parsed.id || !parsed.name || !parsed.weightsFilename)
+            return undefined;
+        if (storageRoots &&
+            !(await verifyReadOnlyModelPayload(storageRoots, modelsRoot, id, parsed.fileSha256))) {
+            return undefined;
+        }
+        const aux = [];
+        for (const a of parsed.auxiliaryFiles ?? []) {
+            if (a.role && a.filename)
+                aux.push({ role: a.role, path: join(itemDir, a.filename) });
+        }
+        return {
+            id: parsed.id,
+            name: parsed.name,
+            approxSizeBytes: parsed.approxSizeBytes ?? 0,
+            installedAt: parsed.installedAt ?? new Date().toISOString(),
+            weightsPath: join(itemDir, parsed.weightsFilename),
+            weightsKind: parsed.weightsKind ?? 'checkpoint',
+            auxiliaryFiles: aux,
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Resolve a specific installed model by id (the one a request targets). */
+async function findInstalledModel(storageRoots, id) {
+    const root = await findModelRoot(storageRoots, id);
+    return root ? resolveInstalledModel(root, id, storageRoots) : undefined;
+}
+async function findFirstInstalledModel(storageRoots) {
+    const entries = await listOverlayModelIds(storageRoots);
+    entries.sort();
+    for (const id of entries) {
+        const model = await findInstalledModel(storageRoots, id);
+        if (model)
+            return model;
+    }
+    return undefined;
+}
+/**
+ * Translate a resolved on-disk model into stable-diffusion.cpp's `sd-server`
+ * launch flags. `checkpoint` weights load via `--model`; `diffusion-model`
+ * weights are unet-only and ride alongside `--vae`/`--clip_l`/`--clip_g`/
+ * `--t5xxl` from the auxiliary files. Aux roles map 1:1 to flag names.
+ *
+ * Bind flags use `--listen-ip` / `--listen-port` (NOT `--host` / `--port`)
+ * — that's the upstream stable-diffusion.cpp server CLI as of master-587;
+ * earlier llama.cpp-style `--host` / `--port` will be rejected with
+ * `error: unknown argument: --host` and the supervisor restart-loops.
+ *
+ * `--vae-tiling` is the default because SDXL/Flux-class VAE decode at
+ * 800–1024px wants a single ~7 GB Vulkan buffer, which exceeds the
+ * per-allocation cap most consumer drivers enforce (~25–50% of total
+ * VRAM). The OOM lands at the very last step (sampling completes, then
+ * VAE blows up), wasting the entire generation. Tiled decode splits
+ * that buffer into chunks at ~10–20% decode-speed cost; on a 12 GB
+ * card it's the difference between succeeding and dying on `vae alloc
+ * compute buffer failed`. Quality difference is imperceptible for the
+ * tile sizes sd.cpp picks.
+ *
+ * The text encoder never takes flash attention. ggml's CPU kernel sums the
+ * attention values in FP16, which a Qwen-family encoder overflows: FLUX.2
+ * klein on a Windows-on-ARM CPU rendered unrelated shapes with a 200
+ * (native-v0.1.53). It bites wherever the encoder lands on the CPU, and the
+ * encoder runs once per image, so GPUs lose nothing. `--diffusion-fa` means
+ * `--flash-attn off,diffusion=on`, so that branch already leaves it off.
+ */
+export function buildSdServerArgs(model, port) {
+    const args = [];
+    if (model.weightsKind === 'diffusion-model') {
+        args.push('--diffusion-model', model.weightsPath);
+    }
+    else {
+        args.push('--model', model.weightsPath);
+    }
+    for (const aux of model.auxiliaryFiles) {
+        args.push(`--${aux.role}`, aux.path);
+    }
+    const steps = MODEL_SAMPLE_STEP_DEFAULTS.get(model.id);
+    if (steps)
+        args.push('--steps', String(steps));
+    const profile = sdServerLoadProfile(model.id);
+    if (profile.diffusionFlashAttention)
+        args.push('--diffusion-fa');
+    else
+        args.push('--flash-attn', 'auto,te=off');
+    if (profile.bf16TensorPattern) {
+        args.push('--tensor-type-rules', `${profile.bf16TensorPattern}=bf16`);
+    }
+    if (profile.vaeDecode === 'tiled')
+        args.push('--vae-tiling');
+    if (profile.stepCache)
+        args.push('--cache-mode', profile.stepCache);
+    args.push('--listen-ip', '127.0.0.1', '--listen-port', String(port));
+    return args;
+}

package/dist/providers/image/mock.d.ts

@@ -0,0 +1,23 @@
+/**
+ * MockImageProvider — deterministic image generation for tests and for the
+ * `GEZEL_MOCK_PROVIDER=1` flow. Produces a tiny, valid PNG whose pixel
+ * colour is derived from a hash of the prompt so different prompts yield
+ * visually different (but still trivially small) outputs.
+ */
+import type { ImageEngineHealth, ImageGenerationInput, ImageGenerationOutput, ImageModelPullEvent, ImageModelPullSpec, ImageProvider, InstalledImageModelInfo } from './types.js';
+export declare class MockImageProvider implements ImageProvider {
+    readonly name = "mock";
+    private readonly installed;
+    /** Internal counter driving deterministic seed generation on "random" seeds. */
+    private nextSeed;
+    constructor(opts?: {
+        preinstalled?: InstalledImageModelInfo[];
+    });
+    /** Number of input images received on the most recent generate(). Tests use this. */
+    lastInputImageCount: number;
+    generate(input: ImageGenerationInput): Promise<ImageGenerationOutput>;
+    listInstalledModels(): Promise<InstalledImageModelInfo[]>;
+    pullModel(id: string, spec: ImageModelPullSpec, signal?: AbortSignal): AsyncIterable<ImageModelPullEvent>;
+    deleteModel(id: string): Promise<void>;
+    health(): Promise<ImageEngineHealth>;
+}

package/dist/providers/image/mock.js

@@ -0,0 +1,151 @@
+/**
+ * MockImageProvider — deterministic image generation for tests and for the
+ * `GEZEL_MOCK_PROVIDER=1` flow. Produces a tiny, valid PNG whose pixel
+ * colour is derived from a hash of the prompt so different prompts yield
+ * visually different (but still trivially small) outputs.
+ */
+import { deflateSync } from 'node:zlib';
+export class MockImageProvider {
+    name = 'mock';
+    installed = new Map();
+    /** Internal counter driving deterministic seed generation on "random" seeds. */
+    nextSeed = 1;
+    constructor(opts = {}) {
+        for (const m of opts.preinstalled ?? []) {
+            this.installed.set(m.id, m);
+        }
+    }
+    /** Number of input images received on the most recent generate(). Tests use this. */
+    lastInputImageCount = 0;
+    async generate(input) {
+        this.lastInputImageCount = input.inputImages?.length ?? 0;
+        const width = clamp(input.width ?? 16, 1, 512);
+        const height = clamp(input.height ?? 16, 1, 512);
+        const steps = clamp(input.steps ?? 1, 1, 100);
+        const seed = input.seed ?? this.nextSeed++;
+        const model = input.model ?? [...this.installed.keys()][0] ?? 'mock-model';
+        const rgba = solidFill(width, height, colourForPrompt(input.prompt));
+        const png = encodePng(rgba, width, height);
+        return {
+            png,
+            meta: {
+                model,
+                seed,
+                steps,
+                widthPx: width,
+                heightPx: height,
+                durationMs: 1,
+            },
+        };
+    }
+    async listInstalledModels() {
+        return [...this.installed.values()];
+    }
+    async *pullModel(id, spec, signal) {
+        const total = spec.approxSizeBytes;
+        for (const step of [0.25, 0.5, 0.75, 1]) {
+            if (signal?.aborted) {
+                yield { type: 'error', error: 'download aborted' };
+                yield { type: 'done', id };
+                return;
+            }
+            // Tiny tick so tests can observe a mid-pull state between steps.
+            await new Promise((resolve) => {
+                const t = setTimeout(resolve, 5);
+                t.unref?.();
+            });
+            yield { type: 'progress', bytesWritten: Math.round(total * step), totalBytes: total };
+        }
+        this.installed.set(id, {
+            id,
+            name: spec.name,
+            approxSizeBytes: total,
+            installedAt: new Date().toISOString(),
+        });
+        yield { type: 'done', id };
+    }
+    async deleteModel(id) {
+        this.installed.delete(id);
+    }
+    async health() {
+        return { status: 'ok', baseUrl: 'mock://image' };
+    }
+}
+function clamp(n, min, max) {
+    if (n < min)
+        return min;
+    if (n > max)
+        return max;
+    return n;
+}
+/** Hash the prompt to an `[r, g, b]` tuple so different prompts paint different pixels. */
+function colourForPrompt(prompt) {
+    let h = 2166136261;
+    for (let i = 0; i < prompt.length; i++) {
+        h ^= prompt.charCodeAt(i);
+        h = Math.imul(h, 16777619);
+    }
+    return [h & 0xff, (h >>> 8) & 0xff, (h >>> 16) & 0xff];
+}
+function solidFill(width, height, rgb) {
+    const rgba = new Uint8Array(width * height * 4);
+    for (let i = 0; i < width * height; i++) {
+        rgba[i * 4] = rgb[0];
+        rgba[i * 4 + 1] = rgb[1];
+        rgba[i * 4 + 2] = rgb[2];
+        rgba[i * 4 + 3] = 0xff;
+    }
+    return rgba;
+}
+// ── Minimal PNG encoder (RGBA, no filter) ──────────────────────────
+const CRC_TABLE = (() => {
+    const table = new Uint32Array(256);
+    for (let n = 0; n < 256; n++) {
+        let c = n;
+        for (let k = 0; k < 8; k++)
+            c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+        table[n] = c >>> 0;
+    }
+    return table;
+})();
+function crc32(bytes) {
+    let crc = 0xffffffff;
+    for (let i = 0; i < bytes.length; i++)
+        crc = CRC_TABLE[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
+    return (crc ^ 0xffffffff) >>> 0;
+}
+function chunk(type, data) {
+    const typeBytes = Buffer.from(type, 'ascii');
+    const buf = Buffer.alloc(8 + data.length + 4);
+    buf.writeUInt32BE(data.length, 0);
+    typeBytes.copy(buf, 4);
+    Buffer.from(data).copy(buf, 8);
+    const crcInput = new Uint8Array(typeBytes.length + data.length);
+    crcInput.set(typeBytes, 0);
+    crcInput.set(data, typeBytes.length);
+    buf.writeUInt32BE(crc32(crcInput), 8 + data.length);
+    return buf;
+}
+function encodePng(rgba, width, height) {
+    const sig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+    const ihdr = Buffer.alloc(13);
+    ihdr.writeUInt32BE(width, 0);
+    ihdr.writeUInt32BE(height, 4);
+    ihdr[8] = 8;
+    ihdr[9] = 6;
+    ihdr[10] = 0;
+    ihdr[11] = 0;
+    ihdr[12] = 0;
+    const raw = new Uint8Array(height * (1 + width * 4));
+    for (let y = 0; y < height; y++) {
+        raw[y * (1 + width * 4)] = 0;
+        raw.set(rgba.subarray(y * width * 4, (y + 1) * width * 4), y * (1 + width * 4) + 1);
+    }
+    const idat = deflateSync(raw);
+    return Buffer.concat([
+        sig,
+        chunk('IHDR', ihdr),
+        chunk('IDAT', idat),
+        chunk('IEND', new Uint8Array(0)),
+    ]);
+}

package/dist/providers/image/pull-registry.d.ts

@@ -0,0 +1,106 @@
+import { type ImageModelPullEvent } from '@bendyline/gezel';
+import type { CatalogService } from '@bendyline/gezel-catalog';
+import type { ImageProvider } from './types.js';
+/**
+ * Snapshot of one in-flight (or recently-finished) image-model pull.
+ * Exposed to the UI via `GET /api/image-gen/pulls` so a client that
+ * mounted after the pull started can still render an accurate
+ * progress bar without subscribing to the live SSE.
+ */
+export interface ActiveImagePull {
+    id: string;
+    startedAt: string;
+    bytesWritten: number;
+    /** Best-known total. Zero until the first `progress` event arrives. */
+    totalBytes: number;
+    /** Present when the shared downloader is between retry attempts. */
+    retrying?: {
+        attempt: number;
+        maxAttempts: number;
+        delayMs: number;
+        reason: string;
+    };
+    /** Set once a terminal event has been observed (done or error). */
+    finished: boolean;
+    /** Set when the pull errored — kept on the snapshot so reconnecting clients see the failure. */
+    error?: string;
+}
+type Listener = (event: ImageModelPullEvent) => void;
+/**
+ * Where pulls get their provider: the service's routing manager, or the
+ * hosted inference's local provider. Read on every pull so a provider reset
+ * (an engine binary swap) is picked up.
+ */
+export interface ImageProviderSource {
+    current(): Promise<Pick<ImageProvider, 'pullModel'>>;
+}
+/**
+ * Owns the lifecycle of every in-flight image-model pull.
+ *
+ * Pulls used to be tied 1:1 to the HTTP request that started them — when
+ * the client disconnected (navigated away from Settings → Image
+ * generation), the SSE loop exited, the async generator closed, and the
+ * download was aborted by the .partial-leaves-resumable code path. A
+ * returning client had no record the pull ever existed.
+ *
+ * The registry decouples those two: the pull runs in the background owned
+ * by this object, and HTTP requests are just consumers that subscribe to
+ * the live event stream + the cached snapshot. Disconnect detaches the
+ * consumer; the pull keeps going. Cancel is now an explicit `DELETE
+ * /pulls/:id` rather than a side-effect of closing the fetch.
+ */
+export declare class ImageModelPullRegistry {
+    private readonly entries;
+    private readonly controllers;
+    private readonly imageProvider;
+    private readonly catalog;
+    constructor(opts: {
+        imageProvider: ImageProviderSource;
+        catalog: CatalogService;
+    });
+    /**
+     * Start a pull for `id`. Idempotent — when a pull is already in flight
+     * for this id, returns the existing snapshot and does not kick off a
+     * second download. Throws if the catalog id is unknown.
+     *
+     * The pull itself runs in a detached `void` task; this method returns
+     * synchronously-claimed state so callers can immediately subscribe
+     * without racing the first event.
+     */
+    start(id: string): Promise<{
+        snapshot: ActiveImagePull;
+        alreadyRunning: boolean;
+    }>;
+    list(): ActiveImagePull[];
+    get(id: string): ActiveImagePull | null;
+    /**
+     * Subscribe to live events for an in-flight pull. On subscribe, the
+     * registry immediately delivers a synthetic event reflecting the last
+     * known snapshot — without it, a late subscriber to a paused (between
+     * retry attempts) or just-started pull would see nothing until the
+     * next real event, leaving the UI's progress bar blank.
+     *
+     * Returns `null` when no entry exists for `id` so the caller can
+     * 404. Returns an unsubscribe function otherwise; calling it removes
+     * the listener but does NOT cancel the pull — disconnect ≠ cancel.
+     */
+    subscribe(id: string, listener: Listener): (() => void) | null;
+    /**
+     * Cancel an in-flight pull. The shared downloader observes the abort,
+     * leaves the `.partial` in place for resume, and surfaces an `error`
+     * event whose snapshot stays in the registry for `FINISHED_TTL_MS`.
+     * Returns true when a live entry was aborted.
+     */
+    cancel(id: string): boolean;
+    /** Test-only: drop everything so a fresh test starts clean. */
+    clear(): void;
+    private replaySnapshot;
+    private consume;
+    private applyEvent;
+    private broadcast;
+}
+export declare class UnknownImageModelError extends Error {
+    readonly id: string;
+    constructor(id: string);
+}
+export {};