@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/models/catalog-payload-identity.js

@@ -0,0 +1,254 @@
+/**
+ * Does the catalog still describe the bytes this model was installed from?
+ *
+ * A catalog version bump means one of two very different things. Either the
+ * payload moved — new repo, new quant, rotated shas, an added shard — and the
+ * user really is running an older build; or only metadata changed (tuning,
+ * sizing hints, release date, wording), which the runtime already resolves
+ * live from the catalog on every turn and which nothing on disk has to catch
+ * up with. Comparing version strings cannot tell those apart, so a metadata
+ * edit asked every installed model to re-download itself: five models, ~100 GB
+ * of transfer, to adopt a `residentBytes` removal that had already taken
+ * effect. This module asks the question the version string was standing in
+ * for.
+ *
+ * Three tiers, most conclusive first:
+ *
+ *   1. `payloadFingerprint` — a hash of the catalog's own payload description,
+ *      snapshotted into the install manifest at download time. Recomputing it
+ *      from the current catalog is an exact answer for free.
+ *   2. `fileSha256` — the per-file digests every install records. Compare them
+ *      to the catalog's pins file by file.
+ *   3. On-disk sizes vs the catalog's `sizeBytes`. All that installs predating
+ *      the hash map can offer; strong evidence across a multi-shard payload,
+ *      but evidence, not proof — so it never upgrades an install to tier 1.
+ *
+ * A sibling check lives in `shared-model-migration.ts`
+ * (`assertCatalogPayloadIntegrity`). It answers a different question — may
+ * these bytes be admitted to the machine-wide store — and stays a hard
+ * assertion over a freshly-hashed bundle. This one is a read-only comparison
+ * over what the install already recorded, and its answers steer UI, never
+ * trust.
+ */
+import { createHash } from 'node:crypto';
+import { stat } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+/**
+ * Stable hash of a catalog payload description. Deliberately excludes
+ * `revision`: it is a fetch coordinate, and identical hashes mean identical
+ * bytes no matter which commit served them.
+ */
+export function catalogPayloadFingerprint(payload) {
+    const canonical = JSON.stringify({
+        repo: payload.huggingfaceRepo,
+        files: payload.files
+            .map((file) => [file.name, file.sha256.toLowerCase()])
+            .sort(([left], [right]) => left.localeCompare(right)),
+        ...(payload.chatTemplate ? { chatTemplate: payload.chatTemplate } : {}),
+    });
+    return createHash('sha256').update(canonical).digest('hex');
+}
+/**
+ * Translate a catalog manifest into the payload one engine would install.
+ * The single place that knows each source block's shape, so callers compare
+ * like for like without re-deriving filenames.
+ */
+export function describeCatalogPayload(manifest, engine) {
+    if (engine === 'mlx') {
+        const src = manifest.mlx;
+        if (!src)
+            return null;
+        // The install pipeline merges a template into tokenizer_config.json when
+        // upstream ships none, in which case the file on disk is no longer the
+        // file the catalog hashed. A sidecar in the file list means the same
+        // recovery path is reachable.
+        const templateMayBeInjected = Boolean(src.chatTemplate || src.files.some((file) => basename(file.name) === 'chat_template.jinja'));
+        return {
+            huggingfaceRepo: src.huggingfaceRepo,
+            ...(src.chatTemplate ? { chatTemplate: src.chatTemplate } : {}),
+            files: src.files.map((file) => ({
+                name: file.name,
+                sha256: file.sha256,
+                sizeBytes: file.sizeBytes,
+                ...(templateMayBeInjected && basename(file.name) === 'tokenizer_config.json'
+                    ? { transformed: true }
+                    : {}),
+            })),
+        };
+    }
+    const src = engine === 'ds4' ? manifest.ds4 : manifest.llamaCpp;
+    if (!src)
+        return null;
+    const files = [];
+    if (src.shards && src.shards.length > 0) {
+        for (const shard of src.shards) {
+            files.push({
+                name: basename(shard.name),
+                sha256: shard.sha256,
+                sizeBytes: shard.sizeBytes,
+            });
+        }
+    }
+    else if (src.filename && src.sha256) {
+        files.push({
+            name: basename(src.filename),
+            sha256: src.sha256,
+            sizeBytes: src.approxSizeBytes,
+        });
+    }
+    // Draft companions exist on both engine sources and are fetched
+    // unconditionally by `buildDownloadPlan`, so read them off the already
+    // engine-selected `src` rather than re-narrowing to one engine.
+    if (src.draftModel) {
+        files.push({
+            name: basename(src.draftModel.filename),
+            sha256: src.draftModel.sha256,
+            sizeBytes: src.draftModel.sizeBytes,
+        });
+    }
+    const llamaCpp = engine === 'llama-cpp' ? manifest.llamaCpp : undefined;
+    if (llamaCpp?.mmproj) {
+        // Only fetched when the user opts into vision — its absence says nothing
+        // about whether the weights are current.
+        files.push({
+            name: basename(llamaCpp.mmproj.filename),
+            sha256: llamaCpp.mmproj.sha256,
+            sizeBytes: llamaCpp.mmproj.sizeBytes,
+            optional: true,
+        });
+    }
+    const ds4 = engine === 'ds4' ? manifest.ds4 : undefined;
+    if (ds4?.visionEncoder) {
+        // A ds4 encoder is part of the selected model's advertised capability and
+        // the installer always fetches it. Missing it is a genuine stale/incomplete
+        // payload, not a legitimate text-only variant of the same catalog entry.
+        files.push({
+            name: basename(ds4.visionEncoder.filename),
+            sha256: ds4.visionEncoder.sha256,
+            sizeBytes: ds4.visionEncoder.sizeBytes,
+        });
+    }
+    return files.length > 0 ? { huggingfaceRepo: src.huggingfaceRepo, files } : null;
+}
+export async function comparePayloadIdentity(opts) {
+    const { catalog, installed } = opts;
+    if (catalog.files.length === 0) {
+        return {
+            identity: 'unknown',
+            basis: 'none',
+            reason: 'the catalog entry describes no payload files',
+        };
+    }
+    if (installed.payloadFingerprint) {
+        if (installed.payloadFingerprint === catalogPayloadFingerprint(catalog)) {
+            return {
+                identity: 'same',
+                basis: 'fingerprint',
+                reason: 'the catalog describes the same files this copy was installed from',
+                provenByHash: true,
+            };
+        }
+        // The verdict is settled — the fingerprint covers things the per-file scan
+        // deliberately tolerates (a rotated optional sidecar, a re-pinned chat
+        // template). Walk the files anyway, purely to name one in the message: "the
+        // catalog rebuilt X" is worth far more to the reader than "something moved".
+        return {
+            identity: 'changed',
+            basis: 'fingerprint',
+            reason: firstDifferingFile(catalog, installed.fileSha256) ??
+                'the catalog now describes different model files than this copy holds',
+        };
+    }
+    if (installed.huggingfaceRepo && installed.huggingfaceRepo !== catalog.huggingfaceRepo) {
+        return {
+            identity: 'changed',
+            basis: 'repo',
+            reason: `the catalog now sources this model from ${catalog.huggingfaceRepo} (this copy came from ${installed.huggingfaceRepo})`,
+        };
+    }
+    const recorded = installed.fileSha256;
+    if (recorded && Object.keys(recorded).length > 0) {
+        const difference = firstDifferingFile(catalog, recorded);
+        return difference
+            ? { identity: 'changed', basis: 'hashes', reason: difference }
+            : {
+                identity: 'same',
+                basis: 'hashes',
+                reason: 'every file the catalog pins matches the copy on disk',
+                provenByHash: true,
+            };
+    }
+    const modelDir = opts.modelDir;
+    if (!modelDir) {
+        return {
+            identity: 'unknown',
+            basis: 'none',
+            reason: 'this copy recorded no file hashes and its directory was not supplied',
+        };
+    }
+    const statSize = opts.statSize ?? defaultStatSize;
+    let compared = 0;
+    for (const file of catalog.files) {
+        if (file.sizeBytes === undefined)
+            continue;
+        const size = await statSize(join(modelDir, ...file.name.split('/')));
+        if (size === null) {
+            if (file.optional)
+                continue;
+            return {
+                identity: 'changed',
+                basis: 'sizes',
+                reason: `the catalog now ships ${file.name}, which this copy does not have`,
+            };
+        }
+        if (size !== file.sizeBytes) {
+            return {
+                identity: 'changed',
+                basis: 'sizes',
+                reason: `${file.name} is a different size than the file the catalog now pins`,
+            };
+        }
+        compared += 1;
+    }
+    if (compared === 0) {
+        return {
+            identity: 'unknown',
+            basis: 'none',
+            reason: 'this copy recorded no file hashes and the catalog pins no file sizes',
+        };
+    }
+    return {
+        identity: 'same',
+        basis: 'sizes',
+        reason: `all ${compared} file(s) the catalog pins are present at the expected size`,
+    };
+}
+/**
+ * The first catalog file the recorded digests don't account for, phrased for a
+ * reader, or null when they all check out. A file the install knowingly
+ * rewrote can only be checked for presence, and an optional one may be absent
+ * outright.
+ */
+function firstDifferingFile(catalog, recorded) {
+    if (!recorded || Object.keys(recorded).length === 0)
+        return null;
+    for (const file of catalog.files) {
+        const actual = recorded[file.name];
+        if (actual === undefined) {
+            if (file.optional)
+                continue;
+            return `the catalog now ships ${file.name}, which this copy does not have`;
+        }
+        if (file.transformed)
+            continue;
+        if (actual.toLowerCase() !== file.sha256.toLowerCase()) {
+            return `${file.name} differs from the file the catalog now pins`;
+        }
+    }
+    return null;
+}
+async function defaultStatSize(absolutePath) {
+    return stat(absolutePath)
+        .then((info) => (info.isFile() ? info.size : null))
+        .catch(() => null);
+}

package/dist/models/ensure.d.ts

@@ -0,0 +1,125 @@
+import type { CatalogService } from '@bendyline/gezel-catalog';
+import type { LlamaCppModelManager } from '../providers/llama-cpp/index.js';
+import type { MlxModelManager } from '../providers/mlx/index.js';
+/**
+ * Backends `/v1/models/ensure` understands today. Ollama is intentionally
+ * not yet wired — `/api/ollama/pull` is the current path; deferred to a
+ * later phase.
+ */
+export type EnsureBackend = 'llama-cpp' | 'mlx' | 'ds4';
+/**
+ * One event from an in-flight install job. The shape is a
+ * superset across both backends — fields irrelevant to the active
+ * backend are simply omitted. Apps should branch on `type`.
+ */
+export type EnsureEvent = {
+    type: 'progress';
+    jobId: string;
+    modelId: string;
+    bytesWritten: number;
+    totalBytes: number;
+} | {
+    type: 'verifying';
+    jobId: string;
+    modelId: string;
+    file?: string;
+} | {
+    type: 'extracting-metadata';
+    jobId: string;
+    modelId: string;
+} | {
+    type: 'retrying';
+    jobId: string;
+    modelId: string;
+    attempt: number;
+    maxAttempts: number;
+    delayMs: number;
+    reason: string;
+} | {
+    type: 'done';
+    jobId: string;
+    modelId: string;
+    warning?: string;
+} | {
+    type: 'error';
+    jobId: string;
+    modelId: string;
+    error: string;
+};
+export interface EnsureModelResult {
+    /**
+     * `ready` — the model is already installed; no job started. Callers
+     * can immediately call `POST /v1/chat/completions`.
+     *
+     * `downloading` — an install job is in flight (existing or freshly
+     * started). Poll via `GET /v1/models/ensure/:jobId` or subscribe via
+     * `GET /v1/models/ensure/:jobId/events`.
+     */
+    status: 'ready' | 'downloading';
+    modelId: string;
+    jobId?: string;
+}
+export interface JobSnapshot {
+    jobId: string;
+    modelId: string;
+    status: 'running' | 'done' | 'error';
+    startedAt: number;
+    /**
+     * Cached recent events so a late subscriber can replay history
+     * before live events arrive. Capped to the last 64 to bound memory.
+     */
+    events: EnsureEvent[];
+}
+export interface EnsureModelOrchestrator {
+    /**
+     * Idempotent: if the model is already installed, returns immediately
+     * with `status: 'ready'`. If an install for the same model is in
+     * flight, returns the existing `jobId`. Otherwise starts a new job
+     * and returns its `jobId`. Throws a `KnownEnsureError` for unknown
+     * models, unsupported backends, or unsupported provider sources on
+     * the selected backend (e.g. an MLX-only model requested via the
+     * llama-cpp backend) so the route can map to 404 / 400.
+     */
+    ensure(qualifiedModelId: string): Promise<EnsureModelResult>;
+    getJob(jobId: string): JobSnapshot | null;
+    subscribeJob(jobId: string, listener: (event: EnsureEvent) => void): () => void;
+}
+/**
+ * Error class with a structured `code` so the route can pick the
+ * right HTTP status. Avoids regex-matching error message strings.
+ */
+export declare class KnownEnsureError extends Error {
+    readonly code: 'unknown_model' | 'unsupported_backend' | 'no_source_for_backend' | 'ambiguous_model';
+    constructor(code: KnownEnsureError['code'], message: string);
+}
+export interface CreateEnsureModelOptions {
+    llamaCpp: LlamaCppModelManager;
+    ds4: LlamaCppModelManager;
+    mlx: MlxModelManager;
+    catalog: CatalogService;
+    /**
+     * Fired (fire-and-forget) the moment a fresh install job starts, with
+     * the backend + catalog id. Lets the host kick off engine-side
+     * provisioning that's independent of the weight download — notably
+     * warming the MLX Python venv (uv + mlx-vlm + torch wheels, 1–5 min
+     * cold) in parallel with the download instead of paying it serially
+     * on the first chat turn. Not called when the model is already
+     * installed or an install for it is already in flight. The host owns
+     * error handling — this orchestrator never awaits it.
+     */
+    onInstallStart?: (info: {
+        backend: EnsureBackend;
+        catalogId: string;
+    }) => void;
+}
+/**
+ * Parse a qualified id (`<backend>:<catalogId>`) into its parts. Bare
+ * ids and forward-slash separators are rejected — `/v1/models/ensure`
+ * requires explicit disambiguation so the caller (and the user reading
+ * an error message) can see exactly which backend was selected.
+ */
+export declare function parseQualifiedModelId(qualified: string): {
+    backend: EnsureBackend;
+    catalogId: string;
+} | null;
+export declare function createEnsureModelOrchestrator(opts: CreateEnsureModelOptions): Promise<EnsureModelOrchestrator>;

package/dist/models/ensure.js

@@ -0,0 +1,205 @@
+import { randomUUID } from 'node:crypto';
+/**
+ * Error class with a structured `code` so the route can pick the
+ * right HTTP status. Avoids regex-matching error message strings.
+ */
+export class KnownEnsureError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = 'KnownEnsureError';
+        this.code = code;
+    }
+}
+const RECENT_EVENT_CAP = 64;
+/**
+ * Parse a qualified id (`<backend>:<catalogId>`) into its parts. Bare
+ * ids and forward-slash separators are rejected — `/v1/models/ensure`
+ * requires explicit disambiguation so the caller (and the user reading
+ * an error message) can see exactly which backend was selected.
+ */
+export function parseQualifiedModelId(qualified) {
+    const trimmed = qualified.trim();
+    if (!trimmed)
+        return null;
+    const idx = trimmed.indexOf(':');
+    if (idx < 0)
+        return null;
+    const backendRaw = trimmed.slice(0, idx);
+    const catalogId = trimmed.slice(idx + 1).trim();
+    if (!catalogId)
+        return null;
+    if (backendRaw !== 'llama-cpp' && backendRaw !== 'mlx' && backendRaw !== 'ds4')
+        return null;
+    return { backend: backendRaw, catalogId };
+}
+export async function createEnsureModelOrchestrator(opts) {
+    // Keyed by qualified id for de-duplication: a second `ensure` for the
+    // same model joins the existing job instead of starting a parallel
+    // install (which both backends already reject anyway, but we want a
+    // friendlier behavior at the orchestrator layer).
+    const jobsByModel = new Map();
+    const jobsById = new Map();
+    function emit(job, event) {
+        job.events.push(event);
+        if (job.events.length > RECENT_EVENT_CAP) {
+            job.events.splice(0, job.events.length - RECENT_EVENT_CAP);
+        }
+        for (const listener of job.listeners) {
+            try {
+                listener(event);
+            }
+            catch {
+                // Listener crashes shouldn't kill the job pump.
+            }
+        }
+    }
+    async function isInstalled(backend, catalogId) {
+        const manager = backend === 'llama-cpp' ? opts.llamaCpp : backend === 'ds4' ? opts.ds4 : opts.mlx;
+        const model = await manager.resolveModel(catalogId).catch(() => null);
+        return model !== null;
+    }
+    async function startJob(qualifiedModelId, backend, catalogId) {
+        const jobId = randomUUID();
+        const job = {
+            jobId,
+            modelId: qualifiedModelId,
+            status: 'running',
+            startedAt: Date.now(),
+            events: [],
+            listeners: new Set(),
+        };
+        jobsByModel.set(qualifiedModelId, job);
+        jobsById.set(jobId, job);
+        // Let the host overlap engine-side provisioning with the download
+        // (e.g. warm the MLX venv). Fire-and-forget — never let a host
+        // callback throw into the job-start path.
+        try {
+            opts.onInstallStart?.({ backend, catalogId });
+        }
+        catch {
+            // Host owns its own errors; a throw here must not abort the job.
+        }
+        const manager = backend === 'llama-cpp' ? opts.llamaCpp : backend === 'ds4' ? opts.ds4 : opts.mlx;
+        const iter = manager.install(catalogId);
+        // Drive the install loop in the background. Subscribers get every
+        // event via `emit`; the job's terminal status is set on the last
+        // event of either backend's contract (`done` or `error`).
+        void (async () => {
+            try {
+                for await (const ev of iter) {
+                    const enriched = { ...ev, jobId, modelId: qualifiedModelId };
+                    emit(job, enriched);
+                    if (ev.type === 'done') {
+                        job.status = 'done';
+                    }
+                    else if (ev.type === 'error') {
+                        job.status = 'error';
+                    }
+                }
+                // Defensive: if the iterable closes without emitting a terminal
+                // event, surface a synthetic error so subscribers see a final
+                // state instead of waiting forever.
+                if (job.status === 'running') {
+                    job.status = 'error';
+                    emit(job, {
+                        type: 'error',
+                        jobId,
+                        modelId: qualifiedModelId,
+                        error: 'install loop closed without a terminal event',
+                    });
+                }
+            }
+            catch (err) {
+                job.status = 'error';
+                emit(job, {
+                    type: 'error',
+                    jobId,
+                    modelId: qualifiedModelId,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+            finally {
+                // Free the per-model slot so a follow-up `ensure` can start a
+                // fresh job (e.g. after a failed install the user retries).
+                // The completed-job snapshot stays available by `jobId` for a
+                // while so polling clients can still read it.
+                jobsByModel.delete(qualifiedModelId);
+                // Keep the completed job in `jobsById` for ~10 minutes so SSE
+                // reconnects + late status checks still find it.
+                setTimeout(() => jobsById.delete(jobId), 10 * 60_000).unref?.();
+            }
+        })();
+        return jobId;
+    }
+    return {
+        async ensure(qualifiedModelId) {
+            const parsed = parseQualifiedModelId(qualifiedModelId);
+            if (!parsed) {
+                throw new KnownEnsureError('ambiguous_model', `Expected "<backend>:<catalogId>" — got "${qualifiedModelId}". Call /v1/models for the catalog.`);
+            }
+            const { backend, catalogId } = parsed;
+            // Catalog validation: confirm the model exists and has a source
+            // for the requested backend before either resolving or starting
+            // a job. Catches typos and "MLX-only model on a non-Apple host"
+            // up front instead of as a job error 30s later.
+            const detail = await opts.catalog.get('chat-model', catalogId).catch(() => null);
+            if (!detail || detail.manifest.kind !== 'chat-model') {
+                throw new KnownEnsureError('unknown_model', `chat-model "${catalogId}" is not in the catalog`);
+            }
+            const source = backend === 'llama-cpp'
+                ? detail.manifest.llamaCpp
+                : backend === 'ds4'
+                    ? detail.manifest.ds4
+                    : detail.manifest.mlx;
+            if (!source) {
+                const alternatives = [
+                    detail.manifest.llamaCpp ? `llama-cpp:${catalogId}` : null,
+                    detail.manifest.mlx ? `mlx:${catalogId}` : null,
+                    detail.manifest.ds4 ? `ds4:${catalogId}` : null,
+                ].filter((id) => Boolean(id));
+                throw new KnownEnsureError('no_source_for_backend', `chat-model "${catalogId}" has no ${backend} source${alternatives.length > 0 ? ` — try ${alternatives.map((id) => `"${id}"`).join(' or ')}` : ''}`);
+            }
+            if (await isInstalled(backend, catalogId)) {
+                return { status: 'ready', modelId: qualifiedModelId };
+            }
+            const existing = jobsByModel.get(qualifiedModelId);
+            if (existing) {
+                return { status: 'downloading', modelId: qualifiedModelId, jobId: existing.jobId };
+            }
+            const jobId = await startJob(qualifiedModelId, backend, catalogId);
+            return { status: 'downloading', modelId: qualifiedModelId, jobId };
+        },
+        getJob(jobId) {
+            const job = jobsById.get(jobId);
+            if (!job)
+                return null;
+            return {
+                jobId: job.jobId,
+                modelId: job.modelId,
+                status: job.status,
+                startedAt: job.startedAt,
+                events: [...job.events],
+            };
+        },
+        subscribeJob(jobId, listener) {
+            const job = jobsById.get(jobId);
+            if (!job)
+                return () => { };
+            // Replay buffered events so a late subscriber doesn't miss the
+            // earlier `progress` chunks (same pattern as ChatEventBus).
+            for (const past of job.events)
+                listener(past);
+            // If the job is already terminal, no live events are coming. Tell
+            // the caller via a synthetic empty unsubscribe — the buffered
+            // replay above already delivered the final state.
+            if (job.status !== 'running') {
+                return () => { };
+            }
+            job.listeners.add(listener);
+            return () => {
+                job.listeners.delete(listener);
+            };
+        },
+    };
+}

package/dist/models/gezmodel.d.ts

@@ -0,0 +1,47 @@
+import { type Readable } from 'node:stream';
+import { type GezmodelBundleManifest, type GezmodelEngine, type GezmodelImportProgress, type GezmodelImportReview } from '@bendyline/gezel';
+import type { CatalogService } from '@bendyline/gezel-catalog';
+import type { LlamaCppModelManager } from '../providers/llama-cpp/models.js';
+import type { MlxModelManager } from '../providers/mlx/models.js';
+export interface GezmodelExport {
+    stream: NodeJS.ReadableStream;
+    filename: string;
+    manifest: GezmodelBundleManifest;
+}
+export interface GezmodelScanOptions {
+    importId?: string;
+    bytesTotal?: number;
+    signal?: AbortSignal;
+    onProgress?: (progress: GezmodelImportProgress) => void;
+    onUploadComplete?: () => void;
+}
+export declare class GezmodelManager {
+    private readonly importsRoot;
+    private readonly catalog;
+    private readonly owners;
+    constructor(opts: {
+        home: string;
+        catalog: CatalogService;
+        llamaCpp: LlamaCppModelManager;
+        mlx: MlxModelManager;
+        ds4: LlamaCppModelManager;
+    });
+    /** Hash model files, buffer the small manifests, then stream the ZIP payload. */
+    export(engine: GezmodelEngine, id: string): Promise<GezmodelExport>;
+    /**
+     * Stream an uploaded archive into private transaction staging, inspect its
+     * central directory, extract through hash-verifying streams, then return a
+     * review token. No engine directory is touched here.
+     */
+    scanUpload(input: Readable, opts?: GezmodelScanOptions): Promise<GezmodelImportReview>;
+    /** Re-verify staged bytes after the review pause, then atomically install. */
+    confirmImport(importId: string, replace: boolean): Promise<{
+        engine: GezmodelEngine;
+        id: string;
+    }>;
+    cancelImport(importId: string): Promise<void>;
+    private owner;
+    private stagePath;
+    private loadReview;
+    private cleanupExpired;
+}