@phamkhachoabk/dsh-vision-describe-mlx 0.1.0

package/README.md

@@ -0,0 +1,48 @@
+# @phamkhachoabk/dsh-vision-describe-mlx
+
+Service Provider for `ctx.visionDescribe`: a vision-language model held in
+memory on this machine by [MLX](https://github.com/ml-explore/mlx), answering
+what each image shows.
+
+**Apple Silicon only.** MLX is built on Metal; there is no Linux or Intel
+build, so this provider reports itself `unusable` everywhere else and the
+harness keeps running without it.
+
+## Requirements
+
+```
+python3 -m pip install mlx-vlm
+```
+
+The checkpoint downloads itself into the Hugging Face cache on first use —
+about 2.5 GB for the default `mlx-community/Qwen3-VL-4B-Instruct-4bit`. Nothing
+is downloaded until the plugin first starts, and nothing leaves the machine
+afterwards.
+
+Name a virtualenv's interpreter in `python` to keep it out of the system
+environment.
+
+## How it runs
+
+A checkpoint costs seconds to load, so the process is long-lived: one
+`python/sidecar.py` is spawned, loads the model once, and then answers NDJSON
+requests on stdin for the life of the plugin. `warmUp()` starts it at boot so a
+user's first image does not pay for the load.
+
+A process that dies is replaced on the next call under a doubling backoff
+(500 ms to 30 s, ten attempts). A request that outruns its budget takes the
+process with it: the model generates inside a blocking call that cannot be
+asked to stop, so an abandoned one is replaced rather than reused.
+
+## Config
+
+| Key | Default | Meaning |
+| --- | --- | --- |
+| `python` | `python3` | Interpreter owning the MLX install |
+| `model` | `mlx-community/Qwen3-VL-4B-Instruct-4bit` | MLX model id or local checkpoint |
+| `maxTokens` | `512` | Longest description for one image |
+| `startupTimeoutMs` | `900000` | Budget for the first start, which may download the checkpoint |
+| `warmUpOnStart` | `true` | Load the model in the background at boot |
+
+A larger checkpoint answers better and costs more memory; `model` is the only
+change needed to swap one in.

package/lib/engine.d.ts

@@ -0,0 +1,28 @@
+/** The MLX vision-language engine. @module @phamkhachoabk/dsh-vision-describe-mlx */
+import type { Context } from '@deepseek-ai/cordis';
+import { VisionDescribeEngine, type EngineAvailability, type VisionDescribeAttempt, type VisionDescribeInput, type VisionDescribeSpec } from '@phamkhachoabk/dsh-vision-describe';
+import { type MlxSidecarConfig } from './sidecar.ts';
+/**
+ * Whole-image description from a vision-language model held in memory on this
+ * machine. Tier 1 because it is, today, the only engine that can answer the
+ * question at all; a provider for another platform registers beside it.
+ */
+export declare class MlxVisionDescribeEngine extends VisionDescribeEngine {
+    private readonly ctx;
+    private readonly config;
+    readonly id = "mlx-vlm";
+    readonly tier = 1;
+    private readonly sidecar;
+    constructor(ctx: Context, config: MlxSidecarConfig);
+    /** The checkpoint decides what an answer looks like, so it is the build identity. */
+    get version(): string;
+    probe(signal?: AbortSignal): Promise<EngineAvailability>;
+    /**
+     * Describe each image in turn. The model answers one request at a time, so
+     * issuing them serially is what keeps each image's budget meaningful.
+     */
+    describe(inputs: readonly VisionDescribeInput[], spec: VisionDescribeSpec, signal: AbortSignal): Promise<readonly VisionDescribeAttempt[]>;
+    /** Load the model before a user's first image has to wait for it. */
+    warmUp(signal?: AbortSignal): Promise<void>;
+    private describeOne;
+}

package/lib/engine.js

@@ -0,0 +1,68 @@
+/** The MLX vision-language engine. @module @phamkhachoabk/dsh-vision-describe-mlx */
+import { VisionDescribeEngine, } from '@phamkhachoabk/dsh-vision-describe';
+import { MlxSidecar, SidecarTimeout, probeRuntime } from "./sidecar.js";
+/**
+ * Whole-image description from a vision-language model held in memory on this
+ * machine. Tier 1 because it is, today, the only engine that can answer the
+ * question at all; a provider for another platform registers beside it.
+ */
+export class MlxVisionDescribeEngine extends VisionDescribeEngine {
+    ctx;
+    config;
+    id = 'mlx-vlm';
+    tier = 1;
+    sidecar;
+    constructor(ctx, config) {
+        super();
+        this.ctx = ctx;
+        this.config = config;
+        this.sidecar = new MlxSidecar(ctx, config);
+        ctx.effect(() => () => { this.sidecar.stop(); });
+    }
+    /** The checkpoint decides what an answer looks like, so it is the build identity. */
+    get version() {
+        return this.config.model;
+    }
+    async probe(signal) {
+        return probeRuntime(this.ctx, this.config, signal);
+    }
+    /**
+     * Describe each image in turn. The model answers one request at a time, so
+     * issuing them serially is what keeps each image's budget meaningful.
+     */
+    async describe(inputs, spec, signal) {
+        const attempts = [];
+        for (const input of inputs) {
+            attempts.push(await this.describeOne(input, spec, signal));
+        }
+        return attempts;
+    }
+    /** Load the model before a user's first image has to wait for it. */
+    async warmUp(signal) {
+        await this.sidecar.ensure(signal);
+    }
+    async describeOne(input, spec, signal) {
+        const started = Date.now();
+        try {
+            const answer = await this.sidecar.request(input.path, spec.prompt, spec.perImageTimeoutMs, signal);
+            const description = answer.description.trim();
+            return {
+                attachmentId: input.attachmentId,
+                outcome: description === '' ? 'empty' : 'ok',
+                ms: answer.ms,
+                description,
+                warnings: [],
+            };
+        }
+        catch (error) {
+            return {
+                attachmentId: input.attachmentId,
+                outcome: error instanceof SidecarTimeout ? 'timeout' : 'error',
+                ms: Date.now() - started,
+                description: '',
+                warnings: [],
+                detail: String(error),
+            };
+        }
+    }
+}

package/lib/index.d.ts

@@ -0,0 +1,31 @@
+/**
+ * MLX vision-description provider. Registers one engine on `ctx.visionDescribe`
+ * backed by a vision-language model kept loaded on this machine.
+ *
+ * @module @phamkhachoabk/dsh-vision-describe-mlx
+ */
+import type { Context } from '@deepseek-ai/cordis';
+import z from '@deepseek-ai/schemastery';
+export { MlxVisionDescribeEngine } from './engine.ts';
+export { MlxSidecar, SidecarTimeout, probeRuntime, SIDECAR_SCRIPT } from './sidecar.ts';
+export type { MlxSidecarConfig, SidecarReply } from './sidecar.ts';
+/** Cordis plugin name used by loader diagnostics. */
+export declare const name = "vision-describe-mlx";
+/** `ctx.visionDescribe` owns the ladder; `ctx.subprocess` runs the model in the host world. */
+export declare const inject: readonly ["visionDescribe", "subprocess"];
+export interface Config {
+    python: string;
+    model: string;
+    maxTokens: number;
+    graceMs: number;
+    stderrMaxBytes: number;
+    startupTimeoutMs: number;
+    warmUpOnStart: boolean;
+}
+export declare const Config: z<Config>;
+/**
+ * Register the engine for the lifetime of `ctx`.
+ * @param ctx - plugin context; the registration and the process are disposed with it.
+ * @param config - interpreter, model, and process settings.
+ */
+export declare function apply(ctx: Context, config: Config): void;

package/lib/index.js

@@ -0,0 +1,55 @@
+/**
+ * MLX vision-description provider. Registers one engine on `ctx.visionDescribe`
+ * backed by a vision-language model kept loaded on this machine.
+ *
+ * @module @phamkhachoabk/dsh-vision-describe-mlx
+ */
+import z from '@deepseek-ai/schemastery';
+import { MlxVisionDescribeEngine } from "./engine.js";
+export { MlxVisionDescribeEngine } from "./engine.js";
+export { MlxSidecar, SidecarTimeout, probeRuntime, SIDECAR_SCRIPT } from "./sidecar.js";
+/** Cordis plugin name used by loader diagnostics. */
+export const name = 'vision-describe-mlx';
+/** `ctx.visionDescribe` owns the ladder; `ctx.subprocess` runs the model in the host world. */
+export const inject = ['visionDescribe', 'subprocess'];
+export const Config = z.object({
+    python: z.string().default('python3')
+        .description('Interpreter owning the MLX install; name a virtualenv\'s python to use it.'),
+    model: z.string().default('mlx-community/Qwen3-VL-4B-Instruct-4bit')
+        .description('MLX model id or local checkpoint path. Downloaded on first use.'),
+    maxTokens: z.number().step(1).min(16).default(512)
+        .description('Longest description the model may produce for one image.'),
+    graceMs: z.number().step(1).min(1).default(2000)
+        .description('SIGTERM to SIGKILL window for the sidecar process tree.'),
+    stderrMaxBytes: z.number().step(1).min(1024).default(64 * 1024)
+        .description('Diagnostic tail retained from the sidecar stderr.'),
+    startupTimeoutMs: z.number().step(1).min(1000).default(900_000)
+        .description('Budget for the first start, which downloads the checkpoint if the Hugging Face '
+        + 'cache does not already hold it. Later starts only reload from disk.'),
+    warmUpOnStart: z.boolean().default(true)
+        .description('Load the model in the background when the plugin starts.'),
+});
+/**
+ * Register the engine for the lifetime of `ctx`.
+ * @param ctx - plugin context; the registration and the process are disposed with it.
+ * @param config - interpreter, model, and process settings.
+ */
+export function apply(ctx, config) {
+    if (process.platform !== 'darwin' || process.arch !== 'arm64') {
+        // Fail-closed and loud once: the harness keeps running without image
+        // descriptions rather than refusing to boot over an optional capability.
+        ctx.logger.warn('vision-describe-mlx: MLX runs on Apple Silicon only, '
+            + `not ${process.platform}-${process.arch} — image description disabled`);
+        return;
+    }
+    const engine = new MlxVisionDescribeEngine(ctx, config);
+    ctx.effect(() => ctx.visionDescribe.registerEngine(engine));
+    if (!config.warmUpOnStart)
+        return;
+    const controller = new AbortController();
+    ctx.effect(() => () => { controller.abort(); });
+    // Deliberately not awaited: booting must not block on a checkpoint download.
+    void ctx.visionDescribe.warmUp(controller.signal).catch((error) => {
+        ctx.logger.warn(`vision-describe-mlx: warm-up failed: ${String(error)}`);
+    });
+}

package/lib/sidecar.d.ts

@@ -0,0 +1,113 @@
+/** The persistent MLX sidecar process. @module @phamkhachoabk/dsh-vision-describe-mlx */
+import type { Context } from '@deepseek-ai/cordis';
+/**
+ * Path of the bundled Python entry point. It ships as source rather than as a
+ * per-platform binary: there is nothing to compile, and the heavy parts (MLX,
+ * the model weights) live in the user's own Python environment and Hugging
+ * Face cache, not in this package.
+ */
+export declare const SIDECAR_SCRIPT: string;
+/** One answered request. */
+export interface SidecarReply {
+    description: string;
+    ms: number;
+}
+/**
+ * A request that outran its budget. The model generates inside a blocking call
+ * the process cannot be asked to abandon, so this always ends with the process
+ * being replaced rather than reused.
+ */
+export declare class SidecarTimeout extends Error {
+    constructor(ms: number);
+}
+export interface MlxSidecarConfig {
+    /** Interpreter that owns the MLX install; `python3` unless a venv is named. */
+    python: string;
+    /** MLX model id or local checkpoint path. */
+    model: string;
+    maxTokens: number;
+    /** SIGTERM → grace → SIGKILL window for the process tree. */
+    graceMs: number;
+    /** Bounded diagnostic tail kept from the sidecar's stderr. */
+    stderrMaxBytes: number;
+    /** Budget for import, checkpoint download and model load on first start. */
+    startupTimeoutMs: number;
+}
+/**
+ * Ask the bundled script whether its runtime exists, without loading a model.
+ *
+ * Exit 3 is the script's word for "mlx_vlm is not importable", which is the
+ * one failure a user can act on, so it is reported in those terms rather than
+ * as a bare exit code.
+ *
+ * @param ctx - plugin context supplying `ctx.subprocess`.
+ * @param config - interpreter, model and diagnostic limits.
+ * @param signal - cancellation for the probe.
+ * @returns readiness, or the reason this machine cannot run the sidecar.
+ */
+export declare function probeRuntime(ctx: Context, config: MlxSidecarConfig, signal?: AbortSignal): Promise<{
+    kind: 'ready';
+} | {
+    kind: 'unusable';
+    reason: string;
+}>;
+/**
+ * Owns one long-lived model process.
+ *
+ * Every call reuses the same process, so the checkpoint is loaded once for the
+ * life of the plugin rather than once per image. A process that dies is
+ * replaced on the next call under a bounded backoff, and one that has failed
+ * {@link MAX_RESTARTS} times in a row is left alone until the plugin reloads:
+ * a machine without the runtime should say so, not respawn forever.
+ */
+export declare class MlxSidecar {
+    private readonly ctx;
+    private readonly config;
+    private handle;
+    private controller;
+    private starting;
+    private readonly waiting;
+    /** Tail of the in-flight chain; see {@link request}. */
+    private queue;
+    private sequence;
+    private restarts;
+    private blockedUntil;
+    private lastFailure?;
+    private stopped;
+    constructor(ctx: Context, config: MlxSidecarConfig);
+    /** Whether a process is currently loaded and answering. */
+    get running(): boolean;
+    /**
+     * Ensure a loaded process exists, starting one if needed.
+     * @param signal - cancellation for a start that is under way.
+     * @throws when the runtime is missing, the model cannot load, or the backoff
+     * window has not elapsed.
+     */
+    ensure(signal?: AbortSignal): Promise<void>;
+    /**
+     * Describe one image on the loaded process.
+     * @param path - absolute host path of the image.
+     * @param prompt - the resolved instruction.
+     * @param timeoutMs - budget for this one image.
+     * @param signal - cancellation for the whole call.
+     * @returns the model's description and the time it took.
+     * @throws {@link SidecarTimeout} when the budget elapses, or an `Error` when
+     * the process could not be started or died mid-request.
+     */
+    request(path: string, prompt: string, timeoutMs: number, signal: AbortSignal): Promise<SidecarReply>;
+    private send;
+    /** Terminate the process and fail everything still waiting. Safe to call twice. */
+    stop(): void;
+    private replace;
+    private start;
+    private noteFailure;
+    /**
+     * Start the dispatch loop and resolve once the model reports itself loaded.
+     * The loop keeps running for the life of the process, routing each later
+     * line to whoever is waiting on that id.
+     */
+    private readUntilReady;
+    private deliver;
+    /** A process that exits on its own leaves the next call to start a new one. */
+    private onExit;
+}

package/lib/sidecar.js

@@ -0,0 +1,362 @@
+/** The persistent MLX sidecar process. @module @phamkhachoabk/dsh-vision-describe-mlx */
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { z } from 'zod';
+const require = createRequire(import.meta.url);
+/**
+ * Path of the bundled Python entry point. It ships as source rather than as a
+ * per-platform binary: there is nothing to compile, and the heavy parts (MLX,
+ * the model weights) live in the user's own Python environment and Hugging
+ * Face cache, not in this package.
+ */
+export const SIDECAR_SCRIPT = join(dirname(require.resolve('@phamkhachoabk/dsh-vision-describe-mlx/package.json')), 'python', 'sidecar.py');
+const readyEvent = z.object({
+    event: z.literal('ready'),
+    model: z.string(),
+    ms: z.number(),
+});
+const errorEvent = z.object({
+    event: z.literal('error'),
+    detail: z.string(),
+});
+const reply = z.object({
+    id: z.string().nullish(),
+    ok: z.boolean(),
+    description: z.string().optional(),
+    error: z.string().optional(),
+    ms: z.number().optional(),
+});
+const sidecarLine = z.union([readyEvent, errorEvent, reply]);
+/**
+ * A request that outran its budget. The model generates inside a blocking call
+ * the process cannot be asked to abandon, so this always ends with the process
+ * being replaced rather than reused.
+ */
+export class SidecarTimeout extends Error {
+    constructor(ms) {
+        super(`the model did not answer within ${String(ms)}ms`);
+        this.name = 'SidecarTimeout';
+    }
+}
+/**
+ * Ask the bundled script whether its runtime exists, without loading a model.
+ *
+ * Exit 3 is the script's word for "mlx_vlm is not importable", which is the
+ * one failure a user can act on, so it is reported in those terms rather than
+ * as a bare exit code.
+ *
+ * @param ctx - plugin context supplying `ctx.subprocess`.
+ * @param config - interpreter, model and diagnostic limits.
+ * @param signal - cancellation for the probe.
+ * @returns readiness, or the reason this machine cannot run the sidecar.
+ */
+export async function probeRuntime(ctx, config, signal) {
+    if (process.platform !== 'darwin' || process.arch !== 'arm64') {
+        return {
+            kind: 'unusable',
+            reason: `MLX runs on Apple Silicon only, not ${process.platform}-${process.arch}`,
+        };
+    }
+    try {
+        const handle = ctx.subprocess.spawn({
+            argv: [config.python, SIDECAR_SCRIPT, '--model', config.model, '--probe'],
+            cwd: tmpdir(),
+            stdio: {
+                stdin: 'ignore',
+                stdout: { maxBytes: 4096 },
+                stderr: { maxBytes: config.stderrMaxBytes },
+            },
+            graceMs: config.graceMs,
+            ...(signal === undefined ? {} : { signal }),
+        });
+        const outcome = await handle.done;
+        if (outcome.exitCode === 0)
+            return { kind: 'ready' };
+        const tail = handle.collected.stderr?.readFrom(0).text.trim() ?? '';
+        if (outcome.exitCode === 3) {
+            return {
+                kind: 'unusable',
+                reason: `${config.python} cannot import mlx_vlm — install it with `
+                    + `\`${config.python} -m pip install mlx-vlm\``,
+            };
+        }
+        return {
+            kind: 'unusable',
+            reason: `${config.python} could not run the sidecar (exit ${String(outcome.exitCode)})`
+                + (tail === '' ? '' : `: ${tail}`),
+        };
+    }
+    catch (error) {
+        return { kind: 'unusable', reason: `${config.python} could not be spawned: ${String(error)}` };
+    }
+}
+/** Restart policy, mirroring the one the harness's MCP client uses for its servers. */
+const BACKOFF_BASE_MS = 500;
+const BACKOFF_MAX_MS = 30_000;
+const MAX_RESTARTS = 10;
+/**
+ * Owns one long-lived model process.
+ *
+ * Every call reuses the same process, so the checkpoint is loaded once for the
+ * life of the plugin rather than once per image. A process that dies is
+ * replaced on the next call under a bounded backoff, and one that has failed
+ * {@link MAX_RESTARTS} times in a row is left alone until the plugin reloads:
+ * a machine without the runtime should say so, not respawn forever.
+ */
+export class MlxSidecar {
+    ctx;
+    config;
+    handle;
+    controller;
+    starting;
+    waiting = new Map();
+    /** Tail of the in-flight chain; see {@link request}. */
+    queue = Promise.resolve();
+    sequence = 0;
+    restarts = 0;
+    blockedUntil = 0;
+    lastFailure;
+    stopped = false;
+    constructor(ctx, config) {
+        this.ctx = ctx;
+        this.config = config;
+    }
+    /** Whether a process is currently loaded and answering. */
+    get running() {
+        return this.handle !== undefined;
+    }
+    /**
+     * Ensure a loaded process exists, starting one if needed.
+     * @param signal - cancellation for a start that is under way.
+     * @throws when the runtime is missing, the model cannot load, or the backoff
+     * window has not elapsed.
+     */
+    async ensure(signal) {
+        if (this.stopped)
+            throw new Error('the sidecar was stopped');
+        if (this.handle !== undefined)
+            return;
+        if (this.restarts >= MAX_RESTARTS) {
+            throw new Error(`gave up after ${String(MAX_RESTARTS)} failed starts: ${this.lastFailure ?? 'no detail'}`);
+        }
+        const wait = this.blockedUntil - Date.now();
+        if (wait > 0) {
+            throw new Error(`still in the restart backoff for ${String(wait)}ms: ${this.lastFailure ?? 'no detail'}`);
+        }
+        this.starting ??= this.start(signal).finally(() => { this.starting = undefined; });
+        await this.starting;
+    }
+    /**
+     * Describe one image on the loaded process.
+     * @param path - absolute host path of the image.
+     * @param prompt - the resolved instruction.
+     * @param timeoutMs - budget for this one image.
+     * @param signal - cancellation for the whole call.
+     * @returns the model's description and the time it took.
+     * @throws {@link SidecarTimeout} when the budget elapses, or an `Error` when
+     * the process could not be started or died mid-request.
+     */
+    async request(path, prompt, timeoutMs, signal) {
+        // One at a time. The model answers serially, so a second caller's budget
+        // must not start ticking while the first is still generating — a queued
+        // request would time out and take the running one's process with it.
+        const turn = this.queue.then(async () => this.send(path, prompt, timeoutMs, signal));
+        this.queue = turn.then(() => undefined, () => undefined);
+        return turn;
+    }
+    async send(path, prompt, timeoutMs, signal) {
+        if (signal.aborted)
+            throw new Error('cancelled');
+        await this.ensure(signal);
+        const handle = this.handle;
+        if (handle?.stdin === undefined)
+            throw new Error('the sidecar has no stdin to write to');
+        this.sequence += 1;
+        const id = String(this.sequence);
+        const answered = new Promise((resolve, reject) => {
+            this.waiting.set(id, { resolve, reject });
+        });
+        handle.stdin.write(`${JSON.stringify({
+            id, path, prompt, maxTokens: this.config.maxTokens,
+        })}\n`);
+        let timer;
+        const expired = new Promise((_resolve, reject) => {
+            timer = setTimeout(() => { reject(new SidecarTimeout(timeoutMs)); }, timeoutMs);
+        });
+        const aborted = new Promise((_resolve, reject) => {
+            if (signal.aborted) {
+                reject(new Error('cancelled'));
+                return;
+            }
+            signal.addEventListener('abort', () => { reject(new Error('cancelled')); }, { once: true });
+        });
+        try {
+            return await Promise.race([answered, expired, aborted]);
+        }
+        catch (error) {
+            // A process mid-generation cannot be handed the next request, so an
+            // abandoned one is replaced rather than reused.
+            this.replace(String(error));
+            throw error;
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+            this.waiting.delete(id);
+        }
+    }
+    /** Terminate the process and fail everything still waiting. Safe to call twice. */
+    stop() {
+        this.stopped = true;
+        this.replace('the sidecar was stopped');
+    }
+    replace(reason) {
+        const handle = this.handle;
+        this.handle = undefined;
+        this.controller?.abort();
+        this.controller = undefined;
+        handle?.terminate();
+        for (const [id, waiter] of this.waiting) {
+            this.waiting.delete(id);
+            waiter.reject(new Error(reason));
+        }
+    }
+    async start(signal) {
+        const controller = new AbortController();
+        if (signal !== undefined) {
+            signal.addEventListener('abort', () => { controller.abort(); }, { once: true });
+        }
+        const handle = this.ctx.subprocess.spawn({
+            argv: [this.config.python, SIDECAR_SCRIPT, '--model', this.config.model],
+            cwd: tmpdir(),
+            stdio: {
+                stdin: 'pipe',
+                stdout: 'pipe',
+                stderr: { maxBytes: this.config.stderrMaxBytes },
+            },
+            graceMs: this.config.graceMs,
+            signal: controller.signal,
+        });
+        const loaded = this.readUntilReady(handle);
+        let timer;
+        const expired = new Promise((_resolve, reject) => {
+            timer = setTimeout(() => { reject(new Error(`the model did not load within ${String(this.config.startupTimeoutMs)}ms`)); }, this.config.startupTimeoutMs);
+        });
+        try {
+            await Promise.race([loaded, expired]);
+            this.handle = handle;
+            this.controller = controller;
+            this.restarts = 0;
+            this.blockedUntil = 0;
+            delete this.lastFailure;
+        }
+        catch (error) {
+            controller.abort();
+            handle.terminate();
+            this.noteFailure(String(error));
+            throw error;
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        }
+    }
+    noteFailure(detail) {
+        this.restarts += 1;
+        this.lastFailure = detail;
+        const backoff = Math.min(BACKOFF_BASE_MS * 2 ** (this.restarts - 1), BACKOFF_MAX_MS);
+        this.blockedUntil = Date.now() + backoff;
+    }
+    /**
+     * Start the dispatch loop and resolve once the model reports itself loaded.
+     * The loop keeps running for the life of the process, routing each later
+     * line to whoever is waiting on that id.
+     */
+    async readUntilReady(handle) {
+        const stream = handle.stdout;
+        if (stream === undefined)
+            throw new Error('the sidecar has no stdout to read from');
+        return new Promise((resolve, reject) => {
+            let settled = false;
+            const fail = (error) => {
+                if (settled)
+                    return;
+                settled = true;
+                reject(error);
+            };
+            const pump = async () => {
+                let buffer = '';
+                for await (const chunk of stream) {
+                    buffer += String(chunk);
+                    let newline = buffer.indexOf('\n');
+                    while (newline !== -1) {
+                        const line = buffer.slice(0, newline).trim();
+                        buffer = buffer.slice(newline + 1);
+                        if (line !== '') {
+                            const parsed = sidecarLine.safeParse(JSON.parse(line));
+                            if (!parsed.success) {
+                                fail(new Error(`unparsable sidecar line: ${line.slice(0, 200)}`));
+                            }
+                            else if ('event' in parsed.data) {
+                                if (parsed.data.event === 'error') {
+                                    fail(new Error(parsed.data.detail));
+                                }
+                                else if (!settled) {
+                                    settled = true;
+                                    resolve();
+                                }
+                            }
+                            else {
+                                this.deliver(parsed.data);
+                            }
+                        }
+                        newline = buffer.indexOf('\n');
+                    }
+                }
+            };
+            void pump().catch((error) => { fail(new Error(String(error))); });
+            void handle.done.then((outcome) => {
+                const tail = handle.collected.stderr?.readFrom(0).text.trim() ?? '';
+                const detail = `sidecar exited with code ${String(outcome.exitCode)}`
+                    + (tail === '' ? '' : `: ${tail}`);
+                fail(new Error(detail));
+                this.onExit(detail);
+            }, (error) => {
+                fail(new Error(String(error)));
+                this.onExit(String(error));
+            });
+        });
+    }
+    deliver(message) {
+        const id = message.id ?? undefined;
+        if (id === undefined) {
+            this.ctx.logger.warn(`vision-describe-mlx: sidecar answered without an id: ${message.error ?? ''}`);
+            return;
+        }
+        const waiter = this.waiting.get(id);
+        if (waiter === undefined)
+            return;
+        this.waiting.delete(id);
+        if (message.ok && message.description !== undefined) {
+            waiter.resolve({ description: message.description, ms: message.ms ?? 0 });
+            return;
+        }
+        waiter.reject(new Error(message.error ?? 'the sidecar reported a failure with no detail'));
+    }
+    /** A process that exits on its own leaves the next call to start a new one. */
+    onExit(detail) {
+        if (this.handle === undefined)
+            return;
+        this.handle = undefined;
+        this.controller = undefined;
+        for (const [id, waiter] of this.waiting) {
+            this.waiting.delete(id);
+            waiter.reject(new Error(detail));
+        }
+        if (this.stopped)
+            return;
+        this.noteFailure(detail);
+        this.ctx.logger.warn(`vision-describe-mlx: ${detail}`);
+    }
+}

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@phamkhachoabk/dsh-vision-describe-mlx",
+  "version": "0.1.0",
+  "description": "Local vision-description provider for DeepSeek Harness: a persistent MLX vision-language model on Apple Silicon, so a text-only model can be told what an image shows",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/",
+    "python/",
+    "README.md"
+  ],
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "zod": "^4.4.3",
+    "@phamkhachoabk/dsh-vision-describe": "^0.1.0"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "^4.0.2",
+    "@deepseek-ai/dsh-subprocess": "^0.1.5-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.2"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.2",
+    "@deepseek-ai/dsh-attachment": "^0.1.5-rc.1",
+    "@deepseek-ai/dsh-subprocess": "^0.1.5-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.2"
+  }
+}

package/python/sidecar.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+"""Persistent MLX vision-language sidecar for @phamkhachoabk/dsh-vision-describe-mlx.
+
+Loads one vision-language model into memory and then answers NDJSON requests on
+stdin with NDJSON responses on stdout, for as long as the parent keeps it alive.
+Loading a multi-gigabyte checkpoint costs seconds, so the process is long-lived
+by design: the parent pays that once, not once per image.
+
+Protocol (one JSON object per line, both directions):
+  <- {"id": "1", "path": "/abs/image.png", "prompt": "...", "maxTokens": 512}
+  -> {"id": "1", "ok": true, "description": "...", "ms": 1234}
+  -> {"id": "1", "ok": false, "error": "..."}
+Startup emits {"event": "ready", "model": "...", "ms": 1234} once, or exits.
+
+Exit codes: 2 usage, 3 runtime missing (mlx_vlm not importable), 4 model load
+failed. The parent turns each into a distinct `unusable` reason.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+
+
+def emit(payload):
+    sys.stdout.write(json.dumps(payload, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def fail(code, detail):
+    emit({"event": "error", "detail": detail})
+    sys.exit(code)
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(prog="dsh-vision-describe-mlx", add_help=True)
+    parser.add_argument("--model", required=True, help="MLX model id or local path")
+    parser.add_argument("--probe", action="store_true", help="load nothing; report importability and exit")
+    try:
+        return parser.parse_args(argv)
+    except SystemExit:
+        raise SystemExit(2)
+
+
+def main(argv):
+    args = parse_args(argv)
+
+    try:
+        import mlx_vlm
+        from mlx_vlm import generate, load
+        from mlx_vlm.prompt_utils import apply_chat_template
+        from mlx_vlm.utils import load_config
+    except Exception as error:  # noqa: BLE001 - any import failure means the same thing
+        fail(3, f"mlx_vlm is not importable: {error}")
+
+    version = getattr(mlx_vlm, "__version__", "unknown")
+    if args.probe:
+        emit({"event": "ready", "model": args.model, "ms": 0, "probe": True, "mlxVlm": version})
+        return 0
+
+    started = time.monotonic()
+    try:
+        model, processor = load(args.model)
+        config = load_config(args.model)
+    except Exception as error:  # noqa: BLE001 - a download or checkpoint failure reads alike
+        fail(4, f"could not load {args.model}: {error}")
+    emit({
+        "event": "ready",
+        "model": args.model,
+        "ms": int((time.monotonic() - started) * 1000),
+        "mlxVlm": version,
+    })
+
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            request = json.loads(line)
+        except ValueError as error:
+            emit({"id": None, "ok": False, "error": f"unparsable request: {error}"})
+            continue
+
+        request_id = request.get("id")
+        began = time.monotonic()
+        try:
+            path = request["path"]
+            if not os.path.isfile(path):
+                raise FileNotFoundError(path)
+            formatted = apply_chat_template(
+                processor, config, request["prompt"], num_images=1,
+            )
+            # `generate` opens the image itself and wants paths, not pixels.
+            output = generate(
+                model, processor, formatted, image=[path],
+                max_tokens=int(request.get("maxTokens", 512)), verbose=False,
+            )
+            # 0.7 returns a GenerationResult carrying the text plus usage stats.
+            text = getattr(output, "text", output)
+            emit({
+                "id": request_id,
+                "ok": True,
+                "description": str(text).strip(),
+                "ms": int((time.monotonic() - began) * 1000),
+            })
+        except Exception as error:  # noqa: BLE001 - one bad image must not end the process
+            emit({
+                "id": request_id,
+                "ok": False,
+                "error": f"{type(error).__name__}: {error}",
+                "ms": int((time.monotonic() - began) * 1000),
+            })
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))