@mlx-node/server 0.0.13 → 0.0.15

package/src/errors.ts

@@ -0,0 +1,120 @@
+/** OpenAI- and Anthropic-compatible JSON error responses. */
+
+import type { ServerResponse } from 'node:http';
+
+export interface APIError {
+  type: string;
+  message: string;
+  code: string | null;
+  param: string | null;
+}
+
+export function sendError(
+  res: ServerResponse,
+  status: number,
+  type: string,
+  message: string,
+  param?: string | null,
+): void {
+  const body: { error: APIError } = {
+    error: {
+      type,
+      message,
+      code: null,
+      param: param ?? null,
+    },
+  };
+  res.writeHead(status, { 'Content-Type': 'application/json' });
+  res.end(JSON.stringify(body));
+}
+
+export function sendBadRequest(res: ServerResponse, message: string, param?: string): void {
+  sendError(res, 400, 'invalid_request_error', message, param);
+}
+
+export function sendNotFound(res: ServerResponse, message: string): void {
+  sendError(res, 404, 'not_found_error', message);
+}
+
+export function sendMethodNotAllowed(res: ServerResponse, allowed: string): void {
+  res.writeHead(405, { Allow: allowed, 'Content-Type': 'application/json' });
+  res.end(
+    JSON.stringify({
+      error: { type: 'invalid_request_error', message: 'Method not allowed', code: null, param: null },
+    }),
+  );
+}
+
+export function sendInternalError(res: ServerResponse, message: string): void {
+  sendError(res, 500, 'server_error', message);
+}
+
+/**
+ * 503 with `type: 'storage_timeout'`. Emitted by the responses endpoint when
+ * an in-flight `store.store(...)` gating a `previous_response_id` continuation
+ * fails to settle within `CHAIN_WRITE_WAIT_TIMEOUT_MS` and the final `getChain`
+ * probe still misses. 503 (not 404) because the write may yet land, so the
+ * same id can be retried — a 404 would wrongly mark it permanently invalid.
+ */
+export function sendStorageTimeout(res: ServerResponse, message: string): void {
+  sendError(res, 503, 'storage_timeout', message);
+}
+
+/**
+ * 429 with `type: 'rate_limit_error'` and `code: 'queue_full'`. Emitted by
+ * `/v1/responses` when the per-model execution queue is already holding
+ * `maxQueueDepth` waiters behind the current dispatch. Always sets
+ * `Retry-After: 1` (string seconds) so clients back off briefly before
+ * retrying — short enough to encourage a retry, long enough to avoid
+ * busy-looping the server.
+ */
+export function sendRateLimit(res: ServerResponse, message: string): void {
+  res.writeHead(429, { 'Retry-After': '1', 'Content-Type': 'application/json' });
+  res.end(
+    JSON.stringify({
+      error: {
+        type: 'rate_limit_error',
+        message,
+        code: 'queue_full',
+        param: null,
+      },
+    }),
+  );
+}
+
+export function sendAnthropicError(res: ServerResponse, status: number, type: string, message: string): void {
+  res.writeHead(status, { 'Content-Type': 'application/json' });
+  res.end(JSON.stringify({ type: 'error', error: { type, message } }));
+}
+
+export function sendAnthropicBadRequest(res: ServerResponse, message: string): void {
+  sendAnthropicError(res, 400, 'invalid_request_error', message);
+}
+
+export function sendAnthropicNotFound(res: ServerResponse, message: string): void {
+  sendAnthropicError(res, 404, 'not_found_error', message);
+}
+
+export function sendAnthropicInternalError(res: ServerResponse, message: string): void {
+  sendAnthropicError(res, 500, 'api_error', message);
+}
+
+export function sendAnthropicNotImplemented(res: ServerResponse, message: string): void {
+  sendAnthropicError(res, 501, 'not_supported_error', message);
+}
+
+export function sendAnthropicMethodNotAllowed(res: ServerResponse, allowed: string): void {
+  res.writeHead(405, { Allow: allowed, 'Content-Type': 'application/json' });
+  res.end(JSON.stringify({ type: 'error', error: { type: 'invalid_request_error', message: 'Method not allowed' } }));
+}
+
+/**
+ * 429 Anthropic-shape rate-limit response. Mirror of {@link sendRateLimit}
+ * for `/v1/messages`. Body uses the `{ type: 'error', error: { type, message } }`
+ * envelope the rest of the Anthropic error helpers use; `Retry-After: 1`
+ * is set verbatim so clients can wait one second before retrying.
+ */
+export function sendAnthropicRateLimit(res: ServerResponse, message: string): void {
+  res.writeHead(429, { 'Retry-After': '1', 'Content-Type': 'application/json' });
+  res.end(JSON.stringify({ type: 'error', error: { type: 'rate_limit_error', message } }));
+}

package/src/handler.ts

@@ -0,0 +1,195 @@
+/** Composable `(req, res)` handler for node:http — usable standalone or mounted into an existing server. */
+
+import type { IncomingMessage, ServerResponse } from 'node:http';
+
+import type { ResponseStore } from '@mlx-node/core';
+
+import { hasCredential, isAuthorized, sendUnauthorized } from './auth.js';
+import { sendInternalError } from './errors.js';
+import { createHealthReporter, type ServerHealth } from './health.js';
+import type { IdleSweeper } from './idle-sweeper.js';
+import { ModelWorkCoordinator } from './model-work-coordinator.js';
+import type { ModelRegistry } from './registry.js';
+import { requestPathname, routeRequest } from './router.js';
+
+/**
+ * Routes reachable WITHOUT a token when one is configured.
+ *
+ * A supervisor has to poll liveness before it can be handed a token (it may
+ * be the thing that generates it). Everything served here is scrubbed to
+ * `{ status, uptimeMs, pid }` by the router when the caller is
+ * unauthenticated — see `toMinimalHealth`.
+ *
+ * `/` is included because it is a pure liveness stub: the router answers it
+ * with a constant `{ service: 'mlx-node' }` and reads no state, so there is
+ * nothing to leak. Claude Code issues `HEAD /` before its first request — it
+ * does send `x-api-key`, so it would pass the check anyway, but gating a
+ * contentless probe would 401 every other client's liveness check for no
+ * security gain.
+ */
+const UNAUTHENTICATED_PATHS = new Set(['/', '/health', '/v1/health']);
+
+/**
+ * Entry in the list returned by `GET /v1/models`. Matches the shape
+ * `ModelRegistry.list()` produces — exported so callers that supply a
+ * custom `listModels` callback (e.g. `mlx launch claude` discovering
+ * every model on disk) can build entries without importing internals.
+ */
+export interface PublicModelEntry {
+  id: string;
+  object: 'model';
+  created: number;
+  owned_by: string;
+}
+
+export interface HandlerOptions {
+  /**
+   * Enable CORS headers.
+   *
+   * Default: `true` when no `authToken` is set (historical behaviour), and
+   * `false` once one is — `Access-Control-Allow-Origin: *` on a
+   * token-protected server would invite any web page to spend a leaked token
+   * from the user's browser. An explicit value always wins.
+   */
+  cors?: boolean;
+  /** Response store for previous_response_id support. */
+  store?: ResponseStore | null;
+  /**
+   * Retention (seconds) stamped as `expires_at` when committing a response row.
+   * When omitted, the endpoint falls back to its own default (see `responses.ts`
+   * and `ServerConfig.responseRetentionSec`).
+   */
+  responseRetentionSec?: number;
+  /**
+   * Optional idle sweeper. Forwarded to `routeRequest` so that the
+   * inference endpoints (`/v1/responses` + `/v1/messages`) can bracket
+   * their native-model dispatch with `beginRequest()` / `endRequest()`.
+   *
+   * Note: the begin/end hooks are DELIBERATELY scoped to the inference
+   * endpoints — wrapping every HTTP call (OPTIONS preflights,
+   * `/v1/models`, `/v1/health`, 404s) would let purely observational
+   * traffic keep the allocator pinned forever.
+   */
+  idleSweeper?: IdleSweeper | null;
+  /**
+   * Optional async callback invoked by `/v1/messages` before it looks
+   * the model up in the registry. The callback should register the
+   * model on demand; on return, the endpoint does `registry.get(name)`
+   * and 404s if still unresolved.
+   */
+  resolveModel?: (name: string) => Promise<void>;
+  /**
+   * Coordinates process-wide MLX work so lazy model loads / warmups do not
+   * overlap live inference on another model.
+   */
+  modelWorkCoordinator?: ModelWorkCoordinator;
+  /**
+   * Optional override for `GET /v1/models`. When provided, that endpoint
+   * returns this list instead of `registry.list()`.
+   */
+  listModels?: () => PublicModelEntry[];
+  /**
+   * Shared secret required on every route except `/health` and `/v1/health`.
+   *
+   * `undefined` (the default) disables the gate entirely and is byte-for-byte
+   * identical to the pre-auth behaviour: no header is inspected, no
+   * `WWW-Authenticate` is emitted, and CORS keeps its `true` default.
+   *
+   * Accepted as `x-api-key: <token>` (checked first — Anthropic clients send
+   * it) or `authorization: Bearer <token>` (scheme case-insensitive).
+   */
+  authToken?: string;
+  /**
+   * Builds the `/health` body. Supplied by `createServer` so the HTTP
+   * endpoint and `ServerInstance.health()` share one uptime origin. When
+   * omitted, `createHandler` builds its own reporter from `registry`,
+   * `idleSweeper` and `modelWorkCoordinator`.
+   */
+  health?: () => ServerHealth;
+}
+
+export function createHandler(
+  registry: ModelRegistry,
+  options?: HandlerOptions,
+): (req: IncomingMessage, res: ServerResponse) => Promise<void> {
+  const authToken = options?.authToken;
+  // CORS defaults follow the auth posture; an explicit value still wins.
+  const cors = options?.cors ?? authToken === undefined;
+  const store = options?.store ?? null;
+  const responseRetentionSec = options?.responseRetentionSec;
+  const idleSweeper = options?.idleSweeper ?? null;
+  const resolveModel = options?.resolveModel;
+  const modelWorkCoordinator =
+    options?.modelWorkCoordinator ?? (resolveModel ? new ModelWorkCoordinator(registry.queueDepthLimit) : undefined);
+  const listModels = options?.listModels;
+  const health =
+    options?.health ?? createHealthReporter({ registry, idleSweeper, modelWorkCoordinator: modelWorkCoordinator });
+
+  return async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
+    // The guard spans the WHOLE listener, not just `routeRequest`. `http.createServer`
+    // discards the returned promise, so anything that escapes here is an unhandled
+    // rejection — and Node's default `--unhandled-rejections=throw` turns that into
+    // process death. A crash-by-request is the one failure a request handler must
+    // not have, so nothing before the routing call gets to be outside the try either.
+    try {
+      if (cors) {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+        res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization, x-api-key, anthropic-version');
+
+        if (req.method === 'OPTIONS') {
+          res.writeHead(204);
+          res.end();
+          return;
+        }
+      }
+
+      // Single auth choke point. `routeRequest` is called from exactly one
+      // place (below), so no route can be added that bypasses this.
+      //
+      // `authToken === undefined` short-circuits before any header is touched:
+      // an unprotected server behaves exactly as it did before auth existed.
+      let authenticated = true;
+      if (authToken !== undefined) {
+        authenticated = isAuthorized(req, authToken);
+        if (!authenticated) {
+          // Host-independent: see `requestPathname`. Building the base from the
+          // `Host` header threw on `Host: [`, from a branch only an
+          // UNAUTHENTICATED request reaches — so enabling auth was what made the
+          // server killable by anyone who could open the socket.
+          const path = requestPathname(req);
+          // `/health` degrades to a liveness-only body instead of 401 — but
+          // ONLY when no credential was offered. A caller who presented a
+          // WRONG token gets the 401 it needs to notice the typo.
+          if (!UNAUTHENTICATED_PATHS.has(path) || hasCredential(req)) {
+            sendUnauthorized(res);
+            return;
+          }
+        }
+      }
+
+      // Returning the promise lets tests await the full lifecycle including
+      // post-`res.end()` bookkeeping (e.g. `SessionRegistry.adopt`). `http.createServer`
+      // ignores the return value, so this is transparent to production callers.
+      await routeRequest(
+        req,
+        res,
+        registry,
+        store,
+        responseRetentionSec,
+        idleSweeper,
+        resolveModel,
+        listModels,
+        modelWorkCoordinator,
+        { health, authenticated },
+      );
+    } catch (err: unknown) {
+      const message = err instanceof Error ? err.message : 'Internal server error';
+      if (!res.headersSent) {
+        sendInternalError(res, message);
+      } else {
+        res.end();
+      }
+    }
+  };
+}

package/src/health.ts

@@ -0,0 +1,213 @@
+/**
+ * Server readiness reporting for supervisors (e.g. an Electron app running
+ * the inference server as a child process).
+ *
+ * The old `/health` returned a constant `{ status: 'ok' }`, which cannot
+ * distinguish four states a supervisor genuinely needs to tell apart:
+ *
+ *   - up, nothing loaded          → keep waiting, this is normal
+ *   - up, model resident          → route traffic
+ *   - wedged mid-load             → keep waiting, do NOT restart
+ *   - wedged behind a full queue  → shed load / warn the user
+ *
+ * Every input already exists in-process; this module only exposes it. The
+ * status ladder itself is a PURE function ({@link deriveHealthStatus}) over a
+ * plain record so it can be unit-tested without a server, a registry, or the
+ * native addon.
+ *
+ * IMPORTANT: nothing here may call into `@mlx-node/core`. `/health` is
+ * deliberately excluded from the idle sweeper's `beginRequest`/`endRequest`
+ * bracket (see `HandlerOptions.idleSweeper`), so a native call from this path
+ * would run outside the in-flight accounting the drain timer relies on.
+ * Every field below is read from plain JavaScript state.
+ */
+
+import type { IdleSweeper } from './idle-sweeper.js';
+import type { ModelWorkCoordinator } from './model-work-coordinator.js';
+import type { ModelRegistry } from './registry.js';
+
+/**
+ * Coarse readiness classification. Ordered by precedence in
+ * {@link deriveHealthStatus} — a higher rung wins even when a lower rung's
+ * condition also holds.
+ */
+export type ServerHealthStatus = 'ok' | 'loading' | 'degraded' | 'error';
+
+/**
+ * Outcome of the most recent load bracket, recorded by
+ * {@link ModelWorkCoordinator} in the `finally` of `withModelLoad` /
+ * `withModelLoadInstrumented`.
+ *
+ * Before this existed, a `resolveModel` failure became an HTTP 500 and was
+ * dropped — a supervisor polling after the fact had no way to learn WHY the
+ * server had no resident model.
+ *
+ * Resident HTTP requests bypass the load writer and do not overwrite this
+ * record with a no-op. The `'error'` rung is nevertheless gated on "no
+ * resident models": a server that is answering requests is not in an error
+ * state regardless of what the last load bracket did.
+ */
+export interface ModelLoadRecord {
+  /** Caller-supplied label, normally the model name. `null` when unlabelled. */
+  label: string | null;
+  /** `Date.now()` at writer-lock acquisition (when the load actually began). */
+  startedAt: number;
+  /** `Date.now()` when the bracket settled, success or failure. */
+  finishedAt: number;
+  ok: boolean;
+  /** Message of the thrown error, or `null` on success. */
+  error: string | null;
+}
+
+/** Inputs to the pure status ladder. Deliberately primitive so tests can fixture them. */
+export interface HealthStatusInputs {
+  /** True while a load holds the coordinator's exclusive writer slot. */
+  writerActive: boolean;
+  /** Loads parked waiting for the writer slot. */
+  waitingWriters: number;
+  /** Inference requests currently bracketed by the idle sweeper. */
+  inFlight: number;
+  /** Distinct model names currently registered. */
+  residentModelCount: number;
+  /** True when at least one per-model queue is holding its configured max waiters. */
+  queueSaturated: boolean;
+  /** Most recent load bracket, or `null` if none has settled. */
+  lastLoad: ModelLoadRecord | null;
+}
+
+/** Full readiness body. `{ status: 'ok' }` remains a strict subset of this. */
+export interface ServerHealth {
+  status: ServerHealthStatus;
+  /** Milliseconds since the reporter was created (≈ server start). */
+  uptimeMs: number;
+  pid: number;
+  models: {
+    /** Registered names, including aliases. */
+    resident: string[];
+    count: number;
+  };
+  work: {
+    inFlight: number;
+    /** True while the idle sweeper has a `clearCache()` drain armed. */
+    drainPending: boolean;
+    writerActive: boolean;
+    waitingWriters: number;
+  };
+  queue: {
+    /** Deepest per-model waiter count across every session registry. */
+    depth: number;
+    /** Configured per-model waiter cap, or `null` when unbounded. */
+    limit: number | null;
+    saturated: boolean;
+  };
+  lastLoad: ModelLoadRecord | null;
+}
+
+/**
+ * The subset served to an UNAUTHENTICATED `/health` poll on a
+ * token-protected server. A supervisor must be able to poll before it holds a
+ * token, but `models.resident` is user data — model names routinely leak
+ * project names and local paths.
+ */
+export interface ServerHealthMinimal {
+  status: ServerHealthStatus;
+  uptimeMs: number;
+  pid: number;
+}
+
+/**
+ * Pure status ladder. No I/O, no clock, no native calls — just the four
+ * documented rungs, in precedence order:
+ *
+ *   1. `writerActive`                        → 'loading'
+ *   2. last load failed AND nothing resident → 'error'
+ *   3. queue saturated, OR a load is parked
+ *      behind live inference                 → 'degraded'
+ *   4. otherwise                             → 'ok'
+ *
+ * `'loading'` outranks `'error'` on purpose: a retry that already holds the
+ * writer slot means the supervisor should wait, not restart the process.
+ */
+export function deriveHealthStatus(input: HealthStatusInputs): ServerHealthStatus {
+  if (input.writerActive) return 'loading';
+  if (input.lastLoad?.ok === false && input.residentModelCount === 0) return 'error';
+  if (input.queueSaturated) return 'degraded';
+  // A writer parked while readers are still running means the swap cannot
+  // proceed until they drain, and every request for the incoming model
+  // stalls behind it. Either condition alone is an ordinary transient.
+  if (input.waitingWriters > 0 && input.inFlight > 0) return 'degraded';
+  return 'ok';
+}
+
+/** Project the full body down to the three fields safe to serve without a token. */
+export function toMinimalHealth(health: ServerHealth): ServerHealthMinimal {
+  return { status: health.status, uptimeMs: health.uptimeMs, pid: health.pid };
+}
+
+export interface HealthReporterDeps {
+  registry: ModelRegistry;
+  /** Optional: supplies `inFlight` / `drainPending`. Absent ⇒ both read as idle. */
+  idleSweeper?: IdleSweeper | null;
+  /** Optional: supplies writer state + `lastLoad`. Absent ⇒ no load has ever run. */
+  modelWorkCoordinator?: ModelWorkCoordinator | null;
+  /** Defaults to `Date.now()` at construction. */
+  startedAt?: number;
+  /** Injectable clock for tests. */
+  now?: () => number;
+}
+
+/**
+ * Build a zero-argument reporter closing over the live server objects. Each
+ * call re-reads current state; nothing is cached, so a supervisor polling on
+ * an interval always sees the present moment.
+ */
+export function createHealthReporter(deps: HealthReporterDeps): () => ServerHealth {
+  const now = deps.now ?? ((): number => Date.now());
+  const startedAt = deps.startedAt ?? now();
+
+  return (): ServerHealth => {
+    const resident = deps.registry.list().map((entry) => entry.id);
+
+    // Queue saturation is inherently per-model: one wedged model should
+    // surface even while others are idle. We report the WORST case.
+    let depth = 0;
+    let limit: number | null = null;
+    let saturated = false;
+    for (const sessionRegistry of deps.registry.listSessionRegistries()) {
+      const registryDepth = sessionRegistry.queueDepth;
+      if (registryDepth > depth) depth = registryDepth;
+      const registryLimit = sessionRegistry.queueDepthLimit;
+      if (registryLimit !== undefined) {
+        if (limit === null || registryLimit < limit) limit = registryLimit;
+        if (registryDepth >= registryLimit) saturated = true;
+      }
+    }
+
+    const writerActive = deps.modelWorkCoordinator?.writerActive ?? false;
+    const waitingWriters = deps.modelWorkCoordinator?.waitingWriters ?? 0;
+    const lastLoad = deps.modelWorkCoordinator?.lastLoad ?? null;
+    const inFlight = deps.idleSweeper?.inFlight ?? 0;
+    const drainPending = deps.idleSweeper?.isPending ?? false;
+
+    const status = deriveHealthStatus({
+      writerActive,
+      waitingWriters,
+      inFlight,
+      residentModelCount: resident.length,
+      queueSaturated: saturated,
+      lastLoad,
+    });
+
+    return {
+      status,
+      // Clamped: a backwards clock step (NTP, fake timers in a sibling test)
+      // must not surface a negative uptime a supervisor might read as a wrap.
+      uptimeMs: Math.max(0, now() - startedAt),
+      pid: process.pid,
+      models: { resident, count: resident.length },
+      work: { inFlight, drainPending, writerActive, waitingWriters },
+      queue: { depth, limit, saturated },
+      lastLoad,
+    };
+  };
+}

package/src/host/discover.ts

@@ -0,0 +1,25 @@
+/** Discover locally-downloaded generative models under a given directory. */
+
+import type { LaunchPreset, ModelType } from '@mlx-node/lm/family-data';
+import { discoverLocalChatModels } from '@mlx-node/lm/model-discovery';
+
+/** A locally-downloaded model paired with its sampling preset. */
+export interface DiscoveredModel {
+  name: string;
+  path: string;
+  modelType: ModelType;
+  preset: LaunchPreset;
+}
+
+/**
+ * Use the same checkpoint IDs and paths as the agent and setup UI, including
+ * supported GGUF files and their quant variants. No weights are loaded here.
+ */
+export async function discoverModels(dir: string): Promise<DiscoveredModel[]> {
+  return (await discoverLocalChatModels(dir)).map(({ name, path, modelType, preset }) => ({
+    name,
+    path,
+    modelType,
+    preset,
+  }));
+}

package/src/host/env-policy.ts

@@ -0,0 +1,81 @@
+/**
+ * Engine tuning that is LAUNCHER POLICY, not an engine default.
+ *
+ * The native engine reads these knobs from the process environment through a
+ * `OnceLock`: the FIRST read latches the value for the life of the process, so
+ * every variable here must be in place before any model is loaded and must
+ * never be mutated afterwards. That one-shot latch is why the policy is a
+ * plain data object rather than something a caller can toggle at runtime.
+ *
+ * Two application shapes, deliberately different:
+ *
+ *   - In-process host (`mlx serve`, `mlx launch claude`): call
+ *     {@link applyEnginePolicy} to write `process.env` BEFORE the first load.
+ *     A value the user already set in their shell always wins — this is a
+ *     default, not an override.
+ *   - Out-of-process host (Electron `utilityProcess.fork`): pass
+ *     {@link engineEnvFor} into `fork({ env })` and never touch
+ *     `process.env` in the child. The returned map is unconditional: it
+ *     describes the child's starting environment, and the parent is expected
+ *     to spread the user's own env under it if it wants shell values to win.
+ *
+ * Keeping the two apart matters. `MLX_PAGED_PREFILL_CHUNK_SIZE` reads as
+ * `0` ("no chunking") in Rust when unset; 2048 is the value
+ * `mlx launch claude` has historically applied to bound the cold-prefill
+ * memory peak, and baking it into the engine would silently change every
+ * other embedder's behaviour.
+ */
+
+/** Env var names this module owns. Exported so tests can assert the exact set. */
+export const ENGINE_POLICY_ENV_VARS = ['MLX_PAGED_PREFILL_CHUNK_SIZE'] as const;
+
+export interface EnginePolicy {
+  /**
+   * `MLX_PAGED_PREFILL_CHUNK_SIZE` — tokens per paged-prefill chunk.
+   * `0` disables chunking (the Rust default when the var is unset).
+   */
+  pagedPrefillChunkSize?: number;
+}
+
+/**
+ * The policy `mlx launch claude` has applied since the flag existed, and the
+ * one `mlx serve` and the desktop sidecar inherit.
+ *
+ * 2048 matches the mlx-lm / mlx-vlm default and reduces per-chunk overhead
+ * versus the older 1024 for long Qwen dense contexts.
+ */
+export const LAUNCHER_ENGINE_POLICY: EnginePolicy = Object.freeze({ pagedPrefillChunkSize: 2048 });
+
+/**
+ * Render a policy as the environment map a forked child should start with.
+ *
+ * Unconditional by construction: there is no "already set" to respect in a
+ * child that does not exist yet.
+ */
+export function engineEnvFor(policy: EnginePolicy): Record<string, string> {
+  const env: Record<string, string> = {};
+  if (policy.pagedPrefillChunkSize !== undefined) {
+    env.MLX_PAGED_PREFILL_CHUNK_SIZE = String(policy.pagedPrefillChunkSize);
+  }
+  return env;
+}
+
+/**
+ * Apply a policy to an in-process environment as a DEFAULT.
+ *
+ * Only writes vars that are currently unset — matching the historical
+ * `if (process.env.X == null)` guard, so an explicit `X=` (empty string) in
+ * the user's shell is treated as "the user has an opinion" and left alone.
+ *
+ * Returns the names actually written, so a caller can log or assert on them.
+ */
+export function applyEnginePolicy(policy: EnginePolicy, env: NodeJS.ProcessEnv = process.env): string[] {
+  const applied: string[] = [];
+  for (const [name, value] of Object.entries(engineEnvFor(policy))) {
+    if (env[name] == null) {
+      env[name] = value;
+      applied.push(name);
+    }
+  }
+  return applied;
+}