@mlx-bun/inference 0.0.0-stage → 1.0.0

package/src/contracts/mlx/media.ts

@@ -0,0 +1,113 @@
+import type { MlxArray } from '@mlx-bun/mlx/array';
+import type { Cache } from './cache';
+
+/** Prepared embeddings for vision/audio prompts. Qualified models consume
+ * this through the media prompt input the graph binds (`bindMediaInput`).
+ * Token-only prefix caching remains disabled for media. */
+export type Vision = {
+  /** Producer identity includes all media and preceding rendered tokens. */
+  prefixIdentity?: string;
+  embeddings: MlxArray;
+  /** bool [L] image-token mask for the bidirectional attention overlay.
+   *  Absent when the prompt carries ANY audio — audio(-containing) prompts
+   *  run fully causal (`02d723a:docs/design/generic-model-support.md` §6.6). */
+  imageMask?: MlxArray;
+  /** bool [L] union multimodal soft-token mask (image | audio) for
+   *  per-layer-input id zeroing. Absent on the legacy vision-only shape,
+   *  where zeroing falls back to imageMask. */
+  multimodalMask?: MlxArray;
+  /** Qwen3.5/3.8 vision: the request's mRoPE positions + decode delta,
+   *  owned by the media prompt input the graph binds. */
+  mrope?: import("./positions").MropeRequestState;
+};
+
+/** Returns unscaled text embeddings; the caller owns the returned tensor. */
+export interface TextEmbeddingModel {
+  readonly embed: { encode(ids: MlxArray): MlxArray };
+}
+export interface AudioEncoder {
+  readonly cacheIdentity?: string;
+  readonly embedScale: number;
+  features(mel: Float32Array, frames: number, preDivide?: boolean): MlxArray;
+}
+
+/** Prepared native input with an indivisible prefill. The preparation owner
+ * retains its tensors; the graph's binding returns an owned final hidden row.
+ * Decode, sampling and row retirement use the ordinary execution group. */
+export interface MlxPromptInput {
+  /** The vision prefill phase: one sequence's prompt from position `start`,
+   * the prepared embeddings at 0 and its token ids after a reused prefix, at
+   * the request's positions. `phase` and `width` are the scheduler's: the
+   * planner's name for the piece (one position after a reused prefix appends
+   * as a decode, more as a tail) and the class of its rows. Returns the hidden
+   * of the prompt's last position. */
+  prefill(ids: MlxArray, caches: Cache[], start: number, phase: "tail" | "decode", width: number): MlxArray;
+  readonly decodeState?: MlxDecodeState;
+}
+
+/** Additional graph-owned decode state. Equal keys declare compatible row
+ * state; the backend supplies the current row order after joins/retirement. */
+export interface MlxDecodeState {
+  readonly key: string;
+  /** One new token per row at the positions each row's state names; `width`
+   * is the class of the rows, named by the scheduler. */
+  decode(ids: MlxArray, caches: Cache[], rows: readonly MlxDecodeState[], width: number): MlxArray;
+}
+
+/** Common contract for image encoders: preprocess image bytes into an
+ * encoder-specific representation, then turn it into language-space soft
+ * tokens [1, softTokens, hidden] (pre-divided by embed_scale). Encoders own
+ * their preprocessing. */
+export interface VisionEncoder<P extends { softTokens: number } = { softTokens: number }> {
+  readonly cacheIdentity?: string;
+  preprocess(bytes: Uint8Array): Promise<P>;
+  features(pre: P): MlxArray;
+  dispose?(): void;
+}
+
+/** Raw image pixels that a denoising method takes as its own prefill input. */
+export interface PixelInput {
+  /** Pixels are owned by the caller once returned. */
+  preprocess(bytes: Uint8Array): Promise<{ pixels: MlxArray; softTokens: number }>;
+  /** Insert `softTokens` image placeholders (one image) into rendered prompt ids. */
+  spliceTokens(rawIds: number[], softTokens: number, tokenIds: { image: number; boi: number; eoi: number }): number[];
+}
+
+/** Lazy image/audio encoders loaded from a checkpoint's sidecars; null where
+ * the checkpoint ships none. Loading is deferred until a request needs it. */
+export interface MediaEncoders {
+  vision: (() => VisionEncoder) | null;
+  audio: (() => AudioEncoder & { dispose?(): void }) | null;
+}
+
+/** Facts about a checkpoint's files that only the caller's storage can answer. */
+export interface MediaSidecarProbes {
+  /** Whether the sidecar at `path` ships the audio encoder's tensors (header-only read). */
+  shipsAudioTower(path: string): boolean;
+}
+
+
+/** Fixed embedding prefill over prepared image/audio embeddings. */
+export type MlxEmbeddingsForward = (embeddings: MlxArray, state: Cache[], imageMask: MlxArray | null,
+  ids: MlxArray, multimodalMask: MlxArray | null) => MlxArray;
+export interface MlxEmbeddingsRange {
+  readonly from: number;
+  readonly to: number;
+  readonly operation: MlxEmbeddingsForward;
+}
+
+/** Fixed token forward at request-owned rotary positions. */
+export type MlxPositionedForward = (ids: MlxArray, state: Cache[], positions: MlxArray) => MlxArray;
+export interface MlxPositionedRange {
+  readonly from: number;
+  readonly to: number;
+  readonly operation: MlxPositionedForward;
+}
+
+/** Fixed embedding prefill at request-owned rotary positions. */
+export type MlxPositionedEmbeddingsForward = (embeddings: MlxArray, state: Cache[], positions: MlxArray) => MlxArray;
+export interface MlxPositionedEmbeddingsRange {
+  readonly from: number;
+  readonly to: number;
+  readonly operation: MlxPositionedEmbeddingsForward;
+}

package/src/contracts/mlx/positions.ts

@@ -0,0 +1,16 @@
+import type { MlxArray } from "@mlx-bun/mlx/array";
+
+export interface MropeRequestState {
+  /** Full-grid positions for the prompt, one Int32Array per t/h/w stream. */
+  positions: [Int32Array, Int32Array, Int32Array];
+  /** (max position + 1) - promptLength; decode positions = offset + delta. */
+  delta: number;
+}
+
+export interface MropeForwardState {
+  /** Effective positions [3, B, L] int32 for the current forward window. */
+  posIds: MlxArray;
+  /** Borrowed model-owned f32 inv_freq (mropeInvFreq) — not disposed here. */
+  invFreq: MlxArray;
+  rotaryDims: number;
+}

package/src/contracts/mlx/row-table.ts

@@ -0,0 +1,2 @@
+/** Authored exact positive row entries and, only when explicitly written, a default operation. */
+export type MlxRowTable<Operation> = Readonly<Partial<Record<number, Operation>> & { readonly default?: Operation }>;

package/src/contracts/mlx/trainable.ts

@@ -0,0 +1,103 @@
+import type { MlxArray } from "@mlx-bun/mlx/array";
+import type * as ops from "@mlx-bun/mlx/ops";
+import type { SharedKv } from "./cache";
+
+/** The quantized LM head and final-logit softcap the fused linear-CE heads
+ *  read. The head is not a LoRA target, so the base weights are authoritative. */
+export interface HeadQuant {
+  w: MlxArray; scales: MlxArray; biases: MlxArray | null;
+  spec: ops.QuantSpec; softcap: number | null;
+}
+
+/** Layout of a prefix-shared concat [prompt(P); chosen(Rc); rejected(Rr)]. */
+export interface PrefixLayout { P: number; Rc: number; Rr: number }
+
+/** A LoRA leaf pair as the trainer holds it (`LoraWeights`). */
+export interface LoraLeaves { a: MlxArray; b: MlxArray; scale: number; rank: number }
+
+/** One sequence streamed through the layer stack in segments. Borrows the ids
+ *  it began with; owns its caches, masks and per-layer inputs until `dispose`. */
+export interface SegmentedPass {
+  /** The input embeddings [1, T, hidden] (the first segment boundary). The
+   *  caller owns and disposes it. */
+  readonly input: MlxArray;
+  /** Layers `[lo, hi)` over `h`, returning the residual stream after layer
+   *  hi-1 (no final norm) and the fetched K/V of the reused donors in the range.
+   *  `h` is never disposed; `donorKvIn` holds earlier segments' donor K/V
+   *  (caller-owned). The caller owns and disposes what comes back. */
+  runRange(h: MlxArray, lo: number, hi: number, donorKvIn: Map<number, SharedKv>):
+    { h: MlxArray; donorKvOut: Map<number, SharedKv> };
+  /** Release the pass's caches and masks. */
+  dispose(): void;
+}
+
+/** Backward that streams a sequence layer-segment by layer-segment. The driver
+ *  owns the autograd (per-segment vjps over detached boundaries); the graph
+ *  owns how a range of its layers runs and what crosses segment boundaries. */
+export interface SegmentedTraining {
+  readonly layerCount: number;
+  /** K/V a layer reuses from an earlier layer of the same forward.
+   *  `donorOf[i]` is the layer whose fetched K/V layer `i` consumes, or null
+   *  when `i` attends its own; `reusedDonors` are the layers others consume,
+   *  whose K/V must cross segment boundaries. Both are empty/null for a graph
+   *  without KV sharing. */
+  readonly sharedKv: { readonly reusedDonors: ReadonlySet<number>; readonly donorOf: readonly (number | null)[] };
+  /** The final norm applied to the last boundary before the LM head. */
+  finalNorm(h: MlxArray): MlxArray;
+  /** Begin one pass over `ids` [1, T]. With a `prefix` layout the pass ropes and
+   *  masks the prefix-shared concat (only for a graph that also declares
+   *  `prefixShared`); the pass's caches own the layout (`TrainingAttentionCache.ropePositions`). */
+  begin(ids: MlxArray, prefix?: PrefixLayout): SegmentedPass;
+}
+
+/** One forward over a prefix-shared concat instead of two [prompt; response]
+ *  forwards: block-sparse mask plus block-wise rope, bit-exact with the pair. */
+export interface PrefixSharedTraining {
+  /** Post-final-norm hidden [1, T, hidden] of the concat `ids` under `layout`.
+   *  Borrows `ids`; the caller owns the result. */
+  prefixSharedHidden(ids: MlxArray, layout: PrefixLayout): MlxArray;
+}
+
+/** A gradient-checkpointed run of one training loop. */
+export interface GradCheckpointRun {
+  /** Free the checkpoints the last backward held (after its grads are eval'd). */
+  releaseStep(): void;
+  /** Forward-only evaluation: switch checkpointing off / back on. */
+  suspend(): void;
+  resume(): void;
+  /** Switch checkpointing off and free everything it holds. */
+  end(): void;
+}
+
+/** Recompute layer activations in the backward pass (numerically identical to
+ *  off: a memory-for-compute trade). */
+export interface GradCheckpointTraining {
+  /** Enable for the adapter's targets; `splitMlp` also checkpoints each layer's
+   *  attention and MLP sub-blocks separately. */
+  enable(targets: readonly { modulePath: string; lw: LoraLeaves }[], splitMlp: boolean): GradCheckpointRun;
+}
+
+/** Whether the hand-rolled flash kernel may serve the training attention
+ *  (`MLX_BUN_TRAIN_ATTN=flash`). A refusal states why. */
+export type FlashTrainingAttention = { readonly supported: true } | { readonly supported: false; readonly reason: string };
+
+/** The denoising objective of a `method: "denoising"` graph. */
+export interface DenoisingTraining {
+  readonly vocabSize: number;
+  /** LoRA module path of each decoder layer (`<path>.self_attn.q_proj`, ...). */
+  readonly layerModulePaths: readonly string[];
+  /** Logits [1, L, vocab] (float32) of denoising `canvas` [1, L] given the prompt. */
+  canvasLogits(promptIds: number[], canvas: MlxArray): MlxArray;
+}
+
+/** What the trainer consumes from a graph. A graph declares only the parts it
+ *  implements; the trainer names the missing declaration when it needs one. */
+export interface TrainableGraph {
+  /** The quantized LM head for fused/flash linear-CE heads. */
+  lmHead?(): HeadQuant;
+  readonly flashAttention?: FlashTrainingAttention;
+  readonly segmented?: SegmentedTraining;
+  readonly prefixShared?: PrefixSharedTraining;
+  readonly gradCheckpoint?: GradCheckpointTraining;
+  readonly denoising?: DenoisingTraining;
+}

package/src/contracts/mlx/whisper.ts

@@ -0,0 +1,96 @@
+import type { MlxArray } from "@mlx-bun/mlx/array";
+import type { Dtype } from "@mlx-bun/mlx/ffi";
+
+/** Encoder/decoder geometry a speech graph reads from its checkpoint. */
+export interface WhisperDims {
+  nMels: number;
+  nAudioCtx: number;
+  nAudioState: number;
+  nAudioHead: number;
+  nAudioLayer: number;
+  nVocab: number;
+  nTextCtx: number;
+  nTextState: number;
+  nTextHead: number;
+  nTextLayer: number;
+}
+
+/** Decoder state one decode owns (self-attention K/V and the cross K/V from the encoder output).
+ * The graph that made it is the only one that reads it; the transcription loop only advances,
+ * reorders and releases it. */
+export interface WhisperDecoderCache {
+  /** Tokens already in the self-attention cache. */
+  readonly offset: number;
+  /** Beam reorder: keep rows `indices` (take along the batch axis). */
+  rearrange(indices: number[]): void;
+  dispose(): void;
+}
+
+/** The optimized path's cache: its self-attention K/V arrays per layer, so a pipelined loop can
+ * schedule their evaluation with the step outputs. */
+export interface WhisperFastCache extends WhisperDecoderCache {
+  readonly k: readonly MlxArray[];
+  readonly v: readonly MlxArray[];
+}
+
+/** Token filters the optimized decode steps apply on device. */
+export interface FastFilterConfig {
+  nVocab: number;
+  eot: number;
+  timestampBegin: number;
+  noTimestamps: number;
+  /** Ids masked at every step (non-speech + task/special tokens). */
+  suppressIds: number[];
+  /** Ids masked at the first sampled position (blank + eot), or null. */
+  blankIds: number[] | null;
+  useTimestampRules: boolean;
+  maxInitialTimestampIndex: number | null;
+}
+
+/** The optimized execution path of a speech graph (temperature-0 decoding, beam search). Every step
+ * returns owned arrays. */
+export interface WhisperFastPath {
+  makeCache(): WhisperFastCache;
+  /** `audioCtx` (Lab): encode only the first 2·audioCtx mel frames. */
+  encode(mel: MlxArray, audioCtx?: number | null): MlxArray;
+  /** Project the encoder output into every decoder layer's cross K/V. */
+  crossKv(features: MlxArray, cache: WhisperFastCache): void;
+  /** Multi-token prefill: fills the self-KV cache, returns logits [B, T, V]. */
+  prefill(tokens: MlxArray, cache: WhisperFastCache): MlxArray;
+  /** Pipelined greedy step over token and timestamp-state arrays. */
+  greedyStep(
+    st: { tok: MlxArray; lastTok: MlxArray; penultTok: MlxArray; lastTs: MlxArray },
+    cache: WhisperFastCache, cfg: FastFilterConfig,
+  ): { pre: MlxArray; next: MlxArray; lp: MlxArray; lastTs: MlxArray };
+  beamStep(
+    tokens: number[], cache: WhisperFastCache, cfg: FastFilterConfig, k: number,
+    state: { lastTok: number[]; penultTok: number[]; lastTs: number[] },
+  ): { pre: MlxArray; idx: MlxArray; vals: MlxArray };
+  /** One compiled decode step: pre-filter and filtered last-position logits [B, V] f32. */
+  step(
+    tokens: number[], cache: WhisperFastCache, cfg: FastFilterConfig,
+    state: { lastTok: number[]; penultTok: number[]; lastTs: number[]; atBegin: boolean },
+  ): { pre: MlxArray; filtered: MlxArray };
+}
+
+/** What the transcription engine needs of a speech-recognition graph: the faithful encoder/decoder,
+ * the optimized path, and the checkpoint facts window decoding and word timing read. The engine
+ * never names the class that implements it. */
+export interface WhisperGraph {
+  readonly dims: WhisperDims;
+  /** The checkpoint's weight dtype (mel windows are cast to it). */
+  readonly dtype: Dtype;
+  readonly isMultilingual: boolean;
+  /** Cross-attention heads (layer, head) used for word timestamps. */
+  readonly alignmentHeads: [number, number][];
+  readonly fast: WhisperFastPath;
+  makeCache(): WhisperDecoderCache;
+  /** mel [B, 3000, n_mels] (weight dtype) → [B, 1500, D]. */
+  encode(mel: MlxArray): MlxArray;
+  /** tokens [B, T] int32 + audio features → logits [B, T, n_vocab] and, on request, the per-layer
+   * raw cross-attention qk. Advances `cache`. */
+  decode(
+    tokens: MlxArray, audioFeatures: MlxArray, cache: WhisperDecoderCache,
+    opts?: { wantCrossQk?: boolean },
+  ): { logits: MlxArray; crossQk: (MlxArray | null)[] };
+}

package/src/contracts/portable/composition.ts

@@ -0,0 +1,106 @@
+/** The read kernels of a composition's affine-quantized caches. `tiled-causal`
+ * is OptiQ's rule: a window through the N-tiled fused port while the cache's own
+ * mask is plain causal (where the port takes the configuration), every other
+ * read through the unfused port. `unfused` is mlx-lm's port for every read.
+ * `--fused-sdpa on` composes the first and `off` the second; unset, a per-layer
+ * scheme (`--kv-quant config`) composes `tiled-causal` and a uniform one
+ * `unfused`, which keeps each scheme's oracle parity. */
+export type AffineKernelChoice = "tiled-causal" | "unfused";
+
+/** How a composition's attention caches store keys and values. A quantized
+ * scheme carries `quantizedKvStart` only when it is nonzero: storage stays
+ * bf16 until a sequence holds that many tokens, then converts in place. */
+export type KvStorageScheme =
+  | { readonly kind: "bf16" }
+  /** Every attention layer, affine-quantized with one bit width and group size. */
+  | { readonly kind: "affine"; readonly bits: number; readonly groupSize: number; readonly quantizedKvStart?: number;
+      readonly kernels: AffineKernelChoice }
+  /** The layers listed (cache index), each with its own bit width and group size; the others stay bf16. */
+  | { readonly kind: "affine-layers"; readonly layers: readonly AffineKvLayer[]; readonly quantizedKvStart?: number;
+      readonly kernels: AffineKernelChoice }
+  /** Full-attention layers TurboQuant-encoded; sliding-window layers stay bf16. */
+  | { readonly kind: "turbo"; readonly kBits: number; readonly vBits: number; readonly quantizedKvStart?: number };
+
+/** Full-attention keys and values held in pages (`--paged-kv`). */
+export interface PagedKvComposition {
+  /** Positions per page. */
+  readonly blockSize: number;
+  /** The pages' read of at most eight queries without an array mask (decode,
+   * and prefill and verification windows that short): the direct Metal kernel
+   * over the blocks (Lab numerics) where true, otherwise the gathered blocks
+   * through the fused SDPA (the numerical control). Pages the kernel does not
+   * read (TurboQuant, affine widths that straddle a 32-bit word) read gathered.
+   * Pages the kernel reads have no compiled decode form (the kernel's
+   * partitions follow the context length): their composition resolves
+   * `compiledDecode` off. */
+  readonly direct: boolean;
+}
+
+export interface AffineKvLayer {
+  readonly layer: number;
+  readonly bits: number;
+  readonly groupSize: number;
+}
+
+/** The bf16 KV scheme: what a graph whose composition names no scheme stores. */
+export const BF16_KV: KvStorageScheme = Object.freeze({ kind: "bf16" });
+
+/** Drafted tokens per verify round: a fixed count (0 when the composition has
+ * no drafter), or adaptive, where the scheduler picks each round's count up to
+ * `max`. */
+export type DraftDepth = number | { readonly adaptive: true; readonly max: number };
+
+/** The facts that fix one served configuration, resolved once by the
+ * application before the loader builds the graph and consumed as data. Every
+ * field holds a resolved value; nothing here is "unset" or "auto". */
+export interface Composition {
+  /** The GPU's architecture identifier as MLX reports it (for example `applegpu_g16s`). */
+  readonly device: string;
+  /** The storage and reads of every attention cache the graph's `makeCache` composes. */
+  readonly kv: KvStorageScheme;
+  /** Full-attention keys and values held in pages of `blockSize` positions
+   * (`--paged-kv`), or null. Every request's state takes the paged lego in place
+   * of the plain full-attention storage the graph composes; its pages store
+   * what `kv` names. */
+  readonly pagedKv: PagedKvComposition | null;
+  readonly draftDepth: DraftDepth;
+  /** The prefill chunk size, in prompt tokens, the composition is built for.
+   * The prefill planner may hand the graph fewer rows than this as a prompt's
+   * tail. Its per-prompt halving for a graph with a bounded prefill workspace
+   * is the planner's business and does not change what the composition built. */
+  readonly prefillChunkTokens: number;
+  /** The most rows one forward carries (`--batch`, mlx-lm's `--batch-size`). */
+  readonly maxRows: number;
+  /** A decode step whose live rows are all plain greedy samples them with one
+   * argmax over the batch instead of one sampler call per row (default on): the
+   * same per-row arithmetic in one graph. Off, every row samples alone. */
+  readonly vectorGreedySampling: boolean;
+  /** One forward carries a decode group and a prefill group where the graph
+   * declares mixed rows (default off). */
+  readonly mixedPrefill: boolean;
+  /** Prompt tokens one mixed forward carries beside its decode rows (default 256). */
+  readonly mixedTokenBudget: number;
+  /** A prompt's last token prefills alone, as the first one-position step
+   * (mlx-lm's convention, default on); off, the final chunk prefills whole and
+   * token zero samples from it. */
+  readonly prefillTailSplit: boolean;
+  /** Grammar-forced continuations enter the KV without one forward per token
+   * (jump-forward decoding, default off). */
+  readonly grammarJump: boolean;
+  /** Every speculative row drafts by argmax instead of the request sampler's
+   * coupled draw (default off). The draft moves acceptance and draft cost, not
+   * the output distribution (see `generation/speculative/draft-policy`). */
+  readonly greedyDraft: boolean;
+}
+
+/** The value of each composition fact a caller leaves unstated: what
+ * `resolveComposition` resolves, and what a graph built without a record runs. */
+export const COMPOSITION_DEFAULTS = Object.freeze({
+  pagedKv: null,
+  vectorGreedySampling: true,
+  mixedPrefill: false,
+  mixedTokenBudget: 256,
+  prefillTailSplit: true,
+  grammarJump: false,
+  greedyDraft: false,
+} satisfies Partial<Composition>);

package/src/contracts/portable/denoising.ts

@@ -0,0 +1,17 @@
+import type { GraphDescriptor } from "./graph";
+
+/** Denoising has canvas state and feedback, independent of AR token caches.
+ * Returned tensors/state are owned by the run. Inputs remain borrowed. A
+ * fused method may bypass this graph and implement InferenceMethod directly. */
+export interface DenoisingGraph<Tensor, State> {
+  readonly descriptor: GraphDescriptor;
+  readonly vocabSize: number;
+  readonly canvasLength: number;
+  readonly embedScale: number;
+  prefill(promptIds: number[], vision?: Tensor): State;
+  extendPrefill(tokens: Tensor, state: State): void;
+  decoderLogits(canvas: Tensor, state: State, feedback: Tensor | null): Tensor;
+  dequantEmbedWeight(): Tensor;
+  softEmbeddings(logits: Tensor, weight: Tensor): Tensor;
+  closeState(state: State): void;
+}

package/src/contracts/portable/execution.ts

@@ -0,0 +1,79 @@
+/** Request facts only. Native resources remain with the preparation owner. */
+export interface ExecutionRequirements {
+  /** Prepared autoregressive media (embeddings for a media input binding).
+   * Denoising image pixels are that method's own prefill input, not media. */
+  readonly hasVision: boolean;
+  readonly hasPreparedPrefixIdentity?: boolean;
+  readonly hasAdapters: boolean;
+  readonly hasRepetitionPenalty: boolean;
+  readonly userSeed: boolean;
+  readonly hasLogitsExtras: boolean;
+  readonly hasGrammar: boolean;
+  readonly wantsLogprobs: boolean;
+  readonly hasDraft: boolean;
+}
+
+export interface ExecutionCapabilities {
+  readonly method: "autoregressive" | "denoising";
+  /** Executor coordination, not a model capability: the shared group drives
+   * each row's own grammar state (wait for its mask, advance it with the
+   * emitted token, stop the row when it terminates). The shared sampler masks
+   * any model's logits; grammar jump and speculation are qualified separately. */
+  readonly grammarBatch: boolean;
+  /** Prepared media can enter the ordinary shared execution group. */
+  readonly mediaBatch?: boolean;
+  readonly mediaPrefixCache?: boolean;
+  readonly checkpoints: boolean;
+  /** Bound ordinary driver can restore/capture this request configuration. */
+  readonly sharedCheckpoints?: boolean;
+  readonly speculativeLogprobs?: boolean;
+  /** The grouped provider supports the target's mounted adapter context. */
+  readonly sharedSpeculativeAdapters?: boolean;
+  /** Methods supplied by the model's shared execution binding. */
+  readonly groupedMethods?: readonly string[];
+  /** A shared method accepts request-owned grammar continuation proposals. */
+  readonly sharedGrammarProposals?: boolean;
+  /** An AR method appends grammar-accepted spans without speculative verification. */
+  readonly sharedGrammarJump?: boolean;
+  /** A grouped method can consume this request's strict known continuations. */
+  readonly sharedFill?: boolean;
+  /** A mounted provider can consume verified external continuations. */
+  readonly sharedSpeculativeEcho?: boolean;
+  /** The graph owns a compiled decode step. */
+  readonly compiledDecode?: boolean;
+}
+
+export interface ExecutionFeatures {
+  readonly fill: boolean;
+  readonly compiledDecode?: boolean;
+  readonly grammarJump?: boolean;
+}
+
+/** Why a request is refused or not served as asked. Refusals
+ * (`mechanism: "unsupported"`) name the capability the executor or graph lacks;
+ * the rest name a requested feature the request's other choices make incompatible. */
+export type ExecutionReason =
+  | "media-batch-unsupported" | "grammar-batch-unsupported"
+  | "method-batch-unsupported"
+  | "draft-method-unsupported" | "logprobs-method-unsupported" | "repetition-penalty-method-unsupported"
+  | "logits-extras-method-unsupported" | "fill-method-unsupported"
+  | "draft-incompatible-with-request"
+  | "fill-incompatible-with-request" | "compiled-decode-unavailable-for-request"
+  | "grammar-jump-incompatible-with-request";
+
+/** Selected once; consumers execute/report these values without reselecting. */
+export interface ResolvedExecution {
+  /** Implementation-owned method ID. The built-in planner retains its known
+   * methods; another model may register a different set without editing this contract. */
+  readonly method: string;
+  /** `unsupported`: no shared executor serves this request; `reasons` names why. */
+  readonly mechanism: "continuous" | "unsupported";
+  readonly promptCache: boolean;
+  readonly checkpoint: boolean;
+  readonly fill: boolean;
+  /** Permission to attempt the bound compiled step, never a promise that
+   * every dynamic row/state shape supports replay. */
+  readonly compiledDecode: boolean;
+  readonly grammarJump: boolean;
+  readonly reasons: readonly ExecutionReason[];
+}

package/src/contracts/portable/generation.ts

@@ -0,0 +1,103 @@
+/** Portable execution contracts. No transport, model, tensor, or runtime imports. */
+export type CancelReason = "requested" | "consumer_closed" | "consumer_idle" | "engine_closed";
+
+export interface Cancellation {
+  readonly reason: CancelReason | undefined;
+  /** Invoked immediately when already cancelled. Returns an idempotent unsubscribe. */
+  subscribe(listener: (reason: CancelReason) => void): () => void;
+}
+
+export interface TokenLogprobs {
+  /** Selected-token logprob, before sampler truncation. */
+  logprob?: number;
+  /** Top-k pairs, sorted by descending logprob. */
+  top?: { id: number; logprob: number }[];
+}
+
+export type GenerationEvent =
+  | { readonly type: "committed"; readonly sequence: number;
+      readonly tokenIds: readonly number[]; readonly logprobs?: readonly (TokenLogprobs | undefined)[] }
+  | { readonly type: "progress"; readonly completed: number; readonly total?: number };
+
+/** Methods publish in order and await each publication before publishing again.
+ * Only committed tokens cross this boundary. Tentative canvases/drafts stay private. */
+export interface GenerationOutput {
+  /** False requests a normal stop. The method must stop publishing and settle
+   * its metrics after releasing its private execution state. */
+  commit(tokenIds: readonly number[], logprobs?: readonly (TokenLogprobs | undefined)[]): Promise<void | false>;
+  progress(completed: number, total?: number): Promise<void>;
+}
+
+export interface MethodResult<Metrics> {
+  readonly finishReason: "stop" | "length";
+  readonly metrics: Metrics;
+}
+
+/** One attempt's private state, including any pending device work.
+ * close must await safe resource release, including on execute failure. */
+export interface MethodRun<Metrics> {
+  execute(output: GenerationOutput, cancellation: Cancellation): Promise<MethodResult<Metrics>>;
+  close(): Promise<void>;
+}
+
+export interface InferenceMethod<Metrics> {
+  readonly id: string;
+  /** A rejected creation must release its own partial allocations. */
+  createRun(cancellation: Cancellation): Promise<MethodRun<Metrics>>;
+}
+
+export interface ExecutionPlan<Metrics> {
+  readonly id: string;
+  readonly outputTokenLimit: number;
+  readonly method: InferenceMethod<Metrics>;
+}
+
+export interface ExecutionPlanner<Request, Metrics> {
+  /** Planning selects a bound implementation; it does not allocate device state. */
+  plan(request: Request, cancellation: Cancellation): Promise<ExecutionPlan<Metrics>>;
+}
+
+export type GenerationOutcome<Metrics> =
+  | { readonly status: "completed"; readonly committedTokens: number;
+      readonly output?: Uint32Array; readonly result: MethodResult<Metrics> }
+  | { readonly status: "cancelled"; readonly committedTokens: number;
+      readonly output?: Uint32Array; readonly reason: CancelReason }
+  | { readonly status: "failed"; readonly committedTokens: number;
+      readonly output?: Uint32Array; readonly error: { readonly code: string; readonly message: string;
+        readonly cleanupError?: string } };
+
+export type SessionState = "created" | "preparing" | "running" | "settling" | "terminal";
+
+export type RunControl = {
+  /** collect returns token IDs; per-token logprobs require stream consumption. */
+  readonly output: "stream" | "collect";
+  readonly cancellation?: Cancellation;
+} | {
+  /** Direct delivery for consumers that already own bounded output storage.
+   * The callback is awaited before execution continues; false is a normal
+   * token stop. Borrowed arrays are valid until the callback settles. */
+  readonly output: "callback";
+  readonly onTokens: GenerationOutput["commit"];
+  readonly cancellation?: Cancellation;
+};
+
+export interface GenerationSession<Metrics> {
+  readonly id: string;
+  readonly state: SessionState;
+  /** Single consumer. Stream mode starts on first next(); collect mode has no events. */
+  readonly events: AsyncIterable<GenerationEvent>;
+  /** Always resolves with one terminal outcome, after native resources are released. */
+  readonly result: Promise<GenerationOutcome<Metrics>>;
+  cancel(reason?: CancelReason): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface InferenceEngine<Request, Metrics> {
+  open(request: Request, control: RunControl): Promise<GenerationSession<Metrics>>;
+  close(): Promise<void>;
+}
+
+/** Injected host timer. The portable engine has no DOM/Node/Bun dependency. */
+export interface Timer {
+  after(milliseconds: number, callback: () => void): () => void;
+}