@simulatte/doppler 0.1.8 → 0.1.9

package/src/inference/pipelines/vision/encoder.js

@@ -0,0 +1,386 @@
+
+
+import { trace } from '../../../debug/index.js';
+import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
+import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
+import {
+  doLayerNorm, doMatmul, doGelu, doResidualAdd, doCast,
+} from './ops.js';
+
+/**
+ * Run the Qwen3-VL vision encoder on preprocessed image patches.
+ *
+ * Architecture:
+ *   patch_embed (conv2d 3->hidden, stride=patchSize) -> [numPatches, hiddenSize]
+ *   for each ViT block:
+ *     x = layerNorm(x)
+ *     x = x + selfAttention(x)    (no KV cache — full prefill attention)
+ *     x = layerNorm(x)
+ *     x = x + FFN(x)              (gelu activation)
+ *   spatialMerge(x) -> [numMergedPatches, outHiddenSize]
+ *
+ * @param {object} params
+ * @param {GPUBuffer}  params.patchBuffer    Preprocessed patches [numPatches, hiddenSize] on GPU
+ * @param {number}     params.numPatches     Total number of patches
+ * @param {object}     params.visionConfig   Vision config from manifest
+ * @param {object}     params.weights        Vision encoder weight buffers keyed by tensor name
+ * @param {object}     params.pipelineState  Shared pipeline state for buffer tracking
+ * @returns {Promise<{ features: GPUBuffer, numTokens: number }>}
+ */
+export async function runVisionEncoder(params) {
+  const {
+    patchBuffer,
+    numPatches,
+    visionConfig,
+    weights,
+    pipelineState,
+  } = params;
+
+  const {
+    depth,
+    hiddenSize,
+    intermediateSize,
+    numHeads,
+    outHiddenSize,
+    spatialMergeSize,
+    eps = 1e-6,
+  } = visionConfig;
+
+  const headDim = Math.floor(hiddenSize / numHeads);
+  const device = getDevice();
+
+  trace('vision', `encoder: depth=${depth} hidden=${hiddenSize} heads=${numHeads} patches=${numPatches}`);
+
+  let hidden = patchBuffer;
+
+  // Run ViT transformer blocks.
+  for (let i = 0; i < depth; i++) {
+    const prefix = `visual.blocks.${i}`;
+
+    // Pre-attention layer norm.
+    const normed1 = await doLayerNorm(hidden, weights[`${prefix}.norm1.weight`], weights[`${prefix}.norm1.bias`], {
+      seqLen: numPatches, hiddenSize, eps,
+    });
+
+    // Self-attention (full, no KV cache).
+    const attnOut = await visionSelfAttention({
+      input: normed1,
+      seqLen: numPatches,
+      hiddenSize,
+      numHeads,
+      headDim,
+      qkvWeight: weights[`${prefix}.attn.qkv.weight`],
+      qkvBias: weights[`${prefix}.attn.qkv.bias`],
+      projWeight: weights[`${prefix}.attn.proj.weight`],
+      projBias: weights[`${prefix}.attn.proj.bias`],
+    });
+
+    releaseBuffer(normed1);
+
+    // Residual add.
+    const residual1 = await doResidualAdd(hidden, attnOut, { count: numPatches * hiddenSize });
+    releaseBuffer(hidden);
+    releaseBuffer(attnOut);
+
+    // Pre-FFN layer norm.
+    const normed2 = await doLayerNorm(residual1, weights[`${prefix}.norm2.weight`], weights[`${prefix}.norm2.bias`], {
+      seqLen: numPatches, hiddenSize, eps,
+    });
+
+    // FFN: linear -> gelu -> linear.
+    const ffnOut = await visionFFN({
+      input: normed2,
+      seqLen: numPatches,
+      hiddenSize,
+      intermediateSize,
+      fc1Weight: weights[`${prefix}.mlp.fc1.weight`],
+      fc1Bias: weights[`${prefix}.mlp.fc1.bias`],
+      fc2Weight: weights[`${prefix}.mlp.fc2.weight`],
+      fc2Bias: weights[`${prefix}.mlp.fc2.bias`],
+    });
+
+    releaseBuffer(normed2);
+
+    // Residual add.
+    hidden = await doResidualAdd(residual1, ffnOut, { count: numPatches * hiddenSize });
+    releaseBuffer(residual1);
+    releaseBuffer(ffnOut);
+
+    trace('vision', `  block ${i}/${depth} done`);
+  }
+
+  // Spatial merge projector: merge 2x2 patches -> outHiddenSize.
+  const mergedTokens = Math.floor(numPatches / (spatialMergeSize * spatialMergeSize));
+  const merged = await spatialMergeProject({
+    input: hidden,
+    numPatches,
+    hiddenSize,
+    outHiddenSize,
+    spatialMergeSize,
+    weights,
+  });
+
+  releaseBuffer(hidden);
+
+  trace('vision', `encoder done: ${numPatches} patches -> ${mergedTokens} tokens (${outHiddenSize}d)`);
+
+  return { features: merged, numTokens: mergedTokens };
+}
+
+/**
+ * Vision self-attention (full prefill, no KV cache).
+ * QKV are fused into one weight matrix [3*hiddenSize, hiddenSize].
+ */
+async function visionSelfAttention(params) {
+  const {
+    input, seqLen, hiddenSize, numHeads, headDim,
+    qkvWeight, qkvBias, projWeight, projBias,
+  } = params;
+
+  // QKV projection: [seqLen, hiddenSize] @ [hiddenSize, 3*hiddenSize] -> [seqLen, 3*hiddenSize]
+  const qkv = await doMatmul(input, qkvWeight, {
+    M: seqLen, K: hiddenSize, N: 3 * hiddenSize, bias: qkvBias,
+  });
+
+  // Split Q, K, V and compute scaled dot-product attention on GPU.
+  // This uses the existing attention kernel infrastructure in prefill mode.
+  const attnResult = await computeVisionAttention({
+    qkv, seqLen, numHeads, headDim, hiddenSize,
+  });
+
+  releaseBuffer(qkv);
+
+  // Output projection: [seqLen, hiddenSize] @ [hiddenSize, hiddenSize] -> [seqLen, hiddenSize]
+  const output = await doMatmul(attnResult, projWeight, {
+    M: seqLen, K: hiddenSize, N: hiddenSize, bias: projBias,
+  });
+
+  releaseBuffer(attnResult);
+
+  return output;
+}
+
+/**
+ * Compute scaled dot-product attention for vision encoder.
+ * No KV cache, no causal mask — full bidirectional attention.
+ *
+ * Input: fused QKV buffer [seqLen, 3*hiddenSize]
+ * Output: attention output [seqLen, hiddenSize]
+ */
+async function computeVisionAttention(params) {
+  const { qkv, seqLen, numHeads, headDim, hiddenSize } = params;
+  const device = getDevice();
+  const scale = 1.0 / Math.sqrt(headDim);
+
+  // For the initial implementation, read QKV back to CPU, compute attention,
+  // and upload the result. This will be replaced with a GPU kernel.
+  //
+  // TODO(perf): Replace with GPU-native vision attention kernel.
+  // The text decoder attention kernels assume causal masking and KV cache,
+  // which don't apply to the vision encoder's bidirectional full attention.
+  const qkvSize = seqLen * 3 * hiddenSize;
+  const qkvData = new Float32Array(qkvSize);
+  {
+    const staging = device.createBuffer({
+      size: qkvSize * 4,
+      usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
+    });
+    const encoder = device.createCommandEncoder();
+    encoder.copyBufferToBuffer(qkv, 0, staging, 0, qkvSize * 4);
+    device.queue.submit([encoder.finish()]);
+    await staging.mapAsync(GPUMapMode.READ);
+    qkvData.set(new Float32Array(staging.getMappedRange()));
+    staging.unmap();
+    staging.destroy();
+  }
+
+  // Split into Q, K, V: each [numHeads, seqLen, headDim]
+  const Q = new Float32Array(numHeads * seqLen * headDim);
+  const K = new Float32Array(numHeads * seqLen * headDim);
+  const V = new Float32Array(numHeads * seqLen * headDim);
+
+  for (let s = 0; s < seqLen; s++) {
+    for (let h = 0; h < numHeads; h++) {
+      for (let d = 0; d < headDim; d++) {
+        const srcBase = s * 3 * hiddenSize;
+        const headOffset = h * headDim + d;
+        Q[(h * seqLen + s) * headDim + d] = qkvData[srcBase + headOffset];
+        K[(h * seqLen + s) * headDim + d] = qkvData[srcBase + hiddenSize + headOffset];
+        V[(h * seqLen + s) * headDim + d] = qkvData[srcBase + 2 * hiddenSize + headOffset];
+      }
+    }
+  }
+
+  // Compute attention: softmax(Q @ K^T / sqrt(d)) @ V per head.
+  const output = new Float32Array(seqLen * hiddenSize);
+
+  for (let h = 0; h < numHeads; h++) {
+    // Scores: [seqLen, seqLen]
+    const scores = new Float32Array(seqLen * seqLen);
+    for (let i = 0; i < seqLen; i++) {
+      for (let j = 0; j < seqLen; j++) {
+        let dot = 0;
+        for (let d = 0; d < headDim; d++) {
+          dot += Q[(h * seqLen + i) * headDim + d] * K[(h * seqLen + j) * headDim + d];
+        }
+        scores[i * seqLen + j] = dot * scale;
+      }
+    }
+
+    // Softmax per row.
+    for (let i = 0; i < seqLen; i++) {
+      let maxVal = -Infinity;
+      for (let j = 0; j < seqLen; j++) {
+        if (scores[i * seqLen + j] > maxVal) maxVal = scores[i * seqLen + j];
+      }
+      let sumExp = 0;
+      for (let j = 0; j < seqLen; j++) {
+        scores[i * seqLen + j] = Math.exp(scores[i * seqLen + j] - maxVal);
+        sumExp += scores[i * seqLen + j];
+      }
+      for (let j = 0; j < seqLen; j++) {
+        scores[i * seqLen + j] /= sumExp;
+      }
+    }
+
+    // Weighted sum: [seqLen, headDim]
+    for (let i = 0; i < seqLen; i++) {
+      for (let d = 0; d < headDim; d++) {
+        let val = 0;
+        for (let j = 0; j < seqLen; j++) {
+          val += scores[i * seqLen + j] * V[(h * seqLen + j) * headDim + d];
+        }
+        output[i * hiddenSize + h * headDim + d] = val;
+      }
+    }
+  }
+
+  // Upload result to GPU.
+  const outBuffer = acquireBuffer(seqLen * hiddenSize * 4, 'vision-attn-output');
+  device.queue.writeBuffer(outBuffer, 0, output);
+
+  return outBuffer;
+}
+
+/**
+ * Vision FFN: fc1 -> gelu -> fc2.
+ */
+async function visionFFN(params) {
+  const {
+    input, seqLen, hiddenSize, intermediateSize,
+    fc1Weight, fc1Bias, fc2Weight, fc2Bias,
+  } = params;
+
+  // fc1: [seqLen, hiddenSize] -> [seqLen, intermediateSize]
+  const fc1Out = await doMatmul(input, fc1Weight, {
+    M: seqLen, K: hiddenSize, N: intermediateSize, bias: fc1Bias,
+  });
+
+  // GELU activation.
+  const activated = await doGelu(fc1Out, { count: seqLen * intermediateSize });
+  releaseBuffer(fc1Out);
+
+  // fc2: [seqLen, intermediateSize] -> [seqLen, hiddenSize]
+  const fc2Out = await doMatmul(activated, fc2Weight, {
+    M: seqLen, K: intermediateSize, N: hiddenSize, bias: fc2Bias,
+  });
+  releaseBuffer(activated);
+
+  return fc2Out;
+}
+
+/**
+ * Spatial merge projector.
+ *
+ * Takes [numPatches, hiddenSize] vision features and merges spatialMergeSize x spatialMergeSize
+ * adjacent patches into single tokens via concatenation + linear projection.
+ *
+ * Input:  [numPatches, hiddenSize] where numPatches = gridH * gridW
+ * Output: [mergedPatches, outHiddenSize] where mergedPatches = (gridH/m) * (gridW/m), m = spatialMergeSize
+ */
+async function spatialMergeProject(params) {
+  const {
+    input, numPatches, hiddenSize, outHiddenSize, spatialMergeSize, weights,
+  } = params;
+
+  const device = getDevice();
+  const m = spatialMergeSize;
+  const concatDim = m * m * hiddenSize;
+
+  // Read vision features back for spatial rearrangement.
+  // TODO(perf): GPU kernel for spatial merge gather.
+  const inputSize = numPatches * hiddenSize;
+  const inputData = new Float32Array(inputSize);
+  {
+    const staging = device.createBuffer({
+      size: inputSize * 4,
+      usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
+    });
+    const encoder = device.createCommandEncoder();
+    encoder.copyBufferToBuffer(input, 0, staging, 0, inputSize * 4);
+    device.queue.submit([encoder.finish()]);
+    await staging.mapAsync(GPUMapMode.READ);
+    inputData.set(new Float32Array(staging.getMappedRange()));
+    staging.unmap();
+    staging.destroy();
+  }
+
+  // Assume patches are laid out as [gridH, gridW, hiddenSize].
+  // We need gridH and gridW — derive from numPatches assuming square-ish grid.
+  // The actual grid dimensions should be passed in; for now infer from sqrt.
+  const gridSide = Math.round(Math.sqrt(numPatches));
+  const gridH = gridSide;
+  const gridW = Math.floor(numPatches / gridH);
+
+  const mergedH = Math.floor(gridH / m);
+  const mergedW = Math.floor(gridW / m);
+  const mergedCount = mergedH * mergedW;
+
+  // Concatenate m x m patches into single vectors of dimension concatDim.
+  const concatenated = new Float32Array(mergedCount * concatDim);
+  for (let mh = 0; mh < mergedH; mh++) {
+    for (let mw = 0; mw < mergedW; mw++) {
+      const outIdx = mh * mergedW + mw;
+      let offset = 0;
+      for (let dh = 0; dh < m; dh++) {
+        for (let dw = 0; dw < m; dw++) {
+          const srcH = mh * m + dh;
+          const srcW = mw * m + dw;
+          const srcIdx = srcH * gridW + srcW;
+          for (let d = 0; d < hiddenSize; d++) {
+            concatenated[outIdx * concatDim + offset] = inputData[srcIdx * hiddenSize + d];
+            offset++;
+          }
+        }
+      }
+    }
+  }
+
+  // Upload concatenated data.
+  const concatBuffer = acquireBuffer(mergedCount * concatDim * 4, 'vision-merge-concat');
+  device.queue.writeBuffer(concatBuffer, 0, concatenated);
+
+  // Linear projection: [mergedCount, concatDim] @ [concatDim, outHiddenSize] -> [mergedCount, outHiddenSize]
+  const projected = await doMatmul(concatBuffer, weights['visual.merger.mlp.0.weight'], {
+    M: mergedCount,
+    K: concatDim,
+    N: outHiddenSize,
+    bias: weights['visual.merger.mlp.0.bias'],
+  });
+
+  releaseBuffer(concatBuffer);
+
+  // GELU + second linear layer.
+  const activated = await doGelu(projected, { count: mergedCount * outHiddenSize });
+  releaseBuffer(projected);
+
+  const output = await doMatmul(activated, weights['visual.merger.mlp.2.weight'], {
+    M: mergedCount,
+    K: outHiddenSize,
+    N: outHiddenSize,
+    bias: weights['visual.merger.mlp.2.bias'],
+  });
+  releaseBuffer(activated);
+
+  return output;
+}

package/src/inference/pipelines/vision/image-preprocess.js

@@ -0,0 +1,151 @@
+
+
+import { trace } from '../../../debug/index.js';
+
+/**
+ * Preprocess an image for Qwen3-VL vision encoder.
+ *
+ * Accepts raw pixel data (Uint8Array RGBA or RGB, or Float32Array normalized)
+ * and returns a GPU-ready Float32Array of shape [C, H, W] after:
+ *   1. Resize to fit min/max pixel constraints
+ *   2. Pad to patch-aligned dimensions
+ *   3. Normalize with mean/std
+ *   4. Extract temporal patches (for video; single frame for images)
+ *
+ * @param {Uint8Array|Float32Array} pixels   Raw pixel data (RGBA or RGB)
+ * @param {number}                  width    Source image width
+ * @param {number}                  height   Source image height
+ * @param {object}                  config   Vision config from manifest/preset
+ * @returns {{ data: Float32Array, gridThw: [number, number, number], patchedHeight: number, patchedWidth: number }}
+ */
+export function preprocessImage(pixels, width, height, config) {
+  const {
+    patchSize = 16,
+    spatialMergeSize = 2,
+    temporalPatchSize = 2,
+    minPixels = 3136,
+    maxPixels = 1003520,
+    normalization = {},
+  } = config;
+
+  const mean = normalization.mean || [0.48145466, 0.4578275, 0.40821073];
+  const std = normalization.std || [0.26862954, 0.26130258, 0.27577711];
+
+  // Step 1: Compute target dimensions respecting pixel constraints and patch alignment.
+  const mergedPatch = patchSize * spatialMergeSize;
+  const { targetWidth, targetHeight } = computeTargetDimensions(
+    width, height, minPixels, maxPixels, mergedPatch,
+  );
+
+  trace('vision', `preprocess: ${width}x${height} -> ${targetWidth}x${targetHeight} (patch=${patchSize}, merge=${spatialMergeSize})`);
+
+  // Step 2: Resize to target dimensions (bilinear interpolation on CPU).
+  const channels = 3;
+  const resized = resizeBilinear(pixels, width, height, targetWidth, targetHeight, channels);
+
+  // Step 3: Normalize to [0,1] then apply mean/std normalization.
+  const normalized = new Float32Array(channels * targetHeight * targetWidth);
+  for (let c = 0; c < channels; c++) {
+    const m = mean[c];
+    const s = std[c];
+    for (let y = 0; y < targetHeight; y++) {
+      for (let x = 0; x < targetWidth; x++) {
+        const srcIdx = (y * targetWidth + x) * channels + c;
+        const dstIdx = c * targetHeight * targetWidth + y * targetWidth + x;
+        normalized[dstIdx] = (resized[srcIdx] / 255.0 - m) / s;
+      }
+    }
+  }
+
+  // Step 4: Compute grid dimensions for the LLM.
+  //   gridT = 1 for single image (temporalPatchSize frames per temporal patch)
+  //   gridH = targetHeight / patchSize
+  //   gridW = targetWidth / patchSize
+  const gridT = 1;
+  const gridH = Math.floor(targetHeight / patchSize);
+  const gridW = Math.floor(targetWidth / patchSize);
+
+  return {
+    data: normalized,
+    width: targetWidth,
+    height: targetHeight,
+    channels,
+    gridThw: [gridT, gridH, gridW],
+    patchedHeight: targetHeight,
+    patchedWidth: targetWidth,
+  };
+}
+
+/**
+ * Compute target dimensions that satisfy:
+ *   - Total pixels >= minPixels and <= maxPixels
+ *   - Both dimensions are multiples of mergedPatch
+ *   - Aspect ratio is preserved as closely as possible
+ */
+function computeTargetDimensions(width, height, minPixels, maxPixels, mergedPatch) {
+  const aspectRatio = width / height;
+
+  // Start from the geometric mean of min/max pixel counts.
+  let targetPixels = Math.sqrt(minPixels * maxPixels);
+  targetPixels = Math.max(minPixels, Math.min(maxPixels, targetPixels));
+
+  // Compute dimensions preserving aspect ratio.
+  let h = Math.sqrt(targetPixels / aspectRatio);
+  let w = h * aspectRatio;
+
+  // Round to nearest mergedPatch multiple.
+  h = Math.max(mergedPatch, Math.round(h / mergedPatch) * mergedPatch);
+  w = Math.max(mergedPatch, Math.round(w / mergedPatch) * mergedPatch);
+
+  // Clamp total pixels.
+  if (h * w > maxPixels) {
+    const scale = Math.sqrt(maxPixels / (h * w));
+    h = Math.max(mergedPatch, Math.round((h * scale) / mergedPatch) * mergedPatch);
+    w = Math.max(mergedPatch, Math.round((w * scale) / mergedPatch) * mergedPatch);
+  }
+  if (h * w < minPixels) {
+    const scale = Math.sqrt(minPixels / (h * w));
+    h = Math.max(mergedPatch, Math.round((h * scale) / mergedPatch) * mergedPatch);
+    w = Math.max(mergedPatch, Math.round((w * scale) / mergedPatch) * mergedPatch);
+  }
+
+  return { targetWidth: w, targetHeight: h };
+}
+
+/**
+ * Bilinear resize of interleaved RGB(A) pixel data.
+ * Input: Uint8Array or Float32Array in [H, W, C] layout (C >= 3, only first 3 used).
+ * Output: Float32Array in [H, W, 3] layout with values in [0, 255].
+ */
+function resizeBilinear(src, srcW, srcH, dstW, dstH, channels) {
+  const srcChannels = src.length / (srcW * srcH);
+  const out = new Float32Array(dstH * dstW * channels);
+  const scaleX = srcW / dstW;
+  const scaleY = srcH / dstH;
+
+  for (let y = 0; y < dstH; y++) {
+    const srcY = y * scaleY;
+    const y0 = Math.min(Math.floor(srcY), srcH - 1);
+    const y1 = Math.min(y0 + 1, srcH - 1);
+    const fy = srcY - y0;
+
+    for (let x = 0; x < dstW; x++) {
+      const srcX = x * scaleX;
+      const x0 = Math.min(Math.floor(srcX), srcW - 1);
+      const x1 = Math.min(x0 + 1, srcW - 1);
+      const fx = srcX - x0;
+
+      for (let c = 0; c < channels; c++) {
+        const v00 = src[(y0 * srcW + x0) * srcChannels + c];
+        const v01 = src[(y0 * srcW + x1) * srcChannels + c];
+        const v10 = src[(y1 * srcW + x0) * srcChannels + c];
+        const v11 = src[(y1 * srcW + x1) * srcChannels + c];
+        const top = v00 + (v01 - v00) * fx;
+        const bot = v10 + (v11 - v10) * fx;
+        out[(y * dstW + x) * channels + c] = top + (bot - top) * fy;
+      }
+    }
+  }
+
+  return out;
+}

package/src/inference/pipelines/vision/index.js

@@ -0,0 +1,173 @@
+
+
+import { trace } from '../../../debug/index.js';
+import { getDevice } from '../../../gpu/device.js';
+import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
+import { preprocessImage } from './image-preprocess.js';
+import { patchEmbed } from './patch-embed.js';
+import { runVisionEncoder } from './encoder.js';
+
+/**
+ * Encode an image through the Qwen3-VL vision pipeline.
+ *
+ * Full flow:
+ *   raw pixels -> preprocess -> patch embed -> ViT blocks -> spatial merge -> visual tokens
+ *
+ * @param {object} params
+ * @param {Uint8Array|Float32Array} params.pixels   Raw image pixel data (RGBA or RGB)
+ * @param {number}                  params.width    Image width
+ * @param {number}                  params.height   Image height
+ * @param {object}                  params.visionConfig  Vision config from manifest
+ * @param {object}                  params.weights  Vision encoder weight buffers
+ * @returns {Promise<VisionEncodeResult>}
+ */
+export async function encodeImage(params) {
+  const { pixels, width, height, visionConfig, weights } = params;
+
+  trace('vision', `encodeImage: ${width}x${height} input`);
+
+  // Step 1: Preprocess — resize, normalize, compute grid.
+  const preprocessed = preprocessImage(pixels, width, height, visionConfig);
+
+  // Step 2: Patch embedding — conv2d patches -> [numPatches, hiddenSize].
+  const { patchBuffer, numPatches } = await patchEmbed({
+    imageData: preprocessed.data,
+    height: preprocessed.height,
+    width: preprocessed.width,
+    channels: preprocessed.channels,
+    visionConfig,
+    weights,
+  });
+
+  // Step 3: Vision encoder — ViT blocks + spatial merge.
+  const { features, numTokens } = await runVisionEncoder({
+    patchBuffer,
+    numPatches,
+    visionConfig,
+    weights,
+  });
+
+  return {
+    features,
+    numTokens,
+    gridThw: preprocessed.gridThw,
+    imageWidth: preprocessed.width,
+    imageHeight: preprocessed.height,
+  };
+}
+
+/**
+ * Inject visual tokens into text token embeddings.
+ *
+ * Replaces positions in the embedding sequence where image_token_id appears
+ * with the encoded visual features from the vision encoder.
+ *
+ * For Qwen3-VL with DeepStack, visual tokens are injected at specific decoder
+ * layers (deepstackVisualIndexes), not at the input embedding level.
+ * This function handles the simpler input-level injection case.
+ * DeepStack injection is handled in the decoder layer loop.
+ *
+ * @param {object} params
+ * @param {Float32Array}  params.textEmbeddings    [seqLen, hiddenSize]
+ * @param {Int32Array}    params.tokenIds          [seqLen]
+ * @param {GPUBuffer}     params.visualFeatures    [numVisualTokens, outHiddenSize]
+ * @param {number}        params.numVisualTokens   Number of visual tokens
+ * @param {number}        params.imageTokenId      Token ID marking image positions
+ * @param {number}        params.hiddenSize        Text model hidden size
+ * @returns {{ mergedEmbeddings: Float32Array, mergedLength: number }}
+ */
+export function mergeVisualTokens(params) {
+  const {
+    textEmbeddings, tokenIds, visualFeatures,
+    numVisualTokens, imageTokenId, hiddenSize,
+  } = params;
+
+  // Count image token positions.
+  const imagePositions = [];
+  for (let i = 0; i < tokenIds.length; i++) {
+    if (tokenIds[i] === imageTokenId) {
+      imagePositions.push(i);
+    }
+  }
+
+  if (imagePositions.length === 0) {
+    trace('vision', 'mergeVisualTokens: no image tokens found, returning text-only');
+    return { mergedEmbeddings: textEmbeddings, mergedLength: tokenIds.length };
+  }
+
+  trace('vision', `mergeVisualTokens: replacing ${imagePositions.length} image tokens with ${numVisualTokens} visual tokens`);
+
+  // The merged sequence replaces contiguous image_token_id runs with visual features.
+  // For Qwen3-VL: image tokens appear as a block between vision_start and vision_end tokens.
+  // The visual features replace the entire image token block.
+
+  // Find contiguous image token ranges.
+  const ranges = [];
+  let rangeStart = imagePositions[0];
+  let rangeEnd = imagePositions[0];
+  for (let i = 1; i < imagePositions.length; i++) {
+    if (imagePositions[i] === rangeEnd + 1) {
+      rangeEnd = imagePositions[i];
+    } else {
+      ranges.push([rangeStart, rangeEnd]);
+      rangeStart = imagePositions[i];
+      rangeEnd = imagePositions[i];
+    }
+  }
+  ranges.push([rangeStart, rangeEnd]);
+
+  // Build merged sequence: text tokens (non-image) + visual tokens replacing each range.
+  const textLen = tokenIds.length;
+  const replacedCount = imagePositions.length;
+  const mergedLength = textLen - replacedCount + numVisualTokens;
+  const merged = new Float32Array(mergedLength * hiddenSize);
+
+  let srcPos = 0;
+  let dstPos = 0;
+  let visualOffset = 0;
+
+  for (const [start, end] of ranges) {
+    // Copy text tokens before this range.
+    const textBefore = start - srcPos;
+    if (textBefore > 0) {
+      merged.set(
+        textEmbeddings.subarray(srcPos * hiddenSize, start * hiddenSize),
+        dstPos * hiddenSize,
+      );
+      dstPos += textBefore;
+    }
+
+    // Insert visual tokens replacing this range.
+    const rangeLen = end - start + 1;
+    const tokensToInsert = Math.min(numVisualTokens - visualOffset, rangeLen);
+    // Copy from visual features buffer (CPU side).
+    for (let i = 0; i < tokensToInsert; i++) {
+      for (let d = 0; d < hiddenSize; d++) {
+        merged[(dstPos + i) * hiddenSize + d] = visualFeatures[(visualOffset + i) * hiddenSize + d];
+      }
+    }
+    dstPos += tokensToInsert;
+    visualOffset += tokensToInsert;
+    srcPos = end + 1;
+  }
+
+  // Copy remaining text tokens after last range.
+  if (srcPos < textLen) {
+    merged.set(
+      textEmbeddings.subarray(srcPos * hiddenSize, textLen * hiddenSize),
+      dstPos * hiddenSize,
+    );
+    dstPos += textLen - srcPos;
+  }
+
+  return { mergedEmbeddings: merged, mergedLength: dstPos };
+}
+
+/**
+ * @typedef {object} VisionEncodeResult
+ * @property {GPUBuffer}  features     Encoded visual tokens [numTokens, outHiddenSize]
+ * @property {number}     numTokens    Number of visual tokens after spatial merge
+ * @property {number[]}   gridThw      [temporal, height, width] grid dimensions
+ * @property {number}     imageWidth   Processed image width
+ * @property {number}     imageHeight  Processed image height
+ */