@simulatte/doppler 0.1.9 → 0.2.0

package/src/inference/pipelines/vision/encoder.js

@@ -1,386 +0,0 @@
-
-
-import { trace } from '../../../debug/index.js';
-import { getDevice, getKernelCapabilities } from '../../../gpu/device.js';
-import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
-import {
-  doLayerNorm, doMatmul, doGelu, doResidualAdd, doCast,
-} from './ops.js';
-
-/**
- * Run the Qwen3-VL vision encoder on preprocessed image patches.
- *
- * Architecture:
- *   patch_embed (conv2d 3->hidden, stride=patchSize) -> [numPatches, hiddenSize]
- *   for each ViT block:
- *     x = layerNorm(x)
- *     x = x + selfAttention(x)    (no KV cache — full prefill attention)
- *     x = layerNorm(x)
- *     x = x + FFN(x)              (gelu activation)
- *   spatialMerge(x) -> [numMergedPatches, outHiddenSize]
- *
- * @param {object} params
- * @param {GPUBuffer}  params.patchBuffer    Preprocessed patches [numPatches, hiddenSize] on GPU
- * @param {number}     params.numPatches     Total number of patches
- * @param {object}     params.visionConfig   Vision config from manifest
- * @param {object}     params.weights        Vision encoder weight buffers keyed by tensor name
- * @param {object}     params.pipelineState  Shared pipeline state for buffer tracking
- * @returns {Promise<{ features: GPUBuffer, numTokens: number }>}
- */
-export async function runVisionEncoder(params) {
-  const {
-    patchBuffer,
-    numPatches,
-    visionConfig,
-    weights,
-    pipelineState,
-  } = params;
-
-  const {
-    depth,
-    hiddenSize,
-    intermediateSize,
-    numHeads,
-    outHiddenSize,
-    spatialMergeSize,
-    eps = 1e-6,
-  } = visionConfig;
-
-  const headDim = Math.floor(hiddenSize / numHeads);
-  const device = getDevice();
-
-  trace('vision', `encoder: depth=${depth} hidden=${hiddenSize} heads=${numHeads} patches=${numPatches}`);
-
-  let hidden = patchBuffer;
-
-  // Run ViT transformer blocks.
-  for (let i = 0; i < depth; i++) {
-    const prefix = `visual.blocks.${i}`;
-
-    // Pre-attention layer norm.
-    const normed1 = await doLayerNorm(hidden, weights[`${prefix}.norm1.weight`], weights[`${prefix}.norm1.bias`], {
-      seqLen: numPatches, hiddenSize, eps,
-    });
-
-    // Self-attention (full, no KV cache).
-    const attnOut = await visionSelfAttention({
-      input: normed1,
-      seqLen: numPatches,
-      hiddenSize,
-      numHeads,
-      headDim,
-      qkvWeight: weights[`${prefix}.attn.qkv.weight`],
-      qkvBias: weights[`${prefix}.attn.qkv.bias`],
-      projWeight: weights[`${prefix}.attn.proj.weight`],
-      projBias: weights[`${prefix}.attn.proj.bias`],
-    });
-
-    releaseBuffer(normed1);
-
-    // Residual add.
-    const residual1 = await doResidualAdd(hidden, attnOut, { count: numPatches * hiddenSize });
-    releaseBuffer(hidden);
-    releaseBuffer(attnOut);
-
-    // Pre-FFN layer norm.
-    const normed2 = await doLayerNorm(residual1, weights[`${prefix}.norm2.weight`], weights[`${prefix}.norm2.bias`], {
-      seqLen: numPatches, hiddenSize, eps,
-    });
-
-    // FFN: linear -> gelu -> linear.
-    const ffnOut = await visionFFN({
-      input: normed2,
-      seqLen: numPatches,
-      hiddenSize,
-      intermediateSize,
-      fc1Weight: weights[`${prefix}.mlp.fc1.weight`],
-      fc1Bias: weights[`${prefix}.mlp.fc1.bias`],
-      fc2Weight: weights[`${prefix}.mlp.fc2.weight`],
-      fc2Bias: weights[`${prefix}.mlp.fc2.bias`],
-    });
-
-    releaseBuffer(normed2);
-
-    // Residual add.
-    hidden = await doResidualAdd(residual1, ffnOut, { count: numPatches * hiddenSize });
-    releaseBuffer(residual1);
-    releaseBuffer(ffnOut);
-
-    trace('vision', `  block ${i}/${depth} done`);
-  }
-
-  // Spatial merge projector: merge 2x2 patches -> outHiddenSize.
-  const mergedTokens = Math.floor(numPatches / (spatialMergeSize * spatialMergeSize));
-  const merged = await spatialMergeProject({
-    input: hidden,
-    numPatches,
-    hiddenSize,
-    outHiddenSize,
-    spatialMergeSize,
-    weights,
-  });
-
-  releaseBuffer(hidden);
-
-  trace('vision', `encoder done: ${numPatches} patches -> ${mergedTokens} tokens (${outHiddenSize}d)`);
-
-  return { features: merged, numTokens: mergedTokens };
-}
-
-/**
- * Vision self-attention (full prefill, no KV cache).
- * QKV are fused into one weight matrix [3*hiddenSize, hiddenSize].
- */
-async function visionSelfAttention(params) {
-  const {
-    input, seqLen, hiddenSize, numHeads, headDim,
-    qkvWeight, qkvBias, projWeight, projBias,
-  } = params;
-
-  // QKV projection: [seqLen, hiddenSize] @ [hiddenSize, 3*hiddenSize] -> [seqLen, 3*hiddenSize]
-  const qkv = await doMatmul(input, qkvWeight, {
-    M: seqLen, K: hiddenSize, N: 3 * hiddenSize, bias: qkvBias,
-  });
-
-  // Split Q, K, V and compute scaled dot-product attention on GPU.
-  // This uses the existing attention kernel infrastructure in prefill mode.
-  const attnResult = await computeVisionAttention({
-    qkv, seqLen, numHeads, headDim, hiddenSize,
-  });
-
-  releaseBuffer(qkv);
-
-  // Output projection: [seqLen, hiddenSize] @ [hiddenSize, hiddenSize] -> [seqLen, hiddenSize]
-  const output = await doMatmul(attnResult, projWeight, {
-    M: seqLen, K: hiddenSize, N: hiddenSize, bias: projBias,
-  });
-
-  releaseBuffer(attnResult);
-
-  return output;
-}
-
-/**
- * Compute scaled dot-product attention for vision encoder.
- * No KV cache, no causal mask — full bidirectional attention.
- *
- * Input: fused QKV buffer [seqLen, 3*hiddenSize]
- * Output: attention output [seqLen, hiddenSize]
- */
-async function computeVisionAttention(params) {
-  const { qkv, seqLen, numHeads, headDim, hiddenSize } = params;
-  const device = getDevice();
-  const scale = 1.0 / Math.sqrt(headDim);
-
-  // For the initial implementation, read QKV back to CPU, compute attention,
-  // and upload the result. This will be replaced with a GPU kernel.
-  //
-  // TODO(perf): Replace with GPU-native vision attention kernel.
-  // The text decoder attention kernels assume causal masking and KV cache,
-  // which don't apply to the vision encoder's bidirectional full attention.
-  const qkvSize = seqLen * 3 * hiddenSize;
-  const qkvData = new Float32Array(qkvSize);
-  {
-    const staging = device.createBuffer({
-      size: qkvSize * 4,
-      usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
-    });
-    const encoder = device.createCommandEncoder();
-    encoder.copyBufferToBuffer(qkv, 0, staging, 0, qkvSize * 4);
-    device.queue.submit([encoder.finish()]);
-    await staging.mapAsync(GPUMapMode.READ);
-    qkvData.set(new Float32Array(staging.getMappedRange()));
-    staging.unmap();
-    staging.destroy();
-  }
-
-  // Split into Q, K, V: each [numHeads, seqLen, headDim]
-  const Q = new Float32Array(numHeads * seqLen * headDim);
-  const K = new Float32Array(numHeads * seqLen * headDim);
-  const V = new Float32Array(numHeads * seqLen * headDim);
-
-  for (let s = 0; s < seqLen; s++) {
-    for (let h = 0; h < numHeads; h++) {
-      for (let d = 0; d < headDim; d++) {
-        const srcBase = s * 3 * hiddenSize;
-        const headOffset = h * headDim + d;
-        Q[(h * seqLen + s) * headDim + d] = qkvData[srcBase + headOffset];
-        K[(h * seqLen + s) * headDim + d] = qkvData[srcBase + hiddenSize + headOffset];
-        V[(h * seqLen + s) * headDim + d] = qkvData[srcBase + 2 * hiddenSize + headOffset];
-      }
-    }
-  }
-
-  // Compute attention: softmax(Q @ K^T / sqrt(d)) @ V per head.
-  const output = new Float32Array(seqLen * hiddenSize);
-
-  for (let h = 0; h < numHeads; h++) {
-    // Scores: [seqLen, seqLen]
-    const scores = new Float32Array(seqLen * seqLen);
-    for (let i = 0; i < seqLen; i++) {
-      for (let j = 0; j < seqLen; j++) {
-        let dot = 0;
-        for (let d = 0; d < headDim; d++) {
-          dot += Q[(h * seqLen + i) * headDim + d] * K[(h * seqLen + j) * headDim + d];
-        }
-        scores[i * seqLen + j] = dot * scale;
-      }
-    }
-
-    // Softmax per row.
-    for (let i = 0; i < seqLen; i++) {
-      let maxVal = -Infinity;
-      for (let j = 0; j < seqLen; j++) {
-        if (scores[i * seqLen + j] > maxVal) maxVal = scores[i * seqLen + j];
-      }
-      let sumExp = 0;
-      for (let j = 0; j < seqLen; j++) {
-        scores[i * seqLen + j] = Math.exp(scores[i * seqLen + j] - maxVal);
-        sumExp += scores[i * seqLen + j];
-      }
-      for (let j = 0; j < seqLen; j++) {
-        scores[i * seqLen + j] /= sumExp;
-      }
-    }
-
-    // Weighted sum: [seqLen, headDim]
-    for (let i = 0; i < seqLen; i++) {
-      for (let d = 0; d < headDim; d++) {
-        let val = 0;
-        for (let j = 0; j < seqLen; j++) {
-          val += scores[i * seqLen + j] * V[(h * seqLen + j) * headDim + d];
-        }
-        output[i * hiddenSize + h * headDim + d] = val;
-      }
-    }
-  }
-
-  // Upload result to GPU.
-  const outBuffer = acquireBuffer(seqLen * hiddenSize * 4, 'vision-attn-output');
-  device.queue.writeBuffer(outBuffer, 0, output);
-
-  return outBuffer;
-}
-
-/**
- * Vision FFN: fc1 -> gelu -> fc2.
- */
-async function visionFFN(params) {
-  const {
-    input, seqLen, hiddenSize, intermediateSize,
-    fc1Weight, fc1Bias, fc2Weight, fc2Bias,
-  } = params;
-
-  // fc1: [seqLen, hiddenSize] -> [seqLen, intermediateSize]
-  const fc1Out = await doMatmul(input, fc1Weight, {
-    M: seqLen, K: hiddenSize, N: intermediateSize, bias: fc1Bias,
-  });
-
-  // GELU activation.
-  const activated = await doGelu(fc1Out, { count: seqLen * intermediateSize });
-  releaseBuffer(fc1Out);
-
-  // fc2: [seqLen, intermediateSize] -> [seqLen, hiddenSize]
-  const fc2Out = await doMatmul(activated, fc2Weight, {
-    M: seqLen, K: intermediateSize, N: hiddenSize, bias: fc2Bias,
-  });
-  releaseBuffer(activated);
-
-  return fc2Out;
-}
-
-/**
- * Spatial merge projector.
- *
- * Takes [numPatches, hiddenSize] vision features and merges spatialMergeSize x spatialMergeSize
- * adjacent patches into single tokens via concatenation + linear projection.
- *
- * Input:  [numPatches, hiddenSize] where numPatches = gridH * gridW
- * Output: [mergedPatches, outHiddenSize] where mergedPatches = (gridH/m) * (gridW/m), m = spatialMergeSize
- */
-async function spatialMergeProject(params) {
-  const {
-    input, numPatches, hiddenSize, outHiddenSize, spatialMergeSize, weights,
-  } = params;
-
-  const device = getDevice();
-  const m = spatialMergeSize;
-  const concatDim = m * m * hiddenSize;
-
-  // Read vision features back for spatial rearrangement.
-  // TODO(perf): GPU kernel for spatial merge gather.
-  const inputSize = numPatches * hiddenSize;
-  const inputData = new Float32Array(inputSize);
-  {
-    const staging = device.createBuffer({
-      size: inputSize * 4,
-      usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
-    });
-    const encoder = device.createCommandEncoder();
-    encoder.copyBufferToBuffer(input, 0, staging, 0, inputSize * 4);
-    device.queue.submit([encoder.finish()]);
-    await staging.mapAsync(GPUMapMode.READ);
-    inputData.set(new Float32Array(staging.getMappedRange()));
-    staging.unmap();
-    staging.destroy();
-  }
-
-  // Assume patches are laid out as [gridH, gridW, hiddenSize].
-  // We need gridH and gridW — derive from numPatches assuming square-ish grid.
-  // The actual grid dimensions should be passed in; for now infer from sqrt.
-  const gridSide = Math.round(Math.sqrt(numPatches));
-  const gridH = gridSide;
-  const gridW = Math.floor(numPatches / gridH);
-
-  const mergedH = Math.floor(gridH / m);
-  const mergedW = Math.floor(gridW / m);
-  const mergedCount = mergedH * mergedW;
-
-  // Concatenate m x m patches into single vectors of dimension concatDim.
-  const concatenated = new Float32Array(mergedCount * concatDim);
-  for (let mh = 0; mh < mergedH; mh++) {
-    for (let mw = 0; mw < mergedW; mw++) {
-      const outIdx = mh * mergedW + mw;
-      let offset = 0;
-      for (let dh = 0; dh < m; dh++) {
-        for (let dw = 0; dw < m; dw++) {
-          const srcH = mh * m + dh;
-          const srcW = mw * m + dw;
-          const srcIdx = srcH * gridW + srcW;
-          for (let d = 0; d < hiddenSize; d++) {
-            concatenated[outIdx * concatDim + offset] = inputData[srcIdx * hiddenSize + d];
-            offset++;
-          }
-        }
-      }
-    }
-  }
-
-  // Upload concatenated data.
-  const concatBuffer = acquireBuffer(mergedCount * concatDim * 4, 'vision-merge-concat');
-  device.queue.writeBuffer(concatBuffer, 0, concatenated);
-
-  // Linear projection: [mergedCount, concatDim] @ [concatDim, outHiddenSize] -> [mergedCount, outHiddenSize]
-  const projected = await doMatmul(concatBuffer, weights['visual.merger.mlp.0.weight'], {
-    M: mergedCount,
-    K: concatDim,
-    N: outHiddenSize,
-    bias: weights['visual.merger.mlp.0.bias'],
-  });
-
-  releaseBuffer(concatBuffer);
-
-  // GELU + second linear layer.
-  const activated = await doGelu(projected, { count: mergedCount * outHiddenSize });
-  releaseBuffer(projected);
-
-  const output = await doMatmul(activated, weights['visual.merger.mlp.2.weight'], {
-    M: mergedCount,
-    K: outHiddenSize,
-    N: outHiddenSize,
-    bias: weights['visual.merger.mlp.2.bias'],
-  });
-  releaseBuffer(activated);
-
-  return output;
-}

package/src/inference/pipelines/vision/image-preprocess.js

@@ -1,151 +0,0 @@
-
-
-import { trace } from '../../../debug/index.js';
-
-/**
- * Preprocess an image for Qwen3-VL vision encoder.
- *
- * Accepts raw pixel data (Uint8Array RGBA or RGB, or Float32Array normalized)
- * and returns a GPU-ready Float32Array of shape [C, H, W] after:
- *   1. Resize to fit min/max pixel constraints
- *   2. Pad to patch-aligned dimensions
- *   3. Normalize with mean/std
- *   4. Extract temporal patches (for video; single frame for images)
- *
- * @param {Uint8Array|Float32Array} pixels   Raw pixel data (RGBA or RGB)
- * @param {number}                  width    Source image width
- * @param {number}                  height   Source image height
- * @param {object}                  config   Vision config from manifest/preset
- * @returns {{ data: Float32Array, gridThw: [number, number, number], patchedHeight: number, patchedWidth: number }}
- */
-export function preprocessImage(pixels, width, height, config) {
-  const {
-    patchSize = 16,
-    spatialMergeSize = 2,
-    temporalPatchSize = 2,
-    minPixels = 3136,
-    maxPixels = 1003520,
-    normalization = {},
-  } = config;
-
-  const mean = normalization.mean || [0.48145466, 0.4578275, 0.40821073];
-  const std = normalization.std || [0.26862954, 0.26130258, 0.27577711];
-
-  // Step 1: Compute target dimensions respecting pixel constraints and patch alignment.
-  const mergedPatch = patchSize * spatialMergeSize;
-  const { targetWidth, targetHeight } = computeTargetDimensions(
-    width, height, minPixels, maxPixels, mergedPatch,
-  );
-
-  trace('vision', `preprocess: ${width}x${height} -> ${targetWidth}x${targetHeight} (patch=${patchSize}, merge=${spatialMergeSize})`);
-
-  // Step 2: Resize to target dimensions (bilinear interpolation on CPU).
-  const channels = 3;
-  const resized = resizeBilinear(pixels, width, height, targetWidth, targetHeight, channels);
-
-  // Step 3: Normalize to [0,1] then apply mean/std normalization.
-  const normalized = new Float32Array(channels * targetHeight * targetWidth);
-  for (let c = 0; c < channels; c++) {
-    const m = mean[c];
-    const s = std[c];
-    for (let y = 0; y < targetHeight; y++) {
-      for (let x = 0; x < targetWidth; x++) {
-        const srcIdx = (y * targetWidth + x) * channels + c;
-        const dstIdx = c * targetHeight * targetWidth + y * targetWidth + x;
-        normalized[dstIdx] = (resized[srcIdx] / 255.0 - m) / s;
-      }
-    }
-  }
-
-  // Step 4: Compute grid dimensions for the LLM.
-  //   gridT = 1 for single image (temporalPatchSize frames per temporal patch)
-  //   gridH = targetHeight / patchSize
-  //   gridW = targetWidth / patchSize
-  const gridT = 1;
-  const gridH = Math.floor(targetHeight / patchSize);
-  const gridW = Math.floor(targetWidth / patchSize);
-
-  return {
-    data: normalized,
-    width: targetWidth,
-    height: targetHeight,
-    channels,
-    gridThw: [gridT, gridH, gridW],
-    patchedHeight: targetHeight,
-    patchedWidth: targetWidth,
-  };
-}
-
-/**
- * Compute target dimensions that satisfy:
- *   - Total pixels >= minPixels and <= maxPixels
- *   - Both dimensions are multiples of mergedPatch
- *   - Aspect ratio is preserved as closely as possible
- */
-function computeTargetDimensions(width, height, minPixels, maxPixels, mergedPatch) {
-  const aspectRatio = width / height;
-
-  // Start from the geometric mean of min/max pixel counts.
-  let targetPixels = Math.sqrt(minPixels * maxPixels);
-  targetPixels = Math.max(minPixels, Math.min(maxPixels, targetPixels));
-
-  // Compute dimensions preserving aspect ratio.
-  let h = Math.sqrt(targetPixels / aspectRatio);
-  let w = h * aspectRatio;
-
-  // Round to nearest mergedPatch multiple.
-  h = Math.max(mergedPatch, Math.round(h / mergedPatch) * mergedPatch);
-  w = Math.max(mergedPatch, Math.round(w / mergedPatch) * mergedPatch);
-
-  // Clamp total pixels.
-  if (h * w > maxPixels) {
-    const scale = Math.sqrt(maxPixels / (h * w));
-    h = Math.max(mergedPatch, Math.round((h * scale) / mergedPatch) * mergedPatch);
-    w = Math.max(mergedPatch, Math.round((w * scale) / mergedPatch) * mergedPatch);
-  }
-  if (h * w < minPixels) {
-    const scale = Math.sqrt(minPixels / (h * w));
-    h = Math.max(mergedPatch, Math.round((h * scale) / mergedPatch) * mergedPatch);
-    w = Math.max(mergedPatch, Math.round((w * scale) / mergedPatch) * mergedPatch);
-  }
-
-  return { targetWidth: w, targetHeight: h };
-}
-
-/**
- * Bilinear resize of interleaved RGB(A) pixel data.
- * Input: Uint8Array or Float32Array in [H, W, C] layout (C >= 3, only first 3 used).
- * Output: Float32Array in [H, W, 3] layout with values in [0, 255].
- */
-function resizeBilinear(src, srcW, srcH, dstW, dstH, channels) {
-  const srcChannels = src.length / (srcW * srcH);
-  const out = new Float32Array(dstH * dstW * channels);
-  const scaleX = srcW / dstW;
-  const scaleY = srcH / dstH;
-
-  for (let y = 0; y < dstH; y++) {
-    const srcY = y * scaleY;
-    const y0 = Math.min(Math.floor(srcY), srcH - 1);
-    const y1 = Math.min(y0 + 1, srcH - 1);
-    const fy = srcY - y0;
-
-    for (let x = 0; x < dstW; x++) {
-      const srcX = x * scaleX;
-      const x0 = Math.min(Math.floor(srcX), srcW - 1);
-      const x1 = Math.min(x0 + 1, srcW - 1);
-      const fx = srcX - x0;
-
-      for (let c = 0; c < channels; c++) {
-        const v00 = src[(y0 * srcW + x0) * srcChannels + c];
-        const v01 = src[(y0 * srcW + x1) * srcChannels + c];
-        const v10 = src[(y1 * srcW + x0) * srcChannels + c];
-        const v11 = src[(y1 * srcW + x1) * srcChannels + c];
-        const top = v00 + (v01 - v00) * fx;
-        const bot = v10 + (v11 - v10) * fx;
-        out[(y * dstW + x) * channels + c] = top + (bot - top) * fy;
-      }
-    }
-  }
-
-  return out;
-}

package/src/inference/pipelines/vision/index.js

@@ -1,173 +0,0 @@
-
-
-import { trace } from '../../../debug/index.js';
-import { getDevice } from '../../../gpu/device.js';
-import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
-import { preprocessImage } from './image-preprocess.js';
-import { patchEmbed } from './patch-embed.js';
-import { runVisionEncoder } from './encoder.js';
-
-/**
- * Encode an image through the Qwen3-VL vision pipeline.
- *
- * Full flow:
- *   raw pixels -> preprocess -> patch embed -> ViT blocks -> spatial merge -> visual tokens
- *
- * @param {object} params
- * @param {Uint8Array|Float32Array} params.pixels   Raw image pixel data (RGBA or RGB)
- * @param {number}                  params.width    Image width
- * @param {number}                  params.height   Image height
- * @param {object}                  params.visionConfig  Vision config from manifest
- * @param {object}                  params.weights  Vision encoder weight buffers
- * @returns {Promise<VisionEncodeResult>}
- */
-export async function encodeImage(params) {
-  const { pixels, width, height, visionConfig, weights } = params;
-
-  trace('vision', `encodeImage: ${width}x${height} input`);
-
-  // Step 1: Preprocess — resize, normalize, compute grid.
-  const preprocessed = preprocessImage(pixels, width, height, visionConfig);
-
-  // Step 2: Patch embedding — conv2d patches -> [numPatches, hiddenSize].
-  const { patchBuffer, numPatches } = await patchEmbed({
-    imageData: preprocessed.data,
-    height: preprocessed.height,
-    width: preprocessed.width,
-    channels: preprocessed.channels,
-    visionConfig,
-    weights,
-  });
-
-  // Step 3: Vision encoder — ViT blocks + spatial merge.
-  const { features, numTokens } = await runVisionEncoder({
-    patchBuffer,
-    numPatches,
-    visionConfig,
-    weights,
-  });
-
-  return {
-    features,
-    numTokens,
-    gridThw: preprocessed.gridThw,
-    imageWidth: preprocessed.width,
-    imageHeight: preprocessed.height,
-  };
-}
-
-/**
- * Inject visual tokens into text token embeddings.
- *
- * Replaces positions in the embedding sequence where image_token_id appears
- * with the encoded visual features from the vision encoder.
- *
- * For Qwen3-VL with DeepStack, visual tokens are injected at specific decoder
- * layers (deepstackVisualIndexes), not at the input embedding level.
- * This function handles the simpler input-level injection case.
- * DeepStack injection is handled in the decoder layer loop.
- *
- * @param {object} params
- * @param {Float32Array}  params.textEmbeddings    [seqLen, hiddenSize]
- * @param {Int32Array}    params.tokenIds          [seqLen]
- * @param {GPUBuffer}     params.visualFeatures    [numVisualTokens, outHiddenSize]
- * @param {number}        params.numVisualTokens   Number of visual tokens
- * @param {number}        params.imageTokenId      Token ID marking image positions
- * @param {number}        params.hiddenSize        Text model hidden size
- * @returns {{ mergedEmbeddings: Float32Array, mergedLength: number }}
- */
-export function mergeVisualTokens(params) {
-  const {
-    textEmbeddings, tokenIds, visualFeatures,
-    numVisualTokens, imageTokenId, hiddenSize,
-  } = params;
-
-  // Count image token positions.
-  const imagePositions = [];
-  for (let i = 0; i < tokenIds.length; i++) {
-    if (tokenIds[i] === imageTokenId) {
-      imagePositions.push(i);
-    }
-  }
-
-  if (imagePositions.length === 0) {
-    trace('vision', 'mergeVisualTokens: no image tokens found, returning text-only');
-    return { mergedEmbeddings: textEmbeddings, mergedLength: tokenIds.length };
-  }
-
-  trace('vision', `mergeVisualTokens: replacing ${imagePositions.length} image tokens with ${numVisualTokens} visual tokens`);
-
-  // The merged sequence replaces contiguous image_token_id runs with visual features.
-  // For Qwen3-VL: image tokens appear as a block between vision_start and vision_end tokens.
-  // The visual features replace the entire image token block.
-
-  // Find contiguous image token ranges.
-  const ranges = [];
-  let rangeStart = imagePositions[0];
-  let rangeEnd = imagePositions[0];
-  for (let i = 1; i < imagePositions.length; i++) {
-    if (imagePositions[i] === rangeEnd + 1) {
-      rangeEnd = imagePositions[i];
-    } else {
-      ranges.push([rangeStart, rangeEnd]);
-      rangeStart = imagePositions[i];
-      rangeEnd = imagePositions[i];
-    }
-  }
-  ranges.push([rangeStart, rangeEnd]);
-
-  // Build merged sequence: text tokens (non-image) + visual tokens replacing each range.
-  const textLen = tokenIds.length;
-  const replacedCount = imagePositions.length;
-  const mergedLength = textLen - replacedCount + numVisualTokens;
-  const merged = new Float32Array(mergedLength * hiddenSize);
-
-  let srcPos = 0;
-  let dstPos = 0;
-  let visualOffset = 0;
-
-  for (const [start, end] of ranges) {
-    // Copy text tokens before this range.
-    const textBefore = start - srcPos;
-    if (textBefore > 0) {
-      merged.set(
-        textEmbeddings.subarray(srcPos * hiddenSize, start * hiddenSize),
-        dstPos * hiddenSize,
-      );
-      dstPos += textBefore;
-    }
-
-    // Insert visual tokens replacing this range.
-    const rangeLen = end - start + 1;
-    const tokensToInsert = Math.min(numVisualTokens - visualOffset, rangeLen);
-    // Copy from visual features buffer (CPU side).
-    for (let i = 0; i < tokensToInsert; i++) {
-      for (let d = 0; d < hiddenSize; d++) {
-        merged[(dstPos + i) * hiddenSize + d] = visualFeatures[(visualOffset + i) * hiddenSize + d];
-      }
-    }
-    dstPos += tokensToInsert;
-    visualOffset += tokensToInsert;
-    srcPos = end + 1;
-  }
-
-  // Copy remaining text tokens after last range.
-  if (srcPos < textLen) {
-    merged.set(
-      textEmbeddings.subarray(srcPos * hiddenSize, textLen * hiddenSize),
-      dstPos * hiddenSize,
-    );
-    dstPos += textLen - srcPos;
-  }
-
-  return { mergedEmbeddings: merged, mergedLength: dstPos };
-}
-
-/**
- * @typedef {object} VisionEncodeResult
- * @property {GPUBuffer}  features     Encoded visual tokens [numTokens, outHiddenSize]
- * @property {number}     numTokens    Number of visual tokens after spatial merge
- * @property {number[]}   gridThw      [temporal, height, width] grid dimensions
- * @property {number}     imageWidth   Processed image width
- * @property {number}     imageHeight  Processed image height
- */