@huggingface/transformers 4.0.0-next.6 → 4.0.0-next.8

package/src/models/models.js

@@ -2,7 +2,7 @@ export * from './albert/modeling_albert.js';
 export * from './apertus/modeling_apertus.js';
 export * from './afmoe/modeling_afmoe.js';
 export * from './arcee/modeling_arcee.js';
-export * from './ast/modeling_ast.js';
+export * from './audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js';
 export * from './bart/modeling_bart.js';
 export * from './beit/modeling_beit.js';
 export * from './bert/modeling_bert.js';
@@ -12,6 +12,7 @@ export * from './bloom/modeling_bloom.js';
 export * from './camembert/modeling_camembert.js';
 export * from './chatterbox/modeling_chatterbox.js';
 export * from './chinese_clip/modeling_chinese_clip.js';
+export * from './chmv2/modeling_chmv2.js';
 export * from './clap/modeling_clap.js';
 export * from './clip/modeling_clip.js';
 export * from './clipseg/modeling_clipseg.js';
@@ -24,6 +25,7 @@ export * from './convnextv2/modeling_convnextv2.js';
 export * from './d_fine/modeling_d_fine.js';
 export * from './dac/modeling_dac.js';
 export * from './deberta/modeling_deberta.js';
+export * from './deepseek_v3/modeling_deepseek_v3.js';
 export * from './deberta_v2/modeling_deberta_v2.js';
 export * from './decision_transformer/modeling_decision_transformer.js';
 export * from './deit/modeling_deit.js';
@@ -41,6 +43,7 @@ export * from './efficientnet/modeling_efficientnet.js';
 export * from './electra/modeling_electra.js';
 export * from './ernie4_5/modeling_ernie4_5.js';
 export * from './esm/modeling_esm.js';
+export * from './eurobert/modeling_eurobert.js';
 export * from './exaone/modeling_exaone.js';
 export * from './falcon/modeling_falcon.js';
 export * from './falcon_h1/modeling_falcon_h1.js';
@@ -51,6 +54,8 @@ export * from './gemma2/modeling_gemma2.js';
 export * from './gemma3/modeling_gemma3.js';
 export * from './gemma3n/modeling_gemma3n.js';
 export * from './glm/modeling_glm.js';
+export * from './glm_moe_dsa/modeling_glm_moe_dsa.js';
+export * from './glm_ocr/modeling_glm_ocr.js';
 export * from './glpn/modeling_glpn.js';
 export * from './gpt_bigcode/modeling_gpt_bigcode.js';
 export * from './gpt_neo/modeling_gpt_neo.js';
@@ -60,6 +65,7 @@ export * from './gpt2/modeling_gpt2.js';
 export * from './gptj/modeling_gptj.js';
 export * from './granite/modeling_granite.js';
 export * from './granitemoehybrid/modeling_granitemoehybrid.js';
+export * from './granite_speech/modeling_granite_speech.js';
 export * from './grounding_dino/modeling_grounding_dino.js';
 export * from './groupvit/modeling_groupvit.js';
 export * from './helium/modeling_helium.js';
@@ -71,7 +77,9 @@ export * from './ijepa/modeling_ijepa.js';
 export * from './jais/modeling_jais.js';
 export * from './jina_clip/modeling_jina_clip.js';
 export * from './lfm2/modeling_lfm2.js';
+export * from './lighton_ocr/modeling_lighton_ocr.js';
 export * from './lfm2_moe/modeling_lfm2_moe.js';
+export * from './lfm2_vl/modeling_lfm2_vl.js';
 export * from './llama/modeling_llama.js';
 export * from './llama4/modeling_llama4.js';
 export * from './llava/modeling_llava.js';
@@ -86,6 +94,7 @@ export * from './metric3dv2/modeling_metric3dv2.js';
 export * from './mgp_str/modeling_mgp_str.js';
 export * from './mimi/modeling_mimi.js';
 export * from './mistral/modeling_mistral.js';
+export * from './mistral4/modeling_mistral4.js';
 export * from './mobilebert/modeling_mobilebert.js';
 export * from './mobilellm/modeling_mobilellm.js';
 export * from './mobilenet_v1/modeling_mobilenet_v1.js';
@@ -103,6 +112,7 @@ export * from './mt5/modeling_mt5.js';
 export * from './multi_modality/modeling_multi_modality.js';
 export * from './musicgen/modeling_musicgen.js';
 export * from './nanochat/modeling_nanochat.js';
+export * from './nemotron_h/modeling_nemotron_h.js';
 export * from './neobert/modeling_neobert.js';
 export * from './nomic_bert/modeling_nomic_bert.js';
 export * from './olmo/modeling_olmo.js';
@@ -146,6 +156,7 @@ export * from './segformer/modeling_segformer.js';
 export * from './siglip/modeling_siglip.js';
 export * from './smollm3/modeling_smollm3.js';
 export * from './snac/modeling_snac.js';
+export * from './solar_open/modeling_solar_open.js';
 export * from './speecht5/modeling_speecht5.js';
 export * from './squeezebert/modeling_squeezebert.js';
 export * from './stablelm/modeling_stablelm.js';
@@ -168,6 +179,8 @@ export * from './vit_msn/modeling_vit_msn.js';
 export * from './vitmatte/modeling_vitmatte.js';
 export * from './vitpose/modeling_vitpose.js';
 export * from './vits/modeling_vits.js';
+export * from './voxtral/modeling_voxtral.js';
+export * from './voxtral_realtime/modeling_voxtral_realtime.js';
 export * from './wav2vec2/modeling_wav2vec2.js';
 export * from './wav2vec2_bert/modeling_wav2vec2_bert.js';
 export * from './wavlm/modeling_wavlm.js';

package/src/models/nemotron_h/modeling_nemotron_h.js

@@ -0,0 +1,5 @@
+import { PreTrainedModel } from '../modeling_utils.js';
+
+export class NemotronHPreTrainedModel extends PreTrainedModel {}
+export class NemotronHModel extends NemotronHPreTrainedModel {}
+export class NemotronHForCausalLM extends NemotronHPreTrainedModel {}

package/src/models/paligemma/modeling_paligemma.js

@@ -1,26 +1,3 @@
-import { PreTrainedModel, default_merge_input_ids_with_image_features } from '../modeling_utils.js';
+import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js';
 
-export class PaliGemmaPreTrainedModel extends PreTrainedModel {
-    forward_params = [
-        'input_ids',
-        // 'inputs_embeds',
-        'attention_mask',
-        'pixel_values',
-        'position_ids',
-        'past_key_values',
-    ];
-}
-
-export class PaliGemmaForConditionalGeneration extends PaliGemmaPreTrainedModel {
-    _merge_input_ids_with_image_features(kwargs) {
-        const vision_hidden_size = kwargs.image_features.dims.at(-1);
-        const reshaped_image_hidden_states = kwargs.image_features.view(-1, vision_hidden_size);
-
-        return default_merge_input_ids_with_image_features({
-            // @ts-ignore
-            image_token_id: this.config.image_token_index,
-            ...kwargs,
-            image_features: reshaped_image_hidden_states,
-        });
-    }
-}
+export class PaliGemmaForConditionalGeneration extends LlavaForConditionalGeneration {}

package/src/models/processors.js

@@ -1,10 +1,13 @@
 export * from './chatterbox/processing_chatterbox.js';
 export * from './florence2/processing_florence2.js';
 export * from './gemma3n/processing_gemma3n.js';
+export * from './glm46v/processing_glm46v.js';
+export * from './granite_speech/processing_granite_speech.js';
 export * from './grounding_dino/processing_grounding_dino.js';
 export * from './idefics3/processing_idefics3.js';
 export * from './janus/processing_janus.js';
 export * from './jina_clip/processing_jina_clip.js';
+export * from './lfm2_vl/processing_lfm2_vl.js';
 export * from './llava/processing_llava.js';
 export * from './mgp_str/processing_mgp_str.js';
 export * from './moonshine/processing_moonshine.js';
@@ -22,6 +25,7 @@ export * from './smolvlm/processing_smolvlm.js';
 export * from './speecht5/processing_speecht5.js';
 export * from './ultravox/processing_ultravox.js';
 export * from './voxtral/processing_voxtral.js';
+export * from './voxtral_realtime/processing_voxtral_realtime.js';
 export * from './wav2vec2/processing_wav2vec2.js';
 export * from './wav2vec2_with_lm/processing_wav2vec2_with_lm.js';
 export * from './whisper/processing_whisper.js';

package/src/models/qwen2_5_vl/modeling_qwen2_5_vl.js

@@ -1,5 +1,9 @@
-import { Qwen2VLForConditionalGeneration } from '../qwen2_vl/modeling_qwen2_vl.js';
+import { Qwen2VLForConditionalGeneration, Qwen2VLForCausalLM } from '../qwen2_vl/modeling_qwen2_vl.js';
 
 export class Qwen2_5_VLForConditionalGeneration extends Qwen2VLForConditionalGeneration {
     image_grid_thw_name = 'image_grid_thw';
 }
+
+export class Qwen2_5_VLForCausalLM extends Qwen2VLForCausalLM {
+    image_grid_thw_name = 'image_grid_thw';
+}

package/src/models/qwen2_vl/image_processing_qwen2_vl.js

@@ -1,46 +1,6 @@
-import { ImageProcessor } from '../../image_processors_utils.js';
+import { ImageProcessor, smart_resize } from '../../image_processors_utils.js';
 import { cat, Tensor } from '../../utils/tensor.js';
 
-/**
- * Rescales the image so that the following conditions are met:
- *
- * 1. Both dimensions (height and width) are divisible by 'factor'.
- * 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
- * 3. The aspect ratio of the image is maintained as closely as possible.
- *
- * @param {number} height The height of the image.
- * @param {number} width The width of the image.
- * @param {number} [factor=28] The factor to use for resizing.
- * @param {number} [min_pixels=56*56] The minimum number of pixels.
- * @param {number} [max_pixels=14*14*4*1280] The maximum number of pixels.
- * @returns {[number, number]} The new height and width of the image.
- * @throws {Error} If the height or width is smaller than the factor.
- */
-function smart_resize(height, width, factor = 28, min_pixels = 56 * 56, max_pixels = 14 * 14 * 4 * 1280) {
-    if (height < factor || width < factor) {
-        throw new Error(`height:${height} or width:${width} must be larger than factor:${factor}`);
-    } else if (Math.max(height, width) / Math.min(height, width) > 200) {
-        throw new Error(
-            `absolute aspect ratio must be smaller than 200, got ${Math.max(height, width) / Math.min(height, width)}`,
-        );
-    }
-
-    let h_bar = Math.round(height / factor) * factor;
-    let w_bar = Math.round(width / factor) * factor;
-
-    if (h_bar * w_bar > max_pixels) {
-        const beta = Math.sqrt((height * width) / max_pixels);
-        h_bar = Math.floor(height / beta / factor) * factor;
-        w_bar = Math.floor(width / beta / factor) * factor;
-    } else if (h_bar * w_bar < min_pixels) {
-        const beta = Math.sqrt(min_pixels / (height * width));
-        h_bar = Math.ceil((height * beta) / factor) * factor;
-        w_bar = Math.ceil((width * beta) / factor) * factor;
-    }
-
-    return [h_bar, w_bar];
-}
-
 export class Qwen2VLImageProcessor extends ImageProcessor {
     constructor(config) {
         super(config);

package/src/models/qwen2_vl/modeling_qwen2_vl.js

@@ -1,9 +1,4 @@
-import {
-    PreTrainedModel,
-    cumsum_masked_fill,
-    default_merge_input_ids_with_image_features,
-    getPastLength,
-} from '../modeling_utils.js';
+import { PreTrainedModel, cumsum_masked_fill, default_merge_input_ids_with_image_features } from '../modeling_utils.js';
 import { sessionRun } from '../session.js';
 import { stack, Tensor, ones_like, zeros } from '../../utils/tensor.js';
 import { max } from '../../utils/maths.js';
@@ -22,8 +17,172 @@ export class Qwen2VLPreTrainedModel extends PreTrainedModel {
     ];
 }
 export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
+    // NOTE: This is used as the base class for all Qwen VL models and their CausalLM variants.
+    // CausalLM variants (e.g., Qwen2VLForCausalLM) extend this class but load only
+    // embed_tokens + decoder_model_merged (no vision_encoder) via MultimodalLanguageModelOnly type.
     image_grid_thw_name = 'grid_thw';
 
+    /**
+     * Compute text-only 3D rope position IDs (all 3 dims get the same 1D positions).
+     * @param {Tensor} input_ids
+     * @param {Tensor} attention_mask
+     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas]
+     */
+    _get_text_only_rope_index(input_ids, attention_mask) {
+        if (attention_mask) {
+            const { data, dims } = cumsum_masked_fill(attention_mask);
+
+            const position_ids = BigInt64Array.from({ length: 3 * data.length }, (_, i) => data[i % data.length]);
+            /** @type {bigint[]} */
+            const mrope_position_deltas = Array.from(
+                { length: dims[0] },
+                (_, i) => max(data.subarray(dims[1] * i, dims[1] * (i + 1)))[0] + 1n + BigInt(dims[1]),
+            );
+
+            return [
+                new Tensor('int64', position_ids, [3, ...dims]),
+                new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
+            ];
+        } else {
+            const [batch_size, seq_length] = input_ids.dims;
+            const position_ids = BigInt64Array.from({ length: 3 * batch_size * seq_length }, (_, i) =>
+                BigInt(Math.floor((i % seq_length) / batch_size)),
+            );
+
+            return [new Tensor('int64', position_ids, [3, ...input_ids.dims]), zeros([batch_size, 1])];
+        }
+    }
+
+    /**
+     * Reorder per-segment position ID lists from [seg1[t,h,w], seg2[t,h,w], ...] into
+     * global [all_t, all_h, all_w] order, then write back into the position_ids array
+     * respecting attention mask.
+     * @param {number[][]} llm_pos_ids_list List of per-segment position arrays, each of length 3*seg_len
+     * @param {number[]} attn_mask Attention mask for this batch element
+     * @param {number[][][]} position_ids_list [3][batch][seq] output array to write into
+     * @param {number} batch_idx Current batch index
+     * @returns {number[]} Flat reordered positions of length total_len
+     */
+    _reorder_and_write_positions(llm_pos_ids_list, attn_mask, position_ids_list, batch_idx) {
+        const total_len = llm_pos_ids_list.reduce((acc, x) => acc + x.length, 0);
+        const llm_positions = new Array(total_len);
+        let index = 0;
+        for (let x = 0; x < 3; ++x) {
+            for (const val of llm_pos_ids_list) {
+                const seg_len = val.length / 3;
+                for (let z = x * seg_len; z < (x + 1) * seg_len; ++z) {
+                    llm_positions[index++] = val[z];
+                }
+            }
+        }
+
+        let count = 0;
+        for (let y = 0; y < attn_mask.length; ++y) {
+            if (attn_mask[y] == 1) {
+                for (let x = 0; x < 3; ++x) {
+                    position_ids_list[x][batch_idx][y] = llm_positions[(x * total_len) / 3 + count];
+                }
+                ++count;
+            }
+        }
+
+        return llm_positions;
+    }
+
+    /**
+     * Build per-batch position ID segments for multimodal rope.
+     * Override this in subclasses to change how vision/text segments are identified and positioned.
+     * @param {object} params
+     * @param {any[]} params.filtered_ids - attention-masked token IDs for this batch element
+     * @param {any[][]} params.image_grid_thw_list - all image grid dimensions
+     * @param {any[][]} params.video_grid_thw_list - all video grid dimensions
+     * @param {number} params.spatial_merge_size
+     * @param {{image_index: number, video_index: number}} params.state - mutable counters shared across batches
+     * @returns {number[][]} llm_pos_ids_list - segments of [t..., h..., w...] positions
+     */
+    _get_multimodal_rope_positions({
+        filtered_ids,
+        image_grid_thw_list,
+        video_grid_thw_list,
+        spatial_merge_size,
+        state,
+    }) {
+        // @ts-ignore
+        const { image_token_id, video_token_id, vision_start_token_id } = this.config;
+
+        const ids = filtered_ids;
+        const vision_start_indices = ids.reduce((acc, x, idx) => {
+            if (x == vision_start_token_id) acc.push(idx);
+            return acc;
+        }, []);
+
+        const vision_tokens = vision_start_indices.map((x) => ids[x + 1]);
+        const image_nums = vision_tokens.filter((x) => x == image_token_id).length;
+        const video_nums = vision_tokens.filter((x) => x == video_token_id).length;
+
+        /** @type {number[][]} */
+        const llm_pos_ids_list = [];
+        let st = 0;
+        let remain_images = image_nums;
+        let remain_videos = video_nums;
+        for (let j = 0; j < vision_tokens.length; ++j) {
+            const next_image_token = ids.findIndex((x, i) => i > st && x == image_token_id);
+            const next_video_token = ids.findIndex((x, i) => i > st && x == video_token_id);
+
+            const ed_image = remain_images > 0 && next_image_token !== -1 ? next_image_token : ids.length + 1;
+            const ed_video = remain_videos > 0 && next_video_token !== -1 ? next_video_token : ids.length + 1;
+
+            let ed;
+            let t, h, w;
+            if (ed_image < ed_video) {
+                [t, h, w] = image_grid_thw_list[state.image_index];
+                ++state.image_index;
+                --remain_images;
+                ed = ed_image;
+            } else {
+                [t, h, w] = video_grid_thw_list[state.video_index];
+                ++state.video_index;
+                --remain_videos;
+                ed = ed_video;
+            }
+
+            const [llm_grid_t, llm_grid_h, llm_grid_w] = [
+                Number(t),
+                Math.floor(Number(h) / spatial_merge_size),
+                Math.floor(Number(w) / spatial_merge_size),
+            ];
+            const text_len = ed - st;
+            const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
+
+            llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
+
+            const offset = text_len + st_idx;
+            const grid_size = llm_grid_t * llm_grid_h * llm_grid_w;
+            const t_index = Array.from(
+                { length: grid_size },
+                (_, i) => offset + Math.floor(i / (llm_grid_h * llm_grid_w)),
+            );
+            const h_index = Array.from(
+                { length: grid_size },
+                (_, i) => offset + (Math.floor(i / llm_grid_w) % llm_grid_h),
+            );
+            const w_index = Array.from({ length: grid_size }, (_, i) => offset + (i % llm_grid_w));
+
+            llm_pos_ids_list.push([t_index, h_index, w_index].flat());
+
+            st = ed + grid_size;
+        }
+
+        if (st < ids.length) {
+            const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
+            const text_len = ids.length - st;
+
+            llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
+        }
+
+        return llm_pos_ids_list;
+    }
+
     /**
      * Calculate the 3D rope index based on image and video's temporal, height and width in LLM.
      *
@@ -53,137 +212,49 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
      * @param {Tensor} input_ids Indices of input sequence tokens in the vocabulary. Tensor of shape `(batch_size, sequence_length)`.
      * @param {Tensor} image_grid_thw (Optional) The temporal, height and width of feature shape of each image in LLM. Tensor of shape `(num_images, 3)`.
      * @param {Tensor} video_grid_thw (Optional) The temporal, height and width of feature shape of each video in LLM. Tensor of shape `(num_videos, 3)`.
-     * @param {Tensor} attention_mask (Optional) Mask to avoid performing attention on padding token indices. Tensor of shape `(batch_size, sequence_length)`. Mask values selected in `[0, 1]`:
-     * - 1 for tokens that are **not masked**,
-     * - 0 for tokens that are **masked**.
-     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas] with:
-     * - position_ids: Tensor of shape `(3, batch_size, sequence_length)`.
-     * - mrope_position_deltas: Tensor of shape `(batch_size)`.
+     * @param {Tensor} attention_mask (Optional) Mask to avoid performing attention on padding token indices. Tensor of shape `(batch_size, sequence_length)`.
+     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas]
      */
     get_rope_index(input_ids, image_grid_thw, video_grid_thw, attention_mask) {
         // @ts-ignore
-        const { vision_config, image_token_id, video_token_id, vision_start_token_id } = this.config;
+        const { vision_config } = this.config;
         const spatial_merge_size = vision_config.spatial_merge_size ?? 2;
 
-        const mrope_position_deltas = [];
         if (image_grid_thw || video_grid_thw) {
-            let total_input_ids = input_ids.tolist();
+            const total_input_ids = input_ids.tolist();
             if (!attention_mask) {
                 attention_mask = ones_like(input_ids);
             }
 
             const attention_mask_list = attention_mask.tolist();
-            const position_ids_list = Array.from({ length: 3 }, (_) =>
-                Array.from({ length: input_ids.dims[0] }, (_) => Array.from({ length: input_ids.dims[1] }, (_) => 1)),
+            const position_ids_list = Array.from({ length: 3 }, () =>
+                Array.from({ length: input_ids.dims[0] }, () => Array.from({ length: input_ids.dims[1] }, () => 0)),
             );
 
             const image_grid_thw_list = image_grid_thw ? image_grid_thw.tolist() : [];
             const video_grid_thw_list = video_grid_thw ? video_grid_thw.tolist() : [];
+            const state = { image_index: 0, video_index: 0 };
 
-            let image_index = 0;
-            let video_index = 0;
+            const mrope_position_deltas = [];
             for (let i = 0; i < total_input_ids.length; ++i) {
-                const ids = total_input_ids[i].filter((_, j) => attention_mask_list[i][j] == 1);
-
-                const vision_start_indices = ids.reduce((acc, x, idx) => {
-                    if (x == vision_start_token_id) acc.push(idx);
-                    return acc;
-                }, []);
-
-                const vision_tokens = vision_start_indices.map((x) => ids[x + 1]);
-                const image_nums = vision_tokens.filter((x) => x == image_token_id).length;
-                const video_nums = vision_tokens.filter((x) => x == video_token_id).length;
-
-                /** @type {number[][]} */
-                let llm_pos_ids_list = [];
-                let st = 0;
-                let remain_images = image_nums;
-                let remain_videos = video_nums;
-                for (let j = 0; j < vision_tokens.length; ++j) {
-                    const next_image_token = ids.findIndex((x, i) => i > st && x == image_token_id);
-                    const next_video_token = ids.findIndex((x, i) => i > st && x == video_token_id);
-
-                    const ed_image = remain_images > 0 && next_image_token !== -1 ? next_image_token : ids.length + 1;
-
-                    const ed_video = remain_videos > 0 && next_video_token !== -1 ? next_video_token : ids.length + 1;
-
-                    let ed;
-                    let t, h, w;
-                    if (ed_image < ed_video) {
-                        [t, h, w] = image_grid_thw_list[image_index];
-                        ++image_index;
-                        --remain_images;
-                        ed = ed_image;
-                    } else {
-                        [t, h, w] = video_grid_thw_list[video_index];
-                        ++video_index;
-                        --remain_videos;
-                        ed = ed_video;
-                    }
-
-                    const [llm_grid_t, llm_grid_h, llm_grid_w] = [
-                        Number(t),
-                        Math.floor(Number(h) / spatial_merge_size),
-                        Math.floor(Number(w) / spatial_merge_size),
-                    ];
-                    const text_len = ed - st;
-                    const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
-
-                    llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
-
-                    const offset = text_len + st_idx;
-                    const grid_size = llm_grid_t * llm_grid_h * llm_grid_w;
-                    const t_index = Array.from(
-                        { length: grid_size },
-                        (_, i) => offset + Math.floor(i / (llm_grid_h * llm_grid_w)),
-                    );
-                    const h_index = Array.from(
-                        { length: grid_size },
-                        (_, i) => offset + (Math.floor(i / llm_grid_w) % llm_grid_h),
-                    );
-                    const w_index = Array.from({ length: grid_size }, (_, i) => offset + (i % llm_grid_w));
-
-                    llm_pos_ids_list.push([t_index, h_index, w_index].flat());
-
-                    st = ed + grid_size;
-                }
-
-                if (st < ids.length) {
-                    const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
-                    const text_len = ids.length - st;
-
-                    llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
-                }
-
-                // NOTE: Each item in llm_pos_ids_list is an array of shape (3, text_len),
-                // meaning to perform concatenation along dim=1, we can do the following:
-                const num_items = llm_pos_ids_list.reduce((acc, x) => acc + x.length, 0);
-                /** @type {number[]} */
-                const llm_positions = new Array(num_items);
-                let index = 0;
-                for (let x = 0; x < 3; ++x) {
-                    for (let y = 0; y < llm_pos_ids_list.length; ++y) {
-                        const val = llm_pos_ids_list[y];
-                        const text_len = val.length / 3;
-                        for (let z = x * text_len; z < (x + 1) * text_len; ++z) {
-                            llm_positions[index++] = val[z];
-                        }
-                    }
-                }
-
-                let count = 0;
-                const attn_mask = attention_mask_list[i];
-                for (let y = 0; y < attn_mask.length; ++y) {
-                    if (attn_mask[y] == 1) {
-                        for (let x = 0; x < 3; ++x) {
-                            position_ids_list[x][i][y] = llm_positions[(x * num_items) / 3 + count];
-                        }
-                        ++count;
-                    }
-                }
+                const filtered_ids = total_input_ids[i].filter((_, j) => attention_mask_list[i][j] == 1);
+
+                const llm_pos_ids_list = this._get_multimodal_rope_positions({
+                    filtered_ids,
+                    image_grid_thw_list,
+                    video_grid_thw_list,
+                    spatial_merge_size,
+                    state,
+                });
+
+                const llm_positions = this._reorder_and_write_positions(
+                    llm_pos_ids_list,
+                    attention_mask_list[i],
+                    position_ids_list,
+                    i,
+                );
 
-                const max_llm_positions = max(llm_positions)[0];
-                mrope_position_deltas.push(max_llm_positions + 1 - total_input_ids[i].length);
+                mrope_position_deltas.push(max(llm_positions)[0] + 1 - total_input_ids[i].length);
             }
 
             return [
@@ -191,29 +262,7 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
                 new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
             ];
         } else {
-            // Text-only
-            if (attention_mask) {
-                const { data, dims } = cumsum_masked_fill(attention_mask);
-
-                const position_ids = BigInt64Array.from({ length: 3 * data.length }, (_, i) => data[i % data.length]);
-                /** @type {bigint[]} */
-                const mrope_position_deltas = Array.from(
-                    { length: dims[0] },
-                    (_, i) => max(data.subarray(dims[1] * i, dims[1] * (i + 1)))[0] + 1n + BigInt(dims[1]),
-                );
-
-                return [
-                    new Tensor('int64', position_ids, [3, ...dims]),
-                    new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
-                ];
-            } else {
-                const [batch_size, seq_length] = input_ids.dims;
-                const position_ids = BigInt64Array.from({ length: 3 * batch_size * seq_length }, (_, i) =>
-                    BigInt(Math.floor((i % seq_length) / batch_size)),
-                );
-
-                return [new Tensor('int64', position_ids, [3, ...input_ids.dims]), zeros([batch_size, 1])];
-            }
+            return this._get_text_only_rope_index(input_ids, attention_mask);
         }
     }
 
@@ -250,7 +299,7 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
                 model_inputs.pixel_values = null;
                 // model_inputs.pixel_values_videos = null;
 
-                const past_length = getPastLength(model_inputs.past_key_values);
+                const past_length = model_inputs.past_key_values.get_seq_length();
 
                 if (past_length < model_inputs.input_ids.dims[1]) {
                     // Externally provided `past_key_values` with full input_ids:
@@ -287,3 +336,5 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
         return model_inputs;
     }
 }
+
+export class Qwen2VLForCausalLM extends Qwen2VLForConditionalGeneration {}

package/src/models/qwen2_vl/processing_qwen2_vl.js

@@ -6,6 +6,7 @@ import { RawImage } from '../../utils/image.js';
 export class Qwen2VLProcessor extends Processor {
     static image_processor_class = AutoImageProcessor;
     static tokenizer_class = AutoTokenizer;
+    static image_token = '<|image_pad|>';
 
     /**
      *
@@ -31,13 +32,14 @@ export class Qwen2VLProcessor extends Processor {
             let merge_length = this.image_processor.config.merge_size ** 2;
             let index = 0;
 
+            const image_token = /** @type {typeof Qwen2VLProcessor} */ (this.constructor).image_token;
             const image_grid_thw_list = image_grid_thw.tolist();
             text = text.map((t) => {
-                while (t.includes('<|image_pad|>')) {
+                while (t.includes(image_token)) {
                     const prod = Number(image_grid_thw_list[index++].reduce((a, b) => a * b, 1n));
-                    t = t.replace('<|image_pad|>', '<|placeholder|>'.repeat(Math.floor(prod / merge_length)));
+                    t = t.replace(image_token, '<|placeholder|>'.repeat(Math.floor(prod / merge_length)));
                 }
-                return t.replaceAll('<|placeholder|>', '<|image_pad|>');
+                return t.replaceAll('<|placeholder|>', image_token);
             });
         }
 
@@ -46,7 +48,6 @@ export class Qwen2VLProcessor extends Processor {
         return {
             ...text_inputs,
             ...image_inputs,
-            // TODO: ...videos_inputs,
         };
     }
 }

package/src/models/qwen3_5/modeling_qwen3_5.js

@@ -1,3 +1,4 @@
 import { Qwen3VLForConditionalGeneration } from '../qwen3_vl/modeling_qwen3_vl.js';
 
 export class Qwen3_5ForConditionalGeneration extends Qwen3VLForConditionalGeneration {}
+export class Qwen3_5ForCausalLM extends Qwen3_5ForConditionalGeneration {}

package/src/models/qwen3_5_moe/modeling_qwen3_5_moe.js

@@ -1,3 +1,4 @@
-import { Qwen3_5ForConditionalGeneration } from '../qwen3_5/modeling_qwen3_5.js';
+import { Qwen3_5ForConditionalGeneration, Qwen3_5ForCausalLM } from '../qwen3_5/modeling_qwen3_5.js';
 
 export class Qwen3_5MoeForConditionalGeneration extends Qwen3_5ForConditionalGeneration {}
+export class Qwen3_5MoeForCausalLM extends Qwen3_5ForCausalLM {}

package/src/models/qwen3_vl/modeling_qwen3_vl.js

@@ -1,3 +1,4 @@
-import { Qwen2_5_VLForConditionalGeneration } from '../qwen2_5_vl/modeling_qwen2_5_vl.js';
+import { Qwen2_5_VLForConditionalGeneration, Qwen2_5_VLForCausalLM } from '../qwen2_5_vl/modeling_qwen2_5_vl.js';
 
 export class Qwen3VLForConditionalGeneration extends Qwen2_5_VLForConditionalGeneration {}
+export class Qwen3VLForCausalLM extends Qwen2_5_VLForCausalLM {}

package/src/models/qwen3_vl_moe/modeling_qwen3_vl_moe.js

@@ -1,3 +1,4 @@
-import { Qwen3VLForConditionalGeneration } from '../qwen3_vl/modeling_qwen3_vl.js';
+import { Qwen3VLForConditionalGeneration, Qwen3VLForCausalLM } from '../qwen3_vl/modeling_qwen3_vl.js';
 
 export class Qwen3VLMoeForConditionalGeneration extends Qwen3VLForConditionalGeneration {}
+export class Qwen3VLMoeForCausalLM extends Qwen3VLForCausalLM {}