@huggingface/transformers 4.0.0-next.7 → 4.0.0-next.9

package/src/models/models.js

@@ -2,7 +2,7 @@ export * from './albert/modeling_albert.js';
 export * from './apertus/modeling_apertus.js';
 export * from './afmoe/modeling_afmoe.js';
 export * from './arcee/modeling_arcee.js';
-export * from './ast/modeling_ast.js';
+export * from './audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.js';
 export * from './bart/modeling_bart.js';
 export * from './beit/modeling_beit.js';
 export * from './bert/modeling_bert.js';
@@ -12,6 +12,7 @@ export * from './bloom/modeling_bloom.js';
 export * from './camembert/modeling_camembert.js';
 export * from './chatterbox/modeling_chatterbox.js';
 export * from './chinese_clip/modeling_chinese_clip.js';
+export * from './chmv2/modeling_chmv2.js';
 export * from './clap/modeling_clap.js';
 export * from './clip/modeling_clip.js';
 export * from './clipseg/modeling_clipseg.js';
@@ -24,6 +25,7 @@ export * from './convnextv2/modeling_convnextv2.js';
 export * from './d_fine/modeling_d_fine.js';
 export * from './dac/modeling_dac.js';
 export * from './deberta/modeling_deberta.js';
+export * from './deepseek_v3/modeling_deepseek_v3.js';
 export * from './deberta_v2/modeling_deberta_v2.js';
 export * from './decision_transformer/modeling_decision_transformer.js';
 export * from './deit/modeling_deit.js';
@@ -41,6 +43,7 @@ export * from './efficientnet/modeling_efficientnet.js';
 export * from './electra/modeling_electra.js';
 export * from './ernie4_5/modeling_ernie4_5.js';
 export * from './esm/modeling_esm.js';
+export * from './eurobert/modeling_eurobert.js';
 export * from './exaone/modeling_exaone.js';
 export * from './falcon/modeling_falcon.js';
 export * from './falcon_h1/modeling_falcon_h1.js';
@@ -51,6 +54,8 @@ export * from './gemma2/modeling_gemma2.js';
 export * from './gemma3/modeling_gemma3.js';
 export * from './gemma3n/modeling_gemma3n.js';
 export * from './glm/modeling_glm.js';
+export * from './glm_moe_dsa/modeling_glm_moe_dsa.js';
+export * from './glm_ocr/modeling_glm_ocr.js';
 export * from './glpn/modeling_glpn.js';
 export * from './gpt_bigcode/modeling_gpt_bigcode.js';
 export * from './gpt_neo/modeling_gpt_neo.js';
@@ -72,6 +77,7 @@ export * from './ijepa/modeling_ijepa.js';
 export * from './jais/modeling_jais.js';
 export * from './jina_clip/modeling_jina_clip.js';
 export * from './lfm2/modeling_lfm2.js';
+export * from './lighton_ocr/modeling_lighton_ocr.js';
 export * from './lfm2_moe/modeling_lfm2_moe.js';
 export * from './lfm2_vl/modeling_lfm2_vl.js';
 export * from './llama/modeling_llama.js';
@@ -88,6 +94,7 @@ export * from './metric3dv2/modeling_metric3dv2.js';
 export * from './mgp_str/modeling_mgp_str.js';
 export * from './mimi/modeling_mimi.js';
 export * from './mistral/modeling_mistral.js';
+export * from './mistral4/modeling_mistral4.js';
 export * from './mobilebert/modeling_mobilebert.js';
 export * from './mobilellm/modeling_mobilellm.js';
 export * from './mobilenet_v1/modeling_mobilenet_v1.js';
@@ -105,6 +112,7 @@ export * from './mt5/modeling_mt5.js';
 export * from './multi_modality/modeling_multi_modality.js';
 export * from './musicgen/modeling_musicgen.js';
 export * from './nanochat/modeling_nanochat.js';
+export * from './nemotron_h/modeling_nemotron_h.js';
 export * from './neobert/modeling_neobert.js';
 export * from './nomic_bert/modeling_nomic_bert.js';
 export * from './olmo/modeling_olmo.js';
@@ -148,6 +156,7 @@ export * from './segformer/modeling_segformer.js';
 export * from './siglip/modeling_siglip.js';
 export * from './smollm3/modeling_smollm3.js';
 export * from './snac/modeling_snac.js';
+export * from './solar_open/modeling_solar_open.js';
 export * from './speecht5/modeling_speecht5.js';
 export * from './squeezebert/modeling_squeezebert.js';
 export * from './stablelm/modeling_stablelm.js';

package/src/models/nemotron_h/modeling_nemotron_h.js

@@ -0,0 +1,5 @@
+import { PreTrainedModel } from '../modeling_utils.js';
+
+export class NemotronHPreTrainedModel extends PreTrainedModel {}
+export class NemotronHModel extends NemotronHPreTrainedModel {}
+export class NemotronHForCausalLM extends NemotronHPreTrainedModel {}

package/src/models/processors.js

@@ -1,6 +1,8 @@
 export * from './chatterbox/processing_chatterbox.js';
 export * from './florence2/processing_florence2.js';
+export * from './gemma3/processing_gemma3.js';
 export * from './gemma3n/processing_gemma3n.js';
+export * from './glm46v/processing_glm46v.js';
 export * from './granite_speech/processing_granite_speech.js';
 export * from './grounding_dino/processing_grounding_dino.js';
 export * from './idefics3/processing_idefics3.js';

package/src/models/qwen2_vl/modeling_qwen2_vl.js

@@ -22,6 +22,167 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
     // embed_tokens + decoder_model_merged (no vision_encoder) via MultimodalLanguageModelOnly type.
     image_grid_thw_name = 'grid_thw';
 
+    /**
+     * Compute text-only 3D rope position IDs (all 3 dims get the same 1D positions).
+     * @param {Tensor} input_ids
+     * @param {Tensor} attention_mask
+     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas]
+     */
+    _get_text_only_rope_index(input_ids, attention_mask) {
+        if (attention_mask) {
+            const { data, dims } = cumsum_masked_fill(attention_mask);
+
+            const position_ids = BigInt64Array.from({ length: 3 * data.length }, (_, i) => data[i % data.length]);
+            /** @type {bigint[]} */
+            const mrope_position_deltas = Array.from(
+                { length: dims[0] },
+                (_, i) => max(data.subarray(dims[1] * i, dims[1] * (i + 1)))[0] + 1n + BigInt(dims[1]),
+            );
+
+            return [
+                new Tensor('int64', position_ids, [3, ...dims]),
+                new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
+            ];
+        } else {
+            const [batch_size, seq_length] = input_ids.dims;
+            const position_ids = BigInt64Array.from({ length: 3 * batch_size * seq_length }, (_, i) =>
+                BigInt(Math.floor((i % seq_length) / batch_size)),
+            );
+
+            return [new Tensor('int64', position_ids, [3, ...input_ids.dims]), zeros([batch_size, 1])];
+        }
+    }
+
+    /**
+     * Reorder per-segment position ID lists from [seg1[t,h,w], seg2[t,h,w], ...] into
+     * global [all_t, all_h, all_w] order, then write back into the position_ids array
+     * respecting attention mask.
+     * @param {number[][]} llm_pos_ids_list List of per-segment position arrays, each of length 3*seg_len
+     * @param {number[]} attn_mask Attention mask for this batch element
+     * @param {number[][][]} position_ids_list [3][batch][seq] output array to write into
+     * @param {number} batch_idx Current batch index
+     * @returns {number[]} Flat reordered positions of length total_len
+     */
+    _reorder_and_write_positions(llm_pos_ids_list, attn_mask, position_ids_list, batch_idx) {
+        const total_len = llm_pos_ids_list.reduce((acc, x) => acc + x.length, 0);
+        const llm_positions = new Array(total_len);
+        let index = 0;
+        for (let x = 0; x < 3; ++x) {
+            for (const val of llm_pos_ids_list) {
+                const seg_len = val.length / 3;
+                for (let z = x * seg_len; z < (x + 1) * seg_len; ++z) {
+                    llm_positions[index++] = val[z];
+                }
+            }
+        }
+
+        let count = 0;
+        for (let y = 0; y < attn_mask.length; ++y) {
+            if (attn_mask[y] == 1) {
+                for (let x = 0; x < 3; ++x) {
+                    position_ids_list[x][batch_idx][y] = llm_positions[(x * total_len) / 3 + count];
+                }
+                ++count;
+            }
+        }
+
+        return llm_positions;
+    }
+
+    /**
+     * Build per-batch position ID segments for multimodal rope.
+     * Override this in subclasses to change how vision/text segments are identified and positioned.
+     * @param {object} params
+     * @param {any[]} params.filtered_ids - attention-masked token IDs for this batch element
+     * @param {any[][]} params.image_grid_thw_list - all image grid dimensions
+     * @param {any[][]} params.video_grid_thw_list - all video grid dimensions
+     * @param {number} params.spatial_merge_size
+     * @param {{image_index: number, video_index: number}} params.state - mutable counters shared across batches
+     * @returns {number[][]} llm_pos_ids_list - segments of [t..., h..., w...] positions
+     */
+    _get_multimodal_rope_positions({
+        filtered_ids,
+        image_grid_thw_list,
+        video_grid_thw_list,
+        spatial_merge_size,
+        state,
+    }) {
+        // @ts-ignore
+        const { image_token_id, video_token_id, vision_start_token_id } = this.config;
+
+        const ids = filtered_ids;
+        const vision_start_indices = ids.reduce((acc, x, idx) => {
+            if (x == vision_start_token_id) acc.push(idx);
+            return acc;
+        }, []);
+
+        const vision_tokens = vision_start_indices.map((x) => ids[x + 1]);
+        const image_nums = vision_tokens.filter((x) => x == image_token_id).length;
+        const video_nums = vision_tokens.filter((x) => x == video_token_id).length;
+
+        /** @type {number[][]} */
+        const llm_pos_ids_list = [];
+        let st = 0;
+        let remain_images = image_nums;
+        let remain_videos = video_nums;
+        for (let j = 0; j < vision_tokens.length; ++j) {
+            const next_image_token = ids.findIndex((x, i) => i > st && x == image_token_id);
+            const next_video_token = ids.findIndex((x, i) => i > st && x == video_token_id);
+
+            const ed_image = remain_images > 0 && next_image_token !== -1 ? next_image_token : ids.length + 1;
+            const ed_video = remain_videos > 0 && next_video_token !== -1 ? next_video_token : ids.length + 1;
+
+            let ed;
+            let t, h, w;
+            if (ed_image < ed_video) {
+                [t, h, w] = image_grid_thw_list[state.image_index];
+                ++state.image_index;
+                --remain_images;
+                ed = ed_image;
+            } else {
+                [t, h, w] = video_grid_thw_list[state.video_index];
+                ++state.video_index;
+                --remain_videos;
+                ed = ed_video;
+            }
+
+            const [llm_grid_t, llm_grid_h, llm_grid_w] = [
+                Number(t),
+                Math.floor(Number(h) / spatial_merge_size),
+                Math.floor(Number(w) / spatial_merge_size),
+            ];
+            const text_len = ed - st;
+            const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
+
+            llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
+
+            const offset = text_len + st_idx;
+            const grid_size = llm_grid_t * llm_grid_h * llm_grid_w;
+            const t_index = Array.from(
+                { length: grid_size },
+                (_, i) => offset + Math.floor(i / (llm_grid_h * llm_grid_w)),
+            );
+            const h_index = Array.from(
+                { length: grid_size },
+                (_, i) => offset + (Math.floor(i / llm_grid_w) % llm_grid_h),
+            );
+            const w_index = Array.from({ length: grid_size }, (_, i) => offset + (i % llm_grid_w));
+
+            llm_pos_ids_list.push([t_index, h_index, w_index].flat());
+
+            st = ed + grid_size;
+        }
+
+        if (st < ids.length) {
+            const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
+            const text_len = ids.length - st;
+
+            llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
+        }
+
+        return llm_pos_ids_list;
+    }
+
     /**
      * Calculate the 3D rope index based on image and video's temporal, height and width in LLM.
      *
@@ -51,137 +212,49 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
      * @param {Tensor} input_ids Indices of input sequence tokens in the vocabulary. Tensor of shape `(batch_size, sequence_length)`.
      * @param {Tensor} image_grid_thw (Optional) The temporal, height and width of feature shape of each image in LLM. Tensor of shape `(num_images, 3)`.
      * @param {Tensor} video_grid_thw (Optional) The temporal, height and width of feature shape of each video in LLM. Tensor of shape `(num_videos, 3)`.
-     * @param {Tensor} attention_mask (Optional) Mask to avoid performing attention on padding token indices. Tensor of shape `(batch_size, sequence_length)`. Mask values selected in `[0, 1]`:
-     * - 1 for tokens that are **not masked**,
-     * - 0 for tokens that are **masked**.
-     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas] with:
-     * - position_ids: Tensor of shape `(3, batch_size, sequence_length)`.
-     * - mrope_position_deltas: Tensor of shape `(batch_size)`.
+     * @param {Tensor} attention_mask (Optional) Mask to avoid performing attention on padding token indices. Tensor of shape `(batch_size, sequence_length)`.
+     * @returns {[Tensor, Tensor]} [position_ids, mrope_position_deltas]
      */
     get_rope_index(input_ids, image_grid_thw, video_grid_thw, attention_mask) {
         // @ts-ignore
-        const { vision_config, image_token_id, video_token_id, vision_start_token_id } = this.config;
+        const { vision_config } = this.config;
         const spatial_merge_size = vision_config.spatial_merge_size ?? 2;
 
-        const mrope_position_deltas = [];
         if (image_grid_thw || video_grid_thw) {
-            let total_input_ids = input_ids.tolist();
+            const total_input_ids = input_ids.tolist();
             if (!attention_mask) {
                 attention_mask = ones_like(input_ids);
             }
 
             const attention_mask_list = attention_mask.tolist();
-            const position_ids_list = Array.from({ length: 3 }, (_) =>
-                Array.from({ length: input_ids.dims[0] }, (_) => Array.from({ length: input_ids.dims[1] }, (_) => 1)),
+            const position_ids_list = Array.from({ length: 3 }, () =>
+                Array.from({ length: input_ids.dims[0] }, () => Array.from({ length: input_ids.dims[1] }, () => 0)),
             );
 
             const image_grid_thw_list = image_grid_thw ? image_grid_thw.tolist() : [];
             const video_grid_thw_list = video_grid_thw ? video_grid_thw.tolist() : [];
+            const state = { image_index: 0, video_index: 0 };
 
-            let image_index = 0;
-            let video_index = 0;
+            const mrope_position_deltas = [];
             for (let i = 0; i < total_input_ids.length; ++i) {
-                const ids = total_input_ids[i].filter((_, j) => attention_mask_list[i][j] == 1);
-
-                const vision_start_indices = ids.reduce((acc, x, idx) => {
-                    if (x == vision_start_token_id) acc.push(idx);
-                    return acc;
-                }, []);
-
-                const vision_tokens = vision_start_indices.map((x) => ids[x + 1]);
-                const image_nums = vision_tokens.filter((x) => x == image_token_id).length;
-                const video_nums = vision_tokens.filter((x) => x == video_token_id).length;
-
-                /** @type {number[][]} */
-                let llm_pos_ids_list = [];
-                let st = 0;
-                let remain_images = image_nums;
-                let remain_videos = video_nums;
-                for (let j = 0; j < vision_tokens.length; ++j) {
-                    const next_image_token = ids.findIndex((x, i) => i > st && x == image_token_id);
-                    const next_video_token = ids.findIndex((x, i) => i > st && x == video_token_id);
-
-                    const ed_image = remain_images > 0 && next_image_token !== -1 ? next_image_token : ids.length + 1;
-
-                    const ed_video = remain_videos > 0 && next_video_token !== -1 ? next_video_token : ids.length + 1;
-
-                    let ed;
-                    let t, h, w;
-                    if (ed_image < ed_video) {
-                        [t, h, w] = image_grid_thw_list[image_index];
-                        ++image_index;
-                        --remain_images;
-                        ed = ed_image;
-                    } else {
-                        [t, h, w] = video_grid_thw_list[video_index];
-                        ++video_index;
-                        --remain_videos;
-                        ed = ed_video;
-                    }
-
-                    const [llm_grid_t, llm_grid_h, llm_grid_w] = [
-                        Number(t),
-                        Math.floor(Number(h) / spatial_merge_size),
-                        Math.floor(Number(w) / spatial_merge_size),
-                    ];
-                    const text_len = ed - st;
-                    const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
-
-                    llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
-
-                    const offset = text_len + st_idx;
-                    const grid_size = llm_grid_t * llm_grid_h * llm_grid_w;
-                    const t_index = Array.from(
-                        { length: grid_size },
-                        (_, i) => offset + Math.floor(i / (llm_grid_h * llm_grid_w)),
-                    );
-                    const h_index = Array.from(
-                        { length: grid_size },
-                        (_, i) => offset + (Math.floor(i / llm_grid_w) % llm_grid_h),
-                    );
-                    const w_index = Array.from({ length: grid_size }, (_, i) => offset + (i % llm_grid_w));
-
-                    llm_pos_ids_list.push([t_index, h_index, w_index].flat());
-
-                    st = ed + grid_size;
-                }
-
-                if (st < ids.length) {
-                    const st_idx = llm_pos_ids_list.length > 0 ? max(llm_pos_ids_list.at(-1))[0] + 1 : 0;
-                    const text_len = ids.length - st;
-
-                    llm_pos_ids_list.push(Array.from({ length: 3 * text_len }, (_, i) => st_idx + (i % text_len)));
-                }
-
-                // NOTE: Each item in llm_pos_ids_list is an array of shape (3, text_len),
-                // meaning to perform concatenation along dim=1, we can do the following:
-                const num_items = llm_pos_ids_list.reduce((acc, x) => acc + x.length, 0);
-                /** @type {number[]} */
-                const llm_positions = new Array(num_items);
-                let index = 0;
-                for (let x = 0; x < 3; ++x) {
-                    for (let y = 0; y < llm_pos_ids_list.length; ++y) {
-                        const val = llm_pos_ids_list[y];
-                        const text_len = val.length / 3;
-                        for (let z = x * text_len; z < (x + 1) * text_len; ++z) {
-                            llm_positions[index++] = val[z];
-                        }
-                    }
-                }
-
-                let count = 0;
-                const attn_mask = attention_mask_list[i];
-                for (let y = 0; y < attn_mask.length; ++y) {
-                    if (attn_mask[y] == 1) {
-                        for (let x = 0; x < 3; ++x) {
-                            position_ids_list[x][i][y] = llm_positions[(x * num_items) / 3 + count];
-                        }
-                        ++count;
-                    }
-                }
+                const filtered_ids = total_input_ids[i].filter((_, j) => attention_mask_list[i][j] == 1);
+
+                const llm_pos_ids_list = this._get_multimodal_rope_positions({
+                    filtered_ids,
+                    image_grid_thw_list,
+                    video_grid_thw_list,
+                    spatial_merge_size,
+                    state,
+                });
+
+                const llm_positions = this._reorder_and_write_positions(
+                    llm_pos_ids_list,
+                    attention_mask_list[i],
+                    position_ids_list,
+                    i,
+                );
 
-                const max_llm_positions = max(llm_positions)[0];
-                mrope_position_deltas.push(max_llm_positions + 1 - total_input_ids[i].length);
+                mrope_position_deltas.push(max(llm_positions)[0] + 1 - total_input_ids[i].length);
             }
 
             return [
@@ -189,29 +262,7 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
                 new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
             ];
         } else {
-            // Text-only
-            if (attention_mask) {
-                const { data, dims } = cumsum_masked_fill(attention_mask);
-
-                const position_ids = BigInt64Array.from({ length: 3 * data.length }, (_, i) => data[i % data.length]);
-                /** @type {bigint[]} */
-                const mrope_position_deltas = Array.from(
-                    { length: dims[0] },
-                    (_, i) => max(data.subarray(dims[1] * i, dims[1] * (i + 1)))[0] + 1n + BigInt(dims[1]),
-                );
-
-                return [
-                    new Tensor('int64', position_ids, [3, ...dims]),
-                    new Tensor('int64', mrope_position_deltas, [mrope_position_deltas.length, 1]),
-                ];
-            } else {
-                const [batch_size, seq_length] = input_ids.dims;
-                const position_ids = BigInt64Array.from({ length: 3 * batch_size * seq_length }, (_, i) =>
-                    BigInt(Math.floor((i % seq_length) / batch_size)),
-                );
-
-                return [new Tensor('int64', position_ids, [3, ...input_ids.dims]), zeros([batch_size, 1])];
-            }
+            return this._get_text_only_rope_index(input_ids, attention_mask);
         }
     }
 
@@ -235,50 +286,57 @@ export class Qwen2VLForConditionalGeneration extends Qwen2VLPreTrainedModel {
 
     prepare_inputs_for_generation(input_ids, model_inputs, generation_config) {
         // Overwritten -- in specific circumstances we don't want to forward image inputs to the model
-        if (model_inputs.attention_mask && !model_inputs.position_ids) {
-            // Calculate position_ids and rope_deltas
-            if (!model_inputs.past_key_values) {
-                [model_inputs.position_ids, model_inputs.rope_deltas] = this.get_rope_index(
+        if (!model_inputs.attention_mask || model_inputs.position_ids) {
+            return model_inputs;
+        }
+
+        const session = this.sessions['decoder_model_merged'] ?? this.sessions['model'];
+        if (!session.inputNames.includes('position_ids')) {
+            return model_inputs;
+        }
+
+        // Calculate position_ids and rope_deltas
+        if (!model_inputs.past_key_values) {
+            [model_inputs.position_ids, model_inputs.rope_deltas] = this.get_rope_index(
+                model_inputs.input_ids,
+                model_inputs.image_grid_thw,
+                model_inputs.video_grid_thw,
+                model_inputs.attention_mask,
+            );
+        } else {
+            model_inputs.pixel_values = null;
+            // model_inputs.pixel_values_videos = null;
+
+            const past_length = model_inputs.past_key_values.get_seq_length();
+
+            if (past_length < model_inputs.input_ids.dims[1]) {
+                // Externally provided `past_key_values` with full input_ids:
+                // Compute full position_ids, then slice to only the new (unprocessed) tokens.
+                const [full_position_ids, rope_deltas] = this.get_rope_index(
                     model_inputs.input_ids,
                     model_inputs.image_grid_thw,
                     model_inputs.video_grid_thw,
                     model_inputs.attention_mask,
                 );
+                model_inputs.rope_deltas = rope_deltas;
+                model_inputs.position_ids = full_position_ids.slice(null, null, [past_length, null]);
+                model_inputs.input_ids = model_inputs.input_ids.slice(null, [past_length, null]);
             } else {
-                model_inputs.pixel_values = null;
-                // model_inputs.pixel_values_videos = null;
-
-                const past_length = model_inputs.past_key_values.get_seq_length();
-
-                if (past_length < model_inputs.input_ids.dims[1]) {
-                    // Externally provided `past_key_values` with full input_ids:
-                    // Compute full position_ids, then slice to only the new (unprocessed) tokens.
-                    const [full_position_ids, rope_deltas] = this.get_rope_index(
+                // Auto-regressive case: single new token.
+                // `rope_deltas` may be absent when generation starts from externally provided `past_key_values`.
+                // In that case, recompute from current inputs instead of relying on persisted model state.
+                if (!model_inputs.rope_deltas) {
+                    [, model_inputs.rope_deltas] = this.get_rope_index(
                         model_inputs.input_ids,
                         model_inputs.image_grid_thw,
                         model_inputs.video_grid_thw,
                         model_inputs.attention_mask,
                     );
-                    model_inputs.rope_deltas = rope_deltas;
-                    model_inputs.position_ids = full_position_ids.slice(null, null, [past_length, null]);
-                    model_inputs.input_ids = model_inputs.input_ids.slice(null, [past_length, null]);
-                } else {
-                    // Auto-regressive case: single new token.
-                    // `rope_deltas` may be absent when generation starts from externally provided `past_key_values`.
-                    // In that case, recompute from current inputs instead of relying on persisted model state.
-                    if (!model_inputs.rope_deltas) {
-                        [, model_inputs.rope_deltas] = this.get_rope_index(
-                            model_inputs.input_ids,
-                            model_inputs.image_grid_thw,
-                            model_inputs.video_grid_thw,
-                            model_inputs.attention_mask,
-                        );
-                    }
-
-                    const delta = BigInt(past_length);
-                    const rope_deltas_list = model_inputs.rope_deltas.map((x) => delta + x);
-                    model_inputs.position_ids = stack([rope_deltas_list, rope_deltas_list, rope_deltas_list], 0);
                 }
+
+                const delta = BigInt(past_length);
+                const rope_deltas_list = model_inputs.rope_deltas.map((x) => delta + x);
+                model_inputs.position_ids = stack([rope_deltas_list, rope_deltas_list, rope_deltas_list], 0);
             }
         }
 

package/src/models/qwen2_vl/processing_qwen2_vl.js

@@ -6,6 +6,7 @@ import { RawImage } from '../../utils/image.js';
 export class Qwen2VLProcessor extends Processor {
     static image_processor_class = AutoImageProcessor;
     static tokenizer_class = AutoTokenizer;
+    static image_token = '<|image_pad|>';
 
     /**
      *
@@ -31,13 +32,14 @@ export class Qwen2VLProcessor extends Processor {
             let merge_length = this.image_processor.config.merge_size ** 2;
             let index = 0;
 
+            const image_token = /** @type {typeof Qwen2VLProcessor} */ (this.constructor).image_token;
             const image_grid_thw_list = image_grid_thw.tolist();
             text = text.map((t) => {
-                while (t.includes('<|image_pad|>')) {
+                while (t.includes(image_token)) {
                     const prod = Number(image_grid_thw_list[index++].reduce((a, b) => a * b, 1n));
-                    t = t.replace('<|image_pad|>', '<|placeholder|>'.repeat(Math.floor(prod / merge_length)));
+                    t = t.replace(image_token, '<|placeholder|>'.repeat(Math.floor(prod / merge_length)));
                 }
-                return t.replaceAll('<|placeholder|>', '<|image_pad|>');
+                return t.replaceAll('<|placeholder|>', image_token);
             });
         }
 
@@ -46,7 +48,6 @@ export class Qwen2VLProcessor extends Processor {
         return {
             ...text_inputs,
             ...image_inputs,
-            // TODO: ...videos_inputs,
         };
     }
 }