@huggingface/transformers 4.0.0-next.5 → 4.0.0-next.7

package/src/models/detr/image_processing_detr.js

@@ -8,7 +8,7 @@ import {
 import { full } from '../../utils/tensor.js';
 
 /**
- * @typedef {object} DetrFeatureExtractorResultProps
+ * @typedef {Object} DetrFeatureExtractorResultProps
  * @property {import('../../utils/tensor.js').Tensor} pixel_mask
  * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & DetrFeatureExtractorResultProps} DetrFeatureExtractorResult
  */

package/src/models/feature_extractors.js

@@ -4,6 +4,7 @@ export * from './chatterbox/feature_extraction_chatterbox.js';
 export * from './clap/feature_extraction_clap.js';
 export * from './dac/feature_extraction_dac.js';
 export * from './gemma3n/feature_extraction_gemma3n.js';
+export * from './granite_speech/feature_extraction_granite_speech.js';
 export * from './moonshine/feature_extraction_moonshine.js';
 export * from './parakeet/feature_extraction_parakeet.js';
 export * from './pyannote/feature_extraction_pyannote.js';
@@ -12,6 +13,7 @@ export * from './snac/feature_extraction_snac.js';
 export * from './speecht5/feature_extraction_speecht5.js';
 export * from './wav2vec2/feature_extraction_wav2vec2.js';
 export * from './wespeaker/feature_extraction_wespeaker.js';
+export * from './voxtral_realtime/feature_extraction_voxtral_realtime.js';
 export * from './whisper/feature_extraction_whisper.js';
 
 export { FeatureExtractor } from '../feature_extraction_utils.js';

package/src/models/gemma3n/modeling_gemma3n.js

@@ -115,3 +115,5 @@ export class Gemma3nForConditionalGeneration extends Gemma3nPreTrainedModel {
         });
     }
 }
+
+export class Gemma3nForCausalLM extends Gemma3nForConditionalGeneration {}

package/src/models/granite_speech/feature_extraction_granite_speech.js

@@ -0,0 +1,58 @@
+import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js';
+import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js';
+import { Tensor } from '../../utils/tensor.js';
+
+export class GraniteSpeechFeatureExtractor extends FeatureExtractor {
+    constructor(config) {
+        super(config);
+
+        const { n_fft, win_length, n_mels, sample_rate } = config.melspec_kwargs;
+
+        // torchaudio uses HTK mel scale with no norm by default
+        this.mel_filters = mel_filter_bank(
+            Math.floor(1 + n_fft / 2), // num_frequency_bins = 257
+            n_mels, // 80
+            0, // min_frequency
+            sample_rate / 2, // max_frequency = 8000
+            sample_rate, // 16000
+            null, // norm (torchaudio default: no norm)
+            'htk', // mel_scale (torchaudio default)
+        );
+
+        // torchaudio center-pads the window when win_length < n_fft:
+        // pad_amount = (n_fft - win_length) // 2 on each side
+        const raw_window = window_function(win_length, 'hann');
+        this.window = new Float64Array(n_fft);
+        const pad = Math.floor((n_fft - win_length) / 2);
+        this.window.set(raw_window, pad);
+    }
+
+    /**
+     * Extract mel spectrogram features from audio, matching the Python GraniteSpeechFeatureExtractor.
+     * @param {Float32Array|Float64Array} audio The audio waveform.
+     * @returns {Promise<{input_features: Tensor}>}
+     */
+    async _call(audio) {
+        validate_audio_inputs(audio, 'GraniteSpeechFeatureExtractor');
+
+        const { n_fft, hop_length, n_mels } = this.config.melspec_kwargs;
+
+        // Truncate to even number of frames for pair-stacking
+        const num_frames = 1 + Math.floor((audio.length - 1) / hop_length);
+        const max_num_frames = num_frames - (num_frames % 2);
+
+        const mel = await spectrogram(audio, this.window, n_fft, hop_length, {
+            power: 2.0,
+            mel_filters: this.mel_filters,
+            log_mel: 'log10_max_norm',
+            transpose: true, // [time, n_mels]
+            max_num_frames,
+            do_pad: false,
+        });
+
+        // Stack adjacent frame pairs: [time, n_mels] → [1, time/2, 2*n_mels]
+        const input_features = mel.view(-1, 2 * n_mels).unsqueeze_(0);
+
+        return { input_features };
+    }
+}

package/src/models/granite_speech/modeling_granite_speech.js

@@ -0,0 +1,5 @@
+import { UltravoxModel } from '../ultravox/modeling_ultravox.js';
+
+export class GraniteSpeechForConditionalGeneration extends UltravoxModel {
+    forward_params = ['input_ids', 'attention_mask', 'input_features', 'past_key_values'];
+}

package/src/models/granite_speech/processing_granite_speech.js

@@ -0,0 +1,62 @@
+import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js';
+import { AutoTokenizer } from '../auto/tokenization_auto.js';
+import { Processor } from '../../processing_utils.js';
+import { Tensor } from '../../utils/tensor.js';
+
+export class GraniteSpeechProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer;
+    static feature_extractor_class = AutoFeatureExtractor;
+    static uses_processor_config = true;
+
+    /**
+     * Compute the number of audio tokens for a given raw audio length.
+     * @param {number} audioLength Raw audio sample count.
+     * @returns {number} Number of projector output tokens.
+     */
+    _get_num_audio_features(audioLength) {
+        const { hop_length } = this.feature_extractor.config.melspec_kwargs;
+        const { projector_window_size, projector_downsample_rate } = this.feature_extractor.config;
+        const effective_window_size = Math.floor(projector_window_size / projector_downsample_rate);
+        const mel_length = Math.floor(audioLength / hop_length) + 1;
+        const encoder_length = Math.floor(mel_length / 2);
+        const nblocks = Math.ceil(encoder_length / projector_window_size);
+        return nblocks * effective_window_size;
+    }
+
+    /**
+     * @param {string} text The text input to process.
+     * @param {Float32Array} audio The audio input to process.
+     */
+    async _call(text, audio = null, kwargs = {}) {
+        if (Array.isArray(text)) {
+            throw new Error('Batched inputs are not supported yet.');
+        }
+
+        let audio_inputs = {};
+        if (audio) {
+            const { input_features } = await this.feature_extractor(audio);
+            audio_inputs['input_features'] = input_features;
+
+            // Compute audio embed sizes and mask in the processor
+            const audio_embed_size = this._get_num_audio_features(audio.length);
+            const mask_data = new Uint8Array(audio_embed_size).fill(1);
+            audio_inputs['input_features_mask'] = new Tensor('bool', mask_data, [1, audio_embed_size]);
+
+            const audio_token = this.config.audio_token ?? '<|audio|>';
+            if (!text.includes(audio_token)) {
+                throw new Error(`The input text does not contain the audio token ${audio_token}.`);
+            }
+            text = text.replaceAll(audio_token, audio_token.repeat(audio_embed_size));
+        }
+
+        const text_inputs = this.tokenizer(text, {
+            add_special_tokens: false,
+            ...kwargs,
+        });
+
+        return {
+            ...text_inputs,
+            ...audio_inputs,
+        };
+    }
+}

package/src/models/grounding_dino/image_processing_grounding_dino.js

@@ -2,7 +2,7 @@ import { ImageProcessor } from '../../image_processors_utils.js';
 import { ones } from '../../utils/tensor.js';
 
 /**
- * @typedef {object} GroundingDinoFeatureExtractorResultProps
+ * @typedef {Object} GroundingDinoFeatureExtractorResultProps
  * @property {import('../../utils/tensor.js').Tensor} pixel_mask
  * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & GroundingDinoFeatureExtractorResultProps} GroundingDinoFeatureExtractorResult
  */

package/src/models/idefics3/modeling_idefics3.js

@@ -1,7 +1,9 @@
-import { PreTrainedModel, default_merge_input_ids_with_image_features } from '../modeling_utils.js';
-import { sessionRun } from '../session.js';
+import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js';
 
-export class Idefics3PreTrainedModel extends PreTrainedModel {
+/**
+ * The Idefics3 model which consists of a vision backbone and a language model.
+ */
+export class Idefics3ForConditionalGeneration extends LlavaForConditionalGeneration {
     forward_params = [
         'input_ids',
         'attention_mask',
@@ -11,32 +13,3 @@ export class Idefics3PreTrainedModel extends PreTrainedModel {
         'past_key_values',
     ];
 }
-
-/**
- * The Idefics3 model which consists of a vision backbone and a language model.
- */
-export class Idefics3ForConditionalGeneration extends Idefics3PreTrainedModel {
-    async encode_image({ pixel_values, pixel_attention_mask }) {
-        const features = (await sessionRun(this.sessions['vision_encoder'], { pixel_values, pixel_attention_mask }))
-            .image_features;
-        return features;
-    }
-
-    _merge_input_ids_with_image_features(kwargs) {
-        const vision_hidden_size = kwargs.image_features.dims.at(-1);
-        const reshaped_image_hidden_states = kwargs.image_features.view(-1, vision_hidden_size);
-
-        return default_merge_input_ids_with_image_features({
-            // @ts-ignore
-            image_token_id: this.config.image_token_id,
-            ...kwargs,
-            image_features: reshaped_image_hidden_states,
-        });
-    }
-}
-
-/**
- * The SmolVLM Model with a language modeling head.
- * It is made up a SigLIP vision encoder, with a language modeling head on top.
- */
-export class SmolVLMForConditionalGeneration extends Idefics3ForConditionalGeneration {}

package/src/models/image_processors.js

@@ -14,6 +14,7 @@ export * from './grounding_dino/image_processing_grounding_dino.js';
 export * from './idefics3/image_processing_idefics3.js';
 export * from './janus/image_processing_janus.js';
 export * from './jina_clip/image_processing_jina_clip.js';
+export * from './lfm2_vl/image_processing_lfm2_vl.js';
 export * from './llava_onevision/image_processing_llava_onevision.js';
 export * from './mask2former/image_processing_mask2former.js';
 export * from './maskformer/image_processing_maskformer.js';

package/src/models/lfm2_vl/image_processing_lfm2_vl.js

@@ -0,0 +1,305 @@
+import { ImageProcessor, smart_resize } from '../../image_processors_utils.js';
+import { Tensor, cat, interpolate_4d, stack } from '../../utils/tensor.js';
+
+/**
+ * @typedef {import('../../utils/image.js').RawImage} RawImage
+ */
+
+/**
+ * Returns the closest integer to `number` that is divisible by `factor`.
+ * @param {number} number
+ * @param {number} factor
+ * @returns {number}
+ */
+function round_by_factor(number, factor) {
+    return Math.round(number / factor) * factor;
+}
+
+/**
+ * Find the closest aspect ratio from target_ratios to match the input aspect ratio.
+ * @param {number} aspect_ratio
+ * @param {number[][]} target_ratios
+ * @param {number} width
+ * @param {number} height
+ * @param {number} image_size
+ * @returns {number[]}
+ */
+function find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size) {
+    let best_ratio_diff = Infinity;
+    let best_ratio = [1, 1];
+    const area = width * height;
+    for (const ratio of target_ratios) {
+        const ratio_diff = Math.abs(aspect_ratio - ratio[0] / ratio[1]);
+        if (ratio_diff < best_ratio_diff) {
+            best_ratio_diff = ratio_diff;
+            best_ratio = ratio;
+        } else if (ratio_diff === best_ratio_diff && area > 0.5 * image_size * image_size * ratio[0] * ratio[1]) {
+            best_ratio = ratio;
+        }
+    }
+    return best_ratio;
+}
+
+/**
+ * Compute all valid (width, height) tile ratios for the given range.
+ * @param {number} min_tiles
+ * @param {number} max_tiles
+ * @returns {number[][]}
+ */
+function get_target_ratios(min_tiles, max_tiles) {
+    /** @type {number[][]} */
+    const ratios = [];
+    const seen = new Set();
+    for (let n = min_tiles; n <= max_tiles; ++n) {
+        for (let w = 1; w <= n; ++w) {
+            for (let h = 1; h <= n; ++h) {
+                const product = w * h;
+                if (product >= min_tiles && product <= max_tiles) {
+                    const key = (w << 16) | h;
+                    if (!seen.has(key)) {
+                        seen.add(key);
+                        ratios.push([w, h]);
+                    }
+                }
+            }
+        }
+    }
+    return ratios.sort((a, b) => a[0] * a[1] - b[0] * b[1]);
+}
+
+/**
+ * Convert image tensor to flattened patches.
+ *
+ * Equivalent to PyTorch: `images.reshape(B, C, ph, ps, pw, ps).permute(0, 2, 4, 3, 5, 1).reshape(B, ph*pw, -1)`
+ * @param {Tensor} images Shape: [batch, channels, height, width]
+ * @param {number} patch_size
+ * @returns {Tensor} Shape: [batch, num_patches, patch_size * patch_size * channels]
+ */
+function convert_image_to_patches(images, patch_size) {
+    const [B, C, H, W] = images.dims;
+    const ph = Math.floor(H / patch_size),
+        pw = Math.floor(W / patch_size);
+    const patch_dim = patch_size * patch_size * C;
+    const data = /** @type {Float32Array} */ (images.data);
+    const result = new Float32Array(B * ph * pw * patch_dim);
+    const ch_stride = H * W;
+
+    for (let b = 0; b < B; ++b) {
+        const b_src = b * C * ch_stride;
+        const b_dst = b * ph * pw * patch_dim;
+        for (let py = 0; py < ph; ++py) {
+            for (let px = 0; px < pw; ++px) {
+                let off = b_dst + (py * pw + px) * patch_dim;
+                for (let dy = 0; dy < patch_size; ++dy) {
+                    const row = (py * patch_size + dy) * W + px * patch_size;
+                    for (let dx = 0; dx < patch_size; ++dx) {
+                        const pixel = row + dx;
+                        for (let c = 0; c < C; ++c) {
+                            result[off++] = data[b_src + c * ch_stride + pixel];
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    return new Tensor('float32', result, [B, ph * pw, patch_dim]);
+}
+
+/**
+ * Pad patches along the patch dimension to `target_length`.
+ * @param {Tensor} patches Shape: [1, current_length, patch_dim]
+ * @param {number} target_length
+ * @returns {{ padded: Tensor, mask: Tensor }}
+ */
+function pad_along_first_dim(patches, target_length) {
+    const [, len, dim] = patches.dims;
+    const mask_data = new BigInt64Array(target_length);
+    mask_data.fill(1n, 0, len);
+
+    let padded = patches;
+    if (len < target_length) {
+        const padded_data = new Float32Array(target_length * dim);
+        padded_data.set(/** @type {Float32Array} */ (patches.data));
+        padded = new Tensor('float32', padded_data, [1, target_length, dim]);
+    }
+
+    return { padded, mask: new Tensor('int64', mask_data, [target_length]) };
+}
+
+export class Lfm2VlImageProcessor extends ImageProcessor {
+    constructor(/** @type {Record<string, any>} */ config) {
+        super(config);
+        this.downsample_factor = config.downsample_factor ?? 2;
+        this.do_image_splitting = config.do_image_splitting ?? true;
+        this.min_tiles = config.min_tiles ?? 2;
+        this.max_tiles = config.max_tiles ?? 10;
+        this.use_thumbnail = config.use_thumbnail ?? true;
+        this.min_image_tokens = config.min_image_tokens ?? 64;
+        this.max_image_tokens = config.max_image_tokens ?? 256;
+        this.encoder_patch_size = config.encoder_patch_size ?? config.patch_size ?? 16;
+        this.tile_size = config.tile_size ?? 512;
+        this.max_pixels_tolerance = config.max_pixels_tolerance ?? 2.0;
+        this.return_row_col_info = config.return_row_col_info ?? false;
+
+        const max_thumbnail_patches = this.max_image_tokens * this.downsample_factor ** 2;
+        const tile_size_patches = this.do_image_splitting ? (this.tile_size / this.encoder_patch_size) ** 2 : 0;
+        this.max_num_patches = Math.max(max_thumbnail_patches, tile_size_patches);
+    }
+
+    /**
+     * Check if the image is too large to be processed as a single tile.
+     * @param {number} height
+     * @param {number} width
+     * @returns {boolean}
+     */
+    _is_image_too_large(height, width) {
+        const total_factor = this.encoder_patch_size * this.downsample_factor;
+        const h_bar = Math.max(this.encoder_patch_size, round_by_factor(height, total_factor));
+        const w_bar = Math.max(this.encoder_patch_size, round_by_factor(width, total_factor));
+        return (
+            h_bar * w_bar >
+            this.max_image_tokens * (this.encoder_patch_size * this.downsample_factor) ** 2 * this.max_pixels_tolerance
+        );
+    }
+
+    /**
+     * Get the grid layout for tiling a large image.
+     * @param {number} height
+     * @param {number} width
+     * @returns {{ grid_width: number, grid_height: number, target_width: number, target_height: number }}
+     */
+    _get_grid_layout(height, width) {
+        const target_ratios = get_target_ratios(this.min_tiles, this.max_tiles);
+        const [grid_width, grid_height] = find_closest_aspect_ratio(
+            width / height,
+            target_ratios,
+            width,
+            height,
+            this.tile_size,
+        );
+        return {
+            grid_width,
+            grid_height,
+            target_width: this.tile_size * grid_width,
+            target_height: this.tile_size * grid_height,
+        };
+    }
+
+    /** @param {RawImage|RawImage[]|RawImage[][]} images */
+    // @ts-expect-error
+    async _call(images, { return_row_col_info = null } = {}) {
+        /** @type {RawImage[][]} */
+        let batched_images;
+        if (!Array.isArray(images)) {
+            batched_images = [[images]];
+        } else if (!Array.isArray(images[0])) {
+            batched_images = [/** @type {RawImage[]} */ (images)];
+        } else {
+            batched_images = /** @type {RawImage[][]} */ (images);
+        }
+
+        /** @type {Tensor[]} */
+        const all_pixel_values = [];
+        /** @type {Tensor[]} */
+        const all_pixel_masks = [];
+        /** @type {number[][]} */
+        const all_spatial_shapes = [];
+        /** @type {number[]} */
+        const all_rows = [];
+        /** @type {number[]} */
+        const all_cols = [];
+        /** @type {number[][]} */
+        const all_image_sizes = [];
+
+        for (const image_batch of batched_images) {
+            const preprocessed = await Promise.all(image_batch.map((x) => this.preprocess(x, { do_pad: false })));
+
+            for (const { pixel_values } of preprocessed) {
+                const [, height, width] = pixel_values.dims;
+                const img = pixel_values.unsqueeze_(0);
+
+                const total_factor = this.encoder_patch_size * this.downsample_factor;
+                const f2 = total_factor ** 2;
+                const [new_height, new_width] = smart_resize(
+                    Math.max(total_factor, height),
+                    Math.max(total_factor, width),
+                    total_factor,
+                    this.min_image_tokens * f2,
+                    this.max_image_tokens * f2,
+                ).map((x) => Math.max(total_factor, x));
+
+                /** @type {Tensor[]} */
+                let tiles;
+                let num_rows = 1,
+                    num_cols = 1;
+
+                const is_large = this._is_image_too_large(height, width);
+                const do_splitting = this.do_image_splitting && !(this.min_tiles === 1 && this.max_tiles === 1);
+
+                if (is_large && do_splitting) {
+                    const { grid_width, grid_height, target_width, target_height } = this._get_grid_layout(
+                        height,
+                        width,
+                    );
+                    num_rows = grid_height;
+                    num_cols = grid_width;
+
+                    const resized = await interpolate_4d(img, {
+                        size: [target_height, target_width],
+                    });
+
+                    tiles = [];
+                    for (let r = 0; r < grid_height; ++r) {
+                        for (let c = 0; c < grid_width; ++c) {
+                            const y = r * this.tile_size;
+                            const x = c * this.tile_size;
+                            tiles.push(resized.slice(null, null, [y, y + this.tile_size], [x, x + this.tile_size]));
+                        }
+                    }
+
+                    if (this.use_thumbnail && grid_width * grid_height !== 1) {
+                        tiles.push(await interpolate_4d(img, { size: [new_height, new_width] }));
+                    }
+                } else {
+                    tiles = [await interpolate_4d(img, { size: [new_height, new_width] })];
+                }
+
+                for (const tile of tiles) {
+                    const [, , th, tw] = tile.dims;
+                    const patches = convert_image_to_patches(tile, this.encoder_patch_size);
+                    const { padded, mask } = pad_along_first_dim(patches, this.max_num_patches);
+
+                    all_pixel_values.push(padded);
+                    all_pixel_masks.push(mask);
+                    all_spatial_shapes.push([
+                        Math.floor(th / this.encoder_patch_size),
+                        Math.floor(tw / this.encoder_patch_size),
+                    ]);
+                }
+
+                all_rows.push(num_rows);
+                all_cols.push(num_cols);
+                all_image_sizes.push([new_height, new_width]);
+            }
+        }
+
+        /** @type {Record<string, any>} */
+        const result = {
+            pixel_values: cat(all_pixel_values, 0),
+            pixel_attention_mask: stack(all_pixel_masks, 0),
+            spatial_shapes: new Tensor('int64', BigInt64Array.from(all_spatial_shapes.flat(), BigInt), [
+                all_spatial_shapes.length,
+                2,
+            ]),
+        };
+
+        if (return_row_col_info ?? this.return_row_col_info) {
+            result.image_rows = all_rows;
+            result.image_cols = all_cols;
+            result.image_sizes = all_image_sizes;
+        }
+
+        return result;
+    }
+}

package/src/models/lfm2_vl/modeling_lfm2_vl.js

@@ -0,0 +1,13 @@
+import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js';
+
+export class Lfm2VlForConditionalGeneration extends LlavaForConditionalGeneration {
+    forward_params = [
+        'input_ids',
+        'attention_mask',
+        'pixel_values',
+        'pixel_attention_mask',
+        'spatial_shapes',
+        'position_ids',
+        'past_key_values',
+    ];
+}

package/src/models/lfm2_vl/processing_lfm2_vl.js

@@ -0,0 +1,77 @@
+import { Processor } from '../../processing_utils.js';
+import { AutoImageProcessor } from '../auto/image_processing_auto.js';
+import { AutoTokenizer } from '../auto/tokenization_auto.js';
+
+/**
+ * @typedef {import('../../utils/image.js').RawImage} RawImage
+ */
+
+export class Lfm2VlProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer;
+    static image_processor_class = AutoImageProcessor;
+
+    /**
+     * @param {RawImage|RawImage[]} images
+     * @param {string|string[]|null} [text]
+     * @param {Record<string, any>} [kwargs]
+     */
+    async _call(images, text = null, kwargs = {}) {
+        const { image_rows, image_cols, image_sizes, ...image_inputs } = await this.image_processor(images, {
+            ...kwargs,
+            return_row_col_info: true,
+        });
+
+        if (text) {
+            const image_token = this.config.image_token ?? '<image>';
+            const {
+                tile_size = 512,
+                downsample_factor = 2,
+                encoder_patch_size = 16,
+                use_thumbnail = true,
+            } = /** @type {Record<string, any>} */ (this.image_processor.config);
+
+            const ds = (/** @type {number} */ s) => Math.ceil(Math.floor(s / encoder_patch_size) / downsample_factor);
+            const tokens_per_tile = ds(tile_size) ** 2;
+            const image_start = this.config.image_start_token ?? '<|image_start|>';
+            const image_end = this.config.image_end_token ?? '<|image_end|>';
+            const thumbnail_token = this.config.image_thumbnail ?? '<|img_thumbnail|>';
+
+            if (!Array.isArray(text)) text = [text];
+
+            let image_idx = 0;
+            text = text.map((sample) => {
+                const parts = sample.split(image_token);
+                return (
+                    parts[0] +
+                    parts
+                        .slice(1)
+                        .map((part) => {
+                            const idx = image_idx++;
+                            const [h, w] = image_sizes[idx];
+                            const rows = image_rows[idx],
+                                cols = image_cols[idx];
+                            const tokens_for_image = ds(h) * ds(w);
+
+                            let expanded = image_start;
+                            if (rows > 1 || cols > 1) {
+                                const tile_str = image_token.repeat(tokens_per_tile);
+                                for (let r = 0; r < rows; ++r)
+                                    for (let c = 0; c < cols; ++c)
+                                        expanded += `<|img_row_${r + 1}_col_${c + 1}|>` + tile_str;
+                                if (use_thumbnail) expanded += thumbnail_token + image_token.repeat(tokens_for_image);
+                            } else {
+                                expanded += image_token.repeat(tokens_for_image);
+                            }
+                            return expanded + image_end + part;
+                        })
+                        .join('')
+                );
+            });
+        }
+
+        return {
+            ...image_inputs,
+            ...(text ? this.tokenizer(text, kwargs) : {}),
+        };
+    }
+}

package/src/models/llava/modeling_llava.js

@@ -14,7 +14,7 @@ export class LlavaForConditionalGeneration extends LlavaPreTrainedModel {
 
         return default_merge_input_ids_with_image_features({
             // @ts-ignore
-            image_token_id: this.config.image_token_index,
+            image_token_id: this.config.image_token_index ?? this.config.image_token_id,
             ...kwargs,
             image_features: reshaped_image_hidden_states,
         });

package/src/models/mistral3/modeling_mistral3.js

@@ -1,3 +1,3 @@
-import { LlavaQwen2ForCausalLM } from '../llava/modeling_llava.js';
+import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js';
 
-export class Mistral3ForConditionalGeneration extends LlavaQwen2ForCausalLM {}
+export class Mistral3ForConditionalGeneration extends LlavaForConditionalGeneration {}