@huggingface/transformers 4.0.0-next.6 → 4.0.0-next.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@huggingface/transformers",
-  "version": "4.0.0-next.6",
+  "version": "4.0.0-next.7",
   "description": "State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!",
   "main": "./dist/transformers.node.cjs",
   "types": "./types/transformers.d.ts",
@@ -43,10 +43,10 @@
   },
   "homepage": "https://github.com/huggingface/transformers.js#readme",
   "dependencies": {
-    "@huggingface/jinja": "^0.5.5",
+    "@huggingface/jinja": "^0.5.6",
     "@huggingface/tokenizers": "^0.1.2",
     "onnxruntime-node": "1.24.3",
-    "onnxruntime-web": "1.25.0-dev.20260303-e7e64dc112",
+    "onnxruntime-web": "1.25.0-dev.20260307-d626b568e0",
     "sharp": "^0.34.5"
   },
   "devDependencies": {

package/src/cache_utils.js

@@ -0,0 +1,62 @@
+import { Tensor } from './utils/tensor.js';
+
+/**
+ * A cache class that stores past key values as named tensors.
+ */
+class _DynamicCache {
+    /**
+     * Create a DynamicCache, optionally pre-populated with entries.
+     * @param {Record<string, Tensor>} [entries] Initial name→Tensor mappings.
+     */
+    constructor(entries) {
+        if (!entries) return;
+        for (const key in entries) {
+            if (key in this) {
+                throw new TypeError(`Key "${key}" conflicts with an existing property on DynamicCache`);
+            }
+            const value = entries[key];
+            if (!(value instanceof Tensor)) {
+                throw new TypeError(`Expected a Tensor for key "${key}", got ${typeof value}`);
+            }
+            this[key] = value;
+        }
+    }
+
+    /**
+     * Get the cached sequence length. This requires at least one attention cache entry to be present.
+     * @returns {number} The past sequence length.
+     */
+    get_seq_length() {
+        /** @type {Record<string, Tensor>} */
+        const self = /** @type {any} */ (this);
+        for (const name in self) {
+            if (name.startsWith('past_key_values.')) {
+                return self[name].dims.at(-2);
+            }
+        }
+        throw new Error('Unable to determine sequence length from the cache.');
+    }
+
+    /**
+     * Dispose all contained tensors whose data resides on the GPU.
+     * Returns a promise that resolves when all disposals are complete.
+     * @returns {Promise<void>} Promise that resolves when all GPU tensors are disposed.
+     */
+    async dispose() {
+        const promises = [];
+        for (const t of /** @type {Tensor[]} */ (Object.values(this))) {
+            if (t.location === 'gpu-buffer') {
+                promises.push(t.dispose());
+            }
+        }
+        await Promise.all(promises);
+    }
+}
+
+/**
+ * @typedef {_DynamicCache & Record<string, Tensor>} DynamicCache
+ */
+
+export const DynamicCache = /** @type {new (entries?: Record<string, Tensor>) => DynamicCache} */ (
+    /** @type {unknown} */ (_DynamicCache)
+);

package/src/configs.js

@@ -68,10 +68,13 @@ function getNormalizedConfig(config) {
         case 'florence2':
         case 'llava_onevision':
         case 'idefics3':
+        case 'granite_speech':
         case 'ultravox':
         case 'voxtral':
+        case 'voxtral_realtime':
         case 'smolvlm':
         case 'gemma3n':
+        case 'lfm2_vl':
         case 'chatterbox':
         case 'mistral3':
         case 'qwen2_5_vl':
@@ -131,10 +134,13 @@ function getNormalizedConfig(config) {
         case 'cohere':
         case 'cohere2':
         case 'mistral':
+        case 'voxtral_realtime_text':
+        case 'voxtral_realtime_encoder':
         case 'starcoder2':
         case 'qwen2':
         case 'qwen2_moe':
         case 'qwen2_vl':
+        case 'qwen2_vl_text':
         case 'qwen2_5_vl_text':
         case 'qwen3_moe':
         case 'qwen3_vl_text':
@@ -293,6 +299,9 @@ function getNormalizedConfig(config) {
  * @returns {Record<string, number[]>}
  */
 export function getCacheShapes(config, options) {
+    if (!(config instanceof PretrainedConfig)) {
+        config = new PretrainedConfig(config);
+    }
     if (['lfm2', 'lfm2_moe'].includes(config.model_type)) {
         const pkv_prefix = options?.prefix ?? 'past_key_values';
         const conv_prefix = pkv_prefix === 'present' ? 'present' : 'past';
@@ -401,8 +410,14 @@ export function getCacheShapes(config, options) {
             }
         }
         return cache_values;
-    } else if (['qwen3_5', 'qwen3_5_moe'].includes(config.model_type)) {
-        return getCacheShapes(/**@type {any} */ (config).text_config, options);
+    } else if (['lfm2_vl', 'qwen3_5', 'qwen3_5_moe', 'voxtral_realtime'].includes(config.model_type)) {
+        let subConfig;
+        if (config.model_type === 'voxtral_realtime' && options?.session_name === 'audio_encoder') {
+            subConfig = /** @type {any} */ (config).audio_config;
+        } else {
+            subConfig = /** @type {any} */ (config).text_config;
+        }
+        return getCacheShapes(subConfig, options);
     }
 
     return getKeyValueShapes(config, options);

package/src/env.js

@@ -26,7 +26,7 @@ import fs from 'node:fs';
 import path from 'node:path';
 import url from 'node:url';
 
-const VERSION = '4.0.0-next.6';
+const VERSION = '4.0.0-next.7';
 
 const HAS_SELF = typeof self !== 'undefined';
 
@@ -224,6 +224,11 @@ export const LogLevel = Object.freeze({
  * @property {boolean} useWasmCache Whether to pre-load and cache WASM binaries and the WASM factory (.mjs) for ONNX Runtime.
  * Defaults to `true` when cache is available. This can improve performance and enables offline usage by avoiding repeated downloads.
  * @property {string} cacheKey The cache key to use for storing models and WASM binaries. Defaults to 'transformers-cache'.
+ * @property {boolean} experimental_useCrossOriginStorage Whether to use the Cross-Origin Storage API to cache model files
+ * across origins, allowing different sites to share the same cached model weights. Defaults to `false`.
+ * Requires the Cross-Origin Storage Chrome extension: {@link https://chromewebstore.google.com/detail/cross-origin-storage/denpnpcgjgikjpoglpjefakmdcbmlgih}.
+ * The `experimental_` prefix indicates that the underlying browser API is not yet standardised and may change or be
+ * removed without a major version bump. For more information, see {@link https://github.com/WICG/cross-origin-storage}.
  * @property {(input: string | URL, init?: any) => Promise<any>} fetch The fetch function to use. Defaults to `fetch`.
  */
 
@@ -270,6 +275,8 @@ export const env = {
     useWasmCache: IS_WEB_CACHE_AVAILABLE || IS_FS_AVAILABLE,
     cacheKey: 'transformers-cache',
 
+    experimental_useCrossOriginStorage: false,
+
     /////////////////// Custom fetch /////////////////////
     fetch: DEFAULT_FETCH,
 

package/src/image_processors_utils.js

@@ -14,7 +14,7 @@ import { logger } from './utils/logger.js';
  */
 
 /**
- * @typedef {object} ImageProcessorResult
+ * @typedef {Object} ImageProcessorResult
  * @property {Tensor} pixel_values The pixel values of the batched preprocessed images.
  * @property {HeightWidth[]} original_sizes Array of two-dimensional tuples like [[480, 640]].
  * @property {HeightWidth[]} reshaped_input_sizes Array of two-dimensional tuples like [[1000, 1330]].
@@ -407,7 +407,7 @@ function compute_segments(
  * @returns {[number, number]} The new height and width of the image.
  * @throws {Error} If the height or width is smaller than the factor.
  */
-function smart_resize(height, width, factor = 28, min_pixels = 56 * 56, max_pixels = 14 * 14 * 4 * 1280) {
+export function smart_resize(height, width, factor = 28, min_pixels = 56 * 56, max_pixels = 14 * 14 * 4 * 1280) {
     if (height < factor || width < factor) {
         throw new Error(`height:${height} or width:${width} must be larger than factor:${factor}`);
     } else if (Math.max(height, width) / Math.min(height, width) > 200) {
@@ -886,7 +886,7 @@ export class ImageProcessor extends Callable {
     }
 
     /**
-     * @typedef {object} PreprocessedImage
+     * @typedef {Object} PreprocessedImage
      * @property {HeightWidth} original_size The original size of the image.
      * @property {HeightWidth} reshaped_input_size The reshaped input size of the image.
      * @property {Tensor} pixel_values The pixel values of the preprocessed image.

package/src/models/chatterbox/modeling_chatterbox.js

@@ -105,7 +105,7 @@ export class ChatterboxModel extends ChatterboxPreTrainedModel {
                 if (!past_key_values || target_length !== 1) {
                     throw new Error('Incorrect state encountered during generation.');
                 }
-                const past_length = Object.values(past_key_values)[0].dims.at(-2);
+                const past_length = past_key_values.get_seq_length();
                 attention_mask = ones([inputs_embeds.dims[0], past_length + target_length]);
             }
         }

package/src/models/detr/image_processing_detr.js

@@ -8,7 +8,7 @@ import {
 import { full } from '../../utils/tensor.js';
 
 /**
- * @typedef {object} DetrFeatureExtractorResultProps
+ * @typedef {Object} DetrFeatureExtractorResultProps
  * @property {import('../../utils/tensor.js').Tensor} pixel_mask
  * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & DetrFeatureExtractorResultProps} DetrFeatureExtractorResult
  */

package/src/models/feature_extractors.js

@@ -4,6 +4,7 @@ export * from './chatterbox/feature_extraction_chatterbox.js';
 export * from './clap/feature_extraction_clap.js';
 export * from './dac/feature_extraction_dac.js';
 export * from './gemma3n/feature_extraction_gemma3n.js';
+export * from './granite_speech/feature_extraction_granite_speech.js';
 export * from './moonshine/feature_extraction_moonshine.js';
 export * from './parakeet/feature_extraction_parakeet.js';
 export * from './pyannote/feature_extraction_pyannote.js';
@@ -12,6 +13,7 @@ export * from './snac/feature_extraction_snac.js';
 export * from './speecht5/feature_extraction_speecht5.js';
 export * from './wav2vec2/feature_extraction_wav2vec2.js';
 export * from './wespeaker/feature_extraction_wespeaker.js';
+export * from './voxtral_realtime/feature_extraction_voxtral_realtime.js';
 export * from './whisper/feature_extraction_whisper.js';
 
 export { FeatureExtractor } from '../feature_extraction_utils.js';

package/src/models/gemma3n/modeling_gemma3n.js

@@ -115,3 +115,5 @@ export class Gemma3nForConditionalGeneration extends Gemma3nPreTrainedModel {
         });
     }
 }
+
+export class Gemma3nForCausalLM extends Gemma3nForConditionalGeneration {}

package/src/models/granite_speech/feature_extraction_granite_speech.js

@@ -0,0 +1,58 @@
+import { FeatureExtractor, validate_audio_inputs } from '../../feature_extraction_utils.js';
+import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js';
+import { Tensor } from '../../utils/tensor.js';
+
+export class GraniteSpeechFeatureExtractor extends FeatureExtractor {
+    constructor(config) {
+        super(config);
+
+        const { n_fft, win_length, n_mels, sample_rate } = config.melspec_kwargs;
+
+        // torchaudio uses HTK mel scale with no norm by default
+        this.mel_filters = mel_filter_bank(
+            Math.floor(1 + n_fft / 2), // num_frequency_bins = 257
+            n_mels, // 80
+            0, // min_frequency
+            sample_rate / 2, // max_frequency = 8000
+            sample_rate, // 16000
+            null, // norm (torchaudio default: no norm)
+            'htk', // mel_scale (torchaudio default)
+        );
+
+        // torchaudio center-pads the window when win_length < n_fft:
+        // pad_amount = (n_fft - win_length) // 2 on each side
+        const raw_window = window_function(win_length, 'hann');
+        this.window = new Float64Array(n_fft);
+        const pad = Math.floor((n_fft - win_length) / 2);
+        this.window.set(raw_window, pad);
+    }
+
+    /**
+     * Extract mel spectrogram features from audio, matching the Python GraniteSpeechFeatureExtractor.
+     * @param {Float32Array|Float64Array} audio The audio waveform.
+     * @returns {Promise<{input_features: Tensor}>}
+     */
+    async _call(audio) {
+        validate_audio_inputs(audio, 'GraniteSpeechFeatureExtractor');
+
+        const { n_fft, hop_length, n_mels } = this.config.melspec_kwargs;
+
+        // Truncate to even number of frames for pair-stacking
+        const num_frames = 1 + Math.floor((audio.length - 1) / hop_length);
+        const max_num_frames = num_frames - (num_frames % 2);
+
+        const mel = await spectrogram(audio, this.window, n_fft, hop_length, {
+            power: 2.0,
+            mel_filters: this.mel_filters,
+            log_mel: 'log10_max_norm',
+            transpose: true, // [time, n_mels]
+            max_num_frames,
+            do_pad: false,
+        });
+
+        // Stack adjacent frame pairs: [time, n_mels] → [1, time/2, 2*n_mels]
+        const input_features = mel.view(-1, 2 * n_mels).unsqueeze_(0);
+
+        return { input_features };
+    }
+}

package/src/models/granite_speech/modeling_granite_speech.js

@@ -0,0 +1,5 @@
+import { UltravoxModel } from '../ultravox/modeling_ultravox.js';
+
+export class GraniteSpeechForConditionalGeneration extends UltravoxModel {
+    forward_params = ['input_ids', 'attention_mask', 'input_features', 'past_key_values'];
+}

package/src/models/granite_speech/processing_granite_speech.js

@@ -0,0 +1,62 @@
+import { AutoFeatureExtractor } from '../auto/feature_extraction_auto.js';
+import { AutoTokenizer } from '../auto/tokenization_auto.js';
+import { Processor } from '../../processing_utils.js';
+import { Tensor } from '../../utils/tensor.js';
+
+export class GraniteSpeechProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer;
+    static feature_extractor_class = AutoFeatureExtractor;
+    static uses_processor_config = true;
+
+    /**
+     * Compute the number of audio tokens for a given raw audio length.
+     * @param {number} audioLength Raw audio sample count.
+     * @returns {number} Number of projector output tokens.
+     */
+    _get_num_audio_features(audioLength) {
+        const { hop_length } = this.feature_extractor.config.melspec_kwargs;
+        const { projector_window_size, projector_downsample_rate } = this.feature_extractor.config;
+        const effective_window_size = Math.floor(projector_window_size / projector_downsample_rate);
+        const mel_length = Math.floor(audioLength / hop_length) + 1;
+        const encoder_length = Math.floor(mel_length / 2);
+        const nblocks = Math.ceil(encoder_length / projector_window_size);
+        return nblocks * effective_window_size;
+    }
+
+    /**
+     * @param {string} text The text input to process.
+     * @param {Float32Array} audio The audio input to process.
+     */
+    async _call(text, audio = null, kwargs = {}) {
+        if (Array.isArray(text)) {
+            throw new Error('Batched inputs are not supported yet.');
+        }
+
+        let audio_inputs = {};
+        if (audio) {
+            const { input_features } = await this.feature_extractor(audio);
+            audio_inputs['input_features'] = input_features;
+
+            // Compute audio embed sizes and mask in the processor
+            const audio_embed_size = this._get_num_audio_features(audio.length);
+            const mask_data = new Uint8Array(audio_embed_size).fill(1);
+            audio_inputs['input_features_mask'] = new Tensor('bool', mask_data, [1, audio_embed_size]);
+
+            const audio_token = this.config.audio_token ?? '<|audio|>';
+            if (!text.includes(audio_token)) {
+                throw new Error(`The input text does not contain the audio token ${audio_token}.`);
+            }
+            text = text.replaceAll(audio_token, audio_token.repeat(audio_embed_size));
+        }
+
+        const text_inputs = this.tokenizer(text, {
+            add_special_tokens: false,
+            ...kwargs,
+        });
+
+        return {
+            ...text_inputs,
+            ...audio_inputs,
+        };
+    }
+}

package/src/models/grounding_dino/image_processing_grounding_dino.js

@@ -2,7 +2,7 @@ import { ImageProcessor } from '../../image_processors_utils.js';
 import { ones } from '../../utils/tensor.js';
 
 /**
- * @typedef {object} GroundingDinoFeatureExtractorResultProps
+ * @typedef {Object} GroundingDinoFeatureExtractorResultProps
  * @property {import('../../utils/tensor.js').Tensor} pixel_mask
  * @typedef {import('../../image_processors_utils.js').ImageProcessorResult & GroundingDinoFeatureExtractorResultProps} GroundingDinoFeatureExtractorResult
  */

package/src/models/idefics3/modeling_idefics3.js

@@ -1,7 +1,9 @@
-import { PreTrainedModel, default_merge_input_ids_with_image_features } from '../modeling_utils.js';
-import { sessionRun } from '../session.js';
+import { LlavaForConditionalGeneration } from '../llava/modeling_llava.js';
 
-export class Idefics3PreTrainedModel extends PreTrainedModel {
+/**
+ * The Idefics3 model which consists of a vision backbone and a language model.
+ */
+export class Idefics3ForConditionalGeneration extends LlavaForConditionalGeneration {
     forward_params = [
         'input_ids',
         'attention_mask',
@@ -11,32 +13,3 @@ export class Idefics3PreTrainedModel extends PreTrainedModel {
         'past_key_values',
     ];
 }
-
-/**
- * The Idefics3 model which consists of a vision backbone and a language model.
- */
-export class Idefics3ForConditionalGeneration extends Idefics3PreTrainedModel {
-    async encode_image({ pixel_values, pixel_attention_mask }) {
-        const features = (await sessionRun(this.sessions['vision_encoder'], { pixel_values, pixel_attention_mask }))
-            .image_features;
-        return features;
-    }
-
-    _merge_input_ids_with_image_features(kwargs) {
-        const vision_hidden_size = kwargs.image_features.dims.at(-1);
-        const reshaped_image_hidden_states = kwargs.image_features.view(-1, vision_hidden_size);
-
-        return default_merge_input_ids_with_image_features({
-            // @ts-ignore
-            image_token_id: this.config.image_token_id,
-            ...kwargs,
-            image_features: reshaped_image_hidden_states,
-        });
-    }
-}
-
-/**
- * The SmolVLM Model with a language modeling head.
- * It is made up a SigLIP vision encoder, with a language modeling head on top.
- */
-export class SmolVLMForConditionalGeneration extends Idefics3ForConditionalGeneration {}

package/src/models/image_processors.js

@@ -14,6 +14,7 @@ export * from './grounding_dino/image_processing_grounding_dino.js';
 export * from './idefics3/image_processing_idefics3.js';
 export * from './janus/image_processing_janus.js';
 export * from './jina_clip/image_processing_jina_clip.js';
+export * from './lfm2_vl/image_processing_lfm2_vl.js';
 export * from './llava_onevision/image_processing_llava_onevision.js';
 export * from './mask2former/image_processing_mask2former.js';
 export * from './maskformer/image_processing_maskformer.js';