@huggingface/transformers 3.0.1 → 3.1.0

package/src/base/processing_utils.js

@@ -0,0 +1,145 @@
+
+/**
+ * @file Processors are used to prepare inputs (e.g., text, image or audio) for a model.
+ * 
+ * **Example:** Using a `WhisperProcessor` to prepare an audio input for a model.
+ * ```javascript
+ * import { AutoProcessor, read_audio } from '@huggingface/transformers';
+ *
+ * const processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');
+ * const audio = await read_audio('https://huggingface.co/datasets/Narsil/asr_dummy/resolve/main/mlk.flac', 16000);
+ * const { input_features } = await processor(audio);
+ * // Tensor {
+ * //   data: Float32Array(240000) [0.4752984642982483, 0.5597258806228638, 0.56434166431427, ...],
+ * //   dims: [1, 80, 3000],
+ * //   type: 'float32',
+ * //   size: 240000,
+ * // }
+ * ```
+ * 
+ * @module processors
+ */
+import { PROCESSOR_NAME } from '../utils/constants.js';
+import {
+    Callable,
+} from '../utils/generic.js';
+import { getModelJSON } from '../utils/hub.js';
+
+/**
+ * @typedef {Object} ProcessorProperties Additional processor-specific properties.
+ * @typedef {import('../utils/hub.js').PretrainedOptions & ProcessorProperties} PretrainedProcessorOptions
+ */
+
+
+/**
+ * Represents a Processor that extracts features from an input.
+ */
+export class Processor extends Callable {
+    static classes = [
+        'image_processor_class',
+        'tokenizer_class',
+        'feature_extractor_class',
+    ]
+    static uses_processor_config = false;
+
+    /**
+     * Creates a new Processor with the given components
+     * @param {Object} config 
+     * @param {Record<string, Object>} components 
+     */
+    constructor(config, components) {
+        super();
+        this.config = config;
+        this.components = components;
+    }
+
+    /**
+     * @returns {import('./image_processors_utils.js').ImageProcessor|undefined} The image processor of the processor, if it exists.
+     */
+    get image_processor() {
+        return this.components.image_processor;
+    }
+
+    /**
+     * @returns {import('../tokenizers.js').PreTrainedTokenizer|undefined} The tokenizer of the processor, if it exists.
+     */
+    get tokenizer() {
+        return this.components.tokenizer;
+    }
+
+    /**
+     * @returns {import('./feature_extraction_utils.js').FeatureExtractor|undefined} The feature extractor of the processor, if it exists.
+     */
+    get feature_extractor() {
+        return this.components.feature_extractor;
+    }
+
+    apply_chat_template(messages, options = {}) {
+        if (!this.tokenizer) {
+            throw new Error('Unable to apply chat template without a tokenizer.');
+        }
+        return this.tokenizer.apply_chat_template(messages, {
+            tokenize: false, // default to false
+            ...options,
+        });
+    }
+
+    batch_decode(...args) {
+        if (!this.tokenizer) {
+            throw new Error('Unable to decode without a tokenizer.');
+        }
+        return this.tokenizer.batch_decode(...args);
+    }
+
+
+    /**
+     * Calls the feature_extractor function with the given input.
+     * @param {any} input The input to extract features from.
+     * @param {...any} args Additional arguments.
+     * @returns {Promise<any>} A Promise that resolves with the extracted features.
+     */
+    async _call(input, ...args) {
+        for (const item of [this.image_processor, this.feature_extractor, this.tokenizer]) {
+            if (item) {
+                return item(input, ...args);
+            }
+        }
+        throw new Error('No image processor, feature extractor, or tokenizer found.');
+    }
+
+
+    /**
+     * Instantiate one of the processor classes of the library from a pretrained model.
+     * 
+     * The processor class to instantiate is selected based on the `feature_extractor_type` property of the config object
+     * (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible)
+     * 
+     * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either:
+     * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co.
+     *   Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+     *   user or organization name, like `dbmdz/bert-base-german-cased`.
+     * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`.
+     * @param {PretrainedProcessorOptions} options Additional options for loading the processor.
+     * 
+     * @returns {Promise<Processor>} A new instance of the Processor class.
+     */
+    static async from_pretrained(pretrained_model_name_or_path, options) {
+
+        const [config, components] = await Promise.all([
+            // TODO:
+            this.uses_processor_config
+                ? getModelJSON(pretrained_model_name_or_path, PROCESSOR_NAME, true, options)
+                : {},
+            Promise.all(
+                this.classes
+                    .filter((cls) => cls in this)
+                    .map(async (cls) => {
+                        const component = await this[cls].from_pretrained(pretrained_model_name_or_path, options);
+                        return [cls.replace(/_class$/, ''), component];
+                    })
+            ).then(Object.fromEntries)
+        ]);
+
+        return new this(config, components);
+    }
+}

package/src/configs.js

@@ -36,6 +36,13 @@ import {
  * @typedef {import('./utils/hub.js').PretrainedOptions} PretrainedOptions
  */
 
+/**
+ * @typedef {import('./utils/core.js').ProgressCallback} ProgressCallback
+ */
+
+/**
+ * @typedef {import('./utils/core.js').ProgressInfo} ProgressInfo
+ */
 
 /**
  * Loads a config from the specified path.
@@ -61,6 +68,7 @@ function getNormalizedConfig(config) {
         case 'llava':
         case 'paligemma':
         case 'florence2':
+        case 'llava_onevision':
             init_normalized_config = getNormalizedConfig(config.text_config);
             break;
         case 'moondream1':
@@ -69,6 +77,9 @@ function getNormalizedConfig(config) {
         case 'musicgen':
             init_normalized_config = getNormalizedConfig(config.decoder);
             break;
+        case 'multi_modality':
+            init_normalized_config = getNormalizedConfig(config.language_config);
+            break;
 
         // Decoder-only models
         case 'gpt2':
@@ -91,11 +102,14 @@ function getNormalizedConfig(config) {
             mapping['hidden_size'] = 'hidden_size';
             break;
         case 'llama':
+        case 'olmo':
+        case 'mobilellm':
         case 'granite':
         case 'cohere':
         case 'mistral':
         case 'starcoder2':
         case 'qwen2':
+        case 'qwen2_vl':
             mapping['num_heads'] = 'num_key_value_heads';
             mapping['num_layers'] = 'num_hidden_layers';
             mapping['hidden_size'] = 'hidden_size';
@@ -216,14 +230,12 @@ function getNormalizedConfig(config) {
  */
 export function getKeyValueShapes(config, {
     prefix = 'past_key_values',
+    batch_size=1,
 } = {}) {
     /** @type {Record<string, number[]>} */
     const decoderFeeds = {};
     const normalized_config = config.normalized_config;
 
-    // TODO support batches (i.e., batch_size > 1)
-    const batch_size = 1;
-
     if (normalized_config.is_encoder_decoder && (
         'num_encoder_heads' in normalized_config && 'num_decoder_heads' in normalized_config
     )) {

package/src/env.js

@@ -26,7 +26,7 @@ import fs from 'fs';
 import path from 'path';
 import url from 'url';
 
-const VERSION = '3.0.1';
+const VERSION = '3.1.0';
 
 // Check if various APIs are available (depends on environment)
 const IS_BROWSER_ENV = typeof self !== 'undefined';
@@ -73,9 +73,20 @@ export const apis = Object.freeze({
 });
 
 const RUNNING_LOCALLY = IS_FS_AVAILABLE && IS_PATH_AVAILABLE;
-const dirname__ = RUNNING_LOCALLY
-    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))
-    : './';
+
+let dirname__ = './';
+if (RUNNING_LOCALLY) {
+    // NOTE: We wrap `import.meta` in a call to `Object` to prevent Webpack from trying to bundle it in CommonJS.
+    // Although we get the warning: "Accessing import.meta directly is unsupported (only property access or destructuring is supported)",
+    // it is safe to ignore since the bundled value (`{}`) isn't used for CommonJS environments (we use __dirname instead).
+    const _import_meta_url = Object(import.meta).url;
+
+    if (_import_meta_url) {
+        dirname__ = path.dirname(path.dirname(url.fileURLToPath(_import_meta_url))) // ESM
+    } else if (typeof __dirname !== 'undefined') {
+        dirname__ = path.dirname(__dirname) // CommonJS
+    }
+}
 
 // Only used for environments with access to file system
 const DEFAULT_CACHE_DIR = RUNNING_LOCALLY

package/src/models/audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js

@@ -0,0 +1,90 @@
+import { FeatureExtractor, validate_audio_inputs } from '../../base/feature_extraction_utils.js';
+import { Tensor } from '../../utils/tensor.js';
+import { mel_filter_bank, spectrogram, window_function } from '../../utils/audio.js';
+
+
+export class ASTFeatureExtractor extends FeatureExtractor {
+
+    constructor(config) {
+        super(config);
+
+        const sampling_rate = this.config.sampling_rate;
+        const mel_filters = mel_filter_bank(
+            256, // num_frequency_bins
+            this.config.num_mel_bins, // num_mel_filters
+            20, // min_frequency
+            Math.floor(sampling_rate / 2), // max_frequency
+            sampling_rate, // sampling_rate
+            null, // norm
+            "kaldi", // mel_scale
+            true, // triangularize_in_mel_space
+        );
+
+        // Do padding:
+        for (let i = 0; i < mel_filters.length; ++i) {
+            mel_filters[i].push(0);
+        }
+        this.mel_filters = mel_filters;
+
+        this.window = window_function(400, 'hann', {
+            periodic: false,
+        })
+
+        this.mean = this.config.mean;
+        this.std = this.config.std;
+    }
+
+    /**
+     * Computes the log-Mel spectrogram of the provided audio waveform.
+     * @param {Float32Array|Float64Array} waveform The audio waveform to process.
+     * @param {number} max_length The maximum number of frames to return.
+     * @returns {Promise<Tensor>} An object containing the log-Mel spectrogram data as a Float32Array and its dimensions as an array of numbers.
+     */
+    async _extract_fbank_features(waveform, max_length) {
+        // NOTE: We don't pad/truncate since that is passed in as `max_num_frames`
+        return spectrogram(
+            waveform,
+            this.window, // window
+            400, // frame_length
+            160, // hop_length
+            {
+                fft_length: 512,
+                power: 2.0,
+                center: false,
+                preemphasis: 0.97,
+                mel_filters: this.mel_filters,
+                log_mel: 'log',
+                mel_floor: 1.192092955078125e-07,
+                remove_dc_offset: true,
+
+                // Custom
+                max_num_frames: max_length,
+                transpose: true,
+            }
+        )
+    }
+
+
+    /**
+     * Asynchronously extracts features from a given audio using the provided configuration.
+     * @param {Float32Array|Float64Array} audio The audio data as a Float32Array/Float64Array.
+     * @returns {Promise<{ input_values: Tensor }>} A Promise resolving to an object containing the extracted input features as a Tensor.
+     */
+    async _call(audio) {
+        validate_audio_inputs(audio, 'ASTFeatureExtractor');
+
+        const features = await this._extract_fbank_features(audio, this.config.max_length);
+        if (this.config.do_normalize) {
+            // Normalize the input audio spectrogram to have mean=0, std=0.5
+            const denom = this.std * 2;
+            const features_data = features.data;
+            for (let i = 0; i < features_data.length; ++i) {
+                features_data[i] = (features_data[i] - this.mean) / denom;
+            }
+        }
+
+        return {
+            input_values: features.unsqueeze_(0)
+        };
+    }
+}

package/src/models/auto/feature_extraction_auto.js

@@ -0,0 +1,41 @@
+
+import { FEATURE_EXTRACTOR_NAME, GITHUB_ISSUE_URL } from '../../utils/constants.js';
+import { getModelJSON } from '../../utils/hub.js';
+import { FeatureExtractor } from '../../base/feature_extraction_utils.js';
+import * as AllFeatureExtractors from '../feature_extractors.js';
+
+export class AutoFeatureExtractor {
+
+    /**
+     * Instantiate one of the feature extractor classes of the library from a pretrained model.
+     * 
+     * The processor class to instantiate is selected based on the `feature_extractor_type` property of
+     * the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible)
+     * 
+     * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either:
+     * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co.
+     *   Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+     *   user or organization name, like `dbmdz/bert-base-german-cased`.
+     * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`.
+     * @param {import('../../utils/hub.js').PretrainedOptions} options Additional options for loading the processor.
+     * 
+     * @returns {Promise<AllFeatureExtractors.ImageProcessor>} A new instance of the Processor class.
+     */
+
+    /** @type {typeof FeatureExtractor.from_pretrained} */
+    static async from_pretrained(pretrained_model_name_or_path, options={}) {
+
+        const preprocessorConfig = await getModelJSON(pretrained_model_name_or_path, FEATURE_EXTRACTOR_NAME, true, options);
+
+        // Determine feature extractor class
+        const key = preprocessorConfig.feature_extractor_type;
+        const feature_extractor_class = AllFeatureExtractors[key];
+
+        if (!feature_extractor_class) {
+            throw new Error(`Unknown feature_extractor_type: '${key}'. Please report this at ${GITHUB_ISSUE_URL}.`);
+        }
+
+        // Instantiate feature extractor
+        return new feature_extractor_class(preprocessorConfig);
+    }
+}

package/src/models/auto/image_processing_auto.js

@@ -0,0 +1,29 @@
+
+import { GITHUB_ISSUE_URL, IMAGE_PROCESSOR_NAME } from '../../utils/constants.js';
+import { getModelJSON } from '../../utils/hub.js';
+import { ImageProcessor } from '../../base/image_processors_utils.js';
+import * as AllImageProcessors from '../image_processors.js';
+
+export class AutoImageProcessor {
+
+    /** @type {typeof ImageProcessor.from_pretrained} */
+    static async from_pretrained(pretrained_model_name_or_path, options={}) {
+
+        const preprocessorConfig = await getModelJSON(pretrained_model_name_or_path, IMAGE_PROCESSOR_NAME, true, options);
+
+        // Determine image processor class
+        const key = preprocessorConfig.image_processor_type ?? preprocessorConfig.feature_extractor_type;
+        let image_processor_class = AllImageProcessors[key];
+
+        if (!image_processor_class) {
+            if (key !== undefined) {
+                // Only log a warning if the class is not found and the key is set.
+                console.warn(`Image processor type '${key}' not found, assuming base ImageProcessor. Please report this at ${GITHUB_ISSUE_URL}.`)
+            }
+            image_processor_class = ImageProcessor;
+        }
+
+        // Instantiate image processor
+        return new image_processor_class(preprocessorConfig);
+    }
+}

package/src/models/auto/processing_auto.js

@@ -0,0 +1,100 @@
+
+
+import { IMAGE_PROCESSOR_NAME } from '../../utils/constants.js';
+import { getModelJSON } from '../../utils/hub.js';
+import { Processor } from '../../base/processing_utils.js';
+
+import * as AllProcessors from '../processors.js';
+import * as AllImageProcessors from '../image_processors.js';
+import * as AllFeatureExtractors from '../feature_extractors.js';
+
+/**
+ * Helper class which is used to instantiate pretrained processors with the `from_pretrained` function.
+ * The chosen processor class is determined by the type specified in the processor config.
+ * 
+ * **Example:** Load a processor using `from_pretrained`.
+ * ```javascript
+ * let processor = await AutoProcessor.from_pretrained('openai/whisper-tiny.en');
+ * ```
+ * 
+ * **Example:** Run an image through a processor.
+ * ```javascript
+ * let processor = await AutoProcessor.from_pretrained('Xenova/clip-vit-base-patch16');
+ * let image = await RawImage.read('https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/football-match.jpg');
+ * let image_inputs = await processor(image);
+ * // {
+ * //   "pixel_values": {
+ * //     "dims": [ 1, 3, 224, 224 ],
+ * //     "type": "float32",
+ * //     "data": Float32Array [ -1.558687686920166, -1.558687686920166, -1.5440893173217773, ... ],
+ * //     "size": 150528
+ * //   },
+ * //   "original_sizes": [
+ * //     [ 533, 800 ]
+ * //   ],
+ * //   "reshaped_input_sizes": [
+ * //     [ 224, 224 ]
+ * //   ]
+ * // }
+ * ```
+ */
+export class AutoProcessor {
+
+    /**
+     * Instantiate one of the processor classes of the library from a pretrained model.
+     * 
+     * The processor class to instantiate is selected based on the `image_processor_type` (or `feature_extractor_type`; legacy)
+     * property of the config object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible)
+     * 
+     * @param {string} pretrained_model_name_or_path The name or path of the pretrained model. Can be either:
+     * - A string, the *model id* of a pretrained processor hosted inside a model repo on huggingface.co.
+     *   Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+     *   user or organization name, like `dbmdz/bert-base-german-cased`.
+     * - A path to a *directory* containing processor files, e.g., `./my_model_directory/`.
+     * @param {import('../../utils/hub.js').PretrainedOptions} options Additional options for loading the processor.
+     * 
+     * @returns {Promise<Processor>} A new instance of the Processor class.
+     */
+
+    /** @type {typeof Processor.from_pretrained} */
+    static async from_pretrained(pretrained_model_name_or_path, options={}) {
+
+        // TODO: first check for processor.json 
+        const preprocessorConfig = await getModelJSON(pretrained_model_name_or_path, IMAGE_PROCESSOR_NAME, true, options);
+
+        const { image_processor_type, feature_extractor_type, processor_class } = preprocessorConfig;
+        if (processor_class && AllProcessors[processor_class]) {
+            return AllProcessors[processor_class].from_pretrained(pretrained_model_name_or_path, options);
+        }
+
+        if (!image_processor_type && !feature_extractor_type) {
+            throw new Error('No `image_processor_type` or `feature_extractor_type` found in the config.');
+        }
+
+        const components = {};
+        if (image_processor_type) {
+            const image_processor_class = AllImageProcessors[image_processor_type];
+            if (!image_processor_class) {
+                throw new Error(`Unknown image_processor_type: '${image_processor_type}'.`);
+            }
+            components.image_processor = new image_processor_class(preprocessorConfig);
+        }
+
+        if (feature_extractor_type) {
+            const image_processor_class = AllImageProcessors[feature_extractor_type];
+            if (image_processor_class) {
+                // Handle legacy case where image processors were specified as feature extractors
+                components.image_processor = new image_processor_class(preprocessorConfig);
+            } else {
+                const feature_extractor_class = AllFeatureExtractors[feature_extractor_type];
+                if (!feature_extractor_class) {
+                    throw new Error(`Unknown feature_extractor_type: '${feature_extractor_type}'.`);
+                }
+                components.feature_extractor = new feature_extractor_class(preprocessorConfig);
+            }
+        }
+
+        const config = {};
+        return new Processor(config, components);
+    }
+}

package/src/models/beit/image_processing_beit.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class BeitFeatureExtractor extends ImageProcessor { }

package/src/models/bit/image_processing_bit.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class BitImageProcessor extends ImageProcessor { }

package/src/models/chinese_clip/image_processing_chinese_clip.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class ChineseCLIPFeatureExtractor extends ImageProcessor { }