@huggingface/transformers 4.0.0-next.4 → 4.0.0-next.6

package/src/utils/hub/utils.js

@@ -1,4 +1,5 @@
 import { ERROR_MAPPING, REPO_ID_REGEX } from './constants.js';
+import { logger } from '../logger.js';
 
 /**
  * Joins multiple parts of a path into a single path, while handling leading and trailing slashes.
@@ -79,16 +80,21 @@ export function handleError(status, remoteURL, fatal) {
 /**
  * Read and track progress when reading a Response object
  *
- * @param {Response|import('./files.js').FileResponse} response The Response object to read
+ * @param {Response|import('./FileResponse.js').FileResponse} response The Response object to read
  * @param {(data: {progress: number, loaded: number, total: number}) => void} progress_callback The function to call with progress updates
+ * @param {number} [expectedSize] The expected size of the file (used when content-length header is missing)
  * @returns {Promise<Uint8Array>} A Promise that resolves with the Uint8Array buffer
  */
-export async function readResponse(response, progress_callback) {
+export async function readResponse(response, progress_callback, expectedSize) {
     const contentLength = response.headers.get('Content-Length');
-    if (contentLength === null) {
-        console.warn('Unable to determine content-length from response headers. Will expand buffer when needed.');
+
+    // Use content-length if available, otherwise fall back to expectedSize (from metadata)
+    let total = contentLength ? parseInt(contentLength, 10) : (expectedSize ?? 0);
+
+    if (contentLength === null && !expectedSize) {
+        logger.warn('Unable to determine content-length from response headers. Will expand buffer when needed.');
     }
-    let total = parseInt(contentLength ?? '0');
+
     let buffer = new Uint8Array(total);
     let loaded = 0;
 
@@ -126,3 +132,37 @@ export async function readResponse(response, progress_callback) {
 
     return buffer;
 }
+
+/**
+ * Checks if the given URL is a blob URL (created via URL.createObjectURL).
+ * Blob URLs should not be cached as they are temporary in-memory references.
+ * @param {string} url - The URL to check.
+ * @returns {boolean} True if the URL is a blob URL, false otherwise.
+ */
+export function isBlobURL(url) {
+    return isValidUrl(url, ['blob:']);
+}
+
+/**
+ * Converts any URL to an absolute URL if needed.
+ * If the URL is already absolute (http://, https://, or blob:), returns it unchanged (handled by new URL(...)).
+ * Otherwise, resolves it relative to the current page location (browser) or module location (Node/Bun/Deno).
+ * @param {string} url - The URL to convert (can be relative or absolute).
+ * @returns {string} The absolute URL.
+ */
+export function toAbsoluteURL(url) {
+    let baseURL;
+
+    if (typeof location !== 'undefined' && location.href) {
+        // Browser environment: use location.href
+        baseURL = location.href;
+    } else if (typeof import.meta !== 'undefined' && import.meta.url) {
+        // Node.js/Bun/Deno module environment: use import.meta.url
+        baseURL = import.meta.url;
+    } else {
+        // Fallback: if no base is available, return the URL unchanged
+        return url;
+    }
+
+    return new URL(url, baseURL).href;
+}

package/src/utils/hub.js

@@ -6,9 +6,12 @@
 
 import { apis, env } from '../env.js';
 import { dispatchCallback } from './core.js';
-import { FileResponse, FileCache } from './hub/files.js';
+import { FileResponse } from './hub/FileResponse.js';
+import { FileCache } from './cache/FileCache.js';
 import { handleError, isValidUrl, pathJoin, isValidHfModelId, readResponse } from './hub/utils.js';
 import { getCache, tryCache } from './cache.js';
+import { get_file_metadata } from './model_registry/get_file_metadata.js';
+import { logger } from './logger.js';
 
 export { MAX_EXTERNAL_DATA_CHUNKS } from './hub/constants.js';
 
@@ -37,7 +40,7 @@ export { MAX_EXTERNAL_DATA_CHUNKS } from './hub/constants.js';
  * @typedef {Object} ModelSpecificPretrainedOptions Options for loading a pretrained model.
  * @property {string} [subfolder='onnx'] In case the relevant files are located inside a subfolder of the model repo on huggingface.co,
  * you can specify the folder name here.
- * @property {string} [model_file_name=null] If specified, load the model with this name (excluding the .onnx suffix). Currently only valid for encoder- or decoder-only models.
+ * @property {string} [model_file_name=null] If specified, load the model with this name (excluding the dtype and .onnx suffixes). Currently only valid for encoder- or decoder-only models.
  * @property {import("./devices.js").DeviceType|Record<string, import("./devices.js").DeviceType>} [device=null] The device to run the model on. If not specified, the device will be chosen from the environment settings.
  * @property {import("./dtypes.js").DataType|Record<string, import("./dtypes.js").DataType>} [dtype=null] The data type to use for the model. If not specified, the data type will be chosen from the environment settings.
  * @property {ExternalData|Record<string, ExternalData>} [use_external_data_format=false] Whether to load the model using the external data format (used for models >= 2GB in size).
@@ -63,14 +66,30 @@ export async function getFile(urlOrPath) {
                     : urlOrPath.toString()
                 : urlOrPath,
         );
-    } else if (typeof process !== 'undefined' && process?.release?.name === 'node') {
+    } else {
+        return env.fetch(urlOrPath, {
+            headers: getFetchHeaders(urlOrPath),
+        });
+    }
+}
+
+/**
+ * Generates appropriate HTTP headers for fetching resources.
+ * In Node.js environments, adds User-Agent and Authorization headers when applicable.
+ * In browser environments, returns minimal headers for security.
+ *
+ * @param {URL|string} urlOrPath The URL or path being fetched.
+ * @returns {Headers} A Headers object with appropriate headers for the request.
+ */
+export function getFetchHeaders(urlOrPath) {
+    const isNode = typeof process !== 'undefined' && process?.release?.name === 'node';
+    const headers = new Headers();
+
+    if (isNode) {
         const IS_CI = !!process.env?.TESTING_REMOTELY;
         const version = env.version;
-
-        const headers = new Headers();
         headers.set('User-Agent', `transformers.js/${version}; is_ci/${IS_CI};`);
 
-        // Check whether we are making a request to the Hugging Face Hub.
         const isHFURL = isValidUrl(urlOrPath, ['http:', 'https:'], ['huggingface.co', 'hf.co']);
         if (isHFURL) {
             // If an access token is present in the environment variables,
@@ -81,13 +100,13 @@ export async function getFile(urlOrPath) {
                 headers.set('Authorization', `Bearer ${token}`);
             }
         }
-        return fetch(urlOrPath, { headers });
     } else {
         // Running in a browser-environment, so we use default headers
         // NOTE: We do not allow passing authorization headers in the browser,
         // since this would require exposing the token to the client.
-        return fetch(urlOrPath);
     }
+
+    return headers;
 }
 
 /**
@@ -103,7 +122,7 @@ export async function getFile(urlOrPath) {
  * @returns {{ requestURL: string, localPath: string, remoteURL: string, proposedCacheKey: string, validModelId: boolean }}
  * An object containing all the paths and URLs for the resource.
  */
-function buildResourcePaths(path_or_repo_id, filename, options = {}, cache = null) {
+export function buildResourcePaths(path_or_repo_id, filename, options = {}, cache = null) {
     const revision = options.revision ?? 'main';
     const requestURL = pathJoin(path_or_repo_id, filename);
 
@@ -142,7 +161,7 @@ function buildResourcePaths(path_or_repo_id, filename, options = {}, cache = nul
  * @param {import('./cache.js').CacheInterface | null} cache The cache instance to check.
  * @param {string} localPath The local path to try first.
  * @param {string} proposedCacheKey The proposed cache key to try second.
- * @returns {Promise<Response|import('./hub/files.js').FileResponse|undefined|string>}
+ * @returns {Promise<Response|import('./hub/FileResponse.js').FileResponse|undefined|string>}
  * The cached response if found, undefined otherwise.
  */
 export async function checkCachedResource(cache, localPath, proposedCacheKey) {
@@ -164,7 +183,7 @@ export async function checkCachedResource(cache, localPath, proposedCacheKey) {
  * @param {string} filename The name of the file to cache.
  * @param {import('./cache.js').CacheInterface} cache The cache instance to store in.
  * @param {string} cacheKey The cache key to use.
- * @param {Response|import('./hub/files.js').FileResponse} response The response to cache.
+ * @param {Response|import('./hub/FileResponse.js').FileResponse} response The response to cache.
  * @param {Uint8Array} [result] The result buffer if already read.
  * @param {PretrainedOptions} [options] Options containing progress callback and context for progress updates.
  * @returns {Promise<void>}
@@ -200,7 +219,7 @@ export async function storeCachedResource(path_or_repo_id, filename, cache, cach
             .catch((err) => {
                 // Do not crash if unable to add to cache (e.g., QuotaExceededError).
                 // Rather, log a warning and proceed with execution.
-                console.warn(`Unable to add response to browser cache: ${err}.`);
+                logger.warn(`Unable to add response to browser cache: ${err}.`);
             });
     }
 }
@@ -241,7 +260,7 @@ export async function loadResourceFile(
     // Whether to cache the final response in the end.
     let toCacheResponse = false;
 
-    /** @type {Response|import('./hub/files.js').FileResponse|undefined|string} */
+    /** @type {Response|import('./hub/FileResponse.js').FileResponse|undefined|string} */
     let response;
 
     // Check cache
@@ -262,7 +281,7 @@ export async function loadResourceFile(
                 } catch (e) {
                     // Something went wrong while trying to get the file locally.
                     // NOTE: error handling is done in the next step (since `response` will be undefined)
-                    console.warn(`Unable to load from local path "${localPath}": "${e}"`);
+                    logger.warn(`Unable to load from local path "${localPath}": "${e}"`);
                 }
             } else if (options.local_files_only) {
                 throw new Error(`\`local_files_only=true\`, but attempted to load a remote file from: ${requestURL}.`);
@@ -354,14 +373,36 @@ export async function loadResourceFile(
                     total: buffer.length,
                 });
             } else {
-                buffer = await readResponse(response, (data) => {
-                    dispatchCallback(options.progress_callback, {
-                        status: 'progress',
-                        name: path_or_repo_id,
-                        file: filename,
-                        ...data,
-                    });
-                });
+                // Get expected file size from response headers or metadata
+                // This helps with progress tracking when content-length is missing
+                let expectedSize;
+                const contentLength = response.headers.get('content-length');
+                if (contentLength) {
+                    expectedSize = parseInt(contentLength, 10);
+                } else {
+                    // Try to get size from metadata (useful when content-length is missing during download)
+                    try {
+                        const metadata = await get_file_metadata(path_or_repo_id, filename, options);
+                        if (metadata.size) {
+                            expectedSize = metadata.size;
+                        }
+                    } catch (e) {
+                        // Ignore metadata fetch errors
+                    }
+                }
+
+                buffer = await readResponse(
+                    response,
+                    (data) => {
+                        dispatchCallback(options.progress_callback, {
+                            status: 'progress',
+                            name: path_or_repo_id,
+                            file: filename,
+                            ...data,
+                        });
+                    },
+                    expectedSize,
+                );
             }
         }
         result = buffer;

package/src/utils/image.js

@@ -15,16 +15,16 @@ import { saveBlob } from './io.js';
 
 // Will be empty (or not used) if running in browser or web-worker
 import sharp from 'sharp';
+import { logger } from './logger.js';
 
 let createCanvasFunction;
 let ImageDataClass;
 let loadImageFunction;
-const IS_BROWSER_OR_WEBWORKER = apis.IS_BROWSER_ENV || apis.IS_WEBWORKER_ENV;
-if (IS_BROWSER_OR_WEBWORKER) {
+if (apis.IS_WEB_ENV) {
     // Running in browser or web-worker
     createCanvasFunction = (/** @type {number} */ width, /** @type {number} */ height) => {
         if (!self.OffscreenCanvas) {
-            throw new Error('OffscreenCanvas not supported by this browser.');
+            throw new Error('OffscreenCanvas not supported by this environment.');
         }
         return new self.OffscreenCanvas(width, height);
     };
@@ -133,7 +133,7 @@ export class RawImage {
      * @returns {RawImage} The image object.
      */
     static fromCanvas(canvas) {
-        if (!IS_BROWSER_OR_WEBWORKER) {
+        if (!apis.IS_WEB_ENV) {
             throw new Error('fromCanvas() is only supported in browser environments.');
         }
 
@@ -164,7 +164,7 @@ export class RawImage {
      * @returns {Promise<RawImage>} The image object.
      */
     static async fromBlob(blob) {
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             // Running in environment with canvas
             const img = await loadImageFunction(blob);
 
@@ -378,7 +378,7 @@ export class RawImage {
             height = (width / this.width) * this.height;
         }
 
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             // TODO use `resample` in browser environment
 
             // Store number of channels before resizing
@@ -406,7 +406,7 @@ export class RawImage {
                 case 'box':
                 case 'hamming':
                     if (resampleMethod === 'box' || resampleMethod === 'hamming') {
-                        console.warn(
+                        logger.warn(
                             `Resampling method ${resampleMethod} is not yet supported. Using bilinear instead.`,
                         );
                         resampleMethod = 'bilinear';
@@ -452,7 +452,7 @@ export class RawImage {
             return this;
         }
 
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             // Store number of channels before padding
             const numChannels = this.channels;
 
@@ -494,7 +494,7 @@ export class RawImage {
         const crop_width = x_max - x_min + 1;
         const crop_height = y_max - y_min + 1;
 
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             // Store number of channels before resizing
             const numChannels = this.channels;
 
@@ -541,7 +541,7 @@ export class RawImage {
         const width_offset = (this.width - crop_width) / 2;
         const height_offset = (this.height - crop_height) / 2;
 
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             // Store number of channels before resizing
             const numChannels = this.channels;
 
@@ -649,7 +649,7 @@ export class RawImage {
     }
 
     async toBlob(type = 'image/png', quality = 1) {
-        if (!IS_BROWSER_OR_WEBWORKER) {
+        if (!apis.IS_WEB_ENV) {
             throw new Error('toBlob() is only supported in browser environments.');
         }
 
@@ -672,7 +672,7 @@ export class RawImage {
     }
 
     toCanvas() {
-        if (!IS_BROWSER_OR_WEBWORKER) {
+        if (!apis.IS_WEB_ENV) {
             throw new Error('toCanvas() is only supported in browser environments.');
         }
 
@@ -773,7 +773,7 @@ export class RawImage {
      * @returns {Promise<void>}
      */
     async save(path) {
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             if (apis.IS_WEBWORKER_ENV) {
                 throw new Error('Unable to save an image from a Web Worker.');
             }
@@ -798,7 +798,7 @@ export class RawImage {
      * @returns {import('sharp').Sharp} The Sharp instance.
      */
     toSharp() {
-        if (IS_BROWSER_OR_WEBWORKER) {
+        if (apis.IS_WEB_ENV) {
             throw new Error('toSharp() is only supported in server-side environments.');
         }
 

package/src/utils/logger.js

@@ -0,0 +1,67 @@
+/**
+ * @file Logger utility for Transformers.js with configurable log levels.
+ *
+ * @module utils/logger
+ */
+
+import { env, LogLevel } from '../env.js';
+
+/**
+ * Logger that respects the configured log level in env.logLevel.
+ *
+ * @example
+ * import { logger } from './utils/logger.js';
+ * logger.info('Model loaded successfully');
+ * logger.warn('Deprecated method used');
+ * logger.error('Failed to load model');
+ * logger.debug('Token count:', tokens.length);
+ */
+export const logger = {
+    /**
+     * Log an error message. Only suppressed when logLevel is NONE.
+     * @param  {...any} args - Arguments to log
+     */
+    error(...args) {
+        if (env.logLevel <= LogLevel.ERROR) {
+            console.error(...args);
+        }
+    },
+
+    /**
+     * Log a warning message. Shown when logLevel <= WARNING.
+     * @param  {...any} args - Arguments to log
+     */
+    warn(...args) {
+        if (env.logLevel <= LogLevel.WARNING) {
+            console.warn(...args);
+        }
+    },
+
+    /**
+     * Log an info message. Shown when logLevel <= INFO.
+     * @param  {...any} args - Arguments to log
+     */
+    info(...args) {
+        if (env.logLevel <= LogLevel.INFO) {
+            console.log(...args);
+        }
+    },
+
+    /**
+     * Log a debug message. Only shown when logLevel is DEBUG.
+     * @param  {...any} args - Arguments to log
+     */
+    debug(...args) {
+        if (env.logLevel <= LogLevel.DEBUG) {
+            console.log(...args);
+        }
+    },
+
+    /**
+     * Log a message (alias for info). Shown when logLevel <= INFO.
+     * @param  {...any} args - Arguments to log
+     */
+    log(...args) {
+        this.info(...args);
+    },
+};

package/src/utils/model-loader.js

@@ -1,6 +1,37 @@
 import { getModelFile, MAX_EXTERNAL_DATA_CHUNKS } from './hub.js';
 import { apis } from '../env.js';
 
+/**
+ * Resolves an `use_external_data_format` config value to the number of data chunks for a given file.
+ * @param {import('./hub.js').ExternalData|Record<string, import('./hub.js').ExternalData>|null|undefined} config The external data format configuration.
+ * @param {string} fullName The full ONNX file name (e.g., "model_quantized.onnx").
+ * @param {string} fileName The base file name (e.g., "model").
+ * @returns {number} The number of external data chunks (0 if none).
+ */
+export function resolveExternalDataFormat(config, fullName, fileName) {
+    if (!config) return 0;
+    if (typeof config === 'object' && config !== null) {
+        if (config.hasOwnProperty(fullName)) return +config[fullName];
+        if (config.hasOwnProperty(fileName)) return +config[fileName];
+        return 0;
+    }
+    return +config; // (false=0, true=1, number remains the same)
+}
+
+/**
+ * Generates the file names for external data chunks.
+ * @param {string} fullName The full ONNX file name (e.g., "model_quantized.onnx").
+ * @param {number} numChunks The number of external data chunks.
+ * @returns {string[]} Array of external data file names.
+ */
+export function getExternalDataChunkNames(fullName, numChunks) {
+    const names = [];
+    for (let i = 0; i < numChunks; ++i) {
+        names.push(`${fullName}_data${i === 0 ? '' : '_' + i}`);
+    }
+    return names;
+}
+
 /**
  * Loads the core model file.
  *
@@ -42,28 +73,15 @@ export async function getModelDataFiles(
     /** @type {Promise<string|{path: string, data: Uint8Array}>[]} */
     let externalDataPromises = [];
 
-    if (use_external_data_format) {
-        let external_data_format;
-        if (typeof use_external_data_format === 'object') {
-            if (use_external_data_format.hasOwnProperty(baseName)) {
-                external_data_format = use_external_data_format[baseName];
-            } else if (use_external_data_format.hasOwnProperty(fileName)) {
-                external_data_format = use_external_data_format[fileName];
-            } else {
-                external_data_format = false;
-            }
-        } else {
-            external_data_format = use_external_data_format;
-        }
-
-        const num_chunks = +external_data_format; // (false=0, true=1, number remains the same)
+    const num_chunks = resolveExternalDataFormat(use_external_data_format, baseName, fileName);
+    if (num_chunks > 0) {
         if (num_chunks > MAX_EXTERNAL_DATA_CHUNKS) {
             throw new Error(
                 `The number of external data chunks (${num_chunks}) exceeds the maximum allowed value (${MAX_EXTERNAL_DATA_CHUNKS}).`,
             );
         }
-        for (let i = 0; i < num_chunks; ++i) {
-            const path = `${baseName}_data${i === 0 ? '' : '_' + i}`;
+        const chunkNames = getExternalDataChunkNames(baseName, num_chunks);
+        for (const path of chunkNames) {
             const fullPath = `${options.subfolder ?? ''}/${path}`;
             externalDataPromises.push(
                 new Promise(async (resolve, reject) => {