@huggingface/transformers 4.0.0-next.4 → 4.0.0-next.6

package/src/utils/model_registry/ModelRegistry.js

@@ -0,0 +1,346 @@
+/**
+ * @file Model registry for cache and file operations
+ *
+ * Provides static methods for:
+ * - Discovering which files a model needs
+ * - Getting file metadata
+ * - Checking cache status
+ *
+ * **Example:** Get all files needed for a model
+ * ```javascript
+ * const files = await ModelRegistry.get_files(
+ *   "onnx-community/all-MiniLM-L6-v2-ONNX",
+ *   { dtype: "fp16" },
+ * );
+ * console.log(files); // [ 'config.json', 'onnx/model_fp16.onnx', 'onnx/model_fp16.onnx_data', 'tokenizer.json', 'tokenizer_config.json' ]
+ * ```
+ *
+ * **Example:** Get all files needed for a specific pipeline task
+ * ```javascript
+ * const files = await ModelRegistry.get_pipeline_files(
+ *   "text-generation",
+ *   "onnx-community/Qwen3-0.6B-ONNX",
+ *   { dtype: "q4" },
+ * );
+ * console.log(files); // [ 'config.json', 'onnx/model_q4.onnx', 'generation_config.json', 'tokenizer.json', 'tokenizer_config.json' ]
+ * ```
+ *
+ * **Example:** Get specific component files
+ * ```javascript
+ * const modelFiles = await ModelRegistry.get_model_files("onnx-community/all-MiniLM-L6-v2-ONNX", { dtype: "q4" });
+ * const tokenizerFiles = await ModelRegistry.get_tokenizer_files("onnx-community/all-MiniLM-L6-v2-ONNX");
+ * const processorFiles = await ModelRegistry.get_processor_files("onnx-community/all-MiniLM-L6-v2-ONNX");
+ * console.log(modelFiles); // [ 'config.json', 'onnx/model_q4.onnx', 'onnx/model_q4.onnx_data' ]
+ * console.log(tokenizerFiles); // [ 'tokenizer.json', 'tokenizer_config.json' ]
+ * console.log(processorFiles); // [ ]
+ * ```
+ *
+ * **Example:** Check file metadata without downloading
+ * ```javascript
+ * const metadata = await ModelRegistry.get_file_metadata(
+ *   "onnx-community/Qwen3-0.6B-ONNX",
+ *   "config.json"
+ * );
+ * console.log(metadata); // { exists: true, size: 912, contentType: 'application/json', fromCache: true }
+ * ```
+ *
+ * **Example:** Model cache management
+ * ```javascript
+ * const modelId = "onnx-community/Qwen3-0.6B-ONNX";
+ * const options = { dtype: "q4" };
+ *
+ * // Quickly check if the model is cached (probably false)
+ * let cached = await ModelRegistry.is_cached(modelId, options);
+ * console.log(cached); // false
+ *
+ * // Get per-file cache detail
+ * let cacheStatus = await ModelRegistry.is_cached_files(modelId, options);
+ * console.log(cacheStatus);
+ * // {
+ * //   allCached: false,
+ * //   files: [ { file: 'config.json', cached: true }, { file: 'onnx/model_q4.onnx', cached: false }, { file: 'generation_config.json', cached: false }, { file: 'tokenizer.json', cached: false }, { file: 'tokenizer_config.json', cached: false } ]
+ * // }
+ *
+ * // Download the model by instantiating a pipeline
+ * const generator = await pipeline("text-generation", modelId, options);
+ * const output = await generator(
+ *   [{ role: "user", content: "What is the capital of France?" }],
+ *   { max_new_tokens: 256, do_sample: false },
+ * );
+ * console.log(output[0].generated_text.at(-1).content); // <think>...</think>\n\nThe capital of France is **Paris**.
+ *
+ * // Check if the model is cached (should be true now)
+ * cached = await ModelRegistry.is_cached(modelId, options);
+ * console.log(cached); // true
+ *
+ * // Clear the cache
+ * const clearResult = await ModelRegistry.clear_cache(modelId, options);
+ * console.log(clearResult);
+ * // {
+ * //   filesDeleted: 5,
+ * //   filesCached: 5,
+ * //   files: [ { file: 'config.json', deleted: true, wasCached: true }, { file: 'onnx/model_q4.onnx', deleted: true, wasCached: true }, { file: 'generation_config.json', deleted: true, wasCached: true }, { file: 'tokenizer.json', deleted: true, wasCached: true }, { file: 'tokenizer_config.json', deleted: true, wasCached: true } ]
+ * // }
+ *
+ * // Check if the model is cached (should be false again)
+ * cached = await ModelRegistry.is_cached(modelId, options);
+ * console.log(cached); // false
+ * ```
+ *
+ * @module utils/model_registry
+ */
+
+import { get_files } from './get_files.js';
+import { get_pipeline_files } from './get_pipeline_files.js';
+import { get_model_files } from './get_model_files.js';
+import { get_tokenizer_files } from './get_tokenizer_files.js';
+import { get_processor_files } from './get_processor_files.js';
+import { is_cached, is_cached_files, is_pipeline_cached, is_pipeline_cached_files } from './is_cached.js';
+import { get_file_metadata } from './get_file_metadata.js';
+import { clear_cache, clear_pipeline_cache } from './clear_cache.js';
+
+/**
+ * Static class for cache and file management operations.
+ * @hideconstructor
+ */
+export class ModelRegistry {
+    /**
+     * Get all files (model, tokenizer, processor) needed for a model.
+     *
+     * @param {string} modelId - The model id (e.g., "onnx-community/bert-base-uncased-ONNX")
+     * @param {Object} [options] - Optional parameters
+     * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix)
+     * @param {boolean} [options.include_tokenizer=true] - Whether to check for tokenizer files
+     * @param {boolean} [options.include_processor=true] - Whether to check for processor files
+     * @returns {Promise<string[]>} Array of file paths
+     *
+     * @example
+     * const files = await ModelRegistry.get_files('onnx-community/gpt2-ONNX');
+     * console.log(files); // ['config.json', 'tokenizer.json', 'onnx/model_q4.onnx', ...]
+     */
+    static async get_files(modelId, options = {}) {
+        return get_files(modelId, options);
+    }
+
+    /**
+     * Get all files needed for a specific pipeline task.
+     * Automatically determines which components are needed based on the task.
+     *
+     * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal")
+     * @param {string} modelId - The model id (e.g., "onnx-community/bert-base-uncased-ONNX")
+     * @param {Object} [options] - Optional parameters
+     * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix)
+     * @returns {Promise<string[]>} Array of file paths
+     *
+     * @example
+     * const files = await ModelRegistry.get_pipeline_files('text-generation', 'onnx-community/gpt2-ONNX');
+     * console.log(files); // ['config.json', 'tokenizer.json', 'onnx/model_q4.onnx', ...]
+     */
+    static async get_pipeline_files(task, modelId, options = {}) {
+        return get_pipeline_files(task, modelId, options);
+    }
+
+    /**
+     * Get model files needed for a specific model.
+     *
+     * @param {string} modelId - The model id
+     * @param {Object} [options] - Optional parameters
+     * @param {import('../../configs.js').PretrainedConfig} [options.config=null] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @param {string} [options.model_file_name=null] - Override the model file name (excluding .onnx suffix)
+     * @returns {Promise<string[]>} Array of model file paths
+     *
+     * @example
+     * const files = await ModelRegistry.get_model_files('onnx-community/bert-base-uncased-ONNX');
+     * console.log(files); // ['config.json', 'onnx/model_q4.onnx', 'generation_config.json']
+     */
+    static async get_model_files(modelId, options = {}) {
+        return get_model_files(modelId, options);
+    }
+
+    /**
+     * Get tokenizer files needed for a specific model.
+     *
+     * @param {string} modelId - The model id
+     * @returns {Promise<string[]>} Array of tokenizer file paths
+     *
+     * @example
+     * const files = await ModelRegistry.get_tokenizer_files('onnx-community/gpt2-ONNX');
+     * console.log(files); // ['tokenizer.json', 'tokenizer_config.json']
+     */
+    static async get_tokenizer_files(modelId) {
+        return get_tokenizer_files(modelId);
+    }
+
+    /**
+     * Get processor files needed for a specific model.
+     *
+     * @param {string} modelId - The model id
+     * @returns {Promise<string[]>} Array of processor file paths
+     *
+     * @example
+     * const files = await ModelRegistry.get_processor_files('onnx-community/vit-base-patch16-224-ONNX');
+     * console.log(files); // ['preprocessor_config.json']
+     */
+    static async get_processor_files(modelId) {
+        return get_processor_files(modelId);
+    }
+
+    /**
+     * Quickly checks if a model is fully cached by verifying `config.json` is present,
+     * then confirming all required files are cached.
+     * Returns a plain boolean — use `is_cached_files` if you need per-file detail.
+     *
+     * @param {string} modelId - The model id
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @returns {Promise<boolean>} Whether all required files are cached
+     *
+     * @example
+     * const cached = await ModelRegistry.is_cached('onnx-community/bert-base-uncased-ONNX');
+     * console.log(cached); // true or false
+     */
+    static async is_cached(modelId, options = {}) {
+        return is_cached(modelId, options);
+    }
+
+    /**
+     * Checks if all files for a given model are already cached, with per-file detail.
+     * Automatically determines which files are needed using get_files().
+     *
+     * @param {string} modelId - The model id
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @returns {Promise<import('./is_cached.js').CacheCheckResult>} Object with allCached boolean and files array with cache status
+     *
+     * @example
+     * const status = await ModelRegistry.is_cached_files('onnx-community/bert-base-uncased-ONNX');
+     * console.log(status.allCached); // true or false
+     * console.log(status.files); // [{ file: 'config.json', cached: true }, ...]
+     */
+    static async is_cached_files(modelId, options = {}) {
+        return is_cached_files(modelId, options);
+    }
+
+    /**
+     * Quickly checks if all files for a specific pipeline task are cached by verifying
+     * `config.json` is present, then confirming all required files are cached.
+     * Returns a plain boolean — use `is_pipeline_cached_files` if you need per-file detail.
+     *
+     * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal")
+     * @param {string} modelId - The model id
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @returns {Promise<boolean>} Whether all required files are cached
+     *
+     * @example
+     * const cached = await ModelRegistry.is_pipeline_cached('text-generation', 'onnx-community/gpt2-ONNX');
+     * console.log(cached); // true or false
+     */
+    static async is_pipeline_cached(task, modelId, options = {}) {
+        return is_pipeline_cached(task, modelId, options);
+    }
+
+    /**
+     * Checks if all files for a specific pipeline task are already cached, with per-file detail.
+     * Automatically determines which components are needed based on the task.
+     *
+     * @param {string} task - The pipeline task (e.g., "text-generation", "background-removal")
+     * @param {string} modelId - The model id
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] - Override device
+     * @returns {Promise<import('./is_cached.js').CacheCheckResult>} Object with allCached boolean and files array with cache status
+     *
+     * @example
+     * const status = await ModelRegistry.is_pipeline_cached_files('text-generation', 'onnx-community/gpt2-ONNX');
+     * console.log(status.allCached); // true or false
+     * console.log(status.files); // [{ file: 'config.json', cached: true }, ...]
+     */
+    static async is_pipeline_cached_files(task, modelId, options = {}) {
+        return is_pipeline_cached_files(task, modelId, options);
+    }
+
+    /**
+     * Get metadata for a specific file without downloading it.
+     *
+     * @param {string} path_or_repo_id - Model id or path
+     * @param {string} filename - The file name
+     * @param {import('../hub.js').PretrainedOptions} [options] - Optional parameters
+     * @returns {Promise<{exists: boolean, size?: number, contentType?: string, fromCache?: boolean}>} File metadata
+     *
+     * @example
+     * const metadata = await ModelRegistry.get_file_metadata('onnx-community/gpt2-ONNX', 'config.json');
+     * console.log(metadata.exists, metadata.size); // true, 665
+     */
+    static async get_file_metadata(path_or_repo_id, filename, options = {}) {
+        return get_file_metadata(path_or_repo_id, filename, options);
+    }
+
+    /**
+     * Clears all cached files for a given model.
+     * Automatically determines which files are needed and removes them from the cache.
+     *
+     * @param {string} modelId - The model id (e.g., "onnx-community/gpt2-ONNX")
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device] - Override device
+     * @param {boolean} [options.include_tokenizer=true] - Whether to clear tokenizer files
+     * @param {boolean} [options.include_processor=true] - Whether to clear processor files
+     * @returns {Promise<import('./clear_cache.js').CacheClearResult>} Object with deletion statistics and file status
+     *
+     * @example
+     * const result = await ModelRegistry.clear_cache('onnx-community/bert-base-uncased-ONNX');
+     * console.log(`Deleted ${result.filesDeleted} of ${result.filesCached} cached files`);
+     */
+    static async clear_cache(modelId, options = {}) {
+        return clear_cache(modelId, options);
+    }
+
+    /**
+     * Clears all cached files for a specific pipeline task.
+     * Automatically determines which components are needed based on the task.
+     *
+     * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification")
+     * @param {string} modelId - The model id (e.g., "onnx-community/gpt2-ONNX")
+     * @param {Object} [options] - Optional parameters
+     * @param {string} [options.cache_dir] - Custom cache directory
+     * @param {string} [options.revision] - Model revision (default: 'main')
+     * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+     * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype] - Override dtype
+     * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device] - Override device
+     * @returns {Promise<import('./clear_cache.js').CacheClearResult>} Object with deletion statistics and file status
+     *
+     * @example
+     * const result = await ModelRegistry.clear_pipeline_cache('text-generation', 'onnx-community/gpt2-ONNX');
+     * console.log(`Deleted ${result.filesDeleted} of ${result.filesCached} cached files`);
+     */
+    static async clear_pipeline_cache(task, modelId, options = {}) {
+        return clear_pipeline_cache(task, modelId, options);
+    }
+}

package/src/utils/model_registry/clear_cache.js

@@ -0,0 +1,128 @@
+/**
+ * @file Cache clearing utilities for model files
+ *
+ * Provides functions to clear cached model files from the cache system.
+ */
+
+import { getCache } from '../cache.js';
+import { buildResourcePaths, checkCachedResource } from '../hub.js';
+import { get_files } from './get_files.js';
+import { get_pipeline_files } from './get_pipeline_files.js';
+
+/**
+ * @typedef {Object} FileClearStatus
+ * @property {string} file - The file path
+ * @property {boolean} deleted - Whether the file was successfully deleted
+ * @property {boolean} wasCached - Whether the file was cached before deletion
+ */
+
+/**
+ * @typedef {Object} CacheClearResult
+ * @property {number} filesDeleted - Number of files successfully deleted
+ * @property {number} filesCached - Number of files that were in cache
+ * @property {FileClearStatus[]} files - Array of files with their deletion status
+ */
+
+/**
+ * Internal helper to clear cached files.
+ *
+ *
+ * @private
+ * @param {string} modelId - The model id
+ * @param {string[]} files - List of file paths to clear
+ * @param {Object} options - Options including cache_dir
+ * @returns {Promise<CacheClearResult>}
+ */
+async function clear_files_from_cache(modelId, files, options = {}) {
+    const cache = await getCache(options?.cache_dir);
+
+    if (!cache) {
+        return {
+            filesDeleted: 0,
+            filesCached: 0,
+            files: files.map((filename) => ({ file: filename, deleted: false, wasCached: false })),
+        };
+    }
+
+    if (!cache.delete) {
+        throw new Error('Cache does not support delete operation');
+    }
+
+    const results = await Promise.all(
+        files.map(async (filename) => {
+            const { localPath, proposedCacheKey } = buildResourcePaths(modelId, filename, options, cache);
+
+            const cached = await checkCachedResource(cache, localPath, proposedCacheKey);
+            const wasCached = !!cached;
+
+            let deleted = false;
+            if (wasCached) {
+                // Try proposedCacheKey first (remote URL for browser Cache API, request path for FileCache),
+                // then fall back to localPath in case the entry was cached under the local key instead.
+                const deletedWithProposed = await cache.delete(proposedCacheKey);
+                const deletedWithLocal =
+                    !deletedWithProposed && proposedCacheKey !== localPath ? await cache.delete(localPath) : false;
+
+                deleted = deletedWithProposed || deletedWithLocal;
+            }
+
+            return { file: filename, deleted, wasCached };
+        }),
+    );
+
+    return {
+        filesDeleted: results.filter((r) => r.deleted).length,
+        filesCached: results.filter((r) => r.wasCached).length,
+        files: results,
+    };
+}
+
+/**
+ * Clears all cached files for a given model.
+ * Automatically determines which files are needed using get_files().
+ *
+ * @param {string} modelId - The model id (e.g., "Xenova/gpt2")
+ * @param {Object} [options] - Optional parameters
+ * @param {string} [options.cache_dir] - Custom cache directory
+ * @param {string} [options.revision] - Model revision (default: 'main')
+ * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+ * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype] - Override dtype
+ * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device] - Override device
+ * @param {boolean} [options.include_tokenizer=true] - Whether to clear tokenizer files
+ * @param {boolean} [options.include_processor=true] - Whether to clear processor files
+ * @returns {Promise<CacheClearResult>} Object with deletion statistics and file status
+ */
+export async function clear_cache(modelId, options = {}) {
+    if (!modelId) {
+        throw new Error('modelId is required');
+    }
+
+    const files = await get_files(modelId, options);
+    return await clear_files_from_cache(modelId, files, options);
+}
+
+/**
+ * Clears all cached files for a specific pipeline task.
+ * Automatically determines which components are needed based on the task.
+ *
+ * @param {string} task - The pipeline task (e.g., "text-generation", "image-classification")
+ * @param {string} modelId - The model id (e.g., "Xenova/gpt2")
+ * @param {Object} [options] - Optional parameters
+ * @param {string} [options.cache_dir] - Custom cache directory
+ * @param {string} [options.revision] - Model revision (default: 'main')
+ * @param {import('../../configs.js').PretrainedConfig} [options.config] - Pre-loaded config
+ * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype] - Override dtype
+ * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device] - Override device
+ * @returns {Promise<CacheClearResult>} Object with deletion statistics and file status
+ */
+export async function clear_pipeline_cache(task, modelId, options = {}) {
+    if (!task) {
+        throw new Error('task is required');
+    }
+    if (!modelId) {
+        throw new Error('modelId is required');
+    }
+
+    const files = await get_pipeline_files(task, modelId, options);
+    return await clear_files_from_cache(modelId, files, options);
+}

package/src/utils/model_registry/get_file_metadata.js

@@ -0,0 +1,149 @@
+/**
+ * @file File metadata utilities for cache-aware operations
+ */
+
+import { env } from '../../env.js';
+import { getCache } from '../cache.js';
+import { buildResourcePaths, checkCachedResource, getFetchHeaders, getFile } from '../hub.js';
+import { isValidUrl } from '../hub/utils.js';
+import { logger } from '../logger.js';
+
+/**
+ * @typedef {import('../hub.js').PretrainedOptions} PretrainedOptions
+ */
+
+/**
+ * Makes a Range request to get file metadata without downloading full content.
+ *
+ * We use a Range request (first byte only) instead of HEAD because:
+ * 1. Ensures we get the uncompressed file size, which is critical for accurate progress tracking
+ * 2. When compression is used, HEAD request content-length reflects compressed size
+ * 3. But during actual download, fetch API decompresses transparently and reports uncompressed bytes
+ * 4. This mismatch causes progress bar inconsistencies when tracking multiple files
+ * 5. Range requests typically aren't compressed, and content-range header shows true uncompressed size
+ *
+ * @param {URL|string} urlOrPath The URL/path of the file.
+ * @returns {Promise<Response|null>} A promise that resolves to a Response object or null if not supported.
+ * @private
+ */
+async function fetch_file_head(urlOrPath) {
+    // Range requests only make sense for HTTP URLs
+    if (!isValidUrl(urlOrPath, ['http:', 'https:'])) {
+        return null;
+    }
+
+    const headers = getFetchHeaders(urlOrPath);
+    headers.set('Range', 'bytes=0-0');
+    return env.fetch(urlOrPath, { method: 'GET', headers });
+}
+
+/**
+ * Gets file metadata (size, content-type, etc.) without downloading the full content.
+ * Uses Range requests for remote files to be efficient.
+ * Can also be used as a lightweight file existence check by checking the `.exists` property.
+ *
+ * @param {string} path_or_repo_id This can be either:
+ * - a string, the *model id* of a model repo on huggingface.co.
+ * - a path to a *directory* potentially containing the file.
+ * @param {string} filename The name of the file to check.
+ * @param {PretrainedOptions} [options] An object containing optional parameters.
+ * @returns {Promise<{exists: boolean, size?: number, contentType?: string, fromCache?: boolean}>} A Promise that resolves to file metadata.
+ */
+export async function get_file_metadata(path_or_repo_id, filename, options = {}) {
+    /** @type {import('../cache.js').CacheInterface | null} */
+    const cache = await getCache(options?.cache_dir);
+    const { localPath, remoteURL, proposedCacheKey, validModelId } = buildResourcePaths(
+        path_or_repo_id,
+        filename,
+        options,
+        cache,
+    );
+
+    // Check cache first - if cached, we can get metadata from the cached response
+    const cachedResponse = await checkCachedResource(cache, localPath, proposedCacheKey);
+    if (cachedResponse !== undefined && typeof cachedResponse !== 'string') {
+        const size = cachedResponse.headers.get('content-length');
+        const contentType = cachedResponse.headers.get('content-type');
+        return {
+            exists: true,
+            size: size ? parseInt(size, 10) : undefined,
+            contentType: contentType || undefined,
+            fromCache: true,
+        };
+    }
+
+    // Check local file system
+    if (env.allowLocalModels) {
+        const isURL = isValidUrl(localPath, ['http:', 'https:']);
+        if (!isURL) {
+            try {
+                const response = await getFile(localPath);
+                if (typeof response !== 'string' && response.status !== 404) {
+                    const size = response.headers.get('content-length');
+                    const contentType = response.headers.get('content-type');
+                    return {
+                        exists: true,
+                        size: size ? parseInt(size, 10) : undefined,
+                        contentType: contentType || undefined,
+                        fromCache: false,
+                    };
+                }
+            } catch (e) {
+                // File doesn't exist locally, continue to remote check
+            }
+        }
+    }
+
+    // Check remote if allowed - use Range request for efficiency
+    if (env.allowRemoteModels && !options.local_files_only && validModelId) {
+        try {
+            // Make a Range request to get metadata without downloading full content
+            const rangeResponse = await fetch_file_head(remoteURL);
+
+            if (rangeResponse && rangeResponse.status >= 200 && rangeResponse.status < 300) {
+                let size;
+                const contentType = rangeResponse.headers.get('content-type');
+
+                // If server supports Range requests, we get a 206 Partial Content response
+                // with a content-range header showing the total uncompressed file size
+                if (rangeResponse.status === 206) {
+                    const contentRange = rangeResponse.headers.get('content-range');
+                    if (contentRange) {
+                        // Format: "bytes 0-0/12345" where 12345 is the total uncompressed size
+                        const match = contentRange.match(/bytes \d+-\d+\/(\d+)/);
+                        if (match) {
+                            size = parseInt(match[1], 10);
+                        }
+                    }
+                } else if (rangeResponse.status === 200) {
+                    // Server doesn't support Range requests and returned the full file (200 OK)
+                    // Cancel the response body immediately to avoid downloading the entire file
+                    try {
+                        await rangeResponse.body?.cancel();
+                    } catch (cancelError) {
+                        // Ignore cancellation errors - body might already be consumed or not cancellable
+                    }
+                }
+
+                // Fallback to content-length if content-range is not available (200 OK response)
+                // This can happen if server doesn't support Range requests
+                if (size === undefined) {
+                    const contentLength = rangeResponse.headers.get('content-length');
+                    size = contentLength ? parseInt(contentLength, 10) : undefined;
+                }
+
+                return {
+                    exists: true,
+                    size,
+                    contentType: contentType || undefined,
+                    fromCache: false,
+                };
+            }
+        } catch (e) {
+            // Range request failed most likely because of a network error, timeout, etc.
+            logger.warn(`Unable to fetch file metadata for "${remoteURL}": ${e}`);
+        }
+    }
+
+    return { exists: false, fromCache: false };
+}

package/src/utils/model_registry/get_files.js

@@ -0,0 +1,42 @@
+import { get_tokenizer_files } from './get_tokenizer_files.js';
+import { get_model_files } from './get_model_files.js';
+import { get_processor_files } from './get_processor_files.js';
+
+/**
+ * Returns the list of files that will be loaded for a model based on its configuration.
+ * Automatically detects which files are needed (tokenizer, processor, model files).
+ *
+ * @param {string} modelId The model id (e.g., "Xenova/llama-2-7b")
+ * @param {Object} [options] Optional parameters
+ * @param {import('../../configs.js').PretrainedConfig} [options.config=null] Pre-loaded model config (optional, will be fetched if not provided)
+ * @param {import('../dtypes.js').DataType|Record<string, import('../dtypes.js').DataType>} [options.dtype=null] Override dtype (use this if passing dtype to pipeline)
+ * @param {import('../devices.js').DeviceType|Record<string, import('../devices.js').DeviceType>} [options.device=null] Override device (use this if passing device to pipeline)
+ * @param {string|null} [options.model_file_name=null|null] Override the model file name (excluding .onnx suffix)
+ * @param {boolean} [options.include_tokenizer=true] Whether to check for tokenizer files (set to false for vision-only models)
+ * @param {boolean} [options.include_processor=true] Whether to check for processor files
+ * @returns {Promise<string[]>} Array of file paths that will be loaded
+ */
+export async function get_files(
+    modelId,
+    {
+        config = null,
+        dtype = null,
+        device = null,
+        model_file_name = null,
+        include_tokenizer = true,
+        include_processor = true,
+    } = {},
+) {
+    const files = await get_model_files(modelId, { config, dtype, device, model_file_name });
+
+    if (include_tokenizer) {
+        const tokenizerFiles = await get_tokenizer_files(modelId);
+        files.push(...tokenizerFiles);
+    }
+    if (include_processor) {
+        const processorFiles = await get_processor_files(modelId);
+        files.push(...processorFiles);
+    }
+
+    return files;
+}