@huggingface/transformers 3.0.2 → 3.1.0

package/src/models/convnext/image_processing_convnext.js

@@ -0,0 +1,45 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class ConvNextImageProcessor extends ImageProcessor {
+    constructor(config) {
+        super(config);
+
+        /**
+         * Percentage of the image to crop. Only has an effect if this.size < 384.
+         */
+        this.crop_pct = this.config.crop_pct ?? (224 / 256);
+    }
+
+    async resize(image) {
+        const shortest_edge = this.size?.shortest_edge;
+        if (shortest_edge === undefined) {
+            throw new Error(`Size dictionary must contain 'shortest_edge' key.`);
+        }
+
+        if (shortest_edge < 384) {
+            // maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
+            const resize_shortest_edge = Math.floor(shortest_edge / this.crop_pct);
+
+            const [newWidth, newHeight] = this.get_resize_output_image_size(image, {
+                shortest_edge: resize_shortest_edge,
+            });
+
+            image = await image.resize(newWidth, newHeight, {
+                resample: this.resample,
+            });
+
+            // then crop to (shortest_edge, shortest_edge)
+            image = await image.center_crop(shortest_edge, shortest_edge);
+        } else {
+            // warping (no cropping) when evaluated at 384 or larger
+            image = await image.resize(shortest_edge, shortest_edge, {
+                resample: this.resample,
+            });
+        }
+
+        return image;
+    }
+}
+export class ConvNextFeatureExtractor extends ConvNextImageProcessor { }

package/src/models/deit/image_processing_deit.js

@@ -0,0 +1,6 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class DeiTImageProcessor extends ImageProcessor { }
+export class DeiTFeatureExtractor extends DeiTImageProcessor { }

package/src/models/detr/image_processing_detr.js

@@ -0,0 +1,52 @@
+import { 
+    ImageProcessor,
+    post_process_object_detection,
+    post_process_panoptic_segmentation,
+    post_process_instance_segmentation,
+} from "../../base/image_processors_utils.js";
+
+import { full } from '../../utils/tensor.js';
+
+
+/**
+ * @typedef {object} DetrFeatureExtractorResultProps
+ * @property {import('../../utils/tensor.js').Tensor} pixel_mask
+ * @typedef {import('../../base/image_processors_utils.js').ImageProcessorResult & DetrFeatureExtractorResultProps} DetrFeatureExtractorResult
+ */
+
+export class DetrImageProcessor extends ImageProcessor {
+    /**
+     * Calls the feature extraction process on an array of images, preprocesses
+     * each image, and concatenates the resulting features into a single Tensor.
+     * @param {import('../../utils/image.js').RawImage[]} images The image(s) to extract features from.
+     * @returns {Promise<DetrFeatureExtractorResult>} An object containing the concatenated pixel values of the preprocessed images.
+     */
+    async _call(images) {
+        const result = await super._call(images);
+
+        // TODO support differently-sized images, for now assume all images are the same size.
+        // TODO support different mask sizes (not just 64x64)
+        // Currently, just fill pixel mask with 1s
+        const maskSize = [result.pixel_values.dims[0], 64, 64];
+        const pixel_mask = full(maskSize, 1n);
+
+        return { ...result, pixel_mask };
+    }
+
+    /** @type {typeof post_process_object_detection} */
+    post_process_object_detection(...args) {
+        return post_process_object_detection(...args);
+    }
+
+    /** @type {typeof post_process_panoptic_segmentation} */
+    post_process_panoptic_segmentation(...args) {
+        return post_process_panoptic_segmentation(...args);
+    }
+
+    /** @type {typeof post_process_instance_segmentation} */
+    post_process_instance_segmentation(...args) {
+        return post_process_instance_segmentation(...args);
+    }
+}
+
+export class DetrFeatureExtractor extends DetrImageProcessor { } // NOTE: extends DetrImageProcessor  

package/src/models/donut/image_processing_donut.js

@@ -0,0 +1,31 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class DonutImageProcessor extends ImageProcessor {
+    pad_image(pixelData, imgDims, padSize, options = {}) {
+        const [imageHeight, imageWidth, imageChannels] = imgDims;
+
+        let image_mean = this.image_mean;
+        if (!Array.isArray(this.image_mean)) {
+            image_mean = new Array(imageChannels).fill(image_mean);
+        }
+
+        let image_std = this.image_std;
+        if (!Array.isArray(image_std)) {
+            image_std = new Array(imageChannels).fill(image_mean);
+        }
+
+        const constant_values = image_mean.map((x, i) => - x / image_std[i]);
+
+        return super.pad_image(pixelData, imgDims, padSize, {
+            center: true,
+
+            // Since normalization is done after padding, we need to use certain constant values to ensure the same behaviour is observed.
+            // For more information, see https://github.com/huggingface/transformers/blob/main/src/transformers/models/donut/image_processing_donut.py#L433-L451
+            constant_values,
+            ...options,
+        });
+    }
+}
+export class DonutFeatureExtractor extends DonutImageProcessor { }

package/src/models/dpt/image_processing_dpt.js

@@ -0,0 +1,6 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class DPTImageProcessor extends ImageProcessor { }
+export class DPTFeatureExtractor extends DPTImageProcessor { } // NOTE: extends DPTImageProcessor

package/src/models/efficientnet/image_processing_efficientnet.js

@@ -0,0 +1,13 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class EfficientNetImageProcessor extends ImageProcessor {
+    constructor(config) {
+        super(config);
+        this.include_top = this.config.include_top ?? true;
+        if (this.include_top) {
+            this.image_std = this.image_std.map(x => x * x);
+        }
+    }
+}

package/src/models/feature_extractors.js

@@ -0,0 +1,12 @@
+
+export * from './audio_spectrogram_transformer/feature_extraction_audio_spectrogram_transformer.js';
+export * from './clap/feature_extraction_clap.js';
+export * from './pyannote/feature_extraction_pyannote.js';
+export * from './seamless_m4t/feature_extraction_seamless_m4t.js';
+export * from './speecht5/feature_extraction_speecht5.js';
+export * from './wav2vec2/feature_extraction_wav2vec2.js';
+export * from './wespeaker/feature_extraction_wespeaker.js';
+export * from './whisper/feature_extraction_whisper.js';
+
+// For legacy support, ImageFeatureExtractor is an alias for ImageProcessor
+export { ImageProcessor as ImageFeatureExtractor } from "../base/image_processors_utils.js";

package/src/models/florence2/processing_florence2.js

@@ -0,0 +1,128 @@
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+
+export class Florence2Processor extends Processor {
+    static tokenizer_class = AutoTokenizer
+    static image_processor_class = AutoImageProcessor
+
+    constructor(config, components) {
+        super(config, components);
+
+        const {
+            tasks_answer_post_processing_type,
+            task_prompts_without_inputs,
+            task_prompts_with_input,
+        } = this.image_processor.config;
+
+        /** @type {Map<string, string>} */
+        this.tasks_answer_post_processing_type = new Map(Object.entries(tasks_answer_post_processing_type ?? {}));
+
+        /** @type {Map<string, string>} */
+        this.task_prompts_without_inputs = new Map(Object.entries(task_prompts_without_inputs ?? {}));
+
+        /** @type {Map<string, string>} */
+        this.task_prompts_with_input = new Map(Object.entries(task_prompts_with_input ?? {}));
+
+        this.regexes = {
+            quad_boxes: /(.+?)<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>/gm,
+            bboxes: /([^<]+)?<loc_(\d+)><loc_(\d+)><loc_(\d+)><loc_(\d+)>/gm,
+        }
+        this.size_per_bin = 1000;
+    }
+
+    /**
+     * Helper function to construct prompts from input texts
+     * @param {string|string[]} text
+     * @returns {string[]}
+     */
+    construct_prompts(text) {
+        if (typeof text === 'string') {
+            text = [text];
+        }
+
+        const prompts = [];
+        for (const t of text) {
+            // 1. fixed task prompts without additional inputs
+            if (this.task_prompts_without_inputs.has(t)) {
+                prompts.push(this.task_prompts_without_inputs.get(t));
+            }
+            // 2. task prompts with additional inputs 
+            else {
+                for (const [task, prompt] of this.task_prompts_with_input) {
+                    if (t.includes(task)) {
+                        prompts.push(prompt.replaceAll('{input}', t).replaceAll(task, ''));
+                        break;
+                    }
+                }
+
+                // 3. default prompt
+                if (prompts.length !== text.length) {
+                    prompts.push(t);
+                }
+            }
+        }
+        return prompts;
+    }
+
+    /**
+     * Post-process the output of the model to each of the task outputs.
+     * @param {string} text The text to post-process.
+     * @param {string} task The task to post-process the text for.
+     * @param {[number, number]} image_size The size of the image. height x width.
+     */
+    post_process_generation(text, task, image_size) {
+        const task_answer_post_processing_type = this.tasks_answer_post_processing_type.get(task) ?? 'pure_text';
+
+        // remove the special tokens
+        text = text.replaceAll('<s>', '').replaceAll('</s>', '');
+
+        let final_answer;
+        switch (task_answer_post_processing_type) {
+            case 'pure_text':
+                final_answer = text;
+                break;
+
+            case 'description_with_bboxes':
+            case 'bboxes':
+            case 'phrase_grounding':
+            case 'ocr':
+                const key = task_answer_post_processing_type === 'ocr' ? 'quad_boxes' : 'bboxes';
+                const matches = text.matchAll(this.regexes[key]);
+                const labels = [];
+                const items = [];
+                for (const [_, label, ...locations] of matches) {
+                    // Push new label, or duplicate the last label
+                    labels.push(label ? label.trim() : labels.at(-1) ?? '');
+                    items.push(locations.map((x, i) =>
+                        // NOTE: Add 0.5 to use the center position of the bin as the coordinate.
+                        (Number(x) + 0.5) / this.size_per_bin * image_size[i % 2])
+                    );
+                }
+                final_answer = { labels, [key]: items };
+                break;
+
+            default:
+                throw new Error(`Task "${task}" (of type "${task_answer_post_processing_type}") not yet implemented.`);
+        }
+
+        return { [task]: final_answer }
+    }
+
+    // NOTE: images and text are switched from the python version
+    // `images` is required, `text` is optional
+    async _call(images, text=null, kwargs = {}) {
+
+        if (!images && !text){
+            throw new Error('Either text or images must be provided');
+        }
+
+        const image_inputs = await this.image_processor(images, kwargs);
+        const text_inputs = text ? this.tokenizer(text, kwargs) : {};
+
+        return {
+            ...image_inputs,
+            ...text_inputs,
+        }
+    }
+}

package/src/models/glpn/image_processing_glpn.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class GLPNFeatureExtractor extends ImageProcessor { }

package/src/models/image_processors.js

@@ -0,0 +1,36 @@
+
+export * from './beit/image_processing_beit.js'
+export * from './bit/image_processing_bit.js'
+export * from './chinese_clip/image_processing_chinese_clip.js'
+export * from './clip/image_processing_clip.js'
+export * from './convnext/image_processing_convnext.js'
+export * from './deit/image_processing_deit.js'
+export * from './detr/image_processing_detr.js'
+export * from './donut/image_processing_donut.js'
+export * from './dpt/image_processing_dpt.js'
+export * from './efficientnet/image_processing_efficientnet.js'
+export * from './glpn/image_processing_glpn.js'
+export * from './janus/image_processing_janus.js'
+export * from './jina_clip/image_processing_jina_clip.js'
+export * from './llava_onevision/image_processing_llava_onevision.js'
+export * from './mask2former/image_processing_mask2former.js'
+export * from './maskformer/image_processing_maskformer.js'
+export * from './mobilenet_v1/image_processing_mobilenet_v1.js'
+export * from './mobilenet_v2/image_processing_mobilenet_v2.js'
+export * from './mobilenet_v3/image_processing_mobilenet_v3.js'
+export * from './mobilenet_v4/image_processing_mobilenet_v4.js'
+export * from './mobilevit/image_processing_mobilevit.js'
+export * from './nougat/image_processing_nougat.js'
+export * from './owlv2/image_processing_owlv2.js'
+export * from './owlvit/image_processing_owlvit.js'
+export * from './pvt/image_processing_pvt.js'
+export * from './qwen2_vl/image_processing_qwen2_vl.js'
+export * from './rt_detr/image_processing_rt_detr.js'
+export * from './sam/image_processing_sam.js'
+export * from './segformer/image_processing_segformer.js'
+export * from './siglip/image_processing_siglip.js'
+export * from './swin2sr/image_processing_swin2sr.js'
+export * from './vit/image_processing_vit.js'
+export * from './vitmatte/image_processing_vitmatte.js'
+export * from './vitpose/image_processing_vitpose.js'
+export * from './yolos/image_processing_yolos.js'

package/src/models/janus/image_processing_janus.js

@@ -0,0 +1,26 @@
+
+import {
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class VLMImageProcessor extends ImageProcessor {
+    constructor(config) {
+        super({
+            do_pad: true,
+            pad_size: {
+                width: config.image_size,
+                height: config.image_size,
+            },
+            ...config,
+        });
+        this.constant_values = this.config.background_color.map(x => x * this.rescale_factor)
+    }
+
+    pad_image(pixelData, imgDims, padSize, options) {
+        return super.pad_image(pixelData, imgDims, padSize, {
+            constant_values: this.constant_values,
+            center: true,
+            ...options,
+        });
+    }
+}

package/src/models/janus/processing_janus.js

@@ -0,0 +1,123 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+import { mergeArrays } from "../../utils/core.js";
+import { Tensor } from "../../utils/tensor.js";
+import { RawImage } from "../../utils/image.js";
+
+export class VLChatProcessor extends Processor {
+    static image_processor_class = AutoImageProcessor
+    static tokenizer_class = AutoTokenizer
+    static uses_processor_config = true;
+
+    constructor(config, components) {
+        super(config, components);
+
+        this.image_tag = this.config.image_tag;
+        this.image_start_tag = this.config.image_start_tag;
+        this.image_end_tag = this.config.image_end_tag;
+        this.num_image_tokens = this.config.num_image_tokens;
+    }
+
+    /**
+     * @typedef {Object} MultimodalMessageProperties Additional properties for multimodal messages.
+     * @property {(RawImage | string | URL)[]} [images] The images in the message.
+     * @typedef {(import('../../tokenizers.js').Message & MultimodalMessageProperties)[]} MultimodalConversation The conversation possibly containing multimodal inputs.
+     */
+
+    /**
+     * @typedef {Object} VLCChatProcessorResult The processed input.
+     * @property {Tensor} input_ids The input IDs.
+     * @property {Tensor} attention_mask The attention mask.
+     * @property {Tensor} images_seq_mask The image sequence mask.
+     * @property {Tensor} images_emb_mask The image embedding mask.
+     */
+
+    /**
+     * @param {MultimodalConversation} conversation The chat messages to process.
+     * @param {Object} options Additional options for processing.
+     * @param {RawImage|RawImage[]} [options.images] The images to process, if not set in the conversation.
+     * @param {string} [options.chat_template="default"] The chat template to use.
+     * @returns {Promise<VLCChatProcessorResult | VLCChatProcessorResult & import('../../base/image_processors_utils.js').ImageProcessorResult>} The processed input.
+     */
+    async _call(conversation, {
+        images = null,
+        chat_template = "default",
+    }={}) {
+        if (!images) {
+            images = await Promise.all(
+                conversation
+                    .filter((msg) => msg.images)
+                    .flatMap((msg) => msg.images)
+                    .map((img) => RawImage.read(img))
+            );
+        } else if (!Array.isArray(images)) {
+            images = [images];
+        }
+
+        const tokenizer = this.tokenizer;
+        const result = tokenizer.apply_chat_template(conversation, {
+            tokenize: false,
+            add_generation_prompt: true,
+            chat_template,
+        });
+
+        const encode = (text) => tokenizer.encode(text, { add_special_tokens: false });
+        const parts = (/** @type {string} */(result))
+            .split(this.image_tag);
+        const num_images = parts.length - 1;
+        if (images.length !== num_images) {
+            throw new Error(`Number of images provided (${images.length}) does not match number of "${this.image_tag}" image tags (${num_images})`);
+        }
+
+        const [
+            image_placeholder_tag_id,
+            image_start_tag_id,
+            image_end_tag_id,
+        ] = tokenizer.model.convert_tokens_to_ids([
+            this.image_tag,
+            this.image_start_tag,
+            this.image_end_tag,
+        ]);
+
+        let input_ids = encode(parts[0]);
+        let images_seq_mask = new Array(input_ids.length).fill(false);
+        for (let i = 1; i < parts.length; ++i) {
+            const placeholder_image_tokens = new Array(this.num_image_tokens).fill(image_placeholder_tag_id);
+            const tokens = encode(parts[i]);
+            input_ids = mergeArrays(
+                input_ids,
+                [image_start_tag_id], placeholder_image_tokens, [image_end_tag_id],
+                tokens,
+            );
+            const image_mask = new Array(this.num_image_tokens).fill(true);
+            images_seq_mask = mergeArrays(
+                images_seq_mask,
+                [false], image_mask, [false],
+                new Array(tokens.length).fill(false),
+            );
+        }
+
+        const dims = [1, input_ids.length];
+        const final = {
+            input_ids: new Tensor('int64', input_ids, dims),
+            attention_mask: new Tensor('int64', new Array(input_ids.length).fill(1), dims),
+            images_seq_mask: new Tensor('bool', images_seq_mask, dims),
+            images_emb_mask: new Tensor(
+                'bool',
+                new Array(num_images * this.num_image_tokens).fill(true),
+                [1, num_images, this.num_image_tokens],
+            ),
+        }
+
+        if (images && images.length > 0) {
+            const image_inputs = await this.image_processor(images);
+            // Set the batch_size dimension to 1
+            image_inputs.pixel_values.unsqueeze_(0);
+            return { ...final, ...image_inputs };
+        }
+
+        return final;
+    }
+}

package/src/models/jina_clip/image_processing_jina_clip.js

@@ -0,0 +1,26 @@
+import {
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class JinaCLIPImageProcessor extends ImageProcessor {
+    constructor(config) {
+        // JinaCLIPImageProcessor uses a custom preprocessor_config.json, so we configure it here
+        const { resize_mode, fill_color, interpolation, size, ...other } = config;
+
+        const new_size = resize_mode === 'squash'
+            ? { width: size, height: size }
+            : resize_mode === 'shortest'
+                ? { shortest_edge: size }
+                : { longest_edge: size };
+
+        const resample = interpolation === 'bicubic' ? 3 : 2;
+        super({
+            ...other,
+            size: new_size,
+            resample,
+            do_center_crop: true,
+            crop_size: size,
+            do_normalize: true,
+        });
+    }
+}

package/src/models/jina_clip/processing_jina_clip.js

@@ -0,0 +1,24 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+
+export class JinaCLIPProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer
+    static image_processor_class = AutoImageProcessor
+
+    async _call(text=null, images=null, kwargs = {}) {
+
+        if (!text && !images){
+            throw new Error('Either text or images must be provided');
+        }
+
+        const text_inputs = text ? this.tokenizer(text, kwargs) : {};
+        const image_inputs = images ? await this.image_processor(images, kwargs) : {};
+
+        return {
+            ...text_inputs,
+            ...image_inputs,
+        }
+    }
+}

package/src/models/llava_onevision/image_processing_llava_onevision.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class LlavaOnevisionImageProcessor extends ImageProcessor {}

package/src/models/mask2former/image_processing_mask2former.js

@@ -0,0 +1,5 @@
+
+import { MaskFormerImageProcessor } from "../maskformer/image_processing_maskformer.js";
+
+// NOTE: extends MaskFormerImageProcessor
+export class Mask2FormerImageProcessor extends MaskFormerImageProcessor { }

package/src/models/maskformer/image_processing_maskformer.js

@@ -0,0 +1,18 @@
+import { 
+    ImageProcessor,
+    post_process_panoptic_segmentation,
+    post_process_instance_segmentation,
+} from "../../base/image_processors_utils.js";
+
+export class MaskFormerImageProcessor extends ImageProcessor {
+
+    /** @type {typeof post_process_panoptic_segmentation} */
+    post_process_panoptic_segmentation(...args) {
+        return post_process_panoptic_segmentation(...args);
+    }
+    /** @type {typeof post_process_instance_segmentation} */
+    post_process_instance_segmentation(...args) {
+        return post_process_instance_segmentation(...args);
+    }
+}
+export class MaskFormerFeatureExtractor extends MaskFormerImageProcessor { }