@huggingface/transformers 3.0.2 → 3.1.1

package/src/models/idefics3/image_processing_idefics3.js

@@ -0,0 +1,219 @@
+
+
+import {
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+import { cat, full, interpolate_4d, stack } from "../../utils/tensor.js";
+
+export class Idefics3ImageProcessor extends ImageProcessor {
+    constructor(config) {
+        super(config);
+
+        this.do_image_splitting = config.do_image_splitting ?? true;
+        this.max_image_size = config.max_image_size;
+    }
+
+    /**
+     * @typedef {import('../../utils/image.js').RawImage} RawImage
+     * @typedef {import('../../utils/tensor.js').Tensor} Tensor
+     */
+
+    /**
+     * Calculate size to resize images to, to be multiples of `vision_encoder_max_size` while preserving the aspect ratio.
+     * @param {Tensor} pixel_values Tensor of the image to resize.
+     * @param {number} vision_encoder_max_size Maximum size of the output image. If the image is larger than this size,
+     * it will be split into patches of this size, and the original image will be concatenated with the patches, resized to max_size.
+     */
+    get_resize_for_vision_encoder(pixel_values, vision_encoder_max_size) {
+        let [height, width] = pixel_values.dims.slice(-2);
+
+        const aspect_ratio = width / height;
+        if (width >= height) {
+            width = Math.ceil(width / vision_encoder_max_size) * vision_encoder_max_size;
+            height = Math.floor(width / aspect_ratio);
+            height = Math.ceil(height / vision_encoder_max_size) * vision_encoder_max_size;
+        } else {
+            height = Math.ceil(height / vision_encoder_max_size) * vision_encoder_max_size;
+            width = Math.floor(height * aspect_ratio);
+            width = Math.ceil(width / vision_encoder_max_size) * vision_encoder_max_size;
+        }
+        return { height, width };
+    }
+
+    /** @param {RawImage|RawImage[]|RawImage[][]} images */
+    async _call(images, {
+        do_image_splitting = null,
+        return_row_col_info = false,
+    } = {}) {
+
+        /** @type {RawImage[][]} */
+        let batched_2d_images;
+        if (!Array.isArray(images)) {
+            batched_2d_images = [[images]];
+        } else {
+            if (images.length === 0 || !images[0]) {
+                throw new Error("No images provided.");
+            }
+            if (!Array.isArray(images[0])) {
+                batched_2d_images = [/** @type {RawImage[]} */(images)];
+            } else {
+                batched_2d_images = /** @type {RawImage[][]} */(images);
+            }
+        }
+
+        // List of tensors, each with shape [patches, channels, height, width]
+        let all_pixel_values = [];
+        let images_list_rows = [];
+        let images_list_cols = [];
+
+        const original_sizes = [];
+        const reshaped_input_sizes = [];
+        for (const image_batch of batched_2d_images) {
+
+            let images_list = await Promise.all(image_batch.map(x => this.preprocess(x)));
+
+            // Original sizes of images
+            original_sizes.push(...images_list.map(x => x.original_size));
+
+            // Reshaped sizes of images, before padding or cropping
+            reshaped_input_sizes.push(...images_list.map(x => x.reshaped_input_size));
+
+            // Convert images to 4D tensors for easier processing
+            images_list.forEach(x => x.pixel_values.unsqueeze_(0));
+
+            const { longest_edge } = this.max_image_size;
+
+            /** @type {Tensor[]} */
+            let images_tensor;
+            if (do_image_splitting ?? this.do_image_splitting) {
+                let image_rows = new Array(images_list.length);
+                let image_cols = new Array(images_list.length);
+
+                // We first resize both height and width of each image to the nearest max_image_size multiple, disregarding the aspect ratio
+                images_tensor = await Promise.all(images_list.map(async (x, i) => {
+                    const new_size = this.get_resize_for_vision_encoder(x.pixel_values, longest_edge);
+
+                    const resized = await interpolate_4d(x.pixel_values, {
+                        size: [new_size.height, new_size.width],
+                    });
+
+                    const { frames, num_splits_h, num_splits_w } = await this.split_image(resized, this.max_image_size);
+                    image_rows[i] = num_splits_h;
+                    image_cols[i] = num_splits_w;
+                    return cat(frames, 0);
+                }));
+
+                images_list_rows.push(image_rows);
+                images_list_cols.push(image_cols);
+
+            } else {
+                /** @type {[number, number]} */
+                const size = [longest_edge, longest_edge];
+                images_tensor = await Promise.all(
+                    images_list.map(x => interpolate_4d(x.pixel_values, { size }))
+                );
+
+                images_list_rows.push(new Array(images_list.length).fill(0));
+                images_list_cols.push(new Array(images_list.length).fill(0));
+            }
+
+            all_pixel_values.push(cat(images_tensor, 0));
+        }
+
+        const batch_size = all_pixel_values.length;
+        const [n, c, h, w] = all_pixel_values[0].dims;
+
+        // Stack pixel values
+        let pixel_values;
+        let pixel_attention_mask;
+        if (batch_size === 1) {
+            pixel_values = all_pixel_values[0].unsqueeze_(0);
+            pixel_attention_mask = full([batch_size, n, h, w], true);
+        } else {
+            // Add padding (if necessary) to images with less patches than the maximum number of patches
+            const max_num_patches = Math.max(...all_pixel_values.map(x => x.dims.at(0)));
+
+            pixel_attention_mask = full([batch_size, max_num_patches, h, w], true);
+            const pixel_attention_mask_data = pixel_attention_mask.data;
+            const pixel_attention_mask_stride = max_num_patches * h * w;
+            for (let i = 0; i < batch_size; ++i) {
+                const num_patches = all_pixel_values[i].dims[0];
+                if (num_patches < max_num_patches) {
+                    all_pixel_values[i] = cat([
+                        all_pixel_values[i],
+                        full([max_num_patches - num_patches, c, h, w], 0),
+                    ], 0);
+
+                    const start_offset = i * pixel_attention_mask_stride + num_patches * h * w;
+                    const end_offset = (i + 1) * pixel_attention_mask_stride;
+                    pixel_attention_mask_data.fill(false, start_offset, end_offset);
+                }
+            }
+            pixel_values = stack(all_pixel_values, 0);
+        }
+
+        return {
+            pixel_values,
+            pixel_attention_mask,
+
+            original_sizes,
+            reshaped_input_sizes,
+            ...(
+                return_row_col_info
+                    ? { rows: images_list_rows, cols: images_list_cols }
+                    : {}
+            ),
+        }
+    }
+
+    async split_image(pixel_values, { longest_edge }) {
+        const max_height = longest_edge;
+        const max_width = longest_edge;
+
+        const frames = [];
+
+        const [height, width] = pixel_values.dims.slice(-2);
+
+        let num_splits_h = 0, num_splits_w = 0;
+
+        if (height > max_height || width > max_width) {
+            // Calculate the number of splits
+            num_splits_h = Math.ceil(height / max_height);
+            num_splits_w = Math.ceil(width / max_width);
+
+            // Calculate the optimal width and height for the sub-images
+            const optimal_height = Math.ceil(height / num_splits_h);
+            const optimal_width = Math.ceil(width / num_splits_w);
+
+            // Iterate through each row and column
+            for (let r = 0; r < num_splits_h; r++) {
+                for (let c = 0; c < num_splits_w; c++) {
+                    // Calculate the starting point of the crop
+                    const start_x = c * optimal_width;
+                    const start_y = r * optimal_height;
+
+                    // Calculate the ending point of the crop
+                    const end_x = Math.min(start_x + optimal_width, width);
+                    const end_y = Math.min(start_y + optimal_height, height);
+
+                    // Crop the image
+                    frames.push(pixel_values.slice(null, null, [start_y, end_y], [start_x, end_x]));
+                }
+            }
+
+            // Resize the global image to match max dimensions for memory efficiency
+            const global_image_height = max_height;
+            const global_image_width = max_width;
+
+            if (height !== global_image_height || width !== global_image_width) {
+                pixel_values = await interpolate_4d(pixel_values, {
+                    size: [global_image_height, global_image_width],
+                })
+            }
+        }
+
+        frames.push(pixel_values);
+
+        return { frames, num_splits_h, num_splits_w };
+    }
+}

package/src/models/idefics3/processing_idefics3.js

@@ -0,0 +1,136 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+import { RawImage } from "../../utils/image.js";
+import { count } from "../../utils/core.js";
+
+/**
+ * Prompt with expanded image tokens for when the image is split into patches.
+ * @private
+ */
+function _prompt_split_image(image_seq_len, image_rows, image_cols, fake_token_around_image, image_token, global_img_token) {
+    let text_split_images = "";
+    for (let n_h = 0; n_h < image_rows; ++n_h) {
+        for (let n_w = 0; n_w < image_cols; ++n_w) {
+            text_split_images += (
+                fake_token_around_image +
+                `<row_${n_h + 1}_col_${n_w + 1}>` +
+                image_token.repeat(image_seq_len)
+            );
+        }
+        text_split_images += "\n";
+    }
+
+    text_split_images += (
+        `\n${fake_token_around_image}` +
+        `${global_img_token}` +
+        image_token.repeat(image_seq_len) +
+        `${fake_token_around_image}`
+    );
+    return text_split_images;
+}
+
+/**
+ * Prompt with expanded image tokens for a single image.
+ * @private
+ */
+function _prompt_single_image(image_seq_len, fake_token_around_image, image_token, global_img_token) {
+    return (
+        `${fake_token_around_image}` +
+        `${global_img_token}` +
+        image_token.repeat(image_seq_len) +
+        `${fake_token_around_image}`
+    );
+}
+
+function get_image_prompt_string(image_rows, image_cols, image_seq_len, fake_token_around_image, image_token, global_img_token) {
+    if (image_rows === 0 && image_cols === 0) {
+        return _prompt_single_image(
+            image_seq_len,
+            fake_token_around_image,
+            image_token,
+            global_img_token
+        );
+    }
+    return _prompt_split_image(
+        image_seq_len, image_rows, image_cols, fake_token_around_image, image_token, global_img_token
+    );
+}
+
+
+export class Idefics3Processor extends Processor {
+    static image_processor_class = AutoImageProcessor
+    static tokenizer_class = AutoTokenizer
+    static uses_processor_config = true;
+
+    fake_image_token = "<fake_token_around_image>";
+    image_token = "<image>";
+    global_img_token = "<global-img>";
+
+    /**
+     * 
+     * @param {string|string[]} text 
+     * @param {RawImage|RawImage[]|RawImage[][]} images  
+     * @returns {Promise<any>}
+     */
+    async _call(text, images = null, options = {}) {
+        options.return_row_col_info ??= true;
+
+        let image_inputs;
+
+        if (images) {
+            image_inputs = await this.image_processor(images, options);
+        }
+
+        // NOTE: We assume text is present
+        if (!Array.isArray(text)) {
+            text = [text];
+        }
+
+        const image_rows = image_inputs.rows ?? [new Array(text.length).fill(0)];
+        const image_cols = image_inputs.cols ?? [new Array(text.length).fill(0)];
+
+        const image_seq_len = this.config.image_seq_len;
+        const n_images_in_text = []
+        const prompt_strings = [];
+        for (let i = 0; i < text.length; ++i) {
+            const sample = text[i];
+            const sample_rows = image_rows[i];
+            const sample_cols = image_cols[i];
+
+            n_images_in_text.push(count(sample, this.image_token));
+
+            // Replace the image token with fake tokens around the expanded image token sequence of length `image_seq_len`
+            const image_prompt_strings = sample_rows.map(
+                (n_rows, j) => get_image_prompt_string(
+                    n_rows,
+                    sample_cols[j],
+                    image_seq_len,
+                    this.fake_image_token,
+                    this.image_token,
+                    this.global_img_token,
+                )
+            );
+
+            const split_sample = sample.split(this.image_token);
+            if (split_sample.length === 0) {
+                throw new Error("The image token should be present in the text.");
+            }
+
+            // Place in the image prompt strings where the image tokens are
+            let new_sample = split_sample[0];
+            for (let j = 0; j < image_prompt_strings.length; ++j) {
+                new_sample += image_prompt_strings[j] + split_sample[j + 1];
+            }
+            prompt_strings.push(new_sample);
+        }
+
+        const text_inputs = this.tokenizer(prompt_strings);
+
+        return {
+            ...text_inputs,
+            ...image_inputs,
+        }
+    }
+}

package/src/models/image_processors.js

@@ -0,0 +1,37 @@
+
+export * from './beit/image_processing_beit.js'
+export * from './bit/image_processing_bit.js'
+export * from './chinese_clip/image_processing_chinese_clip.js'
+export * from './clip/image_processing_clip.js'
+export * from './convnext/image_processing_convnext.js'
+export * from './deit/image_processing_deit.js'
+export * from './detr/image_processing_detr.js'
+export * from './donut/image_processing_donut.js'
+export * from './dpt/image_processing_dpt.js'
+export * from './efficientnet/image_processing_efficientnet.js'
+export * from './glpn/image_processing_glpn.js'
+export * from './idefics3/image_processing_idefics3.js'
+export * from './janus/image_processing_janus.js'
+export * from './jina_clip/image_processing_jina_clip.js'
+export * from './llava_onevision/image_processing_llava_onevision.js'
+export * from './mask2former/image_processing_mask2former.js'
+export * from './maskformer/image_processing_maskformer.js'
+export * from './mobilenet_v1/image_processing_mobilenet_v1.js'
+export * from './mobilenet_v2/image_processing_mobilenet_v2.js'
+export * from './mobilenet_v3/image_processing_mobilenet_v3.js'
+export * from './mobilenet_v4/image_processing_mobilenet_v4.js'
+export * from './mobilevit/image_processing_mobilevit.js'
+export * from './nougat/image_processing_nougat.js'
+export * from './owlv2/image_processing_owlv2.js'
+export * from './owlvit/image_processing_owlvit.js'
+export * from './pvt/image_processing_pvt.js'
+export * from './qwen2_vl/image_processing_qwen2_vl.js'
+export * from './rt_detr/image_processing_rt_detr.js'
+export * from './sam/image_processing_sam.js'
+export * from './segformer/image_processing_segformer.js'
+export * from './siglip/image_processing_siglip.js'
+export * from './swin2sr/image_processing_swin2sr.js'
+export * from './vit/image_processing_vit.js'
+export * from './vitmatte/image_processing_vitmatte.js'
+export * from './vitpose/image_processing_vitpose.js'
+export * from './yolos/image_processing_yolos.js'

package/src/models/janus/image_processing_janus.js

@@ -0,0 +1,26 @@
+
+import {
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class VLMImageProcessor extends ImageProcessor {
+    constructor(config) {
+        super({
+            do_pad: true,
+            pad_size: {
+                width: config.image_size,
+                height: config.image_size,
+            },
+            ...config,
+        });
+        this.constant_values = this.config.background_color.map(x => x * this.rescale_factor)
+    }
+
+    pad_image(pixelData, imgDims, padSize, options) {
+        return super.pad_image(pixelData, imgDims, padSize, {
+            constant_values: this.constant_values,
+            center: true,
+            ...options,
+        });
+    }
+}

package/src/models/janus/processing_janus.js

@@ -0,0 +1,123 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+import { mergeArrays } from "../../utils/core.js";
+import { Tensor } from "../../utils/tensor.js";
+import { RawImage } from "../../utils/image.js";
+
+export class VLChatProcessor extends Processor {
+    static image_processor_class = AutoImageProcessor
+    static tokenizer_class = AutoTokenizer
+    static uses_processor_config = true;
+
+    constructor(config, components) {
+        super(config, components);
+
+        this.image_tag = this.config.image_tag;
+        this.image_start_tag = this.config.image_start_tag;
+        this.image_end_tag = this.config.image_end_tag;
+        this.num_image_tokens = this.config.num_image_tokens;
+    }
+
+    /**
+     * @typedef {Object} MultimodalMessageProperties Additional properties for multimodal messages.
+     * @property {(RawImage | string | URL)[]} [images] The images in the message.
+     * @typedef {(import('../../tokenizers.js').Message & MultimodalMessageProperties)[]} MultimodalConversation The conversation possibly containing multimodal inputs.
+     */
+
+    /**
+     * @typedef {Object} VLCChatProcessorResult The processed input.
+     * @property {Tensor} input_ids The input IDs.
+     * @property {Tensor} attention_mask The attention mask.
+     * @property {Tensor} images_seq_mask The image sequence mask.
+     * @property {Tensor} images_emb_mask The image embedding mask.
+     */
+
+    /**
+     * @param {MultimodalConversation} conversation The chat messages to process.
+     * @param {Object} options Additional options for processing.
+     * @param {RawImage|RawImage[]} [options.images] The images to process, if not set in the conversation.
+     * @param {string} [options.chat_template="default"] The chat template to use.
+     * @returns {Promise<VLCChatProcessorResult | VLCChatProcessorResult & import('../../base/image_processors_utils.js').ImageProcessorResult>} The processed input.
+     */
+    async _call(conversation, {
+        images = null,
+        chat_template = "default",
+    }={}) {
+        if (!images) {
+            images = await Promise.all(
+                conversation
+                    .filter((msg) => msg.images)
+                    .flatMap((msg) => msg.images)
+                    .map((img) => RawImage.read(img))
+            );
+        } else if (!Array.isArray(images)) {
+            images = [images];
+        }
+
+        const tokenizer = this.tokenizer;
+        const result = tokenizer.apply_chat_template(conversation, {
+            tokenize: false,
+            add_generation_prompt: true,
+            chat_template,
+        });
+
+        const encode = (text) => tokenizer.encode(text, { add_special_tokens: false });
+        const parts = (/** @type {string} */(result))
+            .split(this.image_tag);
+        const num_images = parts.length - 1;
+        if (images.length !== num_images) {
+            throw new Error(`Number of images provided (${images.length}) does not match number of "${this.image_tag}" image tags (${num_images})`);
+        }
+
+        const [
+            image_placeholder_tag_id,
+            image_start_tag_id,
+            image_end_tag_id,
+        ] = tokenizer.model.convert_tokens_to_ids([
+            this.image_tag,
+            this.image_start_tag,
+            this.image_end_tag,
+        ]);
+
+        let input_ids = encode(parts[0]);
+        let images_seq_mask = new Array(input_ids.length).fill(false);
+        for (let i = 1; i < parts.length; ++i) {
+            const placeholder_image_tokens = new Array(this.num_image_tokens).fill(image_placeholder_tag_id);
+            const tokens = encode(parts[i]);
+            input_ids = mergeArrays(
+                input_ids,
+                [image_start_tag_id], placeholder_image_tokens, [image_end_tag_id],
+                tokens,
+            );
+            const image_mask = new Array(this.num_image_tokens).fill(true);
+            images_seq_mask = mergeArrays(
+                images_seq_mask,
+                [false], image_mask, [false],
+                new Array(tokens.length).fill(false),
+            );
+        }
+
+        const dims = [1, input_ids.length];
+        const final = {
+            input_ids: new Tensor('int64', input_ids, dims),
+            attention_mask: new Tensor('int64', new Array(input_ids.length).fill(1), dims),
+            images_seq_mask: new Tensor('bool', images_seq_mask, dims),
+            images_emb_mask: new Tensor(
+                'bool',
+                new Array(num_images * this.num_image_tokens).fill(true),
+                [1, num_images, this.num_image_tokens],
+            ),
+        }
+
+        if (images && images.length > 0) {
+            const image_inputs = await this.image_processor(images);
+            // Set the batch_size dimension to 1
+            image_inputs.pixel_values.unsqueeze_(0);
+            return { ...final, ...image_inputs };
+        }
+
+        return final;
+    }
+}

package/src/models/jina_clip/image_processing_jina_clip.js

@@ -0,0 +1,26 @@
+import {
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class JinaCLIPImageProcessor extends ImageProcessor {
+    constructor(config) {
+        // JinaCLIPImageProcessor uses a custom preprocessor_config.json, so we configure it here
+        const { resize_mode, fill_color, interpolation, size, ...other } = config;
+
+        const new_size = resize_mode === 'squash'
+            ? { width: size, height: size }
+            : resize_mode === 'shortest'
+                ? { shortest_edge: size }
+                : { longest_edge: size };
+
+        const resample = interpolation === 'bicubic' ? 3 : 2;
+        super({
+            ...other,
+            size: new_size,
+            resample,
+            do_center_crop: true,
+            crop_size: size,
+            do_normalize: true,
+        });
+    }
+}

package/src/models/jina_clip/processing_jina_clip.js

@@ -0,0 +1,24 @@
+
+import { Processor } from "../../base/processing_utils.js";
+import { AutoImageProcessor } from "../auto/image_processing_auto.js";
+import { AutoTokenizer } from "../../tokenizers.js";
+
+export class JinaCLIPProcessor extends Processor {
+    static tokenizer_class = AutoTokenizer
+    static image_processor_class = AutoImageProcessor
+
+    async _call(text=null, images=null, kwargs = {}) {
+
+        if (!text && !images){
+            throw new Error('Either text or images must be provided');
+        }
+
+        const text_inputs = text ? this.tokenizer(text, kwargs) : {};
+        const image_inputs = images ? await this.image_processor(images, kwargs) : {};
+
+        return {
+            ...text_inputs,
+            ...image_inputs,
+        }
+    }
+}

package/src/models/llava_onevision/image_processing_llava_onevision.js

@@ -0,0 +1,5 @@
+import { 
+    ImageProcessor,
+} from "../../base/image_processors_utils.js";
+
+export class LlavaOnevisionImageProcessor extends ImageProcessor {}

package/src/models/mask2former/image_processing_mask2former.js

@@ -0,0 +1,5 @@
+
+import { MaskFormerImageProcessor } from "../maskformer/image_processing_maskformer.js";
+
+// NOTE: extends MaskFormerImageProcessor
+export class Mask2FormerImageProcessor extends MaskFormerImageProcessor { }

package/src/models/maskformer/image_processing_maskformer.js

@@ -0,0 +1,18 @@
+import { 
+    ImageProcessor,
+    post_process_panoptic_segmentation,
+    post_process_instance_segmentation,
+} from "../../base/image_processors_utils.js";
+
+export class MaskFormerImageProcessor extends ImageProcessor {
+
+    /** @type {typeof post_process_panoptic_segmentation} */
+    post_process_panoptic_segmentation(...args) {
+        return post_process_panoptic_segmentation(...args);
+    }
+    /** @type {typeof post_process_instance_segmentation} */
+    post_process_instance_segmentation(...args) {
+        return post_process_instance_segmentation(...args);
+    }
+}
+export class MaskFormerFeatureExtractor extends MaskFormerImageProcessor { }