@mediapipe/tasks-vision 0.10.7 → 0.10.9-rc.20231208

package/README.md

@@ -66,7 +66,7 @@ const vision = await FilesetResolver.forVisionTasks(
     "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm"
 );
 const gestureRecognizer = await GestureRecognizer.createFromModelPath(vision,
-    "hhttps://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task"
+    "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task"
 );
 const image = document.getElementById("image") as HTMLImageElement;
 const recognitions = gestureRecognizer.recognize(image);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mediapipe/tasks-vision",
-  "version": "0.10.7",
+  "version": "0.10.9-rc.20231208",
   "description": "MediaPipe Vision Tasks",
   "main": "vision_bundle.cjs",
   "browser": "vision_bundle.mjs",

package/vision.d.ts

@@ -103,6 +103,12 @@ export declare interface Category {
     displayName: string;
 }
 
+/**
+ * A category to color mapping that uses either a map or an array to assign
+ * category indexes to RGBA colors.
+ */
+export declare type CategoryToColorMap = Map<number, RGBAColor> | RGBAColor[];
+
 /** Classification results for a given classifier head. */
 export declare interface Classifications {
     /**
@@ -171,6 +177,9 @@ declare interface Connection {
     end: number;
 }
 
+/** A color map with 22 classes. Used in our demos. */
+export declare const DEFAULT_CATEGORY_TO_COLOR_MAP: number[][];
+
 /** Represents one detection by a detection task. */
 export declare interface Detection {
     /** A list of `Category` objects. */
@@ -218,9 +227,23 @@ export declare class DrawingUtils {
     /**
      * Creates a new DrawingUtils class.
      *
-     * @param ctx The canvas to render onto.
+     * @param gpuContext The WebGL canvas rendering context to render into. If
+     *     your Task is using a GPU delegate, the context must be obtained from
+     * its canvas (provided via `setOptions({ canvas: .. })`).
+     */
+    constructor(gpuContext: WebGL2RenderingContext);
+    /**
+     * Creates a new DrawingUtils class.
+     *
+     * @param cpuContext The 2D canvas rendering context to render into. If
+     *     you are rendering GPU data you must also provide `gpuContext` to allow
+     *     for data conversion.
+     * @param gpuContext A WebGL canvas that is used for GPU rendering and for
+     *     converting GPU to CPU data. If your Task is using a GPU delegate, the
+     *     context must be obtained from  its canvas (provided via
+     *     `setOptions({ canvas: .. })`).
      */
-    constructor(ctx: CanvasRenderingContext2D);
+    constructor(cpuContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, gpuContext?: WebGL2RenderingContext);
     /**
      * Restricts a number between two endpoints (order doesn't matter).
      *
@@ -247,6 +270,9 @@ export declare class DrawingUtils {
     /**
      * Draws circles onto the provided landmarks.
      *
+     * This method can only be used when `DrawingUtils` is initialized with a
+     * `CanvasRenderingContext2D`.
+     *
      * @export
      * @param landmarks The landmarks to draw.
      * @param style The style to visualize the landmarks.
@@ -255,6 +281,9 @@ export declare class DrawingUtils {
     /**
      * Draws lines between landmarks (given a connection graph).
      *
+     * This method can only be used when `DrawingUtils` is initialized with a
+     * `CanvasRenderingContext2D`.
+     *
      * @export
      * @param landmarks The landmarks to draw.
      * @param connections The connections array that contains the start and the
@@ -265,11 +294,59 @@ export declare class DrawingUtils {
     /**
      * Draws a bounding box.
      *
+     * This method can only be used when `DrawingUtils` is initialized with a
+     * `CanvasRenderingContext2D`.
+     *
      * @export
      * @param boundingBox The bounding box to draw.
      * @param style The style to visualize the boundin box.
      */
     drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void;
+    /**
+     * Draws a category mask using the provided category-to-color mapping.
+     *
+     * @export
+     * @param mask A category mask that was returned from a segmentation task.
+     * @param categoryToColorMap A map that maps category indices to RGBA
+     *     values. You must specify a map entry for each category.
+     * @param background A color or image to use as the background. Defaults to
+     *     black.
+     */
+    drawCategoryMask(mask: MPMask, categoryToColorMap: Map<number, RGBAColor>, background?: RGBAColor | ImageSource): void;
+    /**
+     * Draws a category mask using the provided color array.
+     *
+     * @export
+     * @param mask A category mask that was returned from a segmentation task.
+     * @param categoryToColorMap An array that maps indices to RGBA values. The
+     *     array's indices must correspond to the category indices of the model
+     *     and an entry must be provided for each category.
+     * @param background A color or image to use as the background. Defaults to
+     *     black.
+     */
+    drawCategoryMask(mask: MPMask, categoryToColorMap: RGBAColor[], background?: RGBAColor | ImageSource): void;
+    /**
+     * Blends two images using the provided confidence mask.
+     *
+     * If you are using an `ImageData` or `HTMLImageElement` as your data source
+     * and drawing the result onto a `WebGL2RenderingContext`, this method uploads
+     * the image data to the GPU. For still image input that gets re-used every
+     * frame, you can reduce the cost of re-uploading these images by passing a
+     * `HTMLCanvasElement` instead.
+     *
+     * @export
+     * @param mask A confidence mask that was returned from a segmentation task.
+     * @param defaultTexture An image or a four-channel color that will be used
+     *     when confidence values are low.
+     * @param overlayTexture An image or four-channel color that will be used when
+     *     confidence values are high.
+     */
+    drawConfidenceMask(mask: MPMask, defaultTexture: RGBAColor | ImageSource, overlayTexture: RGBAColor | ImageSource): void;
+    /**
+     * Frees all WebGL resources held by this class.
+     * @export
+     */
+    close(): void;
 }
 
 /**
@@ -922,7 +999,7 @@ export declare interface GestureRecognizerOptions extends VisionTaskOptions {
 export declare interface GestureRecognizerResult {
     /** Hand landmarks of detected hands. */
     landmarks: NormalizedLandmark[][];
-    /** Hand landmarks in world coordniates of detected hands. */
+    /** Hand landmarks in world coordinates of detected hands. */
     worldLandmarks: Landmark[][];
     /** Handedness of detected hands. */
     handedness: Category[][];
@@ -1880,6 +1957,7 @@ export declare class MPImage {
  * `close()` on the `MPMask` instance.
  */
 export declare class MPMask {
+    readonly interpolateValues: boolean;
     /** Returns the canvas element that the mask is bound to. */
     readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
     /** Returns the width of the mask. */
@@ -2325,6 +2403,17 @@ export declare interface RegionOfInterest {
     scribble?: NormalizedKeypoint[];
 }
 
+/**
+ * A four channel color with values for red, green, blue and alpha
+ * respectively.
+ */
+export declare type RGBAColor = [
+number,
+number,
+number,
+number
+] | number[];
+
 /**
  * The two running modes of a vision task.
  * 1) The image mode for processing single image inputs.