@mediapipe/tasks-vision 0.1.0-alpha-12 → 0.1.0-alpha-13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -160,3 +160,20 @@ const detections = objectDetector.detect(image);
160
160
 
161
161
  For more information, refer to the [Object Detector](https://developers.google.com/mediapipe/solutions/vision/object_detector/web_js) documentation.
162
162
 
163
+
164
+ ## Pose Landmark Detection
165
+
166
+ The MediaPipe Pose Landmarker task lets you detect the landmarks of body poses
167
+ in an image. You can use this Task to localize key points of a pose and render
168
+ visual effects over the body.
169
+
170
+ ```
171
+ const vision = await FilesetResolver.forVisionTasks(
172
+ "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
173
+ );
174
+ const poseLandmarker = await PoseLandmarker.createFromModelPath(vision,
175
+ "model.task"
176
+ );
177
+ const image = document.getElementById("image") as HTMLImageElement;
178
+ const landmarks = poseLandmarker.detect(image);
179
+ ```
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mediapipe/tasks-vision",
3
- "version": "0.1.0-alpha-12",
3
+ "version": "0.1.0-alpha-13",
4
4
  "description": "MediaPipe Vision Tasks",
5
5
  "main": "vision_bundle.js",
6
6
  "author": "mediapipe@google.com",
package/vision.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  /**
2
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
2
+ * Copyright 2022 The MediaPipe Authors.
3
3
  *
4
4
  * Licensed under the Apache License, Version 2.0 (the "License");
5
5
  * you may not use this file except in compliance with the License.
@@ -30,7 +30,7 @@ declare interface BaseOptions_2 {
30
30
  }
31
31
 
32
32
  /**
33
- * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
33
+ * Copyright 2023 The MediaPipe Authors.
34
34
  *
35
35
  * Licensed under the Apache License, Version 2.0 (the "License");
36
36
  * you may not use this file except in compliance with the License.
@@ -63,7 +63,7 @@ export declare interface BoundingBox {
63
63
  export declare type Callback<I, O> = (input: I) => O;
64
64
 
65
65
  /**
66
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
66
+ * Copyright 2022 The MediaPipe Authors.
67
67
  *
68
68
  * Licensed under the Apache License, Version 2.0 (the "License");
69
69
  * you may not use this file except in compliance with the License.
@@ -117,7 +117,7 @@ export declare interface Classifications {
117
117
  }
118
118
 
119
119
  /**
120
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
120
+ * Copyright 2022 The MediaPipe Authors.
121
121
  *
122
122
  * Licensed under the Apache License, Version 2.0 (the "License");
123
123
  * you may not use this file except in compliance with the License.
@@ -261,7 +261,7 @@ export declare class DrawingUtils {
261
261
  }
262
262
 
263
263
  /**
264
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
264
+ * Copyright 2022 The MediaPipe Authors.
265
265
  *
266
266
  * Licensed under the Apache License, Version 2.0 (the "License");
267
267
  * you may not use this file except in compliance with the License.
@@ -294,7 +294,7 @@ declare interface EmbedderOptions {
294
294
  }
295
295
 
296
296
  /**
297
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
297
+ * Copyright 2022 The MediaPipe Authors.
298
298
  *
299
299
  * Licensed under the Apache License, Version 2.0 (the "License");
300
300
  * you may not use this file except in compliance with the License.
@@ -598,7 +598,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
598
598
  * lifetime of the returned data is only guaranteed for the duration of the
599
599
  * callback.
600
600
  */
601
- stylize(image: ImageSource, callback: ImageCallback): void;
601
+ stylize(image: ImageSource, callback: FaceStylizerCallback): void;
602
602
  /**
603
603
  * Performs face stylization on the provided single image. The method returns
604
604
  * synchronously once the callback returns. Only use this method when the
@@ -620,7 +620,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
620
620
  * lifetime of the returned data is only guaranteed for the duration of the
621
621
  * callback.
622
622
  */
623
- stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageCallback): void;
623
+ stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: FaceStylizerCallback): void;
624
624
  /**
625
625
  * Performs face stylization on the provided video frame. Only use this method
626
626
  * when the FaceStylizer is created with the video running mode.
@@ -635,7 +635,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
635
635
  * lifetime of the returned data is only guaranteed for the duration of
636
636
  * the callback.
637
637
  */
638
- stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageCallback): void;
638
+ stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback: FaceStylizerCallback): void;
639
639
  /**
640
640
  * Performs face stylization on the provided video frame. Only use this
641
641
  * method when the FaceStylizer is created with the video running mode.
@@ -661,9 +661,19 @@ export declare class FaceStylizer extends VisionTaskRunner {
661
661
  * lifetime of the returned data is only guaranteed for the duration of
662
662
  * the callback.
663
663
  */
664
- stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageCallback): void;
664
+ stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: FaceStylizerCallback): void;
665
665
  }
666
666
 
667
+ /**
668
+ * A callback that receives an image from the face stylizer, or `null` if no
669
+ * face was detected. The lifetime of the underlying data is limited to the
670
+ * duration of the callback. If asynchronous processing is needed, all data
671
+ * needs to be copied before the callback returns.
672
+ *
673
+ * The `WebGLTexture` output type is reserved for future usage.
674
+ */
675
+ export declare type FaceStylizerCallback = (image: ImageData | WebGLTexture | null, width: number, height: number) => void;
676
+
667
677
  /** Options to configure the MediaPipe Face Stylizer Task */
668
678
  export declare interface FaceStylizerOptions extends VisionTaskOptions {
669
679
  }
@@ -943,22 +953,12 @@ export declare interface HandLandmarkerOptions extends VisionTaskOptions {
943
953
  export declare interface HandLandmarkerResult {
944
954
  /** Hand landmarks of detected hands. */
945
955
  landmarks: NormalizedLandmark[][];
946
- /** Hand landmarks in world coordniates of detected hands. */
956
+ /** Hand landmarks in world coordinates of detected hands. */
947
957
  worldLandmarks: Landmark[][];
948
958
  /** Handedness of detected hands. */
949
959
  handednesses: Category[][];
950
960
  }
951
961
 
952
- /**
953
- * A callback that receives an `ImageData` object from a Vision task. The
954
- * lifetime of the underlying data is limited to the duration of the callback.
955
- * If asynchronous processing is needed, all data needs to be copied before the
956
- * callback returns.
957
- *
958
- * The `WebGLTexture` output type is reserved for future usage.
959
- */
960
- export declare type ImageCallback = (image: ImageData | WebGLTexture, width: number, height: number) => void;
961
-
962
962
  /** Performs classification on images. */
963
963
  export declare class ImageClassifier extends VisionTaskRunner {
964
964
  /**
@@ -1213,7 +1213,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
1213
1213
  * lifetime of the returned data is only guaranteed for the duration of the
1214
1214
  * callback.
1215
1215
  */
1216
- segment(image: ImageSource, callback: ImageSegmenterCallack): void;
1216
+ segment(image: ImageSource, callback: ImageSegmenterCallback): void;
1217
1217
  /**
1218
1218
  * Performs image segmentation on the provided single image and invokes the
1219
1219
  * callback with the response. The method returns synchronously once the
@@ -1227,7 +1227,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
1227
1227
  * lifetime of the returned data is only guaranteed for the duration of the
1228
1228
  * callback.
1229
1229
  */
1230
- segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallack): void;
1230
+ segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallback): void;
1231
1231
  /**
1232
1232
  * Performs image segmentation on the provided video frame and invokes the
1233
1233
  * callback with the response. The method returns synchronously once the
@@ -1240,7 +1240,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
1240
1240
  * lifetime of the returned data is only guaranteed for the duration of the
1241
1241
  * callback.
1242
1242
  */
1243
- segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallack): void;
1243
+ segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallback): void;
1244
1244
  /**
1245
1245
  * Performs image segmentation on the provided video frame and invokes the
1246
1246
  * callback with the response. The method returns synchronously once the
@@ -1255,7 +1255,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
1255
1255
  * lifetime of the returned data is only guaranteed for the duration of the
1256
1256
  * callback.
1257
1257
  */
1258
- segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageSegmenterCallack): void;
1258
+ segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageSegmenterCallback): void;
1259
1259
  /**
1260
1260
  * Get the category label list of the ImageSegmenter can recognize. For
1261
1261
  * `CATEGORY_MASK` type, the index in the category mask corresponds to the
@@ -1276,7 +1276,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
1276
1276
  * asynchronous processing is needed, all data needs to be copied before the
1277
1277
  * callback returns.
1278
1278
  */
1279
- export declare type ImageSegmenterCallack = (result: ImageSegmenterResult) => void;
1279
+ export declare type ImageSegmenterCallback = (result: ImageSegmenterResult) => void;
1280
1280
 
1281
1281
  /** Options to configure the MediaPipe Image Segmenter Task */
1282
1282
  export declare interface ImageSegmenterOptions extends VisionTaskOptions {
@@ -1292,7 +1292,7 @@ export declare interface ImageSegmenterOptions extends VisionTaskOptions {
1292
1292
  }
1293
1293
 
1294
1294
  /**
1295
- * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
1295
+ * Copyright 2023 The MediaPipe Authors.
1296
1296
  *
1297
1297
  * Licensed under the Apache License, Version 2.0 (the "License");
1298
1298
  * you may not use this file except in compliance with the License.
@@ -1413,7 +1413,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
1413
1413
  * lifetime of the returned data is only guaranteed for the duration of the
1414
1414
  * callback.
1415
1415
  */
1416
- segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallack): void;
1416
+ segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallback): void;
1417
1417
  /**
1418
1418
  * Performs interactive segmentation on the provided single image and invokes
1419
1419
  * the callback with the response. The `roi` parameter is used to represent a
@@ -1439,7 +1439,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
1439
1439
  * lifetime of the returned data is only guaranteed for the duration of the
1440
1440
  * callback.
1441
1441
  */
1442
- segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallack): void;
1442
+ segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallback): void;
1443
1443
  }
1444
1444
 
1445
1445
  /**
@@ -1448,7 +1448,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
1448
1448
  * asynchronous processing is needed, all data needs to be copied before the
1449
1449
  * callback returns.
1450
1450
  */
1451
- export declare type InteractiveSegmenterCallack = (result: InteractiveSegmenterResult) => void;
1451
+ export declare type InteractiveSegmenterCallback = (result: InteractiveSegmenterResult) => void;
1452
1452
 
1453
1453
  /** Options to configure the MediaPipe Interactive Segmenter Task */
1454
1454
  export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
@@ -1459,7 +1459,7 @@ export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
1459
1459
  }
1460
1460
 
1461
1461
  /**
1462
- * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
1462
+ * Copyright 2023 The MediaPipe Authors.
1463
1463
  *
1464
1464
  * Licensed under the Apache License, Version 2.0 (the "License");
1465
1465
  * you may not use this file except in compliance with the License.
@@ -1514,7 +1514,7 @@ export declare interface LandmarkData {
1514
1514
  }
1515
1515
 
1516
1516
  /**
1517
- * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
1517
+ * Copyright 2023 The MediaPipe Authors.
1518
1518
  *
1519
1519
  * Licensed under the Apache License, Version 2.0 (the "License");
1520
1520
  * you may not use this file except in compliance with the License.
@@ -1539,7 +1539,249 @@ declare interface Matrix {
1539
1539
  }
1540
1540
 
1541
1541
  /**
1542
- * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
1542
+ * The wrapper class for MediaPipe Image objects.
1543
+ *
1544
+ * Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects.
1545
+ * You can convert the underlying type to any other type by passing the
1546
+ * desired type to `getImage()`. As type conversions can be expensive, it is
1547
+ * recommended to limit these conversions. You can verify what underlying
1548
+ * types are already available by invoking `hasType()`.
1549
+ *
1550
+ * Images that are returned from a MediaPipe Tasks are owned by by the
1551
+ * underlying C++ Task. If you need to extend the lifetime of these objects,
1552
+ * you can invoke the `clone()` method. To free up the resources obtained
1553
+ * during any clone or type conversion operation, it is important to invoke
1554
+ * `close()` on the `MPImage` instance.
1555
+ *
1556
+ * Converting to and from ImageBitmap requires that the MediaPipe task is
1557
+ * initialized with an `OffscreenCanvas`. As we require WebGL2 support, this
1558
+ * places some limitations on Browser support as outlined here:
1559
+ * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1560
+ *
1561
+ * Some MediaPipe tasks return single channel masks. These masks are stored
1562
+ * using an underlying `Uint8ClampedArray` an `Float32Array` (represented as
1563
+ * single-channel arrays). To convert these type to other formats a conversion
1564
+ * function is invoked to convert pixel values between single channel and four
1565
+ * channel RGBA values. To customize this conversion, you can specify these
1566
+ * conversion functions when you invoke `getImage()`. If you use the default
1567
+ * conversion function a warning will be logged to the console.
1568
+ */
1569
+ export declare class MPImage {
1570
+ /** Returns the canvas element that the image is bound to. */
1571
+ readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
1572
+ /** Returns the width of the image. */
1573
+ readonly width: number;
1574
+ /** Returns the height of the image. */
1575
+ readonly height: number;
1576
+ private constructor();
1577
+ /**
1578
+ * Returns whether this `MPImage` stores the image in the desired format.
1579
+ * This method can be called to reduce expensive conversion before invoking
1580
+ * `getType()`.
1581
+ */
1582
+ hasType(type: MPImageStorageType): boolean;
1583
+ /**
1584
+ * Returns the underlying image as a single channel `Uint8ClampedArray`. Note
1585
+ * that this involves an expensive GPU to CPU transfer if the current image is
1586
+ * only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1587
+ * function converts RGBA data pixel-by-pixel to a single channel value by
1588
+ * invoking a conversion function (see class comment for detail).
1589
+ *
1590
+ * @param type The type of image to return.
1591
+ * @param converter A set of conversion functions that will be invoked to
1592
+ * convert the underlying pixel data if necessary. You may omit this
1593
+ * function if the requested conversion does not change the pixel format.
1594
+ * @return The current data as a Uint8ClampedArray.
1595
+ */
1596
+ getImage(type: MPImageStorageType.UINT8_CLAMPED_ARRAY, converter?: MPImageChannelConverter): Uint8ClampedArray;
1597
+ /**
1598
+ * Returns the underlying image as a single channel `Float32Array`. Note
1599
+ * that this involves an expensive GPU to CPU transfer if the current image is
1600
+ * only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1601
+ * function converts RGBA data pixel-by-pixel to a single channel value by
1602
+ * invoking a conversion function (see class comment for detail).
1603
+ *
1604
+ * @param type The type of image to return.
1605
+ * @param converter A set of conversion functions that will be invoked to
1606
+ * convert the underlying pixel data if necessary. You may omit this
1607
+ * function if the requested conversion does not change the pixel format.
1608
+ * @return The current image as a Float32Array.
1609
+ */
1610
+ getImage(type: MPImageStorageType.FLOAT32_ARRAY, converter?: MPImageChannelConverter): Float32Array;
1611
+ /**
1612
+ * Returns the underlying image as an `ImageData` object. Note that this
1613
+ * involves an expensive GPU to CPU transfer if the current image is only
1614
+ * available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1615
+ * function converts single channel pixel values to RGBA by invoking a
1616
+ * conversion function (see class comment for detail).
1617
+ *
1618
+ * @return The current image as an ImageData object.
1619
+ */
1620
+ getImage(type: MPImageStorageType.IMAGE_DATA, converter?: MPImageChannelConverter): ImageData;
1621
+ /**
1622
+ * Returns the underlying image as an `ImageBitmap`. Note that
1623
+ * conversions to `ImageBitmap` are expensive, especially if the data
1624
+ * currently resides on CPU. If necessary, this function first converts single
1625
+ * channel pixel values to RGBA by invoking a conversion function (see class
1626
+ * comment for detail).
1627
+ *
1628
+ * Processing with `ImageBitmap`s requires that the MediaPipe Task was
1629
+ * initialized with an `OffscreenCanvas` with WebGL2 support. See
1630
+ * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1631
+ * for a list of supported platforms.
1632
+ *
1633
+ * @param type The type of image to return.
1634
+ * @param converter A set of conversion functions that will be invoked to
1635
+ * convert the underlying pixel data if necessary. You may omit this
1636
+ * function if the requested conversion does not change the pixel format.
1637
+ * @return The current image as an ImageBitmap object.
1638
+ */
1639
+ getImage(type: MPImageStorageType.IMAGE_BITMAP, converter?: MPImageChannelConverter): ImageBitmap;
1640
+ /**
1641
+ * Returns the underlying image as a `WebGLTexture` object. Note that this
1642
+ * involves a CPU to GPU transfer if the current image is only available as
1643
+ * an `ImageData` object. The returned texture is bound to the current
1644
+ * canvas (see `.canvas`).
1645
+ *
1646
+ * @param type The type of image to return.
1647
+ * @param converter A set of conversion functions that will be invoked to
1648
+ * convert the underlying pixel data if necessary. You may omit this
1649
+ * function if the requested conversion does not change the pixel format.
1650
+ * @return The current image as a WebGLTexture.
1651
+ */
1652
+ getImage(type: MPImageStorageType.WEBGL_TEXTURE, converter?: MPImageChannelConverter): WebGLTexture;
1653
+ /**
1654
+ * Creates a copy of the resources stored in this `MPImage`. You can invoke
1655
+ * this method to extend the lifetime of an image returned by a MediaPipe
1656
+ * Task. Note that performance critical applications should aim to only use
1657
+ * the `MPImage` within the MediaPipe Task callback so that copies can be
1658
+ * avoided.
1659
+ */
1660
+ clone(): MPImage;
1661
+ /**
1662
+ * Frees up any resources owned by this `MPImage` instance.
1663
+ *
1664
+ * Note that this method does not free images that are owned by the C++
1665
+ * Task, as these are freed automatically once you leave the MediaPipe
1666
+ * callback. Additionally, some shared state is freed only once you invoke the
1667
+ * Task's `close()` method.
1668
+ */
1669
+ close(): void;
1670
+ }
1671
+
1672
+ /**
1673
+ * An interface that can be used to provide custom conversion functions. These
1674
+ * functions are invoked to convert pixel values between different channel
1675
+ * counts and value ranges. Any conversion function that is not specified will
1676
+ * result in a default conversion.
1677
+ */
1678
+ export declare interface MPImageChannelConverter {
1679
+ /**
1680
+ * A conversion function to convert a number in the [0.0, 1.0] range to RGBA.
1681
+ * The output is an array with four elemeents whose values range from 0 to 255
1682
+ * inclusive.
1683
+ *
1684
+ * The default conversion function is `[v * 255, v * 255, v * 255, 255]`
1685
+ * and will log a warning if invoked.
1686
+ */
1687
+ floatToRGBAConverter?: (value: number) => [
1688
+ number,
1689
+ number,
1690
+ number,
1691
+ number
1692
+ ];
1693
+ uint8ToRGBAConverter?: (value: number) => [
1694
+ number,
1695
+ number,
1696
+ number,
1697
+ number
1698
+ ];
1699
+ /**
1700
+ * A conversion function to convert an RGBA value in the range of 0 to 255 to
1701
+ * a single value in the [0.0, 1.0] range.
1702
+ *
1703
+ * The default conversion function is `(r / 3 + g / 3 + b / 3) / 255` and will
1704
+ * log a warning if invoked.
1705
+ */
1706
+ rgbaToFloatConverter?: (r: number, g: number, b: number, a: number) => number;
1707
+ /**
1708
+ * A conversion function to convert an RGBA value in the range of 0 to 255 to
1709
+ * a single value in the [0, 255] range.
1710
+ *
1711
+ * The default conversion function is `r / 3 + g / 3 + b / 3` and will log a
1712
+ * warning if invoked.
1713
+ */
1714
+ rgbaToUint8Converter?: (r: number, g: number, b: number, a: number) => number;
1715
+ /**
1716
+ * A conversion function to convert a single value in the 0.0 to 1.0 range to
1717
+ * [0, 255].
1718
+ *
1719
+ * The default conversion function is `r * 255` and will log a warning if
1720
+ * invoked.
1721
+ */
1722
+ floatToUint8Converter?: (value: number) => number;
1723
+ /**
1724
+ * A conversion function to convert a single value in the 0 to 255 range to
1725
+ * [0.0, 1.0] .
1726
+ *
1727
+ * The default conversion function is `r / 255` and will log a warning if
1728
+ * invoked.
1729
+ */
1730
+ uint8ToFloatConverter?: (value: number) => number;
1731
+ }
1732
+
1733
+ /** The supported image formats. For internal usage. */
1734
+ export declare type MPImageNativeContainer = Uint8ClampedArray | Float32Array | ImageData | ImageBitmap | WebGLTexture;
1735
+
1736
+ /**
1737
+ * A class that encapsulates the shaders used by an MPImage. Can be re-used
1738
+ * across MPImages that use the same WebGL2Rendering context.
1739
+ */
1740
+ export declare class MPImageShaderContext {
1741
+ /** Runs the callback using the shader. */
1742
+ run<T>(gl: WebGL2RenderingContext, flipVertically: boolean, callback: () => T): T;
1743
+ /**
1744
+ * Binds a framebuffer to the canvas. If the framebuffer does not yet exist,
1745
+ * creates it first. Binds the provided texture to the framebuffer.
1746
+ */
1747
+ bindFramebuffer(gl: WebGL2RenderingContext, texture: WebGLTexture): void;
1748
+ unbindFramebuffer(): void;
1749
+ close(): void;
1750
+ }
1751
+
1752
+ /**
1753
+ * Copyright 2023 The MediaPipe Authors.
1754
+ *
1755
+ * Licensed under the Apache License, Version 2.0 (the "License");
1756
+ * you may not use this file except in compliance with the License.
1757
+ * You may obtain a copy of the License at
1758
+ *
1759
+ * http://www.apache.org/licenses/LICENSE-2.0
1760
+ *
1761
+ * Unless required by applicable law or agreed to in writing, software
1762
+ * distributed under the License is distributed on an "AS IS" BASIS,
1763
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1764
+ * See the License for the specific language governing permissions and
1765
+ * limitations under the License.
1766
+ */
1767
+ /** The underlying type of the image. */
1768
+ export declare enum MPImageStorageType {
1769
+ /** Represents the native `UInt8ClampedArray` type. */
1770
+ UINT8_CLAMPED_ARRAY = 0,
1771
+ /**
1772
+ * Represents the native `Float32Array` type. Values range from [0.0, 1.0].
1773
+ */
1774
+ FLOAT32_ARRAY = 1,
1775
+ /** Represents the native `ImageData` type. */
1776
+ IMAGE_DATA = 2,
1777
+ /** Represents the native `ImageBitmap` type. */
1778
+ IMAGE_BITMAP = 3,
1779
+ /** Represents the native `WebGLTexture` type. */
1780
+ WEBGL_TEXTURE = 4
1781
+ }
1782
+
1783
+ /**
1784
+ * Copyright 2023 The MediaPipe Authors.
1543
1785
  *
1544
1786
  * Licensed under the Apache License, Version 2.0 (the "License");
1545
1787
  * you may not use this file except in compliance with the License.
@@ -1569,7 +1811,7 @@ declare interface NormalizedKeypoint {
1569
1811
  }
1570
1812
 
1571
1813
  /**
1572
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
1814
+ * Copyright 2022 The MediaPipe Authors.
1573
1815
  *
1574
1816
  * Licensed under the Apache License, Version 2.0 (the "License");
1575
1817
  * you may not use this file except in compliance with the License.
@@ -1667,6 +1909,155 @@ export declare class ObjectDetector extends VisionTaskRunner {
1667
1909
  export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
1668
1910
  }
1669
1911
 
1912
+ /** Performs pose landmarks detection on images. */
1913
+ export declare class PoseLandmarker extends VisionTaskRunner {
1914
+ /**
1915
+ * An array containing the pairs of pose landmark indices to be rendered with
1916
+ * connections.
1917
+ */
1918
+ static POSE_CONNECTIONS: Connection[];
1919
+ /**
1920
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` from the
1921
+ * provided options.
1922
+ * @param wasmFileset A configuration object that provides the location of the
1923
+ * Wasm binary and its loader.
1924
+ * @param poseLandmarkerOptions The options for the PoseLandmarker.
1925
+ * Note that either a path to the model asset or a model buffer needs to
1926
+ * be provided (via `baseOptions`).
1927
+ */
1928
+ static createFromOptions(wasmFileset: WasmFileset, poseLandmarkerOptions: PoseLandmarkerOptions): Promise<PoseLandmarker>;
1929
+ /**
1930
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
1931
+ * the provided model asset buffer.
1932
+ * @param wasmFileset A configuration object that provides the location of the
1933
+ * Wasm binary and its loader.
1934
+ * @param modelAssetBuffer A binary representation of the model.
1935
+ */
1936
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<PoseLandmarker>;
1937
+ /**
1938
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
1939
+ * the path to the model asset.
1940
+ * @param wasmFileset A configuration object that provides the location of the
1941
+ * Wasm binary and its loader.
1942
+ * @param modelAssetPath The path to the model asset.
1943
+ */
1944
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<PoseLandmarker>;
1945
+ private constructor();
1946
+ /**
1947
+ * Sets new options for this `PoseLandmarker`.
1948
+ *
1949
+ * Calling `setOptions()` with a subset of options only affects those options.
1950
+ * You can reset an option back to its default value by explicitly setting it
1951
+ * to `undefined`.
1952
+ *
1953
+ * @param options The options for the pose landmarker.
1954
+ */
1955
+ setOptions(options: PoseLandmarkerOptions): Promise<void>;
1956
+ /**
1957
+ * Performs pose detection on the provided single image and waits
1958
+ * synchronously for the response. Only use this method when the
1959
+ * PoseLandmarker is created with running mode `image`.
1960
+ *
1961
+ * @param image An image to process.
1962
+ * @param callback The callback that is invoked with the result. The
1963
+ * lifetime of the returned masks is only guaranteed for the duration of
1964
+ * the callback.
1965
+ * @return The detected pose landmarks.
1966
+ */
1967
+ detect(image: ImageSource, callback: PoseLandmarkerCallback): void;
1968
+ /**
1969
+ * Performs pose detection on the provided single image and waits
1970
+ * synchronously for the response. Only use this method when the
1971
+ * PoseLandmarker is created with running mode `image`.
1972
+ *
1973
+ * @param image An image to process.
1974
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1975
+ * to process the input image before running inference.
1976
+ * @param callback The callback that is invoked with the result. The
1977
+ * lifetime of the returned masks is only guaranteed for the duration of
1978
+ * the callback.
1979
+ * @return The detected pose landmarks.
1980
+ */
1981
+ detect(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: PoseLandmarkerCallback): void;
1982
+ /**
1983
+ * Performs pose detection on the provided video frame and waits
1984
+ * synchronously for the response. Only use this method when the
1985
+ * PoseLandmarker is created with running mode `video`.
1986
+ *
1987
+ * @param videoFrame A video frame to process.
1988
+ * @param timestamp The timestamp of the current frame, in ms.
1989
+ * @param callback The callback that is invoked with the result. The
1990
+ * lifetime of the returned masks is only guaranteed for the duration of
1991
+ * the callback.
1992
+ * @return The detected pose landmarks.
1993
+ */
1994
+ detectForVideo(videoFrame: ImageSource, timestamp: number, callback: PoseLandmarkerCallback): void;
1995
+ /**
1996
+ * Performs pose detection on the provided video frame and waits
1997
+ * synchronously for the response. Only use this method when the
1998
+ * PoseLandmarker is created with running mode `video`.
1999
+ *
2000
+ * @param videoFrame A video frame to process.
2001
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2002
+ * to process the input image before running inference.
2003
+ * @param timestamp The timestamp of the current frame, in ms.
2004
+ * @param callback The callback that is invoked with the result. The
2005
+ * lifetime of the returned masks is only guaranteed for the duration of
2006
+ * the callback.
2007
+ * @return The detected pose landmarks.
2008
+ */
2009
+ detectForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: PoseLandmarkerCallback): void;
2010
+ }
2011
+
2012
+ /**
2013
+ * A callback that receives the result from the pose detector. The returned
2014
+ * masks are only valid for the duration of the callback. If asynchronous
2015
+ * processing is needed, the masks need to be copied before the callback
2016
+ * returns.
2017
+ */
2018
+ export declare type PoseLandmarkerCallback = (result: PoseLandmarkerResult) => void;
2019
+
2020
+ /** Options to configure the MediaPipe PoseLandmarker Task */
2021
+ export declare interface PoseLandmarkerOptions extends VisionTaskOptions {
2022
+ /**
2023
+ * The maximum number of poses can be detected by the PoseLandmarker.
2024
+ * Defaults to 1.
2025
+ */
2026
+ numPoses?: number | undefined;
2027
+ /**
2028
+ * The minimum confidence score for the pose detection to be considered
2029
+ * successful. Defaults to 0.5.
2030
+ */
2031
+ minPoseDetectionConfidence?: number | undefined;
2032
+ /**
2033
+ * The minimum confidence score of pose presence score in the pose landmark
2034
+ * detection. Defaults to 0.5.
2035
+ */
2036
+ minPosePresenceConfidence?: number | undefined;
2037
+ /**
2038
+ * The minimum confidence score for the pose tracking to be considered
2039
+ * successful. Defaults to 0.5.
2040
+ */
2041
+ minTrackingConfidence?: number | undefined;
2042
+ /** Whether to output segmentation masks. Defaults to false. */
2043
+ outputSegmentationMasks?: boolean | undefined;
2044
+ }
2045
+
2046
+ /**
2047
+ * Represents the pose landmarks deection results generated by `PoseLandmarker`.
2048
+ * Each vector element represents a single pose detected in the image.
2049
+ */
2050
+ export declare interface PoseLandmarkerResult {
2051
+ /** Pose landmarks of detected poses. */
2052
+ landmarks: NormalizedLandmark[];
2053
+ /** Pose landmarks in world coordinates of detected poses. */
2054
+ worldLandmarks: Landmark[];
2055
+ /** Detected auxiliary landmarks, used for deriving ROI for next frame. */
2056
+ auxilaryLandmarks: NormalizedLandmark[];
2057
+ /** Segmentation mask for the detected pose. */
2058
+ segmentationMasks?: Float32Array[] | WebGLTexture[];
2059
+ }
2060
+
1670
2061
  /**
1671
2062
  * Defines a rectangle, used e.g. as part of detection results or as input
1672
2063
  * region-of-interest.
@@ -1743,7 +2134,7 @@ declare abstract class VisionTaskRunner extends TaskRunner {
1743
2134
  }
1744
2135
 
1745
2136
  /**
1746
- * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
2137
+ * Copyright 2022 The MediaPipe Authors.
1747
2138
  *
1748
2139
  * Licensed under the Apache License, Version 2.0 (the "License");
1749
2140
  * you may not use this file except in compliance with the License.