@mediapipe/tasks-vision 0.1.0-alpha-12 → 0.1.0-alpha-13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/package.json +1 -1
- package/vision.d.ts +426 -35
- package/vision_bundle.js +1 -1
- package/wasm/vision_wasm_internal.js +731 -770
- package/wasm/vision_wasm_internal.wasm +0 -0
- package/wasm/vision_wasm_nosimd_internal.js +728 -767
- package/wasm/vision_wasm_nosimd_internal.wasm +0 -0
package/README.md
CHANGED
|
@@ -160,3 +160,20 @@ const detections = objectDetector.detect(image);
|
|
|
160
160
|
|
|
161
161
|
For more information, refer to the [Object Detector](https://developers.google.com/mediapipe/solutions/vision/object_detector/web_js) documentation.
|
|
162
162
|
|
|
163
|
+
|
|
164
|
+
## Pose Landmark Detection
|
|
165
|
+
|
|
166
|
+
The MediaPipe Pose Landmarker task lets you detect the landmarks of body poses
|
|
167
|
+
in an image. You can use this Task to localize key points of a pose and render
|
|
168
|
+
visual effects over the body.
|
|
169
|
+
|
|
170
|
+
```
|
|
171
|
+
const vision = await FilesetResolver.forVisionTasks(
|
|
172
|
+
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
|
|
173
|
+
);
|
|
174
|
+
const poseLandmarker = await PoseLandmarker.createFromModelPath(vision,
|
|
175
|
+
"model.task"
|
|
176
|
+
);
|
|
177
|
+
const image = document.getElementById("image") as HTMLImageElement;
|
|
178
|
+
const landmarks = poseLandmarker.detect(image);
|
|
179
|
+
```
|
package/package.json
CHANGED
package/vision.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
2
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
3
3
|
*
|
|
4
4
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
5
|
* you may not use this file except in compliance with the License.
|
|
@@ -30,7 +30,7 @@ declare interface BaseOptions_2 {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
|
-
* Copyright 2023 The MediaPipe Authors.
|
|
33
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
34
34
|
*
|
|
35
35
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
36
36
|
* you may not use this file except in compliance with the License.
|
|
@@ -63,7 +63,7 @@ export declare interface BoundingBox {
|
|
|
63
63
|
export declare type Callback<I, O> = (input: I) => O;
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
66
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
67
67
|
*
|
|
68
68
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
69
69
|
* you may not use this file except in compliance with the License.
|
|
@@ -117,7 +117,7 @@ export declare interface Classifications {
|
|
|
117
117
|
}
|
|
118
118
|
|
|
119
119
|
/**
|
|
120
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
120
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
121
121
|
*
|
|
122
122
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
123
123
|
* you may not use this file except in compliance with the License.
|
|
@@ -261,7 +261,7 @@ export declare class DrawingUtils {
|
|
|
261
261
|
}
|
|
262
262
|
|
|
263
263
|
/**
|
|
264
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
264
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
265
265
|
*
|
|
266
266
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
267
267
|
* you may not use this file except in compliance with the License.
|
|
@@ -294,7 +294,7 @@ declare interface EmbedderOptions {
|
|
|
294
294
|
}
|
|
295
295
|
|
|
296
296
|
/**
|
|
297
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
297
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
298
298
|
*
|
|
299
299
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
300
300
|
* you may not use this file except in compliance with the License.
|
|
@@ -598,7 +598,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
|
|
|
598
598
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
599
599
|
* callback.
|
|
600
600
|
*/
|
|
601
|
-
stylize(image: ImageSource, callback:
|
|
601
|
+
stylize(image: ImageSource, callback: FaceStylizerCallback): void;
|
|
602
602
|
/**
|
|
603
603
|
* Performs face stylization on the provided single image. The method returns
|
|
604
604
|
* synchronously once the callback returns. Only use this method when the
|
|
@@ -620,7 +620,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
|
|
|
620
620
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
621
621
|
* callback.
|
|
622
622
|
*/
|
|
623
|
-
stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback:
|
|
623
|
+
stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: FaceStylizerCallback): void;
|
|
624
624
|
/**
|
|
625
625
|
* Performs face stylization on the provided video frame. Only use this method
|
|
626
626
|
* when the FaceStylizer is created with the video running mode.
|
|
@@ -635,7 +635,7 @@ export declare class FaceStylizer extends VisionTaskRunner {
|
|
|
635
635
|
* lifetime of the returned data is only guaranteed for the duration of
|
|
636
636
|
* the callback.
|
|
637
637
|
*/
|
|
638
|
-
stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback:
|
|
638
|
+
stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback: FaceStylizerCallback): void;
|
|
639
639
|
/**
|
|
640
640
|
* Performs face stylization on the provided video frame. Only use this
|
|
641
641
|
* method when the FaceStylizer is created with the video running mode.
|
|
@@ -661,9 +661,19 @@ export declare class FaceStylizer extends VisionTaskRunner {
|
|
|
661
661
|
* lifetime of the returned data is only guaranteed for the duration of
|
|
662
662
|
* the callback.
|
|
663
663
|
*/
|
|
664
|
-
stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback:
|
|
664
|
+
stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: FaceStylizerCallback): void;
|
|
665
665
|
}
|
|
666
666
|
|
|
667
|
+
/**
|
|
668
|
+
* A callback that receives an image from the face stylizer, or `null` if no
|
|
669
|
+
* face was detected. The lifetime of the underlying data is limited to the
|
|
670
|
+
* duration of the callback. If asynchronous processing is needed, all data
|
|
671
|
+
* needs to be copied before the callback returns.
|
|
672
|
+
*
|
|
673
|
+
* The `WebGLTexture` output type is reserved for future usage.
|
|
674
|
+
*/
|
|
675
|
+
export declare type FaceStylizerCallback = (image: ImageData | WebGLTexture | null, width: number, height: number) => void;
|
|
676
|
+
|
|
667
677
|
/** Options to configure the MediaPipe Face Stylizer Task */
|
|
668
678
|
export declare interface FaceStylizerOptions extends VisionTaskOptions {
|
|
669
679
|
}
|
|
@@ -943,22 +953,12 @@ export declare interface HandLandmarkerOptions extends VisionTaskOptions {
|
|
|
943
953
|
export declare interface HandLandmarkerResult {
|
|
944
954
|
/** Hand landmarks of detected hands. */
|
|
945
955
|
landmarks: NormalizedLandmark[][];
|
|
946
|
-
/** Hand landmarks in world
|
|
956
|
+
/** Hand landmarks in world coordinates of detected hands. */
|
|
947
957
|
worldLandmarks: Landmark[][];
|
|
948
958
|
/** Handedness of detected hands. */
|
|
949
959
|
handednesses: Category[][];
|
|
950
960
|
}
|
|
951
961
|
|
|
952
|
-
/**
|
|
953
|
-
* A callback that receives an `ImageData` object from a Vision task. The
|
|
954
|
-
* lifetime of the underlying data is limited to the duration of the callback.
|
|
955
|
-
* If asynchronous processing is needed, all data needs to be copied before the
|
|
956
|
-
* callback returns.
|
|
957
|
-
*
|
|
958
|
-
* The `WebGLTexture` output type is reserved for future usage.
|
|
959
|
-
*/
|
|
960
|
-
export declare type ImageCallback = (image: ImageData | WebGLTexture, width: number, height: number) => void;
|
|
961
|
-
|
|
962
962
|
/** Performs classification on images. */
|
|
963
963
|
export declare class ImageClassifier extends VisionTaskRunner {
|
|
964
964
|
/**
|
|
@@ -1213,7 +1213,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
|
|
|
1213
1213
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1214
1214
|
* callback.
|
|
1215
1215
|
*/
|
|
1216
|
-
segment(image: ImageSource, callback:
|
|
1216
|
+
segment(image: ImageSource, callback: ImageSegmenterCallback): void;
|
|
1217
1217
|
/**
|
|
1218
1218
|
* Performs image segmentation on the provided single image and invokes the
|
|
1219
1219
|
* callback with the response. The method returns synchronously once the
|
|
@@ -1227,7 +1227,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
|
|
|
1227
1227
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1228
1228
|
* callback.
|
|
1229
1229
|
*/
|
|
1230
|
-
segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback:
|
|
1230
|
+
segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallback): void;
|
|
1231
1231
|
/**
|
|
1232
1232
|
* Performs image segmentation on the provided video frame and invokes the
|
|
1233
1233
|
* callback with the response. The method returns synchronously once the
|
|
@@ -1240,7 +1240,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
|
|
|
1240
1240
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1241
1241
|
* callback.
|
|
1242
1242
|
*/
|
|
1243
|
-
segmentForVideo(videoFrame: ImageSource, timestamp: number, callback:
|
|
1243
|
+
segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallback): void;
|
|
1244
1244
|
/**
|
|
1245
1245
|
* Performs image segmentation on the provided video frame and invokes the
|
|
1246
1246
|
* callback with the response. The method returns synchronously once the
|
|
@@ -1255,7 +1255,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
|
|
|
1255
1255
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1256
1256
|
* callback.
|
|
1257
1257
|
*/
|
|
1258
|
-
segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback:
|
|
1258
|
+
segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageSegmenterCallback): void;
|
|
1259
1259
|
/**
|
|
1260
1260
|
* Get the category label list of the ImageSegmenter can recognize. For
|
|
1261
1261
|
* `CATEGORY_MASK` type, the index in the category mask corresponds to the
|
|
@@ -1276,7 +1276,7 @@ export declare class ImageSegmenter extends VisionTaskRunner {
|
|
|
1276
1276
|
* asynchronous processing is needed, all data needs to be copied before the
|
|
1277
1277
|
* callback returns.
|
|
1278
1278
|
*/
|
|
1279
|
-
export declare type
|
|
1279
|
+
export declare type ImageSegmenterCallback = (result: ImageSegmenterResult) => void;
|
|
1280
1280
|
|
|
1281
1281
|
/** Options to configure the MediaPipe Image Segmenter Task */
|
|
1282
1282
|
export declare interface ImageSegmenterOptions extends VisionTaskOptions {
|
|
@@ -1292,7 +1292,7 @@ export declare interface ImageSegmenterOptions extends VisionTaskOptions {
|
|
|
1292
1292
|
}
|
|
1293
1293
|
|
|
1294
1294
|
/**
|
|
1295
|
-
* Copyright 2023 The MediaPipe Authors.
|
|
1295
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
1296
1296
|
*
|
|
1297
1297
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1298
1298
|
* you may not use this file except in compliance with the License.
|
|
@@ -1413,7 +1413,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
|
|
|
1413
1413
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1414
1414
|
* callback.
|
|
1415
1415
|
*/
|
|
1416
|
-
segment(image: ImageSource, roi: RegionOfInterest, callback:
|
|
1416
|
+
segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallback): void;
|
|
1417
1417
|
/**
|
|
1418
1418
|
* Performs interactive segmentation on the provided single image and invokes
|
|
1419
1419
|
* the callback with the response. The `roi` parameter is used to represent a
|
|
@@ -1439,7 +1439,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
|
|
|
1439
1439
|
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1440
1440
|
* callback.
|
|
1441
1441
|
*/
|
|
1442
|
-
segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback:
|
|
1442
|
+
segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallback): void;
|
|
1443
1443
|
}
|
|
1444
1444
|
|
|
1445
1445
|
/**
|
|
@@ -1448,7 +1448,7 @@ export declare class InteractiveSegmenter extends VisionTaskRunner {
|
|
|
1448
1448
|
* asynchronous processing is needed, all data needs to be copied before the
|
|
1449
1449
|
* callback returns.
|
|
1450
1450
|
*/
|
|
1451
|
-
export declare type
|
|
1451
|
+
export declare type InteractiveSegmenterCallback = (result: InteractiveSegmenterResult) => void;
|
|
1452
1452
|
|
|
1453
1453
|
/** Options to configure the MediaPipe Interactive Segmenter Task */
|
|
1454
1454
|
export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
|
|
@@ -1459,7 +1459,7 @@ export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
|
|
|
1459
1459
|
}
|
|
1460
1460
|
|
|
1461
1461
|
/**
|
|
1462
|
-
* Copyright 2023 The MediaPipe Authors.
|
|
1462
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
1463
1463
|
*
|
|
1464
1464
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1465
1465
|
* you may not use this file except in compliance with the License.
|
|
@@ -1514,7 +1514,7 @@ export declare interface LandmarkData {
|
|
|
1514
1514
|
}
|
|
1515
1515
|
|
|
1516
1516
|
/**
|
|
1517
|
-
* Copyright 2023 The MediaPipe Authors.
|
|
1517
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
1518
1518
|
*
|
|
1519
1519
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1520
1520
|
* you may not use this file except in compliance with the License.
|
|
@@ -1539,7 +1539,249 @@ declare interface Matrix {
|
|
|
1539
1539
|
}
|
|
1540
1540
|
|
|
1541
1541
|
/**
|
|
1542
|
-
*
|
|
1542
|
+
* The wrapper class for MediaPipe Image objects.
|
|
1543
|
+
*
|
|
1544
|
+
* Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects.
|
|
1545
|
+
* You can convert the underlying type to any other type by passing the
|
|
1546
|
+
* desired type to `getImage()`. As type conversions can be expensive, it is
|
|
1547
|
+
* recommended to limit these conversions. You can verify what underlying
|
|
1548
|
+
* types are already available by invoking `hasType()`.
|
|
1549
|
+
*
|
|
1550
|
+
* Images that are returned from a MediaPipe Tasks are owned by by the
|
|
1551
|
+
* underlying C++ Task. If you need to extend the lifetime of these objects,
|
|
1552
|
+
* you can invoke the `clone()` method. To free up the resources obtained
|
|
1553
|
+
* during any clone or type conversion operation, it is important to invoke
|
|
1554
|
+
* `close()` on the `MPImage` instance.
|
|
1555
|
+
*
|
|
1556
|
+
* Converting to and from ImageBitmap requires that the MediaPipe task is
|
|
1557
|
+
* initialized with an `OffscreenCanvas`. As we require WebGL2 support, this
|
|
1558
|
+
* places some limitations on Browser support as outlined here:
|
|
1559
|
+
* https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
|
|
1560
|
+
*
|
|
1561
|
+
* Some MediaPipe tasks return single channel masks. These masks are stored
|
|
1562
|
+
* using an underlying `Uint8ClampedArray` an `Float32Array` (represented as
|
|
1563
|
+
* single-channel arrays). To convert these type to other formats a conversion
|
|
1564
|
+
* function is invoked to convert pixel values between single channel and four
|
|
1565
|
+
* channel RGBA values. To customize this conversion, you can specify these
|
|
1566
|
+
* conversion functions when you invoke `getImage()`. If you use the default
|
|
1567
|
+
* conversion function a warning will be logged to the console.
|
|
1568
|
+
*/
|
|
1569
|
+
export declare class MPImage {
|
|
1570
|
+
/** Returns the canvas element that the image is bound to. */
|
|
1571
|
+
readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
|
|
1572
|
+
/** Returns the width of the image. */
|
|
1573
|
+
readonly width: number;
|
|
1574
|
+
/** Returns the height of the image. */
|
|
1575
|
+
readonly height: number;
|
|
1576
|
+
private constructor();
|
|
1577
|
+
/**
|
|
1578
|
+
* Returns whether this `MPImage` stores the image in the desired format.
|
|
1579
|
+
* This method can be called to reduce expensive conversion before invoking
|
|
1580
|
+
* `getType()`.
|
|
1581
|
+
*/
|
|
1582
|
+
hasType(type: MPImageStorageType): boolean;
|
|
1583
|
+
/**
|
|
1584
|
+
* Returns the underlying image as a single channel `Uint8ClampedArray`. Note
|
|
1585
|
+
* that this involves an expensive GPU to CPU transfer if the current image is
|
|
1586
|
+
* only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
|
|
1587
|
+
* function converts RGBA data pixel-by-pixel to a single channel value by
|
|
1588
|
+
* invoking a conversion function (see class comment for detail).
|
|
1589
|
+
*
|
|
1590
|
+
* @param type The type of image to return.
|
|
1591
|
+
* @param converter A set of conversion functions that will be invoked to
|
|
1592
|
+
* convert the underlying pixel data if necessary. You may omit this
|
|
1593
|
+
* function if the requested conversion does not change the pixel format.
|
|
1594
|
+
* @return The current data as a Uint8ClampedArray.
|
|
1595
|
+
*/
|
|
1596
|
+
getImage(type: MPImageStorageType.UINT8_CLAMPED_ARRAY, converter?: MPImageChannelConverter): Uint8ClampedArray;
|
|
1597
|
+
/**
|
|
1598
|
+
* Returns the underlying image as a single channel `Float32Array`. Note
|
|
1599
|
+
* that this involves an expensive GPU to CPU transfer if the current image is
|
|
1600
|
+
* only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
|
|
1601
|
+
* function converts RGBA data pixel-by-pixel to a single channel value by
|
|
1602
|
+
* invoking a conversion function (see class comment for detail).
|
|
1603
|
+
*
|
|
1604
|
+
* @param type The type of image to return.
|
|
1605
|
+
* @param converter A set of conversion functions that will be invoked to
|
|
1606
|
+
* convert the underlying pixel data if necessary. You may omit this
|
|
1607
|
+
* function if the requested conversion does not change the pixel format.
|
|
1608
|
+
* @return The current image as a Float32Array.
|
|
1609
|
+
*/
|
|
1610
|
+
getImage(type: MPImageStorageType.FLOAT32_ARRAY, converter?: MPImageChannelConverter): Float32Array;
|
|
1611
|
+
/**
|
|
1612
|
+
* Returns the underlying image as an `ImageData` object. Note that this
|
|
1613
|
+
* involves an expensive GPU to CPU transfer if the current image is only
|
|
1614
|
+
* available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
|
|
1615
|
+
* function converts single channel pixel values to RGBA by invoking a
|
|
1616
|
+
* conversion function (see class comment for detail).
|
|
1617
|
+
*
|
|
1618
|
+
* @return The current image as an ImageData object.
|
|
1619
|
+
*/
|
|
1620
|
+
getImage(type: MPImageStorageType.IMAGE_DATA, converter?: MPImageChannelConverter): ImageData;
|
|
1621
|
+
/**
|
|
1622
|
+
* Returns the underlying image as an `ImageBitmap`. Note that
|
|
1623
|
+
* conversions to `ImageBitmap` are expensive, especially if the data
|
|
1624
|
+
* currently resides on CPU. If necessary, this function first converts single
|
|
1625
|
+
* channel pixel values to RGBA by invoking a conversion function (see class
|
|
1626
|
+
* comment for detail).
|
|
1627
|
+
*
|
|
1628
|
+
* Processing with `ImageBitmap`s requires that the MediaPipe Task was
|
|
1629
|
+
* initialized with an `OffscreenCanvas` with WebGL2 support. See
|
|
1630
|
+
* https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
|
|
1631
|
+
* for a list of supported platforms.
|
|
1632
|
+
*
|
|
1633
|
+
* @param type The type of image to return.
|
|
1634
|
+
* @param converter A set of conversion functions that will be invoked to
|
|
1635
|
+
* convert the underlying pixel data if necessary. You may omit this
|
|
1636
|
+
* function if the requested conversion does not change the pixel format.
|
|
1637
|
+
* @return The current image as an ImageBitmap object.
|
|
1638
|
+
*/
|
|
1639
|
+
getImage(type: MPImageStorageType.IMAGE_BITMAP, converter?: MPImageChannelConverter): ImageBitmap;
|
|
1640
|
+
/**
|
|
1641
|
+
* Returns the underlying image as a `WebGLTexture` object. Note that this
|
|
1642
|
+
* involves a CPU to GPU transfer if the current image is only available as
|
|
1643
|
+
* an `ImageData` object. The returned texture is bound to the current
|
|
1644
|
+
* canvas (see `.canvas`).
|
|
1645
|
+
*
|
|
1646
|
+
* @param type The type of image to return.
|
|
1647
|
+
* @param converter A set of conversion functions that will be invoked to
|
|
1648
|
+
* convert the underlying pixel data if necessary. You may omit this
|
|
1649
|
+
* function if the requested conversion does not change the pixel format.
|
|
1650
|
+
* @return The current image as a WebGLTexture.
|
|
1651
|
+
*/
|
|
1652
|
+
getImage(type: MPImageStorageType.WEBGL_TEXTURE, converter?: MPImageChannelConverter): WebGLTexture;
|
|
1653
|
+
/**
|
|
1654
|
+
* Creates a copy of the resources stored in this `MPImage`. You can invoke
|
|
1655
|
+
* this method to extend the lifetime of an image returned by a MediaPipe
|
|
1656
|
+
* Task. Note that performance critical applications should aim to only use
|
|
1657
|
+
* the `MPImage` within the MediaPipe Task callback so that copies can be
|
|
1658
|
+
* avoided.
|
|
1659
|
+
*/
|
|
1660
|
+
clone(): MPImage;
|
|
1661
|
+
/**
|
|
1662
|
+
* Frees up any resources owned by this `MPImage` instance.
|
|
1663
|
+
*
|
|
1664
|
+
* Note that this method does not free images that are owned by the C++
|
|
1665
|
+
* Task, as these are freed automatically once you leave the MediaPipe
|
|
1666
|
+
* callback. Additionally, some shared state is freed only once you invoke the
|
|
1667
|
+
* Task's `close()` method.
|
|
1668
|
+
*/
|
|
1669
|
+
close(): void;
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
/**
|
|
1673
|
+
* An interface that can be used to provide custom conversion functions. These
|
|
1674
|
+
* functions are invoked to convert pixel values between different channel
|
|
1675
|
+
* counts and value ranges. Any conversion function that is not specified will
|
|
1676
|
+
* result in a default conversion.
|
|
1677
|
+
*/
|
|
1678
|
+
export declare interface MPImageChannelConverter {
|
|
1679
|
+
/**
|
|
1680
|
+
* A conversion function to convert a number in the [0.0, 1.0] range to RGBA.
|
|
1681
|
+
* The output is an array with four elemeents whose values range from 0 to 255
|
|
1682
|
+
* inclusive.
|
|
1683
|
+
*
|
|
1684
|
+
* The default conversion function is `[v * 255, v * 255, v * 255, 255]`
|
|
1685
|
+
* and will log a warning if invoked.
|
|
1686
|
+
*/
|
|
1687
|
+
floatToRGBAConverter?: (value: number) => [
|
|
1688
|
+
number,
|
|
1689
|
+
number,
|
|
1690
|
+
number,
|
|
1691
|
+
number
|
|
1692
|
+
];
|
|
1693
|
+
uint8ToRGBAConverter?: (value: number) => [
|
|
1694
|
+
number,
|
|
1695
|
+
number,
|
|
1696
|
+
number,
|
|
1697
|
+
number
|
|
1698
|
+
];
|
|
1699
|
+
/**
|
|
1700
|
+
* A conversion function to convert an RGBA value in the range of 0 to 255 to
|
|
1701
|
+
* a single value in the [0.0, 1.0] range.
|
|
1702
|
+
*
|
|
1703
|
+
* The default conversion function is `(r / 3 + g / 3 + b / 3) / 255` and will
|
|
1704
|
+
* log a warning if invoked.
|
|
1705
|
+
*/
|
|
1706
|
+
rgbaToFloatConverter?: (r: number, g: number, b: number, a: number) => number;
|
|
1707
|
+
/**
|
|
1708
|
+
* A conversion function to convert an RGBA value in the range of 0 to 255 to
|
|
1709
|
+
* a single value in the [0, 255] range.
|
|
1710
|
+
*
|
|
1711
|
+
* The default conversion function is `r / 3 + g / 3 + b / 3` and will log a
|
|
1712
|
+
* warning if invoked.
|
|
1713
|
+
*/
|
|
1714
|
+
rgbaToUint8Converter?: (r: number, g: number, b: number, a: number) => number;
|
|
1715
|
+
/**
|
|
1716
|
+
* A conversion function to convert a single value in the 0.0 to 1.0 range to
|
|
1717
|
+
* [0, 255].
|
|
1718
|
+
*
|
|
1719
|
+
* The default conversion function is `r * 255` and will log a warning if
|
|
1720
|
+
* invoked.
|
|
1721
|
+
*/
|
|
1722
|
+
floatToUint8Converter?: (value: number) => number;
|
|
1723
|
+
/**
|
|
1724
|
+
* A conversion function to convert a single value in the 0 to 255 range to
|
|
1725
|
+
* [0.0, 1.0] .
|
|
1726
|
+
*
|
|
1727
|
+
* The default conversion function is `r / 255` and will log a warning if
|
|
1728
|
+
* invoked.
|
|
1729
|
+
*/
|
|
1730
|
+
uint8ToFloatConverter?: (value: number) => number;
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1733
|
+
/** The supported image formats. For internal usage. */
|
|
1734
|
+
export declare type MPImageNativeContainer = Uint8ClampedArray | Float32Array | ImageData | ImageBitmap | WebGLTexture;
|
|
1735
|
+
|
|
1736
|
+
/**
|
|
1737
|
+
* A class that encapsulates the shaders used by an MPImage. Can be re-used
|
|
1738
|
+
* across MPImages that use the same WebGL2Rendering context.
|
|
1739
|
+
*/
|
|
1740
|
+
export declare class MPImageShaderContext {
|
|
1741
|
+
/** Runs the callback using the shader. */
|
|
1742
|
+
run<T>(gl: WebGL2RenderingContext, flipVertically: boolean, callback: () => T): T;
|
|
1743
|
+
/**
|
|
1744
|
+
* Binds a framebuffer to the canvas. If the framebuffer does not yet exist,
|
|
1745
|
+
* creates it first. Binds the provided texture to the framebuffer.
|
|
1746
|
+
*/
|
|
1747
|
+
bindFramebuffer(gl: WebGL2RenderingContext, texture: WebGLTexture): void;
|
|
1748
|
+
unbindFramebuffer(): void;
|
|
1749
|
+
close(): void;
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
/**
|
|
1753
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
1754
|
+
*
|
|
1755
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1756
|
+
* you may not use this file except in compliance with the License.
|
|
1757
|
+
* You may obtain a copy of the License at
|
|
1758
|
+
*
|
|
1759
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1760
|
+
*
|
|
1761
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
1762
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1763
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1764
|
+
* See the License for the specific language governing permissions and
|
|
1765
|
+
* limitations under the License.
|
|
1766
|
+
*/
|
|
1767
|
+
/** The underlying type of the image. */
|
|
1768
|
+
export declare enum MPImageStorageType {
|
|
1769
|
+
/** Represents the native `UInt8ClampedArray` type. */
|
|
1770
|
+
UINT8_CLAMPED_ARRAY = 0,
|
|
1771
|
+
/**
|
|
1772
|
+
* Represents the native `Float32Array` type. Values range from [0.0, 1.0].
|
|
1773
|
+
*/
|
|
1774
|
+
FLOAT32_ARRAY = 1,
|
|
1775
|
+
/** Represents the native `ImageData` type. */
|
|
1776
|
+
IMAGE_DATA = 2,
|
|
1777
|
+
/** Represents the native `ImageBitmap` type. */
|
|
1778
|
+
IMAGE_BITMAP = 3,
|
|
1779
|
+
/** Represents the native `WebGLTexture` type. */
|
|
1780
|
+
WEBGL_TEXTURE = 4
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
/**
|
|
1784
|
+
* Copyright 2023 The MediaPipe Authors.
|
|
1543
1785
|
*
|
|
1544
1786
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1545
1787
|
* you may not use this file except in compliance with the License.
|
|
@@ -1569,7 +1811,7 @@ declare interface NormalizedKeypoint {
|
|
|
1569
1811
|
}
|
|
1570
1812
|
|
|
1571
1813
|
/**
|
|
1572
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
1814
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
1573
1815
|
*
|
|
1574
1816
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1575
1817
|
* you may not use this file except in compliance with the License.
|
|
@@ -1667,6 +1909,155 @@ export declare class ObjectDetector extends VisionTaskRunner {
|
|
|
1667
1909
|
export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
|
|
1668
1910
|
}
|
|
1669
1911
|
|
|
1912
|
+
/** Performs pose landmarks detection on images. */
|
|
1913
|
+
export declare class PoseLandmarker extends VisionTaskRunner {
|
|
1914
|
+
/**
|
|
1915
|
+
* An array containing the pairs of pose landmark indices to be rendered with
|
|
1916
|
+
* connections.
|
|
1917
|
+
*/
|
|
1918
|
+
static POSE_CONNECTIONS: Connection[];
|
|
1919
|
+
/**
|
|
1920
|
+
* Initializes the Wasm runtime and creates a new `PoseLandmarker` from the
|
|
1921
|
+
* provided options.
|
|
1922
|
+
* @param wasmFileset A configuration object that provides the location of the
|
|
1923
|
+
* Wasm binary and its loader.
|
|
1924
|
+
* @param poseLandmarkerOptions The options for the PoseLandmarker.
|
|
1925
|
+
* Note that either a path to the model asset or a model buffer needs to
|
|
1926
|
+
* be provided (via `baseOptions`).
|
|
1927
|
+
*/
|
|
1928
|
+
static createFromOptions(wasmFileset: WasmFileset, poseLandmarkerOptions: PoseLandmarkerOptions): Promise<PoseLandmarker>;
|
|
1929
|
+
/**
|
|
1930
|
+
* Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
|
|
1931
|
+
* the provided model asset buffer.
|
|
1932
|
+
* @param wasmFileset A configuration object that provides the location of the
|
|
1933
|
+
* Wasm binary and its loader.
|
|
1934
|
+
* @param modelAssetBuffer A binary representation of the model.
|
|
1935
|
+
*/
|
|
1936
|
+
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<PoseLandmarker>;
|
|
1937
|
+
/**
|
|
1938
|
+
* Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
|
|
1939
|
+
* the path to the model asset.
|
|
1940
|
+
* @param wasmFileset A configuration object that provides the location of the
|
|
1941
|
+
* Wasm binary and its loader.
|
|
1942
|
+
* @param modelAssetPath The path to the model asset.
|
|
1943
|
+
*/
|
|
1944
|
+
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<PoseLandmarker>;
|
|
1945
|
+
private constructor();
|
|
1946
|
+
/**
|
|
1947
|
+
* Sets new options for this `PoseLandmarker`.
|
|
1948
|
+
*
|
|
1949
|
+
* Calling `setOptions()` with a subset of options only affects those options.
|
|
1950
|
+
* You can reset an option back to its default value by explicitly setting it
|
|
1951
|
+
* to `undefined`.
|
|
1952
|
+
*
|
|
1953
|
+
* @param options The options for the pose landmarker.
|
|
1954
|
+
*/
|
|
1955
|
+
setOptions(options: PoseLandmarkerOptions): Promise<void>;
|
|
1956
|
+
/**
|
|
1957
|
+
* Performs pose detection on the provided single image and waits
|
|
1958
|
+
* synchronously for the response. Only use this method when the
|
|
1959
|
+
* PoseLandmarker is created with running mode `image`.
|
|
1960
|
+
*
|
|
1961
|
+
* @param image An image to process.
|
|
1962
|
+
* @param callback The callback that is invoked with the result. The
|
|
1963
|
+
* lifetime of the returned masks is only guaranteed for the duration of
|
|
1964
|
+
* the callback.
|
|
1965
|
+
* @return The detected pose landmarks.
|
|
1966
|
+
*/
|
|
1967
|
+
detect(image: ImageSource, callback: PoseLandmarkerCallback): void;
|
|
1968
|
+
/**
|
|
1969
|
+
* Performs pose detection on the provided single image and waits
|
|
1970
|
+
* synchronously for the response. Only use this method when the
|
|
1971
|
+
* PoseLandmarker is created with running mode `image`.
|
|
1972
|
+
*
|
|
1973
|
+
* @param image An image to process.
|
|
1974
|
+
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1975
|
+
* to process the input image before running inference.
|
|
1976
|
+
* @param callback The callback that is invoked with the result. The
|
|
1977
|
+
* lifetime of the returned masks is only guaranteed for the duration of
|
|
1978
|
+
* the callback.
|
|
1979
|
+
* @return The detected pose landmarks.
|
|
1980
|
+
*/
|
|
1981
|
+
detect(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: PoseLandmarkerCallback): void;
|
|
1982
|
+
/**
|
|
1983
|
+
* Performs pose detection on the provided video frame and waits
|
|
1984
|
+
* synchronously for the response. Only use this method when the
|
|
1985
|
+
* PoseLandmarker is created with running mode `video`.
|
|
1986
|
+
*
|
|
1987
|
+
* @param videoFrame A video frame to process.
|
|
1988
|
+
* @param timestamp The timestamp of the current frame, in ms.
|
|
1989
|
+
* @param callback The callback that is invoked with the result. The
|
|
1990
|
+
* lifetime of the returned masks is only guaranteed for the duration of
|
|
1991
|
+
* the callback.
|
|
1992
|
+
* @return The detected pose landmarks.
|
|
1993
|
+
*/
|
|
1994
|
+
detectForVideo(videoFrame: ImageSource, timestamp: number, callback: PoseLandmarkerCallback): void;
|
|
1995
|
+
/**
|
|
1996
|
+
* Performs pose detection on the provided video frame and waits
|
|
1997
|
+
* synchronously for the response. Only use this method when the
|
|
1998
|
+
* PoseLandmarker is created with running mode `video`.
|
|
1999
|
+
*
|
|
2000
|
+
* @param videoFrame A video frame to process.
|
|
2001
|
+
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
2002
|
+
* to process the input image before running inference.
|
|
2003
|
+
* @param timestamp The timestamp of the current frame, in ms.
|
|
2004
|
+
* @param callback The callback that is invoked with the result. The
|
|
2005
|
+
* lifetime of the returned masks is only guaranteed for the duration of
|
|
2006
|
+
* the callback.
|
|
2007
|
+
* @return The detected pose landmarks.
|
|
2008
|
+
*/
|
|
2009
|
+
detectForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: PoseLandmarkerCallback): void;
|
|
2010
|
+
}
|
|
2011
|
+
|
|
2012
|
+
/**
|
|
2013
|
+
* A callback that receives the result from the pose detector. The returned
|
|
2014
|
+
* masks are only valid for the duration of the callback. If asynchronous
|
|
2015
|
+
* processing is needed, the masks need to be copied before the callback
|
|
2016
|
+
* returns.
|
|
2017
|
+
*/
|
|
2018
|
+
export declare type PoseLandmarkerCallback = (result: PoseLandmarkerResult) => void;
|
|
2019
|
+
|
|
2020
|
+
/** Options to configure the MediaPipe PoseLandmarker Task */
|
|
2021
|
+
export declare interface PoseLandmarkerOptions extends VisionTaskOptions {
|
|
2022
|
+
/**
|
|
2023
|
+
* The maximum number of poses can be detected by the PoseLandmarker.
|
|
2024
|
+
* Defaults to 1.
|
|
2025
|
+
*/
|
|
2026
|
+
numPoses?: number | undefined;
|
|
2027
|
+
/**
|
|
2028
|
+
* The minimum confidence score for the pose detection to be considered
|
|
2029
|
+
* successful. Defaults to 0.5.
|
|
2030
|
+
*/
|
|
2031
|
+
minPoseDetectionConfidence?: number | undefined;
|
|
2032
|
+
/**
|
|
2033
|
+
* The minimum confidence score of pose presence score in the pose landmark
|
|
2034
|
+
* detection. Defaults to 0.5.
|
|
2035
|
+
*/
|
|
2036
|
+
minPosePresenceConfidence?: number | undefined;
|
|
2037
|
+
/**
|
|
2038
|
+
* The minimum confidence score for the pose tracking to be considered
|
|
2039
|
+
* successful. Defaults to 0.5.
|
|
2040
|
+
*/
|
|
2041
|
+
minTrackingConfidence?: number | undefined;
|
|
2042
|
+
/** Whether to output segmentation masks. Defaults to false. */
|
|
2043
|
+
outputSegmentationMasks?: boolean | undefined;
|
|
2044
|
+
}
|
|
2045
|
+
|
|
2046
|
+
/**
|
|
2047
|
+
* Represents the pose landmarks deection results generated by `PoseLandmarker`.
|
|
2048
|
+
* Each vector element represents a single pose detected in the image.
|
|
2049
|
+
*/
|
|
2050
|
+
export declare interface PoseLandmarkerResult {
|
|
2051
|
+
/** Pose landmarks of detected poses. */
|
|
2052
|
+
landmarks: NormalizedLandmark[];
|
|
2053
|
+
/** Pose landmarks in world coordinates of detected poses. */
|
|
2054
|
+
worldLandmarks: Landmark[];
|
|
2055
|
+
/** Detected auxiliary landmarks, used for deriving ROI for next frame. */
|
|
2056
|
+
auxilaryLandmarks: NormalizedLandmark[];
|
|
2057
|
+
/** Segmentation mask for the detected pose. */
|
|
2058
|
+
segmentationMasks?: Float32Array[] | WebGLTexture[];
|
|
2059
|
+
}
|
|
2060
|
+
|
|
1670
2061
|
/**
|
|
1671
2062
|
* Defines a rectangle, used e.g. as part of detection results or as input
|
|
1672
2063
|
* region-of-interest.
|
|
@@ -1743,7 +2134,7 @@ declare abstract class VisionTaskRunner extends TaskRunner {
|
|
|
1743
2134
|
}
|
|
1744
2135
|
|
|
1745
2136
|
/**
|
|
1746
|
-
* Copyright 2022 The MediaPipe Authors.
|
|
2137
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
1747
2138
|
*
|
|
1748
2139
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1749
2140
|
* you may not use this file except in compliance with the License.
|