@mediapipe/tasks-vision 0.0.0-nightly-20230920

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/vision.d.ts ADDED
@@ -0,0 +1,2407 @@
1
+ /**
2
+ * Copyright 2022 The MediaPipe Authors.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+ /** Options to configure MediaPipe model loading and processing. */
17
+ declare interface BaseOptions_2 {
18
+ /**
19
+ * The model path to the model asset file. Only one of `modelAssetPath` or
20
+ * `modelAssetBuffer` can be set.
21
+ */
22
+ modelAssetPath?: string | undefined;
23
+ /**
24
+ * A buffer containing the model aaset. Only one of `modelAssetPath` or
25
+ * `modelAssetBuffer` can be set.
26
+ */
27
+ modelAssetBuffer?: Uint8Array | undefined;
28
+ /** Overrides the default backend to use for the provided model. */
29
+ delegate?: "CPU" | "GPU" | undefined;
30
+ }
31
+
32
+ /**
33
+ * Copyright 2023 The MediaPipe Authors.
34
+ *
35
+ * Licensed under the Apache License, Version 2.0 (the "License");
36
+ * you may not use this file except in compliance with the License.
37
+ * You may obtain a copy of the License at
38
+ *
39
+ * http://www.apache.org/licenses/LICENSE-2.0
40
+ *
41
+ * Unless required by applicable law or agreed to in writing, software
42
+ * distributed under the License is distributed on an "AS IS" BASIS,
43
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
44
+ * See the License for the specific language governing permissions and
45
+ * limitations under the License.
46
+ */
47
+ /** An integer bounding box, axis aligned. */
48
+ export declare interface BoundingBox {
49
+ /** The X coordinate of the top-left corner, in pixels. */
50
+ originX: number;
51
+ /** The Y coordinate of the top-left corner, in pixels. */
52
+ originY: number;
53
+ /** The width of the bounding box, in pixels. */
54
+ width: number;
55
+ /** The height of the bounding box, in pixels. */
56
+ height: number;
57
+ /**
58
+ * Angle of rotation of the original non-rotated box around the top left
59
+ * corner of the original non-rotated box, in clockwise degrees from the
60
+ * horizontal.
61
+ */
62
+ angle: number;
63
+ }
64
+
65
+ /**
66
+ * A user-defined callback to take input data and map it to a custom output
67
+ * value.
68
+ */
69
+ export declare type Callback<I, O> = (input: I) => O;
70
+
71
+ /**
72
+ * Copyright 2022 The MediaPipe Authors.
73
+ *
74
+ * Licensed under the Apache License, Version 2.0 (the "License");
75
+ * you may not use this file except in compliance with the License.
76
+ * You may obtain a copy of the License at
77
+ *
78
+ * http://www.apache.org/licenses/LICENSE-2.0
79
+ *
80
+ * Unless required by applicable law or agreed to in writing, software
81
+ * distributed under the License is distributed on an "AS IS" BASIS,
82
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
83
+ * See the License for the specific language governing permissions and
84
+ * limitations under the License.
85
+ */
86
+ /** A classification category. */
87
+ export declare interface Category {
88
+ /** The probability score of this label category. */
89
+ score: number;
90
+ /** The index of the category in the corresponding label file. */
91
+ index: number;
92
+ /**
93
+ * The label of this category object. Defaults to an empty string if there is
94
+ * no category.
95
+ */
96
+ categoryName: string;
97
+ /**
98
+ * The display name of the label, which may be translated for different
99
+ * locales. For example, a label, "apple", may be translated into Spanish for
100
+ * display purpose, so that the `display_name` is "manzana". Defaults to an
101
+ * empty string if there is no display name.
102
+ */
103
+ displayName: string;
104
+ }
105
+
106
+ /** Classification results for a given classifier head. */
107
+ export declare interface Classifications {
108
+ /**
109
+ * The array of predicted categories, usually sorted by descending scores,
110
+ * e.g., from high to low probability.
111
+ */
112
+ categories: Category[];
113
+ /**
114
+ * The index of the classifier head these categories refer to. This is
115
+ * useful for multi-head models.
116
+ */
117
+ headIndex: number;
118
+ /**
119
+ * The name of the classifier head, which is the corresponding tensor
120
+ * metadata name. Defaults to an empty string if there is no such metadata.
121
+ */
122
+ headName: string;
123
+ }
124
+
125
+ /**
126
+ * Copyright 2022 The MediaPipe Authors.
127
+ *
128
+ * Licensed under the Apache License, Version 2.0 (the "License");
129
+ * you may not use this file except in compliance with the License.
130
+ * You may obtain a copy of the License at
131
+ *
132
+ * http://www.apache.org/licenses/LICENSE-2.0
133
+ *
134
+ * Unless required by applicable law or agreed to in writing, software
135
+ * distributed under the License is distributed on an "AS IS" BASIS,
136
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
137
+ * See the License for the specific language governing permissions and
138
+ * limitations under the License.
139
+ */
140
+ /** Options to configure a MediaPipe Classifier Task. */
141
+ declare interface ClassifierOptions {
142
+ /**
143
+ * The locale to use for display names specified through the TFLite Model
144
+ * Metadata, if any. Defaults to English.
145
+ */
146
+ displayNamesLocale?: string | undefined;
147
+ /** The maximum number of top-scored detection results to return. */
148
+ maxResults?: number | undefined;
149
+ /**
150
+ * Overrides the value provided in the model metadata. Results below this
151
+ * value are rejected.
152
+ */
153
+ scoreThreshold?: number | undefined;
154
+ /**
155
+ * Allowlist of category names. If non-empty, detection results whose category
156
+ * name is not in this set will be filtered out. Duplicate or unknown category
157
+ * names are ignored. Mutually exclusive with `categoryDenylist`.
158
+ */
159
+ categoryAllowlist?: string[] | undefined;
160
+ /**
161
+ * Denylist of category names. If non-empty, detection results whose category
162
+ * name is in this set will be filtered out. Duplicate or unknown category
163
+ * names are ignored. Mutually exclusive with `categoryAllowlist`.
164
+ */
165
+ categoryDenylist?: string[] | undefined;
166
+ }
167
+
168
+ /** A connection between two landmarks. */
169
+ declare interface Connection {
170
+ start: number;
171
+ end: number;
172
+ }
173
+
174
+ /** Represents one detection by a detection task. */
175
+ export declare interface Detection {
176
+ /** A list of `Category` objects. */
177
+ categories: Category[];
178
+ /** The bounding box of the detected objects. */
179
+ boundingBox?: BoundingBox;
180
+ /**
181
+ * List of keypoints associated with the detection. Keypoints represent
182
+ * interesting points related to the detection. For example, the keypoints
183
+ * represent the eye, ear and mouth from face detection model. Or in the
184
+ * template matching detection, e.g. KNIFT, they can represent the feature
185
+ * points for template matching. Contains an empty list if no keypoints are
186
+ * detected.
187
+ */
188
+ keypoints: NormalizedKeypoint[];
189
+ }
190
+
191
+ /** Detection results of a model. */
192
+ declare interface DetectionResult {
193
+ /** A list of Detections. */
194
+ detections: Detection[];
195
+ }
196
+ export { DetectionResult as FaceDetectorResult }
197
+ export { DetectionResult as ObjectDetectorResult }
198
+
199
+ /**
200
+ * Options for customizing the drawing routines
201
+ */
202
+ export declare interface DrawingOptions {
203
+ /** The color that is used to draw the shape. Defaults to white. */
204
+ color?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
205
+ /**
206
+ * The color that is used to fill the shape. Defaults to `.color` (or black
207
+ * if color is not set).
208
+ */
209
+ fillColor?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
210
+ /** The width of the line boundary of the shape. Defaults to 4. */
211
+ lineWidth?: number | Callback<LandmarkData, number>;
212
+ /** The radius of location marker. Defaults to 6. */
213
+ radius?: number | Callback<LandmarkData, number>;
214
+ }
215
+
216
+ /** Helper class to visualize the result of a MediaPipe Vision task. */
217
+ export declare class DrawingUtils {
218
+ /**
219
+ * Creates a new DrawingUtils class.
220
+ *
221
+ * @param ctx The canvas to render onto.
222
+ */
223
+ constructor(ctx: CanvasRenderingContext2D);
224
+ /**
225
+ * Restricts a number between two endpoints (order doesn't matter).
226
+ *
227
+ * @export
228
+ * @param x The number to clamp.
229
+ * @param x0 The first boundary.
230
+ * @param x1 The second boundary.
231
+ * @return The clamped value.
232
+ */
233
+ static clamp(x: number, x0: number, x1: number): number;
234
+ /**
235
+ * Linearly interpolates a value between two points, clamping that value to
236
+ * the endpoints.
237
+ *
238
+ * @export
239
+ * @param x The number to interpolate.
240
+ * @param x0 The x coordinate of the start value.
241
+ * @param x1 The x coordinate of the end value.
242
+ * @param y0 The y coordinate of the start value.
243
+ * @param y1 The y coordinate of the end value.
244
+ * @return The interpolated value.
245
+ */
246
+ static lerp(x: number, x0: number, x1: number, y0: number, y1: number): number;
247
+ /**
248
+ * Draws circles onto the provided landmarks.
249
+ *
250
+ * @export
251
+ * @param landmarks The landmarks to draw.
252
+ * @param style The style to visualize the landmarks.
253
+ */
254
+ drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions): void;
255
+ /**
256
+ * Draws lines between landmarks (given a connection graph).
257
+ *
258
+ * @export
259
+ * @param landmarks The landmarks to draw.
260
+ * @param connections The connections array that contains the start and the
261
+ * end indices for the connections to draw.
262
+ * @param style The style to visualize the landmarks.
263
+ */
264
+ drawConnectors(landmarks?: NormalizedLandmark[], connections?: Connection[], style?: DrawingOptions): void;
265
+ /**
266
+ * Draws a bounding box.
267
+ *
268
+ * @export
269
+ * @param boundingBox The bounding box to draw.
270
+ * @param style The style to visualize the boundin box.
271
+ */
272
+ drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void;
273
+ }
274
+
275
+ /**
276
+ * Copyright 2022 The MediaPipe Authors.
277
+ *
278
+ * Licensed under the Apache License, Version 2.0 (the "License");
279
+ * you may not use this file except in compliance with the License.
280
+ * You may obtain a copy of the License at
281
+ *
282
+ * http://www.apache.org/licenses/LICENSE-2.0
283
+ *
284
+ * Unless required by applicable law or agreed to in writing, software
285
+ * distributed under the License is distributed on an "AS IS" BASIS,
286
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
287
+ * See the License for the specific language governing permissions and
288
+ * limitations under the License.
289
+ */
290
+ /** Options to configure a MediaPipe Embedder Task */
291
+ declare interface EmbedderOptions {
292
+ /**
293
+ * Whether to normalize the returned feature vector with L2 norm. Use this
294
+ * option only if the model does not already contain a native L2_NORMALIZATION
295
+ * TF Lite Op. In most cases, this is already the case and L2 norm is thus
296
+ * achieved through TF Lite inference.
297
+ */
298
+ l2Normalize?: boolean | undefined;
299
+ /**
300
+ * Whether the returned embedding should be quantized to bytes via scalar
301
+ * quantization. Embeddings are implicitly assumed to be unit-norm and
302
+ * therefore any dimension is guaranteed to have a value in [-1.0, 1.0]. Use
303
+ * the l2_normalize option if this is not the case.
304
+ */
305
+ quantize?: boolean | undefined;
306
+ }
307
+
308
+ /**
309
+ * Copyright 2022 The MediaPipe Authors.
310
+ *
311
+ * Licensed under the Apache License, Version 2.0 (the "License");
312
+ * you may not use this file except in compliance with the License.
313
+ * You may obtain a copy of the License at
314
+ *
315
+ * http://www.apache.org/licenses/LICENSE-2.0
316
+ *
317
+ * Unless required by applicable law or agreed to in writing, software
318
+ * distributed under the License is distributed on an "AS IS" BASIS,
319
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
320
+ * See the License for the specific language governing permissions and
321
+ * limitations under the License.
322
+ */
323
+ /**
324
+ * List of embeddings with an optional timestamp.
325
+ *
326
+ * One and only one of the two 'floatEmbedding' and 'quantizedEmbedding' will
327
+ * contain data, based on whether or not the embedder was configured to perform
328
+ * scalar quantization.
329
+ */
330
+ export declare interface Embedding {
331
+ /**
332
+ * Floating-point embedding. Empty if the embedder was configured to perform
333
+ * scalar-quantization.
334
+ */
335
+ floatEmbedding?: number[];
336
+ /**
337
+ * Scalar-quantized embedding. Empty if the embedder was not configured to
338
+ * perform scalar quantization.
339
+ */
340
+ quantizedEmbedding?: Uint8Array;
341
+ /**
342
+ * The index of the classifier head these categories refer to. This is
343
+ * useful for multi-head models.
344
+ */
345
+ headIndex: number;
346
+ /**
347
+ * The name of the classifier head, which is the corresponding tensor
348
+ * metadata name.
349
+ */
350
+ headName: string;
351
+ }
352
+
353
+ /** Performs face detection on images. */
354
+ export declare class FaceDetector extends VisionTaskRunner {
355
+ /**
356
+ * Initializes the Wasm runtime and creates a new face detector from the
357
+ * provided options.
358
+ *
359
+ * @export
360
+ * @param wasmFileset A configuration object that provides the location of the
361
+ * Wasm binary and its loader.
362
+ * @param faceDetectorOptions The options for the FaceDetector. Note that
363
+ * either a path to the model asset or a model buffer needs to be
364
+ * provided (via `baseOptions`).
365
+ */
366
+ static createFromOptions(wasmFileset: WasmFileset, faceDetectorOptions: FaceDetectorOptions): Promise<FaceDetector>;
367
+ /**
368
+ * Initializes the Wasm runtime and creates a new face detector based on the
369
+ * provided model asset buffer.
370
+ *
371
+ * @export
372
+ * @param wasmFileset A configuration object that provides the location of the
373
+ * Wasm binary and its loader.
374
+ * @param modelAssetBuffer A binary representation of the model.
375
+ */
376
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceDetector>;
377
+ /**
378
+ * Initializes the Wasm runtime and creates a new face detector based on the
379
+ * path to the model asset.
380
+ *
381
+ * @export
382
+ * @param wasmFileset A configuration object that provides the location of the
383
+ * Wasm binary and its loader.
384
+ * @param modelAssetPath The path to the model asset.
385
+ */
386
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceDetector>;
387
+ private constructor();
388
+ /**
389
+ * Sets new options for the FaceDetector.
390
+ *
391
+ * Calling `setOptions()` with a subset of options only affects those options.
392
+ * You can reset an option back to its default value by explicitly setting it
393
+ * to `undefined`.
394
+ *
395
+ * @export
396
+ * @param options The options for the FaceDetector.
397
+ */
398
+ setOptions(options: FaceDetectorOptions): Promise<void>;
399
+ /**
400
+ * Performs face detection on the provided single image and waits
401
+ * synchronously for the response. Only use this method when the
402
+ * FaceDetector is created with running mode `image`.
403
+ *
404
+ * @param image An image to process.
405
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
406
+ * to process the input image before running inference.
407
+ * @return A result containing the list of detected faces.
408
+ */
409
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
410
+ /**
411
+ * Performs face detection on the provided video frame and waits
412
+ * synchronously for the response. Only use this method when the
413
+ * FaceDetector is created with running mode `video`.
414
+ *
415
+ * @export
416
+ * @param videoFrame A video frame to process.
417
+ * @param timestamp The timestamp of the current frame, in ms.
418
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
419
+ * to process the input image before running inference.
420
+ * @return A result containing the list of detected faces.
421
+ */
422
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
423
+ }
424
+
425
+ /** Options to configure the MediaPipe Face Detector Task */
426
+ export declare interface FaceDetectorOptions extends VisionTaskOptions {
427
+ /**
428
+ * The minimum confidence score for the face detection to be considered
429
+ * successful. Defaults to 0.5.
430
+ */
431
+ minDetectionConfidence?: number | undefined;
432
+ /**
433
+ * The minimum non-maximum-suppression threshold for face detection to be
434
+ * considered overlapped. Defaults to 0.3.
435
+ */
436
+ minSuppressionThreshold?: number | undefined;
437
+ }
438
+
439
+ /**
440
+ * Performs face landmarks detection on images.
441
+ *
442
+ * This API expects a pre-trained face landmarker model asset bundle.
443
+ */
444
+ export declare class FaceLandmarker extends VisionTaskRunner {
445
+ /**
446
+ * Initializes the Wasm runtime and creates a new `FaceLandmarker` from the
447
+ * provided options.
448
+ * @export
449
+ * @param wasmFileset A configuration object that provides the location of the
450
+ * Wasm binary and its loader.
451
+ * @param faceLandmarkerOptions The options for the FaceLandmarker.
452
+ * Note that either a path to the model asset or a model buffer needs to
453
+ * be provided (via `baseOptions`).
454
+ */
455
+ static createFromOptions(wasmFileset: WasmFileset, faceLandmarkerOptions: FaceLandmarkerOptions): Promise<FaceLandmarker>;
456
+ /**
457
+ * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
458
+ * the provided model asset buffer.
459
+ * @export
460
+ * @param wasmFileset A configuration object that provides the location of the
461
+ * Wasm binary and its loader.
462
+ * @param modelAssetBuffer A binary representation of the model.
463
+ */
464
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceLandmarker>;
465
+ /**
466
+ * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
467
+ * the path to the model asset.
468
+ * @export
469
+ * @param wasmFileset A configuration object that provides the location of the
470
+ * Wasm binary and its loader.
471
+ * @param modelAssetPath The path to the model asset.
472
+ */
473
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceLandmarker>;
474
+ /**
475
+ * Landmark connections to draw the connection between a face's lips.
476
+ * @export
477
+ * @nocollapse
478
+ */
479
+ static FACE_LANDMARKS_LIPS: Connection[];
480
+ /**
481
+ * Landmark connections to draw the connection between a face's left eye.
482
+ * @export
483
+ * @nocollapse
484
+ */
485
+ static FACE_LANDMARKS_LEFT_EYE: Connection[];
486
+ /**
487
+ * Landmark connections to draw the connection between a face's left eyebrow.
488
+ * @export
489
+ * @nocollapse
490
+ */
491
+ static FACE_LANDMARKS_LEFT_EYEBROW: Connection[];
492
+ /**
493
+ * Landmark connections to draw the connection between a face's left iris.
494
+ * @export
495
+ * @nocollapse
496
+ */
497
+ static FACE_LANDMARKS_LEFT_IRIS: Connection[];
498
+ /**
499
+ * Landmark connections to draw the connection between a face's right eye.
500
+ * @export
501
+ * @nocollapse
502
+ */
503
+ static FACE_LANDMARKS_RIGHT_EYE: Connection[];
504
+ /**
505
+ * Landmark connections to draw the connection between a face's right
506
+ * eyebrow.
507
+ * @export
508
+ * @nocollapse
509
+ */
510
+ static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[];
511
+ /**
512
+ * Landmark connections to draw the connection between a face's right iris.
513
+ * @export
514
+ * @nocollapse
515
+ */
516
+ static FACE_LANDMARKS_RIGHT_IRIS: Connection[];
517
+ /**
518
+ * Landmark connections to draw the face's oval.
519
+ * @export
520
+ * @nocollapse
521
+ */
522
+ static FACE_LANDMARKS_FACE_OVAL: Connection[];
523
+ /**
524
+ * Landmark connections to draw the face's contour.
525
+ * @export
526
+ * @nocollapse
527
+ */
528
+ static FACE_LANDMARKS_CONTOURS: Connection[];
529
+ /**
530
+ * Landmark connections to draw the face's tesselation.
531
+ * @export
532
+ * @nocollapse
533
+ */
534
+ static FACE_LANDMARKS_TESSELATION: Connection[];
535
+ private constructor();
536
+ /**
537
+ * Sets new options for this `FaceLandmarker`.
538
+ *
539
+ * Calling `setOptions()` with a subset of options only affects those options.
540
+ * You can reset an option back to its default value by explicitly setting it
541
+ * to `undefined`.
542
+ *
543
+ * @export
544
+ * @param options The options for the face landmarker.
545
+ */
546
+ setOptions(options: FaceLandmarkerOptions): Promise<void>;
547
+ /**
548
+ * Performs face landmarks detection on the provided single image and waits
549
+ * synchronously for the response. Only use this method when the
550
+ * FaceLandmarker is created with running mode `image`.
551
+ *
552
+ * @export
553
+ * @param image An image to process.
554
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
555
+ * to process the input image before running inference.
556
+ * @return The detected face landmarks.
557
+ */
558
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
559
+ /**
560
+ * Performs face landmarks detection on the provided video frame and waits
561
+ * synchronously for the response. Only use this method when the
562
+ * FaceLandmarker is created with running mode `video`.
563
+ *
564
+ * @export
565
+ * @param videoFrame A video frame to process.
566
+ * @param timestamp The timestamp of the current frame, in ms.
567
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
568
+ * to process the input image before running inference.
569
+ * @return The detected face landmarks.
570
+ */
571
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
572
+ }
573
+
574
+ /** Options to configure the MediaPipe FaceLandmarker Task */
575
+ export declare interface FaceLandmarkerOptions extends VisionTaskOptions {
576
+ /**
577
+ * The maximum number of faces can be detected by the FaceLandmarker.
578
+ * Defaults to 1.
579
+ */
580
+ numFaces?: number | undefined;
581
+ /**
582
+ * The minimum confidence score for the face detection to be considered
583
+ * successful. Defaults to 0.5.
584
+ */
585
+ minFaceDetectionConfidence?: number | undefined;
586
+ /**
587
+ * The minimum confidence score of face presence score in the face landmark
588
+ * detection. Defaults to 0.5.
589
+ */
590
+ minFacePresenceConfidence?: number | undefined;
591
+ /**
592
+ * The minimum confidence score for the face tracking to be considered
593
+ * successful. Defaults to 0.5.
594
+ */
595
+ minTrackingConfidence?: number | undefined;
596
+ /**
597
+ * Whether FaceLandmarker outputs face blendshapes classification. Face
598
+ * blendshapes are used for rendering the 3D face model.
599
+ */
600
+ outputFaceBlendshapes?: boolean | undefined;
601
+ /**
602
+ * Whether FaceLandmarker outputs facial transformation_matrix. Facial
603
+ * transformation matrix is used to transform the face landmarks in canonical
604
+ * face to the detected face, so that users can apply face effects on the
605
+ * detected landmarks.
606
+ */
607
+ outputFacialTransformationMatrixes?: boolean | undefined;
608
+ }
609
+
610
+ /**
611
+ * Represents the face landmarks deection results generated by `FaceLandmarker`.
612
+ */
613
+ export declare interface FaceLandmarkerResult {
614
+ /** Detected face landmarks in normalized image coordinates. */
615
+ faceLandmarks: NormalizedLandmark[][];
616
+ /** Optional face blendshapes results. */
617
+ faceBlendshapes: Classifications[];
618
+ /** Optional facial transformation matrix. */
619
+ facialTransformationMatrixes: Matrix[];
620
+ }
621
+
622
+ /** Performs face stylization on images. */
623
+ export declare class FaceStylizer extends VisionTaskRunner {
624
+ /**
625
+ * Initializes the Wasm runtime and creates a new Face Stylizer from the
626
+ * provided options.
627
+ * @export
628
+ * @param wasmFileset A configuration object that provides the location of
629
+ * the Wasm binary and its loader.
630
+ * @param faceStylizerOptions The options for the Face Stylizer. Note
631
+ * that either a path to the model asset or a model buffer needs to be
632
+ * provided (via `baseOptions`).
633
+ */
634
+ static createFromOptions(wasmFileset: WasmFileset, faceStylizerOptions: FaceStylizerOptions): Promise<FaceStylizer>;
635
+ /**
636
+ * Initializes the Wasm runtime and creates a new Face Stylizer based on
637
+ * the provided model asset buffer.
638
+ * @export
639
+ * @param wasmFileset A configuration object that provides the location of
640
+ * the Wasm binary and its loader.
641
+ * @param modelAssetBuffer A binary representation of the model.
642
+ */
643
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceStylizer>;
644
+ /**
645
+ * Initializes the Wasm runtime and creates a new Face Stylizer based on
646
+ * the path to the model asset.
647
+ * @export
648
+ * @param wasmFileset A configuration object that provides the location of
649
+ * the Wasm binary and its loader.
650
+ * @param modelAssetPath The path to the model asset.
651
+ */
652
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceStylizer>;
653
+ private constructor();
654
+ /**
655
+ * Sets new options for the Face Stylizer.
656
+ *
657
+ * Calling `setOptions()` with a subset of options only affects those
658
+ * options. You can reset an option back to its default value by
659
+ * explicitly setting it to `undefined`.
660
+ *
661
+ * @export
662
+ * @param options The options for the Face Stylizer.
663
+ */
664
+ setOptions(options: FaceStylizerOptions): Promise<void>;
665
+ /**
666
+ * Performs face stylization on the provided single image and invokes the
667
+ * callback with result. The method returns synchronously once the callback
668
+ * returns. Only use this method when the FaceStylizer is created with the
669
+ * image running mode.
670
+ *
671
+ * @param image An image to process.
672
+ * @param callback The callback that is invoked with the stylized image or
673
+ * `null` if no face was detected. The lifetime of the returned data is
674
+ * only guaranteed for the duration of the callback.
675
+ */
676
+ stylize(image: ImageSource, callback: FaceStylizerCallback): void;
677
+ /**
678
+ * Performs face stylization on the provided single image and invokes the
679
+ * callback with result. The method returns synchronously once the callback
680
+ * returns. Only use this method when the FaceStylizer is created with the
681
+ * image running mode.
682
+ *
683
+ * The 'imageProcessingOptions' parameter can be used to specify one or all
684
+ * of:
685
+ * - the rotation to apply to the image before performing stylization, by
686
+ * setting its 'rotationDegrees' property.
687
+ * - the region-of-interest on which to perform stylization, by setting its
688
+ * 'regionOfInterest' property. If not specified, the full image is used.
689
+ * If both are specified, the crop around the region-of-interest is extracted
690
+ * first, then the specified rotation is applied to the crop.
691
+ *
692
+ * @param image An image to process.
693
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
694
+ * to process the input image before running inference.
695
+ * @param callback The callback that is invoked with the stylized image or
696
+ * `null` if no face was detected. The lifetime of the returned data is
697
+ * only guaranteed for the duration of the callback.
698
+ */
699
+ stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: FaceStylizerCallback): void;
700
+ /**
701
+ * Performs face stylization on the provided single image and returns the
702
+ * result. This method creates a copy of the resulting image and should not be
703
+ * used in high-throughput applications. Only use this method when the
704
+ * FaceStylizer is created with the image running mode.
705
+ *
706
+ * @param image An image to process.
707
+ * @return A stylized face or `null` if no face was detected. The result is
708
+ * copied to avoid lifetime issues.
709
+ */
710
+ stylize(image: ImageSource): MPImage | null;
711
+ /**
712
+ * Performs face stylization on the provided single image and returns the
713
+ * result. This method creates a copy of the resulting image and should not be
714
+ * used in high-throughput applications. Only use this method when the
715
+ * FaceStylizer is created with the image running mode.
716
+ *
717
+ * The 'imageProcessingOptions' parameter can be used to specify one or all
718
+ * of:
719
+ * - the rotation to apply to the image before performing stylization, by
720
+ * setting its 'rotationDegrees' property.
721
+ * - the region-of-interest on which to perform stylization, by setting its
722
+ * 'regionOfInterest' property. If not specified, the full image is used.
723
+ * If both are specified, the crop around the region-of-interest is extracted
724
+ * first, then the specified rotation is applied to the crop.
725
+ *
726
+ * @param image An image to process.
727
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
728
+ * to process the input image before running inference.
729
+ * @return A stylized face or `null` if no face was detected. The result is
730
+ * copied to avoid lifetime issues.
731
+ */
732
+ stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions): MPImage | null;
733
+ }
734
+
735
+ /**
736
+ * A callback that receives an `MPImage` object from the face stylizer, or
737
+ * `null` if no face was detected. The lifetime of the underlying data is
738
+ * limited to the duration of the callback. If asynchronous processing is
739
+ * needed, all data needs to be copied before the callback returns (via
740
+ * `image.clone()`).
741
+ */
742
+ export declare type FaceStylizerCallback = (image: MPImage | null) => void;
743
+
744
+ /** Options to configure the MediaPipe Face Stylizer Task */
745
+ export declare interface FaceStylizerOptions extends VisionTaskOptions {
746
+ }
747
+
748
+ /**
749
+ * Resolves the files required for the MediaPipe Task APIs.
750
+ *
751
+ * This class verifies whether SIMD is supported in the current environment and
752
+ * loads the SIMD files only if support is detected. The returned filesets
753
+ * require that the Wasm files are published without renaming. If this is not
754
+ * possible, you can invoke the MediaPipe Tasks APIs using a manually created
755
+ * `WasmFileset`.
756
+ */
757
+ export declare class FilesetResolver {
758
+ /**
759
+ * Returns whether SIMD is supported in the current environment.
760
+ *
761
+ * If your environment requires custom locations for the MediaPipe Wasm files,
762
+ * you can use `isSimdSupported()` to decide whether to load the SIMD-based
763
+ * assets.
764
+ *
765
+ * @export
766
+ * @return Whether SIMD support was detected in the current environment.
767
+ */
768
+ static isSimdSupported(): Promise<boolean>;
769
+ /**
770
+ * Creates a fileset for the MediaPipe Audio tasks.
771
+ *
772
+ * @export
773
+ * @param basePath An optional base path to specify the directory the Wasm
774
+ * files should be loaded from. If not specified, the Wasm files are
775
+ * loaded from the host's root directory.
776
+ * @return A `WasmFileset` that can be used to initialize MediaPipe Audio
777
+ * tasks.
778
+ */
779
+ static forAudioTasks(basePath?: string): Promise<WasmFileset>;
780
+ /**
781
+ * Creates a fileset for the MediaPipe Text tasks.
782
+ *
783
+ * @export
784
+ * @param basePath An optional base path to specify the directory the Wasm
785
+ * files should be loaded from. If not specified, the Wasm files are
786
+ * loaded from the host's root directory.
787
+ * @return A `WasmFileset` that can be used to initialize MediaPipe Text
788
+ * tasks.
789
+ */
790
+ static forTextTasks(basePath?: string): Promise<WasmFileset>;
791
+ /**
792
+ * Creates a fileset for the MediaPipe Vision tasks.
793
+ *
794
+ * @export
795
+ * @param basePath An optional base path to specify the directory the Wasm
796
+ * files should be loaded from. If not specified, the Wasm files are
797
+ * loaded from the host's root directory.
798
+ * @return A `WasmFileset` that can be used to initialize MediaPipe Vision
799
+ * tasks.
800
+ */
801
+ static forVisionTasks(basePath?: string): Promise<WasmFileset>;
802
+ }
803
+
804
+ /** Performs hand gesture recognition on images. */
805
+ export declare class GestureRecognizer extends VisionTaskRunner {
806
+ /**
807
+ * An array containing the pairs of hand landmark indices to be rendered with
808
+ * connections.
809
+ * @export
810
+ * @nocollapse
811
+ */
812
+ static HAND_CONNECTIONS: Connection[];
813
+ /**
814
+ * Initializes the Wasm runtime and creates a new gesture recognizer from the
815
+ * provided options.
816
+ * @export
817
+ * @param wasmFileset A configuration object that provides the location of the
818
+ * Wasm binary and its loader.
819
+ * @param gestureRecognizerOptions The options for the gesture recognizer.
820
+ * Note that either a path to the model asset or a model buffer needs to
821
+ * be provided (via `baseOptions`).
822
+ */
823
+ static createFromOptions(wasmFileset: WasmFileset, gestureRecognizerOptions: GestureRecognizerOptions): Promise<GestureRecognizer>;
824
+ /**
825
+ * Initializes the Wasm runtime and creates a new gesture recognizer based on
826
+ * the provided model asset buffer.
827
+ * @export
828
+ * @param wasmFileset A configuration object that provides the location of the
829
+ * Wasm binary and its loader.
830
+ * @param modelAssetBuffer A binary representation of the model.
831
+ */
832
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<GestureRecognizer>;
833
+ /**
834
+ * Initializes the Wasm runtime and creates a new gesture recognizer based on
835
+ * the path to the model asset.
836
+ * @export
837
+ * @param wasmFileset A configuration object that provides the location of the
838
+ * Wasm binary and its loader.
839
+ * @param modelAssetPath The path to the model asset.
840
+ */
841
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<GestureRecognizer>;
842
+ private constructor();
843
+ /**
844
+ * Sets new options for the gesture recognizer.
845
+ *
846
+ * Calling `setOptions()` with a subset of options only affects those options.
847
+ * You can reset an option back to its default value by explicitly setting it
848
+ * to `undefined`.
849
+ *
850
+ * @export
851
+ * @param options The options for the gesture recognizer.
852
+ */
853
+ setOptions(options: GestureRecognizerOptions): Promise<void>;
854
+ /**
855
+ * Performs gesture recognition on the provided single image and waits
856
+ * synchronously for the response. Only use this method when the
857
+ * GestureRecognizer is created with running mode `image`.
858
+ *
859
+ * @export
860
+ * @param image A single image to process.
861
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
862
+ * to process the input image before running inference.
863
+ * @return The detected gestures.
864
+ */
865
+ recognize(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
866
+ /**
867
+ * Performs gesture recognition on the provided video frame and waits
868
+ * synchronously for the response. Only use this method when the
869
+ * GestureRecognizer is created with running mode `video`.
870
+ *
871
+ * @export
872
+ * @param videoFrame A video frame to process.
873
+ * @param timestamp The timestamp of the current frame, in ms.
874
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
875
+ * to process the input image before running inference.
876
+ * @return The detected gestures.
877
+ */
878
+ recognizeForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
879
+ }
880
+
881
+ /** Options to configure the MediaPipe Gesture Recognizer Task */
882
+ export declare interface GestureRecognizerOptions extends VisionTaskOptions {
883
+ /**
884
+ * The maximum number of hands can be detected by the GestureRecognizer.
885
+ * Defaults to 1.
886
+ */
887
+ numHands?: number | undefined;
888
+ /**
889
+ * The minimum confidence score for the hand detection to be considered
890
+ * successful. Defaults to 0.5.
891
+ */
892
+ minHandDetectionConfidence?: number | undefined;
893
+ /**
894
+ * The minimum confidence score of hand presence score in the hand landmark
895
+ * detection. Defaults to 0.5.
896
+ */
897
+ minHandPresenceConfidence?: number | undefined;
898
+ /**
899
+ * The minimum confidence score for the hand tracking to be considered
900
+ * successful. Defaults to 0.5.
901
+ */
902
+ minTrackingConfidence?: number | undefined;
903
+ /**
904
+ * Sets the optional `ClassifierOptions` controlling the canned gestures
905
+ * classifier, such as score threshold, allow list and deny list of gestures.
906
+ * The categories for canned gesture
907
+ * classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up",
908
+ * "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
909
+ */
910
+ cannedGesturesClassifierOptions?: ClassifierOptions | undefined;
911
+ /**
912
+ * Options for configuring the custom gestures classifier, such as score
913
+ * threshold, allow list and deny list of gestures.
914
+ */
915
+ customGesturesClassifierOptions?: ClassifierOptions | undefined;
916
+ }
917
+
918
+ /**
919
+ * Represents the gesture recognition results generated by `GestureRecognizer`.
920
+ */
921
+ export declare interface GestureRecognizerResult {
922
+ /** Hand landmarks of detected hands. */
923
+ landmarks: NormalizedLandmark[][];
924
+ /** Hand landmarks in world coordniates of detected hands. */
925
+ worldLandmarks: Landmark[][];
926
+ /** Handedness of detected hands. */
927
+ handedness: Category[][];
928
+ /**
929
+ * Handedness of detected hands.
930
+ * @deprecated Use `.handedness` instead.
931
+ */
932
+ handednesses: Category[][];
933
+ /**
934
+ * Recognized hand gestures of detected hands. Note that the index of the
935
+ * gesture is always -1, because the raw indices from multiple gesture
936
+ * classifiers cannot consolidate to a meaningful index.
937
+ */
938
+ gestures: Category[][];
939
+ }
940
+
941
+ /** Performs hand landmarks detection on images. */
942
+ export declare class HandLandmarker extends VisionTaskRunner {
943
+ /**
944
+ * An array containing the pairs of hand landmark indices to be rendered with
945
+ * connections.
946
+ * @export
947
+ * @nocollapse
948
+ */
949
+ static HAND_CONNECTIONS: Connection[];
950
+ /**
951
+ * Initializes the Wasm runtime and creates a new `HandLandmarker` from the
952
+ * provided options.
953
+ * @export
954
+ * @param wasmFileset A configuration object that provides the location of the
955
+ * Wasm binary and its loader.
956
+ * @param handLandmarkerOptions The options for the HandLandmarker.
957
+ * Note that either a path to the model asset or a model buffer needs to
958
+ * be provided (via `baseOptions`).
959
+ */
960
+ static createFromOptions(wasmFileset: WasmFileset, handLandmarkerOptions: HandLandmarkerOptions): Promise<HandLandmarker>;
961
+ /**
962
+ * Initializes the Wasm runtime and creates a new `HandLandmarker` based on
963
+ * the provided model asset buffer.
964
+ * @export
965
+ * @param wasmFileset A configuration object that provides the location of the
966
+ * Wasm binary and its loader.
967
+ * @param modelAssetBuffer A binary representation of the model.
968
+ */
969
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<HandLandmarker>;
970
+ /**
971
+ * Initializes the Wasm runtime and creates a new `HandLandmarker` based on
972
+ * the path to the model asset.
973
+ * @export
974
+ * @param wasmFileset A configuration object that provides the location of the
975
+ * Wasm binary and its loader.
976
+ * @param modelAssetPath The path to the model asset.
977
+ */
978
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<HandLandmarker>;
979
+ private constructor();
980
+ /**
981
+ * Sets new options for this `HandLandmarker`.
982
+ *
983
+ * Calling `setOptions()` with a subset of options only affects those options.
984
+ * You can reset an option back to its default value by explicitly setting it
985
+ * to `undefined`.
986
+ *
987
+ * @export
988
+ * @param options The options for the hand landmarker.
989
+ */
990
+ setOptions(options: HandLandmarkerOptions): Promise<void>;
991
+ /**
992
+ * Performs hand landmarks detection on the provided single image and waits
993
+ * synchronously for the response. Only use this method when the
994
+ * HandLandmarker is created with running mode `image`.
995
+ *
996
+ * @export
997
+ * @param image An image to process.
998
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
999
+ * to process the input image before running inference.
1000
+ * @return The detected hand landmarks.
1001
+ */
1002
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
1003
+ /**
1004
+ * Performs hand landmarks detection on the provided video frame and waits
1005
+ * synchronously for the response. Only use this method when the
1006
+ * HandLandmarker is created with running mode `video`.
1007
+ *
1008
+ * @export
1009
+ * @param videoFrame A video frame to process.
1010
+ * @param timestamp The timestamp of the current frame, in ms.
1011
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1012
+ * to process the input image before running inference.
1013
+ * @return The detected hand landmarks.
1014
+ */
1015
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
1016
+ }
1017
+
1018
+ /** Options to configure the MediaPipe HandLandmarker Task */
1019
+ export declare interface HandLandmarkerOptions extends VisionTaskOptions {
1020
+ /**
1021
+ * The maximum number of hands can be detected by the HandLandmarker.
1022
+ * Defaults to 1.
1023
+ */
1024
+ numHands?: number | undefined;
1025
+ /**
1026
+ * The minimum confidence score for the hand detection to be considered
1027
+ * successful. Defaults to 0.5.
1028
+ */
1029
+ minHandDetectionConfidence?: number | undefined;
1030
+ /**
1031
+ * The minimum confidence score of hand presence score in the hand landmark
1032
+ * detection. Defaults to 0.5.
1033
+ */
1034
+ minHandPresenceConfidence?: number | undefined;
1035
+ /**
1036
+ * The minimum confidence score for the hand tracking to be considered
1037
+ * successful. Defaults to 0.5.
1038
+ */
1039
+ minTrackingConfidence?: number | undefined;
1040
+ }
1041
+
1042
+ /**
1043
+ * Represents the hand landmarks deection results generated by `HandLandmarker`.
1044
+ */
1045
+ export declare interface HandLandmarkerResult {
1046
+ /** Hand landmarks of detected hands. */
1047
+ landmarks: NormalizedLandmark[][];
1048
+ /** Hand landmarks in world coordinates of detected hands. */
1049
+ worldLandmarks: Landmark[][];
1050
+ /**
1051
+ * Handedness of detected hands.
1052
+ * @deprecated Use `.handedness` instead.
1053
+ */
1054
+ handednesses: Category[][];
1055
+ /** Handedness of detected hands. */
1056
+ handedness: Category[][];
1057
+ }
1058
+
1059
+ /** Performs classification on images. */
1060
+ export declare class ImageClassifier extends VisionTaskRunner {
1061
+ /**
1062
+ * Initializes the Wasm runtime and creates a new image classifier from the
1063
+ * provided options.
1064
+ * @export
1065
+ * @param wasmFileset A configuration object that provides the location
1066
+ * Wasm binary and its loader.
1067
+ * @param imageClassifierOptions The options for the image classifier. Note
1068
+ * that either a path to the model asset or a model buffer needs to be
1069
+ * provided (via `baseOptions`).
1070
+ */
1071
+ static createFromOptions(wasmFileset: WasmFileset, imageClassifierOptions: ImageClassifierOptions): Promise<ImageClassifier>;
1072
+ /**
1073
+ * Initializes the Wasm runtime and creates a new image classifier based on
1074
+ * the provided model asset buffer.
1075
+ * @export
1076
+ * @param wasmFileset A configuration object that provides the location of the
1077
+ * Wasm binary and its loader.
1078
+ * @param modelAssetBuffer A binary representation of the model.
1079
+ */
1080
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageClassifier>;
1081
+ /**
1082
+ * Initializes the Wasm runtime and creates a new image classifier based on
1083
+ * the path to the model asset.
1084
+ * @export
1085
+ * @param wasmFileset A configuration object that provides the location of the
1086
+ * Wasm binary and its loader.
1087
+ * @param modelAssetPath The path to the model asset.
1088
+ */
1089
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageClassifier>;
1090
+ private constructor();
1091
+ /**
1092
+ * Sets new options for the image classifier.
1093
+ *
1094
+ * Calling `setOptions()` with a subset of options only affects those options.
1095
+ * You can reset an option back to its default value by explicitly setting it
1096
+ * to `undefined`.
1097
+ *
1098
+ * @export
1099
+ * @param options The options for the image classifier.
1100
+ */
1101
+ setOptions(options: ImageClassifierOptions): Promise<void>;
1102
+ /**
1103
+ * Performs image classification on the provided single image and waits
1104
+ * synchronously for the response. Only use this method when the
1105
+ * ImageClassifier is created with running mode `image`.
1106
+ *
1107
+ * @export
1108
+ * @param image An image to process.
1109
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1110
+ * to process the input image before running inference.
1111
+ * @return The classification result of the image
1112
+ */
1113
+ classify(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1114
+ /**
1115
+ * Performs image classification on the provided video frame and waits
1116
+ * synchronously for the response. Only use this method when the
1117
+ * ImageClassifier is created with running mode `video`.
1118
+ *
1119
+ * @export
1120
+ * @param videoFrame A video frame to process.
1121
+ * @param timestamp The timestamp of the current frame, in ms.
1122
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1123
+ * to process the input image before running inference.
1124
+ * @return The classification result of the image
1125
+ */
1126
+ classifyForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1127
+ }
1128
+
1129
+ /** Options to configure the MediaPipe Image Classifier Task. */
1130
+ export declare interface ImageClassifierOptions extends ClassifierOptions, VisionTaskOptions {
1131
+ }
1132
+
1133
+ /** Classification results of a model. */
1134
+ export declare interface ImageClassifierResult {
1135
+ /** The classification results for each head of the model. */
1136
+ classifications: Classifications[];
1137
+ /**
1138
+ * The optional timestamp (in milliseconds) of the start of the chunk of data
1139
+ * corresponding to these results.
1140
+ *
1141
+ * This is only used for classification on time series (e.g. audio
1142
+ * classification). In these use cases, the amount of data to process might
1143
+ * exceed the maximum size that the model can process: to solve this, the
1144
+ * input data is split into multiple chunks starting at different timestamps.
1145
+ */
1146
+ timestampMs?: number;
1147
+ }
1148
+
1149
+ /** Performs embedding extraction on images. */
1150
+ export declare class ImageEmbedder extends VisionTaskRunner {
1151
+ /**
1152
+ * Initializes the Wasm runtime and creates a new image embedder from the
1153
+ * provided options.
1154
+ * @export
1155
+ * @param wasmFileset A configuration object that provides the location of the
1156
+ * Wasm binary and its loader.
1157
+ * @param imageEmbedderOptions The options for the image embedder. Note that
1158
+ * either a path to the TFLite model or the model itself needs to be
1159
+ * provided (via `baseOptions`).
1160
+ */
1161
+ static createFromOptions(wasmFileset: WasmFileset, imageEmbedderOptions: ImageEmbedderOptions): Promise<ImageEmbedder>;
1162
+ /**
1163
+ * Initializes the Wasm runtime and creates a new image embedder based on the
1164
+ * provided model asset buffer.
1165
+ * @export
1166
+ * @param wasmFileset A configuration object that provides the location of the
1167
+ * Wasm binary and its loader.
1168
+ * @param modelAssetBuffer A binary representation of the TFLite model.
1169
+ */
1170
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageEmbedder>;
1171
+ /**
1172
+ * Initializes the Wasm runtime and creates a new image embedder based on the
1173
+ * path to the model asset.
1174
+ * @export
1175
+ * @param wasmFileset A configuration object that provides the location of the
1176
+ * Wasm binary and its loader.
1177
+ * @param modelAssetPath The path to the TFLite model.
1178
+ */
1179
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageEmbedder>;
1180
+ private constructor();
1181
+ /**
1182
+ * Sets new options for the image embedder.
1183
+ *
1184
+ * Calling `setOptions()` with a subset of options only affects those options.
1185
+ * You can reset an option back to its default value by explicitly setting it
1186
+ * to `undefined`.
1187
+ *
1188
+ * @export
1189
+ * @param options The options for the image embedder.
1190
+ */
1191
+ setOptions(options: ImageEmbedderOptions): Promise<void>;
1192
+ /**
1193
+ * Performs embedding extraction on the provided single image and waits
1194
+ * synchronously for the response. Only use this method when the
1195
+ * ImageEmbedder is created with running mode `image`.
1196
+ *
1197
+ * @export
1198
+ * @param image The image to process.
1199
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1200
+ * to process the input image before running inference.
1201
+ * @return The classification result of the image
1202
+ */
1203
+ embed(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1204
+ /**
1205
+ * Performs embedding extraction on the provided video frame and waits
1206
+ * synchronously for the response. Only use this method when the
1207
+ * ImageEmbedder is created with running mode `video`.
1208
+ *
1209
+ * @export
1210
+ * @param imageFrame The image frame to process.
1211
+ * @param timestamp The timestamp of the current frame, in ms.
1212
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1213
+ * to process the input image before running inference.
1214
+ * @return The classification result of the image
1215
+ */
1216
+ embedForVideo(imageFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1217
+ /**
1218
+ * Utility function to compute cosine similarity[1] between two `Embedding`
1219
+ * objects.
1220
+ *
1221
+ * [1]: https://en.wikipedia.org/wiki/Cosine_similarity
1222
+ *
1223
+ * @export
1224
+ * @throws if the embeddings are of different types(float vs. quantized), have
1225
+ * different sizes, or have an L2-norm of 0.
1226
+ */
1227
+ static cosineSimilarity(u: Embedding, v: Embedding): number;
1228
+ }
1229
+
1230
+ /** Options for configuring a MediaPipe Image Embedder task. */
1231
+ export declare interface ImageEmbedderOptions extends EmbedderOptions, VisionTaskOptions {
1232
+ }
1233
+
1234
+ /** Embedding results for a given embedder model. */
1235
+ export declare interface ImageEmbedderResult {
1236
+ /**
1237
+ * The embedding results for each model head, i.e. one for each output tensor.
1238
+ */
1239
+ embeddings: Embedding[];
1240
+ /**
1241
+ * The optional timestamp (in milliseconds) of the start of the chunk of
1242
+ * data corresponding to these results.
1243
+ *
1244
+ * This is only used for embedding extraction on time series (e.g. audio
1245
+ * embedding). In these use cases, the amount of data to process might
1246
+ * exceed the maximum size that the model can process: to solve this, the
1247
+ * input data is split into multiple chunks starting at different timestamps.
1248
+ */
1249
+ timestampMs?: number;
1250
+ }
1251
+
1252
+ /**
1253
+ * Options for image processing.
1254
+ *
1255
+ * If both region-or-interest and rotation are specified, the crop around the
1256
+ * region-of-interest is extracted first, then the specified rotation is applied
1257
+ * to the crop.
1258
+ */
1259
+ declare interface ImageProcessingOptions {
1260
+ /**
1261
+ * The optional region-of-interest to crop from the image. If not specified,
1262
+ * the full image is used.
1263
+ *
1264
+ * Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
1265
+ */
1266
+ regionOfInterest?: RectF;
1267
+ /**
1268
+ * The rotation to apply to the image (or cropped region-of-interest), in
1269
+ * degrees clockwise.
1270
+ *
1271
+ * The rotation must be a multiple (positive or negative) of 90°.
1272
+ */
1273
+ rotationDegrees?: number;
1274
+ }
1275
+
1276
+ /** Performs image segmentation on images. */
1277
+ export declare class ImageSegmenter extends VisionTaskRunner {
1278
+ /**
1279
+ * Initializes the Wasm runtime and creates a new image segmenter from the
1280
+ * provided options.
1281
+ * @export
1282
+ * @param wasmFileset A configuration object that provides the location of
1283
+ * the Wasm binary and its loader.
1284
+ * @param imageSegmenterOptions The options for the Image Segmenter. Note
1285
+ * that either a path to the model asset or a model buffer needs to be
1286
+ * provided (via `baseOptions`).
1287
+ */
1288
+ static createFromOptions(wasmFileset: WasmFileset, imageSegmenterOptions: ImageSegmenterOptions): Promise<ImageSegmenter>;
1289
+ /**
1290
+ * Initializes the Wasm runtime and creates a new image segmenter based on
1291
+ * the provided model asset buffer.
1292
+ * @export
1293
+ * @param wasmFileset A configuration object that provides the location of
1294
+ * the Wasm binary and its loader.
1295
+ * @param modelAssetBuffer A binary representation of the model.
1296
+ */
1297
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageSegmenter>;
1298
+ /**
1299
+ * Initializes the Wasm runtime and creates a new image segmenter based on
1300
+ * the path to the model asset.
1301
+ * @export
1302
+ * @param wasmFileset A configuration object that provides the location of
1303
+ * the Wasm binary and its loader.
1304
+ * @param modelAssetPath The path to the model asset.
1305
+ */
1306
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageSegmenter>;
1307
+ private constructor();
1308
+ /**
1309
+ * Sets new options for the image segmenter.
1310
+ *
1311
+ * Calling `setOptions()` with a subset of options only affects those
1312
+ * options. You can reset an option back to its default value by
1313
+ * explicitly setting it to `undefined`.
1314
+ *
1315
+ * @export
1316
+ * @param options The options for the image segmenter.
1317
+ */
1318
+ setOptions(options: ImageSegmenterOptions): Promise<void>;
1319
+ /**
1320
+ * Performs image segmentation on the provided single image and invokes the
1321
+ * callback with the response. The method returns synchronously once the
1322
+ * callback returns. Only use this method when the ImageSegmenter is
1323
+ * created with running mode `image`.
1324
+ *
1325
+ * @param image An image to process.
1326
+ * @param callback The callback that is invoked with the segmented masks. The
1327
+ * lifetime of the returned data is only guaranteed for the duration of the
1328
+ * callback.
1329
+ */
1330
+ segment(image: ImageSource, callback: ImageSegmenterCallback): void;
1331
+ /**
1332
+ * Performs image segmentation on the provided single image and invokes the
1333
+ * callback with the response. The method returns synchronously once the
1334
+ * callback returns. Only use this method when the ImageSegmenter is
1335
+ * created with running mode `image`.
1336
+ *
1337
+ * @param image An image to process.
1338
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1339
+ * to process the input image before running inference.
1340
+ * @param callback The callback that is invoked with the segmented masks. The
1341
+ * lifetime of the returned data is only guaranteed for the duration of the
1342
+ * callback.
1343
+ */
1344
+ segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallback): void;
1345
+ /**
1346
+ * Performs image segmentation on the provided single image and returns the
1347
+ * segmentation result. This method creates a copy of the resulting masks and
1348
+ * should not be used in high-throughput applications. Only use this method
1349
+ * when the ImageSegmenter is created with running mode `image`.
1350
+ *
1351
+ * @param image An image to process.
1352
+ * @return The segmentation result. The data is copied to avoid lifetime
1353
+ * issues.
1354
+ */
1355
+ segment(image: ImageSource): ImageSegmenterResult;
1356
+ /**
1357
+ * Performs image segmentation on the provided single image and returns the
1358
+ * segmentation result. This method creates a copy of the resulting masks and
1359
+ * should not be used in high-v applications. Only use this method when
1360
+ * the ImageSegmenter is created with running mode `image`.
1361
+ *
1362
+ * @param image An image to process.
1363
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1364
+ * to process the input image before running inference.
1365
+ * @return The segmentation result. The data is copied to avoid lifetime
1366
+ * issues.
1367
+ */
1368
+ segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions): ImageSegmenterResult;
1369
+ /**
1370
+ * Performs image segmentation on the provided video frame and invokes the
1371
+ * callback with the response. The method returns synchronously once the
1372
+ * callback returns. Only use this method when the ImageSegmenter is
1373
+ * created with running mode `video`.
1374
+ *
1375
+ * @param videoFrame A video frame to process.
1376
+ * @param timestamp The timestamp of the current frame, in ms.
1377
+ * @param callback The callback that is invoked with the segmented masks. The
1378
+ * lifetime of the returned data is only guaranteed for the duration of the
1379
+ * callback.
1380
+ */
1381
+ segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallback): void;
1382
+ /**
1383
+ * Performs image segmentation on the provided video frame and invokes the
1384
+ * callback with the response. The method returns synchronously once the
1385
+ * callback returns. Only use this method when the ImageSegmenter is
1386
+ * created with running mode `video`.
1387
+ *
1388
+ * @param videoFrame A video frame to process.
1389
+ * @param timestamp The timestamp of the current frame, in ms.
1390
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1391
+ * to process the input frame before running inference.
1392
+ * @param callback The callback that is invoked with the segmented masks. The
1393
+ * lifetime of the returned data is only guaranteed for the duration of the
1394
+ * callback.
1395
+ */
1396
+ segmentForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallback): void;
1397
+ /**
1398
+ * Performs image segmentation on the provided video frame and returns the
1399
+ * segmentation result. This method creates a copy of the resulting masks and
1400
+ * should not be used in high-throughput applications. Only use this method
1401
+ * when the ImageSegmenter is created with running mode `video`.
1402
+ *
1403
+ * @param videoFrame A video frame to process.
1404
+ * @return The segmentation result. The data is copied to avoid lifetime
1405
+ * issues.
1406
+ */
1407
+ segmentForVideo(videoFrame: ImageSource, timestamp: number): ImageSegmenterResult;
1408
+ /**
1409
+ * Performs image segmentation on the provided video frame and returns the
1410
+ * segmentation result. This method creates a copy of the resulting masks and
1411
+ * should not be used in high-v applications. Only use this method when
1412
+ * the ImageSegmenter is created with running mode `video`.
1413
+ *
1414
+ * @param videoFrame A video frame to process.
1415
+ * @param timestamp The timestamp of the current frame, in ms.
1416
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1417
+ * to process the input frame before running inference.
1418
+ * @return The segmentation result. The data is copied to avoid lifetime
1419
+ * issues.
1420
+ */
1421
+ segmentForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions: ImageProcessingOptions): ImageSegmenterResult;
1422
+ /**
1423
+ * Get the category label list of the ImageSegmenter can recognize. For
1424
+ * `CATEGORY_MASK` type, the index in the category mask corresponds to the
1425
+ * category in the label list. For `CONFIDENCE_MASK` type, the output mask
1426
+ * list at index corresponds to the category in the label list.
1427
+ *
1428
+ * If there is no labelmap provided in the model file, empty label array is
1429
+ * returned.
1430
+ *
1431
+ * @export
1432
+ * @return The labels used by the current model.
1433
+ */
1434
+ getLabels(): string[];
1435
+ }
1436
+
1437
+ /**
1438
+ * A callback that receives the computed masks from the image segmenter. The
1439
+ * returned data is only valid for the duration of the callback. If
1440
+ * asynchronous processing is needed, all data needs to be copied before the
1441
+ * callback returns.
1442
+ */
1443
+ export declare type ImageSegmenterCallback = (result: ImageSegmenterResult) => void;
1444
+
1445
+ /** Options to configure the MediaPipe Image Segmenter Task */
1446
+ export declare interface ImageSegmenterOptions extends VisionTaskOptions {
1447
+ /**
1448
+ * The locale to use for display names specified through the TFLite Model
1449
+ * Metadata, if any. Defaults to English.
1450
+ */
1451
+ displayNamesLocale?: string | undefined;
1452
+ /** Whether to output confidence masks. Defaults to true. */
1453
+ outputConfidenceMasks?: boolean | undefined;
1454
+ /** Whether to output the category masks. Defaults to false. */
1455
+ outputCategoryMask?: boolean | undefined;
1456
+ }
1457
+
1458
+ /** The output result of ImageSegmenter. */
1459
+ export declare class ImageSegmenterResult {
1460
+ /**
1461
+ * Multiple masks represented as `Float32Array` or `WebGLTexture`-backed
1462
+ * `MPImage`s where, for each mask, each pixel represents the prediction
1463
+ * confidence, usually in the [0, 1] range.
1464
+ * @export
1465
+ */
1466
+ readonly confidenceMasks?: MPMask[] | undefined;
1467
+ /**
1468
+ * A category mask represented as a `Uint8ClampedArray` or
1469
+ * `WebGLTexture`-backed `MPImage` where each pixel represents the class
1470
+ * which the pixel in the original image was predicted to belong to.
1471
+ * @export
1472
+ */
1473
+ readonly categoryMask?: MPMask | undefined;
1474
+ /**
1475
+ * The quality scores of the result masks, in the range of [0, 1].
1476
+ * Defaults to `1` if the model doesn't output quality scores. Each
1477
+ * element corresponds to the score of the category in the model outputs.
1478
+ * @export
1479
+ */
1480
+ readonly qualityScores?: number[] | undefined;
1481
+ constructor(
1482
+ /**
1483
+ * Multiple masks represented as `Float32Array` or `WebGLTexture`-backed
1484
+ * `MPImage`s where, for each mask, each pixel represents the prediction
1485
+ * confidence, usually in the [0, 1] range.
1486
+ * @export
1487
+ */
1488
+ confidenceMasks?: MPMask[] | undefined,
1489
+ /**
1490
+ * A category mask represented as a `Uint8ClampedArray` or
1491
+ * `WebGLTexture`-backed `MPImage` where each pixel represents the class
1492
+ * which the pixel in the original image was predicted to belong to.
1493
+ * @export
1494
+ */
1495
+ categoryMask?: MPMask | undefined,
1496
+ /**
1497
+ * The quality scores of the result masks, in the range of [0, 1].
1498
+ * Defaults to `1` if the model doesn't output quality scores. Each
1499
+ * element corresponds to the score of the category in the model outputs.
1500
+ * @export
1501
+ */
1502
+ qualityScores?: number[] | undefined);
1503
+ /**
1504
+ * Frees the resources held by the category and confidence masks.
1505
+ * @export
1506
+ */
1507
+ close(): void;
1508
+ }
1509
+
1510
+ /**
1511
+ * Valid types of image sources which we can run our GraphRunner over.
1512
+ */
1513
+ export declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
1514
+
1515
+ /**
1516
+ * Performs interactive segmentation on images.
1517
+ *
1518
+ * Users can represent user interaction through `RegionOfInterest`, which gives
1519
+ * a hint to InteractiveSegmenter to perform segmentation focusing on the given
1520
+ * region of interest.
1521
+ *
1522
+ * The API expects a TFLite model with mandatory TFLite Model Metadata.
1523
+ *
1524
+ * Input tensor:
1525
+ * (kTfLiteUInt8/kTfLiteFloat32)
1526
+ * - image input of size `[batch x height x width x channels]`.
1527
+ * - batch inference is not supported (`batch` is required to be 1).
1528
+ * - RGB inputs is supported (`channels` is required to be 3).
1529
+ * - if type is kTfLiteFloat32, NormalizationOptions are required to be
1530
+ * attached to the metadata for input normalization.
1531
+ * Output tensors:
1532
+ * (kTfLiteUInt8/kTfLiteFloat32)
1533
+ * - list of segmented masks.
1534
+ * - if `output_type` is CATEGORY_MASK, uint8 Image, Image vector of size 1.
1535
+ * - if `output_type` is CONFIDENCE_MASK, float32 Image list of size
1536
+ * `channels`.
1537
+ * - batch is always 1
1538
+ */
1539
+ export declare class InteractiveSegmenter extends VisionTaskRunner {
1540
+ /**
1541
+ * Initializes the Wasm runtime and creates a new interactive segmenter from
1542
+ * the provided options.
1543
+ * @export
1544
+ * @param wasmFileset A configuration object that provides the location of
1545
+ * the Wasm binary and its loader.
1546
+ * @param interactiveSegmenterOptions The options for the Interactive
1547
+ * Segmenter. Note that either a path to the model asset or a model buffer
1548
+ * needs to be provided (via `baseOptions`).
1549
+ * @return A new `InteractiveSegmenter`.
1550
+ */
1551
+ static createFromOptions(wasmFileset: WasmFileset, interactiveSegmenterOptions: InteractiveSegmenterOptions): Promise<InteractiveSegmenter>;
1552
+ /**
1553
+ * Initializes the Wasm runtime and creates a new interactive segmenter based
1554
+ * on the provided model asset buffer.
1555
+ * @export
1556
+ * @param wasmFileset A configuration object that provides the location of
1557
+ * the Wasm binary and its loader.
1558
+ * @param modelAssetBuffer A binary representation of the model.
1559
+ * @return A new `InteractiveSegmenter`.
1560
+ */
1561
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<InteractiveSegmenter>;
1562
+ /**
1563
+ * Initializes the Wasm runtime and creates a new interactive segmenter based
1564
+ * on the path to the model asset.
1565
+ * @export
1566
+ * @param wasmFileset A configuration object that provides the location of
1567
+ * the Wasm binary and its loader.
1568
+ * @param modelAssetPath The path to the model asset.
1569
+ * @return A new `InteractiveSegmenter`.
1570
+ */
1571
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<InteractiveSegmenter>;
1572
+ private constructor();
1573
+ /**
1574
+ * Sets new options for the interactive segmenter.
1575
+ *
1576
+ * Calling `setOptions()` with a subset of options only affects those
1577
+ * options. You can reset an option back to its default value by
1578
+ * explicitly setting it to `undefined`.
1579
+ *
1580
+ * @export
1581
+ * @param options The options for the interactive segmenter.
1582
+ * @return A Promise that resolves when the settings have been applied.
1583
+ */
1584
+ setOptions(options: InteractiveSegmenterOptions): Promise<void>;
1585
+ /**
1586
+ * Performs interactive segmentation on the provided single image and invokes
1587
+ * the callback with the response. The method returns synchronously once the
1588
+ * callback returns. The `roi` parameter is used to represent a user's region
1589
+ * of interest for segmentation.
1590
+ *
1591
+ * @param image An image to process.
1592
+ * @param roi The region of interest for segmentation.
1593
+ * @param callback The callback that is invoked with the segmented masks. The
1594
+ * lifetime of the returned data is only guaranteed for the duration of the
1595
+ * callback.
1596
+ */
1597
+ segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallback): void;
1598
+ /**
1599
+ * Performs interactive segmentation on the provided single image and invokes
1600
+ * the callback with the response. The method returns synchronously once the
1601
+ * callback returns. The `roi` parameter is used to represent a user's region
1602
+ * of interest for segmentation.
1603
+ *
1604
+ * The 'image_processing_options' parameter can be used to specify the
1605
+ * rotation to apply to the image before performing segmentation, by setting
1606
+ * its 'rotationDegrees' field. Note that specifying a region-of-interest
1607
+ * using the 'regionOfInterest' field is NOT supported and will result in an
1608
+ * error.
1609
+ *
1610
+ * @param image An image to process.
1611
+ * @param roi The region of interest for segmentation.
1612
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1613
+ * to process the input image before running inference.
1614
+ * @param callback The callback that is invoked with the segmented masks. The
1615
+ * lifetime of the returned data is only guaranteed for the duration of the
1616
+ * callback.
1617
+ */
1618
+ segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallback): void;
1619
+ /**
1620
+ * Performs interactive segmentation on the provided video frame and returns
1621
+ * the segmentation result. This method creates a copy of the resulting masks
1622
+ * and should not be used in high-throughput applications. The `roi` parameter
1623
+ * is used to represent a user's region of interest for segmentation.
1624
+ *
1625
+ * @param image An image to process.
1626
+ * @param roi The region of interest for segmentation.
1627
+ * @return The segmentation result. The data is copied to avoid lifetime
1628
+ * limits.
1629
+ */
1630
+ segment(image: ImageSource, roi: RegionOfInterest): InteractiveSegmenterResult;
1631
+ /**
1632
+ * Performs interactive segmentation on the provided video frame and returns
1633
+ * the segmentation result. This method creates a copy of the resulting masks
1634
+ * and should not be used in high-throughput applications. The `roi` parameter
1635
+ * is used to represent a user's region of interest for segmentation.
1636
+ *
1637
+ * The 'image_processing_options' parameter can be used to specify the
1638
+ * rotation to apply to the image before performing segmentation, by setting
1639
+ * its 'rotationDegrees' field. Note that specifying a region-of-interest
1640
+ * using the 'regionOfInterest' field is NOT supported and will result in an
1641
+ * error.
1642
+ *
1643
+ * @param image An image to process.
1644
+ * @param roi The region of interest for segmentation.
1645
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1646
+ * to process the input image before running inference.
1647
+ * @return The segmentation result. The data is copied to avoid lifetime
1648
+ * limits.
1649
+ */
1650
+ segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions): InteractiveSegmenterResult;
1651
+ }
1652
+
1653
+ /**
1654
+ * A callback that receives the computed masks from the interactive segmenter.
1655
+ * The returned data is only valid for the duration of the callback. If
1656
+ * asynchronous processing is needed, all data needs to be copied before the
1657
+ * callback returns.
1658
+ */
1659
+ export declare type InteractiveSegmenterCallback = (result: InteractiveSegmenterResult) => void;
1660
+
1661
+ /** Options to configure the MediaPipe Interactive Segmenter Task */
1662
+ export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
1663
+ /** Whether to output confidence masks. Defaults to true. */
1664
+ outputConfidenceMasks?: boolean | undefined;
1665
+ /** Whether to output the category masks. Defaults to false. */
1666
+ outputCategoryMask?: boolean | undefined;
1667
+ }
1668
+
1669
+ /** The output result of InteractiveSegmenter. */
1670
+ export declare class InteractiveSegmenterResult {
1671
+ /**
1672
+ * Multiple masks represented as `Float32Array` or `WebGLTexture`-backed
1673
+ * `MPImage`s where, for each mask, each pixel represents the prediction
1674
+ * confidence, usually in the [0, 1] range.
1675
+ * @export
1676
+ */
1677
+ readonly confidenceMasks?: MPMask[] | undefined;
1678
+ /**
1679
+ * A category mask represented as a `Uint8ClampedArray` or
1680
+ * `WebGLTexture`-backed `MPImage` where each pixel represents the class
1681
+ * which the pixel in the original image was predicted to belong to.
1682
+ * @export
1683
+ */
1684
+ readonly categoryMask?: MPMask | undefined;
1685
+ /**
1686
+ * The quality scores of the result masks, in the range of [0, 1].
1687
+ * Defaults to `1` if the model doesn't output quality scores. Each
1688
+ * element corresponds to the score of the category in the model outputs.
1689
+ * @export
1690
+ */
1691
+ readonly qualityScores?: number[] | undefined;
1692
+ constructor(
1693
+ /**
1694
+ * Multiple masks represented as `Float32Array` or `WebGLTexture`-backed
1695
+ * `MPImage`s where, for each mask, each pixel represents the prediction
1696
+ * confidence, usually in the [0, 1] range.
1697
+ * @export
1698
+ */
1699
+ confidenceMasks?: MPMask[] | undefined,
1700
+ /**
1701
+ * A category mask represented as a `Uint8ClampedArray` or
1702
+ * `WebGLTexture`-backed `MPImage` where each pixel represents the class
1703
+ * which the pixel in the original image was predicted to belong to.
1704
+ * @export
1705
+ */
1706
+ categoryMask?: MPMask | undefined,
1707
+ /**
1708
+ * The quality scores of the result masks, in the range of [0, 1].
1709
+ * Defaults to `1` if the model doesn't output quality scores. Each
1710
+ * element corresponds to the score of the category in the model outputs.
1711
+ * @export
1712
+ */
1713
+ qualityScores?: number[] | undefined);
1714
+ /**
1715
+ * Frees the resources held by the category and confidence masks.
1716
+ * @export
1717
+ */
1718
+ close(): void;
1719
+ }
1720
+
1721
+ /**
1722
+ * Landmark represents a point in 3D space with x, y, z coordinates. The
1723
+ * landmark coordinates are in meters. z represents the landmark depth,
1724
+ * and the smaller the value the closer the world landmark is to the camera.
1725
+ */
1726
+ export declare interface Landmark {
1727
+ /** The x coordinates of the landmark. */
1728
+ x: number;
1729
+ /** The y coordinates of the landmark. */
1730
+ y: number;
1731
+ /** The z coordinates of the landmark. */
1732
+ z: number;
1733
+ }
1734
+
1735
+ /** Data that a user can use to specialize drawing options. */
1736
+ export declare interface LandmarkData {
1737
+ index?: number;
1738
+ from?: NormalizedLandmark;
1739
+ to?: NormalizedLandmark;
1740
+ }
1741
+
1742
+ /**
1743
+ * Copyright 2023 The MediaPipe Authors.
1744
+ *
1745
+ * Licensed under the Apache License, Version 2.0 (the "License");
1746
+ * you may not use this file except in compliance with the License.
1747
+ * You may obtain a copy of the License at
1748
+ *
1749
+ * http://www.apache.org/licenses/LICENSE-2.0
1750
+ *
1751
+ * Unless required by applicable law or agreed to in writing, software
1752
+ * distributed under the License is distributed on an "AS IS" BASIS,
1753
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1754
+ * See the License for the specific language governing permissions and
1755
+ * limitations under the License.
1756
+ */
1757
+ /** A two-dimensional matrix. */
1758
+ declare interface Matrix {
1759
+ /** The number of rows. */
1760
+ rows: number;
1761
+ /** The number of columns. */
1762
+ columns: number;
1763
+ /** The values as a flattened one-dimensional array. */
1764
+ data: number[];
1765
+ }
1766
+
1767
+ /**
1768
+ * The wrapper class for MediaPipe Image objects.
1769
+ *
1770
+ * Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects.
1771
+ * You can convert the underlying type to any other type by passing the
1772
+ * desired type to `getAs...()`. As type conversions can be expensive, it is
1773
+ * recommended to limit these conversions. You can verify what underlying
1774
+ * types are already available by invoking `has...()`.
1775
+ *
1776
+ * Images that are returned from a MediaPipe Tasks are owned by by the
1777
+ * underlying C++ Task. If you need to extend the lifetime of these objects,
1778
+ * you can invoke the `clone()` method. To free up the resources obtained
1779
+ * during any clone or type conversion operation, it is important to invoke
1780
+ * `close()` on the `MPImage` instance.
1781
+ *
1782
+ * Converting to and from ImageBitmap requires that the MediaPipe task is
1783
+ * initialized with an `OffscreenCanvas`. As we require WebGL2 support, this
1784
+ * places some limitations on Browser support as outlined here:
1785
+ * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1786
+ */
1787
+ export declare class MPImage {
1788
+ /** Returns the canvas element that the image is bound to. */
1789
+ readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
1790
+ /** Returns the width of the image. */
1791
+ readonly width: number;
1792
+ /** Returns the height of the image. */
1793
+ readonly height: number;
1794
+ private constructor();
1795
+ /**
1796
+ * Returns whether this `MPImage` contains a mask of type `ImageData`.
1797
+ * @export
1798
+ */
1799
+ hasImageData(): boolean;
1800
+ /**
1801
+ * Returns whether this `MPImage` contains a mask of type `ImageBitmap`.
1802
+ * @export
1803
+ */
1804
+ hasImageBitmap(): boolean;
1805
+ /**
1806
+ * Returns whether this `MPImage` contains a mask of type `WebGLTexture`.
1807
+ * @export
1808
+ */
1809
+ hasWebGLTexture(): boolean;
1810
+ /**
1811
+ * Returns the underlying image as an `ImageData` object. Note that this
1812
+ * involves an expensive GPU to CPU transfer if the current image is only
1813
+ * available as an `ImageBitmap` or `WebGLTexture`.
1814
+ *
1815
+ * @export
1816
+ * @return The current image as an ImageData object.
1817
+ */
1818
+ getAsImageData(): ImageData;
1819
+ /**
1820
+ * Returns the underlying image as an `ImageBitmap`. Note that
1821
+ * conversions to `ImageBitmap` are expensive, especially if the data
1822
+ * currently resides on CPU.
1823
+ *
1824
+ * Processing with `ImageBitmap`s requires that the MediaPipe Task was
1825
+ * initialized with an `OffscreenCanvas` with WebGL2 support. See
1826
+ * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1827
+ * for a list of supported platforms.
1828
+ *
1829
+ * @export
1830
+ * @return The current image as an ImageBitmap object.
1831
+ */
1832
+ getAsImageBitmap(): ImageBitmap;
1833
+ /**
1834
+ * Returns the underlying image as a `WebGLTexture` object. Note that this
1835
+ * involves a CPU to GPU transfer if the current image is only available as
1836
+ * an `ImageData` object. The returned texture is bound to the current
1837
+ * canvas (see `.canvas`).
1838
+ *
1839
+ * @export
1840
+ * @return The current image as a WebGLTexture.
1841
+ */
1842
+ getAsWebGLTexture(): WebGLTexture;
1843
+ /**
1844
+ * Creates a copy of the resources stored in this `MPImage`. You can invoke
1845
+ * this method to extend the lifetime of an image returned by a MediaPipe
1846
+ * Task. Note that performance critical applications should aim to only use
1847
+ * the `MPImage` within the MediaPipe Task callback so that copies can be
1848
+ * avoided.
1849
+ *
1850
+ * @export
1851
+ */
1852
+ clone(): MPImage;
1853
+ /**
1854
+ * Frees up any resources owned by this `MPImage` instance.
1855
+ *
1856
+ * Note that this method does not free images that are owned by the C++
1857
+ * Task, as these are freed automatically once you leave the MediaPipe
1858
+ * callback. Additionally, some shared state is freed only once you invoke the
1859
+ * Task's `close()` method.
1860
+ *
1861
+ * @export
1862
+ */
1863
+ close(): void;
1864
+ }
1865
+
1866
+ /**
1867
+ * The wrapper class for MediaPipe segmentation masks.
1868
+ *
1869
+ * Masks are stored as `Uint8Array`, `Float32Array` or `WebGLTexture` objects.
1870
+ * You can convert the underlying type to any other type by passing the desired
1871
+ * type to `getAs...()`. As type conversions can be expensive, it is recommended
1872
+ * to limit these conversions. You can verify what underlying types are already
1873
+ * available by invoking `has...()`.
1874
+ *
1875
+ * Masks that are returned from a MediaPipe Tasks are owned by by the
1876
+ * underlying C++ Task. If you need to extend the lifetime of these objects,
1877
+ * you can invoke the `clone()` method. To free up the resources obtained
1878
+ * during any clone or type conversion operation, it is important to invoke
1879
+ * `close()` on the `MPMask` instance.
1880
+ */
1881
+ export declare class MPMask {
1882
+ /** Returns the canvas element that the mask is bound to. */
1883
+ readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
1884
+ /** Returns the width of the mask. */
1885
+ readonly width: number;
1886
+ /** Returns the height of the mask. */
1887
+ readonly height: number;
1888
+ private constructor();
1889
+ /**
1890
+ * Returns whether this `MPMask` contains a mask of type `Uint8Array`.
1891
+ * @export
1892
+ */
1893
+ hasUint8Array(): boolean;
1894
+ /**
1895
+ * Returns whether this `MPMask` contains a mask of type `Float32Array`.
1896
+ * @export
1897
+ */
1898
+ hasFloat32Array(): boolean;
1899
+ /**
1900
+ * Returns whether this `MPMask` contains a mask of type `WebGLTexture`.
1901
+ * @export
1902
+ */
1903
+ hasWebGLTexture(): boolean;
1904
+ /**
1905
+ * Returns the underlying mask as a Uint8Array`. Note that this involves an
1906
+ * expensive GPU to CPU transfer if the current mask is only available as a
1907
+ * `WebGLTexture`.
1908
+ *
1909
+ * @export
1910
+ * @return The current data as a Uint8Array.
1911
+ */
1912
+ getAsUint8Array(): Uint8Array;
1913
+ /**
1914
+ * Returns the underlying mask as a single channel `Float32Array`. Note that
1915
+ * this involves an expensive GPU to CPU transfer if the current mask is
1916
+ * only available as a `WebGLTexture`.
1917
+ *
1918
+ * @export
1919
+ * @return The current mask as a Float32Array.
1920
+ */
1921
+ getAsFloat32Array(): Float32Array;
1922
+ /**
1923
+ * Returns the underlying mask as a `WebGLTexture` object. Note that this
1924
+ * involves a CPU to GPU transfer if the current mask is only available as
1925
+ * a CPU array. The returned texture is bound to the current canvas (see
1926
+ * `.canvas`).
1927
+ *
1928
+ * @export
1929
+ * @return The current mask as a WebGLTexture.
1930
+ */
1931
+ getAsWebGLTexture(): WebGLTexture;
1932
+ /**
1933
+ * Creates a copy of the resources stored in this `MPMask`. You can
1934
+ * invoke this method to extend the lifetime of a mask returned by a
1935
+ * MediaPipe Task. Note that performance critical applications should aim to
1936
+ * only use the `MPMask` within the MediaPipe Task callback so that
1937
+ * copies can be avoided.
1938
+ *
1939
+ * @export
1940
+ */
1941
+ clone(): MPMask;
1942
+ /**
1943
+ * Frees up any resources owned by this `MPMask` instance.
1944
+ *
1945
+ * Note that this method does not free masks that are owned by the C++
1946
+ * Task, as these are freed automatically once you leave the MediaPipe
1947
+ * callback. Additionally, some shared state is freed only once you invoke
1948
+ * the Task's `close()` method.
1949
+ *
1950
+ * @export
1951
+ */
1952
+ close(): void;
1953
+ }
1954
+
1955
+ /**
1956
+ * Copyright 2023 The MediaPipe Authors.
1957
+ *
1958
+ * Licensed under the Apache License, Version 2.0 (the "License");
1959
+ * you may not use this file except in compliance with the License.
1960
+ * You may obtain a copy of the License at
1961
+ *
1962
+ * http://www.apache.org/licenses/LICENSE-2.0
1963
+ *
1964
+ * Unless required by applicable law or agreed to in writing, software
1965
+ * distributed under the License is distributed on an "AS IS" BASIS,
1966
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1967
+ * See the License for the specific language governing permissions and
1968
+ * limitations under the License.
1969
+ */
1970
+ /**
1971
+ * A keypoint, defined by the coordinates (x, y), normalized by the image
1972
+ * dimensions.
1973
+ */
1974
+ declare interface NormalizedKeypoint {
1975
+ /** X in normalized image coordinates. */
1976
+ x: number;
1977
+ /** Y in normalized image coordinates. */
1978
+ y: number;
1979
+ /** Optional label of the keypoint. */
1980
+ label?: string;
1981
+ /** Optional score of the keypoint. */
1982
+ score?: number;
1983
+ }
1984
+
1985
+ /**
1986
+ * Copyright 2022 The MediaPipe Authors.
1987
+ *
1988
+ * Licensed under the Apache License, Version 2.0 (the "License");
1989
+ * you may not use this file except in compliance with the License.
1990
+ * You may obtain a copy of the License at
1991
+ *
1992
+ * http://www.apache.org/licenses/LICENSE-2.0
1993
+ *
1994
+ * Unless required by applicable law or agreed to in writing, software
1995
+ * distributed under the License is distributed on an "AS IS" BASIS,
1996
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1997
+ * See the License for the specific language governing permissions and
1998
+ * limitations under the License.
1999
+ */
2000
+ /**
2001
+ * Normalized Landmark represents a point in 3D space with x, y, z coordinates.
2002
+ * x and y are normalized to [0.0, 1.0] by the image width and height
2003
+ * respectively. z represents the landmark depth, and the smaller the value the
2004
+ * closer the landmark is to the camera. The magnitude of z uses roughly the
2005
+ * same scale as x.
2006
+ */
2007
+ export declare interface NormalizedLandmark {
2008
+ /** The x coordinates of the normalized landmark. */
2009
+ x: number;
2010
+ /** The y coordinates of the normalized landmark. */
2011
+ y: number;
2012
+ /** The z coordinates of the normalized landmark. */
2013
+ z: number;
2014
+ }
2015
+
2016
+ /**
2017
+ * Performs object detection on images.
2018
+ */
2019
+ export declare class ObjectDetector extends VisionTaskRunner {
2020
+ /**
2021
+ * Initializes the Wasm runtime and creates a new object detector from the
2022
+ * provided options.
2023
+ * @export
2024
+ * @param wasmFileset A configuration object that provides the location of the
2025
+ * Wasm binary and its loader.
2026
+ * @param objectDetectorOptions The options for the Object Detector. Note that
2027
+ * either a path to the model asset or a model buffer needs to be
2028
+ * provided (via `baseOptions`).
2029
+ */
2030
+ static createFromOptions(wasmFileset: WasmFileset, objectDetectorOptions: ObjectDetectorOptions): Promise<ObjectDetector>;
2031
+ /**
2032
+ * Initializes the Wasm runtime and creates a new object detector based on the
2033
+ * provided model asset buffer.
2034
+ * @export
2035
+ * @param wasmFileset A configuration object that provides the location of the
2036
+ * Wasm binary and its loader.
2037
+ * @param modelAssetBuffer A binary representation of the model.
2038
+ */
2039
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ObjectDetector>;
2040
+ /**
2041
+ * Initializes the Wasm runtime and creates a new object detector based on the
2042
+ * path to the model asset.
2043
+ * @export
2044
+ * @param wasmFileset A configuration object that provides the location of the
2045
+ * Wasm binary and its loader.
2046
+ * @param modelAssetPath The path to the model asset.
2047
+ */
2048
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ObjectDetector>;
2049
+ private constructor();
2050
+ /**
2051
+ * Sets new options for the object detector.
2052
+ *
2053
+ * Calling `setOptions()` with a subset of options only affects those options.
2054
+ * You can reset an option back to its default value by explicitly setting it
2055
+ * to `undefined`.
2056
+ *
2057
+ * @export
2058
+ * @param options The options for the object detector.
2059
+ */
2060
+ setOptions(options: ObjectDetectorOptions): Promise<void>;
2061
+ /**
2062
+ * Performs object detection on the provided single image and waits
2063
+ * synchronously for the response. Only use this method when the
2064
+ * ObjectDetector is created with running mode `image`.
2065
+ *
2066
+ * @export
2067
+ * @param image An image to process.
2068
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2069
+ * to process the input image before running inference.
2070
+ * @return A result containing a list of detected objects.
2071
+ */
2072
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
2073
+ /**
2074
+ * Performs object detection on the provided video frame and waits
2075
+ * synchronously for the response. Only use this method when the
2076
+ * ObjectDetector is created with running mode `video`.
2077
+ *
2078
+ * @export
2079
+ * @param videoFrame A video frame to process.
2080
+ * @param timestamp The timestamp of the current frame, in ms.
2081
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2082
+ * to process the input image before running inference.
2083
+ * @return A result containing a list of detected objects.
2084
+ */
2085
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
2086
+ }
2087
+
2088
+ /** Options to configure the MediaPipe Object Detector Task */
2089
+ export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
2090
+ }
2091
+
2092
+ /** Performs pose landmarks detection on images. */
2093
+ export declare class PoseLandmarker extends VisionTaskRunner {
2094
+ /**
2095
+ * An array containing the pairs of pose landmark indices to be rendered with
2096
+ * connections.
2097
+ * @export
2098
+ * @nocollapse
2099
+ */
2100
+ static POSE_CONNECTIONS: Connection[];
2101
+ /**
2102
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` from the
2103
+ * provided options.
2104
+ * @export
2105
+ * @param wasmFileset A configuration object that provides the location of the
2106
+ * Wasm binary and its loader.
2107
+ * @param poseLandmarkerOptions The options for the PoseLandmarker.
2108
+ * Note that either a path to the model asset or a model buffer needs to
2109
+ * be provided (via `baseOptions`).
2110
+ */
2111
+ static createFromOptions(wasmFileset: WasmFileset, poseLandmarkerOptions: PoseLandmarkerOptions): Promise<PoseLandmarker>;
2112
+ /**
2113
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
2114
+ * the provided model asset buffer.
2115
+ * @export
2116
+ * @param wasmFileset A configuration object that provides the location of the
2117
+ * Wasm binary and its loader.
2118
+ * @param modelAssetBuffer A binary representation of the model.
2119
+ */
2120
+ static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<PoseLandmarker>;
2121
+ /**
2122
+ * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
2123
+ * the path to the model asset.
2124
+ * @export
2125
+ * @param wasmFileset A configuration object that provides the location of the
2126
+ * Wasm binary and its loader.
2127
+ * @param modelAssetPath The path to the model asset.
2128
+ */
2129
+ static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<PoseLandmarker>;
2130
+ private constructor();
2131
+ /**
2132
+ * Sets new options for this `PoseLandmarker`.
2133
+ *
2134
+ * Calling `setOptions()` with a subset of options only affects those options.
2135
+ * You can reset an option back to its default value by explicitly setting it
2136
+ * to `undefined`.
2137
+ *
2138
+ * @export
2139
+ * @param options The options for the pose landmarker.
2140
+ */
2141
+ setOptions(options: PoseLandmarkerOptions): Promise<void>;
2142
+ /**
2143
+ * Performs pose detection on the provided single image and invokes the
2144
+ * callback with the response. The method returns synchronously once the
2145
+ * callback returns. Only use this method when the PoseLandmarker is created
2146
+ * with running mode `image`.
2147
+ *
2148
+ * @param image An image to process.
2149
+ * @param callback The callback that is invoked with the result. The
2150
+ * lifetime of the returned masks is only guaranteed for the duration of
2151
+ * the callback.
2152
+ */
2153
+ detect(image: ImageSource, callback: PoseLandmarkerCallback): void;
2154
+ /**
2155
+ * Performs pose detection on the provided single image and invokes the
2156
+ * callback with the response. The method returns synchronously once the
2157
+ * callback returns. Only use this method when the PoseLandmarker is created
2158
+ * with running mode `image`.
2159
+ *
2160
+ * @param image An image to process.
2161
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2162
+ * to process the input image before running inference.
2163
+ * @param callback The callback that is invoked with the result. The
2164
+ * lifetime of the returned masks is only guaranteed for the duration of
2165
+ * the callback.
2166
+ */
2167
+ detect(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: PoseLandmarkerCallback): void;
2168
+ /**
2169
+ * Performs pose detection on the provided single image and waits
2170
+ * synchronously for the response. This method creates a copy of the resulting
2171
+ * masks and should not be used in high-throughput applications. Only
2172
+ * use this method when the PoseLandmarker is created with running mode
2173
+ * `image`.
2174
+ *
2175
+ * @param image An image to process.
2176
+ * @return The landmarker result. Any masks are copied to avoid lifetime
2177
+ * limits.
2178
+ * @return The detected pose landmarks.
2179
+ */
2180
+ detect(image: ImageSource): PoseLandmarkerResult;
2181
+ /**
2182
+ * Performs pose detection on the provided single image and waits
2183
+ * synchronously for the response. This method creates a copy of the resulting
2184
+ * masks and should not be used in high-throughput applications. Only
2185
+ * use this method when the PoseLandmarker is created with running mode
2186
+ * `image`.
2187
+ *
2188
+ * @param image An image to process.
2189
+ * @return The landmarker result. Any masks are copied to avoid lifetime
2190
+ * limits.
2191
+ * @return The detected pose landmarks.
2192
+ */
2193
+ detect(image: ImageSource, imageProcessingOptions: ImageProcessingOptions): PoseLandmarkerResult;
2194
+ /**
2195
+ * Performs pose detection on the provided video frame and invokes the
2196
+ * callback with the response. The method returns synchronously once the
2197
+ * callback returns. Only use this method when the PoseLandmarker is created
2198
+ * with running mode `video`.
2199
+ *
2200
+ * @param videoFrame A video frame to process.
2201
+ * @param timestamp The timestamp of the current frame, in ms.
2202
+ * @param callback The callback that is invoked with the result. The
2203
+ * lifetime of the returned masks is only guaranteed for the duration of
2204
+ * the callback.
2205
+ */
2206
+ detectForVideo(videoFrame: ImageSource, timestamp: number, callback: PoseLandmarkerCallback): void;
2207
+ /**
2208
+ * Performs pose detection on the provided video frame and invokes the
2209
+ * callback with the response. The method returns synchronously once the
2210
+ * callback returns. Only use this method when the PoseLandmarker is created
2211
+ * with running mode `video`.
2212
+ *
2213
+ * @param videoFrame A video frame to process.
2214
+ * @param timestamp The timestamp of the current frame, in ms.
2215
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2216
+ * to process the input image before running inference.
2217
+ * @param callback The callback that is invoked with the result. The
2218
+ * lifetime of the returned masks is only guaranteed for the duration of
2219
+ * the callback.
2220
+ */
2221
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions: ImageProcessingOptions, callback: PoseLandmarkerCallback): void;
2222
+ /**
2223
+ * Performs pose detection on the provided video frame and returns the result.
2224
+ * This method creates a copy of the resulting masks and should not be used
2225
+ * in high-throughput applications. Only use this method when the
2226
+ * PoseLandmarker is created with running mode `video`.
2227
+ *
2228
+ * @param videoFrame A video frame to process.
2229
+ * @param timestamp The timestamp of the current frame, in ms.
2230
+ * @return The landmarker result. Any masks are copied to extend the
2231
+ * lifetime of the returned data.
2232
+ */
2233
+ detectForVideo(videoFrame: ImageSource, timestamp: number): PoseLandmarkerResult;
2234
+ /**
2235
+ * Performs pose detection on the provided video frame and returns the result.
2236
+ * This method creates a copy of the resulting masks and should not be used
2237
+ * in high-throughput applications. The method returns synchronously once the
2238
+ * callback returns. Only use this method when the PoseLandmarker is created
2239
+ * with running mode `video`.
2240
+ *
2241
+ * @param videoFrame A video frame to process.
2242
+ * @param timestamp The timestamp of the current frame, in ms.
2243
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2244
+ * to process the input image before running inference.
2245
+ * @return The landmarker result. Any masks are copied to extend the lifetime
2246
+ * of the returned data.
2247
+ */
2248
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions: ImageProcessingOptions): PoseLandmarkerResult;
2249
+ }
2250
+
2251
+ /**
2252
+ * A callback that receives the result from the pose detector. The returned
2253
+ * masks are only valid for the duration of the callback. If asynchronous
2254
+ * processing is needed, the masks need to be copied before the callback
2255
+ * returns.
2256
+ */
2257
+ export declare type PoseLandmarkerCallback = (result: PoseLandmarkerResult) => void;
2258
+
2259
+ /** Options to configure the MediaPipe PoseLandmarker Task */
2260
+ export declare interface PoseLandmarkerOptions extends VisionTaskOptions {
2261
+ /**
2262
+ * The maximum number of poses can be detected by the PoseLandmarker.
2263
+ * Defaults to 1.
2264
+ */
2265
+ numPoses?: number | undefined;
2266
+ /**
2267
+ * The minimum confidence score for the pose detection to be considered
2268
+ * successful. Defaults to 0.5.
2269
+ */
2270
+ minPoseDetectionConfidence?: number | undefined;
2271
+ /**
2272
+ * The minimum confidence score of pose presence score in the pose landmark
2273
+ * detection. Defaults to 0.5.
2274
+ */
2275
+ minPosePresenceConfidence?: number | undefined;
2276
+ /**
2277
+ * The minimum confidence score for the pose tracking to be considered
2278
+ * successful. Defaults to 0.5.
2279
+ */
2280
+ minTrackingConfidence?: number | undefined;
2281
+ /** Whether to output segmentation masks. Defaults to false. */
2282
+ outputSegmentationMasks?: boolean | undefined;
2283
+ }
2284
+
2285
+ /**
2286
+ * Represents the pose landmarks deection results generated by `PoseLandmarker`.
2287
+ * Each vector element represents a single pose detected in the image.
2288
+ */
2289
+ export declare class PoseLandmarkerResult {
2290
+ readonly landmarks: NormalizedLandmark[][];
2291
+ /** Pose landmarks in world coordinates of detected poses. */
2292
+ readonly worldLandmarks: Landmark[][];
2293
+ /** Segmentation mask for the detected pose. */
2294
+ readonly segmentationMasks?: MPMask[] | undefined;
2295
+ constructor(/** Pose landmarks of detected poses. */ landmarks: NormalizedLandmark[][],
2296
+ /** Pose landmarks in world coordinates of detected poses. */
2297
+ worldLandmarks: Landmark[][],
2298
+ /** Segmentation mask for the detected pose. */
2299
+ segmentationMasks?: MPMask[] | undefined);
2300
+ /** Frees the resources held by the segmentation masks. */
2301
+ close(): void;
2302
+ }
2303
+
2304
+ /**
2305
+ * Defines a rectangle, used e.g. as part of detection results or as input
2306
+ * region-of-interest.
2307
+ *
2308
+ * The coordinates are normalized with respect to the image dimensions, i.e.
2309
+ * generally in [0,1] but they may exceed these bounds if describing a region
2310
+ * overlapping the image. The origin is on the top-left corner of the image.
2311
+ */
2312
+ declare interface RectF {
2313
+ left: number;
2314
+ top: number;
2315
+ right: number;
2316
+ bottom: number;
2317
+ }
2318
+
2319
+ /** A Region-Of-Interest (ROI) to represent a region within an image. */
2320
+ export declare interface RegionOfInterest {
2321
+ /** The ROI in keypoint format. */
2322
+ keypoint?: NormalizedKeypoint;
2323
+ /** The ROI as scribbles over the object that the user wants to segment. */
2324
+ scribble?: NormalizedKeypoint[];
2325
+ }
2326
+
2327
+ /**
2328
+ * The two running modes of a vision task.
2329
+ * 1) The image mode for processing single image inputs.
2330
+ * 2) The video mode for processing decoded frames of a video.
2331
+ */
2332
+ declare type RunningMode = "IMAGE" | "VIDEO";
2333
+
2334
+ /** Base class for all MediaPipe Tasks. */
2335
+ declare abstract class TaskRunner {
2336
+ protected constructor();
2337
+ /** Configures the task with custom options. */
2338
+ abstract setOptions(options: TaskRunnerOptions): Promise<void>;
2339
+ /**
2340
+ * Closes and cleans up the resources held by this task.
2341
+ * @export
2342
+ */
2343
+ close(): void;
2344
+ }
2345
+
2346
+ /** Options to configure MediaPipe Tasks in general. */
2347
+ declare interface TaskRunnerOptions {
2348
+ /** Options to configure the loading of the model assets. */
2349
+ baseOptions?: BaseOptions_2;
2350
+ }
2351
+
2352
+ /** The options for configuring a MediaPipe vision task. */
2353
+ declare interface VisionTaskOptions extends TaskRunnerOptions {
2354
+ /**
2355
+ * The canvas element to bind textures to. This has to be set for GPU
2356
+ * processing. The task will initialize a WebGL context and throw an error if
2357
+ * this fails (e.g. if you have already initialized a different type of
2358
+ * context).
2359
+ */
2360
+ canvas?: HTMLCanvasElement | OffscreenCanvas;
2361
+ /**
2362
+ * The running mode of the task. Default to the image mode.
2363
+ * Vision tasks have two running modes:
2364
+ * 1) The image mode for processing single image inputs.
2365
+ * 2) The video mode for processing decoded frames of a video.
2366
+ */
2367
+ runningMode?: RunningMode;
2368
+ }
2369
+
2370
+ /** Base class for all MediaPipe Vision Tasks. */
2371
+ declare abstract class VisionTaskRunner extends TaskRunner {
2372
+ protected constructor();
2373
+ /**
2374
+ * Closes and cleans up the resources held by this task.
2375
+ * @export
2376
+ */
2377
+ close(): void;
2378
+ }
2379
+
2380
+ /**
2381
+ * Copyright 2022 The MediaPipe Authors.
2382
+ *
2383
+ * Licensed under the Apache License, Version 2.0 (the "License");
2384
+ * you may not use this file except in compliance with the License.
2385
+ * You may obtain a copy of the License at
2386
+ *
2387
+ * http://www.apache.org/licenses/LICENSE-2.0
2388
+ *
2389
+ * Unless required by applicable law or agreed to in writing, software
2390
+ * distributed under the License is distributed on an "AS IS" BASIS,
2391
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2392
+ * See the License for the specific language governing permissions and
2393
+ * limitations under the License.
2394
+ */
2395
+ /** An object containing the locations of the Wasm assets */
2396
+ declare interface WasmFileset {
2397
+ /** The path to the Wasm loader script. */
2398
+ wasmLoaderPath: string;
2399
+ /** The path to the Wasm binary. */
2400
+ wasmBinaryPath: string;
2401
+ /** The optional path to the asset loader script. */
2402
+ assetLoaderPath?: string;
2403
+ /** The optional path to the assets binary. */
2404
+ assetBinaryPath?: string;
2405
+ }
2406
+
2407
+ export { }