@mediapipe/tasks-vision 0.1.0-alpha-13 → 0.1.0-alpha-14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/vision.d.ts DELETED
@@ -1,2159 +0,0 @@
1
- /**
2
- * Copyright 2022 The MediaPipe Authors.
3
- *
4
- * Licensed under the Apache License, Version 2.0 (the "License");
5
- * you may not use this file except in compliance with the License.
6
- * You may obtain a copy of the License at
7
- *
8
- * http://www.apache.org/licenses/LICENSE-2.0
9
- *
10
- * Unless required by applicable law or agreed to in writing, software
11
- * distributed under the License is distributed on an "AS IS" BASIS,
12
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- * See the License for the specific language governing permissions and
14
- * limitations under the License.
15
- */
16
- /** Options to configure MediaPipe model loading and processing. */
17
- declare interface BaseOptions_2 {
18
- /**
19
- * The model path to the model asset file. Only one of `modelAssetPath` or
20
- * `modelAssetBuffer` can be set.
21
- */
22
- modelAssetPath?: string | undefined;
23
- /**
24
- * A buffer containing the model aaset. Only one of `modelAssetPath` or
25
- * `modelAssetBuffer` can be set.
26
- */
27
- modelAssetBuffer?: Uint8Array | undefined;
28
- /** Overrides the default backend to use for the provided model. */
29
- delegate?: "CPU" | "GPU" | undefined;
30
- }
31
-
32
- /**
33
- * Copyright 2023 The MediaPipe Authors.
34
- *
35
- * Licensed under the Apache License, Version 2.0 (the "License");
36
- * you may not use this file except in compliance with the License.
37
- * You may obtain a copy of the License at
38
- *
39
- * http://www.apache.org/licenses/LICENSE-2.0
40
- *
41
- * Unless required by applicable law or agreed to in writing, software
42
- * distributed under the License is distributed on an "AS IS" BASIS,
43
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
44
- * See the License for the specific language governing permissions and
45
- * limitations under the License.
46
- */
47
- /** An integer bounding box, axis aligned. */
48
- export declare interface BoundingBox {
49
- /** The X coordinate of the top-left corner, in pixels. */
50
- originX: number;
51
- /** The Y coordinate of the top-left corner, in pixels. */
52
- originY: number;
53
- /** The width of the bounding box, in pixels. */
54
- width: number;
55
- /** The height of the bounding box, in pixels. */
56
- height: number;
57
- }
58
-
59
- /**
60
- * A user-defined callback to take input data and map it to a custom output
61
- * value.
62
- */
63
- export declare type Callback<I, O> = (input: I) => O;
64
-
65
- /**
66
- * Copyright 2022 The MediaPipe Authors.
67
- *
68
- * Licensed under the Apache License, Version 2.0 (the "License");
69
- * you may not use this file except in compliance with the License.
70
- * You may obtain a copy of the License at
71
- *
72
- * http://www.apache.org/licenses/LICENSE-2.0
73
- *
74
- * Unless required by applicable law or agreed to in writing, software
75
- * distributed under the License is distributed on an "AS IS" BASIS,
76
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
77
- * See the License for the specific language governing permissions and
78
- * limitations under the License.
79
- */
80
- /** A classification category. */
81
- export declare interface Category {
82
- /** The probability score of this label category. */
83
- score: number;
84
- /** The index of the category in the corresponding label file. */
85
- index: number;
86
- /**
87
- * The label of this category object. Defaults to an empty string if there is
88
- * no category.
89
- */
90
- categoryName: string;
91
- /**
92
- * The display name of the label, which may be translated for different
93
- * locales. For example, a label, "apple", may be translated into Spanish for
94
- * display purpose, so that the `display_name` is "manzana". Defaults to an
95
- * empty string if there is no display name.
96
- */
97
- displayName: string;
98
- }
99
-
100
- /** Classification results for a given classifier head. */
101
- export declare interface Classifications {
102
- /**
103
- * The array of predicted categories, usually sorted by descending scores,
104
- * e.g., from high to low probability.
105
- */
106
- categories: Category[];
107
- /**
108
- * The index of the classifier head these categories refer to. This is
109
- * useful for multi-head models.
110
- */
111
- headIndex: number;
112
- /**
113
- * The name of the classifier head, which is the corresponding tensor
114
- * metadata name. Defaults to an empty string if there is no such metadata.
115
- */
116
- headName: string;
117
- }
118
-
119
- /**
120
- * Copyright 2022 The MediaPipe Authors.
121
- *
122
- * Licensed under the Apache License, Version 2.0 (the "License");
123
- * you may not use this file except in compliance with the License.
124
- * You may obtain a copy of the License at
125
- *
126
- * http://www.apache.org/licenses/LICENSE-2.0
127
- *
128
- * Unless required by applicable law or agreed to in writing, software
129
- * distributed under the License is distributed on an "AS IS" BASIS,
130
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
131
- * See the License for the specific language governing permissions and
132
- * limitations under the License.
133
- */
134
- /** Options to configure a MediaPipe Classifier Task. */
135
- declare interface ClassifierOptions {
136
- /**
137
- * The locale to use for display names specified through the TFLite Model
138
- * Metadata, if any. Defaults to English.
139
- */
140
- displayNamesLocale?: string | undefined;
141
- /** The maximum number of top-scored detection results to return. */
142
- maxResults?: number | undefined;
143
- /**
144
- * Overrides the value provided in the model metadata. Results below this
145
- * value are rejected.
146
- */
147
- scoreThreshold?: number | undefined;
148
- /**
149
- * Allowlist of category names. If non-empty, detection results whose category
150
- * name is not in this set will be filtered out. Duplicate or unknown category
151
- * names are ignored. Mutually exclusive with `categoryDenylist`.
152
- */
153
- categoryAllowlist?: string[] | undefined;
154
- /**
155
- * Denylist of category names. If non-empty, detection results whose category
156
- * name is in this set will be filtered out. Duplicate or unknown category
157
- * names are ignored. Mutually exclusive with `categoryAllowlist`.
158
- */
159
- categoryDenylist?: string[] | undefined;
160
- }
161
-
162
- /** A connection between two landmarks. */
163
- declare interface Connection {
164
- start: number;
165
- end: number;
166
- }
167
-
168
- /** Represents one detection by a detection task. */
169
- export declare interface Detection {
170
- /** A list of `Category` objects. */
171
- categories: Category[];
172
- /** The bounding box of the detected objects. */
173
- boundingBox?: BoundingBox;
174
- /**
175
- * Optional list of keypoints associated with the detection. Keypoints
176
- * represent interesting points related to the detection. For example, the
177
- * keypoints represent the eye, ear and mouth from face detection model. Or
178
- * in the template matching detection, e.g. KNIFT, they can represent the
179
- * feature points for template matching.
180
- */
181
- keypoints?: NormalizedKeypoint[];
182
- }
183
-
184
- /** Detection results of a model. */
185
- declare interface DetectionResult {
186
- /** A list of Detections. */
187
- detections: Detection[];
188
- }
189
- export { DetectionResult as FaceDetectorResult }
190
- export { DetectionResult as ObjectDetectorResult }
191
-
192
- /**
193
- * Options for customizing the drawing routines
194
- */
195
- export declare interface DrawingOptions {
196
- /** The color that is used to draw the shape. Defaults to white. */
197
- color?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
198
- /**
199
- * The color that is used to fill the shape. Defaults to `.color` (or black
200
- * if color is not set).
201
- */
202
- fillColor?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
203
- /** The width of the line boundary of the shape. Defaults to 4. */
204
- lineWidth?: number | Callback<LandmarkData, number>;
205
- /** The radius of location marker. Defaults to 6. */
206
- radius?: number | Callback<LandmarkData, number>;
207
- }
208
-
209
- /** Helper class to visualize the result of a MediaPipe Vision task. */
210
- export declare class DrawingUtils {
211
- /**
212
- * Creates a new DrawingUtils class.
213
- *
214
- * @param ctx The canvas to render onto.
215
- */
216
- constructor(ctx: CanvasRenderingContext2D);
217
- /**
218
- * Restricts a number between two endpoints (order doesn't matter).
219
- *
220
- * @param x The number to clamp.
221
- * @param x0 The first boundary.
222
- * @param x1 The second boundary.
223
- * @return The clamped value.
224
- */
225
- static clamp(x: number, x0: number, x1: number): number;
226
- /**
227
- * Linearly interpolates a value between two points, clamping that value to
228
- * the endpoints.
229
- *
230
- * @param x The number to interpolate.
231
- * @param x0 The x coordinate of the start value.
232
- * @param x1 The x coordinate of the end value.
233
- * @param y0 The y coordinate of the start value.
234
- * @param y1 The y coordinate of the end value.
235
- * @return The interpolated value.
236
- */
237
- static lerp(x: number, x0: number, x1: number, y0: number, y1: number): number;
238
- /**
239
- * Draws circles onto the provided landmarks.
240
- *
241
- * @param landmarks The landmarks to draw.
242
- * @param style The style to visualize the landmarks.
243
- */
244
- drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions): void;
245
- /**
246
- * Draws lines between landmarks (given a connection graph).
247
- *
248
- * @param landmarks The landmarks to draw.
249
- * @param connections The connections array that contains the start and the
250
- * end indices for the connections to draw.
251
- * @param style The style to visualize the landmarks.
252
- */
253
- drawConnectors(landmarks?: NormalizedLandmark[], connections?: Connection[], style?: DrawingOptions): void;
254
- /**
255
- * Draws a bounding box.
256
- *
257
- * @param boundingBox The bounding box to draw.
258
- * @param style The style to visualize the boundin box.
259
- */
260
- drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void;
261
- }
262
-
263
- /**
264
- * Copyright 2022 The MediaPipe Authors.
265
- *
266
- * Licensed under the Apache License, Version 2.0 (the "License");
267
- * you may not use this file except in compliance with the License.
268
- * You may obtain a copy of the License at
269
- *
270
- * http://www.apache.org/licenses/LICENSE-2.0
271
- *
272
- * Unless required by applicable law or agreed to in writing, software
273
- * distributed under the License is distributed on an "AS IS" BASIS,
274
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
275
- * See the License for the specific language governing permissions and
276
- * limitations under the License.
277
- */
278
- /** Options to configure a MediaPipe Embedder Task */
279
- declare interface EmbedderOptions {
280
- /**
281
- * Whether to normalize the returned feature vector with L2 norm. Use this
282
- * option only if the model does not already contain a native L2_NORMALIZATION
283
- * TF Lite Op. In most cases, this is already the case and L2 norm is thus
284
- * achieved through TF Lite inference.
285
- */
286
- l2Normalize?: boolean | undefined;
287
- /**
288
- * Whether the returned embedding should be quantized to bytes via scalar
289
- * quantization. Embeddings are implicitly assumed to be unit-norm and
290
- * therefore any dimension is guaranteed to have a value in [-1.0, 1.0]. Use
291
- * the l2_normalize option if this is not the case.
292
- */
293
- quantize?: boolean | undefined;
294
- }
295
-
296
- /**
297
- * Copyright 2022 The MediaPipe Authors.
298
- *
299
- * Licensed under the Apache License, Version 2.0 (the "License");
300
- * you may not use this file except in compliance with the License.
301
- * You may obtain a copy of the License at
302
- *
303
- * http://www.apache.org/licenses/LICENSE-2.0
304
- *
305
- * Unless required by applicable law or agreed to in writing, software
306
- * distributed under the License is distributed on an "AS IS" BASIS,
307
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
308
- * See the License for the specific language governing permissions and
309
- * limitations under the License.
310
- */
311
- /**
312
- * List of embeddings with an optional timestamp.
313
- *
314
- * One and only one of the two 'floatEmbedding' and 'quantizedEmbedding' will
315
- * contain data, based on whether or not the embedder was configured to perform
316
- * scalar quantization.
317
- */
318
- export declare interface Embedding {
319
- /**
320
- * Floating-point embedding. Empty if the embedder was configured to perform
321
- * scalar-quantization.
322
- */
323
- floatEmbedding?: number[];
324
- /**
325
- * Scalar-quantized embedding. Empty if the embedder was not configured to
326
- * perform scalar quantization.
327
- */
328
- quantizedEmbedding?: Uint8Array;
329
- /**
330
- * The index of the classifier head these categories refer to. This is
331
- * useful for multi-head models.
332
- */
333
- headIndex: number;
334
- /**
335
- * The name of the classifier head, which is the corresponding tensor
336
- * metadata name.
337
- */
338
- headName: string;
339
- }
340
-
341
- /** Performs face detection on images. */
342
- export declare class FaceDetector extends VisionTaskRunner {
343
- /**
344
- * Initializes the Wasm runtime and creates a new face detector from the
345
- * provided options.
346
- * @param wasmFileset A configuration object that provides the location of the
347
- * Wasm binary and its loader.
348
- * @param faceDetectorOptions The options for the FaceDetector. Note that
349
- * either a path to the model asset or a model buffer needs to be
350
- * provided (via `baseOptions`).
351
- */
352
- static createFromOptions(wasmFileset: WasmFileset, faceDetectorOptions: FaceDetectorOptions): Promise<FaceDetector>;
353
- /**
354
- * Initializes the Wasm runtime and creates a new face detector based on the
355
- * provided model asset buffer.
356
- * @param wasmFileset A configuration object that provides the location of the
357
- * Wasm binary and its loader.
358
- * @param modelAssetBuffer A binary representation of the model.
359
- */
360
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceDetector>;
361
- /**
362
- * Initializes the Wasm runtime and creates a new face detector based on the
363
- * path to the model asset.
364
- * @param wasmFileset A configuration object that provides the location of the
365
- * Wasm binary and its loader.
366
- * @param modelAssetPath The path to the model asset.
367
- */
368
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceDetector>;
369
- private constructor();
370
- /**
371
- * Sets new options for the FaceDetector.
372
- *
373
- * Calling `setOptions()` with a subset of options only affects those options.
374
- * You can reset an option back to its default value by explicitly setting it
375
- * to `undefined`.
376
- *
377
- * @param options The options for the FaceDetector.
378
- */
379
- setOptions(options: FaceDetectorOptions): Promise<void>;
380
- /**
381
- * Performs face detection on the provided single image and waits
382
- * synchronously for the response. Only use this method when the
383
- * FaceDetector is created with running mode `image`.
384
- *
385
- * @param image An image to process.
386
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
387
- * to process the input image before running inference.
388
- * @return A result containing the list of detected faces.
389
- */
390
- detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
391
- /**
392
- * Performs face detection on the provided video frame and waits
393
- * synchronously for the response. Only use this method when the
394
- * FaceDetector is created with running mode `video`.
395
- *
396
- * @param videoFrame A video frame to process.
397
- * @param timestamp The timestamp of the current frame, in ms.
398
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
399
- * to process the input image before running inference.
400
- * @return A result containing the list of detected faces.
401
- */
402
- detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
403
- }
404
-
405
- /** Options to configure the MediaPipe Face Detector Task */
406
- export declare interface FaceDetectorOptions extends VisionTaskOptions {
407
- /**
408
- * The minimum confidence score for the face detection to be considered
409
- * successful. Defaults to 0.5.
410
- */
411
- minDetectionConfidence?: number | undefined;
412
- /**
413
- * The minimum non-maximum-suppression threshold for face detection to be
414
- * considered overlapped. Defaults to 0.3.
415
- */
416
- minSuppressionThreshold?: number | undefined;
417
- }
418
-
419
- /**
420
- * Performs face landmarks detection on images.
421
- *
422
- * This API expects a pre-trained face landmarker model asset bundle.
423
- */
424
- export declare class FaceLandmarker extends VisionTaskRunner {
425
- /**
426
- * Initializes the Wasm runtime and creates a new `FaceLandmarker` from the
427
- * provided options.
428
- * @param wasmFileset A configuration object that provides the location of the
429
- * Wasm binary and its loader.
430
- * @param faceLandmarkerOptions The options for the FaceLandmarker.
431
- * Note that either a path to the model asset or a model buffer needs to
432
- * be provided (via `baseOptions`).
433
- */
434
- static createFromOptions(wasmFileset: WasmFileset, faceLandmarkerOptions: FaceLandmarkerOptions): Promise<FaceLandmarker>;
435
- /**
436
- * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
437
- * the provided model asset buffer.
438
- * @param wasmFileset A configuration object that provides the location of the
439
- * Wasm binary and its loader.
440
- * @param modelAssetBuffer A binary representation of the model.
441
- */
442
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceLandmarker>;
443
- /**
444
- * Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
445
- * the path to the model asset.
446
- * @param wasmFileset A configuration object that provides the location of the
447
- * Wasm binary and its loader.
448
- * @param modelAssetPath The path to the model asset.
449
- */
450
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceLandmarker>;
451
- private constructor();
452
- /**
453
- * Sets new options for this `FaceLandmarker`.
454
- *
455
- * Calling `setOptions()` with a subset of options only affects those options.
456
- * You can reset an option back to its default value by explicitly setting it
457
- * to `undefined`.
458
- *
459
- * @param options The options for the face landmarker.
460
- */
461
- setOptions(options: FaceLandmarkerOptions): Promise<void>;
462
- /**
463
- * Performs face landmarks detection on the provided single image and waits
464
- * synchronously for the response. Only use this method when the
465
- * FaceLandmarker is created with running mode `image`.
466
- *
467
- * @param image An image to process.
468
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
469
- * to process the input image before running inference.
470
- * @return The detected face landmarks.
471
- */
472
- detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
473
- /**
474
- * Performs face landmarks detection on the provided video frame and waits
475
- * synchronously for the response. Only use this method when the
476
- * FaceLandmarker is created with running mode `video`.
477
- *
478
- * @param videoFrame A video frame to process.
479
- * @param timestamp The timestamp of the current frame, in ms.
480
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
481
- * to process the input image before running inference.
482
- * @return The detected face landmarks.
483
- */
484
- detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
485
- }
486
-
487
- /** Options to configure the MediaPipe FaceLandmarker Task */
488
- export declare interface FaceLandmarkerOptions extends VisionTaskOptions {
489
- /**
490
- * The maximum number of faces can be detected by the FaceLandmarker.
491
- * Defaults to 1.
492
- */
493
- numFaces?: number | undefined;
494
- /**
495
- * The minimum confidence score for the face detection to be considered
496
- * successful. Defaults to 0.5.
497
- */
498
- minFaceDetectionConfidence?: number | undefined;
499
- /**
500
- * The minimum confidence score of face presence score in the face landmark
501
- * detection. Defaults to 0.5.
502
- */
503
- minFacePresenceConfidence?: number | undefined;
504
- /**
505
- * The minimum confidence score for the face tracking to be considered
506
- * successful. Defaults to 0.5.
507
- */
508
- minTrackingConfidence?: number | undefined;
509
- /**
510
- * Whether FaceLandmarker outputs face blendshapes classification. Face
511
- * blendshapes are used for rendering the 3D face model.
512
- */
513
- outputFaceBlendshapes?: boolean | undefined;
514
- /**
515
- * Whether FaceLandmarker outputs facial transformation_matrix. Facial
516
- * transformation matrix is used to transform the face landmarks in canonical
517
- * face to the detected face, so that users can apply face effects on the
518
- * detected landmarks.
519
- */
520
- outputFacialTransformationMatrixes?: boolean | undefined;
521
- }
522
-
523
- /**
524
- * Represents the face landmarks deection results generated by `FaceLandmarker`.
525
- */
526
- export declare interface FaceLandmarkerResult {
527
- /** Detected face landmarks in normalized image coordinates. */
528
- faceLandmarks: NormalizedLandmark[][];
529
- /** Optional face blendshapes results. */
530
- faceBlendshapes?: Classifications[];
531
- /** Optional facial transformation matrix. */
532
- facialTransformationMatrixes?: Matrix[];
533
- }
534
-
535
- /**
536
- * A class containing the pairs of landmark indices to be rendered with
537
- * connections.
538
- */
539
- export declare class FaceLandmarksConnections {
540
- static FACE_LANDMARKS_LIPS: Connection[];
541
- static FACE_LANDMARKS_LEFT_EYE: Connection[];
542
- static FACE_LANDMARKS_LEFT_EYEBROW: Connection[];
543
- static FACE_LANDMARKS_LEFT_IRIS: Connection[];
544
- static FACE_LANDMARKS_RIGHT_EYE: Connection[];
545
- static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[];
546
- static FACE_LANDMARKS_RIGHT_IRIS: Connection[];
547
- static FACE_LANDMARKS_FACE_OVAL: Connection[];
548
- static FACE_LANDMARKS_CONTOURS: Connection[];
549
- static FACE_LANDMARKS_TESSELATION: Connection[];
550
- }
551
-
552
- /** Performs face stylization on images. */
553
- export declare class FaceStylizer extends VisionTaskRunner {
554
- /**
555
- * Initializes the Wasm runtime and creates a new Face Stylizer from the
556
- * provided options.
557
- * @param wasmFileset A configuration object that provides the location of
558
- * the Wasm binary and its loader.
559
- * @param faceStylizerOptions The options for the Face Stylizer. Note
560
- * that either a path to the model asset or a model buffer needs to be
561
- * provided (via `baseOptions`).
562
- */
563
- static createFromOptions(wasmFileset: WasmFileset, faceStylizerOptions: FaceStylizerOptions): Promise<FaceStylizer>;
564
- /**
565
- * Initializes the Wasm runtime and creates a new Face Stylizer based on
566
- * the provided model asset buffer.
567
- * @param wasmFileset A configuration object that provides the location of
568
- * the Wasm binary and its loader.
569
- * @param modelAssetBuffer A binary representation of the model.
570
- */
571
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceStylizer>;
572
- /**
573
- * Initializes the Wasm runtime and creates a new Face Stylizer based on
574
- * the path to the model asset.
575
- * @param wasmFileset A configuration object that provides the location of
576
- * the Wasm binary and its loader.
577
- * @param modelAssetPath The path to the model asset.
578
- */
579
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceStylizer>;
580
- private constructor();
581
- /**
582
- * Sets new options for the Face Stylizer.
583
- *
584
- * Calling `setOptions()` with a subset of options only affects those
585
- * options. You can reset an option back to its default value by
586
- * explicitly setting it to `undefined`.
587
- *
588
- * @param options The options for the Face Stylizer.
589
- */
590
- setOptions(options: FaceStylizerOptions): Promise<void>;
591
- /**
592
- * Performs face stylization on the provided single image. The method returns
593
- * synchronously once the callback returns. Only use this method when the
594
- * FaceStylizer is created with the image running mode.
595
- *
596
- * @param image An image to process.
597
- * @param callback The callback that is invoked with the stylized image. The
598
- * lifetime of the returned data is only guaranteed for the duration of the
599
- * callback.
600
- */
601
- stylize(image: ImageSource, callback: FaceStylizerCallback): void;
602
- /**
603
- * Performs face stylization on the provided single image. The method returns
604
- * synchronously once the callback returns. Only use this method when the
605
- * FaceStylizer is created with the image running mode.
606
- *
607
- * The 'imageProcessingOptions' parameter can be used to specify one or all
608
- * of:
609
- * - the rotation to apply to the image before performing stylization, by
610
- * setting its 'rotationDegrees' property.
611
- * - the region-of-interest on which to perform stylization, by setting its
612
- * 'regionOfInterest' property. If not specified, the full image is used.
613
- * If both are specified, the crop around the region-of-interest is extracted
614
- * first, then the specified rotation is applied to the crop.
615
- *
616
- * @param image An image to process.
617
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
618
- * to process the input image before running inference.
619
- * @param callback The callback that is invoked with the stylized image. The
620
- * lifetime of the returned data is only guaranteed for the duration of the
621
- * callback.
622
- */
623
- stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: FaceStylizerCallback): void;
624
- /**
625
- * Performs face stylization on the provided video frame. Only use this method
626
- * when the FaceStylizer is created with the video running mode.
627
- *
628
- * The input frame can be of any size. It's required to provide the video
629
- * frame's timestamp (in milliseconds). The input timestamps must be
630
- * monotonically increasing.
631
- *
632
- * @param videoFrame A video frame to process.
633
- * @param timestamp The timestamp of the current frame, in ms.
634
- * @param callback The callback that is invoked with the stylized image. The
635
- * lifetime of the returned data is only guaranteed for the duration of
636
- * the callback.
637
- */
638
- stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback: FaceStylizerCallback): void;
639
- /**
640
- * Performs face stylization on the provided video frame. Only use this
641
- * method when the FaceStylizer is created with the video running mode.
642
- *
643
- * The 'imageProcessingOptions' parameter can be used to specify one or all
644
- * of:
645
- * - the rotation to apply to the image before performing stylization, by
646
- * setting its 'rotationDegrees' property.
647
- * - the region-of-interest on which to perform stylization, by setting its
648
- * 'regionOfInterest' property. If not specified, the full image is used.
649
- * If both are specified, the crop around the region-of-interest is
650
- * extracted first, then the specified rotation is applied to the crop.
651
- *
652
- * The input frame can be of any size. It's required to provide the video
653
- * frame's timestamp (in milliseconds). The input timestamps must be
654
- * monotonically increasing.
655
- *
656
- * @param videoFrame A video frame to process.
657
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
658
- * to process the input image before running inference.
659
- * @param timestamp The timestamp of the current frame, in ms.
660
- * @param callback The callback that is invoked with the stylized image. The
661
- * lifetime of the returned data is only guaranteed for the duration of
662
- * the callback.
663
- */
664
- stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: FaceStylizerCallback): void;
665
- }
666
-
667
- /**
668
- * A callback that receives an image from the face stylizer, or `null` if no
669
- * face was detected. The lifetime of the underlying data is limited to the
670
- * duration of the callback. If asynchronous processing is needed, all data
671
- * needs to be copied before the callback returns.
672
- *
673
- * The `WebGLTexture` output type is reserved for future usage.
674
- */
675
- export declare type FaceStylizerCallback = (image: ImageData | WebGLTexture | null, width: number, height: number) => void;
676
-
677
- /** Options to configure the MediaPipe Face Stylizer Task */
678
- export declare interface FaceStylizerOptions extends VisionTaskOptions {
679
- }
680
-
681
- /**
682
- * Resolves the files required for the MediaPipe Task APIs.
683
- *
684
- * This class verifies whether SIMD is supported in the current environment and
685
- * loads the SIMD files only if support is detected. The returned filesets
686
- * require that the Wasm files are published without renaming. If this is not
687
- * possible, you can invoke the MediaPipe Tasks APIs using a manually created
688
- * `WasmFileset`.
689
- */
690
- export declare class FilesetResolver {
691
- /**
692
- * Returns whether SIMD is supported in the current environment.
693
- *
694
- * If your environment requires custom locations for the MediaPipe Wasm files,
695
- * you can use `isSimdSupported()` to decide whether to load the SIMD-based
696
- * assets.
697
- *
698
- * @return Whether SIMD support was detected in the current environment.
699
- */
700
- static isSimdSupported(): Promise<boolean>;
701
- /**
702
- * Creates a fileset for the MediaPipe Audio tasks.
703
- *
704
- * @param basePath An optional base path to specify the directory the Wasm
705
- * files should be loaded from. If not specified, the Wasm files are
706
- * loaded from the host's root directory.
707
- * @return A `WasmFileset` that can be used to initialize MediaPipe Audio
708
- * tasks.
709
- */
710
- static forAudioTasks(basePath?: string): Promise<WasmFileset>;
711
- /**
712
- * Creates a fileset for the MediaPipe Text tasks.
713
- *
714
- * @param basePath An optional base path to specify the directory the Wasm
715
- * files should be loaded from. If not specified, the Wasm files are
716
- * loaded from the host's root directory.
717
- * @return A `WasmFileset` that can be used to initialize MediaPipe Text
718
- * tasks.
719
- */
720
- static forTextTasks(basePath?: string): Promise<WasmFileset>;
721
- /**
722
- * Creates a fileset for the MediaPipe Vision tasks.
723
- *
724
- * @param basePath An optional base path to specify the directory the Wasm
725
- * files should be loaded from. If not specified, the Wasm files are
726
- * loaded from the host's root directory.
727
- * @return A `WasmFileset` that can be used to initialize MediaPipe Vision
728
- * tasks.
729
- */
730
- static forVisionTasks(basePath?: string): Promise<WasmFileset>;
731
- }
732
-
733
- /** Performs hand gesture recognition on images. */
734
- export declare class GestureRecognizer extends VisionTaskRunner {
735
- /**
736
- * An array containing the pairs of hand landmark indices to be rendered with
737
- * connections.
738
- */
739
- static HAND_CONNECTIONS: Connection[];
740
- /**
741
- * Initializes the Wasm runtime and creates a new gesture recognizer from the
742
- * provided options.
743
- * @param wasmFileset A configuration object that provides the location of the
744
- * Wasm binary and its loader.
745
- * @param gestureRecognizerOptions The options for the gesture recognizer.
746
- * Note that either a path to the model asset or a model buffer needs to
747
- * be provided (via `baseOptions`).
748
- */
749
- static createFromOptions(wasmFileset: WasmFileset, gestureRecognizerOptions: GestureRecognizerOptions): Promise<GestureRecognizer>;
750
- /**
751
- * Initializes the Wasm runtime and creates a new gesture recognizer based on
752
- * the provided model asset buffer.
753
- * @param wasmFileset A configuration object that provides the location of the
754
- * Wasm binary and its loader.
755
- * @param modelAssetBuffer A binary representation of the model.
756
- */
757
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<GestureRecognizer>;
758
- /**
759
- * Initializes the Wasm runtime and creates a new gesture recognizer based on
760
- * the path to the model asset.
761
- * @param wasmFileset A configuration object that provides the location of the
762
- * Wasm binary and its loader.
763
- * @param modelAssetPath The path to the model asset.
764
- */
765
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<GestureRecognizer>;
766
- private constructor();
767
- /**
768
- * Sets new options for the gesture recognizer.
769
- *
770
- * Calling `setOptions()` with a subset of options only affects those options.
771
- * You can reset an option back to its default value by explicitly setting it
772
- * to `undefined`.
773
- *
774
- * @param options The options for the gesture recognizer.
775
- */
776
- setOptions(options: GestureRecognizerOptions): Promise<void>;
777
- /**
778
- * Performs gesture recognition on the provided single image and waits
779
- * synchronously for the response. Only use this method when the
780
- * GestureRecognizer is created with running mode `image`.
781
- *
782
- * @param image A single image to process.
783
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
784
- * to process the input image before running inference.
785
- * @return The detected gestures.
786
- */
787
- recognize(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
788
- /**
789
- * Performs gesture recognition on the provided video frame and waits
790
- * synchronously for the response. Only use this method when the
791
- * GestureRecognizer is created with running mode `video`.
792
- *
793
- * @param videoFrame A video frame to process.
794
- * @param timestamp The timestamp of the current frame, in ms.
795
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
796
- * to process the input image before running inference.
797
- * @return The detected gestures.
798
- */
799
- recognizeForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
800
- }
801
-
802
- /** Options to configure the MediaPipe Gesture Recognizer Task */
803
- export declare interface GestureRecognizerOptions extends VisionTaskOptions {
804
- /**
805
- * The maximum number of hands can be detected by the GestureRecognizer.
806
- * Defaults to 1.
807
- */
808
- numHands?: number | undefined;
809
- /**
810
- * The minimum confidence score for the hand detection to be considered
811
- * successful. Defaults to 0.5.
812
- */
813
- minHandDetectionConfidence?: number | undefined;
814
- /**
815
- * The minimum confidence score of hand presence score in the hand landmark
816
- * detection. Defaults to 0.5.
817
- */
818
- minHandPresenceConfidence?: number | undefined;
819
- /**
820
- * The minimum confidence score for the hand tracking to be considered
821
- * successful. Defaults to 0.5.
822
- */
823
- minTrackingConfidence?: number | undefined;
824
- /**
825
- * Sets the optional `ClassifierOptions` controlling the canned gestures
826
- * classifier, such as score threshold, allow list and deny list of gestures.
827
- * The categories for canned gesture
828
- * classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up",
829
- * "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
830
- */
831
- cannedGesturesClassifierOptions?: ClassifierOptions | undefined;
832
- /**
833
- * Options for configuring the custom gestures classifier, such as score
834
- * threshold, allow list and deny list of gestures.
835
- */
836
- customGesturesClassifierOptions?: ClassifierOptions | undefined;
837
- }
838
-
839
- /**
840
- * Represents the gesture recognition results generated by `GestureRecognizer`.
841
- */
842
- export declare interface GestureRecognizerResult {
843
- /** Hand landmarks of detected hands. */
844
- landmarks: NormalizedLandmark[][];
845
- /** Hand landmarks in world coordniates of detected hands. */
846
- worldLandmarks: Landmark[][];
847
- /** Handedness of detected hands. */
848
- handednesses: Category[][];
849
- /**
850
- * Recognized hand gestures of detected hands. Note that the index of the
851
- * gesture is always -1, because the raw indices from multiple gesture
852
- * classifiers cannot consolidate to a meaningful index.
853
- */
854
- gestures: Category[][];
855
- }
856
-
857
- /** Performs hand landmarks detection on images. */
858
- export declare class HandLandmarker extends VisionTaskRunner {
859
- /**
860
- * An array containing the pairs of hand landmark indices to be rendered with
861
- * connections.
862
- */
863
- static HAND_CONNECTIONS: Connection[];
864
- /**
865
- * Initializes the Wasm runtime and creates a new `HandLandmarker` from the
866
- * provided options.
867
- * @param wasmFileset A configuration object that provides the location of the
868
- * Wasm binary and its loader.
869
- * @param handLandmarkerOptions The options for the HandLandmarker.
870
- * Note that either a path to the model asset or a model buffer needs to
871
- * be provided (via `baseOptions`).
872
- */
873
- static createFromOptions(wasmFileset: WasmFileset, handLandmarkerOptions: HandLandmarkerOptions): Promise<HandLandmarker>;
874
- /**
875
- * Initializes the Wasm runtime and creates a new `HandLandmarker` based on
876
- * the provided model asset buffer.
877
- * @param wasmFileset A configuration object that provides the location of the
878
- * Wasm binary and its loader.
879
- * @param modelAssetBuffer A binary representation of the model.
880
- */
881
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<HandLandmarker>;
882
- /**
883
- * Initializes the Wasm runtime and creates a new `HandLandmarker` based on
884
- * the path to the model asset.
885
- * @param wasmFileset A configuration object that provides the location of the
886
- * Wasm binary and its loader.
887
- * @param modelAssetPath The path to the model asset.
888
- */
889
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<HandLandmarker>;
890
- private constructor();
891
- /**
892
- * Sets new options for this `HandLandmarker`.
893
- *
894
- * Calling `setOptions()` with a subset of options only affects those options.
895
- * You can reset an option back to its default value by explicitly setting it
896
- * to `undefined`.
897
- *
898
- * @param options The options for the hand landmarker.
899
- */
900
- setOptions(options: HandLandmarkerOptions): Promise<void>;
901
- /**
902
- * Performs hand landmarks detection on the provided single image and waits
903
- * synchronously for the response. Only use this method when the
904
- * HandLandmarker is created with running mode `image`.
905
- *
906
- * @param image An image to process.
907
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
908
- * to process the input image before running inference.
909
- * @return The detected hand landmarks.
910
- */
911
- detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
912
- /**
913
- * Performs hand landmarks detection on the provided video frame and waits
914
- * synchronously for the response. Only use this method when the
915
- * HandLandmarker is created with running mode `video`.
916
- *
917
- * @param videoFrame A video frame to process.
918
- * @param timestamp The timestamp of the current frame, in ms.
919
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
920
- * to process the input image before running inference.
921
- * @return The detected hand landmarks.
922
- */
923
- detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
924
- }
925
-
926
- /** Options to configure the MediaPipe HandLandmarker Task */
927
- export declare interface HandLandmarkerOptions extends VisionTaskOptions {
928
- /**
929
- * The maximum number of hands can be detected by the HandLandmarker.
930
- * Defaults to 1.
931
- */
932
- numHands?: number | undefined;
933
- /**
934
- * The minimum confidence score for the hand detection to be considered
935
- * successful. Defaults to 0.5.
936
- */
937
- minHandDetectionConfidence?: number | undefined;
938
- /**
939
- * The minimum confidence score of hand presence score in the hand landmark
940
- * detection. Defaults to 0.5.
941
- */
942
- minHandPresenceConfidence?: number | undefined;
943
- /**
944
- * The minimum confidence score for the hand tracking to be considered
945
- * successful. Defaults to 0.5.
946
- */
947
- minTrackingConfidence?: number | undefined;
948
- }
949
-
950
- /**
951
- * Represents the hand landmarks deection results generated by `HandLandmarker`.
952
- */
953
- export declare interface HandLandmarkerResult {
954
- /** Hand landmarks of detected hands. */
955
- landmarks: NormalizedLandmark[][];
956
- /** Hand landmarks in world coordinates of detected hands. */
957
- worldLandmarks: Landmark[][];
958
- /** Handedness of detected hands. */
959
- handednesses: Category[][];
960
- }
961
-
962
- /** Performs classification on images. */
963
- export declare class ImageClassifier extends VisionTaskRunner {
964
- /**
965
- * Initializes the Wasm runtime and creates a new image classifier from the
966
- * provided options.
967
- * @param wasmFileset A configuration object that provides the location
968
- * Wasm binary and its loader.
969
- * @param imageClassifierOptions The options for the image classifier. Note
970
- * that either a path to the model asset or a model buffer needs to be
971
- * provided (via `baseOptions`).
972
- */
973
- static createFromOptions(wasmFileset: WasmFileset, imageClassifierOptions: ImageClassifierOptions): Promise<ImageClassifier>;
974
- /**
975
- * Initializes the Wasm runtime and creates a new image classifier based on
976
- * the provided model asset buffer.
977
- * @param wasmFileset A configuration object that provides the location of the
978
- * Wasm binary and its loader.
979
- * @param modelAssetBuffer A binary representation of the model.
980
- */
981
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageClassifier>;
982
- /**
983
- * Initializes the Wasm runtime and creates a new image classifier based on
984
- * the path to the model asset.
985
- * @param wasmFileset A configuration object that provides the location of the
986
- * Wasm binary and its loader.
987
- * @param modelAssetPath The path to the model asset.
988
- */
989
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageClassifier>;
990
- private constructor();
991
- /**
992
- * Sets new options for the image classifier.
993
- *
994
- * Calling `setOptions()` with a subset of options only affects those options.
995
- * You can reset an option back to its default value by explicitly setting it
996
- * to `undefined`.
997
- *
998
- * @param options The options for the image classifier.
999
- */
1000
- setOptions(options: ImageClassifierOptions): Promise<void>;
1001
- /**
1002
- * Performs image classification on the provided single image and waits
1003
- * synchronously for the response. Only use this method when the
1004
- * ImageClassifier is created with running mode `image`.
1005
- *
1006
- * @param image An image to process.
1007
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1008
- * to process the input image before running inference.
1009
- * @return The classification result of the image
1010
- */
1011
- classify(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1012
- /**
1013
- * Performs image classification on the provided video frame and waits
1014
- * synchronously for the response. Only use this method when the
1015
- * ImageClassifier is created with running mode `video`.
1016
- *
1017
- * @param videoFrame A video frame to process.
1018
- * @param timestamp The timestamp of the current frame, in ms.
1019
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1020
- * to process the input image before running inference.
1021
- * @return The classification result of the image
1022
- */
1023
- classifyForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1024
- }
1025
-
1026
- /** Options to configure the MediaPipe Image Classifier Task. */
1027
- export declare interface ImageClassifierOptions extends ClassifierOptions, VisionTaskOptions {
1028
- }
1029
-
1030
- /** Classification results of a model. */
1031
- export declare interface ImageClassifierResult {
1032
- /** The classification results for each head of the model. */
1033
- classifications: Classifications[];
1034
- /**
1035
- * The optional timestamp (in milliseconds) of the start of the chunk of data
1036
- * corresponding to these results.
1037
- *
1038
- * This is only used for classification on time series (e.g. audio
1039
- * classification). In these use cases, the amount of data to process might
1040
- * exceed the maximum size that the model can process: to solve this, the
1041
- * input data is split into multiple chunks starting at different timestamps.
1042
- */
1043
- timestampMs?: number;
1044
- }
1045
-
1046
- /** Performs embedding extraction on images. */
1047
- export declare class ImageEmbedder extends VisionTaskRunner {
1048
- /**
1049
- * Initializes the Wasm runtime and creates a new image embedder from the
1050
- * provided options.
1051
- * @param wasmFileset A configuration object that provides the location of the
1052
- * Wasm binary and its loader.
1053
- * @param imageEmbedderOptions The options for the image embedder. Note that
1054
- * either a path to the TFLite model or the model itself needs to be
1055
- * provided (via `baseOptions`).
1056
- */
1057
- static createFromOptions(wasmFileset: WasmFileset, imageEmbedderOptions: ImageEmbedderOptions): Promise<ImageEmbedder>;
1058
- /**
1059
- * Initializes the Wasm runtime and creates a new image embedder based on the
1060
- * provided model asset buffer.
1061
- * @param wasmFileset A configuration object that provides the location of the
1062
- * Wasm binary and its loader.
1063
- * @param modelAssetBuffer A binary representation of the TFLite model.
1064
- */
1065
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageEmbedder>;
1066
- /**
1067
- * Initializes the Wasm runtime and creates a new image embedder based on the
1068
- * path to the model asset.
1069
- * @param wasmFileset A configuration object that provides the location of the
1070
- * Wasm binary and its loader.
1071
- * @param modelAssetPath The path to the TFLite model.
1072
- */
1073
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageEmbedder>;
1074
- private constructor();
1075
- /**
1076
- * Sets new options for the image embedder.
1077
- *
1078
- * Calling `setOptions()` with a subset of options only affects those options.
1079
- * You can reset an option back to its default value by explicitly setting it
1080
- * to `undefined`.
1081
- *
1082
- * @param options The options for the image embedder.
1083
- */
1084
- setOptions(options: ImageEmbedderOptions): Promise<void>;
1085
- /**
1086
- * Performs embedding extraction on the provided single image and waits
1087
- * synchronously for the response. Only use this method when the
1088
- * ImageEmbedder is created with running mode `image`.
1089
- *
1090
- * @param image The image to process.
1091
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1092
- * to process the input image before running inference.
1093
- * @return The classification result of the image
1094
- */
1095
- embed(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1096
- /**
1097
- * Performs embedding extraction on the provided video frame and waits
1098
- * synchronously for the response. Only use this method when the
1099
- * ImageEmbedder is created with running mode `video`.
1100
- *
1101
- * @param imageFrame The image frame to process.
1102
- * @param timestamp The timestamp of the current frame, in ms.
1103
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1104
- * to process the input image before running inference.
1105
- * @return The classification result of the image
1106
- */
1107
- embedForVideo(imageFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1108
- /**
1109
- * Utility function to compute cosine similarity[1] between two `Embedding`
1110
- * objects.
1111
- *
1112
- * [1]: https://en.wikipedia.org/wiki/Cosine_similarity
1113
- *
1114
- * @throws if the embeddings are of different types(float vs. quantized), have
1115
- * different sizes, or have an L2-norm of 0.
1116
- */
1117
- static cosineSimilarity(u: Embedding, v: Embedding): number;
1118
- }
1119
-
1120
- /** Options for configuring a MediaPipe Image Embedder task. */
1121
- export declare interface ImageEmbedderOptions extends EmbedderOptions, VisionTaskOptions {
1122
- }
1123
-
1124
- /** Embedding results for a given embedder model. */
1125
- export declare interface ImageEmbedderResult {
1126
- /**
1127
- * The embedding results for each model head, i.e. one for each output tensor.
1128
- */
1129
- embeddings: Embedding[];
1130
- /**
1131
- * The optional timestamp (in milliseconds) of the start of the chunk of
1132
- * data corresponding to these results.
1133
- *
1134
- * This is only used for embedding extraction on time series (e.g. audio
1135
- * embedding). In these use cases, the amount of data to process might
1136
- * exceed the maximum size that the model can process: to solve this, the
1137
- * input data is split into multiple chunks starting at different timestamps.
1138
- */
1139
- timestampMs?: number;
1140
- }
1141
-
1142
- /**
1143
- * Options for image processing.
1144
- *
1145
- * If both region-or-interest and rotation are specified, the crop around the
1146
- * region-of-interest is extracted first, then the specified rotation is applied
1147
- * to the crop.
1148
- */
1149
- declare interface ImageProcessingOptions {
1150
- /**
1151
- * The optional region-of-interest to crop from the image. If not specified,
1152
- * the full image is used.
1153
- *
1154
- * Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
1155
- */
1156
- regionOfInterest?: RectF;
1157
- /**
1158
- * The rotation to apply to the image (or cropped region-of-interest), in
1159
- * degrees clockwise.
1160
- *
1161
- * The rotation must be a multiple (positive or negative) of 90°.
1162
- */
1163
- rotationDegrees?: number;
1164
- }
1165
-
1166
- /** Performs image segmentation on images. */
1167
- export declare class ImageSegmenter extends VisionTaskRunner {
1168
- /**
1169
- * Initializes the Wasm runtime and creates a new image segmenter from the
1170
- * provided options.
1171
- * @param wasmFileset A configuration object that provides the location of
1172
- * the Wasm binary and its loader.
1173
- * @param imageSegmenterOptions The options for the Image Segmenter. Note
1174
- * that either a path to the model asset or a model buffer needs to be
1175
- * provided (via `baseOptions`).
1176
- */
1177
- static createFromOptions(wasmFileset: WasmFileset, imageSegmenterOptions: ImageSegmenterOptions): Promise<ImageSegmenter>;
1178
- /**
1179
- * Initializes the Wasm runtime and creates a new image segmenter based on
1180
- * the provided model asset buffer.
1181
- * @param wasmFileset A configuration object that provides the location of
1182
- * the Wasm binary and its loader.
1183
- * @param modelAssetBuffer A binary representation of the model.
1184
- */
1185
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageSegmenter>;
1186
- /**
1187
- * Initializes the Wasm runtime and creates a new image segmenter based on
1188
- * the path to the model asset.
1189
- * @param wasmFileset A configuration object that provides the location of
1190
- * the Wasm binary and its loader.
1191
- * @param modelAssetPath The path to the model asset.
1192
- */
1193
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageSegmenter>;
1194
- private constructor();
1195
- /**
1196
- * Sets new options for the image segmenter.
1197
- *
1198
- * Calling `setOptions()` with a subset of options only affects those
1199
- * options. You can reset an option back to its default value by
1200
- * explicitly setting it to `undefined`.
1201
- *
1202
- * @param options The options for the image segmenter.
1203
- */
1204
- setOptions(options: ImageSegmenterOptions): Promise<void>;
1205
- /**
1206
- * Performs image segmentation on the provided single image and invokes the
1207
- * callback with the response. The method returns synchronously once the
1208
- * callback returns. Only use this method when the ImageSegmenter is
1209
- * created with running mode `image`.
1210
- *
1211
- * @param image An image to process.
1212
- * @param callback The callback that is invoked with the segmented masks. The
1213
- * lifetime of the returned data is only guaranteed for the duration of the
1214
- * callback.
1215
- */
1216
- segment(image: ImageSource, callback: ImageSegmenterCallback): void;
1217
- /**
1218
- * Performs image segmentation on the provided single image and invokes the
1219
- * callback with the response. The method returns synchronously once the
1220
- * callback returns. Only use this method when the ImageSegmenter is
1221
- * created with running mode `image`.
1222
- *
1223
- * @param image An image to process.
1224
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1225
- * to process the input image before running inference.
1226
- * @param callback The callback that is invoked with the segmented masks. The
1227
- * lifetime of the returned data is only guaranteed for the duration of the
1228
- * callback.
1229
- */
1230
- segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallback): void;
1231
- /**
1232
- * Performs image segmentation on the provided video frame and invokes the
1233
- * callback with the response. The method returns synchronously once the
1234
- * callback returns. Only use this method when the ImageSegmenter is
1235
- * created with running mode `video`.
1236
- *
1237
- * @param videoFrame A video frame to process.
1238
- * @param timestamp The timestamp of the current frame, in ms.
1239
- * @param callback The callback that is invoked with the segmented masks. The
1240
- * lifetime of the returned data is only guaranteed for the duration of the
1241
- * callback.
1242
- */
1243
- segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallback): void;
1244
- /**
1245
- * Performs image segmentation on the provided video frame and invokes the
1246
- * callback with the response. The method returns synchronously once the
1247
- * callback returns. Only use this method when the ImageSegmenter is
1248
- * created with running mode `video`.
1249
- *
1250
- * @param videoFrame A video frame to process.
1251
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1252
- * to process the input image before running inference.
1253
- * @param timestamp The timestamp of the current frame, in ms.
1254
- * @param callback The callback that is invoked with the segmented masks. The
1255
- * lifetime of the returned data is only guaranteed for the duration of the
1256
- * callback.
1257
- */
1258
- segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageSegmenterCallback): void;
1259
- /**
1260
- * Get the category label list of the ImageSegmenter can recognize. For
1261
- * `CATEGORY_MASK` type, the index in the category mask corresponds to the
1262
- * category in the label list. For `CONFIDENCE_MASK` type, the output mask
1263
- * list at index corresponds to the category in the label list.
1264
- *
1265
- * If there is no labelmap provided in the model file, empty label array is
1266
- * returned.
1267
- *
1268
- * @return The labels used by the current model.
1269
- */
1270
- getLabels(): string[];
1271
- }
1272
-
1273
- /**
1274
- * A callback that receives the computed masks from the image segmenter. The
1275
- * returned data is only valid for the duration of the callback. If
1276
- * asynchronous processing is needed, all data needs to be copied before the
1277
- * callback returns.
1278
- */
1279
- export declare type ImageSegmenterCallback = (result: ImageSegmenterResult) => void;
1280
-
1281
- /** Options to configure the MediaPipe Image Segmenter Task */
1282
- export declare interface ImageSegmenterOptions extends VisionTaskOptions {
1283
- /**
1284
- * The locale to use for display names specified through the TFLite Model
1285
- * Metadata, if any. Defaults to English.
1286
- */
1287
- displayNamesLocale?: string | undefined;
1288
- /** Whether to output confidence masks. Defaults to true. */
1289
- outputConfidenceMasks?: boolean | undefined;
1290
- /** Whether to output the category masks. Defaults to false. */
1291
- outputCategoryMask?: boolean | undefined;
1292
- }
1293
-
1294
- /**
1295
- * Copyright 2023 The MediaPipe Authors.
1296
- *
1297
- * Licensed under the Apache License, Version 2.0 (the "License");
1298
- * you may not use this file except in compliance with the License.
1299
- * You may obtain a copy of the License at
1300
- *
1301
- * http://www.apache.org/licenses/LICENSE-2.0
1302
- *
1303
- * Unless required by applicable law or agreed to in writing, software
1304
- * distributed under the License is distributed on an "AS IS" BASIS,
1305
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1306
- * See the License for the specific language governing permissions and
1307
- * limitations under the License.
1308
- */
1309
- /** The output result of ImageSegmenter. */
1310
- export declare interface ImageSegmenterResult {
1311
- /**
1312
- * Multiple masks as Float32Arrays or WebGLTextures where, for each mask, each
1313
- * pixel represents the prediction confidence, usually in the [0, 1] range.
1314
- */
1315
- confidenceMasks?: Float32Array[] | WebGLTexture[];
1316
- /**
1317
- * A category mask as a Uint8ClampedArray or WebGLTexture where each
1318
- * pixel represents the class which the pixel in the original image was
1319
- * predicted to belong to.
1320
- */
1321
- categoryMask?: Uint8ClampedArray | WebGLTexture;
1322
- /** The width of the masks. */
1323
- width: number;
1324
- /** The height of the masks. */
1325
- height: number;
1326
- }
1327
-
1328
- /**
1329
- * Valid types of image sources which we can run our GraphRunner over.
1330
- */
1331
- export declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
1332
-
1333
- /**
1334
- * Performs interactive segmentation on images.
1335
- *
1336
- * Users can represent user interaction through `RegionOfInterest`, which gives
1337
- * a hint to InteractiveSegmenter to perform segmentation focusing on the given
1338
- * region of interest.
1339
- *
1340
- * The API expects a TFLite model with mandatory TFLite Model Metadata.
1341
- *
1342
- * Input tensor:
1343
- * (kTfLiteUInt8/kTfLiteFloat32)
1344
- * - image input of size `[batch x height x width x channels]`.
1345
- * - batch inference is not supported (`batch` is required to be 1).
1346
- * - RGB inputs is supported (`channels` is required to be 3).
1347
- * - if type is kTfLiteFloat32, NormalizationOptions are required to be
1348
- * attached to the metadata for input normalization.
1349
- * Output tensors:
1350
- * (kTfLiteUInt8/kTfLiteFloat32)
1351
- * - list of segmented masks.
1352
- * - if `output_type` is CATEGORY_MASK, uint8 Image, Image vector of size 1.
1353
- * - if `output_type` is CONFIDENCE_MASK, float32 Image list of size
1354
- * `channels`.
1355
- * - batch is always 1
1356
- */
1357
- export declare class InteractiveSegmenter extends VisionTaskRunner {
1358
- /**
1359
- * Initializes the Wasm runtime and creates a new interactive segmenter from
1360
- * the provided options.
1361
- * @param wasmFileset A configuration object that provides the location of
1362
- * the Wasm binary and its loader.
1363
- * @param interactiveSegmenterOptions The options for the Interactive
1364
- * Segmenter. Note that either a path to the model asset or a model buffer
1365
- * needs to be provided (via `baseOptions`).
1366
- * @return A new `InteractiveSegmenter`.
1367
- */
1368
- static createFromOptions(wasmFileset: WasmFileset, interactiveSegmenterOptions: InteractiveSegmenterOptions): Promise<InteractiveSegmenter>;
1369
- /**
1370
- * Initializes the Wasm runtime and creates a new interactive segmenter based
1371
- * on the provided model asset buffer.
1372
- * @param wasmFileset A configuration object that provides the location of
1373
- * the Wasm binary and its loader.
1374
- * @param modelAssetBuffer A binary representation of the model.
1375
- * @return A new `InteractiveSegmenter`.
1376
- */
1377
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<InteractiveSegmenter>;
1378
- /**
1379
- * Initializes the Wasm runtime and creates a new interactive segmenter based
1380
- * on the path to the model asset.
1381
- * @param wasmFileset A configuration object that provides the location of
1382
- * the Wasm binary and its loader.
1383
- * @param modelAssetPath The path to the model asset.
1384
- * @return A new `InteractiveSegmenter`.
1385
- */
1386
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<InteractiveSegmenter>;
1387
- private constructor();
1388
- /**
1389
- * Sets new options for the interactive segmenter.
1390
- *
1391
- * Calling `setOptions()` with a subset of options only affects those
1392
- * options. You can reset an option back to its default value by
1393
- * explicitly setting it to `undefined`.
1394
- *
1395
- * @param options The options for the interactive segmenter.
1396
- * @return A Promise that resolves when the settings have been applied.
1397
- */
1398
- setOptions(options: InteractiveSegmenterOptions): Promise<void>;
1399
- /**
1400
- * Performs interactive segmentation on the provided single image and invokes
1401
- * the callback with the response. The `roi` parameter is used to represent a
1402
- * user's region of interest for segmentation.
1403
- *
1404
- * If the output_type is `CATEGORY_MASK`, the callback is invoked with vector
1405
- * of images that represent per-category segmented image mask. If the
1406
- * output_type is `CONFIDENCE_MASK`, the callback is invoked with a vector of
1407
- * images that contains only one confidence image mask. The method returns
1408
- * synchronously once the callback returns.
1409
- *
1410
- * @param image An image to process.
1411
- * @param roi The region of interest for segmentation.
1412
- * @param callback The callback that is invoked with the segmented masks. The
1413
- * lifetime of the returned data is only guaranteed for the duration of the
1414
- * callback.
1415
- */
1416
- segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallback): void;
1417
- /**
1418
- * Performs interactive segmentation on the provided single image and invokes
1419
- * the callback with the response. The `roi` parameter is used to represent a
1420
- * user's region of interest for segmentation.
1421
- *
1422
- * The 'image_processing_options' parameter can be used to specify the
1423
- * rotation to apply to the image before performing segmentation, by setting
1424
- * its 'rotationDegrees' field. Note that specifying a region-of-interest
1425
- * using the 'regionOfInterest' field is NOT supported and will result in an
1426
- * error.
1427
- *
1428
- * If the output_type is `CATEGORY_MASK`, the callback is invoked with vector
1429
- * of images that represent per-category segmented image mask. If the
1430
- * output_type is `CONFIDENCE_MASK`, the callback is invoked with a vector of
1431
- * images that contains only one confidence image mask. The method returns
1432
- * synchronously once the callback returns.
1433
- *
1434
- * @param image An image to process.
1435
- * @param roi The region of interest for segmentation.
1436
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1437
- * to process the input image before running inference.
1438
- * @param callback The callback that is invoked with the segmented masks. The
1439
- * lifetime of the returned data is only guaranteed for the duration of the
1440
- * callback.
1441
- */
1442
- segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallback): void;
1443
- }
1444
-
1445
- /**
1446
- * A callback that receives the computed masks from the interactive segmenter.
1447
- * The returned data is only valid for the duration of the callback. If
1448
- * asynchronous processing is needed, all data needs to be copied before the
1449
- * callback returns.
1450
- */
1451
- export declare type InteractiveSegmenterCallback = (result: InteractiveSegmenterResult) => void;
1452
-
1453
- /** Options to configure the MediaPipe Interactive Segmenter Task */
1454
- export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
1455
- /** Whether to output confidence masks. Defaults to true. */
1456
- outputConfidenceMasks?: boolean | undefined;
1457
- /** Whether to output the category masks. Defaults to false. */
1458
- outputCategoryMask?: boolean | undefined;
1459
- }
1460
-
1461
- /**
1462
- * Copyright 2023 The MediaPipe Authors.
1463
- *
1464
- * Licensed under the Apache License, Version 2.0 (the "License");
1465
- * you may not use this file except in compliance with the License.
1466
- * You may obtain a copy of the License at
1467
- *
1468
- * http://www.apache.org/licenses/LICENSE-2.0
1469
- *
1470
- * Unless required by applicable law or agreed to in writing, software
1471
- * distributed under the License is distributed on an "AS IS" BASIS,
1472
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1473
- * See the License for the specific language governing permissions and
1474
- * limitations under the License.
1475
- */
1476
- /** The output result of InteractiveSegmenter. */
1477
- export declare interface InteractiveSegmenterResult {
1478
- /**
1479
- * Multiple masks as Float32Arrays or WebGLTextures where, for each mask, each
1480
- * pixel represents the prediction confidence, usually in the [0, 1] range.
1481
- */
1482
- confidenceMasks?: Float32Array[] | WebGLTexture[];
1483
- /**
1484
- * A category mask as a Uint8ClampedArray or WebGLTexture where each
1485
- * pixel represents the class which the pixel in the original image was
1486
- * predicted to belong to.
1487
- */
1488
- categoryMask?: Uint8ClampedArray | WebGLTexture;
1489
- /** The width of the masks. */
1490
- width: number;
1491
- /** The height of the masks. */
1492
- height: number;
1493
- }
1494
-
1495
- /**
1496
- * Landmark represents a point in 3D space with x, y, z coordinates. The
1497
- * landmark coordinates are in meters. z represents the landmark depth,
1498
- * and the smaller the value the closer the world landmark is to the camera.
1499
- */
1500
- export declare interface Landmark {
1501
- /** The x coordinates of the landmark. */
1502
- x: number;
1503
- /** The y coordinates of the landmark. */
1504
- y: number;
1505
- /** The z coordinates of the landmark. */
1506
- z: number;
1507
- }
1508
-
1509
- /** Data that a user can use to specialize drawing options. */
1510
- export declare interface LandmarkData {
1511
- index?: number;
1512
- from?: NormalizedLandmark;
1513
- to?: NormalizedLandmark;
1514
- }
1515
-
1516
- /**
1517
- * Copyright 2023 The MediaPipe Authors.
1518
- *
1519
- * Licensed under the Apache License, Version 2.0 (the "License");
1520
- * you may not use this file except in compliance with the License.
1521
- * You may obtain a copy of the License at
1522
- *
1523
- * http://www.apache.org/licenses/LICENSE-2.0
1524
- *
1525
- * Unless required by applicable law or agreed to in writing, software
1526
- * distributed under the License is distributed on an "AS IS" BASIS,
1527
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1528
- * See the License for the specific language governing permissions and
1529
- * limitations under the License.
1530
- */
1531
- /** A two-dimensional matrix. */
1532
- declare interface Matrix {
1533
- /** The number of rows. */
1534
- rows: number;
1535
- /** The number of columns. */
1536
- columns: number;
1537
- /** The values as a flattened one-dimensional array. */
1538
- data: number[];
1539
- }
1540
-
1541
- /**
1542
- * The wrapper class for MediaPipe Image objects.
1543
- *
1544
- * Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects.
1545
- * You can convert the underlying type to any other type by passing the
1546
- * desired type to `getImage()`. As type conversions can be expensive, it is
1547
- * recommended to limit these conversions. You can verify what underlying
1548
- * types are already available by invoking `hasType()`.
1549
- *
1550
- * Images that are returned from a MediaPipe Tasks are owned by by the
1551
- * underlying C++ Task. If you need to extend the lifetime of these objects,
1552
- * you can invoke the `clone()` method. To free up the resources obtained
1553
- * during any clone or type conversion operation, it is important to invoke
1554
- * `close()` on the `MPImage` instance.
1555
- *
1556
- * Converting to and from ImageBitmap requires that the MediaPipe task is
1557
- * initialized with an `OffscreenCanvas`. As we require WebGL2 support, this
1558
- * places some limitations on Browser support as outlined here:
1559
- * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1560
- *
1561
- * Some MediaPipe tasks return single channel masks. These masks are stored
1562
- * using an underlying `Uint8ClampedArray` an `Float32Array` (represented as
1563
- * single-channel arrays). To convert these type to other formats a conversion
1564
- * function is invoked to convert pixel values between single channel and four
1565
- * channel RGBA values. To customize this conversion, you can specify these
1566
- * conversion functions when you invoke `getImage()`. If you use the default
1567
- * conversion function a warning will be logged to the console.
1568
- */
1569
- export declare class MPImage {
1570
- /** Returns the canvas element that the image is bound to. */
1571
- readonly canvas: HTMLCanvasElement | OffscreenCanvas | undefined;
1572
- /** Returns the width of the image. */
1573
- readonly width: number;
1574
- /** Returns the height of the image. */
1575
- readonly height: number;
1576
- private constructor();
1577
- /**
1578
- * Returns whether this `MPImage` stores the image in the desired format.
1579
- * This method can be called to reduce expensive conversion before invoking
1580
- * `getType()`.
1581
- */
1582
- hasType(type: MPImageStorageType): boolean;
1583
- /**
1584
- * Returns the underlying image as a single channel `Uint8ClampedArray`. Note
1585
- * that this involves an expensive GPU to CPU transfer if the current image is
1586
- * only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1587
- * function converts RGBA data pixel-by-pixel to a single channel value by
1588
- * invoking a conversion function (see class comment for detail).
1589
- *
1590
- * @param type The type of image to return.
1591
- * @param converter A set of conversion functions that will be invoked to
1592
- * convert the underlying pixel data if necessary. You may omit this
1593
- * function if the requested conversion does not change the pixel format.
1594
- * @return The current data as a Uint8ClampedArray.
1595
- */
1596
- getImage(type: MPImageStorageType.UINT8_CLAMPED_ARRAY, converter?: MPImageChannelConverter): Uint8ClampedArray;
1597
- /**
1598
- * Returns the underlying image as a single channel `Float32Array`. Note
1599
- * that this involves an expensive GPU to CPU transfer if the current image is
1600
- * only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1601
- * function converts RGBA data pixel-by-pixel to a single channel value by
1602
- * invoking a conversion function (see class comment for detail).
1603
- *
1604
- * @param type The type of image to return.
1605
- * @param converter A set of conversion functions that will be invoked to
1606
- * convert the underlying pixel data if necessary. You may omit this
1607
- * function if the requested conversion does not change the pixel format.
1608
- * @return The current image as a Float32Array.
1609
- */
1610
- getImage(type: MPImageStorageType.FLOAT32_ARRAY, converter?: MPImageChannelConverter): Float32Array;
1611
- /**
1612
- * Returns the underlying image as an `ImageData` object. Note that this
1613
- * involves an expensive GPU to CPU transfer if the current image is only
1614
- * available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
1615
- * function converts single channel pixel values to RGBA by invoking a
1616
- * conversion function (see class comment for detail).
1617
- *
1618
- * @return The current image as an ImageData object.
1619
- */
1620
- getImage(type: MPImageStorageType.IMAGE_DATA, converter?: MPImageChannelConverter): ImageData;
1621
- /**
1622
- * Returns the underlying image as an `ImageBitmap`. Note that
1623
- * conversions to `ImageBitmap` are expensive, especially if the data
1624
- * currently resides on CPU. If necessary, this function first converts single
1625
- * channel pixel values to RGBA by invoking a conversion function (see class
1626
- * comment for detail).
1627
- *
1628
- * Processing with `ImageBitmap`s requires that the MediaPipe Task was
1629
- * initialized with an `OffscreenCanvas` with WebGL2 support. See
1630
- * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
1631
- * for a list of supported platforms.
1632
- *
1633
- * @param type The type of image to return.
1634
- * @param converter A set of conversion functions that will be invoked to
1635
- * convert the underlying pixel data if necessary. You may omit this
1636
- * function if the requested conversion does not change the pixel format.
1637
- * @return The current image as an ImageBitmap object.
1638
- */
1639
- getImage(type: MPImageStorageType.IMAGE_BITMAP, converter?: MPImageChannelConverter): ImageBitmap;
1640
- /**
1641
- * Returns the underlying image as a `WebGLTexture` object. Note that this
1642
- * involves a CPU to GPU transfer if the current image is only available as
1643
- * an `ImageData` object. The returned texture is bound to the current
1644
- * canvas (see `.canvas`).
1645
- *
1646
- * @param type The type of image to return.
1647
- * @param converter A set of conversion functions that will be invoked to
1648
- * convert the underlying pixel data if necessary. You may omit this
1649
- * function if the requested conversion does not change the pixel format.
1650
- * @return The current image as a WebGLTexture.
1651
- */
1652
- getImage(type: MPImageStorageType.WEBGL_TEXTURE, converter?: MPImageChannelConverter): WebGLTexture;
1653
- /**
1654
- * Creates a copy of the resources stored in this `MPImage`. You can invoke
1655
- * this method to extend the lifetime of an image returned by a MediaPipe
1656
- * Task. Note that performance critical applications should aim to only use
1657
- * the `MPImage` within the MediaPipe Task callback so that copies can be
1658
- * avoided.
1659
- */
1660
- clone(): MPImage;
1661
- /**
1662
- * Frees up any resources owned by this `MPImage` instance.
1663
- *
1664
- * Note that this method does not free images that are owned by the C++
1665
- * Task, as these are freed automatically once you leave the MediaPipe
1666
- * callback. Additionally, some shared state is freed only once you invoke the
1667
- * Task's `close()` method.
1668
- */
1669
- close(): void;
1670
- }
1671
-
1672
- /**
1673
- * An interface that can be used to provide custom conversion functions. These
1674
- * functions are invoked to convert pixel values between different channel
1675
- * counts and value ranges. Any conversion function that is not specified will
1676
- * result in a default conversion.
1677
- */
1678
- export declare interface MPImageChannelConverter {
1679
- /**
1680
- * A conversion function to convert a number in the [0.0, 1.0] range to RGBA.
1681
- * The output is an array with four elemeents whose values range from 0 to 255
1682
- * inclusive.
1683
- *
1684
- * The default conversion function is `[v * 255, v * 255, v * 255, 255]`
1685
- * and will log a warning if invoked.
1686
- */
1687
- floatToRGBAConverter?: (value: number) => [
1688
- number,
1689
- number,
1690
- number,
1691
- number
1692
- ];
1693
- uint8ToRGBAConverter?: (value: number) => [
1694
- number,
1695
- number,
1696
- number,
1697
- number
1698
- ];
1699
- /**
1700
- * A conversion function to convert an RGBA value in the range of 0 to 255 to
1701
- * a single value in the [0.0, 1.0] range.
1702
- *
1703
- * The default conversion function is `(r / 3 + g / 3 + b / 3) / 255` and will
1704
- * log a warning if invoked.
1705
- */
1706
- rgbaToFloatConverter?: (r: number, g: number, b: number, a: number) => number;
1707
- /**
1708
- * A conversion function to convert an RGBA value in the range of 0 to 255 to
1709
- * a single value in the [0, 255] range.
1710
- *
1711
- * The default conversion function is `r / 3 + g / 3 + b / 3` and will log a
1712
- * warning if invoked.
1713
- */
1714
- rgbaToUint8Converter?: (r: number, g: number, b: number, a: number) => number;
1715
- /**
1716
- * A conversion function to convert a single value in the 0.0 to 1.0 range to
1717
- * [0, 255].
1718
- *
1719
- * The default conversion function is `r * 255` and will log a warning if
1720
- * invoked.
1721
- */
1722
- floatToUint8Converter?: (value: number) => number;
1723
- /**
1724
- * A conversion function to convert a single value in the 0 to 255 range to
1725
- * [0.0, 1.0] .
1726
- *
1727
- * The default conversion function is `r / 255` and will log a warning if
1728
- * invoked.
1729
- */
1730
- uint8ToFloatConverter?: (value: number) => number;
1731
- }
1732
-
1733
- /** The supported image formats. For internal usage. */
1734
- export declare type MPImageNativeContainer = Uint8ClampedArray | Float32Array | ImageData | ImageBitmap | WebGLTexture;
1735
-
1736
- /**
1737
- * A class that encapsulates the shaders used by an MPImage. Can be re-used
1738
- * across MPImages that use the same WebGL2Rendering context.
1739
- */
1740
- export declare class MPImageShaderContext {
1741
- /** Runs the callback using the shader. */
1742
- run<T>(gl: WebGL2RenderingContext, flipVertically: boolean, callback: () => T): T;
1743
- /**
1744
- * Binds a framebuffer to the canvas. If the framebuffer does not yet exist,
1745
- * creates it first. Binds the provided texture to the framebuffer.
1746
- */
1747
- bindFramebuffer(gl: WebGL2RenderingContext, texture: WebGLTexture): void;
1748
- unbindFramebuffer(): void;
1749
- close(): void;
1750
- }
1751
-
1752
- /**
1753
- * Copyright 2023 The MediaPipe Authors.
1754
- *
1755
- * Licensed under the Apache License, Version 2.0 (the "License");
1756
- * you may not use this file except in compliance with the License.
1757
- * You may obtain a copy of the License at
1758
- *
1759
- * http://www.apache.org/licenses/LICENSE-2.0
1760
- *
1761
- * Unless required by applicable law or agreed to in writing, software
1762
- * distributed under the License is distributed on an "AS IS" BASIS,
1763
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1764
- * See the License for the specific language governing permissions and
1765
- * limitations under the License.
1766
- */
1767
- /** The underlying type of the image. */
1768
- export declare enum MPImageStorageType {
1769
- /** Represents the native `UInt8ClampedArray` type. */
1770
- UINT8_CLAMPED_ARRAY = 0,
1771
- /**
1772
- * Represents the native `Float32Array` type. Values range from [0.0, 1.0].
1773
- */
1774
- FLOAT32_ARRAY = 1,
1775
- /** Represents the native `ImageData` type. */
1776
- IMAGE_DATA = 2,
1777
- /** Represents the native `ImageBitmap` type. */
1778
- IMAGE_BITMAP = 3,
1779
- /** Represents the native `WebGLTexture` type. */
1780
- WEBGL_TEXTURE = 4
1781
- }
1782
-
1783
- /**
1784
- * Copyright 2023 The MediaPipe Authors.
1785
- *
1786
- * Licensed under the Apache License, Version 2.0 (the "License");
1787
- * you may not use this file except in compliance with the License.
1788
- * You may obtain a copy of the License at
1789
- *
1790
- * http://www.apache.org/licenses/LICENSE-2.0
1791
- *
1792
- * Unless required by applicable law or agreed to in writing, software
1793
- * distributed under the License is distributed on an "AS IS" BASIS,
1794
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1795
- * See the License for the specific language governing permissions and
1796
- * limitations under the License.
1797
- */
1798
- /**
1799
- * A keypoint, defined by the coordinates (x, y), normalized by the image
1800
- * dimensions.
1801
- */
1802
- declare interface NormalizedKeypoint {
1803
- /** X in normalized image coordinates. */
1804
- x: number;
1805
- /** Y in normalized image coordinates. */
1806
- y: number;
1807
- /** Optional label of the keypoint. */
1808
- label?: string;
1809
- /** Optional score of the keypoint. */
1810
- score?: number;
1811
- }
1812
-
1813
- /**
1814
- * Copyright 2022 The MediaPipe Authors.
1815
- *
1816
- * Licensed under the Apache License, Version 2.0 (the "License");
1817
- * you may not use this file except in compliance with the License.
1818
- * You may obtain a copy of the License at
1819
- *
1820
- * http://www.apache.org/licenses/LICENSE-2.0
1821
- *
1822
- * Unless required by applicable law or agreed to in writing, software
1823
- * distributed under the License is distributed on an "AS IS" BASIS,
1824
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1825
- * See the License for the specific language governing permissions and
1826
- * limitations under the License.
1827
- */
1828
- /**
1829
- * Normalized Landmark represents a point in 3D space with x, y, z coordinates.
1830
- * x and y are normalized to [0.0, 1.0] by the image width and height
1831
- * respectively. z represents the landmark depth, and the smaller the value the
1832
- * closer the landmark is to the camera. The magnitude of z uses roughly the
1833
- * same scale as x.
1834
- */
1835
- export declare interface NormalizedLandmark {
1836
- /** The x coordinates of the normalized landmark. */
1837
- x: number;
1838
- /** The y coordinates of the normalized landmark. */
1839
- y: number;
1840
- /** The z coordinates of the normalized landmark. */
1841
- z: number;
1842
- }
1843
-
1844
- /** Performs object detection on images. */
1845
- export declare class ObjectDetector extends VisionTaskRunner {
1846
- /**
1847
- * Initializes the Wasm runtime and creates a new object detector from the
1848
- * provided options.
1849
- * @param wasmFileset A configuration object that provides the location of the
1850
- * Wasm binary and its loader.
1851
- * @param objectDetectorOptions The options for the Object Detector. Note that
1852
- * either a path to the model asset or a model buffer needs to be
1853
- * provided (via `baseOptions`).
1854
- */
1855
- static createFromOptions(wasmFileset: WasmFileset, objectDetectorOptions: ObjectDetectorOptions): Promise<ObjectDetector>;
1856
- /**
1857
- * Initializes the Wasm runtime and creates a new object detector based on the
1858
- * provided model asset buffer.
1859
- * @param wasmFileset A configuration object that provides the location of the
1860
- * Wasm binary and its loader.
1861
- * @param modelAssetBuffer A binary representation of the model.
1862
- */
1863
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ObjectDetector>;
1864
- /**
1865
- * Initializes the Wasm runtime and creates a new object detector based on the
1866
- * path to the model asset.
1867
- * @param wasmFileset A configuration object that provides the location of the
1868
- * Wasm binary and its loader.
1869
- * @param modelAssetPath The path to the model asset.
1870
- */
1871
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ObjectDetector>;
1872
- private constructor();
1873
- /**
1874
- * Sets new options for the object detector.
1875
- *
1876
- * Calling `setOptions()` with a subset of options only affects those options.
1877
- * You can reset an option back to its default value by explicitly setting it
1878
- * to `undefined`.
1879
- *
1880
- * @param options The options for the object detector.
1881
- */
1882
- setOptions(options: ObjectDetectorOptions): Promise<void>;
1883
- /**
1884
- * Performs object detection on the provided single image and waits
1885
- * synchronously for the response. Only use this method when the
1886
- * ObjectDetector is created with running mode `image`.
1887
- *
1888
- * @param image An image to process.
1889
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1890
- * to process the input image before running inference.
1891
- * @return A result containing a list of detected objects.
1892
- */
1893
- detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
1894
- /**
1895
- * Performs object detection on the provided video frame and waits
1896
- * synchronously for the response. Only use this method when the
1897
- * ObjectDetector is created with running mode `video`.
1898
- *
1899
- * @param videoFrame A video frame to process.
1900
- * @param timestamp The timestamp of the current frame, in ms.
1901
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1902
- * to process the input image before running inference.
1903
- * @return A result containing a list of detected objects.
1904
- */
1905
- detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
1906
- }
1907
-
1908
- /** Options to configure the MediaPipe Object Detector Task */
1909
- export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
1910
- }
1911
-
1912
- /** Performs pose landmarks detection on images. */
1913
- export declare class PoseLandmarker extends VisionTaskRunner {
1914
- /**
1915
- * An array containing the pairs of pose landmark indices to be rendered with
1916
- * connections.
1917
- */
1918
- static POSE_CONNECTIONS: Connection[];
1919
- /**
1920
- * Initializes the Wasm runtime and creates a new `PoseLandmarker` from the
1921
- * provided options.
1922
- * @param wasmFileset A configuration object that provides the location of the
1923
- * Wasm binary and its loader.
1924
- * @param poseLandmarkerOptions The options for the PoseLandmarker.
1925
- * Note that either a path to the model asset or a model buffer needs to
1926
- * be provided (via `baseOptions`).
1927
- */
1928
- static createFromOptions(wasmFileset: WasmFileset, poseLandmarkerOptions: PoseLandmarkerOptions): Promise<PoseLandmarker>;
1929
- /**
1930
- * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
1931
- * the provided model asset buffer.
1932
- * @param wasmFileset A configuration object that provides the location of the
1933
- * Wasm binary and its loader.
1934
- * @param modelAssetBuffer A binary representation of the model.
1935
- */
1936
- static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<PoseLandmarker>;
1937
- /**
1938
- * Initializes the Wasm runtime and creates a new `PoseLandmarker` based on
1939
- * the path to the model asset.
1940
- * @param wasmFileset A configuration object that provides the location of the
1941
- * Wasm binary and its loader.
1942
- * @param modelAssetPath The path to the model asset.
1943
- */
1944
- static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<PoseLandmarker>;
1945
- private constructor();
1946
- /**
1947
- * Sets new options for this `PoseLandmarker`.
1948
- *
1949
- * Calling `setOptions()` with a subset of options only affects those options.
1950
- * You can reset an option back to its default value by explicitly setting it
1951
- * to `undefined`.
1952
- *
1953
- * @param options The options for the pose landmarker.
1954
- */
1955
- setOptions(options: PoseLandmarkerOptions): Promise<void>;
1956
- /**
1957
- * Performs pose detection on the provided single image and waits
1958
- * synchronously for the response. Only use this method when the
1959
- * PoseLandmarker is created with running mode `image`.
1960
- *
1961
- * @param image An image to process.
1962
- * @param callback The callback that is invoked with the result. The
1963
- * lifetime of the returned masks is only guaranteed for the duration of
1964
- * the callback.
1965
- * @return The detected pose landmarks.
1966
- */
1967
- detect(image: ImageSource, callback: PoseLandmarkerCallback): void;
1968
- /**
1969
- * Performs pose detection on the provided single image and waits
1970
- * synchronously for the response. Only use this method when the
1971
- * PoseLandmarker is created with running mode `image`.
1972
- *
1973
- * @param image An image to process.
1974
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
1975
- * to process the input image before running inference.
1976
- * @param callback The callback that is invoked with the result. The
1977
- * lifetime of the returned masks is only guaranteed for the duration of
1978
- * the callback.
1979
- * @return The detected pose landmarks.
1980
- */
1981
- detect(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: PoseLandmarkerCallback): void;
1982
- /**
1983
- * Performs pose detection on the provided video frame and waits
1984
- * synchronously for the response. Only use this method when the
1985
- * PoseLandmarker is created with running mode `video`.
1986
- *
1987
- * @param videoFrame A video frame to process.
1988
- * @param timestamp The timestamp of the current frame, in ms.
1989
- * @param callback The callback that is invoked with the result. The
1990
- * lifetime of the returned masks is only guaranteed for the duration of
1991
- * the callback.
1992
- * @return The detected pose landmarks.
1993
- */
1994
- detectForVideo(videoFrame: ImageSource, timestamp: number, callback: PoseLandmarkerCallback): void;
1995
- /**
1996
- * Performs pose detection on the provided video frame and waits
1997
- * synchronously for the response. Only use this method when the
1998
- * PoseLandmarker is created with running mode `video`.
1999
- *
2000
- * @param videoFrame A video frame to process.
2001
- * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
2002
- * to process the input image before running inference.
2003
- * @param timestamp The timestamp of the current frame, in ms.
2004
- * @param callback The callback that is invoked with the result. The
2005
- * lifetime of the returned masks is only guaranteed for the duration of
2006
- * the callback.
2007
- * @return The detected pose landmarks.
2008
- */
2009
- detectForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: PoseLandmarkerCallback): void;
2010
- }
2011
-
2012
- /**
2013
- * A callback that receives the result from the pose detector. The returned
2014
- * masks are only valid for the duration of the callback. If asynchronous
2015
- * processing is needed, the masks need to be copied before the callback
2016
- * returns.
2017
- */
2018
- export declare type PoseLandmarkerCallback = (result: PoseLandmarkerResult) => void;
2019
-
2020
- /** Options to configure the MediaPipe PoseLandmarker Task */
2021
- export declare interface PoseLandmarkerOptions extends VisionTaskOptions {
2022
- /**
2023
- * The maximum number of poses can be detected by the PoseLandmarker.
2024
- * Defaults to 1.
2025
- */
2026
- numPoses?: number | undefined;
2027
- /**
2028
- * The minimum confidence score for the pose detection to be considered
2029
- * successful. Defaults to 0.5.
2030
- */
2031
- minPoseDetectionConfidence?: number | undefined;
2032
- /**
2033
- * The minimum confidence score of pose presence score in the pose landmark
2034
- * detection. Defaults to 0.5.
2035
- */
2036
- minPosePresenceConfidence?: number | undefined;
2037
- /**
2038
- * The minimum confidence score for the pose tracking to be considered
2039
- * successful. Defaults to 0.5.
2040
- */
2041
- minTrackingConfidence?: number | undefined;
2042
- /** Whether to output segmentation masks. Defaults to false. */
2043
- outputSegmentationMasks?: boolean | undefined;
2044
- }
2045
-
2046
- /**
2047
- * Represents the pose landmarks deection results generated by `PoseLandmarker`.
2048
- * Each vector element represents a single pose detected in the image.
2049
- */
2050
- export declare interface PoseLandmarkerResult {
2051
- /** Pose landmarks of detected poses. */
2052
- landmarks: NormalizedLandmark[];
2053
- /** Pose landmarks in world coordinates of detected poses. */
2054
- worldLandmarks: Landmark[];
2055
- /** Detected auxiliary landmarks, used for deriving ROI for next frame. */
2056
- auxilaryLandmarks: NormalizedLandmark[];
2057
- /** Segmentation mask for the detected pose. */
2058
- segmentationMasks?: Float32Array[] | WebGLTexture[];
2059
- }
2060
-
2061
- /**
2062
- * Defines a rectangle, used e.g. as part of detection results or as input
2063
- * region-of-interest.
2064
- *
2065
- * The coordinates are normalized with respect to the image dimensions, i.e.
2066
- * generally in [0,1] but they may exceed these bounds if describing a region
2067
- * overlapping the image. The origin is on the top-left corner of the image.
2068
- */
2069
- declare interface RectF {
2070
- left: number;
2071
- top: number;
2072
- right: number;
2073
- bottom: number;
2074
- }
2075
-
2076
- /** A Region-Of-Interest (ROI) to represent a region within an image. */
2077
- export declare interface RegionOfInterest {
2078
- /** The ROI in keypoint format. */
2079
- keypoint: NormalizedKeypoint;
2080
- }
2081
-
2082
- /**
2083
- * The two running modes of a vision task.
2084
- * 1) The image mode for processing single image inputs.
2085
- * 2) The video mode for processing decoded frames of a video.
2086
- */
2087
- declare type RunningMode = "IMAGE" | "VIDEO";
2088
-
2089
- /**
2090
- * The segmentation tasks return the segmentation either as a WebGLTexture (when
2091
- * the output is on GPU) or as a typed JavaScript arrays for CPU-based
2092
- * category or confidence masks. `Uint8ClampedArray`s are used to represent
2093
- * CPU-based category masks and `Float32Array`s are used for CPU-based
2094
- * confidence masks.
2095
- */
2096
- export declare type SegmentationMask = Uint8ClampedArray | Float32Array | WebGLTexture;
2097
-
2098
- /** Base class for all MediaPipe Tasks. */
2099
- declare abstract class TaskRunner {
2100
- protected constructor();
2101
- /** Configures the task with custom options. */
2102
- abstract setOptions(options: TaskRunnerOptions): Promise<void>;
2103
- }
2104
-
2105
- /** Options to configure MediaPipe Tasks in general. */
2106
- declare interface TaskRunnerOptions {
2107
- /** Options to configure the loading of the model assets. */
2108
- baseOptions?: BaseOptions_2;
2109
- }
2110
-
2111
- /** The options for configuring a MediaPipe vision task. */
2112
- declare interface VisionTaskOptions extends TaskRunnerOptions {
2113
- /**
2114
- * The canvas element to bind textures to. This has to be set for GPU
2115
- * processing. The task will initialize a WebGL context and throw an error if
2116
- * this fails (e.g. if you have already initialized a different type of
2117
- * context).
2118
- */
2119
- canvas?: HTMLCanvasElement | OffscreenCanvas;
2120
- /**
2121
- * The running mode of the task. Default to the image mode.
2122
- * Vision tasks have two running modes:
2123
- * 1) The image mode for processing single image inputs.
2124
- * 2) The video mode for processing decoded frames of a video.
2125
- */
2126
- runningMode?: RunningMode;
2127
- }
2128
-
2129
- /** Base class for all MediaPipe Vision Tasks. */
2130
- declare abstract class VisionTaskRunner extends TaskRunner {
2131
- protected constructor();
2132
- /** Configures the shared options of a vision task. */
2133
- applyOptions(options: VisionTaskOptions): Promise<void>;
2134
- }
2135
-
2136
- /**
2137
- * Copyright 2022 The MediaPipe Authors.
2138
- *
2139
- * Licensed under the Apache License, Version 2.0 (the "License");
2140
- * you may not use this file except in compliance with the License.
2141
- * You may obtain a copy of the License at
2142
- *
2143
- * http://www.apache.org/licenses/LICENSE-2.0
2144
- *
2145
- * Unless required by applicable law or agreed to in writing, software
2146
- * distributed under the License is distributed on an "AS IS" BASIS,
2147
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2148
- * See the License for the specific language governing permissions and
2149
- * limitations under the License.
2150
- */
2151
- /** An object containing the locations of the Wasm assets */
2152
- declare interface WasmFileset {
2153
- /** The path to the Wasm loader script. */
2154
- wasmLoaderPath: string;
2155
- /** The path to the Wasm binary. */
2156
- wasmBinaryPath: string;
2157
- }
2158
-
2159
- export { }