@mediapipe/tasks-vision 0.1.0-alpha-12 → 0.1.0-alpha-14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/package.json +1 -1
- package/vision_bundle.js +1 -1
- package/wasm/vision_wasm_internal.js +739 -774
- package/wasm/vision_wasm_internal.wasm +0 -0
- package/wasm/vision_wasm_nosimd_internal.js +736 -771
- package/wasm/vision_wasm_nosimd_internal.wasm +0 -0
- package/vision.d.ts +0 -1768
package/vision.d.ts
DELETED
|
@@ -1,1768 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
3
|
-
*
|
|
4
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
-
* you may not use this file except in compliance with the License.
|
|
6
|
-
* You may obtain a copy of the License at
|
|
7
|
-
*
|
|
8
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
-
*
|
|
10
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
-
* See the License for the specific language governing permissions and
|
|
14
|
-
* limitations under the License.
|
|
15
|
-
*/
|
|
16
|
-
/** Options to configure MediaPipe model loading and processing. */
|
|
17
|
-
declare interface BaseOptions_2 {
|
|
18
|
-
/**
|
|
19
|
-
* The model path to the model asset file. Only one of `modelAssetPath` or
|
|
20
|
-
* `modelAssetBuffer` can be set.
|
|
21
|
-
*/
|
|
22
|
-
modelAssetPath?: string | undefined;
|
|
23
|
-
/**
|
|
24
|
-
* A buffer containing the model aaset. Only one of `modelAssetPath` or
|
|
25
|
-
* `modelAssetBuffer` can be set.
|
|
26
|
-
*/
|
|
27
|
-
modelAssetBuffer?: Uint8Array | undefined;
|
|
28
|
-
/** Overrides the default backend to use for the provided model. */
|
|
29
|
-
delegate?: "CPU" | "GPU" | undefined;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
|
34
|
-
*
|
|
35
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
36
|
-
* you may not use this file except in compliance with the License.
|
|
37
|
-
* You may obtain a copy of the License at
|
|
38
|
-
*
|
|
39
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
40
|
-
*
|
|
41
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
42
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
43
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
44
|
-
* See the License for the specific language governing permissions and
|
|
45
|
-
* limitations under the License.
|
|
46
|
-
*/
|
|
47
|
-
/** An integer bounding box, axis aligned. */
|
|
48
|
-
export declare interface BoundingBox {
|
|
49
|
-
/** The X coordinate of the top-left corner, in pixels. */
|
|
50
|
-
originX: number;
|
|
51
|
-
/** The Y coordinate of the top-left corner, in pixels. */
|
|
52
|
-
originY: number;
|
|
53
|
-
/** The width of the bounding box, in pixels. */
|
|
54
|
-
width: number;
|
|
55
|
-
/** The height of the bounding box, in pixels. */
|
|
56
|
-
height: number;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* A user-defined callback to take input data and map it to a custom output
|
|
61
|
-
* value.
|
|
62
|
-
*/
|
|
63
|
-
export declare type Callback<I, O> = (input: I) => O;
|
|
64
|
-
|
|
65
|
-
/**
|
|
66
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
67
|
-
*
|
|
68
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
69
|
-
* you may not use this file except in compliance with the License.
|
|
70
|
-
* You may obtain a copy of the License at
|
|
71
|
-
*
|
|
72
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
73
|
-
*
|
|
74
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
75
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
76
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
77
|
-
* See the License for the specific language governing permissions and
|
|
78
|
-
* limitations under the License.
|
|
79
|
-
*/
|
|
80
|
-
/** A classification category. */
|
|
81
|
-
export declare interface Category {
|
|
82
|
-
/** The probability score of this label category. */
|
|
83
|
-
score: number;
|
|
84
|
-
/** The index of the category in the corresponding label file. */
|
|
85
|
-
index: number;
|
|
86
|
-
/**
|
|
87
|
-
* The label of this category object. Defaults to an empty string if there is
|
|
88
|
-
* no category.
|
|
89
|
-
*/
|
|
90
|
-
categoryName: string;
|
|
91
|
-
/**
|
|
92
|
-
* The display name of the label, which may be translated for different
|
|
93
|
-
* locales. For example, a label, "apple", may be translated into Spanish for
|
|
94
|
-
* display purpose, so that the `display_name` is "manzana". Defaults to an
|
|
95
|
-
* empty string if there is no display name.
|
|
96
|
-
*/
|
|
97
|
-
displayName: string;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/** Classification results for a given classifier head. */
|
|
101
|
-
export declare interface Classifications {
|
|
102
|
-
/**
|
|
103
|
-
* The array of predicted categories, usually sorted by descending scores,
|
|
104
|
-
* e.g., from high to low probability.
|
|
105
|
-
*/
|
|
106
|
-
categories: Category[];
|
|
107
|
-
/**
|
|
108
|
-
* The index of the classifier head these categories refer to. This is
|
|
109
|
-
* useful for multi-head models.
|
|
110
|
-
*/
|
|
111
|
-
headIndex: number;
|
|
112
|
-
/**
|
|
113
|
-
* The name of the classifier head, which is the corresponding tensor
|
|
114
|
-
* metadata name. Defaults to an empty string if there is no such metadata.
|
|
115
|
-
*/
|
|
116
|
-
headName: string;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
/**
|
|
120
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
121
|
-
*
|
|
122
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
123
|
-
* you may not use this file except in compliance with the License.
|
|
124
|
-
* You may obtain a copy of the License at
|
|
125
|
-
*
|
|
126
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
127
|
-
*
|
|
128
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
129
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
130
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
131
|
-
* See the License for the specific language governing permissions and
|
|
132
|
-
* limitations under the License.
|
|
133
|
-
*/
|
|
134
|
-
/** Options to configure a MediaPipe Classifier Task. */
|
|
135
|
-
declare interface ClassifierOptions {
|
|
136
|
-
/**
|
|
137
|
-
* The locale to use for display names specified through the TFLite Model
|
|
138
|
-
* Metadata, if any. Defaults to English.
|
|
139
|
-
*/
|
|
140
|
-
displayNamesLocale?: string | undefined;
|
|
141
|
-
/** The maximum number of top-scored detection results to return. */
|
|
142
|
-
maxResults?: number | undefined;
|
|
143
|
-
/**
|
|
144
|
-
* Overrides the value provided in the model metadata. Results below this
|
|
145
|
-
* value are rejected.
|
|
146
|
-
*/
|
|
147
|
-
scoreThreshold?: number | undefined;
|
|
148
|
-
/**
|
|
149
|
-
* Allowlist of category names. If non-empty, detection results whose category
|
|
150
|
-
* name is not in this set will be filtered out. Duplicate or unknown category
|
|
151
|
-
* names are ignored. Mutually exclusive with `categoryDenylist`.
|
|
152
|
-
*/
|
|
153
|
-
categoryAllowlist?: string[] | undefined;
|
|
154
|
-
/**
|
|
155
|
-
* Denylist of category names. If non-empty, detection results whose category
|
|
156
|
-
* name is in this set will be filtered out. Duplicate or unknown category
|
|
157
|
-
* names are ignored. Mutually exclusive with `categoryAllowlist`.
|
|
158
|
-
*/
|
|
159
|
-
categoryDenylist?: string[] | undefined;
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
/** A connection between two landmarks. */
|
|
163
|
-
declare interface Connection {
|
|
164
|
-
start: number;
|
|
165
|
-
end: number;
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/** Represents one detection by a detection task. */
|
|
169
|
-
export declare interface Detection {
|
|
170
|
-
/** A list of `Category` objects. */
|
|
171
|
-
categories: Category[];
|
|
172
|
-
/** The bounding box of the detected objects. */
|
|
173
|
-
boundingBox?: BoundingBox;
|
|
174
|
-
/**
|
|
175
|
-
* Optional list of keypoints associated with the detection. Keypoints
|
|
176
|
-
* represent interesting points related to the detection. For example, the
|
|
177
|
-
* keypoints represent the eye, ear and mouth from face detection model. Or
|
|
178
|
-
* in the template matching detection, e.g. KNIFT, they can represent the
|
|
179
|
-
* feature points for template matching.
|
|
180
|
-
*/
|
|
181
|
-
keypoints?: NormalizedKeypoint[];
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
/** Detection results of a model. */
|
|
185
|
-
declare interface DetectionResult {
|
|
186
|
-
/** A list of Detections. */
|
|
187
|
-
detections: Detection[];
|
|
188
|
-
}
|
|
189
|
-
export { DetectionResult as FaceDetectorResult }
|
|
190
|
-
export { DetectionResult as ObjectDetectorResult }
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* Options for customizing the drawing routines
|
|
194
|
-
*/
|
|
195
|
-
export declare interface DrawingOptions {
|
|
196
|
-
/** The color that is used to draw the shape. Defaults to white. */
|
|
197
|
-
color?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
|
|
198
|
-
/**
|
|
199
|
-
* The color that is used to fill the shape. Defaults to `.color` (or black
|
|
200
|
-
* if color is not set).
|
|
201
|
-
*/
|
|
202
|
-
fillColor?: string | CanvasGradient | CanvasPattern | Callback<LandmarkData, string | CanvasGradient | CanvasPattern>;
|
|
203
|
-
/** The width of the line boundary of the shape. Defaults to 4. */
|
|
204
|
-
lineWidth?: number | Callback<LandmarkData, number>;
|
|
205
|
-
/** The radius of location marker. Defaults to 6. */
|
|
206
|
-
radius?: number | Callback<LandmarkData, number>;
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
/** Helper class to visualize the result of a MediaPipe Vision task. */
|
|
210
|
-
export declare class DrawingUtils {
|
|
211
|
-
/**
|
|
212
|
-
* Creates a new DrawingUtils class.
|
|
213
|
-
*
|
|
214
|
-
* @param ctx The canvas to render onto.
|
|
215
|
-
*/
|
|
216
|
-
constructor(ctx: CanvasRenderingContext2D);
|
|
217
|
-
/**
|
|
218
|
-
* Restricts a number between two endpoints (order doesn't matter).
|
|
219
|
-
*
|
|
220
|
-
* @param x The number to clamp.
|
|
221
|
-
* @param x0 The first boundary.
|
|
222
|
-
* @param x1 The second boundary.
|
|
223
|
-
* @return The clamped value.
|
|
224
|
-
*/
|
|
225
|
-
static clamp(x: number, x0: number, x1: number): number;
|
|
226
|
-
/**
|
|
227
|
-
* Linearly interpolates a value between two points, clamping that value to
|
|
228
|
-
* the endpoints.
|
|
229
|
-
*
|
|
230
|
-
* @param x The number to interpolate.
|
|
231
|
-
* @param x0 The x coordinate of the start value.
|
|
232
|
-
* @param x1 The x coordinate of the end value.
|
|
233
|
-
* @param y0 The y coordinate of the start value.
|
|
234
|
-
* @param y1 The y coordinate of the end value.
|
|
235
|
-
* @return The interpolated value.
|
|
236
|
-
*/
|
|
237
|
-
static lerp(x: number, x0: number, x1: number, y0: number, y1: number): number;
|
|
238
|
-
/**
|
|
239
|
-
* Draws circles onto the provided landmarks.
|
|
240
|
-
*
|
|
241
|
-
* @param landmarks The landmarks to draw.
|
|
242
|
-
* @param style The style to visualize the landmarks.
|
|
243
|
-
*/
|
|
244
|
-
drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions): void;
|
|
245
|
-
/**
|
|
246
|
-
* Draws lines between landmarks (given a connection graph).
|
|
247
|
-
*
|
|
248
|
-
* @param landmarks The landmarks to draw.
|
|
249
|
-
* @param connections The connections array that contains the start and the
|
|
250
|
-
* end indices for the connections to draw.
|
|
251
|
-
* @param style The style to visualize the landmarks.
|
|
252
|
-
*/
|
|
253
|
-
drawConnectors(landmarks?: NormalizedLandmark[], connections?: Connection[], style?: DrawingOptions): void;
|
|
254
|
-
/**
|
|
255
|
-
* Draws a bounding box.
|
|
256
|
-
*
|
|
257
|
-
* @param boundingBox The bounding box to draw.
|
|
258
|
-
* @param style The style to visualize the boundin box.
|
|
259
|
-
*/
|
|
260
|
-
drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
/**
|
|
264
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
265
|
-
*
|
|
266
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
267
|
-
* you may not use this file except in compliance with the License.
|
|
268
|
-
* You may obtain a copy of the License at
|
|
269
|
-
*
|
|
270
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
271
|
-
*
|
|
272
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
273
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
274
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
275
|
-
* See the License for the specific language governing permissions and
|
|
276
|
-
* limitations under the License.
|
|
277
|
-
*/
|
|
278
|
-
/** Options to configure a MediaPipe Embedder Task */
|
|
279
|
-
declare interface EmbedderOptions {
|
|
280
|
-
/**
|
|
281
|
-
* Whether to normalize the returned feature vector with L2 norm. Use this
|
|
282
|
-
* option only if the model does not already contain a native L2_NORMALIZATION
|
|
283
|
-
* TF Lite Op. In most cases, this is already the case and L2 norm is thus
|
|
284
|
-
* achieved through TF Lite inference.
|
|
285
|
-
*/
|
|
286
|
-
l2Normalize?: boolean | undefined;
|
|
287
|
-
/**
|
|
288
|
-
* Whether the returned embedding should be quantized to bytes via scalar
|
|
289
|
-
* quantization. Embeddings are implicitly assumed to be unit-norm and
|
|
290
|
-
* therefore any dimension is guaranteed to have a value in [-1.0, 1.0]. Use
|
|
291
|
-
* the l2_normalize option if this is not the case.
|
|
292
|
-
*/
|
|
293
|
-
quantize?: boolean | undefined;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
/**
|
|
297
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
298
|
-
*
|
|
299
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
300
|
-
* you may not use this file except in compliance with the License.
|
|
301
|
-
* You may obtain a copy of the License at
|
|
302
|
-
*
|
|
303
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
304
|
-
*
|
|
305
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
306
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
307
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
308
|
-
* See the License for the specific language governing permissions and
|
|
309
|
-
* limitations under the License.
|
|
310
|
-
*/
|
|
311
|
-
/**
|
|
312
|
-
* List of embeddings with an optional timestamp.
|
|
313
|
-
*
|
|
314
|
-
* One and only one of the two 'floatEmbedding' and 'quantizedEmbedding' will
|
|
315
|
-
* contain data, based on whether or not the embedder was configured to perform
|
|
316
|
-
* scalar quantization.
|
|
317
|
-
*/
|
|
318
|
-
export declare interface Embedding {
|
|
319
|
-
/**
|
|
320
|
-
* Floating-point embedding. Empty if the embedder was configured to perform
|
|
321
|
-
* scalar-quantization.
|
|
322
|
-
*/
|
|
323
|
-
floatEmbedding?: number[];
|
|
324
|
-
/**
|
|
325
|
-
* Scalar-quantized embedding. Empty if the embedder was not configured to
|
|
326
|
-
* perform scalar quantization.
|
|
327
|
-
*/
|
|
328
|
-
quantizedEmbedding?: Uint8Array;
|
|
329
|
-
/**
|
|
330
|
-
* The index of the classifier head these categories refer to. This is
|
|
331
|
-
* useful for multi-head models.
|
|
332
|
-
*/
|
|
333
|
-
headIndex: number;
|
|
334
|
-
/**
|
|
335
|
-
* The name of the classifier head, which is the corresponding tensor
|
|
336
|
-
* metadata name.
|
|
337
|
-
*/
|
|
338
|
-
headName: string;
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
/** Performs face detection on images. */
|
|
342
|
-
export declare class FaceDetector extends VisionTaskRunner {
|
|
343
|
-
/**
|
|
344
|
-
* Initializes the Wasm runtime and creates a new face detector from the
|
|
345
|
-
* provided options.
|
|
346
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
347
|
-
* Wasm binary and its loader.
|
|
348
|
-
* @param faceDetectorOptions The options for the FaceDetector. Note that
|
|
349
|
-
* either a path to the model asset or a model buffer needs to be
|
|
350
|
-
* provided (via `baseOptions`).
|
|
351
|
-
*/
|
|
352
|
-
static createFromOptions(wasmFileset: WasmFileset, faceDetectorOptions: FaceDetectorOptions): Promise<FaceDetector>;
|
|
353
|
-
/**
|
|
354
|
-
* Initializes the Wasm runtime and creates a new face detector based on the
|
|
355
|
-
* provided model asset buffer.
|
|
356
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
357
|
-
* Wasm binary and its loader.
|
|
358
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
359
|
-
*/
|
|
360
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceDetector>;
|
|
361
|
-
/**
|
|
362
|
-
* Initializes the Wasm runtime and creates a new face detector based on the
|
|
363
|
-
* path to the model asset.
|
|
364
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
365
|
-
* Wasm binary and its loader.
|
|
366
|
-
* @param modelAssetPath The path to the model asset.
|
|
367
|
-
*/
|
|
368
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceDetector>;
|
|
369
|
-
private constructor();
|
|
370
|
-
/**
|
|
371
|
-
* Sets new options for the FaceDetector.
|
|
372
|
-
*
|
|
373
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
374
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
375
|
-
* to `undefined`.
|
|
376
|
-
*
|
|
377
|
-
* @param options The options for the FaceDetector.
|
|
378
|
-
*/
|
|
379
|
-
setOptions(options: FaceDetectorOptions): Promise<void>;
|
|
380
|
-
/**
|
|
381
|
-
* Performs face detection on the provided single image and waits
|
|
382
|
-
* synchronously for the response. Only use this method when the
|
|
383
|
-
* FaceDetector is created with running mode `image`.
|
|
384
|
-
*
|
|
385
|
-
* @param image An image to process.
|
|
386
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
387
|
-
* to process the input image before running inference.
|
|
388
|
-
* @return A result containing the list of detected faces.
|
|
389
|
-
*/
|
|
390
|
-
detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
|
|
391
|
-
/**
|
|
392
|
-
* Performs face detection on the provided video frame and waits
|
|
393
|
-
* synchronously for the response. Only use this method when the
|
|
394
|
-
* FaceDetector is created with running mode `video`.
|
|
395
|
-
*
|
|
396
|
-
* @param videoFrame A video frame to process.
|
|
397
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
398
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
399
|
-
* to process the input image before running inference.
|
|
400
|
-
* @return A result containing the list of detected faces.
|
|
401
|
-
*/
|
|
402
|
-
detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
/** Options to configure the MediaPipe Face Detector Task */
|
|
406
|
-
export declare interface FaceDetectorOptions extends VisionTaskOptions {
|
|
407
|
-
/**
|
|
408
|
-
* The minimum confidence score for the face detection to be considered
|
|
409
|
-
* successful. Defaults to 0.5.
|
|
410
|
-
*/
|
|
411
|
-
minDetectionConfidence?: number | undefined;
|
|
412
|
-
/**
|
|
413
|
-
* The minimum non-maximum-suppression threshold for face detection to be
|
|
414
|
-
* considered overlapped. Defaults to 0.3.
|
|
415
|
-
*/
|
|
416
|
-
minSuppressionThreshold?: number | undefined;
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
/**
|
|
420
|
-
* Performs face landmarks detection on images.
|
|
421
|
-
*
|
|
422
|
-
* This API expects a pre-trained face landmarker model asset bundle.
|
|
423
|
-
*/
|
|
424
|
-
export declare class FaceLandmarker extends VisionTaskRunner {
|
|
425
|
-
/**
|
|
426
|
-
* Initializes the Wasm runtime and creates a new `FaceLandmarker` from the
|
|
427
|
-
* provided options.
|
|
428
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
429
|
-
* Wasm binary and its loader.
|
|
430
|
-
* @param faceLandmarkerOptions The options for the FaceLandmarker.
|
|
431
|
-
* Note that either a path to the model asset or a model buffer needs to
|
|
432
|
-
* be provided (via `baseOptions`).
|
|
433
|
-
*/
|
|
434
|
-
static createFromOptions(wasmFileset: WasmFileset, faceLandmarkerOptions: FaceLandmarkerOptions): Promise<FaceLandmarker>;
|
|
435
|
-
/**
|
|
436
|
-
* Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
|
|
437
|
-
* the provided model asset buffer.
|
|
438
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
439
|
-
* Wasm binary and its loader.
|
|
440
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
441
|
-
*/
|
|
442
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceLandmarker>;
|
|
443
|
-
/**
|
|
444
|
-
* Initializes the Wasm runtime and creates a new `FaceLandmarker` based on
|
|
445
|
-
* the path to the model asset.
|
|
446
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
447
|
-
* Wasm binary and its loader.
|
|
448
|
-
* @param modelAssetPath The path to the model asset.
|
|
449
|
-
*/
|
|
450
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceLandmarker>;
|
|
451
|
-
private constructor();
|
|
452
|
-
/**
|
|
453
|
-
* Sets new options for this `FaceLandmarker`.
|
|
454
|
-
*
|
|
455
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
456
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
457
|
-
* to `undefined`.
|
|
458
|
-
*
|
|
459
|
-
* @param options The options for the face landmarker.
|
|
460
|
-
*/
|
|
461
|
-
setOptions(options: FaceLandmarkerOptions): Promise<void>;
|
|
462
|
-
/**
|
|
463
|
-
* Performs face landmarks detection on the provided single image and waits
|
|
464
|
-
* synchronously for the response. Only use this method when the
|
|
465
|
-
* FaceLandmarker is created with running mode `image`.
|
|
466
|
-
*
|
|
467
|
-
* @param image An image to process.
|
|
468
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
469
|
-
* to process the input image before running inference.
|
|
470
|
-
* @return The detected face landmarks.
|
|
471
|
-
*/
|
|
472
|
-
detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
|
|
473
|
-
/**
|
|
474
|
-
* Performs face landmarks detection on the provided video frame and waits
|
|
475
|
-
* synchronously for the response. Only use this method when the
|
|
476
|
-
* FaceLandmarker is created with running mode `video`.
|
|
477
|
-
*
|
|
478
|
-
* @param videoFrame A video frame to process.
|
|
479
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
480
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
481
|
-
* to process the input image before running inference.
|
|
482
|
-
* @return The detected face landmarks.
|
|
483
|
-
*/
|
|
484
|
-
detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): FaceLandmarkerResult;
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
/** Options to configure the MediaPipe FaceLandmarker Task */
|
|
488
|
-
export declare interface FaceLandmarkerOptions extends VisionTaskOptions {
|
|
489
|
-
/**
|
|
490
|
-
* The maximum number of faces can be detected by the FaceLandmarker.
|
|
491
|
-
* Defaults to 1.
|
|
492
|
-
*/
|
|
493
|
-
numFaces?: number | undefined;
|
|
494
|
-
/**
|
|
495
|
-
* The minimum confidence score for the face detection to be considered
|
|
496
|
-
* successful. Defaults to 0.5.
|
|
497
|
-
*/
|
|
498
|
-
minFaceDetectionConfidence?: number | undefined;
|
|
499
|
-
/**
|
|
500
|
-
* The minimum confidence score of face presence score in the face landmark
|
|
501
|
-
* detection. Defaults to 0.5.
|
|
502
|
-
*/
|
|
503
|
-
minFacePresenceConfidence?: number | undefined;
|
|
504
|
-
/**
|
|
505
|
-
* The minimum confidence score for the face tracking to be considered
|
|
506
|
-
* successful. Defaults to 0.5.
|
|
507
|
-
*/
|
|
508
|
-
minTrackingConfidence?: number | undefined;
|
|
509
|
-
/**
|
|
510
|
-
* Whether FaceLandmarker outputs face blendshapes classification. Face
|
|
511
|
-
* blendshapes are used for rendering the 3D face model.
|
|
512
|
-
*/
|
|
513
|
-
outputFaceBlendshapes?: boolean | undefined;
|
|
514
|
-
/**
|
|
515
|
-
* Whether FaceLandmarker outputs facial transformation_matrix. Facial
|
|
516
|
-
* transformation matrix is used to transform the face landmarks in canonical
|
|
517
|
-
* face to the detected face, so that users can apply face effects on the
|
|
518
|
-
* detected landmarks.
|
|
519
|
-
*/
|
|
520
|
-
outputFacialTransformationMatrixes?: boolean | undefined;
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
/**
|
|
524
|
-
* Represents the face landmarks deection results generated by `FaceLandmarker`.
|
|
525
|
-
*/
|
|
526
|
-
export declare interface FaceLandmarkerResult {
|
|
527
|
-
/** Detected face landmarks in normalized image coordinates. */
|
|
528
|
-
faceLandmarks: NormalizedLandmark[][];
|
|
529
|
-
/** Optional face blendshapes results. */
|
|
530
|
-
faceBlendshapes?: Classifications[];
|
|
531
|
-
/** Optional facial transformation matrix. */
|
|
532
|
-
facialTransformationMatrixes?: Matrix[];
|
|
533
|
-
}
|
|
534
|
-
|
|
535
|
-
/**
|
|
536
|
-
* A class containing the pairs of landmark indices to be rendered with
|
|
537
|
-
* connections.
|
|
538
|
-
*/
|
|
539
|
-
export declare class FaceLandmarksConnections {
|
|
540
|
-
static FACE_LANDMARKS_LIPS: Connection[];
|
|
541
|
-
static FACE_LANDMARKS_LEFT_EYE: Connection[];
|
|
542
|
-
static FACE_LANDMARKS_LEFT_EYEBROW: Connection[];
|
|
543
|
-
static FACE_LANDMARKS_LEFT_IRIS: Connection[];
|
|
544
|
-
static FACE_LANDMARKS_RIGHT_EYE: Connection[];
|
|
545
|
-
static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[];
|
|
546
|
-
static FACE_LANDMARKS_RIGHT_IRIS: Connection[];
|
|
547
|
-
static FACE_LANDMARKS_FACE_OVAL: Connection[];
|
|
548
|
-
static FACE_LANDMARKS_CONTOURS: Connection[];
|
|
549
|
-
static FACE_LANDMARKS_TESSELATION: Connection[];
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
/** Performs face stylization on images. */
|
|
553
|
-
export declare class FaceStylizer extends VisionTaskRunner {
|
|
554
|
-
/**
|
|
555
|
-
* Initializes the Wasm runtime and creates a new Face Stylizer from the
|
|
556
|
-
* provided options.
|
|
557
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
558
|
-
* the Wasm binary and its loader.
|
|
559
|
-
* @param faceStylizerOptions The options for the Face Stylizer. Note
|
|
560
|
-
* that either a path to the model asset or a model buffer needs to be
|
|
561
|
-
* provided (via `baseOptions`).
|
|
562
|
-
*/
|
|
563
|
-
static createFromOptions(wasmFileset: WasmFileset, faceStylizerOptions: FaceStylizerOptions): Promise<FaceStylizer>;
|
|
564
|
-
/**
|
|
565
|
-
* Initializes the Wasm runtime and creates a new Face Stylizer based on
|
|
566
|
-
* the provided model asset buffer.
|
|
567
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
568
|
-
* the Wasm binary and its loader.
|
|
569
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
570
|
-
*/
|
|
571
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<FaceStylizer>;
|
|
572
|
-
/**
|
|
573
|
-
* Initializes the Wasm runtime and creates a new Face Stylizer based on
|
|
574
|
-
* the path to the model asset.
|
|
575
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
576
|
-
* the Wasm binary and its loader.
|
|
577
|
-
* @param modelAssetPath The path to the model asset.
|
|
578
|
-
*/
|
|
579
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<FaceStylizer>;
|
|
580
|
-
private constructor();
|
|
581
|
-
/**
|
|
582
|
-
* Sets new options for the Face Stylizer.
|
|
583
|
-
*
|
|
584
|
-
* Calling `setOptions()` with a subset of options only affects those
|
|
585
|
-
* options. You can reset an option back to its default value by
|
|
586
|
-
* explicitly setting it to `undefined`.
|
|
587
|
-
*
|
|
588
|
-
* @param options The options for the Face Stylizer.
|
|
589
|
-
*/
|
|
590
|
-
setOptions(options: FaceStylizerOptions): Promise<void>;
|
|
591
|
-
/**
|
|
592
|
-
* Performs face stylization on the provided single image. The method returns
|
|
593
|
-
* synchronously once the callback returns. Only use this method when the
|
|
594
|
-
* FaceStylizer is created with the image running mode.
|
|
595
|
-
*
|
|
596
|
-
* @param image An image to process.
|
|
597
|
-
* @param callback The callback that is invoked with the stylized image. The
|
|
598
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
599
|
-
* callback.
|
|
600
|
-
*/
|
|
601
|
-
stylize(image: ImageSource, callback: ImageCallback): void;
|
|
602
|
-
/**
|
|
603
|
-
* Performs face stylization on the provided single image. The method returns
|
|
604
|
-
* synchronously once the callback returns. Only use this method when the
|
|
605
|
-
* FaceStylizer is created with the image running mode.
|
|
606
|
-
*
|
|
607
|
-
* The 'imageProcessingOptions' parameter can be used to specify one or all
|
|
608
|
-
* of:
|
|
609
|
-
* - the rotation to apply to the image before performing stylization, by
|
|
610
|
-
* setting its 'rotationDegrees' property.
|
|
611
|
-
* - the region-of-interest on which to perform stylization, by setting its
|
|
612
|
-
* 'regionOfInterest' property. If not specified, the full image is used.
|
|
613
|
-
* If both are specified, the crop around the region-of-interest is extracted
|
|
614
|
-
* first, then the specified rotation is applied to the crop.
|
|
615
|
-
*
|
|
616
|
-
* @param image An image to process.
|
|
617
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
618
|
-
* to process the input image before running inference.
|
|
619
|
-
* @param callback The callback that is invoked with the stylized image. The
|
|
620
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
621
|
-
* callback.
|
|
622
|
-
*/
|
|
623
|
-
stylize(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageCallback): void;
|
|
624
|
-
/**
|
|
625
|
-
* Performs face stylization on the provided video frame. Only use this method
|
|
626
|
-
* when the FaceStylizer is created with the video running mode.
|
|
627
|
-
*
|
|
628
|
-
* The input frame can be of any size. It's required to provide the video
|
|
629
|
-
* frame's timestamp (in milliseconds). The input timestamps must be
|
|
630
|
-
* monotonically increasing.
|
|
631
|
-
*
|
|
632
|
-
* @param videoFrame A video frame to process.
|
|
633
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
634
|
-
* @param callback The callback that is invoked with the stylized image. The
|
|
635
|
-
* lifetime of the returned data is only guaranteed for the duration of
|
|
636
|
-
* the callback.
|
|
637
|
-
*/
|
|
638
|
-
stylizeForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageCallback): void;
|
|
639
|
-
/**
|
|
640
|
-
* Performs face stylization on the provided video frame. Only use this
|
|
641
|
-
* method when the FaceStylizer is created with the video running mode.
|
|
642
|
-
*
|
|
643
|
-
* The 'imageProcessingOptions' parameter can be used to specify one or all
|
|
644
|
-
* of:
|
|
645
|
-
* - the rotation to apply to the image before performing stylization, by
|
|
646
|
-
* setting its 'rotationDegrees' property.
|
|
647
|
-
* - the region-of-interest on which to perform stylization, by setting its
|
|
648
|
-
* 'regionOfInterest' property. If not specified, the full image is used.
|
|
649
|
-
* If both are specified, the crop around the region-of-interest is
|
|
650
|
-
* extracted first, then the specified rotation is applied to the crop.
|
|
651
|
-
*
|
|
652
|
-
* The input frame can be of any size. It's required to provide the video
|
|
653
|
-
* frame's timestamp (in milliseconds). The input timestamps must be
|
|
654
|
-
* monotonically increasing.
|
|
655
|
-
*
|
|
656
|
-
* @param videoFrame A video frame to process.
|
|
657
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
658
|
-
* to process the input image before running inference.
|
|
659
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
660
|
-
* @param callback The callback that is invoked with the stylized image. The
|
|
661
|
-
* lifetime of the returned data is only guaranteed for the duration of
|
|
662
|
-
* the callback.
|
|
663
|
-
*/
|
|
664
|
-
stylizeForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageCallback): void;
|
|
665
|
-
}
|
|
666
|
-
|
|
667
|
-
/** Options to configure the MediaPipe Face Stylizer Task */
|
|
668
|
-
export declare interface FaceStylizerOptions extends VisionTaskOptions {
|
|
669
|
-
}
|
|
670
|
-
|
|
671
|
-
/**
|
|
672
|
-
* Resolves the files required for the MediaPipe Task APIs.
|
|
673
|
-
*
|
|
674
|
-
* This class verifies whether SIMD is supported in the current environment and
|
|
675
|
-
* loads the SIMD files only if support is detected. The returned filesets
|
|
676
|
-
* require that the Wasm files are published without renaming. If this is not
|
|
677
|
-
* possible, you can invoke the MediaPipe Tasks APIs using a manually created
|
|
678
|
-
* `WasmFileset`.
|
|
679
|
-
*/
|
|
680
|
-
export declare class FilesetResolver {
|
|
681
|
-
/**
|
|
682
|
-
* Returns whether SIMD is supported in the current environment.
|
|
683
|
-
*
|
|
684
|
-
* If your environment requires custom locations for the MediaPipe Wasm files,
|
|
685
|
-
* you can use `isSimdSupported()` to decide whether to load the SIMD-based
|
|
686
|
-
* assets.
|
|
687
|
-
*
|
|
688
|
-
* @return Whether SIMD support was detected in the current environment.
|
|
689
|
-
*/
|
|
690
|
-
static isSimdSupported(): Promise<boolean>;
|
|
691
|
-
/**
|
|
692
|
-
* Creates a fileset for the MediaPipe Audio tasks.
|
|
693
|
-
*
|
|
694
|
-
* @param basePath An optional base path to specify the directory the Wasm
|
|
695
|
-
* files should be loaded from. If not specified, the Wasm files are
|
|
696
|
-
* loaded from the host's root directory.
|
|
697
|
-
* @return A `WasmFileset` that can be used to initialize MediaPipe Audio
|
|
698
|
-
* tasks.
|
|
699
|
-
*/
|
|
700
|
-
static forAudioTasks(basePath?: string): Promise<WasmFileset>;
|
|
701
|
-
/**
|
|
702
|
-
* Creates a fileset for the MediaPipe Text tasks.
|
|
703
|
-
*
|
|
704
|
-
* @param basePath An optional base path to specify the directory the Wasm
|
|
705
|
-
* files should be loaded from. If not specified, the Wasm files are
|
|
706
|
-
* loaded from the host's root directory.
|
|
707
|
-
* @return A `WasmFileset` that can be used to initialize MediaPipe Text
|
|
708
|
-
* tasks.
|
|
709
|
-
*/
|
|
710
|
-
static forTextTasks(basePath?: string): Promise<WasmFileset>;
|
|
711
|
-
/**
|
|
712
|
-
* Creates a fileset for the MediaPipe Vision tasks.
|
|
713
|
-
*
|
|
714
|
-
* @param basePath An optional base path to specify the directory the Wasm
|
|
715
|
-
* files should be loaded from. If not specified, the Wasm files are
|
|
716
|
-
* loaded from the host's root directory.
|
|
717
|
-
* @return A `WasmFileset` that can be used to initialize MediaPipe Vision
|
|
718
|
-
* tasks.
|
|
719
|
-
*/
|
|
720
|
-
static forVisionTasks(basePath?: string): Promise<WasmFileset>;
|
|
721
|
-
}
|
|
722
|
-
|
|
723
|
-
/** Performs hand gesture recognition on images. */
|
|
724
|
-
export declare class GestureRecognizer extends VisionTaskRunner {
|
|
725
|
-
/**
|
|
726
|
-
* An array containing the pairs of hand landmark indices to be rendered with
|
|
727
|
-
* connections.
|
|
728
|
-
*/
|
|
729
|
-
static HAND_CONNECTIONS: Connection[];
|
|
730
|
-
/**
|
|
731
|
-
* Initializes the Wasm runtime and creates a new gesture recognizer from the
|
|
732
|
-
* provided options.
|
|
733
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
734
|
-
* Wasm binary and its loader.
|
|
735
|
-
* @param gestureRecognizerOptions The options for the gesture recognizer.
|
|
736
|
-
* Note that either a path to the model asset or a model buffer needs to
|
|
737
|
-
* be provided (via `baseOptions`).
|
|
738
|
-
*/
|
|
739
|
-
static createFromOptions(wasmFileset: WasmFileset, gestureRecognizerOptions: GestureRecognizerOptions): Promise<GestureRecognizer>;
|
|
740
|
-
/**
|
|
741
|
-
* Initializes the Wasm runtime and creates a new gesture recognizer based on
|
|
742
|
-
* the provided model asset buffer.
|
|
743
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
744
|
-
* Wasm binary and its loader.
|
|
745
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
746
|
-
*/
|
|
747
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<GestureRecognizer>;
|
|
748
|
-
/**
|
|
749
|
-
* Initializes the Wasm runtime and creates a new gesture recognizer based on
|
|
750
|
-
* the path to the model asset.
|
|
751
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
752
|
-
* Wasm binary and its loader.
|
|
753
|
-
* @param modelAssetPath The path to the model asset.
|
|
754
|
-
*/
|
|
755
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<GestureRecognizer>;
|
|
756
|
-
private constructor();
|
|
757
|
-
/**
|
|
758
|
-
* Sets new options for the gesture recognizer.
|
|
759
|
-
*
|
|
760
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
761
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
762
|
-
* to `undefined`.
|
|
763
|
-
*
|
|
764
|
-
* @param options The options for the gesture recognizer.
|
|
765
|
-
*/
|
|
766
|
-
setOptions(options: GestureRecognizerOptions): Promise<void>;
|
|
767
|
-
/**
|
|
768
|
-
* Performs gesture recognition on the provided single image and waits
|
|
769
|
-
* synchronously for the response. Only use this method when the
|
|
770
|
-
* GestureRecognizer is created with running mode `image`.
|
|
771
|
-
*
|
|
772
|
-
* @param image A single image to process.
|
|
773
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
774
|
-
* to process the input image before running inference.
|
|
775
|
-
* @return The detected gestures.
|
|
776
|
-
*/
|
|
777
|
-
recognize(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
|
|
778
|
-
/**
|
|
779
|
-
* Performs gesture recognition on the provided video frame and waits
|
|
780
|
-
* synchronously for the response. Only use this method when the
|
|
781
|
-
* GestureRecognizer is created with running mode `video`.
|
|
782
|
-
*
|
|
783
|
-
* @param videoFrame A video frame to process.
|
|
784
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
785
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
786
|
-
* to process the input image before running inference.
|
|
787
|
-
* @return The detected gestures.
|
|
788
|
-
*/
|
|
789
|
-
recognizeForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
/** Options to configure the MediaPipe Gesture Recognizer Task */
|
|
793
|
-
export declare interface GestureRecognizerOptions extends VisionTaskOptions {
|
|
794
|
-
/**
|
|
795
|
-
* The maximum number of hands can be detected by the GestureRecognizer.
|
|
796
|
-
* Defaults to 1.
|
|
797
|
-
*/
|
|
798
|
-
numHands?: number | undefined;
|
|
799
|
-
/**
|
|
800
|
-
* The minimum confidence score for the hand detection to be considered
|
|
801
|
-
* successful. Defaults to 0.5.
|
|
802
|
-
*/
|
|
803
|
-
minHandDetectionConfidence?: number | undefined;
|
|
804
|
-
/**
|
|
805
|
-
* The minimum confidence score of hand presence score in the hand landmark
|
|
806
|
-
* detection. Defaults to 0.5.
|
|
807
|
-
*/
|
|
808
|
-
minHandPresenceConfidence?: number | undefined;
|
|
809
|
-
/**
|
|
810
|
-
* The minimum confidence score for the hand tracking to be considered
|
|
811
|
-
* successful. Defaults to 0.5.
|
|
812
|
-
*/
|
|
813
|
-
minTrackingConfidence?: number | undefined;
|
|
814
|
-
/**
|
|
815
|
-
* Sets the optional `ClassifierOptions` controlling the canned gestures
|
|
816
|
-
* classifier, such as score threshold, allow list and deny list of gestures.
|
|
817
|
-
* The categories for canned gesture
|
|
818
|
-
* classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up",
|
|
819
|
-
* "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
|
|
820
|
-
*/
|
|
821
|
-
cannedGesturesClassifierOptions?: ClassifierOptions | undefined;
|
|
822
|
-
/**
|
|
823
|
-
* Options for configuring the custom gestures classifier, such as score
|
|
824
|
-
* threshold, allow list and deny list of gestures.
|
|
825
|
-
*/
|
|
826
|
-
customGesturesClassifierOptions?: ClassifierOptions | undefined;
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
/**
|
|
830
|
-
* Represents the gesture recognition results generated by `GestureRecognizer`.
|
|
831
|
-
*/
|
|
832
|
-
export declare interface GestureRecognizerResult {
|
|
833
|
-
/** Hand landmarks of detected hands. */
|
|
834
|
-
landmarks: NormalizedLandmark[][];
|
|
835
|
-
/** Hand landmarks in world coordniates of detected hands. */
|
|
836
|
-
worldLandmarks: Landmark[][];
|
|
837
|
-
/** Handedness of detected hands. */
|
|
838
|
-
handednesses: Category[][];
|
|
839
|
-
/**
|
|
840
|
-
* Recognized hand gestures of detected hands. Note that the index of the
|
|
841
|
-
* gesture is always -1, because the raw indices from multiple gesture
|
|
842
|
-
* classifiers cannot consolidate to a meaningful index.
|
|
843
|
-
*/
|
|
844
|
-
gestures: Category[][];
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
/** Performs hand landmarks detection on images. */
|
|
848
|
-
export declare class HandLandmarker extends VisionTaskRunner {
|
|
849
|
-
/**
|
|
850
|
-
* An array containing the pairs of hand landmark indices to be rendered with
|
|
851
|
-
* connections.
|
|
852
|
-
*/
|
|
853
|
-
static HAND_CONNECTIONS: Connection[];
|
|
854
|
-
/**
|
|
855
|
-
* Initializes the Wasm runtime and creates a new `HandLandmarker` from the
|
|
856
|
-
* provided options.
|
|
857
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
858
|
-
* Wasm binary and its loader.
|
|
859
|
-
* @param handLandmarkerOptions The options for the HandLandmarker.
|
|
860
|
-
* Note that either a path to the model asset or a model buffer needs to
|
|
861
|
-
* be provided (via `baseOptions`).
|
|
862
|
-
*/
|
|
863
|
-
static createFromOptions(wasmFileset: WasmFileset, handLandmarkerOptions: HandLandmarkerOptions): Promise<HandLandmarker>;
|
|
864
|
-
/**
|
|
865
|
-
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
|
|
866
|
-
* the provided model asset buffer.
|
|
867
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
868
|
-
* Wasm binary and its loader.
|
|
869
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
870
|
-
*/
|
|
871
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<HandLandmarker>;
|
|
872
|
-
/**
|
|
873
|
-
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
|
|
874
|
-
* the path to the model asset.
|
|
875
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
876
|
-
* Wasm binary and its loader.
|
|
877
|
-
* @param modelAssetPath The path to the model asset.
|
|
878
|
-
*/
|
|
879
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<HandLandmarker>;
|
|
880
|
-
private constructor();
|
|
881
|
-
/**
|
|
882
|
-
* Sets new options for this `HandLandmarker`.
|
|
883
|
-
*
|
|
884
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
885
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
886
|
-
* to `undefined`.
|
|
887
|
-
*
|
|
888
|
-
* @param options The options for the hand landmarker.
|
|
889
|
-
*/
|
|
890
|
-
setOptions(options: HandLandmarkerOptions): Promise<void>;
|
|
891
|
-
/**
|
|
892
|
-
* Performs hand landmarks detection on the provided single image and waits
|
|
893
|
-
* synchronously for the response. Only use this method when the
|
|
894
|
-
* HandLandmarker is created with running mode `image`.
|
|
895
|
-
*
|
|
896
|
-
* @param image An image to process.
|
|
897
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
898
|
-
* to process the input image before running inference.
|
|
899
|
-
* @return The detected hand landmarks.
|
|
900
|
-
*/
|
|
901
|
-
detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
|
|
902
|
-
/**
|
|
903
|
-
* Performs hand landmarks detection on the provided video frame and waits
|
|
904
|
-
* synchronously for the response. Only use this method when the
|
|
905
|
-
* HandLandmarker is created with running mode `video`.
|
|
906
|
-
*
|
|
907
|
-
* @param videoFrame A video frame to process.
|
|
908
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
909
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
910
|
-
* to process the input image before running inference.
|
|
911
|
-
* @return The detected hand landmarks.
|
|
912
|
-
*/
|
|
913
|
-
detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
/** Options to configure the MediaPipe HandLandmarker Task */
|
|
917
|
-
export declare interface HandLandmarkerOptions extends VisionTaskOptions {
|
|
918
|
-
/**
|
|
919
|
-
* The maximum number of hands can be detected by the HandLandmarker.
|
|
920
|
-
* Defaults to 1.
|
|
921
|
-
*/
|
|
922
|
-
numHands?: number | undefined;
|
|
923
|
-
/**
|
|
924
|
-
* The minimum confidence score for the hand detection to be considered
|
|
925
|
-
* successful. Defaults to 0.5.
|
|
926
|
-
*/
|
|
927
|
-
minHandDetectionConfidence?: number | undefined;
|
|
928
|
-
/**
|
|
929
|
-
* The minimum confidence score of hand presence score in the hand landmark
|
|
930
|
-
* detection. Defaults to 0.5.
|
|
931
|
-
*/
|
|
932
|
-
minHandPresenceConfidence?: number | undefined;
|
|
933
|
-
/**
|
|
934
|
-
* The minimum confidence score for the hand tracking to be considered
|
|
935
|
-
* successful. Defaults to 0.5.
|
|
936
|
-
*/
|
|
937
|
-
minTrackingConfidence?: number | undefined;
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
/**
|
|
941
|
-
* Represents the hand landmarks deection results generated by `HandLandmarker`.
|
|
942
|
-
*/
|
|
943
|
-
export declare interface HandLandmarkerResult {
|
|
944
|
-
/** Hand landmarks of detected hands. */
|
|
945
|
-
landmarks: NormalizedLandmark[][];
|
|
946
|
-
/** Hand landmarks in world coordniates of detected hands. */
|
|
947
|
-
worldLandmarks: Landmark[][];
|
|
948
|
-
/** Handedness of detected hands. */
|
|
949
|
-
handednesses: Category[][];
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
/**
|
|
953
|
-
* A callback that receives an `ImageData` object from a Vision task. The
|
|
954
|
-
* lifetime of the underlying data is limited to the duration of the callback.
|
|
955
|
-
* If asynchronous processing is needed, all data needs to be copied before the
|
|
956
|
-
* callback returns.
|
|
957
|
-
*
|
|
958
|
-
* The `WebGLTexture` output type is reserved for future usage.
|
|
959
|
-
*/
|
|
960
|
-
export declare type ImageCallback = (image: ImageData | WebGLTexture, width: number, height: number) => void;
|
|
961
|
-
|
|
962
|
-
/** Performs classification on images. */
|
|
963
|
-
export declare class ImageClassifier extends VisionTaskRunner {
|
|
964
|
-
/**
|
|
965
|
-
* Initializes the Wasm runtime and creates a new image classifier from the
|
|
966
|
-
* provided options.
|
|
967
|
-
* @param wasmFileset A configuration object that provides the location
|
|
968
|
-
* Wasm binary and its loader.
|
|
969
|
-
* @param imageClassifierOptions The options for the image classifier. Note
|
|
970
|
-
* that either a path to the model asset or a model buffer needs to be
|
|
971
|
-
* provided (via `baseOptions`).
|
|
972
|
-
*/
|
|
973
|
-
static createFromOptions(wasmFileset: WasmFileset, imageClassifierOptions: ImageClassifierOptions): Promise<ImageClassifier>;
|
|
974
|
-
/**
|
|
975
|
-
* Initializes the Wasm runtime and creates a new image classifier based on
|
|
976
|
-
* the provided model asset buffer.
|
|
977
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
978
|
-
* Wasm binary and its loader.
|
|
979
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
980
|
-
*/
|
|
981
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageClassifier>;
|
|
982
|
-
/**
|
|
983
|
-
* Initializes the Wasm runtime and creates a new image classifier based on
|
|
984
|
-
* the path to the model asset.
|
|
985
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
986
|
-
* Wasm binary and its loader.
|
|
987
|
-
* @param modelAssetPath The path to the model asset.
|
|
988
|
-
*/
|
|
989
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageClassifier>;
|
|
990
|
-
private constructor();
|
|
991
|
-
/**
|
|
992
|
-
* Sets new options for the image classifier.
|
|
993
|
-
*
|
|
994
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
995
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
996
|
-
* to `undefined`.
|
|
997
|
-
*
|
|
998
|
-
* @param options The options for the image classifier.
|
|
999
|
-
*/
|
|
1000
|
-
setOptions(options: ImageClassifierOptions): Promise<void>;
|
|
1001
|
-
/**
|
|
1002
|
-
* Performs image classification on the provided single image and waits
|
|
1003
|
-
* synchronously for the response. Only use this method when the
|
|
1004
|
-
* ImageClassifier is created with running mode `image`.
|
|
1005
|
-
*
|
|
1006
|
-
* @param image An image to process.
|
|
1007
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1008
|
-
* to process the input image before running inference.
|
|
1009
|
-
* @return The classification result of the image
|
|
1010
|
-
*/
|
|
1011
|
-
classify(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
|
|
1012
|
-
/**
|
|
1013
|
-
* Performs image classification on the provided video frame and waits
|
|
1014
|
-
* synchronously for the response. Only use this method when the
|
|
1015
|
-
* ImageClassifier is created with running mode `video`.
|
|
1016
|
-
*
|
|
1017
|
-
* @param videoFrame A video frame to process.
|
|
1018
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
1019
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1020
|
-
* to process the input image before running inference.
|
|
1021
|
-
* @return The classification result of the image
|
|
1022
|
-
*/
|
|
1023
|
-
classifyForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
/** Options to configure the MediaPipe Image Classifier Task. */
|
|
1027
|
-
export declare interface ImageClassifierOptions extends ClassifierOptions, VisionTaskOptions {
|
|
1028
|
-
}
|
|
1029
|
-
|
|
1030
|
-
/** Classification results of a model. */
|
|
1031
|
-
export declare interface ImageClassifierResult {
|
|
1032
|
-
/** The classification results for each head of the model. */
|
|
1033
|
-
classifications: Classifications[];
|
|
1034
|
-
/**
|
|
1035
|
-
* The optional timestamp (in milliseconds) of the start of the chunk of data
|
|
1036
|
-
* corresponding to these results.
|
|
1037
|
-
*
|
|
1038
|
-
* This is only used for classification on time series (e.g. audio
|
|
1039
|
-
* classification). In these use cases, the amount of data to process might
|
|
1040
|
-
* exceed the maximum size that the model can process: to solve this, the
|
|
1041
|
-
* input data is split into multiple chunks starting at different timestamps.
|
|
1042
|
-
*/
|
|
1043
|
-
timestampMs?: number;
|
|
1044
|
-
}
|
|
1045
|
-
|
|
1046
|
-
/** Performs embedding extraction on images. */
|
|
1047
|
-
export declare class ImageEmbedder extends VisionTaskRunner {
|
|
1048
|
-
/**
|
|
1049
|
-
* Initializes the Wasm runtime and creates a new image embedder from the
|
|
1050
|
-
* provided options.
|
|
1051
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1052
|
-
* Wasm binary and its loader.
|
|
1053
|
-
* @param imageEmbedderOptions The options for the image embedder. Note that
|
|
1054
|
-
* either a path to the TFLite model or the model itself needs to be
|
|
1055
|
-
* provided (via `baseOptions`).
|
|
1056
|
-
*/
|
|
1057
|
-
static createFromOptions(wasmFileset: WasmFileset, imageEmbedderOptions: ImageEmbedderOptions): Promise<ImageEmbedder>;
|
|
1058
|
-
/**
|
|
1059
|
-
* Initializes the Wasm runtime and creates a new image embedder based on the
|
|
1060
|
-
* provided model asset buffer.
|
|
1061
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1062
|
-
* Wasm binary and its loader.
|
|
1063
|
-
* @param modelAssetBuffer A binary representation of the TFLite model.
|
|
1064
|
-
*/
|
|
1065
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageEmbedder>;
|
|
1066
|
-
/**
|
|
1067
|
-
* Initializes the Wasm runtime and creates a new image embedder based on the
|
|
1068
|
-
* path to the model asset.
|
|
1069
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1070
|
-
* Wasm binary and its loader.
|
|
1071
|
-
* @param modelAssetPath The path to the TFLite model.
|
|
1072
|
-
*/
|
|
1073
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageEmbedder>;
|
|
1074
|
-
private constructor();
|
|
1075
|
-
/**
|
|
1076
|
-
* Sets new options for the image embedder.
|
|
1077
|
-
*
|
|
1078
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
1079
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
1080
|
-
* to `undefined`.
|
|
1081
|
-
*
|
|
1082
|
-
* @param options The options for the image embedder.
|
|
1083
|
-
*/
|
|
1084
|
-
setOptions(options: ImageEmbedderOptions): Promise<void>;
|
|
1085
|
-
/**
|
|
1086
|
-
* Performs embedding extraction on the provided single image and waits
|
|
1087
|
-
* synchronously for the response. Only use this method when the
|
|
1088
|
-
* ImageEmbedder is created with running mode `image`.
|
|
1089
|
-
*
|
|
1090
|
-
* @param image The image to process.
|
|
1091
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1092
|
-
* to process the input image before running inference.
|
|
1093
|
-
* @return The classification result of the image
|
|
1094
|
-
*/
|
|
1095
|
-
embed(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
|
|
1096
|
-
/**
|
|
1097
|
-
* Performs embedding extraction on the provided video frame and waits
|
|
1098
|
-
* synchronously for the response. Only use this method when the
|
|
1099
|
-
* ImageEmbedder is created with running mode `video`.
|
|
1100
|
-
*
|
|
1101
|
-
* @param imageFrame The image frame to process.
|
|
1102
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
1103
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1104
|
-
* to process the input image before running inference.
|
|
1105
|
-
* @return The classification result of the image
|
|
1106
|
-
*/
|
|
1107
|
-
embedForVideo(imageFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
|
|
1108
|
-
/**
|
|
1109
|
-
* Utility function to compute cosine similarity[1] between two `Embedding`
|
|
1110
|
-
* objects.
|
|
1111
|
-
*
|
|
1112
|
-
* [1]: https://en.wikipedia.org/wiki/Cosine_similarity
|
|
1113
|
-
*
|
|
1114
|
-
* @throws if the embeddings are of different types(float vs. quantized), have
|
|
1115
|
-
* different sizes, or have an L2-norm of 0.
|
|
1116
|
-
*/
|
|
1117
|
-
static cosineSimilarity(u: Embedding, v: Embedding): number;
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
/** Options for configuring a MediaPipe Image Embedder task. */
|
|
1121
|
-
export declare interface ImageEmbedderOptions extends EmbedderOptions, VisionTaskOptions {
|
|
1122
|
-
}
|
|
1123
|
-
|
|
1124
|
-
/** Embedding results for a given embedder model. */
|
|
1125
|
-
export declare interface ImageEmbedderResult {
|
|
1126
|
-
/**
|
|
1127
|
-
* The embedding results for each model head, i.e. one for each output tensor.
|
|
1128
|
-
*/
|
|
1129
|
-
embeddings: Embedding[];
|
|
1130
|
-
/**
|
|
1131
|
-
* The optional timestamp (in milliseconds) of the start of the chunk of
|
|
1132
|
-
* data corresponding to these results.
|
|
1133
|
-
*
|
|
1134
|
-
* This is only used for embedding extraction on time series (e.g. audio
|
|
1135
|
-
* embedding). In these use cases, the amount of data to process might
|
|
1136
|
-
* exceed the maximum size that the model can process: to solve this, the
|
|
1137
|
-
* input data is split into multiple chunks starting at different timestamps.
|
|
1138
|
-
*/
|
|
1139
|
-
timestampMs?: number;
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
/**
|
|
1143
|
-
* Options for image processing.
|
|
1144
|
-
*
|
|
1145
|
-
* If both region-or-interest and rotation are specified, the crop around the
|
|
1146
|
-
* region-of-interest is extracted first, then the specified rotation is applied
|
|
1147
|
-
* to the crop.
|
|
1148
|
-
*/
|
|
1149
|
-
declare interface ImageProcessingOptions {
|
|
1150
|
-
/**
|
|
1151
|
-
* The optional region-of-interest to crop from the image. If not specified,
|
|
1152
|
-
* the full image is used.
|
|
1153
|
-
*
|
|
1154
|
-
* Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
|
|
1155
|
-
*/
|
|
1156
|
-
regionOfInterest?: RectF;
|
|
1157
|
-
/**
|
|
1158
|
-
* The rotation to apply to the image (or cropped region-of-interest), in
|
|
1159
|
-
* degrees clockwise.
|
|
1160
|
-
*
|
|
1161
|
-
* The rotation must be a multiple (positive or negative) of 90°.
|
|
1162
|
-
*/
|
|
1163
|
-
rotationDegrees?: number;
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
/** Performs image segmentation on images. */
|
|
1167
|
-
export declare class ImageSegmenter extends VisionTaskRunner {
|
|
1168
|
-
/**
|
|
1169
|
-
* Initializes the Wasm runtime and creates a new image segmenter from the
|
|
1170
|
-
* provided options.
|
|
1171
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1172
|
-
* the Wasm binary and its loader.
|
|
1173
|
-
* @param imageSegmenterOptions The options for the Image Segmenter. Note
|
|
1174
|
-
* that either a path to the model asset or a model buffer needs to be
|
|
1175
|
-
* provided (via `baseOptions`).
|
|
1176
|
-
*/
|
|
1177
|
-
static createFromOptions(wasmFileset: WasmFileset, imageSegmenterOptions: ImageSegmenterOptions): Promise<ImageSegmenter>;
|
|
1178
|
-
/**
|
|
1179
|
-
* Initializes the Wasm runtime and creates a new image segmenter based on
|
|
1180
|
-
* the provided model asset buffer.
|
|
1181
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1182
|
-
* the Wasm binary and its loader.
|
|
1183
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
1184
|
-
*/
|
|
1185
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ImageSegmenter>;
|
|
1186
|
-
/**
|
|
1187
|
-
* Initializes the Wasm runtime and creates a new image segmenter based on
|
|
1188
|
-
* the path to the model asset.
|
|
1189
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1190
|
-
* the Wasm binary and its loader.
|
|
1191
|
-
* @param modelAssetPath The path to the model asset.
|
|
1192
|
-
*/
|
|
1193
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageSegmenter>;
|
|
1194
|
-
private constructor();
|
|
1195
|
-
/**
|
|
1196
|
-
* Sets new options for the image segmenter.
|
|
1197
|
-
*
|
|
1198
|
-
* Calling `setOptions()` with a subset of options only affects those
|
|
1199
|
-
* options. You can reset an option back to its default value by
|
|
1200
|
-
* explicitly setting it to `undefined`.
|
|
1201
|
-
*
|
|
1202
|
-
* @param options The options for the image segmenter.
|
|
1203
|
-
*/
|
|
1204
|
-
setOptions(options: ImageSegmenterOptions): Promise<void>;
|
|
1205
|
-
/**
|
|
1206
|
-
* Performs image segmentation on the provided single image and invokes the
|
|
1207
|
-
* callback with the response. The method returns synchronously once the
|
|
1208
|
-
* callback returns. Only use this method when the ImageSegmenter is
|
|
1209
|
-
* created with running mode `image`.
|
|
1210
|
-
*
|
|
1211
|
-
* @param image An image to process.
|
|
1212
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1213
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1214
|
-
* callback.
|
|
1215
|
-
*/
|
|
1216
|
-
segment(image: ImageSource, callback: ImageSegmenterCallack): void;
|
|
1217
|
-
/**
|
|
1218
|
-
* Performs image segmentation on the provided single image and invokes the
|
|
1219
|
-
* callback with the response. The method returns synchronously once the
|
|
1220
|
-
* callback returns. Only use this method when the ImageSegmenter is
|
|
1221
|
-
* created with running mode `image`.
|
|
1222
|
-
*
|
|
1223
|
-
* @param image An image to process.
|
|
1224
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1225
|
-
* to process the input image before running inference.
|
|
1226
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1227
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1228
|
-
* callback.
|
|
1229
|
-
*/
|
|
1230
|
-
segment(image: ImageSource, imageProcessingOptions: ImageProcessingOptions, callback: ImageSegmenterCallack): void;
|
|
1231
|
-
/**
|
|
1232
|
-
* Performs image segmentation on the provided video frame and invokes the
|
|
1233
|
-
* callback with the response. The method returns synchronously once the
|
|
1234
|
-
* callback returns. Only use this method when the ImageSegmenter is
|
|
1235
|
-
* created with running mode `video`.
|
|
1236
|
-
*
|
|
1237
|
-
* @param videoFrame A video frame to process.
|
|
1238
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
1239
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1240
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1241
|
-
* callback.
|
|
1242
|
-
*/
|
|
1243
|
-
segmentForVideo(videoFrame: ImageSource, timestamp: number, callback: ImageSegmenterCallack): void;
|
|
1244
|
-
/**
|
|
1245
|
-
* Performs image segmentation on the provided video frame and invokes the
|
|
1246
|
-
* callback with the response. The method returns synchronously once the
|
|
1247
|
-
* callback returns. Only use this method when the ImageSegmenter is
|
|
1248
|
-
* created with running mode `video`.
|
|
1249
|
-
*
|
|
1250
|
-
* @param videoFrame A video frame to process.
|
|
1251
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1252
|
-
* to process the input image before running inference.
|
|
1253
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
1254
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1255
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1256
|
-
* callback.
|
|
1257
|
-
*/
|
|
1258
|
-
segmentForVideo(videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, timestamp: number, callback: ImageSegmenterCallack): void;
|
|
1259
|
-
/**
|
|
1260
|
-
* Get the category label list of the ImageSegmenter can recognize. For
|
|
1261
|
-
* `CATEGORY_MASK` type, the index in the category mask corresponds to the
|
|
1262
|
-
* category in the label list. For `CONFIDENCE_MASK` type, the output mask
|
|
1263
|
-
* list at index corresponds to the category in the label list.
|
|
1264
|
-
*
|
|
1265
|
-
* If there is no labelmap provided in the model file, empty label array is
|
|
1266
|
-
* returned.
|
|
1267
|
-
*
|
|
1268
|
-
* @return The labels used by the current model.
|
|
1269
|
-
*/
|
|
1270
|
-
getLabels(): string[];
|
|
1271
|
-
}
|
|
1272
|
-
|
|
1273
|
-
/**
|
|
1274
|
-
* A callback that receives the computed masks from the image segmenter. The
|
|
1275
|
-
* returned data is only valid for the duration of the callback. If
|
|
1276
|
-
* asynchronous processing is needed, all data needs to be copied before the
|
|
1277
|
-
* callback returns.
|
|
1278
|
-
*/
|
|
1279
|
-
export declare type ImageSegmenterCallack = (result: ImageSegmenterResult) => void;
|
|
1280
|
-
|
|
1281
|
-
/** Options to configure the MediaPipe Image Segmenter Task */
|
|
1282
|
-
export declare interface ImageSegmenterOptions extends VisionTaskOptions {
|
|
1283
|
-
/**
|
|
1284
|
-
* The locale to use for display names specified through the TFLite Model
|
|
1285
|
-
* Metadata, if any. Defaults to English.
|
|
1286
|
-
*/
|
|
1287
|
-
displayNamesLocale?: string | undefined;
|
|
1288
|
-
/** Whether to output confidence masks. Defaults to true. */
|
|
1289
|
-
outputConfidenceMasks?: boolean | undefined;
|
|
1290
|
-
/** Whether to output the category masks. Defaults to false. */
|
|
1291
|
-
outputCategoryMask?: boolean | undefined;
|
|
1292
|
-
}
|
|
1293
|
-
|
|
1294
|
-
/**
|
|
1295
|
-
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
|
1296
|
-
*
|
|
1297
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1298
|
-
* you may not use this file except in compliance with the License.
|
|
1299
|
-
* You may obtain a copy of the License at
|
|
1300
|
-
*
|
|
1301
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1302
|
-
*
|
|
1303
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1304
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1305
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1306
|
-
* See the License for the specific language governing permissions and
|
|
1307
|
-
* limitations under the License.
|
|
1308
|
-
*/
|
|
1309
|
-
/** The output result of ImageSegmenter. */
|
|
1310
|
-
export declare interface ImageSegmenterResult {
|
|
1311
|
-
/**
|
|
1312
|
-
* Multiple masks as Float32Arrays or WebGLTextures where, for each mask, each
|
|
1313
|
-
* pixel represents the prediction confidence, usually in the [0, 1] range.
|
|
1314
|
-
*/
|
|
1315
|
-
confidenceMasks?: Float32Array[] | WebGLTexture[];
|
|
1316
|
-
/**
|
|
1317
|
-
* A category mask as a Uint8ClampedArray or WebGLTexture where each
|
|
1318
|
-
* pixel represents the class which the pixel in the original image was
|
|
1319
|
-
* predicted to belong to.
|
|
1320
|
-
*/
|
|
1321
|
-
categoryMask?: Uint8ClampedArray | WebGLTexture;
|
|
1322
|
-
/** The width of the masks. */
|
|
1323
|
-
width: number;
|
|
1324
|
-
/** The height of the masks. */
|
|
1325
|
-
height: number;
|
|
1326
|
-
}
|
|
1327
|
-
|
|
1328
|
-
/**
|
|
1329
|
-
* Valid types of image sources which we can run our GraphRunner over.
|
|
1330
|
-
*/
|
|
1331
|
-
export declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
|
|
1332
|
-
|
|
1333
|
-
/**
|
|
1334
|
-
* Performs interactive segmentation on images.
|
|
1335
|
-
*
|
|
1336
|
-
* Users can represent user interaction through `RegionOfInterest`, which gives
|
|
1337
|
-
* a hint to InteractiveSegmenter to perform segmentation focusing on the given
|
|
1338
|
-
* region of interest.
|
|
1339
|
-
*
|
|
1340
|
-
* The API expects a TFLite model with mandatory TFLite Model Metadata.
|
|
1341
|
-
*
|
|
1342
|
-
* Input tensor:
|
|
1343
|
-
* (kTfLiteUInt8/kTfLiteFloat32)
|
|
1344
|
-
* - image input of size `[batch x height x width x channels]`.
|
|
1345
|
-
* - batch inference is not supported (`batch` is required to be 1).
|
|
1346
|
-
* - RGB inputs is supported (`channels` is required to be 3).
|
|
1347
|
-
* - if type is kTfLiteFloat32, NormalizationOptions are required to be
|
|
1348
|
-
* attached to the metadata for input normalization.
|
|
1349
|
-
* Output tensors:
|
|
1350
|
-
* (kTfLiteUInt8/kTfLiteFloat32)
|
|
1351
|
-
* - list of segmented masks.
|
|
1352
|
-
* - if `output_type` is CATEGORY_MASK, uint8 Image, Image vector of size 1.
|
|
1353
|
-
* - if `output_type` is CONFIDENCE_MASK, float32 Image list of size
|
|
1354
|
-
* `channels`.
|
|
1355
|
-
* - batch is always 1
|
|
1356
|
-
*/
|
|
1357
|
-
export declare class InteractiveSegmenter extends VisionTaskRunner {
|
|
1358
|
-
/**
|
|
1359
|
-
* Initializes the Wasm runtime and creates a new interactive segmenter from
|
|
1360
|
-
* the provided options.
|
|
1361
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1362
|
-
* the Wasm binary and its loader.
|
|
1363
|
-
* @param interactiveSegmenterOptions The options for the Interactive
|
|
1364
|
-
* Segmenter. Note that either a path to the model asset or a model buffer
|
|
1365
|
-
* needs to be provided (via `baseOptions`).
|
|
1366
|
-
* @return A new `InteractiveSegmenter`.
|
|
1367
|
-
*/
|
|
1368
|
-
static createFromOptions(wasmFileset: WasmFileset, interactiveSegmenterOptions: InteractiveSegmenterOptions): Promise<InteractiveSegmenter>;
|
|
1369
|
-
/**
|
|
1370
|
-
* Initializes the Wasm runtime and creates a new interactive segmenter based
|
|
1371
|
-
* on the provided model asset buffer.
|
|
1372
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1373
|
-
* the Wasm binary and its loader.
|
|
1374
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
1375
|
-
* @return A new `InteractiveSegmenter`.
|
|
1376
|
-
*/
|
|
1377
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<InteractiveSegmenter>;
|
|
1378
|
-
/**
|
|
1379
|
-
* Initializes the Wasm runtime and creates a new interactive segmenter based
|
|
1380
|
-
* on the path to the model asset.
|
|
1381
|
-
* @param wasmFileset A configuration object that provides the location of
|
|
1382
|
-
* the Wasm binary and its loader.
|
|
1383
|
-
* @param modelAssetPath The path to the model asset.
|
|
1384
|
-
* @return A new `InteractiveSegmenter`.
|
|
1385
|
-
*/
|
|
1386
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<InteractiveSegmenter>;
|
|
1387
|
-
private constructor();
|
|
1388
|
-
/**
|
|
1389
|
-
* Sets new options for the interactive segmenter.
|
|
1390
|
-
*
|
|
1391
|
-
* Calling `setOptions()` with a subset of options only affects those
|
|
1392
|
-
* options. You can reset an option back to its default value by
|
|
1393
|
-
* explicitly setting it to `undefined`.
|
|
1394
|
-
*
|
|
1395
|
-
* @param options The options for the interactive segmenter.
|
|
1396
|
-
* @return A Promise that resolves when the settings have been applied.
|
|
1397
|
-
*/
|
|
1398
|
-
setOptions(options: InteractiveSegmenterOptions): Promise<void>;
|
|
1399
|
-
/**
|
|
1400
|
-
* Performs interactive segmentation on the provided single image and invokes
|
|
1401
|
-
* the callback with the response. The `roi` parameter is used to represent a
|
|
1402
|
-
* user's region of interest for segmentation.
|
|
1403
|
-
*
|
|
1404
|
-
* If the output_type is `CATEGORY_MASK`, the callback is invoked with vector
|
|
1405
|
-
* of images that represent per-category segmented image mask. If the
|
|
1406
|
-
* output_type is `CONFIDENCE_MASK`, the callback is invoked with a vector of
|
|
1407
|
-
* images that contains only one confidence image mask. The method returns
|
|
1408
|
-
* synchronously once the callback returns.
|
|
1409
|
-
*
|
|
1410
|
-
* @param image An image to process.
|
|
1411
|
-
* @param roi The region of interest for segmentation.
|
|
1412
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1413
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1414
|
-
* callback.
|
|
1415
|
-
*/
|
|
1416
|
-
segment(image: ImageSource, roi: RegionOfInterest, callback: InteractiveSegmenterCallack): void;
|
|
1417
|
-
/**
|
|
1418
|
-
* Performs interactive segmentation on the provided single image and invokes
|
|
1419
|
-
* the callback with the response. The `roi` parameter is used to represent a
|
|
1420
|
-
* user's region of interest for segmentation.
|
|
1421
|
-
*
|
|
1422
|
-
* The 'image_processing_options' parameter can be used to specify the
|
|
1423
|
-
* rotation to apply to the image before performing segmentation, by setting
|
|
1424
|
-
* its 'rotationDegrees' field. Note that specifying a region-of-interest
|
|
1425
|
-
* using the 'regionOfInterest' field is NOT supported and will result in an
|
|
1426
|
-
* error.
|
|
1427
|
-
*
|
|
1428
|
-
* If the output_type is `CATEGORY_MASK`, the callback is invoked with vector
|
|
1429
|
-
* of images that represent per-category segmented image mask. If the
|
|
1430
|
-
* output_type is `CONFIDENCE_MASK`, the callback is invoked with a vector of
|
|
1431
|
-
* images that contains only one confidence image mask. The method returns
|
|
1432
|
-
* synchronously once the callback returns.
|
|
1433
|
-
*
|
|
1434
|
-
* @param image An image to process.
|
|
1435
|
-
* @param roi The region of interest for segmentation.
|
|
1436
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1437
|
-
* to process the input image before running inference.
|
|
1438
|
-
* @param callback The callback that is invoked with the segmented masks. The
|
|
1439
|
-
* lifetime of the returned data is only guaranteed for the duration of the
|
|
1440
|
-
* callback.
|
|
1441
|
-
*/
|
|
1442
|
-
segment(image: ImageSource, roi: RegionOfInterest, imageProcessingOptions: ImageProcessingOptions, callback: InteractiveSegmenterCallack): void;
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
/**
|
|
1446
|
-
* A callback that receives the computed masks from the interactive segmenter.
|
|
1447
|
-
* The returned data is only valid for the duration of the callback. If
|
|
1448
|
-
* asynchronous processing is needed, all data needs to be copied before the
|
|
1449
|
-
* callback returns.
|
|
1450
|
-
*/
|
|
1451
|
-
export declare type InteractiveSegmenterCallack = (result: InteractiveSegmenterResult) => void;
|
|
1452
|
-
|
|
1453
|
-
/** Options to configure the MediaPipe Interactive Segmenter Task */
|
|
1454
|
-
export declare interface InteractiveSegmenterOptions extends TaskRunnerOptions {
|
|
1455
|
-
/** Whether to output confidence masks. Defaults to true. */
|
|
1456
|
-
outputConfidenceMasks?: boolean | undefined;
|
|
1457
|
-
/** Whether to output the category masks. Defaults to false. */
|
|
1458
|
-
outputCategoryMask?: boolean | undefined;
|
|
1459
|
-
}
|
|
1460
|
-
|
|
1461
|
-
/**
|
|
1462
|
-
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
|
1463
|
-
*
|
|
1464
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1465
|
-
* you may not use this file except in compliance with the License.
|
|
1466
|
-
* You may obtain a copy of the License at
|
|
1467
|
-
*
|
|
1468
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1469
|
-
*
|
|
1470
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1471
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1472
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1473
|
-
* See the License for the specific language governing permissions and
|
|
1474
|
-
* limitations under the License.
|
|
1475
|
-
*/
|
|
1476
|
-
/** The output result of InteractiveSegmenter. */
|
|
1477
|
-
export declare interface InteractiveSegmenterResult {
|
|
1478
|
-
/**
|
|
1479
|
-
* Multiple masks as Float32Arrays or WebGLTextures where, for each mask, each
|
|
1480
|
-
* pixel represents the prediction confidence, usually in the [0, 1] range.
|
|
1481
|
-
*/
|
|
1482
|
-
confidenceMasks?: Float32Array[] | WebGLTexture[];
|
|
1483
|
-
/**
|
|
1484
|
-
* A category mask as a Uint8ClampedArray or WebGLTexture where each
|
|
1485
|
-
* pixel represents the class which the pixel in the original image was
|
|
1486
|
-
* predicted to belong to.
|
|
1487
|
-
*/
|
|
1488
|
-
categoryMask?: Uint8ClampedArray | WebGLTexture;
|
|
1489
|
-
/** The width of the masks. */
|
|
1490
|
-
width: number;
|
|
1491
|
-
/** The height of the masks. */
|
|
1492
|
-
height: number;
|
|
1493
|
-
}
|
|
1494
|
-
|
|
1495
|
-
/**
|
|
1496
|
-
* Landmark represents a point in 3D space with x, y, z coordinates. The
|
|
1497
|
-
* landmark coordinates are in meters. z represents the landmark depth,
|
|
1498
|
-
* and the smaller the value the closer the world landmark is to the camera.
|
|
1499
|
-
*/
|
|
1500
|
-
export declare interface Landmark {
|
|
1501
|
-
/** The x coordinates of the landmark. */
|
|
1502
|
-
x: number;
|
|
1503
|
-
/** The y coordinates of the landmark. */
|
|
1504
|
-
y: number;
|
|
1505
|
-
/** The z coordinates of the landmark. */
|
|
1506
|
-
z: number;
|
|
1507
|
-
}
|
|
1508
|
-
|
|
1509
|
-
/** Data that a user can use to specialize drawing options. */
|
|
1510
|
-
export declare interface LandmarkData {
|
|
1511
|
-
index?: number;
|
|
1512
|
-
from?: NormalizedLandmark;
|
|
1513
|
-
to?: NormalizedLandmark;
|
|
1514
|
-
}
|
|
1515
|
-
|
|
1516
|
-
/**
|
|
1517
|
-
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
|
1518
|
-
*
|
|
1519
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1520
|
-
* you may not use this file except in compliance with the License.
|
|
1521
|
-
* You may obtain a copy of the License at
|
|
1522
|
-
*
|
|
1523
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1524
|
-
*
|
|
1525
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1526
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1527
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1528
|
-
* See the License for the specific language governing permissions and
|
|
1529
|
-
* limitations under the License.
|
|
1530
|
-
*/
|
|
1531
|
-
/** A two-dimensional matrix. */
|
|
1532
|
-
declare interface Matrix {
|
|
1533
|
-
/** The number of rows. */
|
|
1534
|
-
rows: number;
|
|
1535
|
-
/** The number of columns. */
|
|
1536
|
-
columns: number;
|
|
1537
|
-
/** The values as a flattened one-dimensional array. */
|
|
1538
|
-
data: number[];
|
|
1539
|
-
}
|
|
1540
|
-
|
|
1541
|
-
/**
|
|
1542
|
-
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
|
1543
|
-
*
|
|
1544
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1545
|
-
* you may not use this file except in compliance with the License.
|
|
1546
|
-
* You may obtain a copy of the License at
|
|
1547
|
-
*
|
|
1548
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1549
|
-
*
|
|
1550
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1551
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1552
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1553
|
-
* See the License for the specific language governing permissions and
|
|
1554
|
-
* limitations under the License.
|
|
1555
|
-
*/
|
|
1556
|
-
/**
|
|
1557
|
-
* A keypoint, defined by the coordinates (x, y), normalized by the image
|
|
1558
|
-
* dimensions.
|
|
1559
|
-
*/
|
|
1560
|
-
declare interface NormalizedKeypoint {
|
|
1561
|
-
/** X in normalized image coordinates. */
|
|
1562
|
-
x: number;
|
|
1563
|
-
/** Y in normalized image coordinates. */
|
|
1564
|
-
y: number;
|
|
1565
|
-
/** Optional label of the keypoint. */
|
|
1566
|
-
label?: string;
|
|
1567
|
-
/** Optional score of the keypoint. */
|
|
1568
|
-
score?: number;
|
|
1569
|
-
}
|
|
1570
|
-
|
|
1571
|
-
/**
|
|
1572
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
1573
|
-
*
|
|
1574
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1575
|
-
* you may not use this file except in compliance with the License.
|
|
1576
|
-
* You may obtain a copy of the License at
|
|
1577
|
-
*
|
|
1578
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1579
|
-
*
|
|
1580
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1581
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1582
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1583
|
-
* See the License for the specific language governing permissions and
|
|
1584
|
-
* limitations under the License.
|
|
1585
|
-
*/
|
|
1586
|
-
/**
|
|
1587
|
-
* Normalized Landmark represents a point in 3D space with x, y, z coordinates.
|
|
1588
|
-
* x and y are normalized to [0.0, 1.0] by the image width and height
|
|
1589
|
-
* respectively. z represents the landmark depth, and the smaller the value the
|
|
1590
|
-
* closer the landmark is to the camera. The magnitude of z uses roughly the
|
|
1591
|
-
* same scale as x.
|
|
1592
|
-
*/
|
|
1593
|
-
export declare interface NormalizedLandmark {
|
|
1594
|
-
/** The x coordinates of the normalized landmark. */
|
|
1595
|
-
x: number;
|
|
1596
|
-
/** The y coordinates of the normalized landmark. */
|
|
1597
|
-
y: number;
|
|
1598
|
-
/** The z coordinates of the normalized landmark. */
|
|
1599
|
-
z: number;
|
|
1600
|
-
}
|
|
1601
|
-
|
|
1602
|
-
/** Performs object detection on images. */
|
|
1603
|
-
export declare class ObjectDetector extends VisionTaskRunner {
|
|
1604
|
-
/**
|
|
1605
|
-
* Initializes the Wasm runtime and creates a new object detector from the
|
|
1606
|
-
* provided options.
|
|
1607
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1608
|
-
* Wasm binary and its loader.
|
|
1609
|
-
* @param objectDetectorOptions The options for the Object Detector. Note that
|
|
1610
|
-
* either a path to the model asset or a model buffer needs to be
|
|
1611
|
-
* provided (via `baseOptions`).
|
|
1612
|
-
*/
|
|
1613
|
-
static createFromOptions(wasmFileset: WasmFileset, objectDetectorOptions: ObjectDetectorOptions): Promise<ObjectDetector>;
|
|
1614
|
-
/**
|
|
1615
|
-
* Initializes the Wasm runtime and creates a new object detector based on the
|
|
1616
|
-
* provided model asset buffer.
|
|
1617
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1618
|
-
* Wasm binary and its loader.
|
|
1619
|
-
* @param modelAssetBuffer A binary representation of the model.
|
|
1620
|
-
*/
|
|
1621
|
-
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array): Promise<ObjectDetector>;
|
|
1622
|
-
/**
|
|
1623
|
-
* Initializes the Wasm runtime and creates a new object detector based on the
|
|
1624
|
-
* path to the model asset.
|
|
1625
|
-
* @param wasmFileset A configuration object that provides the location of the
|
|
1626
|
-
* Wasm binary and its loader.
|
|
1627
|
-
* @param modelAssetPath The path to the model asset.
|
|
1628
|
-
*/
|
|
1629
|
-
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ObjectDetector>;
|
|
1630
|
-
private constructor();
|
|
1631
|
-
/**
|
|
1632
|
-
* Sets new options for the object detector.
|
|
1633
|
-
*
|
|
1634
|
-
* Calling `setOptions()` with a subset of options only affects those options.
|
|
1635
|
-
* You can reset an option back to its default value by explicitly setting it
|
|
1636
|
-
* to `undefined`.
|
|
1637
|
-
*
|
|
1638
|
-
* @param options The options for the object detector.
|
|
1639
|
-
*/
|
|
1640
|
-
setOptions(options: ObjectDetectorOptions): Promise<void>;
|
|
1641
|
-
/**
|
|
1642
|
-
* Performs object detection on the provided single image and waits
|
|
1643
|
-
* synchronously for the response. Only use this method when the
|
|
1644
|
-
* ObjectDetector is created with running mode `image`.
|
|
1645
|
-
*
|
|
1646
|
-
* @param image An image to process.
|
|
1647
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1648
|
-
* to process the input image before running inference.
|
|
1649
|
-
* @return A result containing a list of detected objects.
|
|
1650
|
-
*/
|
|
1651
|
-
detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
|
|
1652
|
-
/**
|
|
1653
|
-
* Performs object detection on the provided video frame and waits
|
|
1654
|
-
* synchronously for the response. Only use this method when the
|
|
1655
|
-
* ObjectDetector is created with running mode `video`.
|
|
1656
|
-
*
|
|
1657
|
-
* @param videoFrame A video frame to process.
|
|
1658
|
-
* @param timestamp The timestamp of the current frame, in ms.
|
|
1659
|
-
* @param imageProcessingOptions the `ImageProcessingOptions` specifying how
|
|
1660
|
-
* to process the input image before running inference.
|
|
1661
|
-
* @return A result containing a list of detected objects.
|
|
1662
|
-
*/
|
|
1663
|
-
detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): DetectionResult;
|
|
1664
|
-
}
|
|
1665
|
-
|
|
1666
|
-
/** Options to configure the MediaPipe Object Detector Task */
|
|
1667
|
-
export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
|
|
1668
|
-
}
|
|
1669
|
-
|
|
1670
|
-
/**
|
|
1671
|
-
* Defines a rectangle, used e.g. as part of detection results or as input
|
|
1672
|
-
* region-of-interest.
|
|
1673
|
-
*
|
|
1674
|
-
* The coordinates are normalized with respect to the image dimensions, i.e.
|
|
1675
|
-
* generally in [0,1] but they may exceed these bounds if describing a region
|
|
1676
|
-
* overlapping the image. The origin is on the top-left corner of the image.
|
|
1677
|
-
*/
|
|
1678
|
-
declare interface RectF {
|
|
1679
|
-
left: number;
|
|
1680
|
-
top: number;
|
|
1681
|
-
right: number;
|
|
1682
|
-
bottom: number;
|
|
1683
|
-
}
|
|
1684
|
-
|
|
1685
|
-
/** A Region-Of-Interest (ROI) to represent a region within an image. */
|
|
1686
|
-
export declare interface RegionOfInterest {
|
|
1687
|
-
/** The ROI in keypoint format. */
|
|
1688
|
-
keypoint: NormalizedKeypoint;
|
|
1689
|
-
}
|
|
1690
|
-
|
|
1691
|
-
/**
|
|
1692
|
-
* The two running modes of a vision task.
|
|
1693
|
-
* 1) The image mode for processing single image inputs.
|
|
1694
|
-
* 2) The video mode for processing decoded frames of a video.
|
|
1695
|
-
*/
|
|
1696
|
-
declare type RunningMode = "IMAGE" | "VIDEO";
|
|
1697
|
-
|
|
1698
|
-
/**
|
|
1699
|
-
* The segmentation tasks return the segmentation either as a WebGLTexture (when
|
|
1700
|
-
* the output is on GPU) or as a typed JavaScript arrays for CPU-based
|
|
1701
|
-
* category or confidence masks. `Uint8ClampedArray`s are used to represent
|
|
1702
|
-
* CPU-based category masks and `Float32Array`s are used for CPU-based
|
|
1703
|
-
* confidence masks.
|
|
1704
|
-
*/
|
|
1705
|
-
export declare type SegmentationMask = Uint8ClampedArray | Float32Array | WebGLTexture;
|
|
1706
|
-
|
|
1707
|
-
/** Base class for all MediaPipe Tasks. */
|
|
1708
|
-
declare abstract class TaskRunner {
|
|
1709
|
-
protected constructor();
|
|
1710
|
-
/** Configures the task with custom options. */
|
|
1711
|
-
abstract setOptions(options: TaskRunnerOptions): Promise<void>;
|
|
1712
|
-
}
|
|
1713
|
-
|
|
1714
|
-
/** Options to configure MediaPipe Tasks in general. */
|
|
1715
|
-
declare interface TaskRunnerOptions {
|
|
1716
|
-
/** Options to configure the loading of the model assets. */
|
|
1717
|
-
baseOptions?: BaseOptions_2;
|
|
1718
|
-
}
|
|
1719
|
-
|
|
1720
|
-
/** The options for configuring a MediaPipe vision task. */
|
|
1721
|
-
declare interface VisionTaskOptions extends TaskRunnerOptions {
|
|
1722
|
-
/**
|
|
1723
|
-
* The canvas element to bind textures to. This has to be set for GPU
|
|
1724
|
-
* processing. The task will initialize a WebGL context and throw an error if
|
|
1725
|
-
* this fails (e.g. if you have already initialized a different type of
|
|
1726
|
-
* context).
|
|
1727
|
-
*/
|
|
1728
|
-
canvas?: HTMLCanvasElement | OffscreenCanvas;
|
|
1729
|
-
/**
|
|
1730
|
-
* The running mode of the task. Default to the image mode.
|
|
1731
|
-
* Vision tasks have two running modes:
|
|
1732
|
-
* 1) The image mode for processing single image inputs.
|
|
1733
|
-
* 2) The video mode for processing decoded frames of a video.
|
|
1734
|
-
*/
|
|
1735
|
-
runningMode?: RunningMode;
|
|
1736
|
-
}
|
|
1737
|
-
|
|
1738
|
-
/** Base class for all MediaPipe Vision Tasks. */
|
|
1739
|
-
declare abstract class VisionTaskRunner extends TaskRunner {
|
|
1740
|
-
protected constructor();
|
|
1741
|
-
/** Configures the shared options of a vision task. */
|
|
1742
|
-
applyOptions(options: VisionTaskOptions): Promise<void>;
|
|
1743
|
-
}
|
|
1744
|
-
|
|
1745
|
-
/**
|
|
1746
|
-
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
|
1747
|
-
*
|
|
1748
|
-
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
1749
|
-
* you may not use this file except in compliance with the License.
|
|
1750
|
-
* You may obtain a copy of the License at
|
|
1751
|
-
*
|
|
1752
|
-
* http://www.apache.org/licenses/LICENSE-2.0
|
|
1753
|
-
*
|
|
1754
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
1755
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
1756
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
1757
|
-
* See the License for the specific language governing permissions and
|
|
1758
|
-
* limitations under the License.
|
|
1759
|
-
*/
|
|
1760
|
-
/** An object containing the locations of the Wasm assets */
|
|
1761
|
-
declare interface WasmFileset {
|
|
1762
|
-
/** The path to the Wasm loader script. */
|
|
1763
|
-
wasmLoaderPath: string;
|
|
1764
|
-
/** The path to the Wasm binary. */
|
|
1765
|
-
wasmBinaryPath: string;
|
|
1766
|
-
}
|
|
1767
|
-
|
|
1768
|
-
export { }
|