@mediapipe/tasks-vision 0.1.0-alpha-1 → 0.1.0-alpha-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/vision.d.ts CHANGED
@@ -1,49 +1,3 @@
1
- /// <reference types="@types/offscreencanvas" />
2
-
3
- import * as jspb from 'google-protobuf';
4
-
5
- declare class Acceleration extends jspb.Message {
6
- hasXnnpack(): boolean;
7
- clearXnnpack(): void;
8
- getXnnpack(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack | undefined;
9
- setXnnpack(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack): void;
10
-
11
- hasGpu(): boolean;
12
- clearGpu(): void;
13
- getGpu(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu | undefined;
14
- setGpu(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu): void;
15
-
16
- hasTflite(): boolean;
17
- clearTflite(): void;
18
- getTflite(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite | undefined;
19
- setTflite(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite): void;
20
-
21
- getDelegateCase(): Acceleration.DelegateCase;
22
- serializeBinary(): Uint8Array;
23
- toObject(includeInstance?: boolean): Acceleration.AsObject;
24
- static toObject(includeInstance: boolean, msg: Acceleration): Acceleration.AsObject;
25
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
26
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
27
- static serializeBinaryToWriter(message: Acceleration, writer: jspb.BinaryWriter): void;
28
- static deserializeBinary(bytes: Uint8Array): Acceleration;
29
- static deserializeBinaryFromReader(message: Acceleration, reader: jspb.BinaryReader): Acceleration;
30
- }
31
-
32
- declare namespace Acceleration {
33
- type AsObject = {
34
- xnnpack?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack.AsObject,
35
- gpu?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu.AsObject,
36
- tflite?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite.AsObject,
37
- }
38
-
39
- enum DelegateCase {
40
- DELEGATE_NOT_SET = 0,
41
- XNNPACK = 1,
42
- GPU = 2,
43
- TFLITE = 4,
44
- }
45
- }
46
-
47
1
  /**
48
2
  * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
49
3
  *
@@ -60,7 +14,7 @@ declare namespace Acceleration {
60
14
  * limitations under the License.
61
15
  */
62
16
  /** Options to configure MediaPipe model loading and processing. */
63
- declare interface BaseOptions {
17
+ declare interface BaseOptions_2 {
64
18
  /**
65
19
  * The model path to the model asset file. Only one of `modelAssetPath` or
66
20
  * `modelAssetBuffer` can be set.
@@ -72,41 +26,7 @@ declare interface BaseOptions {
72
26
  */
73
27
  modelAssetBuffer?: Uint8Array | undefined;
74
28
  /** Overrides the default backend to use for the provided model. */
75
- delegate?: 'cpu' | 'gpu' | undefined;
76
- }
77
-
78
- declare class BaseOptions_2 extends jspb.Message {
79
- hasModelAsset(): boolean;
80
- clearModelAsset(): void;
81
- getModelAsset(): mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile | undefined;
82
- setModelAsset(value?: mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile): void;
83
-
84
- hasUseStreamMode(): boolean;
85
- clearUseStreamMode(): void;
86
- getUseStreamMode(): boolean | undefined;
87
- setUseStreamMode(value: boolean): void;
88
-
89
- hasAcceleration(): boolean;
90
- clearAcceleration(): void;
91
- getAcceleration(): mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration | undefined;
92
- setAcceleration(value?: mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration): void;
93
-
94
- serializeBinary(): Uint8Array;
95
- toObject(includeInstance?: boolean): BaseOptions_2.AsObject;
96
- static toObject(includeInstance: boolean, msg: BaseOptions_2): BaseOptions_2.AsObject;
97
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
98
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
99
- static serializeBinaryToWriter(message: BaseOptions_2, writer: jspb.BinaryWriter): void;
100
- static deserializeBinary(bytes: Uint8Array): BaseOptions_2;
101
- static deserializeBinaryFromReader(message: BaseOptions_2, reader: jspb.BinaryReader): BaseOptions_2;
102
- }
103
-
104
- declare namespace BaseOptions_2 {
105
- type AsObject = {
106
- modelAsset?: mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile.AsObject,
107
- useStreamMode?: boolean,
108
- acceleration?: mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration.AsObject,
109
- }
29
+ delegate?: "CPU" | "GPU" | undefined;
110
30
  }
111
31
 
112
32
  /** An integer bounding box, axis aligned. */
@@ -304,113 +224,6 @@ export declare interface Embedding {
304
224
  headName: string;
305
225
  }
306
226
 
307
- /** A listener that will be invoked with an absl::StatusCode and message. */
308
- declare type ErrorListener = (code: number, message: string) => void;
309
-
310
- declare class ExternalFile extends jspb.Message {
311
- hasFileContent(): boolean;
312
- clearFileContent(): void;
313
- getFileContent(): Uint8Array | string;
314
- getFileContent_asU8(): Uint8Array;
315
- getFileContent_asB64(): string;
316
- setFileContent(value: Uint8Array | string): void;
317
-
318
- hasFileName(): boolean;
319
- clearFileName(): void;
320
- getFileName(): string | undefined;
321
- setFileName(value: string): void;
322
-
323
- hasFileDescriptorMeta(): boolean;
324
- clearFileDescriptorMeta(): void;
325
- getFileDescriptorMeta(): FileDescriptorMeta | undefined;
326
- setFileDescriptorMeta(value?: FileDescriptorMeta): void;
327
-
328
- hasFilePointerMeta(): boolean;
329
- clearFilePointerMeta(): void;
330
- getFilePointerMeta(): FilePointerMeta | undefined;
331
- setFilePointerMeta(value?: FilePointerMeta): void;
332
-
333
- serializeBinary(): Uint8Array;
334
- toObject(includeInstance?: boolean): ExternalFile.AsObject;
335
- static toObject(includeInstance: boolean, msg: ExternalFile): ExternalFile.AsObject;
336
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
337
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
338
- static serializeBinaryToWriter(message: ExternalFile, writer: jspb.BinaryWriter): void;
339
- static deserializeBinary(bytes: Uint8Array): ExternalFile;
340
- static deserializeBinaryFromReader(message: ExternalFile, reader: jspb.BinaryReader): ExternalFile;
341
- }
342
-
343
- declare namespace ExternalFile {
344
- type AsObject = {
345
- fileContent: Uint8Array | string,
346
- fileName?: string,
347
- fileDescriptorMeta?: FileDescriptorMeta.AsObject,
348
- filePointerMeta?: FilePointerMeta.AsObject,
349
- }
350
- }
351
-
352
- declare class FileDescriptorMeta extends jspb.Message {
353
- hasFd(): boolean;
354
- clearFd(): void;
355
- getFd(): number | undefined;
356
- setFd(value: number): void;
357
-
358
- hasLength(): boolean;
359
- clearLength(): void;
360
- getLength(): number | undefined;
361
- setLength(value: number): void;
362
-
363
- hasOffset(): boolean;
364
- clearOffset(): void;
365
- getOffset(): number | undefined;
366
- setOffset(value: number): void;
367
-
368
- serializeBinary(): Uint8Array;
369
- toObject(includeInstance?: boolean): FileDescriptorMeta.AsObject;
370
- static toObject(includeInstance: boolean, msg: FileDescriptorMeta): FileDescriptorMeta.AsObject;
371
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
372
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
373
- static serializeBinaryToWriter(message: FileDescriptorMeta, writer: jspb.BinaryWriter): void;
374
- static deserializeBinary(bytes: Uint8Array): FileDescriptorMeta;
375
- static deserializeBinaryFromReader(message: FileDescriptorMeta, reader: jspb.BinaryReader): FileDescriptorMeta;
376
- }
377
-
378
- declare namespace FileDescriptorMeta {
379
- type AsObject = {
380
- fd?: number,
381
- length?: number,
382
- offset?: number,
383
- }
384
- }
385
-
386
- declare class FilePointerMeta extends jspb.Message {
387
- hasPointer(): boolean;
388
- clearPointer(): void;
389
- getPointer(): number | undefined;
390
- setPointer(value: number): void;
391
-
392
- hasLength(): boolean;
393
- clearLength(): void;
394
- getLength(): number | undefined;
395
- setLength(value: number): void;
396
-
397
- serializeBinary(): Uint8Array;
398
- toObject(includeInstance?: boolean): FilePointerMeta.AsObject;
399
- static toObject(includeInstance: boolean, msg: FilePointerMeta): FilePointerMeta.AsObject;
400
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
401
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
402
- static serializeBinaryToWriter(message: FilePointerMeta, writer: jspb.BinaryWriter): void;
403
- static deserializeBinary(bytes: Uint8Array): FilePointerMeta;
404
- static deserializeBinaryFromReader(message: FilePointerMeta, reader: jspb.BinaryReader): FilePointerMeta;
405
- }
406
-
407
- declare namespace FilePointerMeta {
408
- type AsObject = {
409
- pointer?: number,
410
- length?: number,
411
- }
412
- }
413
-
414
227
  /**
415
228
  * Resolves the files required for the MediaPipe Task APIs.
416
229
  *
@@ -464,16 +277,7 @@ export declare class FilesetResolver {
464
277
  }
465
278
 
466
279
  /** Performs hand gesture recognition on images. */
467
- export declare class GestureRecognizer extends VisionTaskRunner<GestureRecognizerResult> {
468
- private gestures;
469
- private landmarks;
470
- private worldLandmarks;
471
- private handednesses;
472
- private readonly options;
473
- private readonly handLandmarkerGraphOptions;
474
- private readonly handLandmarksDetectorGraphOptions;
475
- private readonly handDetectorGraphOptions;
476
- private readonly handGestureRecognizerGraphOptions;
280
+ export declare class GestureRecognizer extends VisionTaskRunner {
477
281
  /**
478
282
  * Initializes the Wasm runtime and creates a new gesture recognizer from the
479
283
  * provided options.
@@ -500,9 +304,7 @@ export declare class GestureRecognizer extends VisionTaskRunner<GestureRecognize
500
304
  * @param modelAssetPath The path to the model asset.
501
305
  */
502
306
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<GestureRecognizer>;
503
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
504
- protected get baseOptions(): BaseOptions_2;
505
- protected set baseOptions(proto: BaseOptions_2);
307
+ private constructor();
506
308
  /**
507
309
  * Sets new options for the gesture recognizer.
508
310
  *
@@ -519,9 +321,11 @@ export declare class GestureRecognizer extends VisionTaskRunner<GestureRecognize
519
321
  * GestureRecognizer is created with running mode `image`.
520
322
  *
521
323
  * @param image A single image to process.
324
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
325
+ * to process the input image before running inference.
522
326
  * @return The detected gestures.
523
327
  */
524
- recognize(image: ImageSource): GestureRecognizerResult;
328
+ recognize(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
525
329
  /**
526
330
  * Performs gesture recognition on the provided video frame and waits
527
331
  * synchronously for the response. Only use this method when the
@@ -529,24 +333,11 @@ export declare class GestureRecognizer extends VisionTaskRunner<GestureRecognize
529
333
  *
530
334
  * @param videoFrame A video frame to process.
531
335
  * @param timestamp The timestamp of the current frame, in ms.
336
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
337
+ * to process the input image before running inference.
532
338
  * @return The detected gestures.
533
339
  */
534
- recognizeForVideo(videoFrame: ImageSource, timestamp: number): GestureRecognizerResult;
535
- /** Runs the gesture recognition and blocks on the response. */
536
- protected process(imageSource: ImageSource, timestamp: number): GestureRecognizerResult;
537
- /** Sets the default values for the graph. */
538
- private initDefaults;
539
- /** Converts the proto data to a Category[][] structure. */
540
- private toJsCategories;
541
- /** Converts raw data into a landmark, and adds it to our landmarks list. */
542
- private addJsLandmarks;
543
- /**
544
- * Converts raw data into a landmark, and adds it to our worldLandmarks
545
- * list.
546
- */
547
- private adddJsWorldLandmarks;
548
- /** Updates the MediaPipe graph configuration. */
549
- private refreshGraph;
340
+ recognizeForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): GestureRecognizerResult;
550
341
  }
551
342
 
552
343
  /** Options to configure the MediaPipe Gesture Recognizer Task */
@@ -596,569 +387,16 @@ export declare interface GestureRecognizerResult {
596
387
  worldLandmarks: Landmark[][];
597
388
  /** Handedness of detected hands. */
598
389
  handednesses: Category[][];
599
- /** Recognized hand gestures of detected hands */
600
- gestures: Category[][];
601
- }
602
-
603
- /**
604
- * Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
605
- * as created by wasm_mediapipe_demo BUILD macro), and either render results
606
- * into canvas, or else return the output WebGLTexture. Takes a WebAssembly
607
- * Module (must be instantiated to self.Module).
608
- */
609
- declare class GraphRunner {
610
- readonly wasmModule: WasmModule;
611
- readonly hasMultiStreamSupport: boolean;
612
- autoResizeCanvas: boolean;
613
- audioPtr: number | null;
614
- audioSize: number;
615
- /**
616
- * Creates a new MediaPipe WASM module. Must be called *after* wasm Module has
617
- * initialized. Note that we take control of the GL canvas from here on out,
618
- * and will resize it to fit input.
619
- *
620
- * @param module The underlying Wasm Module to use.
621
- * @param glCanvas The type of the GL canvas to use, or `null` if no GL
622
- * canvas should be initialzed. Initializes an offscreen canvas if not
623
- * provided.
624
- */
625
- constructor(module: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
626
- /**
627
- * Convenience helper to load a MediaPipe graph from a file and pass it to
628
- * setGraph.
629
- * @param graphFile The url of the MediaPipe graph file to load.
630
- */
631
- initializeGraph(graphFile: string): Promise<void>;
632
- /**
633
- * Convenience helper for calling setGraph with a string representing a text
634
- * proto config.
635
- * @param graphConfig The text proto graph config, expected to be a string in
636
- * default JavaScript UTF-16 format.
637
- */
638
- setGraphFromString(graphConfig: string): void;
639
- /**
640
- * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
641
- * over the video stream. Will replace the previously running MediaPipe graph,
642
- * if there is one.
643
- * @param graphData The raw MediaPipe graph data, either in binary
644
- * protobuffer format (.binarypb), or else in raw text format (.pbtxt or
645
- * .textproto).
646
- * @param isBinary This should be set to true if the graph is in
647
- * binary format, and false if it is in human-readable text format.
648
- */
649
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
650
- /**
651
- * Configures the current graph to handle audio processing in a certain way
652
- * for all its audio input streams. Additionally can configure audio headers
653
- * (both input side packets as well as input stream headers), but these
654
- * configurations only take effect if called before the graph is set/started.
655
- * @param numChannels The number of channels of audio input. Only 1
656
- * is supported for now.
657
- * @param numSamples The number of samples that are taken in each
658
- * audio capture.
659
- * @param sampleRate The rate, in Hz, of the sampling.
660
- * @param streamName The optional name of the input stream to additionally
661
- * configure with audio information. This configuration only occurs before
662
- * the graph is set/started. If unset, a default stream name will be used.
663
- * @param headerName The optional name of the header input side packet to
664
- * additionally configure with audio information. This configuration only
665
- * occurs before the graph is set/started. If unset, a default header name
666
- * will be used.
667
- */
668
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string, headerName?: string): void;
669
- /**
670
- * Allows disabling automatic canvas resizing, in case clients want to control
671
- * control this.
672
- * @param resize True will re-enable automatic canvas resizing, while false
673
- * will disable the feature.
674
- */
675
- setAutoResizeCanvas(resize: boolean): void;
676
- /**
677
- * Allows disabling the automatic render-to-screen code, in case clients don't
678
- * need/want this. In particular, this removes the requirement for pipelines
679
- * to have access to GPU resources, as well as the requirement for graphs to
680
- * have "input_frames_gpu" and "output_frames_gpu" streams defined, so pure
681
- * CPU pipelines and non-video pipelines can be created.
682
- * NOTE: This only affects future graph initializations (via setGraph or
683
- * initializeGraph), and does NOT affect the currently running graph, so
684
- * calls to this should be made *before* setGraph/initializeGraph for the
685
- * graph file being targeted.
686
- * @param enabled True will re-enable automatic render-to-screen code and
687
- * cause GPU resources to once again be requested, while false will
688
- * disable the feature.
689
- */
690
- setAutoRenderToScreen(enabled: boolean): void;
691
- /**
692
- * Bind texture to our internal canvas, and upload image source to GPU.
693
- * Returns tuple [width, height] of texture. Intended for internal usage.
694
- */
695
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number): [
696
- number,
697
- number
698
- ];
699
- /**
700
- * Takes the raw data from a JS image source, and sends it to C++ to be
701
- * processed, waiting synchronously for the response. Note that we will resize
702
- * our GL canvas to fit the input, so input size should only change
703
- * infrequently.
704
- * @param imageSource An image source to process.
705
- * @param timestamp The timestamp of the current frame, in ms.
706
- * @return texture? The WebGL texture reference, if one was produced.
707
- */
708
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
709
- /**
710
- * Converts JavaScript string input parameters into C++ c-string pointers.
711
- * See b/204830158 for more details. Intended for internal usage.
712
- */
713
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
714
- /**
715
- * Converts JavaScript string input parameters into C++ c-string pointers.
716
- * See b/204830158 for more details.
717
- */
718
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
719
- /**
720
- * Ensures existence of the simple listeners table and registers the callback.
721
- * Intended for internal usage.
722
- */
723
- setListener<T>(outputStreamName: string, callbackFcn: (data: T) => void): void;
724
- /**
725
- * Ensures existence of the vector listeners table and registers the callback.
726
- * Intended for internal usage.
727
- */
728
- setVectorListener<T>(outputStreamName: string, callbackFcn: (data: T[]) => void): void;
729
- /**
730
- * Attaches a listener that will be invoked when the MediaPipe framework
731
- * returns an error.
732
- */
733
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void;
734
- /**
735
- * Takes the raw data from a JS audio capture array, and sends it to C++ to be
736
- * processed.
737
- * @param audioData An array of raw audio capture data, like
738
- * from a call to getChannelData on an AudioBuffer.
739
- * @param streamName The name of the MediaPipe graph stream to add the audio
740
- * data to.
741
- * @param timestamp The timestamp of the current frame, in ms.
742
- */
743
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
744
- /**
745
- * Takes the raw data from a JS audio capture array, and sends it to C++ to be
746
- * processed, shaping the audioData array into an audio matrix according to
747
- * the numChannels and numSamples parameters.
748
- * @param audioData An array of raw audio capture data, like
749
- * from a call to getChannelData on an AudioBuffer.
750
- * @param numChannels The number of audio channels this data represents. If 0
751
- * is passed, then the value will be taken from the last call to
752
- * configureAudio.
753
- * @param numSamples The number of audio samples captured in this data packet.
754
- * If 0 is passed, then the value will be taken from the last call to
755
- * configureAudio.
756
- * @param streamName The name of the MediaPipe graph stream to add the audio
757
- * data to.
758
- * @param timestamp The timestamp of the current frame, in ms.
759
- */
760
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
761
- /**
762
- * Takes the relevant information from the HTML video or image element, and
763
- * passes it into the WebGL-based graph for processing on the given stream at
764
- * the given timestamp. Can be used for additional auxiliary GpuBuffer input
765
- * streams. Processing will not occur until a blocking call (like
766
- * processVideoGl or finishProcessing) is made. For use with
767
- * 'gl_graph_runner_internal_multi_input'.
768
- * @param imageSource Reference to the video frame we wish to add into our
769
- * graph.
770
- * @param streamName The name of the MediaPipe graph stream to add the frame
771
- * to.
772
- * @param timestamp The timestamp of the input frame, in ms.
773
- */
774
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
775
- /**
776
- * Sends a boolean packet into the specified stream at the given timestamp.
777
- * @param data The boolean data to send.
778
- * @param streamName The name of the graph input stream to send data into.
779
- * @param timestamp The timestamp of the input data, in ms.
780
- */
781
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
782
- /**
783
- * Sends a double packet into the specified stream at the given timestamp.
784
- * @param data The double data to send.
785
- * @param streamName The name of the graph input stream to send data into.
786
- * @param timestamp The timestamp of the input data, in ms.
787
- */
788
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
789
390
  /**
790
- * Sends a float packet into the specified stream at the given timestamp.
791
- * @param data The float data to send.
792
- * @param streamName The name of the graph input stream to send data into.
793
- * @param timestamp The timestamp of the input data, in ms.
391
+ * Recognized hand gestures of detected hands. Note that the index of the
392
+ * gesture is always -1, because the raw indices from multiple gesture
393
+ * classifiers cannot consolidate to a meaningful index.
794
394
  */
795
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
796
- /**
797
- * Sends an integer packet into the specified stream at the given timestamp.
798
- * @param data The integer data to send.
799
- * @param streamName The name of the graph input stream to send data into.
800
- * @param timestamp The timestamp of the input data, in ms.
801
- */
802
- addIntToStream(data: number, streamName: string, timestamp: number): void;
803
- /**
804
- * Sends a string packet into the specified stream at the given timestamp.
805
- * @param data The string data to send.
806
- * @param streamName The name of the graph input stream to send data into.
807
- * @param timestamp The timestamp of the input data, in ms.
808
- */
809
- addStringToStream(data: string, streamName: string, timestamp: number): void;
810
- /**
811
- * Sends a Record<string, string> packet into the specified stream at the
812
- * given timestamp.
813
- * @param data The records to send (will become a
814
- * std::flat_hash_map<std::string, std::string).
815
- * @param streamName The name of the graph input stream to send data into.
816
- * @param timestamp The timestamp of the input data, in ms.
817
- */
818
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
819
- /**
820
- * Sends a serialized protobuffer packet into the specified stream at the
821
- * given timestamp, to be parsed into the specified protobuffer type.
822
- * @param data The binary (serialized) raw protobuffer data.
823
- * @param protoType The C++ namespaced type this protobuffer data corresponds
824
- * to. It will be converted to this type when output as a packet into the
825
- * graph.
826
- * @param streamName The name of the graph input stream to send data into.
827
- * @param timestamp The timestamp of the input data, in ms.
828
- */
829
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
830
- /**
831
- * Attaches a boolean packet to the specified input_side_packet.
832
- * @param data The boolean data to send.
833
- * @param sidePacketName The name of the graph input side packet to send data
834
- * into.
835
- */
836
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
837
- /**
838
- * Attaches a double packet to the specified input_side_packet.
839
- * @param data The double data to send.
840
- * @param sidePacketName The name of the graph input side packet to send data
841
- * into.
842
- */
843
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
844
- /**
845
- * Attaches a float packet to the specified input_side_packet.
846
- * @param data The float data to send.
847
- * @param sidePacketName The name of the graph input side packet to send data
848
- * into.
849
- */
850
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
851
- /**
852
- * Attaches a integer packet to the specified input_side_packet.
853
- * @param data The integer data to send.
854
- * @param sidePacketName The name of the graph input side packet to send data
855
- * into.
856
- */
857
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
858
- /**
859
- * Attaches a string packet to the specified input_side_packet.
860
- * @param data The string data to send.
861
- * @param sidePacketName The name of the graph input side packet to send data
862
- * into.
863
- */
864
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
865
- /**
866
- * Attaches a serialized proto packet to the specified input_side_packet.
867
- * @param data The binary (serialized) raw protobuffer data.
868
- * @param protoType The C++ namespaced type this protobuffer data corresponds
869
- * to. It will be converted to this type for use in the graph.
870
- * @param sidePacketName The name of the graph input side packet to send data
871
- * into.
872
- */
873
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
874
- /**
875
- * Attaches a boolean packet listener to the specified output_stream.
876
- * @param outputStreamName The name of the graph output stream to grab boolean
877
- * data from.
878
- * @param callbackFcn The function that will be called back with the data, as
879
- * it is received. Note that the data is only guaranteed to exist for the
880
- * duration of the callback, and the callback will be called inline, so it
881
- * should not perform overly complicated (or any async) behavior.
882
- */
883
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
884
- /**
885
- * Attaches a bool[] packet listener to the specified output_stream.
886
- * @param outputStreamName The name of the graph output stream to grab
887
- * std::vector<bool> data from.
888
- * @param callbackFcn The function that will be called back with the data, as
889
- * it is received. Note that the data is only guaranteed to exist for the
890
- * duration of the callback, and the callback will be called inline, so it
891
- * should not perform overly complicated (or any async) behavior.
892
- */
893
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
894
- /**
895
- * Attaches an int packet listener to the specified output_stream.
896
- * @param outputStreamName The name of the graph output stream to grab int
897
- * data from.
898
- * @param callbackFcn The function that will be called back with the data, as
899
- * it is received. Note that the data is only guaranteed to exist for the
900
- * duration of the callback, and the callback will be called inline, so it
901
- * should not perform overly complicated (or any async) behavior.
902
- */
903
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
904
- /**
905
- * Attaches an int[] packet listener to the specified output_stream.
906
- * @param outputStreamName The name of the graph output stream to grab
907
- * std::vector<int> data from.
908
- * @param callbackFcn The function that will be called back with the data, as
909
- * it is received. Note that the data is only guaranteed to exist for the
910
- * duration of the callback, and the callback will be called inline, so it
911
- * should not perform overly complicated (or any async) behavior.
912
- */
913
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
914
- /**
915
- * Attaches a double packet listener to the specified output_stream.
916
- * @param outputStreamName The name of the graph output stream to grab double
917
- * data from.
918
- * @param callbackFcn The function that will be called back with the data, as
919
- * it is received. Note that the data is only guaranteed to exist for the
920
- * duration of the callback, and the callback will be called inline, so it
921
- * should not perform overly complicated (or any async) behavior.
922
- */
923
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
924
- /**
925
- * Attaches a double[] packet listener to the specified output_stream.
926
- * @param outputStreamName The name of the graph output stream to grab
927
- * std::vector<double> data from.
928
- * @param callbackFcn The function that will be called back with the data, as
929
- * it is received. Note that the data is only guaranteed to exist for the
930
- * duration of the callback, and the callback will be called inline, so it
931
- * should not perform overly complicated (or any async) behavior.
932
- */
933
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
934
- /**
935
- * Attaches a float packet listener to the specified output_stream.
936
- * @param outputStreamName The name of the graph output stream to grab float
937
- * data from.
938
- * @param callbackFcn The function that will be called back with the data, as
939
- * it is received. Note that the data is only guaranteed to exist for the
940
- * duration of the callback, and the callback will be called inline, so it
941
- * should not perform overly complicated (or any async) behavior.
942
- */
943
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
944
- /**
945
- * Attaches a float[] packet listener to the specified output_stream.
946
- * @param outputStreamName The name of the graph output stream to grab
947
- * std::vector<float> data from.
948
- * @param callbackFcn The function that will be called back with the data, as
949
- * it is received. Note that the data is only guaranteed to exist for the
950
- * duration of the callback, and the callback will be called inline, so it
951
- * should not perform overly complicated (or any async) behavior.
952
- */
953
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
954
- /**
955
- * Attaches a string packet listener to the specified output_stream.
956
- * @param outputStreamName The name of the graph output stream to grab string
957
- * data from.
958
- * @param callbackFcn The function that will be called back with the data, as
959
- * it is received. Note that the data is only guaranteed to exist for the
960
- * duration of the callback, and the callback will be called inline, so it
961
- * should not perform overly complicated (or any async) behavior.
962
- */
963
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
964
- /**
965
- * Attaches a string[] packet listener to the specified output_stream.
966
- * @param outputStreamName The name of the graph output stream to grab
967
- * std::vector<std::string> data from.
968
- * @param callbackFcn The function that will be called back with the data, as
969
- * it is received. Note that the data is only guaranteed to exist for the
970
- * duration of the callback, and the callback will be called inline, so it
971
- * should not perform overly complicated (or any async) behavior.
972
- */
973
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
974
- /**
975
- * Attaches a serialized proto packet listener to the specified output_stream.
976
- * @param outputStreamName The name of the graph output stream to grab binary
977
- * serialized proto data from (in Uint8Array format).
978
- * @param callbackFcn The function that will be called back with the data, as
979
- * it is received. Note that by default the data is only guaranteed to
980
- * exist for the duration of the callback, and the callback will be called
981
- * inline, so it should not perform overly complicated (or any async)
982
- * behavior. If the proto data needs to be able to outlive the call, you
983
- * may set the optional makeDeepCopy parameter to true, or can manually
984
- * deep-copy the data yourself.
985
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
986
- * will override the default memory management behavior and make a deep
987
- * copy of the underlying data, rather than just returning a view into the
988
- * C++-managed memory. At the cost of a data copy, this allows the
989
- * returned data to outlive the callback lifetime (and it will be cleaned
990
- * up automatically by JS garbage collection whenever the user is finished
991
- * with it).
992
- */
993
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean): void;
994
- /**
995
- * Attaches a listener for an array of serialized proto packets to the
996
- * specified output_stream.
997
- * @param outputStreamName The name of the graph output stream to grab a
998
- * vector of binary serialized proto data from (in Uint8Array[] format).
999
- * @param callbackFcn The function that will be called back with the data, as
1000
- * it is received. Note that by default the data is only guaranteed to
1001
- * exist for the duration of the callback, and the callback will be called
1002
- * inline, so it should not perform overly complicated (or any async)
1003
- * behavior. If the proto data needs to be able to outlive the call, you
1004
- * may set the optional makeDeepCopy parameter to true, or can manually
1005
- * deep-copy the data yourself.
1006
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
1007
- * will override the default memory management behavior and make a deep
1008
- * copy of the underlying data, rather than just returning a view into the
1009
- * C++-managed memory. At the cost of a data copy, this allows the
1010
- * returned data to outlive the callback lifetime (and it will be cleaned
1011
- * up automatically by JS garbage collection whenever the user is finished
1012
- * with it).
1013
- */
1014
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean): void;
1015
- /**
1016
- * Attaches an audio packet listener to the specified output_stream, to be
1017
- * given a Float32Array as output.
1018
- * @param outputStreamName The name of the graph output stream to grab audio
1019
- * data from.
1020
- * @param callbackFcn The function that will be called back with the data, as
1021
- * it is received. Note that the data is only guaranteed to exist for the
1022
- * duration of the callback, and the callback will be called inline, so it
1023
- * should not perform overly complicated (or any async) behavior. If the
1024
- * audio data needs to be able to outlive the call, you may set the
1025
- * optional makeDeepCopy parameter to true, or can manually deep-copy the
1026
- * data yourself.
1027
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
1028
- * will override the default memory management behavior and make a deep
1029
- * copy of the underlying data, rather than just returning a view into the
1030
- * C++-managed memory. At the cost of a data copy, this allows the
1031
- * returned data to outlive the callback lifetime (and it will be cleaned
1032
- * up automatically by JS garbage collection whenever the user is finished
1033
- * with it).
1034
- */
1035
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean): void;
1036
- /**
1037
- * Forces all queued-up packets to be pushed through the MediaPipe graph as
1038
- * far as possible, performing all processing until no more processing can be
1039
- * done.
1040
- */
1041
- finishProcessing(): void;
1042
- }
1043
-
1044
- /** An implementation of the GraphRunner that supports image operations */
1045
- declare class GraphRunnerImageLib extends GraphRunnerImageLibType {
395
+ gestures: Category[][];
1046
396
  }
1047
397
 
1048
- declare const GraphRunnerImageLibType: (new (...args: any[]) => {
1049
- registerModelResourcesGraphService(): void;
1050
- readonly wasmModule: WasmModule;
1051
- readonly hasMultiStreamSupport: boolean;
1052
- autoResizeCanvas: boolean;
1053
- audioPtr: number | null;
1054
- audioSize: number;
1055
- initializeGraph(graphFile: string): Promise<void>;
1056
- setGraphFromString(graphConfig: string): void;
1057
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
1058
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string | undefined, headerName?: string | undefined): void;
1059
- setAutoResizeCanvas(resize: boolean): void;
1060
- setAutoRenderToScreen(enabled: boolean): void;
1061
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number | undefined): [number, number];
1062
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
1063
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
1064
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
1065
- setListener<T>(outputStreamName: string, callbackFcn: (data: T) => void): void;
1066
- setVectorListener<T_1>(outputStreamName: string, callbackFcn: (data: T_1[]) => void): void;
1067
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void;
1068
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
1069
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
1070
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1071
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
1072
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
1073
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
1074
- addIntToStream(data: number, streamName: string, timestamp: number): void;
1075
- addStringToStream(data: string, streamName: string, timestamp: number): void;
1076
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
1077
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
1078
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
1079
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
1080
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
1081
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
1082
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
1083
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
1084
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
1085
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
1086
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1087
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1088
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1089
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1090
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1091
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1092
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
1093
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
1094
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean | undefined): void;
1095
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean | undefined): void;
1096
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean | undefined): void;
1097
- finishProcessing(): void;
1098
- }) & (new (...args: any[]) => {
1099
- addGpuBufferAsImageToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1100
- readonly wasmModule: WasmModule;
1101
- readonly hasMultiStreamSupport: boolean;
1102
- autoResizeCanvas: boolean;
1103
- audioPtr: number | null;
1104
- audioSize: number;
1105
- initializeGraph(graphFile: string): Promise<void>;
1106
- setGraphFromString(graphConfig: string): void;
1107
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
1108
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string | undefined, headerName?: string | undefined): void;
1109
- setAutoResizeCanvas(resize: boolean): void;
1110
- setAutoRenderToScreen(enabled: boolean): void;
1111
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number | undefined): [number, number];
1112
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
1113
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
1114
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
1115
- setListener<T_2>(outputStreamName: string, callbackFcn: (data: T_2) => void): void;
1116
- setVectorListener<T_1_1>(outputStreamName: string, callbackFcn: (data: T_1_1[]) => void): void;
1117
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void; /**
1118
- * Creates a new instance of a Mediapipe Task. Determines if SIMD is
1119
- * supported and loads the relevant WASM binary.
1120
- * @return A fully instantiated instance of `T`.
1121
- */
1122
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
1123
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
1124
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1125
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
1126
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
1127
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
1128
- addIntToStream(data: number, streamName: string, timestamp: number): void;
1129
- addStringToStream(data: string, streamName: string, timestamp: number): void;
1130
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
1131
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
1132
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
1133
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
1134
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
1135
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
1136
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
1137
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
1138
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
1139
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
1140
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1141
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1142
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1143
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1144
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1145
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1146
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
1147
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
1148
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean | undefined): void;
1149
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean | undefined): void;
1150
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean | undefined): void;
1151
- finishProcessing(): void;
1152
- }) & typeof GraphRunner;
1153
-
1154
398
  /** Performs hand landmarks detection on images. */
1155
- export declare class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
1156
- private landmarks;
1157
- private worldLandmarks;
1158
- private handednesses;
1159
- private readonly options;
1160
- private readonly handLandmarksDetectorGraphOptions;
1161
- private readonly handDetectorGraphOptions;
399
+ export declare class HandLandmarker extends VisionTaskRunner {
1162
400
  /**
1163
401
  * Initializes the Wasm runtime and creates a new `HandLandmarker` from the
1164
402
  * provided options.
@@ -1185,9 +423,7 @@ export declare class HandLandmarker extends VisionTaskRunner<HandLandmarkerResul
1185
423
  * @param modelAssetPath The path to the model asset.
1186
424
  */
1187
425
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<HandLandmarker>;
1188
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1189
- protected get baseOptions(): BaseOptions_2;
1190
- protected set baseOptions(proto: BaseOptions_2);
426
+ private constructor();
1191
427
  /**
1192
428
  * Sets new options for this `HandLandmarker`.
1193
429
  *
@@ -1204,9 +440,11 @@ export declare class HandLandmarker extends VisionTaskRunner<HandLandmarkerResul
1204
440
  * HandLandmarker is created with running mode `image`.
1205
441
  *
1206
442
  * @param image An image to process.
443
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
444
+ * to process the input image before running inference.
1207
445
  * @return The detected hand landmarks.
1208
446
  */
1209
- detect(image: ImageSource): HandLandmarkerResult;
447
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
1210
448
  /**
1211
449
  * Performs hand landmarks detection on the provided video frame and waits
1212
450
  * synchronously for the response. Only use this method when the
@@ -1214,24 +452,11 @@ export declare class HandLandmarker extends VisionTaskRunner<HandLandmarkerResul
1214
452
  *
1215
453
  * @param videoFrame A video frame to process.
1216
454
  * @param timestamp The timestamp of the current frame, in ms.
455
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
456
+ * to process the input image before running inference.
1217
457
  * @return The detected hand landmarks.
1218
458
  */
1219
- detectForVideo(videoFrame: ImageSource, timestamp: number): HandLandmarkerResult;
1220
- /** Runs the hand landmarker graph and blocks on the response. */
1221
- protected process(imageSource: ImageSource, timestamp: number): HandLandmarkerResult;
1222
- /** Sets the default values for the graph. */
1223
- private initDefaults;
1224
- /** Converts the proto data to a Category[][] structure. */
1225
- private toJsCategories;
1226
- /** Converts raw data into a landmark, and adds it to our landmarks list. */
1227
- private addJsLandmarks;
1228
- /**
1229
- * Converts raw data into a world landmark, and adds it to our worldLandmarks
1230
- * list.
1231
- */
1232
- private adddJsWorldLandmarks;
1233
- /** Updates the MediaPipe graph configuration. */
1234
- private refreshGraph;
459
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): HandLandmarkerResult;
1235
460
  }
1236
461
 
1237
462
  /** Options to configure the MediaPipe HandLandmarker Task */
@@ -1271,9 +496,7 @@ export declare interface HandLandmarkerResult {
1271
496
  }
1272
497
 
1273
498
  /** Performs classification on images. */
1274
- export declare class ImageClassifier extends VisionTaskRunner<ImageClassifierResult> {
1275
- private classificationResult;
1276
- private readonly options;
499
+ export declare class ImageClassifier extends VisionTaskRunner {
1277
500
  /**
1278
501
  * Initializes the Wasm runtime and creates a new image classifier from the
1279
502
  * provided options.
@@ -1300,9 +523,7 @@ export declare class ImageClassifier extends VisionTaskRunner<ImageClassifierRes
1300
523
  * @param modelAssetPath The path to the model asset.
1301
524
  */
1302
525
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageClassifier>;
1303
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1304
- protected get baseOptions(): BaseOptions_2;
1305
- protected set baseOptions(proto: BaseOptions_2);
526
+ private constructor();
1306
527
  /**
1307
528
  * Sets new options for the image classifier.
1308
529
  *
@@ -1319,9 +540,11 @@ export declare class ImageClassifier extends VisionTaskRunner<ImageClassifierRes
1319
540
  * ImageClassifier is created with running mode `image`.
1320
541
  *
1321
542
  * @param image An image to process.
543
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
544
+ * to process the input image before running inference.
1322
545
  * @return The classification result of the image
1323
546
  */
1324
- classify(image: ImageSource): ImageClassifierResult;
547
+ classify(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1325
548
  /**
1326
549
  * Performs image classification on the provided video frame and waits
1327
550
  * synchronously for the response. Only use this method when the
@@ -1329,13 +552,11 @@ export declare class ImageClassifier extends VisionTaskRunner<ImageClassifierRes
1329
552
  *
1330
553
  * @param videoFrame A video frame to process.
1331
554
  * @param timestamp The timestamp of the current frame, in ms.
555
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
556
+ * to process the input image before running inference.
1332
557
  * @return The classification result of the image
1333
558
  */
1334
- classifyForVideo(videoFrame: ImageSource, timestamp: number): ImageClassifierResult;
1335
- /** Runs the image classification graph and blocks on the response. */
1336
- protected process(imageSource: ImageSource, timestamp: number): ImageClassifierResult;
1337
- /** Updates the MediaPipe graph configuration. */
1338
- private refreshGraph;
559
+ classifyForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageClassifierResult;
1339
560
  }
1340
561
 
1341
562
  /** Options to configure the MediaPipe Image Classifier Task. */
@@ -1359,9 +580,7 @@ export declare interface ImageClassifierResult {
1359
580
  }
1360
581
 
1361
582
  /** Performs embedding extraction on images. */
1362
- export declare class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult> {
1363
- private readonly options;
1364
- private embeddings;
583
+ export declare class ImageEmbedder extends VisionTaskRunner {
1365
584
  /**
1366
585
  * Initializes the Wasm runtime and creates a new image embedder from the
1367
586
  * provided options.
@@ -1388,9 +607,7 @@ export declare class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult>
1388
607
  * @param modelAssetPath The path to the TFLite model.
1389
608
  */
1390
609
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ImageEmbedder>;
1391
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1392
- protected get baseOptions(): BaseOptions_2;
1393
- protected set baseOptions(proto: BaseOptions_2);
610
+ private constructor();
1394
611
  /**
1395
612
  * Sets new options for the image embedder.
1396
613
  *
@@ -1407,9 +624,11 @@ export declare class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult>
1407
624
  * ImageEmbedder is created with running mode `image`.
1408
625
  *
1409
626
  * @param image The image to process.
627
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
628
+ * to process the input image before running inference.
1410
629
  * @return The classification result of the image
1411
630
  */
1412
- embed(image: ImageSource): ImageEmbedderResult;
631
+ embed(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1413
632
  /**
1414
633
  * Performs embedding extraction on the provided video frame and waits
1415
634
  * synchronously for the response. Only use this method when the
@@ -1417,9 +636,11 @@ export declare class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult>
1417
636
  *
1418
637
  * @param imageFrame The image frame to process.
1419
638
  * @param timestamp The timestamp of the current frame, in ms.
639
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
640
+ * to process the input image before running inference.
1420
641
  * @return The classification result of the image
1421
642
  */
1422
- embedForVideo(imageFrame: ImageSource, timestamp: number): ImageEmbedderResult;
643
+ embedForVideo(imageFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): ImageEmbedderResult;
1423
644
  /**
1424
645
  * Utility function to compute cosine similarity[1] between two `Embedding`
1425
646
  * objects.
@@ -1430,15 +651,6 @@ export declare class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult>
1430
651
  * different sizes, or have an L2-norm of 0.
1431
652
  */
1432
653
  static cosineSimilarity(u: Embedding, v: Embedding): number;
1433
- /** Runs the embedding extraction and blocks on the response. */
1434
- protected process(image: ImageSource, timestamp: number): ImageEmbedderResult;
1435
- /**
1436
- * Internal function for converting raw data into an embedding, and setting it
1437
- * as our embeddings result.
1438
- */
1439
- private addJsImageEmdedding;
1440
- /** Updates the MediaPipe graph configuration. */
1441
- private refreshGraph;
1442
654
  }
1443
655
 
1444
656
  /** Options for configuring a MediaPipe Image Embedder task. */
@@ -1464,259 +676,40 @@ export declare interface ImageEmbedderResult {
1464
676
  }
1465
677
 
1466
678
  /**
1467
- * Valid types of image sources which we can run our GraphRunner over.
679
+ * Options for image processing.
680
+ *
681
+ * If both region-or-interest and rotation are specified, the crop around the
682
+ * region-of-interest is extracted first, then the specified rotation is applied
683
+ * to the crop.
1468
684
  */
1469
- export declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
1470
-
1471
- declare class InferenceCalculatorOptions extends jspb.Message {
1472
- hasModelPath(): boolean;
1473
- clearModelPath(): void;
1474
- getModelPath(): string | undefined;
1475
- setModelPath(value: string): void;
1476
-
1477
- hasUseGpu(): boolean;
1478
- clearUseGpu(): void;
1479
- getUseGpu(): boolean | undefined;
1480
- setUseGpu(value: boolean): void;
1481
-
1482
- hasUseNnapi(): boolean;
1483
- clearUseNnapi(): void;
1484
- getUseNnapi(): boolean | undefined;
1485
- setUseNnapi(value: boolean): void;
1486
-
1487
- hasCpuNumThread(): boolean;
1488
- clearCpuNumThread(): void;
1489
- getCpuNumThread(): number | undefined;
1490
- setCpuNumThread(value: number): void;
1491
-
1492
- hasDelegate(): boolean;
1493
- clearDelegate(): void;
1494
- getDelegate(): InferenceCalculatorOptions.Delegate | undefined;
1495
- setDelegate(value?: InferenceCalculatorOptions.Delegate): void;
1496
-
1497
- serializeBinary(): Uint8Array;
1498
- toObject(includeInstance?: boolean): InferenceCalculatorOptions.AsObject;
1499
- static toObject(includeInstance: boolean, msg: InferenceCalculatorOptions): InferenceCalculatorOptions.AsObject;
1500
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1501
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1502
- static serializeBinaryToWriter(message: InferenceCalculatorOptions, writer: jspb.BinaryWriter): void;
1503
- static deserializeBinary(bytes: Uint8Array): InferenceCalculatorOptions;
1504
- static deserializeBinaryFromReader(message: InferenceCalculatorOptions, reader: jspb.BinaryReader): InferenceCalculatorOptions;
685
+ declare interface ImageProcessingOptions {
686
+ /**
687
+ * The optional region-of-interest to crop from the image. If not specified,
688
+ * the full image is used.
689
+ *
690
+ * Coordinates must be in [0,1] with 'left' < 'right' and 'top' < bottom.
691
+ */
692
+ regionOfInterest?: RectF;
693
+ /**
694
+ * The rotation to apply to the image (or cropped region-of-interest), in
695
+ * degrees clockwise.
696
+ *
697
+ * The rotation must be a multiple (positive or negative) of 90°.
698
+ */
699
+ rotationDegrees?: number;
1505
700
  }
1506
701
 
1507
- declare namespace InferenceCalculatorOptions {
1508
- type AsObject = {
1509
- modelPath?: string,
1510
- useGpu?: boolean,
1511
- useNnapi?: boolean,
1512
- cpuNumThread?: number,
1513
- delegate?: InferenceCalculatorOptions.Delegate.AsObject,
1514
- }
1515
-
1516
- class Delegate extends jspb.Message {
1517
- hasTflite(): boolean;
1518
- clearTflite(): void;
1519
- getTflite(): InferenceCalculatorOptions.Delegate.TfLite | undefined;
1520
- setTflite(value?: InferenceCalculatorOptions.Delegate.TfLite): void;
1521
-
1522
- hasGpu(): boolean;
1523
- clearGpu(): void;
1524
- getGpu(): InferenceCalculatorOptions.Delegate.Gpu | undefined;
1525
- setGpu(value?: InferenceCalculatorOptions.Delegate.Gpu): void;
1526
-
1527
- hasNnapi(): boolean;
1528
- clearNnapi(): void;
1529
- getNnapi(): InferenceCalculatorOptions.Delegate.Nnapi | undefined;
1530
- setNnapi(value?: InferenceCalculatorOptions.Delegate.Nnapi): void;
1531
-
1532
- hasXnnpack(): boolean;
1533
- clearXnnpack(): void;
1534
- getXnnpack(): InferenceCalculatorOptions.Delegate.Xnnpack | undefined;
1535
- setXnnpack(value?: InferenceCalculatorOptions.Delegate.Xnnpack): void;
1536
-
1537
- getDelegateCase(): Delegate.DelegateCase;
1538
- serializeBinary(): Uint8Array;
1539
- toObject(includeInstance?: boolean): Delegate.AsObject;
1540
- static toObject(includeInstance: boolean, msg: Delegate): Delegate.AsObject;
1541
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1542
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1543
- static serializeBinaryToWriter(message: Delegate, writer: jspb.BinaryWriter): void;
1544
- static deserializeBinary(bytes: Uint8Array): Delegate;
1545
- static deserializeBinaryFromReader(message: Delegate, reader: jspb.BinaryReader): Delegate;
1546
- }
1547
-
1548
- namespace Delegate {
1549
- type AsObject = {
1550
- tflite?: InferenceCalculatorOptions.Delegate.TfLite.AsObject,
1551
- gpu?: InferenceCalculatorOptions.Delegate.Gpu.AsObject,
1552
- nnapi?: InferenceCalculatorOptions.Delegate.Nnapi.AsObject,
1553
- xnnpack?: InferenceCalculatorOptions.Delegate.Xnnpack.AsObject,
1554
- }
1555
-
1556
- class TfLite extends jspb.Message {
1557
- serializeBinary(): Uint8Array;
1558
- toObject(includeInstance?: boolean): TfLite.AsObject;
1559
- static toObject(includeInstance: boolean, msg: TfLite): TfLite.AsObject;
1560
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1561
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1562
- static serializeBinaryToWriter(message: TfLite, writer: jspb.BinaryWriter): void;
1563
- static deserializeBinary(bytes: Uint8Array): TfLite;
1564
- static deserializeBinaryFromReader(message: TfLite, reader: jspb.BinaryReader): TfLite;
1565
- }
1566
-
1567
- namespace TfLite {
1568
- type AsObject = {
1569
- }
1570
- }
1571
-
1572
- class Gpu extends jspb.Message {
1573
- hasUseAdvancedGpuApi(): boolean;
1574
- clearUseAdvancedGpuApi(): void;
1575
- getUseAdvancedGpuApi(): boolean | undefined;
1576
- setUseAdvancedGpuApi(value: boolean): void;
1577
-
1578
- hasApi(): boolean;
1579
- clearApi(): void;
1580
- getApi(): InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap] | undefined;
1581
- setApi(value: InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap]): void;
1582
-
1583
- hasAllowPrecisionLoss(): boolean;
1584
- clearAllowPrecisionLoss(): void;
1585
- getAllowPrecisionLoss(): boolean | undefined;
1586
- setAllowPrecisionLoss(value: boolean): void;
1587
-
1588
- hasCachedKernelPath(): boolean;
1589
- clearCachedKernelPath(): void;
1590
- getCachedKernelPath(): string | undefined;
1591
- setCachedKernelPath(value: string): void;
1592
-
1593
- hasSerializedModelDir(): boolean;
1594
- clearSerializedModelDir(): void;
1595
- getSerializedModelDir(): string | undefined;
1596
- setSerializedModelDir(value: string): void;
1597
-
1598
- hasModelToken(): boolean;
1599
- clearModelToken(): void;
1600
- getModelToken(): string | undefined;
1601
- setModelToken(value: string): void;
1602
-
1603
- hasUsage(): boolean;
1604
- clearUsage(): void;
1605
- getUsage(): InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap] | undefined;
1606
- setUsage(value: InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap]): void;
1607
-
1608
- serializeBinary(): Uint8Array;
1609
- toObject(includeInstance?: boolean): Gpu.AsObject;
1610
- static toObject(includeInstance: boolean, msg: Gpu): Gpu.AsObject;
1611
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1612
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1613
- static serializeBinaryToWriter(message: Gpu, writer: jspb.BinaryWriter): void;
1614
- static deserializeBinary(bytes: Uint8Array): Gpu;
1615
- static deserializeBinaryFromReader(message: Gpu, reader: jspb.BinaryReader): Gpu;
1616
- }
1617
-
1618
- namespace Gpu {
1619
- type AsObject = {
1620
- useAdvancedGpuApi?: boolean,
1621
- api?: InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap],
1622
- allowPrecisionLoss?: boolean,
1623
- cachedKernelPath?: string,
1624
- serializedModelDir?: string,
1625
- modelToken?: string,
1626
- usage?: InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap],
1627
- }
1628
-
1629
- interface ApiMap {
1630
- ANY: 0;
1631
- OPENGL: 1;
1632
- OPENCL: 2;
1633
- }
1634
-
1635
- const Api: ApiMap;
1636
-
1637
- interface InferenceUsageMap {
1638
- UNSPECIFIED: 0;
1639
- FAST_SINGLE_ANSWER: 1;
1640
- SUSTAINED_SPEED: 2;
1641
- }
1642
-
1643
- const InferenceUsage: InferenceUsageMap;
1644
- }
1645
-
1646
- class Nnapi extends jspb.Message {
1647
- hasCacheDir(): boolean;
1648
- clearCacheDir(): void;
1649
- getCacheDir(): string | undefined;
1650
- setCacheDir(value: string): void;
1651
-
1652
- hasModelToken(): boolean;
1653
- clearModelToken(): void;
1654
- getModelToken(): string | undefined;
1655
- setModelToken(value: string): void;
1656
-
1657
- hasAcceleratorName(): boolean;
1658
- clearAcceleratorName(): void;
1659
- getAcceleratorName(): string | undefined;
1660
- setAcceleratorName(value: string): void;
1661
-
1662
- serializeBinary(): Uint8Array;
1663
- toObject(includeInstance?: boolean): Nnapi.AsObject;
1664
- static toObject(includeInstance: boolean, msg: Nnapi): Nnapi.AsObject;
1665
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1666
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1667
- static serializeBinaryToWriter(message: Nnapi, writer: jspb.BinaryWriter): void;
1668
- static deserializeBinary(bytes: Uint8Array): Nnapi;
1669
- static deserializeBinaryFromReader(message: Nnapi, reader: jspb.BinaryReader): Nnapi;
1670
- }
1671
-
1672
- namespace Nnapi {
1673
- type AsObject = {
1674
- cacheDir?: string,
1675
- modelToken?: string,
1676
- acceleratorName?: string,
1677
- }
1678
- }
1679
-
1680
- class Xnnpack extends jspb.Message {
1681
- hasNumThreads(): boolean;
1682
- clearNumThreads(): void;
1683
- getNumThreads(): number | undefined;
1684
- setNumThreads(value: number): void;
1685
-
1686
- serializeBinary(): Uint8Array;
1687
- toObject(includeInstance?: boolean): Xnnpack.AsObject;
1688
- static toObject(includeInstance: boolean, msg: Xnnpack): Xnnpack.AsObject;
1689
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1690
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1691
- static serializeBinaryToWriter(message: Xnnpack, writer: jspb.BinaryWriter): void;
1692
- static deserializeBinary(bytes: Uint8Array): Xnnpack;
1693
- static deserializeBinaryFromReader(message: Xnnpack, reader: jspb.BinaryReader): Xnnpack;
1694
- }
1695
-
1696
- namespace Xnnpack {
1697
- type AsObject = {
1698
- numThreads?: number,
1699
- }
1700
- }
1701
-
1702
- enum DelegateCase {
1703
- DELEGATE_NOT_SET = 0,
1704
- TFLITE = 1,
1705
- GPU = 2,
1706
- NNAPI = 3,
1707
- XNNPACK = 4,
1708
- }
1709
- }
1710
-
1711
- const ext: jspb.ExtensionFieldInfo<InferenceCalculatorOptions>;
1712
- }
702
+ /**
703
+ * Valid types of image sources which we can run our GraphRunner over.
704
+ */
705
+ export declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
1713
706
 
1714
707
  /**
1715
708
  * Landmark represents a point in 3D space with x, y, z coordinates. The
1716
709
  * landmark coordinates are in meters. z represents the landmark depth,
1717
710
  * and the smaller the value the closer the world landmark is to the camera.
1718
711
  */
1719
- declare interface Landmark {
712
+ export declare interface Landmark {
1720
713
  /** The x coordinates of the landmark. */
1721
714
  x: number;
1722
715
  /** The y coordinates of the landmark. */
@@ -1725,26 +718,6 @@ declare interface Landmark {
1725
718
  z: number;
1726
719
  }
1727
720
 
1728
- declare namespace mediapipe_calculators_tensor_inference_calculator_pb {
1729
- export {
1730
- InferenceCalculatorOptions
1731
- }
1732
- }
1733
-
1734
- declare namespace mediapipe_tasks_cc_core_proto_acceleration_pb {
1735
- export {
1736
- Acceleration
1737
- }
1738
- }
1739
-
1740
- declare namespace mediapipe_tasks_cc_core_proto_external_file_pb {
1741
- export {
1742
- ExternalFile,
1743
- FileDescriptorMeta,
1744
- FilePointerMeta
1745
- }
1746
- }
1747
-
1748
721
  /**
1749
722
  * Copyright 2022 The MediaPipe Authors. All Rights Reserved.
1750
723
  *
@@ -1767,7 +740,7 @@ declare namespace mediapipe_tasks_cc_core_proto_external_file_pb {
1767
740
  * closer the landmark is to the camera. The magnitude of z uses roughly the
1768
741
  * same scale as x.
1769
742
  */
1770
- declare interface NormalizedLandmark {
743
+ export declare interface NormalizedLandmark {
1771
744
  /** The x coordinates of the normalized landmark. */
1772
745
  x: number;
1773
746
  /** The y coordinates of the normalized landmark. */
@@ -1777,9 +750,7 @@ declare interface NormalizedLandmark {
1777
750
  }
1778
751
 
1779
752
  /** Performs object detection on images. */
1780
- export declare class ObjectDetector extends VisionTaskRunner<Detection[]> {
1781
- private detections;
1782
- private readonly options;
753
+ export declare class ObjectDetector extends VisionTaskRunner {
1783
754
  /**
1784
755
  * Initializes the Wasm runtime and creates a new object detector from the
1785
756
  * provided options.
@@ -1806,9 +777,7 @@ export declare class ObjectDetector extends VisionTaskRunner<Detection[]> {
1806
777
  * @param modelAssetPath The path to the model asset.
1807
778
  */
1808
779
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<ObjectDetector>;
1809
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1810
- protected get baseOptions(): BaseOptions_2;
1811
- protected set baseOptions(proto: BaseOptions_2);
780
+ private constructor();
1812
781
  /**
1813
782
  * Sets new options for the object detector.
1814
783
  *
@@ -1825,77 +794,62 @@ export declare class ObjectDetector extends VisionTaskRunner<Detection[]> {
1825
794
  * ObjectDetector is created with running mode `image`.
1826
795
  *
1827
796
  * @param image An image to process.
797
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
798
+ * to process the input image before running inference.
1828
799
  * @return The list of detected objects
1829
800
  */
1830
- detect(image: ImageSource): Detection[];
801
+ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): Detection[];
1831
802
  /**
1832
- * Performs object detection on the provided vidoe frame and waits
803
+ * Performs object detection on the provided video frame and waits
1833
804
  * synchronously for the response. Only use this method when the
1834
805
  * ObjectDetector is created with running mode `video`.
1835
806
  *
1836
807
  * @param videoFrame A video frame to process.
1837
808
  * @param timestamp The timestamp of the current frame, in ms.
809
+ * @param imageProcessingOptions the `ImageProcessingOptions` specifying how
810
+ * to process the input image before running inference.
1838
811
  * @return The list of detected objects
1839
812
  */
1840
- detectForVideo(videoFrame: ImageSource, timestamp: number): Detection[];
1841
- /** Runs the object detector graph and blocks on the response. */
1842
- protected process(imageSource: ImageSource, timestamp: number): Detection[];
1843
- /** Converts raw data into a Detection, and adds it to our detection list. */
1844
- private addJsObjectDetections;
1845
- /** Updates the MediaPipe graph configuration. */
1846
- private refreshGraph;
813
+ detectForVideo(videoFrame: ImageSource, timestamp: number, imageProcessingOptions?: ImageProcessingOptions): Detection[];
1847
814
  }
1848
815
 
1849
816
  /** Options to configure the MediaPipe Object Detector Task */
1850
817
  export declare interface ObjectDetectorOptions extends VisionTaskOptions, ClassifierOptions {
1851
818
  }
1852
819
 
820
+ /**
821
+ * Defines a rectangle, used e.g. as part of detection results or as input
822
+ * region-of-interest.
823
+ *
824
+ * The coordinates are normalized with respect to the image dimensions, i.e.
825
+ * generally in [0,1] but they may exceed these bounds if describing a region
826
+ * overlapping the image. The origin is on the top-left corner of the image.
827
+ */
828
+ declare interface RectF {
829
+ left: number;
830
+ top: number;
831
+ right: number;
832
+ bottom: number;
833
+ }
834
+
1853
835
  /**
1854
836
  * The two running modes of a vision task.
1855
837
  * 1) The image mode for processing single image inputs.
1856
838
  * 2) The video mode for processing decoded frames of a video.
1857
839
  */
1858
- declare type RunningMode = 'image' | 'video';
840
+ declare type RunningMode = "IMAGE" | "VIDEO";
1859
841
 
1860
842
  /** Base class for all MediaPipe Tasks. */
1861
- declare abstract class TaskRunner<O extends TaskRunnerOptions> {
1862
- protected abstract baseOptions: BaseOptions_2;
1863
- protected graphRunner: GraphRunnerImageLib;
1864
- private processingErrors;
1865
- /**
1866
- * Creates a new instance of a Mediapipe Task. Determines if SIMD is
1867
- * supported and loads the relevant WASM binary.
1868
- * @return A fully instantiated instance of `T`.
1869
- */
1870
- protected static createInstance<T extends TaskRunner<O>, O extends TaskRunnerOptions>(type: WasmMediaPipeConstructor<T>, initializeCanvas: boolean, fileset: WasmFileset, options: O): Promise<T>;
1871
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1872
- /** Configures the shared options of a MediaPipe Task. */
1873
- setOptions(options: O): Promise<void>;
1874
- /**
1875
- * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
1876
- * over the video stream. Will replace the previously running MediaPipe graph,
1877
- * if there is one.
1878
- * @param graphData The raw MediaPipe graph data, either in binary
1879
- * protobuffer format (.binarypb), or else in raw text format (.pbtxt or
1880
- * .textproto).
1881
- * @param isBinary This should be set to true if the graph is in
1882
- * binary format, and false if it is in human-readable text format.
1883
- */
1884
- protected setGraph(graphData: Uint8Array, isBinary: boolean): void;
1885
- /**
1886
- * Forces all queued-up packets to be pushed through the MediaPipe graph as
1887
- * far as possible, performing all processing until no more processing can be
1888
- * done.
1889
- */
1890
- protected finishProcessing(): void;
1891
- /** Throws the error from the error listener if an error was raised. */
1892
- private handleErrors;
843
+ declare abstract class TaskRunner {
844
+ protected constructor();
845
+ /** Configures the task with custom options. */
846
+ abstract setOptions(options: TaskRunnerOptions): Promise<void>;
1893
847
  }
1894
848
 
1895
849
  /** Options to configure MediaPipe Tasks in general. */
1896
850
  declare interface TaskRunnerOptions {
1897
851
  /** Options to configure the loading of the model assets. */
1898
- baseOptions?: BaseOptions;
852
+ baseOptions?: BaseOptions_2;
1899
853
  }
1900
854
 
1901
855
  /** The options for configuring a MediaPipe vision task. */
@@ -1910,15 +864,10 @@ declare interface VisionTaskOptions extends TaskRunnerOptions {
1910
864
  }
1911
865
 
1912
866
  /** Base class for all MediaPipe Vision Tasks. */
1913
- declare abstract class VisionTaskRunner<T> extends TaskRunner<VisionTaskOptions> {
867
+ declare abstract class VisionTaskRunner extends TaskRunner {
868
+ protected constructor();
1914
869
  /** Configures the shared options of a vision task. */
1915
- setOptions(options: VisionTaskOptions): Promise<void>;
1916
- /** Sends an image packet to the graph and awaits results. */
1917
- protected abstract process(input: ImageSource, timestamp: number): T;
1918
- /** Sends a single image to the graph and awaits results. */
1919
- protected processImageData(image: ImageSource): T;
1920
- /** Sends a single video frame to the graph and awaits results. */
1921
- protected processVideoData(imageFrame: ImageSource, timestamp: number): T;
870
+ applyOptions(options: VisionTaskOptions): Promise<void>;
1922
871
  }
1923
872
 
1924
873
  /**
@@ -1944,75 +893,4 @@ declare interface WasmFileset {
1944
893
  wasmBinaryPath: string;
1945
894
  }
1946
895
 
1947
- /**
1948
- * Internal type of constructors used for initializing GraphRunner and
1949
- * subclasses.
1950
- */
1951
- declare type WasmMediaPipeConstructor<LibType> = (new (module: WasmModule, canvas?: HTMLCanvasElement | OffscreenCanvas | null) => LibType);
1952
-
1953
- /**
1954
- * Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
1955
- * doesn't break our JS/C++ bridge.
1956
- */
1957
- declare interface WasmModule {
1958
- canvas: HTMLCanvasElement | OffscreenCanvas | null;
1959
- HEAPU8: Uint8Array;
1960
- HEAPU32: Uint32Array;
1961
- HEAPF32: Float32Array;
1962
- HEAPF64: Float64Array;
1963
- errorListener?: ErrorListener;
1964
- _bindTextureToCanvas: () => boolean;
1965
- _changeBinaryGraph: (size: number, dataPtr: number) => void;
1966
- _changeTextGraph: (size: number, dataPtr: number) => void;
1967
- _free: (ptr: number) => void;
1968
- _malloc: (size: number) => number;
1969
- _processFrame: (width: number, height: number, timestamp: number) => void;
1970
- _setAutoRenderToScreen: (enabled: boolean) => void;
1971
- _waitUntilIdle: () => void;
1972
- dataFileDownloads?: {
1973
- [url: string]: {
1974
- loaded: number;
1975
- total: number;
1976
- };
1977
- };
1978
- stringToNewUTF8: (data: string) => number;
1979
- _bindTextureToStream: (streamNamePtr: number) => void;
1980
- _addBoundTextureToStream: (streamNamePtr: number, width: number, height: number, timestamp: number) => void;
1981
- _addBoolToInputStream: (data: boolean, streamNamePtr: number, timestamp: number) => void;
1982
- _addDoubleToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1983
- _addFloatToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1984
- _addIntToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1985
- _addStringToInputStream: (dataPtr: number, streamNamePtr: number, timestamp: number) => void;
1986
- _addFlatHashMapToInputStream: (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, timestamp: number) => void;
1987
- _addProtoToInputStream: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number, timestamp: number) => void;
1988
- _addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
1989
- _addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
1990
- _addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
1991
- _addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
1992
- _addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
1993
- _addProtoToInputSidePacket: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number) => void;
1994
- simpleListeners?: {
1995
- [outputStreamName: string]: (data: unknown) => void;
1996
- };
1997
- vectorListeners?: {
1998
- [outputStreamName: string]: (data: unknown, index: number, length: number) => void;
1999
- };
2000
- _attachBoolListener: (streamNamePtr: number) => void;
2001
- _attachBoolVectorListener: (streamNamePtr: number) => void;
2002
- _attachDoubleListener: (streamNamePtr: number) => void;
2003
- _attachDoubleVectorListener: (streamNamePtr: number) => void;
2004
- _attachFloatListener: (streamNamePtr: number) => void;
2005
- _attachFloatVectorListener: (streamNamePtr: number) => void;
2006
- _attachIntListener: (streamNamePtr: number) => void;
2007
- _attachIntVectorListener: (streamNamePtr: number) => void;
2008
- _attachStringListener: (streamNamePtr: number) => void;
2009
- _attachStringVectorListener: (streamNamePtr: number) => void;
2010
- _attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
2011
- _attachProtoVectorListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
2012
- _attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
2013
- _addAudioToInputStream: (dataPtr: number, numChannels: number, numSamples: number, streamNamePtr: number, timestamp: number) => void;
2014
- _configureAudio: (channels: number, samples: number, sampleRate: number, streamNamePtr: number, headerNamePtr: number) => void;
2015
- _processGl: (frameDataPtr: number) => number;
2016
- }
2017
-
2018
896
  export { }