@mediapipe/tasks-audio 0.1.0-alpha-1 → 0.1.0-alpha-2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/audio.d.ts +10 -1140
  2. package/audio_bundle.js +1 -1
  3. package/package.json +1 -1
package/audio.d.ts CHANGED
@@ -1,53 +1,5 @@
1
- /// <reference types="@types/offscreencanvas" />
2
-
3
- import * as jspb from 'google-protobuf';
4
-
5
- declare class Acceleration extends jspb.Message {
6
- hasXnnpack(): boolean;
7
- clearXnnpack(): void;
8
- getXnnpack(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack | undefined;
9
- setXnnpack(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack): void;
10
-
11
- hasGpu(): boolean;
12
- clearGpu(): void;
13
- getGpu(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu | undefined;
14
- setGpu(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu): void;
15
-
16
- hasTflite(): boolean;
17
- clearTflite(): void;
18
- getTflite(): mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite | undefined;
19
- setTflite(value?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite): void;
20
-
21
- getDelegateCase(): Acceleration.DelegateCase;
22
- serializeBinary(): Uint8Array;
23
- toObject(includeInstance?: boolean): Acceleration.AsObject;
24
- static toObject(includeInstance: boolean, msg: Acceleration): Acceleration.AsObject;
25
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
26
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
27
- static serializeBinaryToWriter(message: Acceleration, writer: jspb.BinaryWriter): void;
28
- static deserializeBinary(bytes: Uint8Array): Acceleration;
29
- static deserializeBinaryFromReader(message: Acceleration, reader: jspb.BinaryReader): Acceleration;
30
- }
31
-
32
- declare namespace Acceleration {
33
- type AsObject = {
34
- xnnpack?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Xnnpack.AsObject,
35
- gpu?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.Gpu.AsObject,
36
- tflite?: mediapipe_calculators_tensor_inference_calculator_pb.InferenceCalculatorOptions.Delegate.TfLite.AsObject,
37
- }
38
-
39
- enum DelegateCase {
40
- DELEGATE_NOT_SET = 0,
41
- XNNPACK = 1,
42
- GPU = 2,
43
- TFLITE = 4,
44
- }
45
- }
46
-
47
1
  /** Performs audio classification. */
48
2
  export declare class AudioClassifier extends AudioTaskRunner<AudioClassifierResult[]> {
49
- private classificationResults;
50
- private readonly options;
51
3
  /**
52
4
  * Initializes the Wasm runtime and creates a new audio classifier from the
53
5
  * provided options.
@@ -74,9 +26,7 @@ export declare class AudioClassifier extends AudioTaskRunner<AudioClassifierResu
74
26
  * @param modelAssetPath The path to the model asset.
75
27
  */
76
28
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<AudioClassifier>;
77
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
78
- protected get baseOptions(): BaseOptions_2;
79
- protected set baseOptions(proto: BaseOptions_2);
29
+ private constructor();
80
30
  /**
81
31
  * Sets new options for the audio classifier.
82
32
  *
@@ -99,15 +49,6 @@ export declare class AudioClassifier extends AudioTaskRunner<AudioClassifierResu
99
49
  * @return The classification result of the audio datas
100
50
  */
101
51
  classify(audioData: Float32Array, sampleRate?: number): AudioClassifierResult[];
102
- /** Sends an audio package to the graph and returns the classifications. */
103
- protected process(audioData: Float32Array, sampleRate: number, timestampMs: number): AudioClassifierResult[];
104
- /**
105
- * Internal function for converting raw data into classification results, and
106
- * adding them to our classfication results list.
107
- **/
108
- private addJsAudioClassificationResults;
109
- /** Updates the MediaPipe graph configuration. */
110
- private refreshGraph;
111
52
  }
112
53
 
113
54
  /** Options to configure the MediaPipe Audio Classifier Task */
@@ -132,8 +73,6 @@ export declare interface AudioClassifierResult {
132
73
 
133
74
  /** Performs embedding extraction on audio. */
134
75
  export declare class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]> {
135
- private embeddingResults;
136
- private readonly options;
137
76
  /**
138
77
  * Initializes the Wasm runtime and creates a new audio embedder from the
139
78
  * provided options.
@@ -160,9 +99,7 @@ export declare class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]
160
99
  * @param modelAssetPath The path to the TFLite model.
161
100
  */
162
101
  static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string): Promise<AudioEmbedder>;
163
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
164
- protected get baseOptions(): BaseOptions_2;
165
- protected set baseOptions(proto: BaseOptions_2);
102
+ private constructor();
166
103
  /**
167
104
  * Sets new options for the audio embedder.
168
105
  *
@@ -195,9 +132,6 @@ export declare class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]
195
132
  * different sizes, or have an L2-norm of 0.
196
133
  */
197
134
  static cosineSimilarity(u: Embedding, v: Embedding): number;
198
- protected process(audioData: Float32Array, sampleRate: number, timestampMs: number): AudioEmbedderResult[];
199
- /** Updates the MediaPipe graph configuration. */
200
- private refreshGraph;
201
135
  }
202
136
 
203
137
  /** Options to configure the MediaPipe Audio Embedder Task */
@@ -223,8 +157,7 @@ export declare interface AudioEmbedderResult {
223
157
  }
224
158
 
225
159
  /** Base class for all MediaPipe Audio Tasks. */
226
- declare abstract class AudioTaskRunner<T> extends TaskRunner<TaskRunnerOptions> {
227
- private defaultSampleRate;
160
+ declare abstract class AudioTaskRunner<T> extends TaskRunner {
228
161
  /**
229
162
  * Sets the sample rate for API calls that omit an explicit sample rate.
230
163
  * `48000` is used as a default if this method is not called.
@@ -232,10 +165,6 @@ declare abstract class AudioTaskRunner<T> extends TaskRunner<TaskRunnerOptions>
232
165
  * @param sampleRate A sample rate (e.g. `44100`).
233
166
  */
234
167
  setDefaultSampleRate(sampleRate: number): void;
235
- /** Sends an audio packet to the graph and awaits results. */
236
- protected abstract process(audioData: Float32Array, sampleRate: number, timestampMs: number): T;
237
- /** Sends a single audio clip to the graph and awaits results. */
238
- protected processAudioClip(audioData: Float32Array, sampleRate?: number): T;
239
168
  }
240
169
 
241
170
  /**
@@ -254,7 +183,7 @@ declare abstract class AudioTaskRunner<T> extends TaskRunner<TaskRunnerOptions>
254
183
  * limitations under the License.
255
184
  */
256
185
  /** Options to configure MediaPipe model loading and processing. */
257
- declare interface BaseOptions {
186
+ declare interface BaseOptions_2 {
258
187
  /**
259
188
  * The model path to the model asset file. Only one of `modelAssetPath` or
260
189
  * `modelAssetBuffer` can be set.
@@ -266,41 +195,7 @@ declare interface BaseOptions {
266
195
  */
267
196
  modelAssetBuffer?: Uint8Array | undefined;
268
197
  /** Overrides the default backend to use for the provided model. */
269
- delegate?: 'cpu' | 'gpu' | undefined;
270
- }
271
-
272
- declare class BaseOptions_2 extends jspb.Message {
273
- hasModelAsset(): boolean;
274
- clearModelAsset(): void;
275
- getModelAsset(): mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile | undefined;
276
- setModelAsset(value?: mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile): void;
277
-
278
- hasUseStreamMode(): boolean;
279
- clearUseStreamMode(): void;
280
- getUseStreamMode(): boolean | undefined;
281
- setUseStreamMode(value: boolean): void;
282
-
283
- hasAcceleration(): boolean;
284
- clearAcceleration(): void;
285
- getAcceleration(): mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration | undefined;
286
- setAcceleration(value?: mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration): void;
287
-
288
- serializeBinary(): Uint8Array;
289
- toObject(includeInstance?: boolean): BaseOptions_2.AsObject;
290
- static toObject(includeInstance: boolean, msg: BaseOptions_2): BaseOptions_2.AsObject;
291
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
292
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
293
- static serializeBinaryToWriter(message: BaseOptions_2, writer: jspb.BinaryWriter): void;
294
- static deserializeBinary(bytes: Uint8Array): BaseOptions_2;
295
- static deserializeBinaryFromReader(message: BaseOptions_2, reader: jspb.BinaryReader): BaseOptions_2;
296
- }
297
-
298
- declare namespace BaseOptions_2 {
299
- type AsObject = {
300
- modelAsset?: mediapipe_tasks_cc_core_proto_external_file_pb.ExternalFile.AsObject,
301
- useStreamMode?: boolean,
302
- acceleration?: mediapipe_tasks_cc_core_proto_acceleration_pb.Acceleration.AsObject,
303
- }
198
+ delegate?: "CPU" | "GPU" | undefined;
304
199
  }
305
200
 
306
201
  /**
@@ -478,113 +373,6 @@ export declare interface Embedding {
478
373
  headName: string;
479
374
  }
480
375
 
481
- /** A listener that will be invoked with an absl::StatusCode and message. */
482
- declare type ErrorListener = (code: number, message: string) => void;
483
-
484
- declare class ExternalFile extends jspb.Message {
485
- hasFileContent(): boolean;
486
- clearFileContent(): void;
487
- getFileContent(): Uint8Array | string;
488
- getFileContent_asU8(): Uint8Array;
489
- getFileContent_asB64(): string;
490
- setFileContent(value: Uint8Array | string): void;
491
-
492
- hasFileName(): boolean;
493
- clearFileName(): void;
494
- getFileName(): string | undefined;
495
- setFileName(value: string): void;
496
-
497
- hasFileDescriptorMeta(): boolean;
498
- clearFileDescriptorMeta(): void;
499
- getFileDescriptorMeta(): FileDescriptorMeta | undefined;
500
- setFileDescriptorMeta(value?: FileDescriptorMeta): void;
501
-
502
- hasFilePointerMeta(): boolean;
503
- clearFilePointerMeta(): void;
504
- getFilePointerMeta(): FilePointerMeta | undefined;
505
- setFilePointerMeta(value?: FilePointerMeta): void;
506
-
507
- serializeBinary(): Uint8Array;
508
- toObject(includeInstance?: boolean): ExternalFile.AsObject;
509
- static toObject(includeInstance: boolean, msg: ExternalFile): ExternalFile.AsObject;
510
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
511
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
512
- static serializeBinaryToWriter(message: ExternalFile, writer: jspb.BinaryWriter): void;
513
- static deserializeBinary(bytes: Uint8Array): ExternalFile;
514
- static deserializeBinaryFromReader(message: ExternalFile, reader: jspb.BinaryReader): ExternalFile;
515
- }
516
-
517
- declare namespace ExternalFile {
518
- type AsObject = {
519
- fileContent: Uint8Array | string,
520
- fileName?: string,
521
- fileDescriptorMeta?: FileDescriptorMeta.AsObject,
522
- filePointerMeta?: FilePointerMeta.AsObject,
523
- }
524
- }
525
-
526
- declare class FileDescriptorMeta extends jspb.Message {
527
- hasFd(): boolean;
528
- clearFd(): void;
529
- getFd(): number | undefined;
530
- setFd(value: number): void;
531
-
532
- hasLength(): boolean;
533
- clearLength(): void;
534
- getLength(): number | undefined;
535
- setLength(value: number): void;
536
-
537
- hasOffset(): boolean;
538
- clearOffset(): void;
539
- getOffset(): number | undefined;
540
- setOffset(value: number): void;
541
-
542
- serializeBinary(): Uint8Array;
543
- toObject(includeInstance?: boolean): FileDescriptorMeta.AsObject;
544
- static toObject(includeInstance: boolean, msg: FileDescriptorMeta): FileDescriptorMeta.AsObject;
545
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
546
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
547
- static serializeBinaryToWriter(message: FileDescriptorMeta, writer: jspb.BinaryWriter): void;
548
- static deserializeBinary(bytes: Uint8Array): FileDescriptorMeta;
549
- static deserializeBinaryFromReader(message: FileDescriptorMeta, reader: jspb.BinaryReader): FileDescriptorMeta;
550
- }
551
-
552
- declare namespace FileDescriptorMeta {
553
- type AsObject = {
554
- fd?: number,
555
- length?: number,
556
- offset?: number,
557
- }
558
- }
559
-
560
- declare class FilePointerMeta extends jspb.Message {
561
- hasPointer(): boolean;
562
- clearPointer(): void;
563
- getPointer(): number | undefined;
564
- setPointer(value: number): void;
565
-
566
- hasLength(): boolean;
567
- clearLength(): void;
568
- getLength(): number | undefined;
569
- setLength(value: number): void;
570
-
571
- serializeBinary(): Uint8Array;
572
- toObject(includeInstance?: boolean): FilePointerMeta.AsObject;
573
- static toObject(includeInstance: boolean, msg: FilePointerMeta): FilePointerMeta.AsObject;
574
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
575
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
576
- static serializeBinaryToWriter(message: FilePointerMeta, writer: jspb.BinaryWriter): void;
577
- static deserializeBinary(bytes: Uint8Array): FilePointerMeta;
578
- static deserializeBinaryFromReader(message: FilePointerMeta, reader: jspb.BinaryReader): FilePointerMeta;
579
- }
580
-
581
- declare namespace FilePointerMeta {
582
- type AsObject = {
583
- pointer?: number,
584
- length?: number,
585
- }
586
- }
587
-
588
376
  /**
589
377
  * Resolves the files required for the MediaPipe Task APIs.
590
378
  *
@@ -637,864 +425,17 @@ export declare class FilesetResolver {
637
425
  static forVisionTasks(basePath?: string): Promise<WasmFileset>;
638
426
  }
639
427
 
640
- /**
641
- * Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
642
- * as created by wasm_mediapipe_demo BUILD macro), and either render results
643
- * into canvas, or else return the output WebGLTexture. Takes a WebAssembly
644
- * Module (must be instantiated to self.Module).
645
- */
646
- declare class GraphRunner {
647
- readonly wasmModule: WasmModule;
648
- readonly hasMultiStreamSupport: boolean;
649
- autoResizeCanvas: boolean;
650
- audioPtr: number | null;
651
- audioSize: number;
652
- /**
653
- * Creates a new MediaPipe WASM module. Must be called *after* wasm Module has
654
- * initialized. Note that we take control of the GL canvas from here on out,
655
- * and will resize it to fit input.
656
- *
657
- * @param module The underlying Wasm Module to use.
658
- * @param glCanvas The type of the GL canvas to use, or `null` if no GL
659
- * canvas should be initialzed. Initializes an offscreen canvas if not
660
- * provided.
661
- */
662
- constructor(module: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
663
- /**
664
- * Convenience helper to load a MediaPipe graph from a file and pass it to
665
- * setGraph.
666
- * @param graphFile The url of the MediaPipe graph file to load.
667
- */
668
- initializeGraph(graphFile: string): Promise<void>;
669
- /**
670
- * Convenience helper for calling setGraph with a string representing a text
671
- * proto config.
672
- * @param graphConfig The text proto graph config, expected to be a string in
673
- * default JavaScript UTF-16 format.
674
- */
675
- setGraphFromString(graphConfig: string): void;
676
- /**
677
- * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
678
- * over the video stream. Will replace the previously running MediaPipe graph,
679
- * if there is one.
680
- * @param graphData The raw MediaPipe graph data, either in binary
681
- * protobuffer format (.binarypb), or else in raw text format (.pbtxt or
682
- * .textproto).
683
- * @param isBinary This should be set to true if the graph is in
684
- * binary format, and false if it is in human-readable text format.
685
- */
686
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
687
- /**
688
- * Configures the current graph to handle audio processing in a certain way
689
- * for all its audio input streams. Additionally can configure audio headers
690
- * (both input side packets as well as input stream headers), but these
691
- * configurations only take effect if called before the graph is set/started.
692
- * @param numChannels The number of channels of audio input. Only 1
693
- * is supported for now.
694
- * @param numSamples The number of samples that are taken in each
695
- * audio capture.
696
- * @param sampleRate The rate, in Hz, of the sampling.
697
- * @param streamName The optional name of the input stream to additionally
698
- * configure with audio information. This configuration only occurs before
699
- * the graph is set/started. If unset, a default stream name will be used.
700
- * @param headerName The optional name of the header input side packet to
701
- * additionally configure with audio information. This configuration only
702
- * occurs before the graph is set/started. If unset, a default header name
703
- * will be used.
704
- */
705
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string, headerName?: string): void;
706
- /**
707
- * Allows disabling automatic canvas resizing, in case clients want to control
708
- * control this.
709
- * @param resize True will re-enable automatic canvas resizing, while false
710
- * will disable the feature.
711
- */
712
- setAutoResizeCanvas(resize: boolean): void;
713
- /**
714
- * Allows disabling the automatic render-to-screen code, in case clients don't
715
- * need/want this. In particular, this removes the requirement for pipelines
716
- * to have access to GPU resources, as well as the requirement for graphs to
717
- * have "input_frames_gpu" and "output_frames_gpu" streams defined, so pure
718
- * CPU pipelines and non-video pipelines can be created.
719
- * NOTE: This only affects future graph initializations (via setGraph or
720
- * initializeGraph), and does NOT affect the currently running graph, so
721
- * calls to this should be made *before* setGraph/initializeGraph for the
722
- * graph file being targeted.
723
- * @param enabled True will re-enable automatic render-to-screen code and
724
- * cause GPU resources to once again be requested, while false will
725
- * disable the feature.
726
- */
727
- setAutoRenderToScreen(enabled: boolean): void;
728
- /**
729
- * Bind texture to our internal canvas, and upload image source to GPU.
730
- * Returns tuple [width, height] of texture. Intended for internal usage.
731
- */
732
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number): [
733
- number,
734
- number
735
- ];
736
- /**
737
- * Takes the raw data from a JS image source, and sends it to C++ to be
738
- * processed, waiting synchronously for the response. Note that we will resize
739
- * our GL canvas to fit the input, so input size should only change
740
- * infrequently.
741
- * @param imageSource An image source to process.
742
- * @param timestamp The timestamp of the current frame, in ms.
743
- * @return texture? The WebGL texture reference, if one was produced.
744
- */
745
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
746
- /**
747
- * Converts JavaScript string input parameters into C++ c-string pointers.
748
- * See b/204830158 for more details. Intended for internal usage.
749
- */
750
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
751
- /**
752
- * Converts JavaScript string input parameters into C++ c-string pointers.
753
- * See b/204830158 for more details.
754
- */
755
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
756
- /**
757
- * Ensures existence of the simple listeners table and registers the callback.
758
- * Intended for internal usage.
759
- */
760
- setListener<T>(outputStreamName: string, callbackFcn: (data: T) => void): void;
761
- /**
762
- * Ensures existence of the vector listeners table and registers the callback.
763
- * Intended for internal usage.
764
- */
765
- setVectorListener<T>(outputStreamName: string, callbackFcn: (data: T[]) => void): void;
766
- /**
767
- * Attaches a listener that will be invoked when the MediaPipe framework
768
- * returns an error.
769
- */
770
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void;
771
- /**
772
- * Takes the raw data from a JS audio capture array, and sends it to C++ to be
773
- * processed.
774
- * @param audioData An array of raw audio capture data, like
775
- * from a call to getChannelData on an AudioBuffer.
776
- * @param streamName The name of the MediaPipe graph stream to add the audio
777
- * data to.
778
- * @param timestamp The timestamp of the current frame, in ms.
779
- */
780
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
781
- /**
782
- * Takes the raw data from a JS audio capture array, and sends it to C++ to be
783
- * processed, shaping the audioData array into an audio matrix according to
784
- * the numChannels and numSamples parameters.
785
- * @param audioData An array of raw audio capture data, like
786
- * from a call to getChannelData on an AudioBuffer.
787
- * @param numChannels The number of audio channels this data represents. If 0
788
- * is passed, then the value will be taken from the last call to
789
- * configureAudio.
790
- * @param numSamples The number of audio samples captured in this data packet.
791
- * If 0 is passed, then the value will be taken from the last call to
792
- * configureAudio.
793
- * @param streamName The name of the MediaPipe graph stream to add the audio
794
- * data to.
795
- * @param timestamp The timestamp of the current frame, in ms.
796
- */
797
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
798
- /**
799
- * Takes the relevant information from the HTML video or image element, and
800
- * passes it into the WebGL-based graph for processing on the given stream at
801
- * the given timestamp. Can be used for additional auxiliary GpuBuffer input
802
- * streams. Processing will not occur until a blocking call (like
803
- * processVideoGl or finishProcessing) is made. For use with
804
- * 'gl_graph_runner_internal_multi_input'.
805
- * @param imageSource Reference to the video frame we wish to add into our
806
- * graph.
807
- * @param streamName The name of the MediaPipe graph stream to add the frame
808
- * to.
809
- * @param timestamp The timestamp of the input frame, in ms.
810
- */
811
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
812
- /**
813
- * Sends a boolean packet into the specified stream at the given timestamp.
814
- * @param data The boolean data to send.
815
- * @param streamName The name of the graph input stream to send data into.
816
- * @param timestamp The timestamp of the input data, in ms.
817
- */
818
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
819
- /**
820
- * Sends a double packet into the specified stream at the given timestamp.
821
- * @param data The double data to send.
822
- * @param streamName The name of the graph input stream to send data into.
823
- * @param timestamp The timestamp of the input data, in ms.
824
- */
825
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
826
- /**
827
- * Sends a float packet into the specified stream at the given timestamp.
828
- * @param data The float data to send.
829
- * @param streamName The name of the graph input stream to send data into.
830
- * @param timestamp The timestamp of the input data, in ms.
831
- */
832
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
833
- /**
834
- * Sends an integer packet into the specified stream at the given timestamp.
835
- * @param data The integer data to send.
836
- * @param streamName The name of the graph input stream to send data into.
837
- * @param timestamp The timestamp of the input data, in ms.
838
- */
839
- addIntToStream(data: number, streamName: string, timestamp: number): void;
840
- /**
841
- * Sends a string packet into the specified stream at the given timestamp.
842
- * @param data The string data to send.
843
- * @param streamName The name of the graph input stream to send data into.
844
- * @param timestamp The timestamp of the input data, in ms.
845
- */
846
- addStringToStream(data: string, streamName: string, timestamp: number): void;
847
- /**
848
- * Sends a Record<string, string> packet into the specified stream at the
849
- * given timestamp.
850
- * @param data The records to send (will become a
851
- * std::flat_hash_map<std::string, std::string).
852
- * @param streamName The name of the graph input stream to send data into.
853
- * @param timestamp The timestamp of the input data, in ms.
854
- */
855
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
856
- /**
857
- * Sends a serialized protobuffer packet into the specified stream at the
858
- * given timestamp, to be parsed into the specified protobuffer type.
859
- * @param data The binary (serialized) raw protobuffer data.
860
- * @param protoType The C++ namespaced type this protobuffer data corresponds
861
- * to. It will be converted to this type when output as a packet into the
862
- * graph.
863
- * @param streamName The name of the graph input stream to send data into.
864
- * @param timestamp The timestamp of the input data, in ms.
865
- */
866
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
867
- /**
868
- * Attaches a boolean packet to the specified input_side_packet.
869
- * @param data The boolean data to send.
870
- * @param sidePacketName The name of the graph input side packet to send data
871
- * into.
872
- */
873
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
874
- /**
875
- * Attaches a double packet to the specified input_side_packet.
876
- * @param data The double data to send.
877
- * @param sidePacketName The name of the graph input side packet to send data
878
- * into.
879
- */
880
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
881
- /**
882
- * Attaches a float packet to the specified input_side_packet.
883
- * @param data The float data to send.
884
- * @param sidePacketName The name of the graph input side packet to send data
885
- * into.
886
- */
887
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
888
- /**
889
- * Attaches a integer packet to the specified input_side_packet.
890
- * @param data The integer data to send.
891
- * @param sidePacketName The name of the graph input side packet to send data
892
- * into.
893
- */
894
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
895
- /**
896
- * Attaches a string packet to the specified input_side_packet.
897
- * @param data The string data to send.
898
- * @param sidePacketName The name of the graph input side packet to send data
899
- * into.
900
- */
901
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
902
- /**
903
- * Attaches a serialized proto packet to the specified input_side_packet.
904
- * @param data The binary (serialized) raw protobuffer data.
905
- * @param protoType The C++ namespaced type this protobuffer data corresponds
906
- * to. It will be converted to this type for use in the graph.
907
- * @param sidePacketName The name of the graph input side packet to send data
908
- * into.
909
- */
910
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
911
- /**
912
- * Attaches a boolean packet listener to the specified output_stream.
913
- * @param outputStreamName The name of the graph output stream to grab boolean
914
- * data from.
915
- * @param callbackFcn The function that will be called back with the data, as
916
- * it is received. Note that the data is only guaranteed to exist for the
917
- * duration of the callback, and the callback will be called inline, so it
918
- * should not perform overly complicated (or any async) behavior.
919
- */
920
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
921
- /**
922
- * Attaches a bool[] packet listener to the specified output_stream.
923
- * @param outputStreamName The name of the graph output stream to grab
924
- * std::vector<bool> data from.
925
- * @param callbackFcn The function that will be called back with the data, as
926
- * it is received. Note that the data is only guaranteed to exist for the
927
- * duration of the callback, and the callback will be called inline, so it
928
- * should not perform overly complicated (or any async) behavior.
929
- */
930
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
931
- /**
932
- * Attaches an int packet listener to the specified output_stream.
933
- * @param outputStreamName The name of the graph output stream to grab int
934
- * data from.
935
- * @param callbackFcn The function that will be called back with the data, as
936
- * it is received. Note that the data is only guaranteed to exist for the
937
- * duration of the callback, and the callback will be called inline, so it
938
- * should not perform overly complicated (or any async) behavior.
939
- */
940
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
941
- /**
942
- * Attaches an int[] packet listener to the specified output_stream.
943
- * @param outputStreamName The name of the graph output stream to grab
944
- * std::vector<int> data from.
945
- * @param callbackFcn The function that will be called back with the data, as
946
- * it is received. Note that the data is only guaranteed to exist for the
947
- * duration of the callback, and the callback will be called inline, so it
948
- * should not perform overly complicated (or any async) behavior.
949
- */
950
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
951
- /**
952
- * Attaches a double packet listener to the specified output_stream.
953
- * @param outputStreamName The name of the graph output stream to grab double
954
- * data from.
955
- * @param callbackFcn The function that will be called back with the data, as
956
- * it is received. Note that the data is only guaranteed to exist for the
957
- * duration of the callback, and the callback will be called inline, so it
958
- * should not perform overly complicated (or any async) behavior.
959
- */
960
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
961
- /**
962
- * Attaches a double[] packet listener to the specified output_stream.
963
- * @param outputStreamName The name of the graph output stream to grab
964
- * std::vector<double> data from.
965
- * @param callbackFcn The function that will be called back with the data, as
966
- * it is received. Note that the data is only guaranteed to exist for the
967
- * duration of the callback, and the callback will be called inline, so it
968
- * should not perform overly complicated (or any async) behavior.
969
- */
970
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
971
- /**
972
- * Attaches a float packet listener to the specified output_stream.
973
- * @param outputStreamName The name of the graph output stream to grab float
974
- * data from.
975
- * @param callbackFcn The function that will be called back with the data, as
976
- * it is received. Note that the data is only guaranteed to exist for the
977
- * duration of the callback, and the callback will be called inline, so it
978
- * should not perform overly complicated (or any async) behavior.
979
- */
980
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
981
- /**
982
- * Attaches a float[] packet listener to the specified output_stream.
983
- * @param outputStreamName The name of the graph output stream to grab
984
- * std::vector<float> data from.
985
- * @param callbackFcn The function that will be called back with the data, as
986
- * it is received. Note that the data is only guaranteed to exist for the
987
- * duration of the callback, and the callback will be called inline, so it
988
- * should not perform overly complicated (or any async) behavior.
989
- */
990
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
991
- /**
992
- * Attaches a string packet listener to the specified output_stream.
993
- * @param outputStreamName The name of the graph output stream to grab string
994
- * data from.
995
- * @param callbackFcn The function that will be called back with the data, as
996
- * it is received. Note that the data is only guaranteed to exist for the
997
- * duration of the callback, and the callback will be called inline, so it
998
- * should not perform overly complicated (or any async) behavior.
999
- */
1000
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
1001
- /**
1002
- * Attaches a string[] packet listener to the specified output_stream.
1003
- * @param outputStreamName The name of the graph output stream to grab
1004
- * std::vector<std::string> data from.
1005
- * @param callbackFcn The function that will be called back with the data, as
1006
- * it is received. Note that the data is only guaranteed to exist for the
1007
- * duration of the callback, and the callback will be called inline, so it
1008
- * should not perform overly complicated (or any async) behavior.
1009
- */
1010
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
1011
- /**
1012
- * Attaches a serialized proto packet listener to the specified output_stream.
1013
- * @param outputStreamName The name of the graph output stream to grab binary
1014
- * serialized proto data from (in Uint8Array format).
1015
- * @param callbackFcn The function that will be called back with the data, as
1016
- * it is received. Note that by default the data is only guaranteed to
1017
- * exist for the duration of the callback, and the callback will be called
1018
- * inline, so it should not perform overly complicated (or any async)
1019
- * behavior. If the proto data needs to be able to outlive the call, you
1020
- * may set the optional makeDeepCopy parameter to true, or can manually
1021
- * deep-copy the data yourself.
1022
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
1023
- * will override the default memory management behavior and make a deep
1024
- * copy of the underlying data, rather than just returning a view into the
1025
- * C++-managed memory. At the cost of a data copy, this allows the
1026
- * returned data to outlive the callback lifetime (and it will be cleaned
1027
- * up automatically by JS garbage collection whenever the user is finished
1028
- * with it).
1029
- */
1030
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean): void;
1031
- /**
1032
- * Attaches a listener for an array of serialized proto packets to the
1033
- * specified output_stream.
1034
- * @param outputStreamName The name of the graph output stream to grab a
1035
- * vector of binary serialized proto data from (in Uint8Array[] format).
1036
- * @param callbackFcn The function that will be called back with the data, as
1037
- * it is received. Note that by default the data is only guaranteed to
1038
- * exist for the duration of the callback, and the callback will be called
1039
- * inline, so it should not perform overly complicated (or any async)
1040
- * behavior. If the proto data needs to be able to outlive the call, you
1041
- * may set the optional makeDeepCopy parameter to true, or can manually
1042
- * deep-copy the data yourself.
1043
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
1044
- * will override the default memory management behavior and make a deep
1045
- * copy of the underlying data, rather than just returning a view into the
1046
- * C++-managed memory. At the cost of a data copy, this allows the
1047
- * returned data to outlive the callback lifetime (and it will be cleaned
1048
- * up automatically by JS garbage collection whenever the user is finished
1049
- * with it).
1050
- */
1051
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean): void;
1052
- /**
1053
- * Attaches an audio packet listener to the specified output_stream, to be
1054
- * given a Float32Array as output.
1055
- * @param outputStreamName The name of the graph output stream to grab audio
1056
- * data from.
1057
- * @param callbackFcn The function that will be called back with the data, as
1058
- * it is received. Note that the data is only guaranteed to exist for the
1059
- * duration of the callback, and the callback will be called inline, so it
1060
- * should not perform overly complicated (or any async) behavior. If the
1061
- * audio data needs to be able to outlive the call, you may set the
1062
- * optional makeDeepCopy parameter to true, or can manually deep-copy the
1063
- * data yourself.
1064
- * @param makeDeepCopy Optional convenience parameter which, if set to true,
1065
- * will override the default memory management behavior and make a deep
1066
- * copy of the underlying data, rather than just returning a view into the
1067
- * C++-managed memory. At the cost of a data copy, this allows the
1068
- * returned data to outlive the callback lifetime (and it will be cleaned
1069
- * up automatically by JS garbage collection whenever the user is finished
1070
- * with it).
1071
- */
1072
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean): void;
1073
- /**
1074
- * Forces all queued-up packets to be pushed through the MediaPipe graph as
1075
- * far as possible, performing all processing until no more processing can be
1076
- * done.
1077
- */
1078
- finishProcessing(): void;
1079
- }
1080
-
1081
- /** An implementation of the GraphRunner that supports image operations */
1082
- declare class GraphRunnerImageLib extends GraphRunnerImageLibType {
1083
- }
1084
-
1085
- declare const GraphRunnerImageLibType: (new (...args: any[]) => {
1086
- registerModelResourcesGraphService(): void;
1087
- readonly wasmModule: WasmModule;
1088
- readonly hasMultiStreamSupport: boolean;
1089
- autoResizeCanvas: boolean;
1090
- audioPtr: number | null;
1091
- audioSize: number;
1092
- initializeGraph(graphFile: string): Promise<void>;
1093
- setGraphFromString(graphConfig: string): void;
1094
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
1095
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string | undefined, headerName?: string | undefined): void;
1096
- setAutoResizeCanvas(resize: boolean): void;
1097
- setAutoRenderToScreen(enabled: boolean): void;
1098
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number | undefined): [number, number];
1099
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
1100
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
1101
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
1102
- setListener<T>(outputStreamName: string, callbackFcn: (data: T) => void): void;
1103
- setVectorListener<T_1>(outputStreamName: string, callbackFcn: (data: T_1[]) => void): void;
1104
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void;
1105
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
1106
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
1107
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1108
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
1109
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
1110
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
1111
- addIntToStream(data: number, streamName: string, timestamp: number): void;
1112
- addStringToStream(data: string, streamName: string, timestamp: number): void;
1113
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
1114
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
1115
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
1116
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
1117
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
1118
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
1119
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
1120
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
1121
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
1122
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
1123
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1124
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1125
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1126
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1127
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1128
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1129
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
1130
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
1131
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean | undefined): void;
1132
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean | undefined): void;
1133
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean | undefined): void;
1134
- finishProcessing(): void;
1135
- }) & (new (...args: any[]) => {
1136
- addGpuBufferAsImageToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1137
- readonly wasmModule: WasmModule;
1138
- readonly hasMultiStreamSupport: boolean;
1139
- autoResizeCanvas: boolean;
1140
- audioPtr: number | null;
1141
- audioSize: number;
1142
- initializeGraph(graphFile: string): Promise<void>;
1143
- setGraphFromString(graphConfig: string): void;
1144
- setGraph(graphData: Uint8Array, isBinary: boolean): void;
1145
- configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string | undefined, headerName?: string | undefined): void;
1146
- setAutoResizeCanvas(resize: boolean): void;
1147
- setAutoRenderToScreen(enabled: boolean): void;
1148
- bindTextureToStream(imageSource: ImageSource, streamNamePtr?: number | undefined): [number, number];
1149
- processGl(imageSource: ImageSource, timestamp: number): WebGLTexture | undefined;
1150
- wrapStringPtr(stringData: string, stringPtrFunc: (ptr: number) => void): void;
1151
- wrapStringPtrPtr(stringData: string[], ptrFunc: (ptr: number) => void): void;
1152
- setListener<T_2>(outputStreamName: string, callbackFcn: (data: T_2) => void): void;
1153
- setVectorListener<T_1_1>(outputStreamName: string, callbackFcn: (data: T_1_1[]) => void): void;
1154
- attachErrorListener(callbackFcn: (code: number, message: string) => void): void; /**
1155
- * Creates a new instance of a Mediapipe Task. Determines if SIMD is
1156
- * supported and loads the relevant WASM binary.
1157
- * @return A fully instantiated instance of `T`.
1158
- */
1159
- addAudioToStream(audioData: Float32Array, streamName: string, timestamp: number): void;
1160
- addAudioToStreamWithShape(audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number): void;
1161
- addGpuBufferToStream(imageSource: ImageSource, streamName: string, timestamp: number): void;
1162
- addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
1163
- addDoubleToStream(data: number, streamName: string, timestamp: number): void;
1164
- addFloatToStream(data: number, streamName: string, timestamp: number): void;
1165
- addIntToStream(data: number, streamName: string, timestamp: number): void;
1166
- addStringToStream(data: string, streamName: string, timestamp: number): void;
1167
- addStringRecordToStream(data: Record<string, string>, streamName: string, timestamp: number): void;
1168
- addProtoToStream(data: Uint8Array, protoType: string, streamName: string, timestamp: number): void;
1169
- addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
1170
- addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
1171
- addFloatToInputSidePacket(data: number, sidePacketName: string): void;
1172
- addIntToInputSidePacket(data: number, sidePacketName: string): void;
1173
- addStringToInputSidePacket(data: string, sidePacketName: string): void;
1174
- addProtoToInputSidePacket(data: Uint8Array, protoType: string, sidePacketName: string): void;
1175
- attachBoolListener(outputStreamName: string, callbackFcn: (data: boolean) => void): void;
1176
- attachBoolVectorListener(outputStreamName: string, callbackFcn: (data: boolean[]) => void): void;
1177
- attachIntListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1178
- attachIntVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1179
- attachDoubleListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1180
- attachDoubleVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1181
- attachFloatListener(outputStreamName: string, callbackFcn: (data: number) => void): void;
1182
- attachFloatVectorListener(outputStreamName: string, callbackFcn: (data: number[]) => void): void;
1183
- attachStringListener(outputStreamName: string, callbackFcn: (data: string) => void): void;
1184
- attachStringVectorListener(outputStreamName: string, callbackFcn: (data: string[]) => void): void;
1185
- attachProtoListener(outputStreamName: string, callbackFcn: (data: Uint8Array) => void, makeDeepCopy?: boolean | undefined): void;
1186
- attachProtoVectorListener(outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, makeDeepCopy?: boolean | undefined): void;
1187
- attachAudioListener(outputStreamName: string, callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean | undefined): void;
1188
- finishProcessing(): void;
1189
- }) & typeof GraphRunner;
1190
-
1191
- /**
1192
- * Valid types of image sources which we can run our GraphRunner over.
1193
- */
1194
- declare type ImageSource = HTMLCanvasElement | HTMLVideoElement | HTMLImageElement | ImageData | ImageBitmap;
1195
-
1196
- declare class InferenceCalculatorOptions extends jspb.Message {
1197
- hasModelPath(): boolean;
1198
- clearModelPath(): void;
1199
- getModelPath(): string | undefined;
1200
- setModelPath(value: string): void;
1201
-
1202
- hasUseGpu(): boolean;
1203
- clearUseGpu(): void;
1204
- getUseGpu(): boolean | undefined;
1205
- setUseGpu(value: boolean): void;
1206
-
1207
- hasUseNnapi(): boolean;
1208
- clearUseNnapi(): void;
1209
- getUseNnapi(): boolean | undefined;
1210
- setUseNnapi(value: boolean): void;
1211
-
1212
- hasCpuNumThread(): boolean;
1213
- clearCpuNumThread(): void;
1214
- getCpuNumThread(): number | undefined;
1215
- setCpuNumThread(value: number): void;
1216
-
1217
- hasDelegate(): boolean;
1218
- clearDelegate(): void;
1219
- getDelegate(): InferenceCalculatorOptions.Delegate | undefined;
1220
- setDelegate(value?: InferenceCalculatorOptions.Delegate): void;
1221
-
1222
- serializeBinary(): Uint8Array;
1223
- toObject(includeInstance?: boolean): InferenceCalculatorOptions.AsObject;
1224
- static toObject(includeInstance: boolean, msg: InferenceCalculatorOptions): InferenceCalculatorOptions.AsObject;
1225
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1226
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1227
- static serializeBinaryToWriter(message: InferenceCalculatorOptions, writer: jspb.BinaryWriter): void;
1228
- static deserializeBinary(bytes: Uint8Array): InferenceCalculatorOptions;
1229
- static deserializeBinaryFromReader(message: InferenceCalculatorOptions, reader: jspb.BinaryReader): InferenceCalculatorOptions;
1230
- }
1231
-
1232
- declare namespace InferenceCalculatorOptions {
1233
- type AsObject = {
1234
- modelPath?: string,
1235
- useGpu?: boolean,
1236
- useNnapi?: boolean,
1237
- cpuNumThread?: number,
1238
- delegate?: InferenceCalculatorOptions.Delegate.AsObject,
1239
- }
1240
-
1241
- class Delegate extends jspb.Message {
1242
- hasTflite(): boolean;
1243
- clearTflite(): void;
1244
- getTflite(): InferenceCalculatorOptions.Delegate.TfLite | undefined;
1245
- setTflite(value?: InferenceCalculatorOptions.Delegate.TfLite): void;
1246
-
1247
- hasGpu(): boolean;
1248
- clearGpu(): void;
1249
- getGpu(): InferenceCalculatorOptions.Delegate.Gpu | undefined;
1250
- setGpu(value?: InferenceCalculatorOptions.Delegate.Gpu): void;
1251
-
1252
- hasNnapi(): boolean;
1253
- clearNnapi(): void;
1254
- getNnapi(): InferenceCalculatorOptions.Delegate.Nnapi | undefined;
1255
- setNnapi(value?: InferenceCalculatorOptions.Delegate.Nnapi): void;
1256
-
1257
- hasXnnpack(): boolean;
1258
- clearXnnpack(): void;
1259
- getXnnpack(): InferenceCalculatorOptions.Delegate.Xnnpack | undefined;
1260
- setXnnpack(value?: InferenceCalculatorOptions.Delegate.Xnnpack): void;
1261
-
1262
- getDelegateCase(): Delegate.DelegateCase;
1263
- serializeBinary(): Uint8Array;
1264
- toObject(includeInstance?: boolean): Delegate.AsObject;
1265
- static toObject(includeInstance: boolean, msg: Delegate): Delegate.AsObject;
1266
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1267
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1268
- static serializeBinaryToWriter(message: Delegate, writer: jspb.BinaryWriter): void;
1269
- static deserializeBinary(bytes: Uint8Array): Delegate;
1270
- static deserializeBinaryFromReader(message: Delegate, reader: jspb.BinaryReader): Delegate;
1271
- }
1272
-
1273
- namespace Delegate {
1274
- type AsObject = {
1275
- tflite?: InferenceCalculatorOptions.Delegate.TfLite.AsObject,
1276
- gpu?: InferenceCalculatorOptions.Delegate.Gpu.AsObject,
1277
- nnapi?: InferenceCalculatorOptions.Delegate.Nnapi.AsObject,
1278
- xnnpack?: InferenceCalculatorOptions.Delegate.Xnnpack.AsObject,
1279
- }
1280
-
1281
- class TfLite extends jspb.Message {
1282
- serializeBinary(): Uint8Array;
1283
- toObject(includeInstance?: boolean): TfLite.AsObject;
1284
- static toObject(includeInstance: boolean, msg: TfLite): TfLite.AsObject;
1285
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1286
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1287
- static serializeBinaryToWriter(message: TfLite, writer: jspb.BinaryWriter): void;
1288
- static deserializeBinary(bytes: Uint8Array): TfLite;
1289
- static deserializeBinaryFromReader(message: TfLite, reader: jspb.BinaryReader): TfLite;
1290
- }
1291
-
1292
- namespace TfLite {
1293
- type AsObject = {
1294
- }
1295
- }
1296
-
1297
- class Gpu extends jspb.Message {
1298
- hasUseAdvancedGpuApi(): boolean;
1299
- clearUseAdvancedGpuApi(): void;
1300
- getUseAdvancedGpuApi(): boolean | undefined;
1301
- setUseAdvancedGpuApi(value: boolean): void;
1302
-
1303
- hasApi(): boolean;
1304
- clearApi(): void;
1305
- getApi(): InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap] | undefined;
1306
- setApi(value: InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap]): void;
1307
-
1308
- hasAllowPrecisionLoss(): boolean;
1309
- clearAllowPrecisionLoss(): void;
1310
- getAllowPrecisionLoss(): boolean | undefined;
1311
- setAllowPrecisionLoss(value: boolean): void;
1312
-
1313
- hasCachedKernelPath(): boolean;
1314
- clearCachedKernelPath(): void;
1315
- getCachedKernelPath(): string | undefined;
1316
- setCachedKernelPath(value: string): void;
1317
-
1318
- hasSerializedModelDir(): boolean;
1319
- clearSerializedModelDir(): void;
1320
- getSerializedModelDir(): string | undefined;
1321
- setSerializedModelDir(value: string): void;
1322
-
1323
- hasModelToken(): boolean;
1324
- clearModelToken(): void;
1325
- getModelToken(): string | undefined;
1326
- setModelToken(value: string): void;
1327
-
1328
- hasUsage(): boolean;
1329
- clearUsage(): void;
1330
- getUsage(): InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap] | undefined;
1331
- setUsage(value: InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap]): void;
1332
-
1333
- serializeBinary(): Uint8Array;
1334
- toObject(includeInstance?: boolean): Gpu.AsObject;
1335
- static toObject(includeInstance: boolean, msg: Gpu): Gpu.AsObject;
1336
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1337
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1338
- static serializeBinaryToWriter(message: Gpu, writer: jspb.BinaryWriter): void;
1339
- static deserializeBinary(bytes: Uint8Array): Gpu;
1340
- static deserializeBinaryFromReader(message: Gpu, reader: jspb.BinaryReader): Gpu;
1341
- }
1342
-
1343
- namespace Gpu {
1344
- type AsObject = {
1345
- useAdvancedGpuApi?: boolean,
1346
- api?: InferenceCalculatorOptions.Delegate.Gpu.ApiMap[keyof InferenceCalculatorOptions.Delegate.Gpu.ApiMap],
1347
- allowPrecisionLoss?: boolean,
1348
- cachedKernelPath?: string,
1349
- serializedModelDir?: string,
1350
- modelToken?: string,
1351
- usage?: InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap[keyof InferenceCalculatorOptions.Delegate.Gpu.InferenceUsageMap],
1352
- }
1353
-
1354
- interface ApiMap {
1355
- ANY: 0;
1356
- OPENGL: 1;
1357
- OPENCL: 2;
1358
- }
1359
-
1360
- const Api: ApiMap;
1361
-
1362
- interface InferenceUsageMap {
1363
- UNSPECIFIED: 0;
1364
- FAST_SINGLE_ANSWER: 1;
1365
- SUSTAINED_SPEED: 2;
1366
- }
1367
-
1368
- const InferenceUsage: InferenceUsageMap;
1369
- }
1370
-
1371
- class Nnapi extends jspb.Message {
1372
- hasCacheDir(): boolean;
1373
- clearCacheDir(): void;
1374
- getCacheDir(): string | undefined;
1375
- setCacheDir(value: string): void;
1376
-
1377
- hasModelToken(): boolean;
1378
- clearModelToken(): void;
1379
- getModelToken(): string | undefined;
1380
- setModelToken(value: string): void;
1381
-
1382
- hasAcceleratorName(): boolean;
1383
- clearAcceleratorName(): void;
1384
- getAcceleratorName(): string | undefined;
1385
- setAcceleratorName(value: string): void;
1386
-
1387
- serializeBinary(): Uint8Array;
1388
- toObject(includeInstance?: boolean): Nnapi.AsObject;
1389
- static toObject(includeInstance: boolean, msg: Nnapi): Nnapi.AsObject;
1390
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1391
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1392
- static serializeBinaryToWriter(message: Nnapi, writer: jspb.BinaryWriter): void;
1393
- static deserializeBinary(bytes: Uint8Array): Nnapi;
1394
- static deserializeBinaryFromReader(message: Nnapi, reader: jspb.BinaryReader): Nnapi;
1395
- }
1396
-
1397
- namespace Nnapi {
1398
- type AsObject = {
1399
- cacheDir?: string,
1400
- modelToken?: string,
1401
- acceleratorName?: string,
1402
- }
1403
- }
1404
-
1405
- class Xnnpack extends jspb.Message {
1406
- hasNumThreads(): boolean;
1407
- clearNumThreads(): void;
1408
- getNumThreads(): number | undefined;
1409
- setNumThreads(value: number): void;
1410
-
1411
- serializeBinary(): Uint8Array;
1412
- toObject(includeInstance?: boolean): Xnnpack.AsObject;
1413
- static toObject(includeInstance: boolean, msg: Xnnpack): Xnnpack.AsObject;
1414
- static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
1415
- static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
1416
- static serializeBinaryToWriter(message: Xnnpack, writer: jspb.BinaryWriter): void;
1417
- static deserializeBinary(bytes: Uint8Array): Xnnpack;
1418
- static deserializeBinaryFromReader(message: Xnnpack, reader: jspb.BinaryReader): Xnnpack;
1419
- }
1420
-
1421
- namespace Xnnpack {
1422
- type AsObject = {
1423
- numThreads?: number,
1424
- }
1425
- }
1426
-
1427
- enum DelegateCase {
1428
- DELEGATE_NOT_SET = 0,
1429
- TFLITE = 1,
1430
- GPU = 2,
1431
- NNAPI = 3,
1432
- XNNPACK = 4,
1433
- }
1434
- }
1435
-
1436
- const ext: jspb.ExtensionFieldInfo<InferenceCalculatorOptions>;
1437
- }
1438
-
1439
- declare namespace mediapipe_calculators_tensor_inference_calculator_pb {
1440
- export {
1441
- InferenceCalculatorOptions
1442
- }
1443
- }
1444
-
1445
- declare namespace mediapipe_tasks_cc_core_proto_acceleration_pb {
1446
- export {
1447
- Acceleration
1448
- }
1449
- }
1450
-
1451
- declare namespace mediapipe_tasks_cc_core_proto_external_file_pb {
1452
- export {
1453
- ExternalFile,
1454
- FileDescriptorMeta,
1455
- FilePointerMeta
1456
- }
1457
- }
1458
-
1459
428
  /** Base class for all MediaPipe Tasks. */
1460
- declare abstract class TaskRunner<O extends TaskRunnerOptions> {
1461
- protected abstract baseOptions: BaseOptions_2;
1462
- protected graphRunner: GraphRunnerImageLib;
1463
- private processingErrors;
1464
- /**
1465
- * Creates a new instance of a Mediapipe Task. Determines if SIMD is
1466
- * supported and loads the relevant WASM binary.
1467
- * @return A fully instantiated instance of `T`.
1468
- */
1469
- protected static createInstance<T extends TaskRunner<O>, O extends TaskRunnerOptions>(type: WasmMediaPipeConstructor<T>, initializeCanvas: boolean, fileset: WasmFileset, options: O): Promise<T>;
1470
- constructor(wasmModule: WasmModule, glCanvas?: HTMLCanvasElement | OffscreenCanvas | null);
1471
- /** Configures the shared options of a MediaPipe Task. */
1472
- setOptions(options: O): Promise<void>;
1473
- /**
1474
- * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
1475
- * over the video stream. Will replace the previously running MediaPipe graph,
1476
- * if there is one.
1477
- * @param graphData The raw MediaPipe graph data, either in binary
1478
- * protobuffer format (.binarypb), or else in raw text format (.pbtxt or
1479
- * .textproto).
1480
- * @param isBinary This should be set to true if the graph is in
1481
- * binary format, and false if it is in human-readable text format.
1482
- */
1483
- protected setGraph(graphData: Uint8Array, isBinary: boolean): void;
1484
- /**
1485
- * Forces all queued-up packets to be pushed through the MediaPipe graph as
1486
- * far as possible, performing all processing until no more processing can be
1487
- * done.
1488
- */
1489
- protected finishProcessing(): void;
1490
- /** Throws the error from the error listener if an error was raised. */
1491
- private handleErrors;
429
+ declare abstract class TaskRunner {
430
+ protected constructor();
431
+ /** Configures the task with custom options. */
432
+ abstract setOptions(options: TaskRunnerOptions): Promise<void>;
1492
433
  }
1493
434
 
1494
435
  /** Options to configure MediaPipe Tasks in general. */
1495
436
  declare interface TaskRunnerOptions {
1496
437
  /** Options to configure the loading of the model assets. */
1497
- baseOptions?: BaseOptions;
438
+ baseOptions?: BaseOptions_2;
1498
439
  }
1499
440
 
1500
441
  /**
@@ -1520,75 +461,4 @@ declare interface WasmFileset {
1520
461
  wasmBinaryPath: string;
1521
462
  }
1522
463
 
1523
- /**
1524
- * Internal type of constructors used for initializing GraphRunner and
1525
- * subclasses.
1526
- */
1527
- declare type WasmMediaPipeConstructor<LibType> = (new (module: WasmModule, canvas?: HTMLCanvasElement | OffscreenCanvas | null) => LibType);
1528
-
1529
- /**
1530
- * Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
1531
- * doesn't break our JS/C++ bridge.
1532
- */
1533
- declare interface WasmModule {
1534
- canvas: HTMLCanvasElement | OffscreenCanvas | null;
1535
- HEAPU8: Uint8Array;
1536
- HEAPU32: Uint32Array;
1537
- HEAPF32: Float32Array;
1538
- HEAPF64: Float64Array;
1539
- errorListener?: ErrorListener;
1540
- _bindTextureToCanvas: () => boolean;
1541
- _changeBinaryGraph: (size: number, dataPtr: number) => void;
1542
- _changeTextGraph: (size: number, dataPtr: number) => void;
1543
- _free: (ptr: number) => void;
1544
- _malloc: (size: number) => number;
1545
- _processFrame: (width: number, height: number, timestamp: number) => void;
1546
- _setAutoRenderToScreen: (enabled: boolean) => void;
1547
- _waitUntilIdle: () => void;
1548
- dataFileDownloads?: {
1549
- [url: string]: {
1550
- loaded: number;
1551
- total: number;
1552
- };
1553
- };
1554
- stringToNewUTF8: (data: string) => number;
1555
- _bindTextureToStream: (streamNamePtr: number) => void;
1556
- _addBoundTextureToStream: (streamNamePtr: number, width: number, height: number, timestamp: number) => void;
1557
- _addBoolToInputStream: (data: boolean, streamNamePtr: number, timestamp: number) => void;
1558
- _addDoubleToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1559
- _addFloatToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1560
- _addIntToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
1561
- _addStringToInputStream: (dataPtr: number, streamNamePtr: number, timestamp: number) => void;
1562
- _addFlatHashMapToInputStream: (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, timestamp: number) => void;
1563
- _addProtoToInputStream: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number, timestamp: number) => void;
1564
- _addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
1565
- _addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
1566
- _addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
1567
- _addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
1568
- _addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
1569
- _addProtoToInputSidePacket: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number) => void;
1570
- simpleListeners?: {
1571
- [outputStreamName: string]: (data: unknown) => void;
1572
- };
1573
- vectorListeners?: {
1574
- [outputStreamName: string]: (data: unknown, index: number, length: number) => void;
1575
- };
1576
- _attachBoolListener: (streamNamePtr: number) => void;
1577
- _attachBoolVectorListener: (streamNamePtr: number) => void;
1578
- _attachDoubleListener: (streamNamePtr: number) => void;
1579
- _attachDoubleVectorListener: (streamNamePtr: number) => void;
1580
- _attachFloatListener: (streamNamePtr: number) => void;
1581
- _attachFloatVectorListener: (streamNamePtr: number) => void;
1582
- _attachIntListener: (streamNamePtr: number) => void;
1583
- _attachIntVectorListener: (streamNamePtr: number) => void;
1584
- _attachStringListener: (streamNamePtr: number) => void;
1585
- _attachStringVectorListener: (streamNamePtr: number) => void;
1586
- _attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
1587
- _attachProtoVectorListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
1588
- _attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
1589
- _addAudioToInputStream: (dataPtr: number, numChannels: number, numSamples: number, streamNamePtr: number, timestamp: number) => void;
1590
- _configureAudio: (channels: number, samples: number, sampleRate: number, streamNamePtr: number, headerNamePtr: number) => void;
1591
- _processGl: (frameDataPtr: number) => number;
1592
- }
1593
-
1594
464
  export { }