@mediapipe/tasks-retrieval 1.1.0-rc.20260929
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +218 -0
- package/NOTICE +32053 -0
- package/README.md +90 -0
- package/package.json +29 -0
- package/retrieval.d.ts +706 -0
- package/retrieval_bundle.cjs +2 -0
- package/retrieval_bundle.cjs.map +1 -0
- package/retrieval_bundle.js +2 -0
- package/retrieval_bundle.js.map +1 -0
- package/retrieval_bundle.mjs +2 -0
- package/retrieval_bundle.mjs.map +1 -0
- package/wasm/retrieval_wasm_internal.js +8628 -0
- package/wasm/retrieval_wasm_internal.wasm +0 -0
- package/wasm/retrieval_wasm_module_internal.js +8624 -0
- package/wasm/retrieval_wasm_module_internal.wasm +0 -0
- package/wasm/retrieval_wasm_nosimd_internal.js +8619 -0
- package/wasm/retrieval_wasm_nosimd_internal.wasm +0 -0
package/retrieval.d.ts
ADDED
|
@@ -0,0 +1,706 @@
|
|
|
1
|
+
/// <reference types="@webgpu/types" />
|
|
2
|
+
|
|
3
|
+
/** Activation data type for model execution. */
|
|
4
|
+
export declare type ActivationDataType = "FLOAT32" | "FLOAT16" | "INT16" | "INT8";
|
|
5
|
+
|
|
6
|
+
/** An audio part in a multimodal content block. */
|
|
7
|
+
export declare interface AudioPart {
|
|
8
|
+
audioData: Float32Array;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Copyright 2026 The MediaPipe Authors.
|
|
13
|
+
*
|
|
14
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
15
|
+
* you may not use this file except in compliance with the License.
|
|
16
|
+
* You may obtain a copy of the License at
|
|
17
|
+
*
|
|
18
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
19
|
+
*
|
|
20
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
21
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
22
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
23
|
+
* See the License for the specific language governing permissions and
|
|
24
|
+
* limitations under the License.
|
|
25
|
+
*/
|
|
26
|
+
/** Chunking mode for document chunking. */
|
|
27
|
+
export declare type ChunkingMode = "CHARACTER" | "WORD";
|
|
28
|
+
|
|
29
|
+
/** A single multimodal content part. */
|
|
30
|
+
export declare type ContentPart = TextPart | ImagePart | AudioPart;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* A default text chunking implementation using native C++ Wasm methods.
|
|
34
|
+
*/
|
|
35
|
+
export declare class DefaultTextChunker implements TextChunker {
|
|
36
|
+
/**
|
|
37
|
+
* Initializes the Wasm runtime and creates a new DefaultTextChunker instance.
|
|
38
|
+
* @export
|
|
39
|
+
*/
|
|
40
|
+
static create(wasmFileset: WasmFileset, chunkSize?: number, chunkOverlap?: number, mode?: ChunkingMode): Promise<DefaultTextChunker>;
|
|
41
|
+
/**
|
|
42
|
+
* Creates a DefaultTextChunker from an already initialized Wasm module.
|
|
43
|
+
* @export
|
|
44
|
+
*/
|
|
45
|
+
static createFromModule(wasmModule: WasmModule, chunkSize?: number, chunkOverlap?: number, mode?: ChunkingMode): DefaultTextChunker;
|
|
46
|
+
private constructor();
|
|
47
|
+
chunk(text: string): string[];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
52
|
+
*
|
|
53
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
54
|
+
* you may not use this file except in compliance with the License.
|
|
55
|
+
* You may obtain a copy of the License at
|
|
56
|
+
*
|
|
57
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
58
|
+
*
|
|
59
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
60
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
61
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
62
|
+
* See the License for the specific language governing permissions and
|
|
63
|
+
* limitations under the License.
|
|
64
|
+
*/
|
|
65
|
+
/**
|
|
66
|
+
* List of embeddings with an optional timestamp.
|
|
67
|
+
*
|
|
68
|
+
* One and only one of the two 'floatEmbedding' and 'quantizedEmbedding' will
|
|
69
|
+
* contain data, based on whether or not the embedder was configured to perform
|
|
70
|
+
* scalar quantization.
|
|
71
|
+
*/
|
|
72
|
+
export declare interface Embedding {
|
|
73
|
+
/**
|
|
74
|
+
* Floating-point embedding. Empty if the embedder was configured to perform
|
|
75
|
+
* scalar-quantization.
|
|
76
|
+
*/
|
|
77
|
+
floatEmbedding?: number[];
|
|
78
|
+
/**
|
|
79
|
+
* Scalar-quantized embedding. Empty if the embedder was not configured to
|
|
80
|
+
* perform scalar quantization.
|
|
81
|
+
*/
|
|
82
|
+
quantizedEmbedding?: Uint8Array;
|
|
83
|
+
/**
|
|
84
|
+
* The index of the classifier head these categories refer to. This is
|
|
85
|
+
* useful for multi-head models.
|
|
86
|
+
*/
|
|
87
|
+
headIndex: number;
|
|
88
|
+
/**
|
|
89
|
+
* The name of the classifier head, which is the corresponding tensor
|
|
90
|
+
* metadata name.
|
|
91
|
+
*/
|
|
92
|
+
headName: string;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Package-neutral interface for generating embeddings across modalities. */
|
|
96
|
+
export declare interface EmbeddingProvider {
|
|
97
|
+
/**
|
|
98
|
+
* Generates a high-dimensional vector embedding for the given content parts.
|
|
99
|
+
* Returns null if the content parts are unsupported by this provider.
|
|
100
|
+
*/
|
|
101
|
+
embedContent(content: readonly ContentPart[]): Promise<Float32Array | number[] | null> | Float32Array | number[] | null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* A listener that receives the current MediaPipe packet timestamp. This is
|
|
106
|
+
* invoked even for empty packet.
|
|
107
|
+
*/
|
|
108
|
+
declare type EmptyPacketListener = (timestamp: number) => void;
|
|
109
|
+
|
|
110
|
+
/** A listener that will be invoked with an absl::StatusCode and message. */
|
|
111
|
+
declare type ErrorListener = (code: number, message: string) => void;
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Resolves the files required for the MediaPipe Task APIs.
|
|
115
|
+
*
|
|
116
|
+
* This class verifies whether SIMD is supported in the current environment and
|
|
117
|
+
* loads the SIMD files only if support is detected. The returned filesets
|
|
118
|
+
* require that the Wasm files are published without renaming. If this is not
|
|
119
|
+
* possible, you can invoke the MediaPipe Tasks APIs using a manually created
|
|
120
|
+
* `WasmFileset`.
|
|
121
|
+
*/
|
|
122
|
+
export declare class FilesetResolver {
|
|
123
|
+
/**
|
|
124
|
+
* Returns whether SIMD is supported in the current environment.
|
|
125
|
+
*
|
|
126
|
+
* If your environment requires custom locations for the MediaPipe Wasm files,
|
|
127
|
+
* you can use `isSimdSupported()` to decide whether to load the SIMD-based
|
|
128
|
+
* assets. Note that for ES6 Modules, SIMD is assumed to be always supported.
|
|
129
|
+
*
|
|
130
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
131
|
+
* @export
|
|
132
|
+
* @return Whether SIMD support was detected in the current environment.
|
|
133
|
+
*/
|
|
134
|
+
static isSimdSupported(useModule?: boolean): Promise<boolean>;
|
|
135
|
+
/**
|
|
136
|
+
* Creates a fileset for the MediaPipe Audio tasks.
|
|
137
|
+
*
|
|
138
|
+
* @export
|
|
139
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
140
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
141
|
+
* loaded from the host's root directory.
|
|
142
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
143
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe Audio
|
|
144
|
+
* tasks.
|
|
145
|
+
*/
|
|
146
|
+
static forAudioTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
147
|
+
/**
|
|
148
|
+
* Creates a fileset for the MediaPipe GenAI tasks.
|
|
149
|
+
*
|
|
150
|
+
* @export
|
|
151
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
152
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
153
|
+
* loaded from the host's root directory.
|
|
154
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
155
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe GenAI
|
|
156
|
+
* tasks.
|
|
157
|
+
*/
|
|
158
|
+
static forGenAiTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
159
|
+
/**
|
|
160
|
+
* Creates a fileset for the MediaPipe Retrieval tasks.
|
|
161
|
+
*
|
|
162
|
+
* @export
|
|
163
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
164
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
165
|
+
* loaded from the host's root directory.
|
|
166
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
167
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe Retrieval
|
|
168
|
+
* tasks.
|
|
169
|
+
*/
|
|
170
|
+
static forRetrievalTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
171
|
+
/**
|
|
172
|
+
* Creates a fileset for the MediaPipe Text tasks.
|
|
173
|
+
*
|
|
174
|
+
* @export
|
|
175
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
176
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
177
|
+
* loaded from the host's root directory.
|
|
178
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
179
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe Text
|
|
180
|
+
* tasks.
|
|
181
|
+
*/
|
|
182
|
+
static forTextTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
183
|
+
/**
|
|
184
|
+
* Creates a fileset for the MediaPipe LiteRT LM Text tasks.
|
|
185
|
+
*
|
|
186
|
+
* @export
|
|
187
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
188
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
189
|
+
* loaded from the host's root directory.
|
|
190
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
191
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe LiteRT LM
|
|
192
|
+
* Text tasks.
|
|
193
|
+
*/
|
|
194
|
+
static forLiteRtLmTextTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
195
|
+
/**
|
|
196
|
+
* Creates a fileset for the MediaPipe Vision tasks.
|
|
197
|
+
*
|
|
198
|
+
* @export
|
|
199
|
+
* @param basePath An optional base path to specify the directory the Wasm
|
|
200
|
+
* files should be loaded from. If not specified, the Wasm files are
|
|
201
|
+
* loaded from the host's root directory.
|
|
202
|
+
* @param useModule Whether to use ES6 Modules for the Wasm files.
|
|
203
|
+
* @return A `WasmFileset` that can be used to initialize MediaPipe Vision
|
|
204
|
+
* tasks.
|
|
205
|
+
*/
|
|
206
|
+
static forVisionTasks(basePath?: string, useModule?: boolean): Promise<WasmFileset>;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** An image part in a multimodal content block. */
|
|
210
|
+
export declare interface ImagePart {
|
|
211
|
+
imageBytes: Uint8Array;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Lightweight, in-memory implementation of VectorStore in TypeScript.
|
|
216
|
+
*/
|
|
217
|
+
export declare class MemoryVectorStore implements VectorStore {
|
|
218
|
+
upsert(records: readonly RetrievalRecord[]): void;
|
|
219
|
+
delete(ids: readonly string[]): void;
|
|
220
|
+
deleteByMetadata(metadataFilter: Record<string, string>): void;
|
|
221
|
+
search(queryEmbedding: Float32Array | readonly number[], topK: number, metadataFilter?: Record<string, string>): RetrievalRecord[];
|
|
222
|
+
get(ids: readonly string[]): RetrievalRecord[];
|
|
223
|
+
getAllRecordIds(): string[];
|
|
224
|
+
deleteAll(): void;
|
|
225
|
+
close(): void;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* Copyright 2026 The MediaPipe Authors.
|
|
230
|
+
*
|
|
231
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
232
|
+
* you may not use this file except in compliance with the License.
|
|
233
|
+
* You may obtain a copy of the License at
|
|
234
|
+
*
|
|
235
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
236
|
+
*
|
|
237
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
238
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
239
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
240
|
+
* See the License for the specific language governing permissions and
|
|
241
|
+
* limitations under the License.
|
|
242
|
+
*/
|
|
243
|
+
/** Options for configuring retrieval queries in SemanticRetriever. */
|
|
244
|
+
export declare interface RetrievalOptions {
|
|
245
|
+
/** The maximum number of results to return. Defaults to 5. */
|
|
246
|
+
readonly limit?: number;
|
|
247
|
+
/** Key-value metadata filter to restrict matching records. */
|
|
248
|
+
readonly metadataFilter?: Record<string, string>;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** Represents a single multi-modal record stored in the vector store. */
|
|
252
|
+
export declare interface RetrievalRecord {
|
|
253
|
+
/** The unique identifier of the record. */
|
|
254
|
+
id: string;
|
|
255
|
+
/** The list of multi-modal parts that make up the record content. */
|
|
256
|
+
content: readonly ContentPart[];
|
|
257
|
+
/** The vector embeddings representing the record. */
|
|
258
|
+
embeddings: Float32Array;
|
|
259
|
+
/** The key-value metadata associated with the record. */
|
|
260
|
+
metadata: Record<string, string>;
|
|
261
|
+
/** The unique identifier of the parent document, if any. */
|
|
262
|
+
parentId?: string;
|
|
263
|
+
/** The identifiers of chunked child records, if any. */
|
|
264
|
+
childIds?: readonly string[];
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/** Represents a retrieved document or multimedia chunk. */
|
|
268
|
+
export declare interface RetrievalResult {
|
|
269
|
+
/** The unique identifier of the retrieved record. */
|
|
270
|
+
readonly id: string;
|
|
271
|
+
/** The list of multi-modal parts in the retrieved record. */
|
|
272
|
+
readonly content: readonly ContentPart[];
|
|
273
|
+
/** The metadata associated with the retrieved record. */
|
|
274
|
+
readonly metadata: Record<string, string>;
|
|
275
|
+
/** The similarity/relevance score of this retrieved result. */
|
|
276
|
+
readonly score: number;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* SemanticRetriever coordinates document chunking, multimodal embedding,
|
|
281
|
+
* and high-dimensional vector search.
|
|
282
|
+
*/
|
|
283
|
+
export declare class SemanticRetriever {
|
|
284
|
+
/**
|
|
285
|
+
* Creates a SemanticRetriever from SemanticRetrieverComponents.
|
|
286
|
+
*
|
|
287
|
+
* @param components The SemanticRetriever components.
|
|
288
|
+
* @return A Promise resolving to the SemanticRetriever instance.
|
|
289
|
+
* @export
|
|
290
|
+
*/
|
|
291
|
+
static createFromComponents(components: SemanticRetrieverComponents): Promise<SemanticRetriever>;
|
|
292
|
+
private constructor();
|
|
293
|
+
/**
|
|
294
|
+
* Inserts a text document into the retriever with automatic chunking.
|
|
295
|
+
* @export
|
|
296
|
+
*/
|
|
297
|
+
insertDocument(id: string, text: string, metadata?: Record<string, string>): Promise<void>;
|
|
298
|
+
/**
|
|
299
|
+
* Inserts multimodal content parts under a single unique ID.
|
|
300
|
+
* @export
|
|
301
|
+
*/
|
|
302
|
+
insertContent(id: string, parts: readonly ContentPart[], metadata?: Record<string, string>): Promise<void>;
|
|
303
|
+
/**
|
|
304
|
+
* Inserts an image into the retriever.
|
|
305
|
+
* @export
|
|
306
|
+
*/
|
|
307
|
+
insertImage(id: string, imageBytes: Uint8Array, metadata?: Record<string, string>): Promise<void>;
|
|
308
|
+
/**
|
|
309
|
+
* Inserts pre-decoded PCM float audio samples into the retriever.
|
|
310
|
+
* @export
|
|
311
|
+
*/
|
|
312
|
+
insertAudio(id: string, audioData: Float32Array, metadata?: Record<string, string>): Promise<void>;
|
|
313
|
+
/**
|
|
314
|
+
* Retrieves top-K nearest matching records based on a text or multimodal query.
|
|
315
|
+
* @export
|
|
316
|
+
*/
|
|
317
|
+
retrieve(query: string | readonly ContentPart[], options?: RetrievalOptions): Promise<RetrievalResult[]>;
|
|
318
|
+
/**
|
|
319
|
+
* Deletes records and their chunks by ID(s), or records matching metadata filter.
|
|
320
|
+
* @export
|
|
321
|
+
*/
|
|
322
|
+
delete(idsOrFilter: string | readonly string[] | Record<string, string>): Promise<void>;
|
|
323
|
+
/**
|
|
324
|
+
* Retrieves all unique top-level record IDs stored in the retriever.
|
|
325
|
+
* @export
|
|
326
|
+
*/
|
|
327
|
+
getAllRecordIds(): Promise<string[]>;
|
|
328
|
+
/**
|
|
329
|
+
* Deletes all records from the retriever.
|
|
330
|
+
* @export
|
|
331
|
+
*/
|
|
332
|
+
deleteAll(): Promise<void>;
|
|
333
|
+
/**
|
|
334
|
+
* Closes the retriever and frees resources.
|
|
335
|
+
* @export
|
|
336
|
+
*/
|
|
337
|
+
close(): void;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Components used to initialize the SemanticRetriever.
|
|
342
|
+
*/
|
|
343
|
+
export declare class SemanticRetrieverComponents {
|
|
344
|
+
/**
|
|
345
|
+
* Adds an embedding provider to the SemanticRetriever.
|
|
346
|
+
*
|
|
347
|
+
* @param provider The embedding provider to add.
|
|
348
|
+
* @return The SemanticRetrieverComponents instance for chaining.
|
|
349
|
+
*/
|
|
350
|
+
addProvider(provider: EmbeddingProvider): this;
|
|
351
|
+
/**
|
|
352
|
+
* Returns the registered embedding providers for the SemanticRetriever.
|
|
353
|
+
*
|
|
354
|
+
* @return A list of registered embedding providers.
|
|
355
|
+
*/
|
|
356
|
+
providers(): readonly EmbeddingProvider[];
|
|
357
|
+
/**
|
|
358
|
+
* Sets the vector store for the SemanticRetriever.
|
|
359
|
+
*
|
|
360
|
+
* @param vectorStore The vector store to set.
|
|
361
|
+
* @return The SemanticRetrieverComponents instance for chaining.
|
|
362
|
+
*/
|
|
363
|
+
setVectorStore(vectorStore: VectorStore): this;
|
|
364
|
+
/**
|
|
365
|
+
* Returns the vector store for the SemanticRetriever.
|
|
366
|
+
*
|
|
367
|
+
* @return The vector store for the SemanticRetriever.
|
|
368
|
+
*/
|
|
369
|
+
vectorStore(): VectorStore | undefined;
|
|
370
|
+
/**
|
|
371
|
+
* Sets the text chunker for the SemanticRetriever.
|
|
372
|
+
*
|
|
373
|
+
* @param textChunker The text chunker to set.
|
|
374
|
+
* @return The SemanticRetrieverComponents instance for chaining.
|
|
375
|
+
*/
|
|
376
|
+
setTextChunker(textChunker: TextChunker): this;
|
|
377
|
+
/**
|
|
378
|
+
* Returns the text chunker for the SemanticRetriever.
|
|
379
|
+
*
|
|
380
|
+
* @return The text chunker for the SemanticRetriever.
|
|
381
|
+
*/
|
|
382
|
+
textChunker(): TextChunker | undefined;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* A listener that receives the contents of a non-empty MediaPipe packet and
|
|
387
|
+
* its timestamp.
|
|
388
|
+
*/
|
|
389
|
+
declare type SimpleListener<T> = (data: T, timestamp: number) => void;
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Interface for chunking long text into smaller segments.
|
|
393
|
+
*/
|
|
394
|
+
export declare interface TextChunker {
|
|
395
|
+
/** Chunks the input text into a list of strings. */
|
|
396
|
+
chunk(text: string): string[] | Promise<string[]>;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Copyright 2026 The MediaPipe Authors.
|
|
401
|
+
*
|
|
402
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
403
|
+
* you may not use this file except in compliance with the License.
|
|
404
|
+
* You may obtain a copy of the License at
|
|
405
|
+
*
|
|
406
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
407
|
+
*
|
|
408
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
409
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
410
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
411
|
+
* See the License for the specific language governing permissions and
|
|
412
|
+
* limitations under the License.
|
|
413
|
+
*/
|
|
414
|
+
/** A text part in a multimodal content block. */
|
|
415
|
+
export declare interface TextPart {
|
|
416
|
+
text: string;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* UniversalEmbedder performs multimodal embedding extraction directly
|
|
421
|
+
* using LiteRT LM.
|
|
422
|
+
*/
|
|
423
|
+
export declare class UniversalEmbedder {
|
|
424
|
+
/**
|
|
425
|
+
* Initializes the Wasm runtime and creates a new UniversalEmbedder instance.
|
|
426
|
+
* @export
|
|
427
|
+
*/
|
|
428
|
+
static createFromOptions(wasmFileset: WasmFileset, options: UniversalEmbedderOptions): Promise<UniversalEmbedder>;
|
|
429
|
+
/**
|
|
430
|
+
* Creates a WebGPU device for GPU-accelerated universal multimodal embedding.
|
|
431
|
+
* @export
|
|
432
|
+
*/
|
|
433
|
+
static createWebGpuDevice(): Promise<GPUDevice>;
|
|
434
|
+
/**
|
|
435
|
+
* Initializes the Wasm runtime and creates a UniversalEmbedder from a model buffer.
|
|
436
|
+
* @export
|
|
437
|
+
*/
|
|
438
|
+
static createFromModelBuffer(wasmFileset: WasmFileset, modelAssetBuffer: Uint8Array | ReadableStreamDefaultReader): Promise<UniversalEmbedder>;
|
|
439
|
+
/**
|
|
440
|
+
* Initializes the Wasm runtime and creates a UniversalEmbedder from a model path.
|
|
441
|
+
* @export
|
|
442
|
+
*/
|
|
443
|
+
static createFromModelPath(wasmFileset: WasmFileset, modelAssetPath: string | string): Promise<UniversalEmbedder>;
|
|
444
|
+
private constructor();
|
|
445
|
+
/**
|
|
446
|
+
* Performs embedding extraction on the input text.
|
|
447
|
+
* @export
|
|
448
|
+
*/
|
|
449
|
+
embedText(text: string): Promise<UniversalEmbedderResult>;
|
|
450
|
+
/**
|
|
451
|
+
* Performs embedding extraction on raw image bytes.
|
|
452
|
+
* @export
|
|
453
|
+
*/
|
|
454
|
+
embedImage(imageBytes: Uint8Array): Promise<UniversalEmbedderResult>;
|
|
455
|
+
/**
|
|
456
|
+
* Performs embedding extraction on audio float samples.
|
|
457
|
+
* @export
|
|
458
|
+
*/
|
|
459
|
+
embedAudio(audioData: Float32Array): Promise<UniversalEmbedderResult>;
|
|
460
|
+
/**
|
|
461
|
+
* Performs embedding extraction on multimodal content parts.
|
|
462
|
+
* @export
|
|
463
|
+
*/
|
|
464
|
+
embedContent(content: readonly ContentPart[]): Promise<UniversalEmbedderResult>;
|
|
465
|
+
/**
|
|
466
|
+
* Computes cosine similarity between two embeddings.
|
|
467
|
+
* @export
|
|
468
|
+
*/
|
|
469
|
+
static cosineSimilarity(u: Embedding, v: Embedding): number;
|
|
470
|
+
/**
|
|
471
|
+
* Returns an {@link EmbeddingProvider} for this embedder.
|
|
472
|
+
* @export
|
|
473
|
+
*/
|
|
474
|
+
getProvider(): EmbeddingProvider;
|
|
475
|
+
/**
|
|
476
|
+
* Shuts down the UniversalEmbedder and releases resources.
|
|
477
|
+
* @export
|
|
478
|
+
*/
|
|
479
|
+
close(): void;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Copyright 2026 The MediaPipe Authors.
|
|
484
|
+
*
|
|
485
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
486
|
+
* you may not use this file except in compliance with the License.
|
|
487
|
+
* You may obtain a copy of the License at
|
|
488
|
+
*
|
|
489
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
490
|
+
*
|
|
491
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
492
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
493
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
494
|
+
* See the License for the specific language governing permissions and
|
|
495
|
+
* limitations under the License.
|
|
496
|
+
*/
|
|
497
|
+
/// <reference types="@webgpu/types" />
|
|
498
|
+
/** Options for configuring the model asset and hardware accelerator. */
|
|
499
|
+
export declare interface UniversalEmbedderBaseOptions {
|
|
500
|
+
/** The model asset buffer containing the .litertlm file. */
|
|
501
|
+
modelAssetBuffer?: Uint8Array | ReadableStreamDefaultReader;
|
|
502
|
+
/** The path to the model asset file. */
|
|
503
|
+
modelAssetPath?: string | string;
|
|
504
|
+
/** The hardware accelerator delegate to use ('CPU' or 'GPU'). */
|
|
505
|
+
delegate?: "CPU" | "GPU";
|
|
506
|
+
/** An optional pre-created GPUDevice to use for GPU inference. */
|
|
507
|
+
device?: GPUDevice;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/** Options to configure the MediaPipe Universal Embedder Task. */
|
|
511
|
+
export declare interface UniversalEmbedderOptions {
|
|
512
|
+
/** Base options specifying the model asset and delegate. */
|
|
513
|
+
baseOptions: UniversalEmbedderBaseOptions;
|
|
514
|
+
/** Whether to L2-normalize the output embedding vector. Defaults to true. */
|
|
515
|
+
l2Normalize?: boolean;
|
|
516
|
+
/** Optional maximum sequence length (in tokens) for text encoder signatures. */
|
|
517
|
+
maxInputLength?: number;
|
|
518
|
+
/** Optional vision tokens per image for vision encoder signatures. */
|
|
519
|
+
visionTokensPerImage?: number;
|
|
520
|
+
/** Optional activation data type for model execution. */
|
|
521
|
+
activationDataType?: ActivationDataType;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
/** Result returned by the Universal Embedder task. */
|
|
525
|
+
export declare interface UniversalEmbedderResult {
|
|
526
|
+
/** The extracted embedding list. */
|
|
527
|
+
embeddings: Embedding[];
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
/**
|
|
531
|
+
* Declarations for the native UniversalEmbedder WebAssembly module.
|
|
532
|
+
*
|
|
533
|
+
* Using `declare interface` instructs Tsickle to generate Closure Compiler
|
|
534
|
+
* externs, preventing properties from being renamed by ADVANCED_OPTIMIZATIONS.
|
|
535
|
+
*/
|
|
536
|
+
export declare interface UniversalEmbedderWasmModule extends WasmModule {
|
|
537
|
+
preinitializedWebGPUDevice?: GPUDevice;
|
|
538
|
+
createUniversalEmbedder(modelPath: string, l2Normalize: boolean, useGpuBackend: boolean, maxInputLength: number, visionTokensPerImage: number, activationDataType: number): number;
|
|
539
|
+
universalEmbedder_embedText(handle: number, text: string): Float32Array;
|
|
540
|
+
universalEmbedder_embedImage(handle: number, imageBytes: Uint8Array | string): Float32Array;
|
|
541
|
+
universalEmbedder_embedAudio(handle: number, audioSamples: number[] | Float32Array): Float32Array;
|
|
542
|
+
universalEmbedder_createContentBuilder(): number;
|
|
543
|
+
universalEmbedder_builderAddText(builderHandle: number, text: string): void;
|
|
544
|
+
universalEmbedder_builderAddImage(builderHandle: number, imageBytes: Uint8Array | string): void;
|
|
545
|
+
universalEmbedder_builderAddAudio(builderHandle: number, audioSamples: number[] | Float32Array): void;
|
|
546
|
+
universalEmbedder_executeEmbedContent(handle: number, builderHandle: number): Float32Array;
|
|
547
|
+
universalEmbedder_freeContentBuilder(builderHandle: number): void;
|
|
548
|
+
universalEmbedder_close(handle: number): void;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* A listener that receives a single element of vector-returning output packet.
|
|
553
|
+
* Receives one element at a time (in order). Once all elements are processed,
|
|
554
|
+
* the listener is invoked with `data` set to `unknown` and `done` set to true.
|
|
555
|
+
* Intended for internal usage.
|
|
556
|
+
*/
|
|
557
|
+
declare type VectorListener<T> = (data: T, done: boolean, timestamp: number) => void;
|
|
558
|
+
|
|
559
|
+
/**
|
|
560
|
+
* Interface for vector storage and similarity search backends.
|
|
561
|
+
*/
|
|
562
|
+
export declare interface VectorStore {
|
|
563
|
+
/** Inserts or updates records in the vector store. */
|
|
564
|
+
upsert(records: readonly RetrievalRecord[]): Promise<void> | void;
|
|
565
|
+
/** Deletes records matching the specified IDs. */
|
|
566
|
+
delete(ids: readonly string[]): Promise<void> | void;
|
|
567
|
+
/** Deletes records matching the specified metadata filter. */
|
|
568
|
+
deleteByMetadata(metadataFilter: Record<string, string>): Promise<void> | void;
|
|
569
|
+
/** Searches top-K nearest records to the query embedding. */
|
|
570
|
+
search(queryEmbedding: Float32Array | readonly number[], topK: number, metadataFilter?: Record<string, string>): Promise<RetrievalRecord[]> | RetrievalRecord[];
|
|
571
|
+
/** Retrieves records matching the specified IDs by key. */
|
|
572
|
+
get(ids: readonly string[]): Promise<RetrievalRecord[]> | RetrievalRecord[];
|
|
573
|
+
/** Retrieves all unique record identifiers. */
|
|
574
|
+
getAllRecordIds(): Promise<string[]> | string[];
|
|
575
|
+
/** Deletes all records from the vector store. */
|
|
576
|
+
deleteAll(): Promise<void> | void;
|
|
577
|
+
/** Releases resources associated with the vector store. */
|
|
578
|
+
close(): void;
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
/**
|
|
582
|
+
* Copyright 2022 The MediaPipe Authors.
|
|
583
|
+
*
|
|
584
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
585
|
+
* you may not use this file except in compliance with the License.
|
|
586
|
+
* You may obtain a copy of the License at
|
|
587
|
+
*
|
|
588
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
589
|
+
*
|
|
590
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
591
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
592
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
593
|
+
* See the License for the specific language governing permissions and
|
|
594
|
+
* limitations under the License.
|
|
595
|
+
*/
|
|
596
|
+
/** An object containing the locations of the Wasm assets */
|
|
597
|
+
declare interface WasmFileset {
|
|
598
|
+
/** The path to the Wasm loader script. */
|
|
599
|
+
wasmLoaderPath: string;
|
|
600
|
+
/** The path to the Wasm binary. */
|
|
601
|
+
wasmBinaryPath: string;
|
|
602
|
+
/** The optional path to the asset loader script. */
|
|
603
|
+
assetLoaderPath?: string;
|
|
604
|
+
/** The optional path to the assets binary. */
|
|
605
|
+
assetBinaryPath?: string;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
|
|
610
|
+
* doesn't break our various JS/C++ bridges. For internal usage.
|
|
611
|
+
*/
|
|
612
|
+
declare interface WasmModule {
|
|
613
|
+
canvas: HTMLCanvasElement | OffscreenCanvas | null;
|
|
614
|
+
HEAPU8: Uint8Array;
|
|
615
|
+
HEAPU32: Uint32Array;
|
|
616
|
+
HEAPF32: Float32Array;
|
|
617
|
+
HEAPF64: Float64Array;
|
|
618
|
+
FS_createDataFile: (parent: string, name: string, data: Uint8Array, canRead: boolean, canWrite: boolean, canOwn: boolean) => void;
|
|
619
|
+
FS_createPath: (parent: string, name: string, canRead: boolean, canWrite: boolean) => void;
|
|
620
|
+
FS_unlink(path: string): void;
|
|
621
|
+
gpuOriginForWebTexturesIsBottomLeft?: boolean;
|
|
622
|
+
errorListener?: ErrorListener;
|
|
623
|
+
_bindTextureToCanvas: () => boolean;
|
|
624
|
+
_changeBinaryGraph: (size: number, dataPtr: number) => void;
|
|
625
|
+
_changeTextGraph: (size: number, dataPtr: number) => void;
|
|
626
|
+
_closeGraph: () => void;
|
|
627
|
+
_free: (ptr: number) => void;
|
|
628
|
+
_malloc: (size: number) => number;
|
|
629
|
+
_processFrame: (width: number, height: number, timestamp: number) => void;
|
|
630
|
+
_setAutoRenderToScreen: (enabled: boolean) => void;
|
|
631
|
+
_waitUntilIdle: () => void;
|
|
632
|
+
dataFileDownloads?: {
|
|
633
|
+
[url: string]: {
|
|
634
|
+
loaded: number;
|
|
635
|
+
total: number;
|
|
636
|
+
};
|
|
637
|
+
};
|
|
638
|
+
stringToNewUTF8: (data: string) => number;
|
|
639
|
+
_bindTextureToStream: (streamNamePtr: number) => void;
|
|
640
|
+
_addBoundTextureToStream: (streamNamePtr: number, width: number, height: number, timestamp: number) => void;
|
|
641
|
+
_addBoolToInputStream: (data: boolean, streamNamePtr: number, timestamp: number) => void;
|
|
642
|
+
_addDoubleToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
|
|
643
|
+
_addFloatToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
|
|
644
|
+
_addIntToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
|
|
645
|
+
_addUintToInputStream: (data: number, streamNamePtr: number, timestamp: number) => void;
|
|
646
|
+
_addStringToInputStream: (dataPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
647
|
+
_addBoolVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
648
|
+
_addDoubleVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
649
|
+
_addFloatVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
650
|
+
_addIntVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
651
|
+
_addUintVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
652
|
+
_addStringVectorToInputStream: (vecPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
653
|
+
_addFlatHashMapToInputStream: (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, timestamp: number) => void;
|
|
654
|
+
_addRawDataSpanToInputStream: (dataPtr: number, dataSize: number, streamNamePtr: number, timestamp: number) => void;
|
|
655
|
+
_addProtoToInputStream: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
656
|
+
_addEmptyPacketToInputStream: (streamNamePtr: number, timestamp: number) => void;
|
|
657
|
+
_addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
|
|
658
|
+
_addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
659
|
+
_addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
660
|
+
_addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
661
|
+
_addUintToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
662
|
+
_addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
|
|
663
|
+
_addRawDataSpanToInputSidePacket: (dataPtr: number, dataSize: number, streamNamePtr: number) => void;
|
|
664
|
+
_addProtoToInputSidePacket: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number) => void;
|
|
665
|
+
_addBoolVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
666
|
+
_addDoubleVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
667
|
+
_addFloatVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
668
|
+
_addIntVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
669
|
+
_addUintVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
670
|
+
_addStringVectorToInputSidePacket: (vecPtr: number, streamNamePtr: number) => void;
|
|
671
|
+
_allocateBoolVector: (size: number) => number;
|
|
672
|
+
_allocateDoubleVector: (size: number) => number;
|
|
673
|
+
_allocateFloatVector: (size: number) => number;
|
|
674
|
+
_allocateIntVector: (size: number) => number;
|
|
675
|
+
_allocateUintVector: (size: number) => number;
|
|
676
|
+
_allocateStringVector: (size: number) => number;
|
|
677
|
+
_addBoolVectorEntry: (vecPtr: number, entry: boolean) => void;
|
|
678
|
+
_addDoubleVectorEntry: (vecPtr: number, entry: number) => void;
|
|
679
|
+
_addFloatVectorEntry: (vecPtr: number, entry: number) => void;
|
|
680
|
+
_addIntVectorEntry: (vecPtr: number, entry: number) => void;
|
|
681
|
+
_addUintVectorEntry: (vecPtr: number, entry: number) => void;
|
|
682
|
+
_addStringVectorEntry: (vecPtr: number, entryStringPtr: number) => void;
|
|
683
|
+
simpleListeners?: Record<string, SimpleListener<unknown> | VectorListener<unknown>>;
|
|
684
|
+
emptyPacketListeners?: Record<string, EmptyPacketListener>;
|
|
685
|
+
_attachBoolListener: (streamNamePtr: number) => void;
|
|
686
|
+
_attachBoolVectorListener: (streamNamePtr: number) => void;
|
|
687
|
+
_attachDoubleListener: (streamNamePtr: number) => void;
|
|
688
|
+
_attachDoubleVectorListener: (streamNamePtr: number) => void;
|
|
689
|
+
_attachFloatListener: (streamNamePtr: number) => void;
|
|
690
|
+
_attachFloatVectorListener: (streamNamePtr: number) => void;
|
|
691
|
+
_attachIntListener: (streamNamePtr: number) => void;
|
|
692
|
+
_attachIntVectorListener: (streamNamePtr: number) => void;
|
|
693
|
+
_attachUintListener: (streamNamePtr: number) => void;
|
|
694
|
+
_attachUintVectorListener: (streamNamePtr: number) => void;
|
|
695
|
+
_attachStringListener: (streamNamePtr: number) => void;
|
|
696
|
+
_attachStringVectorListener: (streamNamePtr: number) => void;
|
|
697
|
+
_attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
698
|
+
_attachProtoVectorListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
699
|
+
_attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
700
|
+
_addAudioToInputStream: (dataPtr: number, numChannels: number, numSamples: number, streamNamePtr: number, timestamp: number) => void;
|
|
701
|
+
_configureAudio: (channels: number, samples: number, sampleRate: number, streamNamePtr: number, headerNamePtr: number) => void;
|
|
702
|
+
_getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
703
|
+
_processGl: (frameDataPtr: number) => number;
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
export { }
|