@bendyline/gezel-knowledge 1.2.3 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -6
- package/dist/index.d.ts +385 -40
- package/dist/index.js +1381 -315
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -12,8 +12,9 @@ npm install @bendyline/gezel-knowledge
|
|
|
12
12
|
|
|
13
13
|
## What a `.gezk` is
|
|
14
14
|
|
|
15
|
-
An open format (format version 0.
|
|
16
|
-
archives still open
|
|
15
|
+
An open format (format version 0.8, preliminary until format 1.0; 0.5–0.7
|
|
16
|
+
archives still open, and the writer emits the oldest generation that can
|
|
17
|
+
express a catalog): a ZIP whose first entry is
|
|
17
18
|
the stored `mimetype` magic (`application/vnd.gezk+zip`), then
|
|
18
19
|
`manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
|
|
19
20
|
directory, brotli bodies, routing centroids) and shard SQLite databases with
|
|
@@ -62,10 +63,19 @@ await compileKnowledgeCatalog({
|
|
|
62
63
|
|
|
63
64
|
Every archive ships a `README.md` and `LICENSES/catalog.txt`; the compiler
|
|
64
65
|
generates minimal ones from the catalog block unless you pass richer texts in
|
|
65
|
-
`extraFiles`. Embedding requires `@huggingface/transformers` at
|
|
66
|
-
module is imported dynamically so everything else works without
|
|
67
|
-
`gezel knowledge` CLI wraps this same API (`init`, `build`,
|
|
68
|
-
`inspect`, `search`), entirely offline — no daemon.
|
|
66
|
+
`extraFiles`. Embedding requires `@huggingface/transformers` (^4.3.1) at
|
|
67
|
+
runtime; the module is imported dynamically so everything else works without
|
|
68
|
+
it. The `gezel knowledge` CLI wraps this same API (`init`, `build`,
|
|
69
|
+
`validate`, `inspect`, `search`), entirely offline — no daemon.
|
|
70
|
+
|
|
71
|
+
With a multimodal profile (`EMBEDDINGGEMMA_2_512_1`), pass `embedMedia` too and
|
|
72
|
+
every image, video and audio file a document references becomes searchable
|
|
73
|
+
media rows (one per image, one per 30-second window), stored after the text
|
|
74
|
+
chunks so text chunk ids do not move. The compiler hands `embedMedia` the
|
|
75
|
+
asset bytes and its modality and expects raw model vectors back; it projects
|
|
76
|
+
them through the profile (`profileUnitVector`). `@bendyline/gezel-service/media`
|
|
77
|
+
exports a ready implementation (`createCatalogMediaEmbedder`) that decodes
|
|
78
|
+
images in pure JS and cuts video and audio with the system ffmpeg.
|
|
69
79
|
|
|
70
80
|
## Reading a catalog
|
|
71
81
|
|
|
@@ -86,6 +96,9 @@ Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
|
|
|
86
96
|
full 200,000-chunk shard) and scores the float query against them through a
|
|
87
97
|
per-byte lookup table (the query is centered first when the catalog's profile
|
|
88
98
|
is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
|
|
99
|
+
A query of the wrong width is refused with `CatalogQueryError('dimension')`.
|
|
100
|
+
Media rows (0.8) are scored exactly in their own lane, `searchMedia`, and
|
|
101
|
+
`assetFile` returns a declared asset's verified path for range streaming.
|
|
89
102
|
|
|
90
103
|
## Stability
|
|
91
104
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as _bendyline_gezk from '@bendyline/gezk';
|
|
2
|
-
import {
|
|
3
|
-
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, KnowledgeTocReference, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
2
|
+
import { KnowledgeAssetModality, KnowledgeEmbeddingProfile, CatalogDocument, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeLocation, KnowledgeRadius, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
3
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, KnowledgeTocReference, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, QUERY_STOP_WORDS, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, formatAtLeast, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, profileUnitVector, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, shardDdlFor } from '@bendyline/gezk';
|
|
4
4
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
5
5
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
6
6
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
@@ -85,8 +85,102 @@ interface CompileAsset {
|
|
|
85
85
|
/** Exactly one of absPath / content. */
|
|
86
86
|
absPath?: string;
|
|
87
87
|
content?: Buffer;
|
|
88
|
+
/**
|
|
89
|
+
* Who made it and under which terms (license, author, sourceUrl, …).
|
|
90
|
+
* Recorded on the asset's media rows, where readers surface it.
|
|
91
|
+
*/
|
|
92
|
+
attribution?: Record<string, unknown>;
|
|
88
93
|
}
|
|
89
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Media rows (format 0.8): the images, video windows and audio windows a
|
|
97
|
+
* catalog's documents reference, embedded into the catalog's own vector
|
|
98
|
+
* space and stored as rows of the shard `chunks` table beside the text.
|
|
99
|
+
*
|
|
100
|
+
* The compiler never decodes media. A producer injects `embedMedia` — the
|
|
101
|
+
* CLI wires the service's media encoder, tests a deterministic fake — which
|
|
102
|
+
* receives one asset and returns one result for an image or one per time
|
|
103
|
+
* window for audio and video. Everything else is decided here: which
|
|
104
|
+
* references count, what text a row carries for full-text search and the
|
|
105
|
+
* relevance judge, its heading context, and its content hash.
|
|
106
|
+
*/
|
|
107
|
+
|
|
108
|
+
/** One asset handed to the producer's media embedder. */
|
|
109
|
+
interface MediaEmbedRequest {
|
|
110
|
+
modality: KnowledgeAssetModality;
|
|
111
|
+
/** Archive path, `assets/…`. */
|
|
112
|
+
path: string;
|
|
113
|
+
bytes: Buffer;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* One embedded piece of an asset: the whole image, or one window of audio or
|
|
117
|
+
* video. `vector` is the model's raw output; the compiler projects it
|
|
118
|
+
* through the profile like a text passage.
|
|
119
|
+
*/
|
|
120
|
+
interface MediaEmbedResult {
|
|
121
|
+
vector: number[];
|
|
122
|
+
startMs?: number;
|
|
123
|
+
endMs?: number;
|
|
124
|
+
width?: number;
|
|
125
|
+
height?: number;
|
|
126
|
+
/** Speech heard in this window, when the producer transcribed it. */
|
|
127
|
+
transcript?: string;
|
|
128
|
+
}
|
|
129
|
+
type MediaEmbedder = (request: MediaEmbedRequest) => Promise<MediaEmbedResult[]>;
|
|
130
|
+
/** One media row as the writer stores it. */
|
|
131
|
+
interface MediaRow {
|
|
132
|
+
modality: KnowledgeAssetModality;
|
|
133
|
+
assetPath: string;
|
|
134
|
+
mimeType: string;
|
|
135
|
+
width: number | null;
|
|
136
|
+
height: number | null;
|
|
137
|
+
startMs: number | null;
|
|
138
|
+
endMs: number | null;
|
|
139
|
+
attributionJson: string | null;
|
|
140
|
+
headingPath: string[];
|
|
141
|
+
line: number;
|
|
142
|
+
text: string;
|
|
143
|
+
contentHash: string;
|
|
144
|
+
vector: number[];
|
|
145
|
+
}
|
|
146
|
+
/** The longest row text kept, in profile tokens: a caption, not an article. */
|
|
147
|
+
declare const MEDIA_ROW_TEXT_MAX_TOKENS = 128;
|
|
148
|
+
/** Media rows one document may carry. */
|
|
149
|
+
declare const MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
|
|
150
|
+
/** Every `` in a body, with its 1-based line and that line's other text. */
|
|
151
|
+
declare function mediaReferences(markdown: string): Array<{
|
|
152
|
+
alt: string;
|
|
153
|
+
path: string;
|
|
154
|
+
line: number;
|
|
155
|
+
caption: string;
|
|
156
|
+
}>;
|
|
157
|
+
/** `assets/diagrams/red-panda_habitat.png` → `red panda habitat`. */
|
|
158
|
+
declare function humanizeAssetName(path: string): string;
|
|
159
|
+
/**
|
|
160
|
+
* Embed every media asset the documents reference (each asset once, however
|
|
161
|
+
* many documents use it) and lay out each document's media rows, ordered by
|
|
162
|
+
* reference line, then start time. A modality the profile does not describe,
|
|
163
|
+
* or one the embedder declines (returns no results for), yields no rows.
|
|
164
|
+
*/
|
|
165
|
+
declare function collectMediaRows(args: {
|
|
166
|
+
documents: Array<{
|
|
167
|
+
id: string;
|
|
168
|
+
title: string;
|
|
169
|
+
markdown: string;
|
|
170
|
+
chunks: MarkdownChunk[];
|
|
171
|
+
}>;
|
|
172
|
+
assets: ReadonlyArray<{
|
|
173
|
+
path: string;
|
|
174
|
+
bytes: Buffer;
|
|
175
|
+
sha256: string;
|
|
176
|
+
attribution?: Record<string, unknown>;
|
|
177
|
+
}>;
|
|
178
|
+
profile: KnowledgeEmbeddingProfile;
|
|
179
|
+
embedMedia: MediaEmbedder;
|
|
180
|
+
countTokens: (text: string) => number;
|
|
181
|
+
onWarning?: (message: string) => void;
|
|
182
|
+
}): Promise<Map<string, MediaRow[]>>;
|
|
183
|
+
|
|
90
184
|
/**
|
|
91
185
|
* `.gezk` compiler (gezk 0.7; spec in bendyline/gezk). One owner of normalization, chunk
|
|
92
186
|
* ids, embedding-profile conformance, SQLite schema, shard assignment,
|
|
@@ -154,6 +248,12 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
154
248
|
embed: (texts: string[]) => Promise<number[][]>;
|
|
155
249
|
/** The profile tokenizer's counter (chunking + header truncation). */
|
|
156
250
|
countTokens: (text: string) => number;
|
|
251
|
+
/**
|
|
252
|
+
* Embeds the images, audio and video the documents reference into the
|
|
253
|
+
* profile's space (format 0.8 media rows; see media-rows.ts). Absent, a
|
|
254
|
+
* catalog carries text rows only, whatever its assets.
|
|
255
|
+
*/
|
|
256
|
+
embedMedia?: MediaEmbedder;
|
|
157
257
|
/** Scratch directory for the staged databases (removed on success). */
|
|
158
258
|
workDir: string;
|
|
159
259
|
smokeQueries?: Array<{
|
|
@@ -242,11 +342,27 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
242
342
|
interface CompileReport {
|
|
243
343
|
manifest: KnowledgeCatalogManifest;
|
|
244
344
|
documents: number;
|
|
345
|
+
/** Every row of the shard `chunks` tables: text chunks plus media rows. */
|
|
245
346
|
chunks: number;
|
|
347
|
+
/** Media rows by modality (all zero without `embedMedia`). */
|
|
348
|
+
media: {
|
|
349
|
+
image: number;
|
|
350
|
+
video: number;
|
|
351
|
+
audio: number;
|
|
352
|
+
};
|
|
246
353
|
shards: number;
|
|
247
354
|
archiveBytes: number;
|
|
248
355
|
}
|
|
249
356
|
declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
|
|
357
|
+
/**
|
|
358
|
+
* The oldest generation that can express this catalog: 0.7, unless the
|
|
359
|
+
* embedding profile uses 0.8 vocabulary or the catalog ships audio or video.
|
|
360
|
+
* Writing the oldest one keeps catalog updates reaching every reader that
|
|
361
|
+
* can open them, instead of cutting installs off at each format bump.
|
|
362
|
+
*/
|
|
363
|
+
declare function catalogFormatVersion(profile: KnowledgeEmbeddingProfile, assets: ReadonlyArray<{
|
|
364
|
+
path: string;
|
|
365
|
+
}>, mediaRows?: number): Extract<GezkFormatVersion, '0.7' | '0.8'>;
|
|
250
366
|
|
|
251
367
|
/**
|
|
252
368
|
* Deterministic `.gezk` ZIP writer: the stored `mimetype` entry first (the
|
|
@@ -331,6 +447,16 @@ declare class CatalogOpenError extends Error {
|
|
|
331
447
|
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt';
|
|
332
448
|
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt');
|
|
333
449
|
}
|
|
450
|
+
/**
|
|
451
|
+
* A query the catalog cannot answer as asked. `dimension`: the query vector's
|
|
452
|
+
* width differs from the catalog's stored width — an embedder that ignored
|
|
453
|
+
* the profile's truncation, or a model the catalog was not built with.
|
|
454
|
+
* Callers treat it as an embedding failure and fall back to keyword search.
|
|
455
|
+
*/
|
|
456
|
+
declare class CatalogQueryError extends Error {
|
|
457
|
+
readonly reason: 'dimension';
|
|
458
|
+
constructor(message: string, reason: 'dimension');
|
|
459
|
+
}
|
|
334
460
|
interface CatalogDb {
|
|
335
461
|
db: DatabaseSync;
|
|
336
462
|
/** The `PRAGMA user_version` generation the file was written with. */
|
|
@@ -398,9 +524,25 @@ interface CatalogAssetInfo {
|
|
|
398
524
|
sizeBytes: number;
|
|
399
525
|
sha256: string;
|
|
400
526
|
}
|
|
527
|
+
/** A declared asset's file on disk, for streaming (extraction already verified its hash). */
|
|
528
|
+
interface CatalogAssetFile extends CatalogAssetInfo {
|
|
529
|
+
absPath: string;
|
|
530
|
+
}
|
|
401
531
|
interface CatalogAssetRead extends CatalogAssetInfo {
|
|
402
532
|
bytes: Uint8Array;
|
|
403
533
|
}
|
|
534
|
+
/** What a media row (0.8) carries beyond its text: the asset, its kind, and the span. */
|
|
535
|
+
interface CatalogChunkMedia {
|
|
536
|
+
modality: 'image' | 'video' | 'audio';
|
|
537
|
+
assetPath: string;
|
|
538
|
+
mimeType: string;
|
|
539
|
+
width?: number;
|
|
540
|
+
height?: number;
|
|
541
|
+
startMs?: number;
|
|
542
|
+
endMs?: number;
|
|
543
|
+
thumbnailAssetPath?: string;
|
|
544
|
+
attribution?: Record<string, unknown>;
|
|
545
|
+
}
|
|
404
546
|
interface CatalogChunkHit {
|
|
405
547
|
chunkUid: string;
|
|
406
548
|
documentId: string;
|
|
@@ -413,6 +555,8 @@ interface CatalogChunkHit {
|
|
|
413
555
|
cosine?: number;
|
|
414
556
|
source: 'vector' | 'fts';
|
|
415
557
|
shardId: number;
|
|
558
|
+
/** Present on a media row (index schema 5). */
|
|
559
|
+
media?: CatalogChunkMedia;
|
|
416
560
|
}
|
|
417
561
|
interface ShardInfo {
|
|
418
562
|
id: number;
|
|
@@ -429,6 +573,7 @@ declare class CatalogHandle {
|
|
|
429
573
|
private readonly router;
|
|
430
574
|
private readonly connections;
|
|
431
575
|
private readonly bitIndexes;
|
|
576
|
+
private readonly mediaIndexes;
|
|
432
577
|
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
433
578
|
private readonly chunkTermRows;
|
|
434
579
|
private readonly spatial;
|
|
@@ -472,6 +617,34 @@ declare class CatalogHandle {
|
|
|
472
617
|
* the statement of how THIS catalog was built.
|
|
473
618
|
*/
|
|
474
619
|
private binaryCenter;
|
|
620
|
+
/** The chunk columns this catalog's schema has. */
|
|
621
|
+
private chunkColumns;
|
|
622
|
+
/** A shard's media rows with their int8 vectors (empty before index schema 5). */
|
|
623
|
+
private shardMedia;
|
|
624
|
+
/** Media rows this catalog holds, by modality (all zero before index schema 5). */
|
|
625
|
+
mediaCounts(): {
|
|
626
|
+
image: number;
|
|
627
|
+
video: number;
|
|
628
|
+
audio: number;
|
|
629
|
+
};
|
|
630
|
+
/**
|
|
631
|
+
* The media lane: every media row of every shard scored exactly against
|
|
632
|
+
* the query (int8 rerank), keeping the best `perModality` of each kind.
|
|
633
|
+
* Images sit at a lower cosine than text in a shared space, so in the text
|
|
634
|
+
* lane's sign-bit pre-filter they would rarely survive the cut; media rows
|
|
635
|
+
* are few enough (capped per catalog) to score them all.
|
|
636
|
+
*/
|
|
637
|
+
searchMedia(queryVector: Float32Array, opts?: {
|
|
638
|
+
perModality?: number;
|
|
639
|
+
allowedDocumentIds?: ReadonlySet<string>;
|
|
640
|
+
}): CatalogChunkHit[];
|
|
641
|
+
private hitFromRow;
|
|
642
|
+
/**
|
|
643
|
+
* Refuse a query vector whose width is not the catalog's stored width,
|
|
644
|
+
* before any scan: routing and rerank would otherwise score a silent
|
|
645
|
+
* prefix or fail deep inside a shard with an untyped error.
|
|
646
|
+
*/
|
|
647
|
+
private assertQueryWidth;
|
|
475
648
|
/** The embedding dimension from the router's profile echo. */
|
|
476
649
|
private dimensions;
|
|
477
650
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
@@ -542,6 +715,12 @@ declare class CatalogHandle {
|
|
|
542
715
|
private assetIndex;
|
|
543
716
|
/** Declared assets, sorted by path. */
|
|
544
717
|
assets(): CatalogAssetInfo[];
|
|
718
|
+
/**
|
|
719
|
+
* Where a declared asset lives on disk, so a large one (video, audio) can be
|
|
720
|
+
* streamed with byte ranges instead of read whole. Checks containment and
|
|
721
|
+
* the manifest size; the content hash was verified once, at extraction.
|
|
722
|
+
*/
|
|
723
|
+
assetFile(path: string): CatalogAssetFile | null;
|
|
545
724
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
546
725
|
readAsset(path: string): CatalogAssetRead | null;
|
|
547
726
|
private documentMeta;
|
|
@@ -718,41 +897,11 @@ interface AsymmetricHit {
|
|
|
718
897
|
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
719
898
|
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
720
899
|
* stored; this function does not know or care which.
|
|
721
|
-
*/
|
|
722
|
-
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number, eligibleChunkIds?: ReadonlySet<number>): AsymmetricHit[];
|
|
723
|
-
|
|
724
|
-
/**
|
|
725
|
-
* Words that carry no subject in a search request. Dropping one from an FTS5
|
|
726
|
-
* OR query is safe: it can only remove hits that matched nothing else, and
|
|
727
|
-
* anything holding a distinctive term still ranks.
|
|
728
|
-
*
|
|
729
|
-
* One list for every keyword-search path — the project index's query builder
|
|
730
|
-
* and token scorers in the service, and the knowledge-catalog reader here.
|
|
731
|
-
* It lives in this package because the reader cannot import the service.
|
|
732
|
-
* There were three lists once: a 24-word one behind the project index's FTS5
|
|
733
|
-
* query, a 14-word one behind area-summary token coverage, and the catalog
|
|
734
|
-
* reader's function words. Neither project list covered the vocabulary of an
|
|
735
|
-
* ordinary request, so "Can you create a PowerPoint about France" reached
|
|
736
|
-
* FTS5 as `"can"* OR "create"* OR "powerpoint"* OR "about"* OR "france"*`.
|
|
737
|
-
* In the shared library `france` matched nothing at all while
|
|
738
|
-
* `can`/`about`/`create` matched 66/25/30 rows, so ranking was decided
|
|
739
|
-
* entirely by the filler: the top document hit was a career memoir, and the
|
|
740
|
-
* top symbol hit was a heading called "All About DocBlocks" whose only
|
|
741
|
-
* matched token was `about`. The catalog reader kept `tell`, `please` and
|
|
742
|
-
* `help` in "Can you tell me how to integrate Azure search with blob
|
|
743
|
-
* storage?".
|
|
744
|
-
*
|
|
745
|
-
* Callers fall back to every token when a query is nothing but these words,
|
|
746
|
-
* so "how to" and a bare search for "create" still search for what was typed.
|
|
747
|
-
*
|
|
748
|
-
* Judgment call worth knowing about: the request verbs `create` and `make`
|
|
749
|
-
* are stopwords, while `build`, `fix`, `read`, `write`, `update`, `delete`
|
|
750
|
-
* and `show` are not. The first two are how a user says "produce a thing" and
|
|
751
|
-
* carry no subject; the rest name real work and real symbols in a codebase.
|
|
752
900
|
*
|
|
753
|
-
*
|
|
901
|
+
* `excludedChunkIds` drops rows outright — media rows, which the text lane
|
|
902
|
+
* leaves to its own exact scan rather than letting them take candidate slots.
|
|
754
903
|
*/
|
|
755
|
-
declare
|
|
904
|
+
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number, eligibleChunkIds?: ReadonlySet<number>, excludedChunkIds?: ReadonlySet<number>): AsymmetricHit[];
|
|
756
905
|
|
|
757
906
|
/**
|
|
758
907
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
@@ -909,6 +1058,7 @@ declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
|
909
1058
|
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
910
1059
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
911
1060
|
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
1061
|
+
declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
|
|
912
1062
|
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
913
1063
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
914
1064
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
@@ -922,6 +1072,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
922
1072
|
revision: string;
|
|
923
1073
|
onnxFile?: string | undefined;
|
|
924
1074
|
onnxDigest?: string | undefined;
|
|
1075
|
+
files?: {
|
|
1076
|
+
path: string;
|
|
1077
|
+
digest?: string | undefined;
|
|
1078
|
+
}[] | undefined;
|
|
925
1079
|
};
|
|
926
1080
|
tokenizer: {
|
|
927
1081
|
kind: string;
|
|
@@ -951,6 +1105,43 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
951
1105
|
center?: number[] | undefined;
|
|
952
1106
|
};
|
|
953
1107
|
};
|
|
1108
|
+
truncation?: {
|
|
1109
|
+
method: "prefix";
|
|
1110
|
+
sourceDimensions: number;
|
|
1111
|
+
} | undefined;
|
|
1112
|
+
media?: {
|
|
1113
|
+
image?: {
|
|
1114
|
+
encoder: {
|
|
1115
|
+
onnxFile: string;
|
|
1116
|
+
onnxDigest?: string | undefined;
|
|
1117
|
+
files?: {
|
|
1118
|
+
path: string;
|
|
1119
|
+
digest?: string | undefined;
|
|
1120
|
+
}[] | undefined;
|
|
1121
|
+
};
|
|
1122
|
+
tokenBudget: number;
|
|
1123
|
+
resample: "bilinear" | "bicubic";
|
|
1124
|
+
alpha: "composite-white";
|
|
1125
|
+
} | undefined;
|
|
1126
|
+
video?: {
|
|
1127
|
+
framesPerSecond: number;
|
|
1128
|
+
maxFrames: number;
|
|
1129
|
+
tokenBudgetPerFrame: number;
|
|
1130
|
+
} | undefined;
|
|
1131
|
+
audio?: {
|
|
1132
|
+
encoder: {
|
|
1133
|
+
onnxFile: string;
|
|
1134
|
+
onnxDigest?: string | undefined;
|
|
1135
|
+
files?: {
|
|
1136
|
+
path: string;
|
|
1137
|
+
digest?: string | undefined;
|
|
1138
|
+
}[] | undefined;
|
|
1139
|
+
};
|
|
1140
|
+
sampleRate: number;
|
|
1141
|
+
channels: 1;
|
|
1142
|
+
maxWindowMs: number;
|
|
1143
|
+
} | undefined;
|
|
1144
|
+
} | undefined;
|
|
954
1145
|
};
|
|
955
1146
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
956
1147
|
declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
@@ -960,6 +1151,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
960
1151
|
revision: string;
|
|
961
1152
|
onnxFile?: string | undefined;
|
|
962
1153
|
onnxDigest?: string | undefined;
|
|
1154
|
+
files?: {
|
|
1155
|
+
path: string;
|
|
1156
|
+
digest?: string | undefined;
|
|
1157
|
+
}[] | undefined;
|
|
963
1158
|
};
|
|
964
1159
|
tokenizer: {
|
|
965
1160
|
kind: string;
|
|
@@ -989,6 +1184,43 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
989
1184
|
center?: number[] | undefined;
|
|
990
1185
|
};
|
|
991
1186
|
};
|
|
1187
|
+
truncation?: {
|
|
1188
|
+
method: "prefix";
|
|
1189
|
+
sourceDimensions: number;
|
|
1190
|
+
} | undefined;
|
|
1191
|
+
media?: {
|
|
1192
|
+
image?: {
|
|
1193
|
+
encoder: {
|
|
1194
|
+
onnxFile: string;
|
|
1195
|
+
onnxDigest?: string | undefined;
|
|
1196
|
+
files?: {
|
|
1197
|
+
path: string;
|
|
1198
|
+
digest?: string | undefined;
|
|
1199
|
+
}[] | undefined;
|
|
1200
|
+
};
|
|
1201
|
+
tokenBudget: number;
|
|
1202
|
+
resample: "bilinear" | "bicubic";
|
|
1203
|
+
alpha: "composite-white";
|
|
1204
|
+
} | undefined;
|
|
1205
|
+
video?: {
|
|
1206
|
+
framesPerSecond: number;
|
|
1207
|
+
maxFrames: number;
|
|
1208
|
+
tokenBudgetPerFrame: number;
|
|
1209
|
+
} | undefined;
|
|
1210
|
+
audio?: {
|
|
1211
|
+
encoder: {
|
|
1212
|
+
onnxFile: string;
|
|
1213
|
+
onnxDigest?: string | undefined;
|
|
1214
|
+
files?: {
|
|
1215
|
+
path: string;
|
|
1216
|
+
digest?: string | undefined;
|
|
1217
|
+
}[] | undefined;
|
|
1218
|
+
};
|
|
1219
|
+
sampleRate: number;
|
|
1220
|
+
channels: 1;
|
|
1221
|
+
maxWindowMs: number;
|
|
1222
|
+
} | undefined;
|
|
1223
|
+
} | undefined;
|
|
992
1224
|
};
|
|
993
1225
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
994
1226
|
declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
@@ -1013,7 +1245,11 @@ declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
|
1013
1245
|
* vector space rather than silently producing slightly different vectors.
|
|
1014
1246
|
*/
|
|
1015
1247
|
|
|
1016
|
-
|
|
1248
|
+
/**
|
|
1249
|
+
* `file`: one of the profile's further pinned files (weight sidecars,
|
|
1250
|
+
* configs); `media`: a media encoder's graph or one of its files.
|
|
1251
|
+
*/
|
|
1252
|
+
type ArtifactRole = 'onnx' | 'tokenizer' | 'file' | 'media';
|
|
1017
1253
|
interface ArtifactCheck {
|
|
1018
1254
|
role: ArtifactRole;
|
|
1019
1255
|
/** Repo-relative file the profile pins. */
|
|
@@ -1038,6 +1274,20 @@ declare class EmbedderArtifactError extends Error {
|
|
|
1038
1274
|
}
|
|
1039
1275
|
/** The transformers.js FileCache key for one repo file. */
|
|
1040
1276
|
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
1277
|
+
/** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
|
|
1278
|
+
declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
|
|
1279
|
+
file: string;
|
|
1280
|
+
expected: string;
|
|
1281
|
+
}>;
|
|
1282
|
+
/**
|
|
1283
|
+
* Hash a profile's media encoder files in a transformers.js cache and
|
|
1284
|
+
* compare, with the same contract as {@link verifyProfileArtifacts}: a media
|
|
1285
|
+
* vector is only as trustworthy as the bytes that produced it.
|
|
1286
|
+
*/
|
|
1287
|
+
declare function verifyMediaArtifacts(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>, opts: {
|
|
1288
|
+
cacheDir: string;
|
|
1289
|
+
revision?: string;
|
|
1290
|
+
}): Promise<VerifiedArtifacts>;
|
|
1041
1291
|
/**
|
|
1042
1292
|
* Hash the profile's pinned files in a transformers.js cache and compare.
|
|
1043
1293
|
* `revision` names the cache key the files were fetched under; it defaults
|
|
@@ -1073,9 +1323,12 @@ interface ProfileEmbedder {
|
|
|
1073
1323
|
readonly profile: KnowledgeEmbeddingProfile;
|
|
1074
1324
|
/** What the loaded model files were checked against after loading. */
|
|
1075
1325
|
readonly verification: VerifiedArtifacts;
|
|
1076
|
-
/**
|
|
1326
|
+
/**
|
|
1327
|
+
* Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's
|
|
1328
|
+
* contract) and returns the model's own width; the compiler projects.
|
|
1329
|
+
*/
|
|
1077
1330
|
embed(texts: string[]): Promise<number[][]>;
|
|
1078
|
-
/** Query embed — applies the profile's
|
|
1331
|
+
/** Query embed — applies the queryInstruction and the profile's projection itself. */
|
|
1079
1332
|
embedQuery(text: string): Promise<Float32Array>;
|
|
1080
1333
|
/** Profile-tokenizer token count (sync once loaded — chunking's contract). */
|
|
1081
1334
|
countTokens(text: string): number;
|
|
@@ -1086,7 +1339,7 @@ interface ProfileEmbedder {
|
|
|
1086
1339
|
* as its optional peer dependency (a test holds the two together). Error
|
|
1087
1340
|
* messages quote it so the install command they print is exact.
|
|
1088
1341
|
*/
|
|
1089
|
-
declare const TRANSFORMERS_PEER_RANGE = "^3.
|
|
1342
|
+
declare const TRANSFORMERS_PEER_RANGE = "^4.3.1";
|
|
1090
1343
|
declare class EmbedderUnavailableError extends Error {
|
|
1091
1344
|
readonly isActionable = true;
|
|
1092
1345
|
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
@@ -1101,12 +1354,41 @@ interface TransformersModule {
|
|
|
1101
1354
|
AutoTokenizer: {
|
|
1102
1355
|
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerFn>;
|
|
1103
1356
|
};
|
|
1357
|
+
/**
|
|
1358
|
+
* Read a repo's config before choosing a loader. Optional so a test fake
|
|
1359
|
+
* that only knows the pipeline keeps working; without it every profile
|
|
1360
|
+
* loads through `pipeline`.
|
|
1361
|
+
*/
|
|
1362
|
+
AutoConfig?: {
|
|
1363
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
|
1364
|
+
};
|
|
1365
|
+
/** Multimodal encoders: loads the text session alone (see `isMultimodalEncoderConfig`). */
|
|
1366
|
+
AutoModel?: {
|
|
1367
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<EncoderModelFn>;
|
|
1368
|
+
};
|
|
1104
1369
|
env: {
|
|
1105
1370
|
cacheDir?: string;
|
|
1106
1371
|
useFSCache?: boolean;
|
|
1107
1372
|
allowRemoteModels?: boolean;
|
|
1108
1373
|
};
|
|
1109
1374
|
}
|
|
1375
|
+
/** A tensor as this module reads it: flat data plus its shape. */
|
|
1376
|
+
interface TensorLike {
|
|
1377
|
+
data: ArrayLike<number>;
|
|
1378
|
+
dims: number[];
|
|
1379
|
+
}
|
|
1380
|
+
/** A tokenizer called on a batch, as the encoder path feeds a model directly. */
|
|
1381
|
+
type BatchTokenizerFn = TokenizerFn & ((texts: string[], options: {
|
|
1382
|
+
padding: boolean;
|
|
1383
|
+
truncation: boolean;
|
|
1384
|
+
max_length: number;
|
|
1385
|
+
}) => {
|
|
1386
|
+
attention_mask?: TensorLike;
|
|
1387
|
+
});
|
|
1388
|
+
/** A text-session encoder: tokenized inputs in, named tensors out. */
|
|
1389
|
+
type EncoderModelFn = ((inputs: unknown) => Promise<Record<string, TensorLike>>) & {
|
|
1390
|
+
dispose?: () => Promise<unknown>;
|
|
1391
|
+
};
|
|
1110
1392
|
type PipelineFn = ((texts: string[], options: {
|
|
1111
1393
|
pooling: string;
|
|
1112
1394
|
normalize: boolean;
|
|
@@ -1143,6 +1425,28 @@ declare function transformersGraphOptions(onnxFile: string): {
|
|
|
1143
1425
|
subfolder: string;
|
|
1144
1426
|
model_file_name: string;
|
|
1145
1427
|
} | null;
|
|
1428
|
+
/**
|
|
1429
|
+
* A multimodal embedding model (one space for text, images, audio) carries
|
|
1430
|
+
* its media encoders' configs beside the text model's. Through the
|
|
1431
|
+
* `feature-extraction` pipeline it would load every encoder; the text path
|
|
1432
|
+
* needs only the text session, which `AutoModel` loads once those configs
|
|
1433
|
+
* are cleared.
|
|
1434
|
+
*/
|
|
1435
|
+
declare function isMultimodalEncoderConfig(config: Record<string, unknown>): boolean;
|
|
1436
|
+
/** How many external-data sidecars the profile pins beside its graph (`<graph>_data`, `<graph>_data_1`, …). */
|
|
1437
|
+
declare function pinnedExternalDataChunks(profile: KnowledgeEmbeddingProfile): number;
|
|
1438
|
+
/**
|
|
1439
|
+
* Make a load of a profile with weight sidecars safe from transformers.js's
|
|
1440
|
+
* sidecar hang (see pinned-files.ts): a local-only load fails fast, naming
|
|
1441
|
+
* the files that are missing; a networked load downloads the sidecars first,
|
|
1442
|
+
* hash-verified. A profile without sidecars loads exactly as before.
|
|
1443
|
+
*/
|
|
1444
|
+
declare function preparePinnedLoad(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1445
|
+
cacheDir: string;
|
|
1446
|
+
localFilesOnly?: boolean;
|
|
1447
|
+
modalities?: ReadonlyArray<'image' | 'video' | 'audio'>;
|
|
1448
|
+
fetchImpl?: typeof fetch;
|
|
1449
|
+
}): Promise<void>;
|
|
1146
1450
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
1147
1451
|
cacheDir?: string;
|
|
1148
1452
|
/** Read cached model files only; never initiate a download. */
|
|
@@ -1161,6 +1465,47 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
1161
1465
|
transformers?: TransformersModule;
|
|
1162
1466
|
}): Promise<ProfileEmbedder>;
|
|
1163
1467
|
|
|
1468
|
+
/**
|
|
1469
|
+
* The pinned files a profile load reads, and getting the weight sidecars into
|
|
1470
|
+
* the cache before transformers.js goes looking for them.
|
|
1471
|
+
*
|
|
1472
|
+
* transformers.js 4.3.1 fetches `.onnx_data` sidecars inside
|
|
1473
|
+
* `new Promise(async (resolve) => { await getModelFile(…) })`. When the file
|
|
1474
|
+
* is missing (a `local_files_only` load of an incomplete install) or its
|
|
1475
|
+
* download fails, the rejection escapes the executor as an unhandled
|
|
1476
|
+
* rejection and the load's promise never settles: a hang, not an error. So a
|
|
1477
|
+
* local-only load checks the cache first, and a networked load downloads the
|
|
1478
|
+
* sidecars itself, hash-verified, before handing over.
|
|
1479
|
+
*/
|
|
1480
|
+
|
|
1481
|
+
interface PinnedLoadFile {
|
|
1482
|
+
path: string;
|
|
1483
|
+
/** `sha256:<hex>` when the profile pins it. */
|
|
1484
|
+
digest?: string;
|
|
1485
|
+
}
|
|
1486
|
+
type MediaModality = 'image' | 'video' | 'audio';
|
|
1487
|
+
/** `onnx/model.onnx_data`, `onnx/model.onnx_data_1`: weights stored beside a graph. */
|
|
1488
|
+
declare function isExternalDataFile(path: string): boolean;
|
|
1489
|
+
/** Every file the profile pins for its text session and, with `modalities`, its media encoders. */
|
|
1490
|
+
declare function pinnedLoadFiles(profile: KnowledgeEmbeddingProfile, modalities?: readonly MediaModality[]): PinnedLoadFile[];
|
|
1491
|
+
/** The pinned files of a load that are not in the transformers.js cache. */
|
|
1492
|
+
declare function missingPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1493
|
+
cacheDir: string;
|
|
1494
|
+
modalities?: readonly MediaModality[];
|
|
1495
|
+
revision?: string;
|
|
1496
|
+
}): Promise<string[]>;
|
|
1497
|
+
/**
|
|
1498
|
+
* Download the profile's missing weight sidecars into the cache, each to a
|
|
1499
|
+
* `.partial`, hashed against its pin and renamed, so the load that follows
|
|
1500
|
+
* finds them cached. Throws on an HTTP error or a digest mismatch.
|
|
1501
|
+
*/
|
|
1502
|
+
declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1503
|
+
cacheDir: string;
|
|
1504
|
+
modalities?: readonly MediaModality[];
|
|
1505
|
+
revision?: string;
|
|
1506
|
+
fetchImpl?: typeof fetch;
|
|
1507
|
+
}): Promise<string[]>;
|
|
1508
|
+
|
|
1164
1509
|
/**
|
|
1165
1510
|
* Tables of contents that documentation trees already carry. Each parser
|
|
1166
1511
|
* turns one tool's outline into the same shape — a tree of topics holding
|
|
@@ -1418,4 +1763,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1418
1763
|
|
|
1419
1764
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1420
1765
|
|
|
1421
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder,
|
|
1766
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|