@bendyline/gezel-knowledge 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -6
- package/dist/index.d.ts +510 -44
- package/dist/index.js +2118 -239
- package/package.json +4 -4
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
|
|
1
|
+
import * as _bendyline_gezk from '@bendyline/gezk';
|
|
2
|
+
import { KnowledgeAssetModality, KnowledgeEmbeddingProfile, CatalogDocument, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeLocation, KnowledgeRadius, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
3
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, KnowledgeTocReference, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, QUERY_STOP_WORDS, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, formatAtLeast, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, profileUnitVector, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, shardDdlFor } from '@bendyline/gezk';
|
|
3
4
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
5
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
5
6
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
@@ -78,8 +79,110 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
|
|
|
78
79
|
/** Split markdown into heading-aware, bounded, overlapping chunks. */
|
|
79
80
|
declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
|
|
80
81
|
|
|
82
|
+
interface CompileAsset {
|
|
83
|
+
/** Archive path, assets/…, matching the format's asset path grammar. */
|
|
84
|
+
path: string;
|
|
85
|
+
/** Exactly one of absPath / content. */
|
|
86
|
+
absPath?: string;
|
|
87
|
+
content?: Buffer;
|
|
88
|
+
/**
|
|
89
|
+
* Who made it and under which terms (license, author, sourceUrl, …).
|
|
90
|
+
* Recorded on the asset's media rows, where readers surface it.
|
|
91
|
+
*/
|
|
92
|
+
attribution?: Record<string, unknown>;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Media rows (format 0.8): the images, video windows and audio windows a
|
|
97
|
+
* catalog's documents reference, embedded into the catalog's own vector
|
|
98
|
+
* space and stored as rows of the shard `chunks` table beside the text.
|
|
99
|
+
*
|
|
100
|
+
* The compiler never decodes media. A producer injects `embedMedia` — the
|
|
101
|
+
* CLI wires the service's media encoder, tests a deterministic fake — which
|
|
102
|
+
* receives one asset and returns one result for an image or one per time
|
|
103
|
+
* window for audio and video. Everything else is decided here: which
|
|
104
|
+
* references count, what text a row carries for full-text search and the
|
|
105
|
+
* relevance judge, its heading context, and its content hash.
|
|
106
|
+
*/
|
|
107
|
+
|
|
108
|
+
/** One asset handed to the producer's media embedder. */
|
|
109
|
+
interface MediaEmbedRequest {
|
|
110
|
+
modality: KnowledgeAssetModality;
|
|
111
|
+
/** Archive path, `assets/…`. */
|
|
112
|
+
path: string;
|
|
113
|
+
bytes: Buffer;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* One embedded piece of an asset: the whole image, or one window of audio or
|
|
117
|
+
* video. `vector` is the model's raw output; the compiler projects it
|
|
118
|
+
* through the profile like a text passage.
|
|
119
|
+
*/
|
|
120
|
+
interface MediaEmbedResult {
|
|
121
|
+
vector: number[];
|
|
122
|
+
startMs?: number;
|
|
123
|
+
endMs?: number;
|
|
124
|
+
width?: number;
|
|
125
|
+
height?: number;
|
|
126
|
+
/** Speech heard in this window, when the producer transcribed it. */
|
|
127
|
+
transcript?: string;
|
|
128
|
+
}
|
|
129
|
+
type MediaEmbedder = (request: MediaEmbedRequest) => Promise<MediaEmbedResult[]>;
|
|
130
|
+
/** One media row as the writer stores it. */
|
|
131
|
+
interface MediaRow {
|
|
132
|
+
modality: KnowledgeAssetModality;
|
|
133
|
+
assetPath: string;
|
|
134
|
+
mimeType: string;
|
|
135
|
+
width: number | null;
|
|
136
|
+
height: number | null;
|
|
137
|
+
startMs: number | null;
|
|
138
|
+
endMs: number | null;
|
|
139
|
+
attributionJson: string | null;
|
|
140
|
+
headingPath: string[];
|
|
141
|
+
line: number;
|
|
142
|
+
text: string;
|
|
143
|
+
contentHash: string;
|
|
144
|
+
vector: number[];
|
|
145
|
+
}
|
|
146
|
+
/** The longest row text kept, in profile tokens: a caption, not an article. */
|
|
147
|
+
declare const MEDIA_ROW_TEXT_MAX_TOKENS = 128;
|
|
148
|
+
/** Media rows one document may carry. */
|
|
149
|
+
declare const MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
|
|
150
|
+
/** Every `` in a body, with its 1-based line and that line's other text. */
|
|
151
|
+
declare function mediaReferences(markdown: string): Array<{
|
|
152
|
+
alt: string;
|
|
153
|
+
path: string;
|
|
154
|
+
line: number;
|
|
155
|
+
caption: string;
|
|
156
|
+
}>;
|
|
157
|
+
/** `assets/diagrams/red-panda_habitat.png` → `red panda habitat`. */
|
|
158
|
+
declare function humanizeAssetName(path: string): string;
|
|
159
|
+
/**
|
|
160
|
+
* Embed every media asset the documents reference (each asset once, however
|
|
161
|
+
* many documents use it) and lay out each document's media rows, ordered by
|
|
162
|
+
* reference line, then start time. A modality the profile does not describe,
|
|
163
|
+
* or one the embedder declines (returns no results for), yields no rows.
|
|
164
|
+
*/
|
|
165
|
+
declare function collectMediaRows(args: {
|
|
166
|
+
documents: Array<{
|
|
167
|
+
id: string;
|
|
168
|
+
title: string;
|
|
169
|
+
markdown: string;
|
|
170
|
+
chunks: MarkdownChunk[];
|
|
171
|
+
}>;
|
|
172
|
+
assets: ReadonlyArray<{
|
|
173
|
+
path: string;
|
|
174
|
+
bytes: Buffer;
|
|
175
|
+
sha256: string;
|
|
176
|
+
attribution?: Record<string, unknown>;
|
|
177
|
+
}>;
|
|
178
|
+
profile: KnowledgeEmbeddingProfile;
|
|
179
|
+
embedMedia: MediaEmbedder;
|
|
180
|
+
countTokens: (text: string) => number;
|
|
181
|
+
onWarning?: (message: string) => void;
|
|
182
|
+
}): Promise<Map<string, MediaRow[]>>;
|
|
183
|
+
|
|
81
184
|
/**
|
|
82
|
-
* `.gezk` compiler (gezk 0.
|
|
185
|
+
* `.gezk` compiler (gezk 0.7; spec in bendyline/gezk). One owner of normalization, chunk
|
|
83
186
|
* ids, embedding-profile conformance, SQLite schema, shard assignment,
|
|
84
187
|
* centroid routing, and deterministic archive layout — the CLI's Markdown
|
|
85
188
|
* adapter and Qualla's Wikipedia adapter both feed this API.
|
|
@@ -104,13 +207,6 @@ interface CompileTopic {
|
|
|
104
207
|
description?: string;
|
|
105
208
|
sortKey?: string;
|
|
106
209
|
}
|
|
107
|
-
interface CompileAsset {
|
|
108
|
-
/** Archive path, `assets/…`, matching the format's asset path grammar. */
|
|
109
|
-
path: string;
|
|
110
|
-
/** Exactly one of `absPath` / `content`. */
|
|
111
|
-
absPath?: string;
|
|
112
|
-
content?: Buffer;
|
|
113
|
-
}
|
|
114
210
|
interface CompileKnowledgeCatalogOptions {
|
|
115
211
|
catalog: {
|
|
116
212
|
id: string;
|
|
@@ -152,6 +248,12 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
152
248
|
embed: (texts: string[]) => Promise<number[][]>;
|
|
153
249
|
/** The profile tokenizer's counter (chunking + header truncation). */
|
|
154
250
|
countTokens: (text: string) => number;
|
|
251
|
+
/**
|
|
252
|
+
* Embeds the images, audio and video the documents reference into the
|
|
253
|
+
* profile's space (format 0.8 media rows; see media-rows.ts). Absent, a
|
|
254
|
+
* catalog carries text rows only, whatever its assets.
|
|
255
|
+
*/
|
|
256
|
+
embedMedia?: MediaEmbedder;
|
|
155
257
|
/** Scratch directory for the staged databases (removed on success). */
|
|
156
258
|
workDir: string;
|
|
157
259
|
smokeQueries?: Array<{
|
|
@@ -189,6 +291,13 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
189
291
|
* and declared in `manifest.files` like every other entry.
|
|
190
292
|
*/
|
|
191
293
|
assets?: CompileAsset[];
|
|
294
|
+
/**
|
|
295
|
+
* Invalid image content (type mismatch, oversized file, or active SVG)
|
|
296
|
+
* fails by default. `warn` omits it and replaces its body references with
|
|
297
|
+
* their text. Invalid paths, duplicates, and aggregate limits still fail.
|
|
298
|
+
*/
|
|
299
|
+
invalidAssets?: 'error' | 'warn';
|
|
300
|
+
onWarning?: (message: string) => void;
|
|
192
301
|
/**
|
|
193
302
|
* Last touch before the manifest is archived — the signing seam
|
|
194
303
|
* (signatures/signing.ts `signManifest`). Must only add/replace the
|
|
@@ -233,11 +342,27 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
233
342
|
interface CompileReport {
|
|
234
343
|
manifest: KnowledgeCatalogManifest;
|
|
235
344
|
documents: number;
|
|
345
|
+
/** Every row of the shard `chunks` tables: text chunks plus media rows. */
|
|
236
346
|
chunks: number;
|
|
347
|
+
/** Media rows by modality (all zero without `embedMedia`). */
|
|
348
|
+
media: {
|
|
349
|
+
image: number;
|
|
350
|
+
video: number;
|
|
351
|
+
audio: number;
|
|
352
|
+
};
|
|
237
353
|
shards: number;
|
|
238
354
|
archiveBytes: number;
|
|
239
355
|
}
|
|
240
356
|
declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
|
|
357
|
+
/**
|
|
358
|
+
* The oldest generation that can express this catalog: 0.7, unless the
|
|
359
|
+
* embedding profile uses 0.8 vocabulary or the catalog ships audio or video.
|
|
360
|
+
* Writing the oldest one keeps catalog updates reaching every reader that
|
|
361
|
+
* can open them, instead of cutting installs off at each format bump.
|
|
362
|
+
*/
|
|
363
|
+
declare function catalogFormatVersion(profile: KnowledgeEmbeddingProfile, assets: ReadonlyArray<{
|
|
364
|
+
path: string;
|
|
365
|
+
}>, mediaRows?: number): Extract<GezkFormatVersion, '0.7' | '0.8'>;
|
|
241
366
|
|
|
242
367
|
/**
|
|
243
368
|
* Deterministic `.gezk` ZIP writer: the stored `mimetype` entry first (the
|
|
@@ -322,6 +447,16 @@ declare class CatalogOpenError extends Error {
|
|
|
322
447
|
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt';
|
|
323
448
|
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt');
|
|
324
449
|
}
|
|
450
|
+
/**
|
|
451
|
+
* A query the catalog cannot answer as asked. `dimension`: the query vector's
|
|
452
|
+
* width differs from the catalog's stored width — an embedder that ignored
|
|
453
|
+
* the profile's truncation, or a model the catalog was not built with.
|
|
454
|
+
* Callers treat it as an embedding failure and fall back to keyword search.
|
|
455
|
+
*/
|
|
456
|
+
declare class CatalogQueryError extends Error {
|
|
457
|
+
readonly reason: 'dimension';
|
|
458
|
+
constructor(message: string, reason: 'dimension');
|
|
459
|
+
}
|
|
325
460
|
interface CatalogDb {
|
|
326
461
|
db: DatabaseSync;
|
|
327
462
|
/** The `PRAGMA user_version` generation the file was written with. */
|
|
@@ -337,34 +472,31 @@ interface CatalogDb {
|
|
|
337
472
|
declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
338
473
|
|
|
339
474
|
/**
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
*
|
|
343
|
-
*
|
|
344
|
-
*
|
|
345
|
-
*
|
|
346
|
-
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
347
|
-
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
348
|
-
* the float query (`asymmetricTopK`), so no vector extension is needed to
|
|
349
|
-
* read a catalog. The catalog's own profile echo says how its bits were
|
|
350
|
-
* derived: a `centered-sign` profile's bits come from `vector − center`, so
|
|
351
|
-
* the query is centered the same way before the scan. The int8 rerank is
|
|
352
|
-
* identical for both kinds of bits.
|
|
353
|
-
*
|
|
354
|
-
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
355
|
-
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
356
|
-
* and tests call it directly.
|
|
475
|
+
* Read-only document point index. The router's ordinary SQLite latitude and
|
|
476
|
+
* longitude indexes provide conservative candidates; the portable gezk
|
|
477
|
+
* spherical predicate decides membership before ranking or pagination.
|
|
478
|
+
* The explicit Qualla adapter keeps already-published 0.6 regional archives
|
|
479
|
+
* useful without interpreting arbitrary producer metadata as coordinates.
|
|
357
480
|
*/
|
|
358
481
|
|
|
482
|
+
interface SpatialMatch {
|
|
483
|
+
distanceMeters: number;
|
|
484
|
+
matchedLocation: KnowledgeLocation;
|
|
485
|
+
}
|
|
486
|
+
interface DocumentLocationRow {
|
|
487
|
+
documentId: string;
|
|
488
|
+
location: KnowledgeLocation;
|
|
489
|
+
}
|
|
490
|
+
|
|
359
491
|
interface CatalogTopic {
|
|
360
492
|
id: string;
|
|
361
493
|
parentId: string | null;
|
|
362
494
|
name: string;
|
|
363
495
|
description: string | null;
|
|
364
496
|
sortKey: string;
|
|
365
|
-
/**
|
|
497
|
+
/** Unique documents listed directly at this topic, including shared references. */
|
|
366
498
|
documentCount: number;
|
|
367
|
-
/**
|
|
499
|
+
/** Unique documents in this topic and its descendants, deduplicated at read time. */
|
|
368
500
|
totalDocumentCount: number;
|
|
369
501
|
}
|
|
370
502
|
interface CatalogDocumentMeta {
|
|
@@ -373,7 +505,7 @@ interface CatalogDocumentMeta {
|
|
|
373
505
|
slug: string;
|
|
374
506
|
summary: string | null;
|
|
375
507
|
language: string;
|
|
376
|
-
/**
|
|
508
|
+
/** Listing placement for a scoped page; primary topic for getDocument/search/global listings. */
|
|
377
509
|
topicId: string;
|
|
378
510
|
/** Listing position among the documents of its topic; null when unordered. */
|
|
379
511
|
ordinal: number | null;
|
|
@@ -383,6 +515,7 @@ interface CatalogDocumentMeta {
|
|
|
383
515
|
attribution: Record<string, string> | null;
|
|
384
516
|
/** Producer-defined metadata (`meta_json`), opaque to the reader. */
|
|
385
517
|
meta: Record<string, unknown> | null;
|
|
518
|
+
locations?: KnowledgeLocation[];
|
|
386
519
|
}
|
|
387
520
|
interface CatalogAssetInfo {
|
|
388
521
|
/** Archive path under `assets/`. */
|
|
@@ -391,9 +524,25 @@ interface CatalogAssetInfo {
|
|
|
391
524
|
sizeBytes: number;
|
|
392
525
|
sha256: string;
|
|
393
526
|
}
|
|
527
|
+
/** A declared asset's file on disk, for streaming (extraction already verified its hash). */
|
|
528
|
+
interface CatalogAssetFile extends CatalogAssetInfo {
|
|
529
|
+
absPath: string;
|
|
530
|
+
}
|
|
394
531
|
interface CatalogAssetRead extends CatalogAssetInfo {
|
|
395
532
|
bytes: Uint8Array;
|
|
396
533
|
}
|
|
534
|
+
/** What a media row (0.8) carries beyond its text: the asset, its kind, and the span. */
|
|
535
|
+
interface CatalogChunkMedia {
|
|
536
|
+
modality: 'image' | 'video' | 'audio';
|
|
537
|
+
assetPath: string;
|
|
538
|
+
mimeType: string;
|
|
539
|
+
width?: number;
|
|
540
|
+
height?: number;
|
|
541
|
+
startMs?: number;
|
|
542
|
+
endMs?: number;
|
|
543
|
+
thumbnailAssetPath?: string;
|
|
544
|
+
attribution?: Record<string, unknown>;
|
|
545
|
+
}
|
|
397
546
|
interface CatalogChunkHit {
|
|
398
547
|
chunkUid: string;
|
|
399
548
|
documentId: string;
|
|
@@ -406,17 +555,28 @@ interface CatalogChunkHit {
|
|
|
406
555
|
cosine?: number;
|
|
407
556
|
source: 'vector' | 'fts';
|
|
408
557
|
shardId: number;
|
|
558
|
+
/** Present on a media row (index schema 5). */
|
|
559
|
+
media?: CatalogChunkMedia;
|
|
409
560
|
}
|
|
410
561
|
interface ShardInfo {
|
|
411
562
|
id: number;
|
|
412
563
|
path: string;
|
|
413
564
|
chunkCount: number;
|
|
414
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* Shards a single-catalog search adds for the pages its query names by
|
|
568
|
+
* title (`titleRouteShards`), on top of the centroid-routed budget.
|
|
569
|
+
*/
|
|
570
|
+
declare const TITLE_ROUTE_SHARDS = 1;
|
|
415
571
|
declare class CatalogHandle {
|
|
416
572
|
readonly rootDir: string;
|
|
417
573
|
private readonly router;
|
|
418
574
|
private readonly connections;
|
|
419
575
|
private readonly bitIndexes;
|
|
576
|
+
private readonly mediaIndexes;
|
|
577
|
+
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
578
|
+
private readonly chunkTermRows;
|
|
579
|
+
private readonly spatial;
|
|
420
580
|
private assetsByPath;
|
|
421
581
|
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
422
582
|
private binaryCenterCache;
|
|
@@ -443,6 +603,13 @@ declare class CatalogHandle {
|
|
|
443
603
|
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
444
604
|
*/
|
|
445
605
|
private shardBits;
|
|
606
|
+
/**
|
|
607
|
+
* Load the next shard's sign bits that are not resident yet, if they fit
|
|
608
|
+
* the resident budget. False when nothing is left to load. One shard per
|
|
609
|
+
* call, so a host can keep answering queries between shards while it warms
|
|
610
|
+
* a large catalog, and never evicts a shard it loaded a moment ago.
|
|
611
|
+
*/
|
|
612
|
+
prewarmNextShard(): boolean;
|
|
446
613
|
/**
|
|
447
614
|
* The stage-1 center the catalog's bits were derived with (a
|
|
448
615
|
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
@@ -450,6 +617,34 @@ declare class CatalogHandle {
|
|
|
450
617
|
* the statement of how THIS catalog was built.
|
|
451
618
|
*/
|
|
452
619
|
private binaryCenter;
|
|
620
|
+
/** The chunk columns this catalog's schema has. */
|
|
621
|
+
private chunkColumns;
|
|
622
|
+
/** A shard's media rows with their int8 vectors (empty before index schema 5). */
|
|
623
|
+
private shardMedia;
|
|
624
|
+
/** Media rows this catalog holds, by modality (all zero before index schema 5). */
|
|
625
|
+
mediaCounts(): {
|
|
626
|
+
image: number;
|
|
627
|
+
video: number;
|
|
628
|
+
audio: number;
|
|
629
|
+
};
|
|
630
|
+
/**
|
|
631
|
+
* The media lane: every media row of every shard scored exactly against
|
|
632
|
+
* the query (int8 rerank), keeping the best `perModality` of each kind.
|
|
633
|
+
* Images sit at a lower cosine than text in a shared space, so in the text
|
|
634
|
+
* lane's sign-bit pre-filter they would rarely survive the cut; media rows
|
|
635
|
+
* are few enough (capped per catalog) to score them all.
|
|
636
|
+
*/
|
|
637
|
+
searchMedia(queryVector: Float32Array, opts?: {
|
|
638
|
+
perModality?: number;
|
|
639
|
+
allowedDocumentIds?: ReadonlySet<string>;
|
|
640
|
+
}): CatalogChunkHit[];
|
|
641
|
+
private hitFromRow;
|
|
642
|
+
/**
|
|
643
|
+
* Refuse a query vector whose width is not the catalog's stored width,
|
|
644
|
+
* before any scan: routing and rerank would otherwise score a silent
|
|
645
|
+
* prefix or fail deep inside a shard with an untyped error.
|
|
646
|
+
*/
|
|
647
|
+
private assertQueryWidth;
|
|
453
648
|
/** The embedding dimension from the router's profile echo. */
|
|
454
649
|
private dimensions;
|
|
455
650
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
@@ -476,9 +671,26 @@ declare class CatalogHandle {
|
|
|
476
671
|
documents: CatalogDocumentMeta[];
|
|
477
672
|
total: number;
|
|
478
673
|
};
|
|
674
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
675
|
+
private sharedDocumentsPage;
|
|
676
|
+
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
677
|
+
tocIntegrity(): {
|
|
678
|
+
placements: number;
|
|
679
|
+
invalidReferences: number;
|
|
680
|
+
missingPrimary: number;
|
|
681
|
+
incorrectCounts: number;
|
|
682
|
+
duplicateReferences: number;
|
|
683
|
+
};
|
|
479
684
|
getDocument(id: string): (CatalogDocumentMeta & {
|
|
480
685
|
markdown: string;
|
|
481
686
|
}) | null;
|
|
687
|
+
/**
|
|
688
|
+
* One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
|
|
689
|
+
* shard the document's chunks live in. Null when the document or the
|
|
690
|
+
* passage is not in this catalog, which is what a fabricated or stale
|
|
691
|
+
* citation looks like.
|
|
692
|
+
*/
|
|
693
|
+
getChunk(documentId: string, chunkUid: string): Omit<CatalogChunkHit, 'cosine' | 'source'> | null;
|
|
482
694
|
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
483
695
|
documentsWithUndeclaredTopic(): number;
|
|
484
696
|
/**
|
|
@@ -503,13 +715,35 @@ declare class CatalogHandle {
|
|
|
503
715
|
private assetIndex;
|
|
504
716
|
/** Declared assets, sorted by path. */
|
|
505
717
|
assets(): CatalogAssetInfo[];
|
|
718
|
+
/**
|
|
719
|
+
* Where a declared asset lives on disk, so a large one (video, audio) can be
|
|
720
|
+
* streamed with byte ranges instead of read whole. Checks containment and
|
|
721
|
+
* the manifest size; the content hash was verified once, at extraction.
|
|
722
|
+
*/
|
|
723
|
+
assetFile(path: string): CatalogAssetFile | null;
|
|
506
724
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
507
725
|
readAsset(path: string): CatalogAssetRead | null;
|
|
726
|
+
private documentMeta;
|
|
727
|
+
documentLocations(documentId: string): KnowledgeLocation[];
|
|
728
|
+
documentLocationRows(): DocumentLocationRow[];
|
|
729
|
+
spatialIntegrity(): {
|
|
730
|
+
hasTable: boolean;
|
|
731
|
+
manifest: _bendyline_gezk.KnowledgeSpatialManifest;
|
|
732
|
+
};
|
|
733
|
+
spatialMatches(radius: KnowledgeRadius): Map<string, SpatialMatch>;
|
|
734
|
+
nearbyDocuments(radius: KnowledgeRadius, opts?: {
|
|
735
|
+
offset?: number;
|
|
736
|
+
limit?: number;
|
|
737
|
+
}): {
|
|
738
|
+
documents: Array<CatalogDocumentMeta & SpatialMatch>;
|
|
739
|
+
total: number;
|
|
740
|
+
};
|
|
741
|
+
shardsForDocuments(ids: ReadonlySet<string>): number[];
|
|
508
742
|
/**
|
|
509
743
|
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
510
744
|
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
511
745
|
*/
|
|
512
|
-
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
746
|
+
searchDocumentsFts(query: string, limit?: number, allowedDocumentIds?: ReadonlySet<string>): Array<{
|
|
513
747
|
documentId: string;
|
|
514
748
|
rank: number;
|
|
515
749
|
}>;
|
|
@@ -533,8 +767,32 @@ declare class CatalogHandle {
|
|
|
533
767
|
shardId: number;
|
|
534
768
|
score: number;
|
|
535
769
|
}>;
|
|
536
|
-
/**
|
|
537
|
-
|
|
770
|
+
/**
|
|
771
|
+
* Top-S shard ids within THIS catalog (single-catalog callers, tests).
|
|
772
|
+
* With the raw `query`, also the shard of the page it names by title
|
|
773
|
+
* (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
|
|
774
|
+
*/
|
|
775
|
+
routeShards(queryVector: Float32Array, shardBudget: number, query?: string): number[];
|
|
776
|
+
/**
|
|
777
|
+
* Title-assisted routing: the shards holding the pages `query` names by
|
|
778
|
+
* title (`namedTitleMatches`), most specific name first, at most `limit`
|
|
779
|
+
* distinct shards. Centroid routing compares a question with averages of
|
|
780
|
+
* whole shards, and e5 scores those in a band a few hundredths wide, so
|
|
781
|
+
* the shard holding the very page a question names can rank 8th of 10:
|
|
782
|
+
* "Who were the members of ABBA?" never reached the ABBA article in the
|
|
783
|
+
* Wikipedia music catalog. Its title does — the vector arm scans that
|
|
784
|
+
* shard too. Measured on the 10-shard music and film-tv catalogs, one such
|
|
785
|
+
* shard lifted end-to-end recall@8 on "What is <title>?" questions from
|
|
786
|
+
* 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
|
|
787
|
+
*
|
|
788
|
+
* `score` is the named match's specificity (fts-query.ts), comparable
|
|
789
|
+
* across catalogs closely enough for the daemon to choose which catalogs'
|
|
790
|
+
* shards to add when several name a page. Empty on any FTS error.
|
|
791
|
+
*/
|
|
792
|
+
titleRouteShards(query: string, limit: number): Array<{
|
|
793
|
+
shardId: number;
|
|
794
|
+
score: number;
|
|
795
|
+
}>;
|
|
538
796
|
/**
|
|
539
797
|
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
540
798
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
@@ -542,14 +800,19 @@ declare class CatalogHandle {
|
|
|
542
800
|
searchSemantic(queryVector: Float32Array, opts?: {
|
|
543
801
|
shardBudget?: number;
|
|
544
802
|
finalK?: number;
|
|
803
|
+
allowedDocumentIds?: ReadonlySet<string>;
|
|
804
|
+
/** The raw query text: adds the shard of the page it names (`routeShards`). */
|
|
805
|
+
query?: string;
|
|
545
806
|
}): CatalogChunkHit[];
|
|
546
807
|
/**
|
|
547
808
|
* The scan stage against an EXPLICIT shard list — what the daemon calls
|
|
548
809
|
* after global cross-catalog routing has already spent the S budget.
|
|
549
810
|
*/
|
|
550
|
-
searchShards(queryVector: Float32Array, shardIds: number[], finalK: number): CatalogChunkHit[];
|
|
811
|
+
searchShards(queryVector: Float32Array, shardIds: number[], finalK: number, allowedDocumentIds?: ReadonlySet<string>): CatalogChunkHit[];
|
|
551
812
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
552
|
-
searchChunksFts(query: string, shardIds: number[], limitPerShard?: number): CatalogChunkHit[];
|
|
813
|
+
searchChunksFts(query: string, shardIds: number[], limitPerShard?: number, allowedDocumentIds?: ReadonlySet<string>): CatalogChunkHit[];
|
|
814
|
+
/** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
|
|
815
|
+
private chunkFtsRows;
|
|
553
816
|
/**
|
|
554
817
|
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
555
818
|
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
@@ -634,8 +897,11 @@ interface AsymmetricHit {
|
|
|
634
897
|
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
635
898
|
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
636
899
|
* stored; this function does not know or care which.
|
|
900
|
+
*
|
|
901
|
+
* `excludedChunkIds` drops rows outright — media rows, which the text lane
|
|
902
|
+
* leaves to its own exact scan rather than letting them take candidate slots.
|
|
637
903
|
*/
|
|
638
|
-
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
|
|
904
|
+
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number, eligibleChunkIds?: ReadonlySet<number>, excludedChunkIds?: ReadonlySet<number>): AsymmetricHit[];
|
|
639
905
|
|
|
640
906
|
/**
|
|
641
907
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
@@ -691,8 +957,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
691
957
|
* bytes.
|
|
692
958
|
*/
|
|
693
959
|
|
|
694
|
-
/**
|
|
695
|
-
declare const PARQUET_EXPORT_VERSION =
|
|
960
|
+
/** 4: document-locations companion, including the explicit legacy Qualla adapter. */
|
|
961
|
+
declare const PARQUET_EXPORT_VERSION = 4;
|
|
696
962
|
type ParquetExportSource = {
|
|
697
963
|
archivePath: string;
|
|
698
964
|
} | {
|
|
@@ -716,7 +982,7 @@ type ParquetExportProgress = {
|
|
|
716
982
|
phase: 'write';
|
|
717
983
|
file: string;
|
|
718
984
|
};
|
|
719
|
-
type ParquetTable = 'documents' | 'chunks' | 'topics';
|
|
985
|
+
type ParquetTable = 'documents' | 'chunks' | 'topics' | 'document_locations';
|
|
720
986
|
interface ParquetExportFile {
|
|
721
987
|
/** Relative to `outDir`, forward slashes. */
|
|
722
988
|
path: string;
|
|
@@ -792,6 +1058,7 @@ declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
|
792
1058
|
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
793
1059
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
794
1060
|
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
1061
|
+
declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
|
|
795
1062
|
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
796
1063
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
797
1064
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
@@ -805,6 +1072,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
805
1072
|
revision: string;
|
|
806
1073
|
onnxFile?: string | undefined;
|
|
807
1074
|
onnxDigest?: string | undefined;
|
|
1075
|
+
files?: {
|
|
1076
|
+
path: string;
|
|
1077
|
+
digest?: string | undefined;
|
|
1078
|
+
}[] | undefined;
|
|
808
1079
|
};
|
|
809
1080
|
tokenizer: {
|
|
810
1081
|
kind: string;
|
|
@@ -834,6 +1105,43 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
834
1105
|
center?: number[] | undefined;
|
|
835
1106
|
};
|
|
836
1107
|
};
|
|
1108
|
+
truncation?: {
|
|
1109
|
+
method: "prefix";
|
|
1110
|
+
sourceDimensions: number;
|
|
1111
|
+
} | undefined;
|
|
1112
|
+
media?: {
|
|
1113
|
+
image?: {
|
|
1114
|
+
encoder: {
|
|
1115
|
+
onnxFile: string;
|
|
1116
|
+
onnxDigest?: string | undefined;
|
|
1117
|
+
files?: {
|
|
1118
|
+
path: string;
|
|
1119
|
+
digest?: string | undefined;
|
|
1120
|
+
}[] | undefined;
|
|
1121
|
+
};
|
|
1122
|
+
tokenBudget: number;
|
|
1123
|
+
resample: "bilinear" | "bicubic";
|
|
1124
|
+
alpha: "composite-white";
|
|
1125
|
+
} | undefined;
|
|
1126
|
+
video?: {
|
|
1127
|
+
framesPerSecond: number;
|
|
1128
|
+
maxFrames: number;
|
|
1129
|
+
tokenBudgetPerFrame: number;
|
|
1130
|
+
} | undefined;
|
|
1131
|
+
audio?: {
|
|
1132
|
+
encoder: {
|
|
1133
|
+
onnxFile: string;
|
|
1134
|
+
onnxDigest?: string | undefined;
|
|
1135
|
+
files?: {
|
|
1136
|
+
path: string;
|
|
1137
|
+
digest?: string | undefined;
|
|
1138
|
+
}[] | undefined;
|
|
1139
|
+
};
|
|
1140
|
+
sampleRate: number;
|
|
1141
|
+
channels: 1;
|
|
1142
|
+
maxWindowMs: number;
|
|
1143
|
+
} | undefined;
|
|
1144
|
+
} | undefined;
|
|
837
1145
|
};
|
|
838
1146
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
839
1147
|
declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
@@ -843,6 +1151,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
843
1151
|
revision: string;
|
|
844
1152
|
onnxFile?: string | undefined;
|
|
845
1153
|
onnxDigest?: string | undefined;
|
|
1154
|
+
files?: {
|
|
1155
|
+
path: string;
|
|
1156
|
+
digest?: string | undefined;
|
|
1157
|
+
}[] | undefined;
|
|
846
1158
|
};
|
|
847
1159
|
tokenizer: {
|
|
848
1160
|
kind: string;
|
|
@@ -872,6 +1184,43 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
872
1184
|
center?: number[] | undefined;
|
|
873
1185
|
};
|
|
874
1186
|
};
|
|
1187
|
+
truncation?: {
|
|
1188
|
+
method: "prefix";
|
|
1189
|
+
sourceDimensions: number;
|
|
1190
|
+
} | undefined;
|
|
1191
|
+
media?: {
|
|
1192
|
+
image?: {
|
|
1193
|
+
encoder: {
|
|
1194
|
+
onnxFile: string;
|
|
1195
|
+
onnxDigest?: string | undefined;
|
|
1196
|
+
files?: {
|
|
1197
|
+
path: string;
|
|
1198
|
+
digest?: string | undefined;
|
|
1199
|
+
}[] | undefined;
|
|
1200
|
+
};
|
|
1201
|
+
tokenBudget: number;
|
|
1202
|
+
resample: "bilinear" | "bicubic";
|
|
1203
|
+
alpha: "composite-white";
|
|
1204
|
+
} | undefined;
|
|
1205
|
+
video?: {
|
|
1206
|
+
framesPerSecond: number;
|
|
1207
|
+
maxFrames: number;
|
|
1208
|
+
tokenBudgetPerFrame: number;
|
|
1209
|
+
} | undefined;
|
|
1210
|
+
audio?: {
|
|
1211
|
+
encoder: {
|
|
1212
|
+
onnxFile: string;
|
|
1213
|
+
onnxDigest?: string | undefined;
|
|
1214
|
+
files?: {
|
|
1215
|
+
path: string;
|
|
1216
|
+
digest?: string | undefined;
|
|
1217
|
+
}[] | undefined;
|
|
1218
|
+
};
|
|
1219
|
+
sampleRate: number;
|
|
1220
|
+
channels: 1;
|
|
1221
|
+
maxWindowMs: number;
|
|
1222
|
+
} | undefined;
|
|
1223
|
+
} | undefined;
|
|
875
1224
|
};
|
|
876
1225
|
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
877
1226
|
declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
@@ -896,7 +1245,11 @@ declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
|
896
1245
|
* vector space rather than silently producing slightly different vectors.
|
|
897
1246
|
*/
|
|
898
1247
|
|
|
899
|
-
|
|
1248
|
+
/**
|
|
1249
|
+
* `file`: one of the profile's further pinned files (weight sidecars,
|
|
1250
|
+
* configs); `media`: a media encoder's graph or one of its files.
|
|
1251
|
+
*/
|
|
1252
|
+
type ArtifactRole = 'onnx' | 'tokenizer' | 'file' | 'media';
|
|
900
1253
|
interface ArtifactCheck {
|
|
901
1254
|
role: ArtifactRole;
|
|
902
1255
|
/** Repo-relative file the profile pins. */
|
|
@@ -921,6 +1274,20 @@ declare class EmbedderArtifactError extends Error {
|
|
|
921
1274
|
}
|
|
922
1275
|
/** The transformers.js FileCache key for one repo file. */
|
|
923
1276
|
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
1277
|
+
/** The pinned files of a profile's media encoders for the given modalities (video reuses image). */
|
|
1278
|
+
declare function mediaEncoderPins(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>): Array<{
|
|
1279
|
+
file: string;
|
|
1280
|
+
expected: string;
|
|
1281
|
+
}>;
|
|
1282
|
+
/**
|
|
1283
|
+
* Hash a profile's media encoder files in a transformers.js cache and
|
|
1284
|
+
* compare, with the same contract as {@link verifyProfileArtifacts}: a media
|
|
1285
|
+
* vector is only as trustworthy as the bytes that produced it.
|
|
1286
|
+
*/
|
|
1287
|
+
declare function verifyMediaArtifacts(profile: KnowledgeEmbeddingProfile, modalities: ReadonlyArray<'image' | 'video' | 'audio'>, opts: {
|
|
1288
|
+
cacheDir: string;
|
|
1289
|
+
revision?: string;
|
|
1290
|
+
}): Promise<VerifiedArtifacts>;
|
|
924
1291
|
/**
|
|
925
1292
|
* Hash the profile's pinned files in a transformers.js cache and compare.
|
|
926
1293
|
* `revision` names the cache key the files were fetched under; it defaults
|
|
@@ -956,9 +1323,12 @@ interface ProfileEmbedder {
|
|
|
956
1323
|
readonly profile: KnowledgeEmbeddingProfile;
|
|
957
1324
|
/** What the loaded model files were checked against after loading. */
|
|
958
1325
|
readonly verification: VerifiedArtifacts;
|
|
959
|
-
/**
|
|
1326
|
+
/**
|
|
1327
|
+
* Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's
|
|
1328
|
+
* contract) and returns the model's own width; the compiler projects.
|
|
1329
|
+
*/
|
|
960
1330
|
embed(texts: string[]): Promise<number[][]>;
|
|
961
|
-
/** Query embed — applies the profile's
|
|
1331
|
+
/** Query embed — applies the queryInstruction and the profile's projection itself. */
|
|
962
1332
|
embedQuery(text: string): Promise<Float32Array>;
|
|
963
1333
|
/** Profile-tokenizer token count (sync once loaded — chunking's contract). */
|
|
964
1334
|
countTokens(text: string): number;
|
|
@@ -969,7 +1339,7 @@ interface ProfileEmbedder {
|
|
|
969
1339
|
* as its optional peer dependency (a test holds the two together). Error
|
|
970
1340
|
* messages quote it so the install command they print is exact.
|
|
971
1341
|
*/
|
|
972
|
-
declare const TRANSFORMERS_PEER_RANGE = "^3.
|
|
1342
|
+
declare const TRANSFORMERS_PEER_RANGE = "^4.3.1";
|
|
973
1343
|
declare class EmbedderUnavailableError extends Error {
|
|
974
1344
|
readonly isActionable = true;
|
|
975
1345
|
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
@@ -984,12 +1354,41 @@ interface TransformersModule {
|
|
|
984
1354
|
AutoTokenizer: {
|
|
985
1355
|
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerFn>;
|
|
986
1356
|
};
|
|
1357
|
+
/**
|
|
1358
|
+
* Read a repo's config before choosing a loader. Optional so a test fake
|
|
1359
|
+
* that only knows the pipeline keeps working; without it every profile
|
|
1360
|
+
* loads through `pipeline`.
|
|
1361
|
+
*/
|
|
1362
|
+
AutoConfig?: {
|
|
1363
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
|
1364
|
+
};
|
|
1365
|
+
/** Multimodal encoders: loads the text session alone (see `isMultimodalEncoderConfig`). */
|
|
1366
|
+
AutoModel?: {
|
|
1367
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<EncoderModelFn>;
|
|
1368
|
+
};
|
|
987
1369
|
env: {
|
|
988
1370
|
cacheDir?: string;
|
|
989
1371
|
useFSCache?: boolean;
|
|
990
1372
|
allowRemoteModels?: boolean;
|
|
991
1373
|
};
|
|
992
1374
|
}
|
|
1375
|
+
/** A tensor as this module reads it: flat data plus its shape. */
|
|
1376
|
+
interface TensorLike {
|
|
1377
|
+
data: ArrayLike<number>;
|
|
1378
|
+
dims: number[];
|
|
1379
|
+
}
|
|
1380
|
+
/** A tokenizer called on a batch, as the encoder path feeds a model directly. */
|
|
1381
|
+
type BatchTokenizerFn = TokenizerFn & ((texts: string[], options: {
|
|
1382
|
+
padding: boolean;
|
|
1383
|
+
truncation: boolean;
|
|
1384
|
+
max_length: number;
|
|
1385
|
+
}) => {
|
|
1386
|
+
attention_mask?: TensorLike;
|
|
1387
|
+
});
|
|
1388
|
+
/** A text-session encoder: tokenized inputs in, named tensors out. */
|
|
1389
|
+
type EncoderModelFn = ((inputs: unknown) => Promise<Record<string, TensorLike>>) & {
|
|
1390
|
+
dispose?: () => Promise<unknown>;
|
|
1391
|
+
};
|
|
993
1392
|
type PipelineFn = ((texts: string[], options: {
|
|
994
1393
|
pooling: string;
|
|
995
1394
|
normalize: boolean;
|
|
@@ -1026,8 +1425,32 @@ declare function transformersGraphOptions(onnxFile: string): {
|
|
|
1026
1425
|
subfolder: string;
|
|
1027
1426
|
model_file_name: string;
|
|
1028
1427
|
} | null;
|
|
1428
|
+
/**
|
|
1429
|
+
* A multimodal embedding model (one space for text, images, audio) carries
|
|
1430
|
+
* its media encoders' configs beside the text model's. Through the
|
|
1431
|
+
* `feature-extraction` pipeline it would load every encoder; the text path
|
|
1432
|
+
* needs only the text session, which `AutoModel` loads once those configs
|
|
1433
|
+
* are cleared.
|
|
1434
|
+
*/
|
|
1435
|
+
declare function isMultimodalEncoderConfig(config: Record<string, unknown>): boolean;
|
|
1436
|
+
/** How many external-data sidecars the profile pins beside its graph (`<graph>_data`, `<graph>_data_1`, …). */
|
|
1437
|
+
declare function pinnedExternalDataChunks(profile: KnowledgeEmbeddingProfile): number;
|
|
1438
|
+
/**
|
|
1439
|
+
* Make a load of a profile with weight sidecars safe from transformers.js's
|
|
1440
|
+
* sidecar hang (see pinned-files.ts): a local-only load fails fast, naming
|
|
1441
|
+
* the files that are missing; a networked load downloads the sidecars first,
|
|
1442
|
+
* hash-verified. A profile without sidecars loads exactly as before.
|
|
1443
|
+
*/
|
|
1444
|
+
declare function preparePinnedLoad(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1445
|
+
cacheDir: string;
|
|
1446
|
+
localFilesOnly?: boolean;
|
|
1447
|
+
modalities?: ReadonlyArray<'image' | 'video' | 'audio'>;
|
|
1448
|
+
fetchImpl?: typeof fetch;
|
|
1449
|
+
}): Promise<void>;
|
|
1029
1450
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
1030
1451
|
cacheDir?: string;
|
|
1452
|
+
/** Read cached model files only; never initiate a download. */
|
|
1453
|
+
localFilesOnly?: boolean;
|
|
1031
1454
|
/**
|
|
1032
1455
|
* onnxruntime session options (e.g. `{ intraOpNumThreads }`), passed
|
|
1033
1456
|
* through to the pipeline. Worker pools use this to divide cores between
|
|
@@ -1042,6 +1465,47 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
1042
1465
|
transformers?: TransformersModule;
|
|
1043
1466
|
}): Promise<ProfileEmbedder>;
|
|
1044
1467
|
|
|
1468
|
+
/**
|
|
1469
|
+
* The pinned files a profile load reads, and getting the weight sidecars into
|
|
1470
|
+
* the cache before transformers.js goes looking for them.
|
|
1471
|
+
*
|
|
1472
|
+
* transformers.js 4.3.1 fetches `.onnx_data` sidecars inside
|
|
1473
|
+
* `new Promise(async (resolve) => { await getModelFile(…) })`. When the file
|
|
1474
|
+
* is missing (a `local_files_only` load of an incomplete install) or its
|
|
1475
|
+
* download fails, the rejection escapes the executor as an unhandled
|
|
1476
|
+
* rejection and the load's promise never settles: a hang, not an error. So a
|
|
1477
|
+
* local-only load checks the cache first, and a networked load downloads the
|
|
1478
|
+
* sidecars itself, hash-verified, before handing over.
|
|
1479
|
+
*/
|
|
1480
|
+
|
|
1481
|
+
interface PinnedLoadFile {
|
|
1482
|
+
path: string;
|
|
1483
|
+
/** `sha256:<hex>` when the profile pins it. */
|
|
1484
|
+
digest?: string;
|
|
1485
|
+
}
|
|
1486
|
+
type MediaModality = 'image' | 'video' | 'audio';
|
|
1487
|
+
/** `onnx/model.onnx_data`, `onnx/model.onnx_data_1`: weights stored beside a graph. */
|
|
1488
|
+
declare function isExternalDataFile(path: string): boolean;
|
|
1489
|
+
/** Every file the profile pins for its text session and, with `modalities`, its media encoders. */
|
|
1490
|
+
declare function pinnedLoadFiles(profile: KnowledgeEmbeddingProfile, modalities?: readonly MediaModality[]): PinnedLoadFile[];
|
|
1491
|
+
/** The pinned files of a load that are not in the transformers.js cache. */
|
|
1492
|
+
declare function missingPinnedFiles(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1493
|
+
cacheDir: string;
|
|
1494
|
+
modalities?: readonly MediaModality[];
|
|
1495
|
+
revision?: string;
|
|
1496
|
+
}): Promise<string[]>;
|
|
1497
|
+
/**
|
|
1498
|
+
* Download the profile's missing weight sidecars into the cache, each to a
|
|
1499
|
+
* `.partial`, hashed against its pin and renamed, so the load that follows
|
|
1500
|
+
* finds them cached. Throws on an HTTP error or a digest mismatch.
|
|
1501
|
+
*/
|
|
1502
|
+
declare function prefetchExternalData(profile: KnowledgeEmbeddingProfile, opts: {
|
|
1503
|
+
cacheDir: string;
|
|
1504
|
+
modalities?: readonly MediaModality[];
|
|
1505
|
+
revision?: string;
|
|
1506
|
+
fetchImpl?: typeof fetch;
|
|
1507
|
+
}): Promise<string[]>;
|
|
1508
|
+
|
|
1045
1509
|
/**
|
|
1046
1510
|
* Tables of contents that documentation trees already carry. Each parser
|
|
1047
1511
|
* turns one tool's outline into the same shape — a tree of topics holding
|
|
@@ -1198,6 +1662,8 @@ interface LoadMarkdownCatalogOptions {
|
|
|
1198
1662
|
};
|
|
1199
1663
|
/** What to do with an image link whose file is missing (default `error`; `warn` for docfx). */
|
|
1200
1664
|
missingAssets?: 'error' | 'warn';
|
|
1665
|
+
/** Keep image alt text without resolving or bundling image files. */
|
|
1666
|
+
skipImages?: boolean;
|
|
1201
1667
|
onWarning?: (message: string) => void;
|
|
1202
1668
|
}
|
|
1203
1669
|
/** The docs directory an `mkdocs.yml` names (`docs_dir`, default `docs`), absolute. */
|
|
@@ -1297,4 +1763,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1297
1763
|
|
|
1298
1764
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1299
1765
|
|
|
1300
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1766
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|