@bendyline/gezel-knowledge 1.0.7 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -17
- package/dist/index.d.ts +606 -161
- package/dist/index.js +2389 -508
- package/package.json +6 -4
package/dist/index.d.ts
CHANGED
|
@@ -1,21 +1,14 @@
|
|
|
1
|
-
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/
|
|
2
|
-
export { CatalogDocument,
|
|
1
|
+
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
3
|
+
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
|
+
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
3
5
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
4
6
|
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
-
declare const GEZK_INDEX_SCHEMA_VERSION = 1;
|
|
8
|
-
/** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
|
|
9
|
-
declare const GEZK_APPLICATION_ID = 1195727435;
|
|
10
|
-
declare const ROUTER_DB_PATH = "index/router.db";
|
|
11
|
-
declare const MANIFEST_PATH = "manifest.json";
|
|
7
|
+
/** Toolchain knobs. Format-level constants come from @bendyline/gezk. */
|
|
8
|
+
|
|
12
9
|
/** Compiler shard sizing (§3.1). */
|
|
13
10
|
declare const SHARD_TARGET_CHUNKS = 200000;
|
|
14
11
|
declare const SHARD_MAX_CHUNKS = 250000;
|
|
15
|
-
/** Bodies below this ship uncompressed (`body_codec = 'none'`). */
|
|
16
|
-
declare const BODY_CODEC_MIN_BYTES = 512;
|
|
17
|
-
/** Maximum decompressed Markdown body returned from one catalog document. */
|
|
18
|
-
declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
|
|
19
12
|
/** Routing (§3.3–3.4). */
|
|
20
13
|
declare const CENTROID_CHUNKS_PER = 12500;
|
|
21
14
|
declare const CENTROID_MAX_PER_SHARD = 32;
|
|
@@ -26,58 +19,17 @@ declare const ROUTE_SHARDS_EXPLICIT = 6;
|
|
|
26
19
|
/** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
|
|
27
20
|
declare const RERANK_FINAL_K = 24;
|
|
28
21
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
29
|
-
/** Fixed ZIP entry timestamp for deterministic archives (§11). */
|
|
30
|
-
declare const ZIP_FIXED_MTIME: Date;
|
|
31
|
-
|
|
32
|
-
/** Frozen v1 DDL (gezk-format-v1.md §6). Readers never migrate these. */
|
|
33
|
-
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
34
|
-
/** Per-shard tables. The vec0 table is created separately (needs sqlite-vec). */
|
|
35
|
-
declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
|
|
36
|
-
/** vec0 table for the hamming stage; `dim` comes from the embedding profile. */
|
|
37
|
-
declare function vecChunksDdl(dim: number): string;
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Stable chunk id (gezk-format-v1.md §5): lowercase hex of the FIRST 16
|
|
41
|
-
* bytes of SHA-256(docId || 0x00 || decimal-ordinal || 0x00 ||
|
|
42
|
-
* SHA-256(chunkText) raw 32 bytes). Content-derived so identical inputs on
|
|
43
|
-
* any toolchain produce identical ids — the anchor of both reproducibility
|
|
44
|
-
* and the `#chunk=` citation fragment.
|
|
45
|
-
*/
|
|
46
|
-
declare function chunkUid(documentId: string, ordinal: number, chunkText: string): string;
|
|
47
|
-
/** Full SHA-256 hex of a chunk's text (`chunks.content_hash`). */
|
|
48
|
-
declare function chunkContentHash(chunkText: string): string;
|
|
49
|
-
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
50
|
-
declare function documentSlug(title: string): string;
|
|
51
|
-
|
|
52
|
-
/**
|
|
53
|
-
* Frozen quantization formulas (gezk-format-v1.md §2). Deliberately computed
|
|
54
|
-
* here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
|
|
55
|
-
* functions, so extension-internal semantics can never drift the format:
|
|
56
|
-
* both the compiler (packing) and the reader (query packing + rerank) call
|
|
57
|
-
* exactly these functions.
|
|
58
|
-
*/
|
|
59
|
-
/** int8: symmetric linear, scale 127, −128 never produced. */
|
|
60
|
-
declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
|
|
61
|
-
/** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
|
|
62
|
-
declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
|
|
63
|
-
/**
|
|
64
|
-
* Rerank score: dot(float32 unit query, dequantized int8 passage). Passages
|
|
65
|
-
* were unit vectors, so this approximates cosine; the query is not quantized.
|
|
66
|
-
*/
|
|
67
|
-
declare function rerankScore(query: ArrayLike<number>, passageInt8: Int8Array): number;
|
|
68
|
-
/** L2-normalize in place-free fashion; throws on a degenerate vector. */
|
|
69
|
-
declare function l2Normalize(vector: ArrayLike<number>): Float32Array;
|
|
70
22
|
|
|
71
23
|
/**
|
|
72
24
|
* The canonical markdown chunker, parameterized over the two frozen
|
|
73
|
-
* profiles (
|
|
25
|
+
* profiles (the gezk spec §7):
|
|
74
26
|
*
|
|
75
|
-
* - `
|
|
27
|
+
* - `markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
|
|
76
28
|
* algorithm lifted from the project indexer
|
|
77
29
|
* (packages/service/src/index-store/docs.ts `chunkMarkdown` /
|
|
78
30
|
* `chunkLineRange`), so the service can later re-point here with zero
|
|
79
31
|
* behavior change.
|
|
80
|
-
* - `
|
|
32
|
+
* - `markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
|
|
81
33
|
* profile. Token counting is injected (`countTokens` — the embedding
|
|
82
34
|
* profile's own tokenizer), packing is per-line with sentence-level
|
|
83
35
|
* splitting for oversized lines, and every chunk carries a structural
|
|
@@ -116,7 +68,7 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
|
|
|
116
68
|
declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
|
|
117
69
|
|
|
118
70
|
/**
|
|
119
|
-
* `.gezk` compiler (gezk
|
|
71
|
+
* `.gezk` compiler (gezk 0.6; spec in bendyline/gezk). One owner of normalization, chunk
|
|
120
72
|
* ids, embedding-profile conformance, SQLite schema, shard assignment,
|
|
121
73
|
* centroid routing, and deterministic archive layout — the CLI's Markdown
|
|
122
74
|
* adapter and Qualla's Wikipedia adapter both feed this API.
|
|
@@ -141,6 +93,13 @@ interface CompileTopic {
|
|
|
141
93
|
description?: string;
|
|
142
94
|
sortKey?: string;
|
|
143
95
|
}
|
|
96
|
+
interface CompileAsset {
|
|
97
|
+
/** Archive path, `assets/…`, matching the format's asset path grammar. */
|
|
98
|
+
path: string;
|
|
99
|
+
/** Exactly one of `absPath` / `content`. */
|
|
100
|
+
absPath?: string;
|
|
101
|
+
content?: Buffer;
|
|
102
|
+
}
|
|
144
103
|
interface CompileKnowledgeCatalogOptions {
|
|
145
104
|
catalog: {
|
|
146
105
|
id: string;
|
|
@@ -157,6 +116,7 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
157
116
|
createdAt: string;
|
|
158
117
|
license: {
|
|
159
118
|
name: string;
|
|
119
|
+
spdx?: string;
|
|
160
120
|
noticePath?: string;
|
|
161
121
|
attributionRequired: boolean;
|
|
162
122
|
};
|
|
@@ -202,9 +162,22 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
202
162
|
* know in advance which titles win the BM25 rank race.
|
|
203
163
|
*/
|
|
204
164
|
smokeQueryPolicy?: 'require' | 'select';
|
|
165
|
+
/** Provenance; defaults to this package's identity. */
|
|
205
166
|
toolchain?: KnowledgeCatalogManifest['toolchain'];
|
|
206
|
-
/**
|
|
167
|
+
/**
|
|
168
|
+
* Extra archive files (path → utf8 content). `README.md` and
|
|
169
|
+
* `LICENSES/catalog.txt` are always shipped: minimal ones are generated
|
|
170
|
+
* from the catalog block unless supplied here. Paths under `assets/` are
|
|
171
|
+
* refused here — images go through `assets`.
|
|
172
|
+
*/
|
|
207
173
|
extraFiles?: Record<string, string>;
|
|
174
|
+
/**
|
|
175
|
+
* Images shipped under `assets/` for document bodies to reference by
|
|
176
|
+
* archive path. Each is validated against the format's asset rules
|
|
177
|
+
* (path grammar, size, leading bytes matching the extension, inert SVG)
|
|
178
|
+
* and declared in `manifest.files` like every other entry.
|
|
179
|
+
*/
|
|
180
|
+
assets?: CompileAsset[];
|
|
208
181
|
/**
|
|
209
182
|
* Last touch before the manifest is archived — the signing seam
|
|
210
183
|
* (signatures/signing.ts `signManifest`). Must only add/replace the
|
|
@@ -219,7 +192,7 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
219
192
|
}) => void;
|
|
220
193
|
embedBatchSize?: number;
|
|
221
194
|
/**
|
|
222
|
-
* TEST/EVAL override of the 200k shard target (gezk
|
|
195
|
+
* TEST/EVAL override of the 200k shard target (the gezk spec §8.1) —
|
|
223
196
|
* lets a small corpus build multi-shard so routing recall can be measured
|
|
224
197
|
* against a scan-all control without a 2M-document corpus. Production
|
|
225
198
|
* builds must not set it: the constant IS the format's scale design.
|
|
@@ -236,11 +209,13 @@ interface CompileReport {
|
|
|
236
209
|
declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
|
|
237
210
|
|
|
238
211
|
/**
|
|
239
|
-
* Deterministic `.gezk` ZIP writer
|
|
240
|
-
* by path bytes, fixed timestamp
|
|
241
|
-
* names, ZIP64 as needed (yazl
|
|
242
|
-
*
|
|
243
|
-
*
|
|
212
|
+
* Deterministic `.gezk` ZIP writer: the stored `mimetype` entry first (the
|
|
213
|
+
* container's magic), then entries sorted by path bytes, fixed timestamp
|
|
214
|
+
* (2000-01-01T00:00:00Z), mode 0644, UTF-8 names, ZIP64 as needed (yazl
|
|
215
|
+
* handles the transition). Every entry is STORED, not deflated: SQLite pages
|
|
216
|
+
* carrying brotli bodies compress little, byte identity then no longer
|
|
217
|
+
* depends on the zlib build, and content-addressed hosting (Hugging Face
|
|
218
|
+
* Xet) can deduplicate unchanged pages across releases.
|
|
244
219
|
*/
|
|
245
220
|
interface ArchiveEntry {
|
|
246
221
|
/** Forward-slashed archive path. */
|
|
@@ -270,8 +245,8 @@ declare const GEZK_ARCHIVE_LIMITS: {
|
|
|
270
245
|
readonly maxCompressionRatio: 400;
|
|
271
246
|
};
|
|
272
247
|
declare class GezkArchiveError extends Error {
|
|
273
|
-
readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io';
|
|
274
|
-
constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io');
|
|
248
|
+
readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io';
|
|
249
|
+
constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io');
|
|
275
250
|
}
|
|
276
251
|
interface GezkArchiveInspection {
|
|
277
252
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -308,43 +283,54 @@ type DatabaseSync = DatabaseSync$1;
|
|
|
308
283
|
* service's `openIndexDatabase`, whose behaviors are all wrong for a signed
|
|
309
284
|
* publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
|
|
310
285
|
* and quarantine-RENAMES a corrupt database. A catalog reader must fail
|
|
311
|
-
* with a typed reason and never touch the bytes.
|
|
286
|
+
* with a typed reason and never touch the bytes. No extension is loaded:
|
|
287
|
+
* every table in a catalog is plain SQLite (plus FTS5).
|
|
312
288
|
*/
|
|
313
289
|
|
|
314
290
|
declare class CatalogOpenError extends Error {
|
|
315
|
-
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | '
|
|
316
|
-
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | '
|
|
291
|
+
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt';
|
|
292
|
+
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt');
|
|
317
293
|
}
|
|
318
294
|
interface CatalogDb {
|
|
319
295
|
db: DatabaseSync;
|
|
296
|
+
/** The `PRAGMA user_version` generation the file was written with. */
|
|
297
|
+
schemaVersion: GezkIndexSchemaVersion;
|
|
320
298
|
close(): void;
|
|
321
299
|
}
|
|
322
300
|
/**
|
|
323
|
-
* Open one catalog SQLite file read-only + immutable
|
|
324
|
-
*
|
|
325
|
-
*
|
|
326
|
-
*
|
|
301
|
+
* Open one catalog SQLite file read-only + immutable and verify the format
|
|
302
|
+
* stamps. `immutable=1` promises SQLite the file cannot change while open —
|
|
303
|
+
* no locking, no journal probing — which is true for a published catalog
|
|
304
|
+
* version and is what makes concurrent readers free.
|
|
327
305
|
*/
|
|
328
306
|
declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
329
307
|
|
|
330
308
|
/**
|
|
331
309
|
* One mounted catalog: the router database plus lazily opened shards, and
|
|
332
|
-
* the two-stage query flow (gezk
|
|
310
|
+
* the two-stage query flow (the gezk spec §8 and §9):
|
|
333
311
|
*
|
|
334
312
|
* embed query (caller) → centroid routing → per shard: bit-hamming KNN
|
|
335
313
|
* top-K → int8 rerank → fuse.
|
|
336
314
|
*
|
|
315
|
+
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
316
|
+
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
|
|
317
|
+
* popcount, so no vector extension is needed to read a catalog.
|
|
318
|
+
*
|
|
337
319
|
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
338
320
|
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
339
321
|
* and tests call it directly.
|
|
340
322
|
*/
|
|
323
|
+
|
|
341
324
|
interface CatalogTopic {
|
|
342
325
|
id: string;
|
|
343
326
|
parentId: string | null;
|
|
344
327
|
name: string;
|
|
345
328
|
description: string | null;
|
|
346
329
|
sortKey: string;
|
|
330
|
+
/** Documents filed directly at this topic (`topics.document_count`). */
|
|
347
331
|
documentCount: number;
|
|
332
|
+
/** Direct plus every descendant topic's documents (computed at read time). */
|
|
333
|
+
totalDocumentCount: number;
|
|
348
334
|
}
|
|
349
335
|
interface CatalogDocumentMeta {
|
|
350
336
|
id: string;
|
|
@@ -352,11 +338,26 @@ interface CatalogDocumentMeta {
|
|
|
352
338
|
slug: string;
|
|
353
339
|
summary: string | null;
|
|
354
340
|
language: string;
|
|
341
|
+
/** The topic the document is filed at — the leaf of its path on 0.6, the root on 0.5. */
|
|
355
342
|
topicId: string;
|
|
343
|
+
/** Listing position among the documents of its topic; null when unordered. */
|
|
344
|
+
ordinal: number | null;
|
|
356
345
|
sourceUrl: string | null;
|
|
357
346
|
sourceRevision: string | null;
|
|
358
347
|
sourceUpdatedAt: string | null;
|
|
359
348
|
attribution: Record<string, string> | null;
|
|
349
|
+
/** Producer-defined metadata (`meta_json`), opaque to the reader. */
|
|
350
|
+
meta: Record<string, unknown> | null;
|
|
351
|
+
}
|
|
352
|
+
interface CatalogAssetInfo {
|
|
353
|
+
/** Archive path under `assets/`. */
|
|
354
|
+
path: string;
|
|
355
|
+
contentType: string;
|
|
356
|
+
sizeBytes: number;
|
|
357
|
+
sha256: string;
|
|
358
|
+
}
|
|
359
|
+
interface CatalogAssetRead extends CatalogAssetInfo {
|
|
360
|
+
bytes: Uint8Array;
|
|
360
361
|
}
|
|
361
362
|
interface CatalogChunkHit {
|
|
362
363
|
chunkUid: string;
|
|
@@ -380,8 +381,12 @@ declare class CatalogHandle {
|
|
|
380
381
|
readonly rootDir: string;
|
|
381
382
|
private readonly router;
|
|
382
383
|
private readonly connections;
|
|
384
|
+
private readonly bitIndexes;
|
|
385
|
+
private assetsByPath;
|
|
383
386
|
readonly meta: Record<string, string>;
|
|
384
387
|
readonly shards: ShardInfo[];
|
|
388
|
+
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
389
|
+
readonly schemaVersion: GezkIndexSchemaVersion;
|
|
385
390
|
private constructor();
|
|
386
391
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
387
392
|
static open(rootDir: string): CatalogHandle;
|
|
@@ -395,13 +400,34 @@ declare class CatalogHandle {
|
|
|
395
400
|
unknownCodecs: number;
|
|
396
401
|
};
|
|
397
402
|
private shardDb;
|
|
403
|
+
/**
|
|
404
|
+
* A shard's sign-bit rows as one contiguous array, loaded on first use and
|
|
405
|
+
* kept while the resident budget allows. Chunk ids must be dense from 1 —
|
|
406
|
+
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
407
|
+
*/
|
|
408
|
+
private shardBits;
|
|
409
|
+
/** The embedding dimension from the router's profile echo. */
|
|
410
|
+
private dimensions;
|
|
398
411
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
399
412
|
resolveCatalogPath(path: string): string;
|
|
413
|
+
/** The format version the router was written with (`meta.format_version`). */
|
|
414
|
+
get formatVersion(): string;
|
|
415
|
+
private documentColumns;
|
|
416
|
+
private documentOrder;
|
|
417
|
+
/**
|
|
418
|
+
* Every topic with its direct count and a rollup over its subtree. On a
|
|
419
|
+
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
420
|
+
*/
|
|
400
421
|
topics(): CatalogTopic[];
|
|
422
|
+
/**
|
|
423
|
+
* A page of documents, by default including those filed under the topic's
|
|
424
|
+
* descendants (`descendants: false` lists only the topic's own).
|
|
425
|
+
*/
|
|
401
426
|
documentsPage(opts?: {
|
|
402
427
|
topicId?: string;
|
|
403
428
|
offset?: number;
|
|
404
429
|
limit?: number;
|
|
430
|
+
descendants?: boolean;
|
|
405
431
|
}): {
|
|
406
432
|
documents: CatalogDocumentMeta[];
|
|
407
433
|
total: number;
|
|
@@ -409,6 +435,32 @@ declare class CatalogHandle {
|
|
|
409
435
|
getDocument(id: string): (CatalogDocumentMeta & {
|
|
410
436
|
markdown: string;
|
|
411
437
|
}) | null;
|
|
438
|
+
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
439
|
+
documentsWithUndeclaredTopic(): number;
|
|
440
|
+
/**
|
|
441
|
+
* Validator support: every `meta_json` must be a JSON object within the
|
|
442
|
+
* byte limit. Returns the offending ids (capped) rather than throwing.
|
|
443
|
+
*/
|
|
444
|
+
checkDocumentMeta(maxBytes: number, cap?: number): {
|
|
445
|
+
invalid: string[];
|
|
446
|
+
oversize: string[];
|
|
447
|
+
};
|
|
448
|
+
/** Validator support: decoded bodies, streamed one at a time. */
|
|
449
|
+
documentBodies(): IterableIterator<{
|
|
450
|
+
id: string;
|
|
451
|
+
markdown: string;
|
|
452
|
+
}>;
|
|
453
|
+
private decodeBody;
|
|
454
|
+
/**
|
|
455
|
+
* The manifest's `files` entries under `assets/`, read lazily: the
|
|
456
|
+
* declaration in the manifest is what authorizes serving a file, since
|
|
457
|
+
* extraction reconciled and hashed every declared entry.
|
|
458
|
+
*/
|
|
459
|
+
private assetIndex;
|
|
460
|
+
/** Declared assets, sorted by path. */
|
|
461
|
+
assets(): CatalogAssetInfo[];
|
|
462
|
+
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
463
|
+
readAsset(path: string): CatalogAssetRead | null;
|
|
412
464
|
/** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
|
|
413
465
|
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
414
466
|
documentId: string;
|
|
@@ -452,15 +504,15 @@ declare class CatalogHandle {
|
|
|
452
504
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
453
505
|
searchChunksFts(query: string, shardIds: number[], limitPerShard?: number): CatalogChunkHit[];
|
|
454
506
|
/**
|
|
455
|
-
* Embedder-free integrity smoke
|
|
456
|
-
*
|
|
457
|
-
*
|
|
507
|
+
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
508
|
+
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
509
|
+
* false instead of throwing (callers report a reason).
|
|
458
510
|
*/
|
|
459
511
|
selfKnnSmoke(shardId: number): boolean;
|
|
460
512
|
}
|
|
461
513
|
|
|
462
514
|
/**
|
|
463
|
-
* Catalog validation (gezk
|
|
515
|
+
* Catalog validation (the gezk spec §3.3 and §5) over an EXTRACTED catalog
|
|
464
516
|
* directory: manifest parse, per-file hash reconciliation, read-only opens,
|
|
465
517
|
* count reconciliation, and — in deep mode — SQLite quick_check, per-shard
|
|
466
518
|
* vector-table alignment, the embedder-free self-KNN smoke, and the
|
|
@@ -483,34 +535,302 @@ interface ValidateCatalogOptions {
|
|
|
483
535
|
deep?: boolean;
|
|
484
536
|
}
|
|
485
537
|
declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
|
|
538
|
+
/** `[…](assets/…)` and `` targets in a body, deduplicated. */
|
|
539
|
+
declare function assetReferences(markdown: string): string[];
|
|
486
540
|
|
|
487
541
|
/**
|
|
488
|
-
*
|
|
489
|
-
*
|
|
490
|
-
*
|
|
491
|
-
*
|
|
542
|
+
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
543
|
+
* 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
|
|
544
|
+
* popcount plus a histogram selection of the top-K is a few milliseconds and
|
|
545
|
+
* needs nothing beyond plain SQLite to load — which is what lets any reader
|
|
546
|
+
* implement the format without a vector extension.
|
|
547
|
+
*/
|
|
548
|
+
interface ShardBitIndex {
|
|
549
|
+
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
550
|
+
bits: Uint8Array;
|
|
551
|
+
bytesPerRow: number;
|
|
552
|
+
rows: number;
|
|
553
|
+
}
|
|
554
|
+
interface HammingHit {
|
|
555
|
+
chunkId: number;
|
|
556
|
+
distance: number;
|
|
557
|
+
}
|
|
558
|
+
/**
|
|
559
|
+
* The K nearest rows by hamming distance, ascending, ties broken by chunk
|
|
560
|
+
* id. Selection is a histogram over the (small, integer) distance range, so
|
|
561
|
+
* the whole query is two linear passes and no heap.
|
|
562
|
+
*/
|
|
563
|
+
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Provenance stamped into every manifest this package compiles. Only the
|
|
567
|
+
* package identity: node/platform details would make two builds of the same
|
|
568
|
+
* inputs differ by machine, and the format promises byte-identical archives
|
|
569
|
+
* for identical inputs and toolchain.
|
|
570
|
+
*/
|
|
571
|
+
declare const KNOWLEDGE_TOOLCHAIN: NonNullable<KnowledgeCatalogManifest['toolchain']>;
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* The exporter's DuckDB seam: one short-lived CLI child per script, SQL on
|
|
575
|
+
* stdin (never in argv), stdout captured. The binary is caller-supplied — the
|
|
576
|
+
* toolchain package never downloads engines — and its reported version is
|
|
577
|
+
* checked against the one the caller expects, because Parquet bytes are only
|
|
578
|
+
* reproducible against a pinned writer.
|
|
579
|
+
*/
|
|
580
|
+
interface DuckdbCli {
|
|
581
|
+
binaryPath: string;
|
|
582
|
+
/** Refuse to run a different release (e.g. `1.5.5`); omit to accept any. */
|
|
583
|
+
expectedVersion?: string;
|
|
584
|
+
}
|
|
585
|
+
declare class DuckdbError extends Error {
|
|
586
|
+
readonly stderr: string;
|
|
587
|
+
constructor(message: string, stderr: string);
|
|
588
|
+
}
|
|
589
|
+
/** `duckdb --version` prints `v1.5.5 (Variegata) d8cdaa33fd`; the tag without its `v`. */
|
|
590
|
+
declare function duckdbVersion(binaryPath: string): Promise<string>;
|
|
591
|
+
declare function assertDuckdbCli(cli: DuckdbCli): Promise<string>;
|
|
592
|
+
/** Run a SQL script against an in-memory database; throws with DuckDB's own text on failure. */
|
|
593
|
+
declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* The Parquet companion of a catalog: `documents-NNN.parquet` and
|
|
597
|
+
* `chunks-NNN.parquet` per shard plus `topics.parquet`, derived from the
|
|
598
|
+
* sealed archive so anyone can load a catalog with DuckDB, pandas, Polars or
|
|
599
|
+
* the `datasets` library without a gezk reader. Column names are the gezk DDL
|
|
600
|
+
* names; embeddings are the int8 and bit vectors EXACTLY as the archive
|
|
601
|
+
* stores them (provably identical, one third of float32's size), with the
|
|
602
|
+
* dequantisation rule recorded in the file metadata (gezk spec §6).
|
|
492
603
|
*
|
|
493
|
-
*
|
|
494
|
-
*
|
|
495
|
-
*
|
|
496
|
-
*
|
|
604
|
+
* Determinism: rows are read in primary-key order and written single-
|
|
605
|
+
* threaded with insertion order preserved, a fixed codec and level, a fixed
|
|
606
|
+
* row-group size, and no wall-clock metadata — so two exports of one archive
|
|
607
|
+
* with one DuckDB release produce byte-identical files. The pinned DuckDB
|
|
608
|
+
* version is part of the report because the writer's framing is part of the
|
|
609
|
+
* bytes.
|
|
497
610
|
*/
|
|
498
611
|
|
|
499
|
-
|
|
500
|
-
|
|
612
|
+
declare const PARQUET_EXPORT_VERSION = 2;
|
|
613
|
+
type ParquetExportSource = {
|
|
614
|
+
archivePath: string;
|
|
615
|
+
} | {
|
|
616
|
+
rootDir: string;
|
|
617
|
+
};
|
|
618
|
+
interface ParquetExportOptions {
|
|
619
|
+
source: ParquetExportSource;
|
|
620
|
+
outDir: string;
|
|
621
|
+
duckdb: DuckdbCli;
|
|
622
|
+
rowGroupSize?: number;
|
|
623
|
+
onProgress?: (event: ParquetExportProgress) => void;
|
|
624
|
+
}
|
|
625
|
+
type ParquetExportProgress = {
|
|
626
|
+
phase: 'extract';
|
|
627
|
+
} | {
|
|
628
|
+
phase: 'stage';
|
|
629
|
+
table: ParquetTable;
|
|
630
|
+
shardId?: number;
|
|
631
|
+
rows: number;
|
|
632
|
+
} | {
|
|
633
|
+
phase: 'write';
|
|
634
|
+
file: string;
|
|
635
|
+
};
|
|
636
|
+
type ParquetTable = 'documents' | 'chunks' | 'topics';
|
|
637
|
+
interface ParquetExportFile {
|
|
638
|
+
/** Relative to `outDir`, forward slashes. */
|
|
639
|
+
path: string;
|
|
640
|
+
table: ParquetTable;
|
|
641
|
+
shardId?: number;
|
|
642
|
+
rows: number;
|
|
643
|
+
sizeBytes: number;
|
|
644
|
+
sha256: string;
|
|
645
|
+
}
|
|
646
|
+
interface ParquetExportReport {
|
|
647
|
+
exportVersion: number;
|
|
648
|
+
catalogId: string;
|
|
649
|
+
version: string;
|
|
650
|
+
publisherId: string;
|
|
651
|
+
formatVersion: string;
|
|
652
|
+
embeddingProfileId: string;
|
|
653
|
+
dimensions: number;
|
|
654
|
+
files: ParquetExportFile[];
|
|
655
|
+
duckdbVersion: string;
|
|
656
|
+
exporter: {
|
|
657
|
+
name: string;
|
|
658
|
+
version: string;
|
|
659
|
+
};
|
|
660
|
+
}
|
|
661
|
+
declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
662
|
+
declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
|
|
663
|
+
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
664
|
+
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
665
|
+
|
|
666
|
+
/**
|
|
667
|
+
* The registered embedding profiles — the single source both the compiler
|
|
668
|
+
* and the reader import. A profile is the FULL vector-space identity:
|
|
669
|
+
* readers refuse to mix vectors across profile ids, and any change to model,
|
|
670
|
+
* instructions, or encoding is a NEW id.
|
|
671
|
+
*
|
|
672
|
+
* Revisions are the Hugging Face commit hashes the profiles were frozen at
|
|
673
|
+
* (2026-08-20). The artifact digests (recorded 2026-09-03) are the sha256 of
|
|
674
|
+
* the exact files at those revisions — the Hub's LFS object ids for the ONNX
|
|
675
|
+
* graphs and the e5 tokenizer, a hashed download for the bge tokenizer — and
|
|
676
|
+
* name the full-precision graph explicitly, so the precision the vectors were
|
|
677
|
+
* produced at is a pinned fact rather than a runtime default. Every embedder
|
|
678
|
+
* built from a profile verifies the files it loaded against these digests.
|
|
679
|
+
*/
|
|
680
|
+
|
|
681
|
+
/** The public profile: multilingual, so world catalogs serve every reader. */
|
|
682
|
+
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
501
683
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
502
|
-
declare const
|
|
684
|
+
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
503
685
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
504
686
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
505
|
-
/** The knowledge chunking profile
|
|
506
|
-
declare const
|
|
687
|
+
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
688
|
+
declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
689
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
690
|
+
declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
691
|
+
id: string;
|
|
692
|
+
model: {
|
|
693
|
+
repo: string;
|
|
694
|
+
revision: string;
|
|
695
|
+
onnxFile?: string | undefined;
|
|
696
|
+
onnxDigest?: string | undefined;
|
|
697
|
+
};
|
|
698
|
+
tokenizer: {
|
|
699
|
+
kind: string;
|
|
700
|
+
file?: string | undefined;
|
|
701
|
+
digest?: string | undefined;
|
|
702
|
+
};
|
|
703
|
+
pooling: "mean" | "cls" | "last";
|
|
704
|
+
normalized: boolean;
|
|
705
|
+
dimensions: number;
|
|
706
|
+
maxTokens: number;
|
|
707
|
+
queryInstruction: string;
|
|
708
|
+
passageInstruction: string;
|
|
709
|
+
vectorEncoding: "bit+int8";
|
|
710
|
+
distance: {
|
|
711
|
+
stage1: "hamming";
|
|
712
|
+
stage2: "cosine";
|
|
713
|
+
};
|
|
714
|
+
quantization: {
|
|
715
|
+
int8: {
|
|
716
|
+
method: "symmetric-linear";
|
|
717
|
+
scale: 127;
|
|
718
|
+
};
|
|
719
|
+
binary: {
|
|
720
|
+
method: "sign";
|
|
721
|
+
threshold: 0;
|
|
722
|
+
packing: "lsb-first";
|
|
723
|
+
};
|
|
724
|
+
};
|
|
725
|
+
};
|
|
726
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
727
|
+
declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
728
|
+
id: string;
|
|
729
|
+
model: {
|
|
730
|
+
repo: string;
|
|
731
|
+
revision: string;
|
|
732
|
+
onnxFile?: string | undefined;
|
|
733
|
+
onnxDigest?: string | undefined;
|
|
734
|
+
};
|
|
735
|
+
tokenizer: {
|
|
736
|
+
kind: string;
|
|
737
|
+
file?: string | undefined;
|
|
738
|
+
digest?: string | undefined;
|
|
739
|
+
};
|
|
740
|
+
pooling: "mean" | "cls" | "last";
|
|
741
|
+
normalized: boolean;
|
|
742
|
+
dimensions: number;
|
|
743
|
+
maxTokens: number;
|
|
744
|
+
queryInstruction: string;
|
|
745
|
+
passageInstruction: string;
|
|
746
|
+
vectorEncoding: "bit+int8";
|
|
747
|
+
distance: {
|
|
748
|
+
stage1: "hamming";
|
|
749
|
+
stage2: "cosine";
|
|
750
|
+
};
|
|
751
|
+
quantization: {
|
|
752
|
+
int8: {
|
|
753
|
+
method: "symmetric-linear";
|
|
754
|
+
scale: 127;
|
|
755
|
+
};
|
|
756
|
+
binary: {
|
|
757
|
+
method: "sign";
|
|
758
|
+
threshold: 0;
|
|
759
|
+
packing: "lsb-first";
|
|
760
|
+
};
|
|
761
|
+
};
|
|
762
|
+
};
|
|
763
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
764
|
+
declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
765
|
+
id: string;
|
|
766
|
+
unit: "chars" | "tokens";
|
|
767
|
+
tokenizer: "profile" | "none";
|
|
768
|
+
target: number;
|
|
769
|
+
overlap: number;
|
|
770
|
+
contextHeader: {
|
|
771
|
+
max: number;
|
|
772
|
+
};
|
|
773
|
+
};
|
|
774
|
+
|
|
775
|
+
/**
|
|
776
|
+
* Verifies the model files an embedder loaded against the digests its
|
|
777
|
+
* profile pins. transformers.js caches every fetched file under a key that
|
|
778
|
+
* mirrors the repo path — `<repo>/<file>` for `main`, `<repo>/<revision>/<file>`
|
|
779
|
+
* for a pinned revision — so after a load the exact bytes are on disk and
|
|
780
|
+
* can be hashed. That turns "same repo and revision" from a convention into
|
|
781
|
+
* a checked fact: a profile that declares digests is served only by weights
|
|
782
|
+
* whose sha256 matches them, and a mismatch is reported as a different
|
|
783
|
+
* vector space rather than silently producing slightly different vectors.
|
|
784
|
+
*/
|
|
785
|
+
|
|
786
|
+
type ArtifactRole = 'onnx' | 'tokenizer';
|
|
787
|
+
interface ArtifactCheck {
|
|
788
|
+
role: ArtifactRole;
|
|
789
|
+
/** Repo-relative file the profile pins. */
|
|
790
|
+
file: string;
|
|
791
|
+
/** Where the loaded copy was looked for. */
|
|
792
|
+
path: string;
|
|
793
|
+
expected: string;
|
|
794
|
+
/** `null` when no file was found at `path`. */
|
|
795
|
+
actual: string | null;
|
|
796
|
+
}
|
|
797
|
+
interface VerifiedArtifacts {
|
|
798
|
+
/** `unpinned` when the profile declares no digests, so nothing was checked. */
|
|
799
|
+
status: 'verified' | 'unpinned';
|
|
800
|
+
checks: ArtifactCheck[];
|
|
801
|
+
}
|
|
802
|
+
declare class EmbedderArtifactError extends Error {
|
|
803
|
+
readonly reason: 'missing' | 'mismatch';
|
|
804
|
+
readonly profileId: string;
|
|
805
|
+
readonly checks: ArtifactCheck[];
|
|
806
|
+
readonly isActionable = true;
|
|
807
|
+
constructor(reason: 'missing' | 'mismatch', profileId: string, checks: ArtifactCheck[]);
|
|
808
|
+
}
|
|
809
|
+
/** The transformers.js FileCache key for one repo file. */
|
|
810
|
+
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
811
|
+
/**
|
|
812
|
+
* Hash the profile's pinned files in a transformers.js cache and compare.
|
|
813
|
+
* `revision` names the cache key the files were fetched under; it defaults
|
|
814
|
+
* to the profile's own revision and is `main` for a pipeline loaded
|
|
815
|
+
* unpinned. Throws `EmbedderArtifactError` on a missing or differing file.
|
|
816
|
+
*/
|
|
817
|
+
declare function verifyProfileArtifacts(profile: KnowledgeEmbeddingProfile, opts: {
|
|
818
|
+
cacheDir: string;
|
|
819
|
+
revision?: string;
|
|
820
|
+
}): Promise<VerifiedArtifacts>;
|
|
507
821
|
|
|
508
822
|
/**
|
|
509
823
|
* Profile-conformant embedder + tokenizer, driven entirely by a declarative
|
|
510
|
-
* KnowledgeEmbeddingProfile — model repo, pinned revision,
|
|
511
|
-
* instruction prefixes all come from the profile object,
|
|
512
|
-
* regexes. This is what the offline CLI wires into the
|
|
513
|
-
* explicit-search path; the daemon wires its own pipeline
|
|
824
|
+
* KnowledgeEmbeddingProfile — model repo, pinned revision, the exact ONNX
|
|
825
|
+
* graph, pooling, and instruction prefixes all come from the profile object,
|
|
826
|
+
* never from env-var regexes. This is what the offline CLI wires into the
|
|
827
|
+
* compiler and the explicit-search path; the daemon wires its own pipeline
|
|
828
|
+
* pool later.
|
|
829
|
+
*
|
|
830
|
+
* After loading, the files transformers.js fetched are hashed and compared
|
|
831
|
+
* with the digests the profile pins (artifact-verify.ts). A profile that
|
|
832
|
+
* declares digests is therefore served only by the very bytes that produced
|
|
833
|
+
* its catalogs; anything else is refused as a different vector space.
|
|
514
834
|
*
|
|
515
835
|
* `@huggingface/transformers` stays a dynamic import (mirroring the
|
|
516
836
|
* service's embed-core): the CLI must start instantly without it, and a
|
|
@@ -521,6 +841,8 @@ declare const GEZEL_MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
|
521
841
|
|
|
522
842
|
interface ProfileEmbedder {
|
|
523
843
|
readonly profile: KnowledgeEmbeddingProfile;
|
|
844
|
+
/** What the loaded model files were checked against after loading. */
|
|
845
|
+
readonly verification: VerifiedArtifacts;
|
|
524
846
|
/** Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's contract). */
|
|
525
847
|
embed(texts: string[]): Promise<number[][]>;
|
|
526
848
|
/** Query embed — applies the profile's queryInstruction itself. */
|
|
@@ -533,6 +855,37 @@ declare class EmbedderUnavailableError extends Error {
|
|
|
533
855
|
readonly isActionable = true;
|
|
534
856
|
constructor(message: string);
|
|
535
857
|
}
|
|
858
|
+
/** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
|
|
859
|
+
interface TransformersModule {
|
|
860
|
+
pipeline: (task: string, model: string, options?: Record<string, unknown>) => Promise<PipelineFn>;
|
|
861
|
+
AutoTokenizer: {
|
|
862
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerFn>;
|
|
863
|
+
};
|
|
864
|
+
env: {
|
|
865
|
+
cacheDir?: string;
|
|
866
|
+
useFSCache?: boolean;
|
|
867
|
+
allowRemoteModels?: boolean;
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
type PipelineFn = ((texts: string[], options: {
|
|
871
|
+
pooling: string;
|
|
872
|
+
normalize: boolean;
|
|
873
|
+
}) => Promise<{
|
|
874
|
+
tolist(): number[][];
|
|
875
|
+
}>) & {
|
|
876
|
+
dispose?: () => Promise<void>;
|
|
877
|
+
};
|
|
878
|
+
interface TokenizerFn {
|
|
879
|
+
encode(text: string): number[];
|
|
880
|
+
}
|
|
881
|
+
/** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
|
|
882
|
+
interface TransformersModelOptions {
|
|
883
|
+
revision: string;
|
|
884
|
+
dtype: string;
|
|
885
|
+
subfolder: string;
|
|
886
|
+
model_file_name: string;
|
|
887
|
+
}
|
|
888
|
+
declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
|
|
536
889
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
537
890
|
cacheDir?: string;
|
|
538
891
|
/**
|
|
@@ -543,80 +896,180 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
543
896
|
* what affects values, and that stays with the caller.
|
|
544
897
|
*/
|
|
545
898
|
sessionOptions?: Record<string, unknown>;
|
|
899
|
+
/** Test seam: a stand-in for `@huggingface/transformers`. */
|
|
900
|
+
transformers?: TransformersModule;
|
|
546
901
|
}): Promise<ProfileEmbedder>;
|
|
547
902
|
|
|
903
|
+
/**
|
|
904
|
+
* Tables of contents that documentation trees already carry. Each parser
|
|
905
|
+
* turns one tool's outline into the same shape — a tree of topics holding
|
|
906
|
+
* ordered document entries — which the Markdown adapter lays over its
|
|
907
|
+
* folder-derived tree: a file the outline names is filed where the outline
|
|
908
|
+
* puts it, with the outline's title and position, and a file it omits stays
|
|
909
|
+
* in its folder. Parsers are pure: they see text and the list of Markdown
|
|
910
|
+
* files, never the disk, so an outline can be checked in a test without a
|
|
911
|
+
* tree on disk.
|
|
912
|
+
*/
|
|
913
|
+
type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book';
|
|
914
|
+
type OutlineFormat = Exclude<TableOfContentsFormat, 'folders' | 'hugo'>;
|
|
915
|
+
interface OutlineEntry {
|
|
916
|
+
/** Root-relative POSIX path of the Markdown file, extension included. */
|
|
917
|
+
file: string;
|
|
918
|
+
/** The title the outline gives the page; wins over the file's own. */
|
|
919
|
+
title?: string;
|
|
920
|
+
/** Listing position among the documents of the containing topic. */
|
|
921
|
+
order?: number;
|
|
922
|
+
}
|
|
923
|
+
interface OutlineTopic {
|
|
924
|
+
name: string;
|
|
925
|
+
description?: string;
|
|
926
|
+
/** Position among sibling topics. */
|
|
927
|
+
order?: number;
|
|
928
|
+
/** When set, the topic takes the resolved title of this document (a root-relative file). */
|
|
929
|
+
titleFromFile?: string;
|
|
930
|
+
children: OutlineTopic[];
|
|
931
|
+
entries: OutlineEntry[];
|
|
932
|
+
}
|
|
933
|
+
interface Outline {
|
|
934
|
+
format: OutlineFormat;
|
|
935
|
+
/** Top-level documents and topics; `root.name` is unused. */
|
|
936
|
+
root: OutlineTopic;
|
|
937
|
+
/** Root-relative files that are the outline itself and never documents. */
|
|
938
|
+
consumed: string[];
|
|
939
|
+
warnings: string[];
|
|
940
|
+
}
|
|
941
|
+
declare function newOutlineTopic(name: string): OutlineTopic;
|
|
942
|
+
/**
|
|
943
|
+
* A link target relative to `baseDir` (root-relative POSIX, `''` for the
|
|
944
|
+
* root) as a root-relative POSIX path, or null when it is empty, absolute
|
|
945
|
+
* or leaves the root. Fragments and queries are dropped.
|
|
946
|
+
*/
|
|
947
|
+
declare function resolveOutlinePath(baseDir: string, target: string): string | null;
|
|
948
|
+
/**
|
|
949
|
+
* Positions become orders wherever the outline gave none. Entries and
|
|
950
|
+
* child topics are ranked separately, each 1-based, because the catalog
|
|
951
|
+
* orders documents by ordinal and topics by sort key.
|
|
952
|
+
*/
|
|
953
|
+
declare function assignOutlineOrders(topic: OutlineTopic): void;
|
|
954
|
+
/**
|
|
955
|
+
* `getting-started` → `Getting Started`. Separators become spaces and each
|
|
956
|
+
* all-lowercase word is capitalized; a word that already carries a capital
|
|
957
|
+
* (`API`, `iPhone`) is left as written.
|
|
958
|
+
*/
|
|
959
|
+
declare function titleFromFolderName(name: string): string;
|
|
960
|
+
/**
|
|
961
|
+
* GitBook's `SUMMARY.md`: a nested list of `[Title](page.md)` items, where a
|
|
962
|
+
* `## Heading` opens a part. An item with children becomes a topic; when
|
|
963
|
+
* it links a page, that page leads the topic. Plain-text items are groups.
|
|
964
|
+
*/
|
|
965
|
+
declare function parseGitbookSummary(text: string, summaryFile: string): Outline;
|
|
966
|
+
/**
|
|
967
|
+
* The `nav` list of an `mkdocs.yml`: `- page.md`, `- Title: page.md`, and
|
|
968
|
+
* `- Section: [ … ]`. Returns null when the config has no `nav`, which in
|
|
969
|
+
* MkDocs means the folder tree is the outline — the adapter's default too.
|
|
970
|
+
* `docsDir` is the docs directory relative to the content root.
|
|
971
|
+
*/
|
|
972
|
+
declare function parseMkdocsNav(config: unknown, docsDir: string, configFile?: string): Outline | null;
|
|
973
|
+
/**
|
|
974
|
+
* Jupyter Book's `_toc.yml`: `format: jb-book` (`root`, `parts` with a
|
|
975
|
+
* `caption`, `chapters`, nested `sections`) or `jb-article` (`root`,
|
|
976
|
+
* `sections`); the pre-1.0 list form is read too. Files are named without
|
|
977
|
+
* extension and resolved against the Markdown files of the tree, so a
|
|
978
|
+
* notebook entry is skipped with a warning. A chapter with sections becomes
|
|
979
|
+
* a topic, led by its own page; `glob` entries expand in sorted order;
|
|
980
|
+
* `url` entries are links, not pages.
|
|
981
|
+
*/
|
|
982
|
+
declare function parseJupyterBookToc(doc: unknown, tocDir: string, files: readonly string[], tocFile?: string): Outline;
|
|
983
|
+
|
|
548
984
|
/**
|
|
549
985
|
* Markdown-folder catalog source — the `gezel knowledge build` input adapter.
|
|
550
986
|
* Folders become the shipped table of contents (a format requirement:
|
|
551
|
-
* gezk
|
|
552
|
-
*
|
|
553
|
-
*
|
|
987
|
+
* the gezk spec §5.2), files become documents, images the bodies reference
|
|
988
|
+
* become assets. A flat corpus — files at the root with no subfolders —
|
|
989
|
+
* gets the single root topic the compiler demands.
|
|
990
|
+
*
|
|
991
|
+
* A tree that already carries an outline keeps it: GitBook's `SUMMARY.md`,
|
|
992
|
+
* an `mkdocs.yml` nav and Jupyter Book's `_toc.yml` are read into the same
|
|
993
|
+
* topic tree (`outline.ts`), and Hugo's conventions — `_index.md` section
|
|
994
|
+
* pages, `weight`, `draft` — are honored on top of the folders. A file an
|
|
995
|
+
* outline omits stays in its folder, with a warning, so nothing is lost.
|
|
996
|
+
*
|
|
997
|
+
* Front matter carries what a documentation tree knows about itself: the
|
|
998
|
+
* title and summary, an explicit `id`, an `order` (the listing ordinal), a
|
|
999
|
+
* `subcategory` shelf below the folder topic, and anything else as opaque
|
|
1000
|
+
* metadata. Two passes keep it deterministic: the first walks files in
|
|
1001
|
+
* sorted order and fixes every id; the second rewrites relative image and
|
|
1002
|
+
* article links now that every target is known.
|
|
554
1003
|
*/
|
|
555
1004
|
|
|
556
1005
|
interface MarkdownCatalogSource {
|
|
557
1006
|
topics: CompileTopic[];
|
|
558
1007
|
documents: CatalogDocument[];
|
|
1008
|
+
/** Images referenced by the bodies, resolved to files under the root. */
|
|
1009
|
+
assets: CompileAsset[];
|
|
1010
|
+
/** The outline that shaped the topics, with its file when it came from one. */
|
|
1011
|
+
toc: {
|
|
1012
|
+
format: TableOfContentsFormat;
|
|
1013
|
+
path?: string;
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
interface TopicOverride {
|
|
1017
|
+
name?: string;
|
|
1018
|
+
/** Listing position among sibling topics; encoded into the sort key. */
|
|
1019
|
+
order?: number;
|
|
1020
|
+
description?: string;
|
|
1021
|
+
}
|
|
1022
|
+
interface TableOfContentsOptions {
|
|
1023
|
+
format: TableOfContentsFormat;
|
|
1024
|
+
/**
|
|
1025
|
+
* The outline file for `gitbook` (`SUMMARY.md`), `mkdocs` (`mkdocs.yml`)
|
|
1026
|
+
* and `jupyter-book` (`_toc.yml`), absolute or relative to the root. By
|
|
1027
|
+
* default the root is searched, and for mkdocs its parent as well.
|
|
1028
|
+
*/
|
|
1029
|
+
path?: string;
|
|
559
1030
|
}
|
|
560
1031
|
interface LoadMarkdownCatalogOptions {
|
|
561
1032
|
language: string;
|
|
562
1033
|
/** Topic id used for root-level files (default `general`). */
|
|
563
1034
|
rootTopicId?: string;
|
|
564
1035
|
rootTopicName?: string;
|
|
1036
|
+
/**
|
|
1037
|
+
* Root-relative POSIX directory path → overrides. An explicit entry wins
|
|
1038
|
+
* over a `_topic.yaml` sidecar in that directory.
|
|
1039
|
+
*/
|
|
1040
|
+
topics?: Record<string, TopicOverride>;
|
|
1041
|
+
/** Root-relative files (POSIX) to leave out, e.g. `['README.md']`. */
|
|
1042
|
+
ignore?: string[];
|
|
1043
|
+
/** Where the table of contents comes from (default: the folders). */
|
|
1044
|
+
toc?: TableOfContentsOptions;
|
|
1045
|
+
/**
|
|
1046
|
+
* When set, relative links to other Markdown files in the tree are
|
|
1047
|
+
* rewritten to `knowledge://` references so a viewer can follow them.
|
|
1048
|
+
*/
|
|
1049
|
+
uri?: {
|
|
1050
|
+
publisherId: string;
|
|
1051
|
+
catalogId: string;
|
|
1052
|
+
};
|
|
1053
|
+
/** What to do with an image link whose file is missing (default `error`). */
|
|
1054
|
+
missingAssets?: 'error' | 'warn';
|
|
1055
|
+
onWarning?: (message: string) => void;
|
|
565
1056
|
}
|
|
1057
|
+
/** The docs directory an `mkdocs.yml` names (`docs_dir`, default `docs`), absolute. */
|
|
1058
|
+
declare function readMkdocsDocsDir(mkdocsPath: string): Promise<string>;
|
|
566
1059
|
/**
|
|
567
|
-
*
|
|
568
|
-
*
|
|
569
|
-
*
|
|
570
|
-
|
|
571
|
-
declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
|
|
572
|
-
|
|
573
|
-
/**
|
|
574
|
-
* Manifest signing (gezk-format-v1.md §7): Ed25519 over the RFC 8785
|
|
575
|
-
* canonical form of the manifest WITHOUT its `signature` field. Key
|
|
576
|
-
* encodings follow the repo's identity conventions (remotes/identity.ts):
|
|
577
|
-
* SPKI/PKCS#8 PEM, base64 signatures, node:crypto one-shot sign/verify.
|
|
578
|
-
* The private key never appears in CI — signing is a manual workstation
|
|
579
|
-
* step (the plan's key-custody stance); verification is what ships.
|
|
580
|
-
*/
|
|
581
|
-
|
|
582
|
-
interface KnowledgeTrustAnchor {
|
|
583
|
-
keyId: string;
|
|
584
|
-
publicKeyPem: string;
|
|
585
|
-
/** Display-only: whose key this is (e.g. `qualla`). */
|
|
586
|
-
publisherId?: string;
|
|
587
|
-
}
|
|
588
|
-
type ManifestSignatureVerdict = {
|
|
589
|
-
ok: true;
|
|
590
|
-
keyId: string;
|
|
591
|
-
} | {
|
|
592
|
-
ok: false;
|
|
593
|
-
reason: 'unsigned' | 'unknown-key' | 'bad-signature' | 'error';
|
|
594
|
-
detail?: string;
|
|
595
|
-
};
|
|
596
|
-
/** Stable key identity: first 16 hex chars of SHA-256 over the SPKI DER. */
|
|
597
|
-
declare function knowledgeKeyId(publicKeyPem: string): string;
|
|
598
|
-
declare function generateKnowledgeSigningKeyPair(): {
|
|
599
|
-
publicKeyPem: string;
|
|
600
|
-
privateKeyPem: string;
|
|
601
|
-
keyId: string;
|
|
602
|
-
};
|
|
603
|
-
/** The exact bytes the signature covers: JCS(manifest minus `signature`). */
|
|
604
|
-
declare function manifestSigningPayload(manifest: KnowledgeCatalogManifest): Buffer;
|
|
605
|
-
declare function signManifest(manifest: KnowledgeCatalogManifest, privateKeyPem: string): KnowledgeCatalogManifest;
|
|
606
|
-
/**
|
|
607
|
-
* Verify against keyId-indexed trust anchors. Key rotation = overlapping
|
|
608
|
-
* anchors for one publisher; an unknown keyId is a hard failure, never a
|
|
609
|
-
* fallback scan across anchors.
|
|
1060
|
+
* Which outline a tree carries: GitBook's `SUMMARY.md` or Jupyter Book's
|
|
1061
|
+
* `_toc.yml` at the root, an `mkdocs.yml` at the root, the project folder
|
|
1062
|
+
* or the root's parent, Hugo's `_index.md` section pages anywhere in the
|
|
1063
|
+
* tree — else the folders themselves.
|
|
610
1064
|
*/
|
|
611
|
-
declare function
|
|
1065
|
+
declare function detectTableOfContents(rootDir: string, projectDir?: string): Promise<TableOfContentsOptions>;
|
|
612
1066
|
/**
|
|
613
|
-
*
|
|
614
|
-
*
|
|
615
|
-
*
|
|
616
|
-
*
|
|
1067
|
+
* Load a folder of Markdown into compiler inputs. Directory chain → topic
|
|
1068
|
+
* chain unless an outline places the file; the relative path (without
|
|
1069
|
+
* extension, `/`-separated) is the stable document id unless the front
|
|
1070
|
+
* matter names one.
|
|
617
1071
|
*/
|
|
618
|
-
declare function
|
|
619
|
-
declare function verifyRegistryIndex(index: KnowledgeRegistryIndex, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
|
|
1072
|
+
declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
|
|
620
1073
|
|
|
621
1074
|
/**
|
|
622
1075
|
* Signed-registry consumption (knowledge-catalogs.md Phase 6): fetch a
|
|
@@ -639,8 +1092,10 @@ declare class KnowledgeRegistryFetchError extends Error {
|
|
|
639
1092
|
interface FetchKnowledgeRegistryOptions {
|
|
640
1093
|
anchors: readonly KnowledgeTrustAnchor[];
|
|
641
1094
|
maxBytes?: number;
|
|
642
|
-
/**
|
|
1095
|
+
/** Budget for the whole fetch when no `signal` is supplied. */
|
|
643
1096
|
timeoutMs?: number;
|
|
1097
|
+
/** Caller-owned deadline (the daemon passes a suspend-aware signal). */
|
|
1098
|
+
signal?: AbortSignal;
|
|
644
1099
|
fetchImpl?: typeof fetch;
|
|
645
1100
|
}
|
|
646
1101
|
/**
|
|
@@ -695,14 +1150,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
695
1150
|
|
|
696
1151
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
697
1152
|
|
|
698
|
-
|
|
699
|
-
* RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
|
|
700
|
-
* to the manifest's Ed25519 signature (gezk-format-v1.md §7). ECMAScript's
|
|
701
|
-
* own JSON.stringify already implements the RFC's number and string
|
|
702
|
-
* serialization; canonicalization adds only (a) object keys sorted by UTF-16
|
|
703
|
-
* code units and (b) a hard rejection of anything without a JSON identity
|
|
704
|
-
* (undefined, functions, non-finite numbers, BigInt).
|
|
705
|
-
*/
|
|
706
|
-
declare function canonicalizeJson(value: unknown): string;
|
|
707
|
-
|
|
708
|
-
export { type ArchiveEntry, BODY_CODEC_MIN_BYTES, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_APPLICATION_ID, GEZK_ARCHIVE_LIMITS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_EMBEDDING_PROFILES, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type KnowledgeTrustAnchor, type LoadMarkdownCatalogOptions, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, type ManifestSignatureVerdict, type MarkdownCatalogSource, type MarkdownChunk, type ProfileEmbedder, RERANK_FINAL_K, ROUTER_DB_PATH, ROUTER_DDL, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_DDL, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type TokenChunkerOptions, type ValidateCatalogOptions, ZIP_FIXED_MTIME, canonicalizeJson, chunkContentHash, chunkMarkdownProfile, chunkUid, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, documentSlug, extractGezkVerified, fetchKnowledgeRegistry, findRegistryEntry, generateKnowledgeSigningKeyPair, inspectGezkArchive, knowledgeEmbeddingProfile, knowledgeKeyId, l2Normalize, loadMarkdownCatalog, manifestSigningPayload, newerRegistryEntries, openCatalogDatabase, quantizeBinary, quantizeInt8, readGezkManifest, rerankK, rerankScore, signManifest, signRegistryIndex, validateExtractedCatalog, vecChunksDdl, verifyManifestSignature, verifyRegistryIndex, writeGezkArchive };
|
|
1153
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|