@bendyline/gezel-knowledge 1.0.6 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -17
- package/dist/index.d.ts +482 -137
- package/dist/index.js +1233 -397
- package/package.json +6 -5
package/dist/index.d.ts
CHANGED
|
@@ -1,20 +1,14 @@
|
|
|
1
|
-
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest } from '@bendyline/
|
|
1
|
+
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
3
|
+
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
|
+
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
2
5
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
3
6
|
|
|
4
|
-
/**
|
|
5
|
-
|
|
6
|
-
declare const GEZK_INDEX_SCHEMA_VERSION = 1;
|
|
7
|
-
/** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
|
|
8
|
-
declare const GEZK_APPLICATION_ID = 1195727435;
|
|
9
|
-
declare const ROUTER_DB_PATH = "index/router.db";
|
|
10
|
-
declare const MANIFEST_PATH = "manifest.json";
|
|
7
|
+
/** Toolchain knobs. Format-level constants come from @bendyline/gezk. */
|
|
8
|
+
|
|
11
9
|
/** Compiler shard sizing (§3.1). */
|
|
12
10
|
declare const SHARD_TARGET_CHUNKS = 200000;
|
|
13
11
|
declare const SHARD_MAX_CHUNKS = 250000;
|
|
14
|
-
/** Bodies below this ship uncompressed (`body_codec = 'none'`). */
|
|
15
|
-
declare const BODY_CODEC_MIN_BYTES = 512;
|
|
16
|
-
/** Maximum decompressed Markdown body returned from one catalog document. */
|
|
17
|
-
declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
|
|
18
12
|
/** Routing (§3.3–3.4). */
|
|
19
13
|
declare const CENTROID_CHUNKS_PER = 12500;
|
|
20
14
|
declare const CENTROID_MAX_PER_SHARD = 32;
|
|
@@ -25,58 +19,17 @@ declare const ROUTE_SHARDS_EXPLICIT = 6;
|
|
|
25
19
|
/** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
|
|
26
20
|
declare const RERANK_FINAL_K = 24;
|
|
27
21
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
28
|
-
/** Fixed ZIP entry timestamp for deterministic archives (§11). */
|
|
29
|
-
declare const ZIP_FIXED_MTIME: Date;
|
|
30
|
-
|
|
31
|
-
/** Frozen v1 DDL (gezk-format-v1.md §6). Readers never migrate these. */
|
|
32
|
-
declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
|
|
33
|
-
/** Per-shard tables. The vec0 table is created separately (needs sqlite-vec). */
|
|
34
|
-
declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
|
|
35
|
-
/** vec0 table for the hamming stage; `dim` comes from the embedding profile. */
|
|
36
|
-
declare function vecChunksDdl(dim: number): string;
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
* Stable chunk id (gezk-format-v1.md §5): lowercase hex of the FIRST 16
|
|
40
|
-
* bytes of SHA-256(docId || 0x00 || decimal-ordinal || 0x00 ||
|
|
41
|
-
* SHA-256(chunkText) raw 32 bytes). Content-derived so identical inputs on
|
|
42
|
-
* any toolchain produce identical ids — the anchor of both reproducibility
|
|
43
|
-
* and the `#chunk=` citation fragment.
|
|
44
|
-
*/
|
|
45
|
-
declare function chunkUid(documentId: string, ordinal: number, chunkText: string): string;
|
|
46
|
-
/** Full SHA-256 hex of a chunk's text (`chunks.content_hash`). */
|
|
47
|
-
declare function chunkContentHash(chunkText: string): string;
|
|
48
|
-
/** URL-ish slug for a document title (display/dedup aid, not an identity). */
|
|
49
|
-
declare function documentSlug(title: string): string;
|
|
50
|
-
|
|
51
|
-
/**
|
|
52
|
-
* Frozen quantization formulas (gezk-format-v1.md §2). Deliberately computed
|
|
53
|
-
* here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
|
|
54
|
-
* functions, so extension-internal semantics can never drift the format:
|
|
55
|
-
* both the compiler (packing) and the reader (query packing + rerank) call
|
|
56
|
-
* exactly these functions.
|
|
57
|
-
*/
|
|
58
|
-
/** int8: symmetric linear, scale 127, −128 never produced. */
|
|
59
|
-
declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
|
|
60
|
-
/** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
|
|
61
|
-
declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
|
|
62
|
-
/**
|
|
63
|
-
* Rerank score: dot(float32 unit query, dequantized int8 passage). Passages
|
|
64
|
-
* were unit vectors, so this approximates cosine; the query is not quantized.
|
|
65
|
-
*/
|
|
66
|
-
declare function rerankScore(query: ArrayLike<number>, passageInt8: Int8Array): number;
|
|
67
|
-
/** L2-normalize in place-free fashion; throws on a degenerate vector. */
|
|
68
|
-
declare function l2Normalize(vector: ArrayLike<number>): Float32Array;
|
|
69
22
|
|
|
70
23
|
/**
|
|
71
24
|
* The canonical markdown chunker, parameterized over the two frozen
|
|
72
|
-
* profiles (
|
|
25
|
+
* profiles (the gezk spec §7):
|
|
73
26
|
*
|
|
74
|
-
* - `
|
|
27
|
+
* - `markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
|
|
75
28
|
* algorithm lifted from the project indexer
|
|
76
29
|
* (packages/service/src/index-store/docs.ts `chunkMarkdown` /
|
|
77
30
|
* `chunkLineRange`), so the service can later re-point here with zero
|
|
78
31
|
* behavior change.
|
|
79
|
-
* - `
|
|
32
|
+
* - `markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
|
|
80
33
|
* profile. Token counting is injected (`countTokens` — the embedding
|
|
81
34
|
* profile's own tokenizer), packing is per-line with sentence-level
|
|
82
35
|
* splitting for oversized lines, and every chunk carries a structural
|
|
@@ -115,7 +68,7 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
|
|
|
115
68
|
declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
|
|
116
69
|
|
|
117
70
|
/**
|
|
118
|
-
* `.gezk` compiler (gezk
|
|
71
|
+
* `.gezk` compiler (gezk 0.5; spec in bendyline/gezk). One owner of normalization, chunk
|
|
119
72
|
* ids, embedding-profile conformance, SQLite schema, shard assignment,
|
|
120
73
|
* centroid routing, and deterministic archive layout — the CLI's Markdown
|
|
121
74
|
* adapter and Qualla's Wikipedia adapter both feed this API.
|
|
@@ -156,6 +109,7 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
156
109
|
createdAt: string;
|
|
157
110
|
license: {
|
|
158
111
|
name: string;
|
|
112
|
+
spdx?: string;
|
|
159
113
|
noticePath?: string;
|
|
160
114
|
attributionRequired: boolean;
|
|
161
115
|
};
|
|
@@ -186,8 +140,28 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
186
140
|
query: string;
|
|
187
141
|
expectedDocumentIds: string[];
|
|
188
142
|
}>;
|
|
143
|
+
/**
|
|
144
|
+
* How to treat `smokeQueries` at seal time. Every recorded query is
|
|
145
|
+
* verified against the BUILT index with the validator's exact semantics
|
|
146
|
+
* (sanitize → fts_documents → top-N) before the manifest is written — a
|
|
147
|
+
* catalog must never ship sanity queries it cannot answer (the arts-pilot
|
|
148
|
+
* "smoke:Abbey (missing 1315)" install failure).
|
|
149
|
+
*
|
|
150
|
+
* - 'require' (default): every query must pass; a miss fails the build
|
|
151
|
+
* with the query, the missing ids, and guidance.
|
|
152
|
+
* - 'select': the list is CANDIDATES — verify in order, record the first
|
|
153
|
+
* three that pass, and fail only when none do. For producers that
|
|
154
|
+
* derive queries mechanically (Qualla uses article titles) and cannot
|
|
155
|
+
* know in advance which titles win the BM25 rank race.
|
|
156
|
+
*/
|
|
157
|
+
smokeQueryPolicy?: 'require' | 'select';
|
|
158
|
+
/** Provenance; defaults to this package's identity. */
|
|
189
159
|
toolchain?: KnowledgeCatalogManifest['toolchain'];
|
|
190
|
-
/**
|
|
160
|
+
/**
|
|
161
|
+
* Extra archive files (path → utf8 content). `README.md` and
|
|
162
|
+
* `LICENSES/catalog.txt` are always shipped: minimal ones are generated
|
|
163
|
+
* from the catalog block unless supplied here.
|
|
164
|
+
*/
|
|
191
165
|
extraFiles?: Record<string, string>;
|
|
192
166
|
/**
|
|
193
167
|
* Last touch before the manifest is archived — the signing seam
|
|
@@ -202,6 +176,13 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
202
176
|
total: number;
|
|
203
177
|
}) => void;
|
|
204
178
|
embedBatchSize?: number;
|
|
179
|
+
/**
|
|
180
|
+
* TEST/EVAL override of the 200k shard target (the gezk spec §8.1) —
|
|
181
|
+
* lets a small corpus build multi-shard so routing recall can be measured
|
|
182
|
+
* against a scan-all control without a 2M-document corpus. Production
|
|
183
|
+
* builds must not set it: the constant IS the format's scale design.
|
|
184
|
+
*/
|
|
185
|
+
shardTargetChunks?: number;
|
|
205
186
|
}
|
|
206
187
|
interface CompileReport {
|
|
207
188
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -213,11 +194,13 @@ interface CompileReport {
|
|
|
213
194
|
declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
|
|
214
195
|
|
|
215
196
|
/**
|
|
216
|
-
* Deterministic `.gezk` ZIP writer
|
|
217
|
-
* by path bytes, fixed timestamp
|
|
218
|
-
* names, ZIP64 as needed (yazl
|
|
219
|
-
*
|
|
220
|
-
*
|
|
197
|
+
* Deterministic `.gezk` ZIP writer: the stored `mimetype` entry first (the
|
|
198
|
+
* container's magic), then entries sorted by path bytes, fixed timestamp
|
|
199
|
+
* (2000-01-01T00:00:00Z), mode 0644, UTF-8 names, ZIP64 as needed (yazl
|
|
200
|
+
* handles the transition). Every entry is STORED, not deflated: SQLite pages
|
|
201
|
+
* carrying brotli bodies compress little, byte identity then no longer
|
|
202
|
+
* depends on the zlib build, and content-addressed hosting (Hugging Face
|
|
203
|
+
* Xet) can deduplicate unchanged pages across releases.
|
|
221
204
|
*/
|
|
222
205
|
interface ArchiveEntry {
|
|
223
206
|
/** Forward-slashed archive path. */
|
|
@@ -247,8 +230,8 @@ declare const GEZK_ARCHIVE_LIMITS: {
|
|
|
247
230
|
readonly maxCompressionRatio: 400;
|
|
248
231
|
};
|
|
249
232
|
declare class GezkArchiveError extends Error {
|
|
250
|
-
readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io';
|
|
251
|
-
constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io');
|
|
233
|
+
readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io';
|
|
234
|
+
constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io');
|
|
252
235
|
}
|
|
253
236
|
interface GezkArchiveInspection {
|
|
254
237
|
manifest: KnowledgeCatalogManifest;
|
|
@@ -285,32 +268,37 @@ type DatabaseSync = DatabaseSync$1;
|
|
|
285
268
|
* service's `openIndexDatabase`, whose behaviors are all wrong for a signed
|
|
286
269
|
* publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
|
|
287
270
|
* and quarantine-RENAMES a corrupt database. A catalog reader must fail
|
|
288
|
-
* with a typed reason and never touch the bytes.
|
|
271
|
+
* with a typed reason and never touch the bytes. No extension is loaded:
|
|
272
|
+
* every table in a 0.5 catalog is plain SQLite (plus FTS5).
|
|
289
273
|
*/
|
|
290
274
|
|
|
291
275
|
declare class CatalogOpenError extends Error {
|
|
292
|
-
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | '
|
|
293
|
-
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | '
|
|
276
|
+
readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt';
|
|
277
|
+
constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt');
|
|
294
278
|
}
|
|
295
279
|
interface CatalogDb {
|
|
296
280
|
db: DatabaseSync;
|
|
297
281
|
close(): void;
|
|
298
282
|
}
|
|
299
283
|
/**
|
|
300
|
-
* Open one catalog SQLite file read-only + immutable
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
284
|
+
* Open one catalog SQLite file read-only + immutable and verify the format
|
|
285
|
+
* stamps. `immutable=1` promises SQLite the file cannot change while open —
|
|
286
|
+
* no locking, no journal probing — which is true for a published catalog
|
|
287
|
+
* version and is what makes concurrent readers free.
|
|
304
288
|
*/
|
|
305
289
|
declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
306
290
|
|
|
307
291
|
/**
|
|
308
292
|
* One mounted catalog: the router database plus lazily opened shards, and
|
|
309
|
-
* the two-stage query flow (gezk
|
|
293
|
+
* the two-stage query flow (the gezk spec §8 and §9):
|
|
310
294
|
*
|
|
311
295
|
* embed query (caller) → centroid routing → per shard: bit-hamming KNN
|
|
312
296
|
* top-K → int8 rerank → fuse.
|
|
313
297
|
*
|
|
298
|
+
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
299
|
+
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
|
|
300
|
+
* popcount, so no vector extension is needed to read a catalog.
|
|
301
|
+
*
|
|
314
302
|
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
315
303
|
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
316
304
|
* and tests call it directly.
|
|
@@ -357,6 +345,7 @@ declare class CatalogHandle {
|
|
|
357
345
|
readonly rootDir: string;
|
|
358
346
|
private readonly router;
|
|
359
347
|
private readonly connections;
|
|
348
|
+
private readonly bitIndexes;
|
|
360
349
|
readonly meta: Record<string, string>;
|
|
361
350
|
readonly shards: ShardInfo[];
|
|
362
351
|
private constructor();
|
|
@@ -372,6 +361,14 @@ declare class CatalogHandle {
|
|
|
372
361
|
unknownCodecs: number;
|
|
373
362
|
};
|
|
374
363
|
private shardDb;
|
|
364
|
+
/**
|
|
365
|
+
* A shard's sign-bit rows as one contiguous array, loaded on first use and
|
|
366
|
+
* kept while the resident budget allows. Chunk ids must be dense from 1 —
|
|
367
|
+
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
368
|
+
*/
|
|
369
|
+
private shardBits;
|
|
370
|
+
/** The embedding dimension from the router's profile echo. */
|
|
371
|
+
private dimensions;
|
|
375
372
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
376
373
|
resolveCatalogPath(path: string): string;
|
|
377
374
|
topics(): CatalogTopic[];
|
|
@@ -391,6 +388,16 @@ declare class CatalogHandle {
|
|
|
391
388
|
documentId: string;
|
|
392
389
|
rank: number;
|
|
393
390
|
}>;
|
|
391
|
+
/**
|
|
392
|
+
* Router-level shape stats: topic-tree size and total route centroids.
|
|
393
|
+
* Read-only counts for inspectors and recall/routing diagnostics — a
|
|
394
|
+
* single-topic catalog fills shards in documentId order, which is the
|
|
395
|
+
* first thing to check when centroid routing underperforms.
|
|
396
|
+
*/
|
|
397
|
+
routerStats(): {
|
|
398
|
+
topics: number;
|
|
399
|
+
routeCentroids: number;
|
|
400
|
+
};
|
|
394
401
|
/**
|
|
395
402
|
* Score every shard against the (unit float32) query: shard score = max
|
|
396
403
|
* cosine over its centroids (§3.4). This is the cross-catalog routing
|
|
@@ -419,15 +426,15 @@ declare class CatalogHandle {
|
|
|
419
426
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
420
427
|
searchChunksFts(query: string, shardIds: number[], limitPerShard?: number): CatalogChunkHit[];
|
|
421
428
|
/**
|
|
422
|
-
* Embedder-free integrity smoke
|
|
423
|
-
*
|
|
424
|
-
*
|
|
429
|
+
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
430
|
+
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
431
|
+
* false instead of throwing (callers report a reason).
|
|
425
432
|
*/
|
|
426
433
|
selfKnnSmoke(shardId: number): boolean;
|
|
427
434
|
}
|
|
428
435
|
|
|
429
436
|
/**
|
|
430
|
-
* Catalog validation (gezk
|
|
437
|
+
* Catalog validation (the gezk spec §3.3 and §5) over an EXTRACTED catalog
|
|
431
438
|
* directory: manifest parse, per-file hash reconciliation, read-only opens,
|
|
432
439
|
* count reconciliation, and — in deep mode — SQLite quick_check, per-shard
|
|
433
440
|
* vector-table alignment, the embedder-free self-KNN smoke, and the
|
|
@@ -452,32 +459,298 @@ interface ValidateCatalogOptions {
|
|
|
452
459
|
declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
|
|
453
460
|
|
|
454
461
|
/**
|
|
455
|
-
*
|
|
456
|
-
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
462
|
+
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
463
|
+
* 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
|
|
464
|
+
* popcount plus a histogram selection of the top-K is a few milliseconds and
|
|
465
|
+
* needs nothing beyond plain SQLite to load — which is what lets any reader
|
|
466
|
+
* implement the format without a vector extension.
|
|
467
|
+
*/
|
|
468
|
+
interface ShardBitIndex {
|
|
469
|
+
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
470
|
+
bits: Uint8Array;
|
|
471
|
+
bytesPerRow: number;
|
|
472
|
+
rows: number;
|
|
473
|
+
}
|
|
474
|
+
interface HammingHit {
|
|
475
|
+
chunkId: number;
|
|
476
|
+
distance: number;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* The K nearest rows by hamming distance, ascending, ties broken by chunk
|
|
480
|
+
* id. Selection is a histogram over the (small, integer) distance range, so
|
|
481
|
+
* the whole query is two linear passes and no heap.
|
|
482
|
+
*/
|
|
483
|
+
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Provenance stamped into every manifest this package compiles. Only the
|
|
487
|
+
* package identity: node/platform details would make two builds of the same
|
|
488
|
+
* inputs differ by machine, and the format promises byte-identical archives
|
|
489
|
+
* for identical inputs and toolchain.
|
|
490
|
+
*/
|
|
491
|
+
declare const KNOWLEDGE_TOOLCHAIN: NonNullable<KnowledgeCatalogManifest['toolchain']>;
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* The exporter's DuckDB seam: one short-lived CLI child per script, SQL on
|
|
495
|
+
* stdin (never in argv), stdout captured. The binary is caller-supplied — the
|
|
496
|
+
* toolchain package never downloads engines — and its reported version is
|
|
497
|
+
* checked against the one the caller expects, because Parquet bytes are only
|
|
498
|
+
* reproducible against a pinned writer.
|
|
499
|
+
*/
|
|
500
|
+
interface DuckdbCli {
|
|
501
|
+
binaryPath: string;
|
|
502
|
+
/** Refuse to run a different release (e.g. `1.5.5`); omit to accept any. */
|
|
503
|
+
expectedVersion?: string;
|
|
504
|
+
}
|
|
505
|
+
declare class DuckdbError extends Error {
|
|
506
|
+
readonly stderr: string;
|
|
507
|
+
constructor(message: string, stderr: string);
|
|
508
|
+
}
|
|
509
|
+
/** `duckdb --version` prints `v1.5.5 (Variegata) d8cdaa33fd`; the tag without its `v`. */
|
|
510
|
+
declare function duckdbVersion(binaryPath: string): Promise<string>;
|
|
511
|
+
declare function assertDuckdbCli(cli: DuckdbCli): Promise<string>;
|
|
512
|
+
/** Run a SQL script against an in-memory database; throws with DuckDB's own text on failure. */
|
|
513
|
+
declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* The Parquet companion of a catalog: `documents-NNN.parquet` and
|
|
517
|
+
* `chunks-NNN.parquet` per shard plus `topics.parquet`, derived from the
|
|
518
|
+
* sealed archive so anyone can load a catalog with DuckDB, pandas, Polars or
|
|
519
|
+
* the `datasets` library without a gezk reader. Column names are the gezk DDL
|
|
520
|
+
* names; embeddings are the int8 and bit vectors EXACTLY as the archive
|
|
521
|
+
* stores them (provably identical, one third of float32's size), with the
|
|
522
|
+
* dequantisation rule recorded in the file metadata (gezk spec §6).
|
|
459
523
|
*
|
|
460
|
-
*
|
|
461
|
-
*
|
|
462
|
-
*
|
|
463
|
-
*
|
|
524
|
+
* Determinism: rows are read in primary-key order and written single-
|
|
525
|
+
* threaded with insertion order preserved, a fixed codec and level, a fixed
|
|
526
|
+
* row-group size, and no wall-clock metadata — so two exports of one archive
|
|
527
|
+
* with one DuckDB release produce byte-identical files. The pinned DuckDB
|
|
528
|
+
* version is part of the report because the writer's framing is part of the
|
|
529
|
+
* bytes.
|
|
464
530
|
*/
|
|
465
531
|
|
|
466
|
-
|
|
467
|
-
|
|
532
|
+
declare const PARQUET_EXPORT_VERSION = 1;
|
|
533
|
+
type ParquetExportSource = {
|
|
534
|
+
archivePath: string;
|
|
535
|
+
} | {
|
|
536
|
+
rootDir: string;
|
|
537
|
+
};
|
|
538
|
+
interface ParquetExportOptions {
|
|
539
|
+
source: ParquetExportSource;
|
|
540
|
+
outDir: string;
|
|
541
|
+
duckdb: DuckdbCli;
|
|
542
|
+
rowGroupSize?: number;
|
|
543
|
+
onProgress?: (event: ParquetExportProgress) => void;
|
|
544
|
+
}
|
|
545
|
+
type ParquetExportProgress = {
|
|
546
|
+
phase: 'extract';
|
|
547
|
+
} | {
|
|
548
|
+
phase: 'stage';
|
|
549
|
+
table: ParquetTable;
|
|
550
|
+
shardId?: number;
|
|
551
|
+
rows: number;
|
|
552
|
+
} | {
|
|
553
|
+
phase: 'write';
|
|
554
|
+
file: string;
|
|
555
|
+
};
|
|
556
|
+
type ParquetTable = 'documents' | 'chunks' | 'topics';
|
|
557
|
+
interface ParquetExportFile {
|
|
558
|
+
/** Relative to `outDir`, forward slashes. */
|
|
559
|
+
path: string;
|
|
560
|
+
table: ParquetTable;
|
|
561
|
+
shardId?: number;
|
|
562
|
+
rows: number;
|
|
563
|
+
sizeBytes: number;
|
|
564
|
+
sha256: string;
|
|
565
|
+
}
|
|
566
|
+
interface ParquetExportReport {
|
|
567
|
+
exportVersion: number;
|
|
568
|
+
catalogId: string;
|
|
569
|
+
version: string;
|
|
570
|
+
publisherId: string;
|
|
571
|
+
formatVersion: string;
|
|
572
|
+
embeddingProfileId: string;
|
|
573
|
+
dimensions: number;
|
|
574
|
+
files: ParquetExportFile[];
|
|
575
|
+
duckdbVersion: string;
|
|
576
|
+
exporter: {
|
|
577
|
+
name: string;
|
|
578
|
+
version: string;
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
582
|
+
declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
|
|
583
|
+
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
584
|
+
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* The registered embedding profiles — the single source both the compiler
|
|
588
|
+
* and the reader import. A profile is the FULL vector-space identity:
|
|
589
|
+
* readers refuse to mix vectors across profile ids, and any change to model,
|
|
590
|
+
* instructions, or encoding is a NEW id.
|
|
591
|
+
*
|
|
592
|
+
* Revisions are the Hugging Face commit hashes the profiles were frozen at
|
|
593
|
+
* (2026-08-20). The artifact digests (recorded 2026-09-03) are the sha256 of
|
|
594
|
+
* the exact files at those revisions — the Hub's LFS object ids for the ONNX
|
|
595
|
+
* graphs and the e5 tokenizer, a hashed download for the bge tokenizer — and
|
|
596
|
+
* name the full-precision graph explicitly, so the precision the vectors were
|
|
597
|
+
* produced at is a pinned fact rather than a runtime default. Every embedder
|
|
598
|
+
* built from a profile verifies the files it loaded against these digests.
|
|
599
|
+
*/
|
|
600
|
+
|
|
601
|
+
/** The public profile: multilingual, so world catalogs serve every reader. */
|
|
602
|
+
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
468
603
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
469
|
-
declare const
|
|
604
|
+
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
470
605
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
471
606
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
472
|
-
/** The knowledge chunking profile
|
|
473
|
-
declare const
|
|
607
|
+
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
608
|
+
declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
609
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
610
|
+
declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
611
|
+
id: string;
|
|
612
|
+
model: {
|
|
613
|
+
repo: string;
|
|
614
|
+
revision: string;
|
|
615
|
+
onnxFile?: string | undefined;
|
|
616
|
+
onnxDigest?: string | undefined;
|
|
617
|
+
};
|
|
618
|
+
tokenizer: {
|
|
619
|
+
kind: string;
|
|
620
|
+
file?: string | undefined;
|
|
621
|
+
digest?: string | undefined;
|
|
622
|
+
};
|
|
623
|
+
pooling: "mean" | "cls" | "last";
|
|
624
|
+
normalized: boolean;
|
|
625
|
+
dimensions: number;
|
|
626
|
+
maxTokens: number;
|
|
627
|
+
queryInstruction: string;
|
|
628
|
+
passageInstruction: string;
|
|
629
|
+
vectorEncoding: "bit+int8";
|
|
630
|
+
distance: {
|
|
631
|
+
stage1: "hamming";
|
|
632
|
+
stage2: "cosine";
|
|
633
|
+
};
|
|
634
|
+
quantization: {
|
|
635
|
+
int8: {
|
|
636
|
+
method: "symmetric-linear";
|
|
637
|
+
scale: 127;
|
|
638
|
+
};
|
|
639
|
+
binary: {
|
|
640
|
+
method: "sign";
|
|
641
|
+
threshold: 0;
|
|
642
|
+
packing: "lsb-first";
|
|
643
|
+
};
|
|
644
|
+
};
|
|
645
|
+
};
|
|
646
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
647
|
+
declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
648
|
+
id: string;
|
|
649
|
+
model: {
|
|
650
|
+
repo: string;
|
|
651
|
+
revision: string;
|
|
652
|
+
onnxFile?: string | undefined;
|
|
653
|
+
onnxDigest?: string | undefined;
|
|
654
|
+
};
|
|
655
|
+
tokenizer: {
|
|
656
|
+
kind: string;
|
|
657
|
+
file?: string | undefined;
|
|
658
|
+
digest?: string | undefined;
|
|
659
|
+
};
|
|
660
|
+
pooling: "mean" | "cls" | "last";
|
|
661
|
+
normalized: boolean;
|
|
662
|
+
dimensions: number;
|
|
663
|
+
maxTokens: number;
|
|
664
|
+
queryInstruction: string;
|
|
665
|
+
passageInstruction: string;
|
|
666
|
+
vectorEncoding: "bit+int8";
|
|
667
|
+
distance: {
|
|
668
|
+
stage1: "hamming";
|
|
669
|
+
stage2: "cosine";
|
|
670
|
+
};
|
|
671
|
+
quantization: {
|
|
672
|
+
int8: {
|
|
673
|
+
method: "symmetric-linear";
|
|
674
|
+
scale: 127;
|
|
675
|
+
};
|
|
676
|
+
binary: {
|
|
677
|
+
method: "sign";
|
|
678
|
+
threshold: 0;
|
|
679
|
+
packing: "lsb-first";
|
|
680
|
+
};
|
|
681
|
+
};
|
|
682
|
+
};
|
|
683
|
+
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
684
|
+
declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
685
|
+
id: string;
|
|
686
|
+
unit: "chars" | "tokens";
|
|
687
|
+
tokenizer: "profile" | "none";
|
|
688
|
+
target: number;
|
|
689
|
+
overlap: number;
|
|
690
|
+
contextHeader: {
|
|
691
|
+
max: number;
|
|
692
|
+
};
|
|
693
|
+
};
|
|
694
|
+
|
|
695
|
+
/**
|
|
696
|
+
* Verifies the model files an embedder loaded against the digests its
|
|
697
|
+
* profile pins. transformers.js caches every fetched file under a key that
|
|
698
|
+
* mirrors the repo path — `<repo>/<file>` for `main`, `<repo>/<revision>/<file>`
|
|
699
|
+
* for a pinned revision — so after a load the exact bytes are on disk and
|
|
700
|
+
* can be hashed. That turns "same repo and revision" from a convention into
|
|
701
|
+
* a checked fact: a profile that declares digests is served only by weights
|
|
702
|
+
* whose sha256 matches them, and a mismatch is reported as a different
|
|
703
|
+
* vector space rather than silently producing slightly different vectors.
|
|
704
|
+
*/
|
|
705
|
+
|
|
706
|
+
type ArtifactRole = 'onnx' | 'tokenizer';
|
|
707
|
+
interface ArtifactCheck {
|
|
708
|
+
role: ArtifactRole;
|
|
709
|
+
/** Repo-relative file the profile pins. */
|
|
710
|
+
file: string;
|
|
711
|
+
/** Where the loaded copy was looked for. */
|
|
712
|
+
path: string;
|
|
713
|
+
expected: string;
|
|
714
|
+
/** `null` when no file was found at `path`. */
|
|
715
|
+
actual: string | null;
|
|
716
|
+
}
|
|
717
|
+
interface VerifiedArtifacts {
|
|
718
|
+
/** `unpinned` when the profile declares no digests, so nothing was checked. */
|
|
719
|
+
status: 'verified' | 'unpinned';
|
|
720
|
+
checks: ArtifactCheck[];
|
|
721
|
+
}
|
|
722
|
+
declare class EmbedderArtifactError extends Error {
|
|
723
|
+
readonly reason: 'missing' | 'mismatch';
|
|
724
|
+
readonly profileId: string;
|
|
725
|
+
readonly checks: ArtifactCheck[];
|
|
726
|
+
readonly isActionable = true;
|
|
727
|
+
constructor(reason: 'missing' | 'mismatch', profileId: string, checks: ArtifactCheck[]);
|
|
728
|
+
}
|
|
729
|
+
/** The transformers.js FileCache key for one repo file. */
|
|
730
|
+
declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
|
|
731
|
+
/**
|
|
732
|
+
* Hash the profile's pinned files in a transformers.js cache and compare.
|
|
733
|
+
* `revision` names the cache key the files were fetched under; it defaults
|
|
734
|
+
* to the profile's own revision and is `main` for a pipeline loaded
|
|
735
|
+
* unpinned. Throws `EmbedderArtifactError` on a missing or differing file.
|
|
736
|
+
*/
|
|
737
|
+
declare function verifyProfileArtifacts(profile: KnowledgeEmbeddingProfile, opts: {
|
|
738
|
+
cacheDir: string;
|
|
739
|
+
revision?: string;
|
|
740
|
+
}): Promise<VerifiedArtifacts>;
|
|
474
741
|
|
|
475
742
|
/**
|
|
476
743
|
* Profile-conformant embedder + tokenizer, driven entirely by a declarative
|
|
477
|
-
* KnowledgeEmbeddingProfile — model repo, pinned revision,
|
|
478
|
-
* instruction prefixes all come from the profile object,
|
|
479
|
-
* regexes. This is what the offline CLI wires into the
|
|
480
|
-
* explicit-search path; the daemon wires its own pipeline
|
|
744
|
+
* KnowledgeEmbeddingProfile — model repo, pinned revision, the exact ONNX
|
|
745
|
+
* graph, pooling, and instruction prefixes all come from the profile object,
|
|
746
|
+
* never from env-var regexes. This is what the offline CLI wires into the
|
|
747
|
+
* compiler and the explicit-search path; the daemon wires its own pipeline
|
|
748
|
+
* pool later.
|
|
749
|
+
*
|
|
750
|
+
* After loading, the files transformers.js fetched are hashed and compared
|
|
751
|
+
* with the digests the profile pins (artifact-verify.ts). A profile that
|
|
752
|
+
* declares digests is therefore served only by the very bytes that produced
|
|
753
|
+
* its catalogs; anything else is refused as a different vector space.
|
|
481
754
|
*
|
|
482
755
|
* `@huggingface/transformers` stays a dynamic import (mirroring the
|
|
483
756
|
* service's embed-core): the CLI must start instantly without it, and a
|
|
@@ -488,6 +761,8 @@ declare const GEZEL_MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
|
488
761
|
|
|
489
762
|
interface ProfileEmbedder {
|
|
490
763
|
readonly profile: KnowledgeEmbeddingProfile;
|
|
764
|
+
/** What the loaded model files were checked against after loading. */
|
|
765
|
+
readonly verification: VerifiedArtifacts;
|
|
491
766
|
/** Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's contract). */
|
|
492
767
|
embed(texts: string[]): Promise<number[][]>;
|
|
493
768
|
/** Query embed — applies the profile's queryInstruction itself. */
|
|
@@ -500,14 +775,55 @@ declare class EmbedderUnavailableError extends Error {
|
|
|
500
775
|
readonly isActionable = true;
|
|
501
776
|
constructor(message: string);
|
|
502
777
|
}
|
|
778
|
+
/** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
|
|
779
|
+
interface TransformersModule {
|
|
780
|
+
pipeline: (task: string, model: string, options?: Record<string, unknown>) => Promise<PipelineFn>;
|
|
781
|
+
AutoTokenizer: {
|
|
782
|
+
from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerFn>;
|
|
783
|
+
};
|
|
784
|
+
env: {
|
|
785
|
+
cacheDir?: string;
|
|
786
|
+
useFSCache?: boolean;
|
|
787
|
+
allowRemoteModels?: boolean;
|
|
788
|
+
};
|
|
789
|
+
}
|
|
790
|
+
type PipelineFn = ((texts: string[], options: {
|
|
791
|
+
pooling: string;
|
|
792
|
+
normalize: boolean;
|
|
793
|
+
}) => Promise<{
|
|
794
|
+
tolist(): number[][];
|
|
795
|
+
}>) & {
|
|
796
|
+
dispose?: () => Promise<void>;
|
|
797
|
+
};
|
|
798
|
+
interface TokenizerFn {
|
|
799
|
+
encode(text: string): number[];
|
|
800
|
+
}
|
|
801
|
+
/** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
|
|
802
|
+
interface TransformersModelOptions {
|
|
803
|
+
revision: string;
|
|
804
|
+
dtype: string;
|
|
805
|
+
subfolder: string;
|
|
806
|
+
model_file_name: string;
|
|
807
|
+
}
|
|
808
|
+
declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
|
|
503
809
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
504
810
|
cacheDir?: string;
|
|
811
|
+
/**
|
|
812
|
+
* onnxruntime session options (e.g. `{ intraOpNumThreads }`), passed
|
|
813
|
+
* through to the pipeline. Worker pools use this to divide cores between
|
|
814
|
+
* concurrent sessions instead of letting every session grab them all.
|
|
815
|
+
* Thread counts change scheduling, never numerics — batch composition is
|
|
816
|
+
* what affects values, and that stays with the caller.
|
|
817
|
+
*/
|
|
818
|
+
sessionOptions?: Record<string, unknown>;
|
|
819
|
+
/** Test seam: a stand-in for `@huggingface/transformers`. */
|
|
820
|
+
transformers?: TransformersModule;
|
|
505
821
|
}): Promise<ProfileEmbedder>;
|
|
506
822
|
|
|
507
823
|
/**
|
|
508
824
|
* Markdown-folder catalog source — the `gezel knowledge build` input adapter.
|
|
509
825
|
* Folders become the shipped table of contents (a format requirement:
|
|
510
|
-
* gezk
|
|
826
|
+
* the gezk spec §5.2), files become documents. A flat corpus — files at
|
|
511
827
|
* the root with no subfolders — gets the single root topic the compiler
|
|
512
828
|
* demands. Deterministic: sorted walk, stable topic-id collision suffixes.
|
|
513
829
|
*/
|
|
@@ -530,53 +846,82 @@ interface LoadMarkdownCatalogOptions {
|
|
|
530
846
|
declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
|
|
531
847
|
|
|
532
848
|
/**
|
|
533
|
-
*
|
|
534
|
-
*
|
|
535
|
-
*
|
|
536
|
-
*
|
|
537
|
-
*
|
|
538
|
-
*
|
|
849
|
+
* Signed-registry consumption (knowledge-catalogs.md Phase 6): fetch a
|
|
850
|
+
* publisher's `_knowledge/registry/index.json`, verify its Ed25519 signature
|
|
851
|
+
* against keyId-indexed trust anchors, and locate archive coordinates.
|
|
852
|
+
*
|
|
853
|
+
* Trust posture: the registry only LOCATES bytes. Content trust stays with
|
|
854
|
+
* the coordinate's `contentDigest` — every consumer re-hashes the downloaded
|
|
855
|
+
* archive against the digest it already trusted, so a compromised registry
|
|
856
|
+
* (or CDN) can at worst fail to locate, never substitute content. Signature
|
|
857
|
+
* verification here protects the locate step itself: an unverified registry
|
|
858
|
+
* cannot steer downloads at attacker-sized garbage or hide/reorder releases.
|
|
539
859
|
*/
|
|
540
|
-
declare function canonicalizeJson(value: unknown): string;
|
|
541
860
|
|
|
861
|
+
type KnowledgeRegistryFetchCode = 'http' | 'too-large' | 'invalid' | 'unsigned-or-untrusted';
|
|
862
|
+
declare class KnowledgeRegistryFetchError extends Error {
|
|
863
|
+
readonly code: KnowledgeRegistryFetchCode;
|
|
864
|
+
constructor(code: KnowledgeRegistryFetchCode, message: string);
|
|
865
|
+
}
|
|
866
|
+
interface FetchKnowledgeRegistryOptions {
|
|
867
|
+
anchors: readonly KnowledgeTrustAnchor[];
|
|
868
|
+
maxBytes?: number;
|
|
869
|
+
/** Budget for the whole fetch when no `signal` is supplied. */
|
|
870
|
+
timeoutMs?: number;
|
|
871
|
+
/** Caller-owned deadline (the daemon passes a suspend-aware signal). */
|
|
872
|
+
signal?: AbortSignal;
|
|
873
|
+
fetchImpl?: typeof fetch;
|
|
874
|
+
}
|
|
542
875
|
/**
|
|
543
|
-
*
|
|
544
|
-
*
|
|
545
|
-
* encodings follow the repo's identity conventions (remotes/identity.ts):
|
|
546
|
-
* SPKI/PKCS#8 PEM, base64 signatures, node:crypto one-shot sign/verify.
|
|
547
|
-
* The private key never appears in CI — signing is a manual workstation
|
|
548
|
-
* step (the plan's key-custody stance); verification is what ships.
|
|
876
|
+
* Fetch + schema-parse + signature-verify a registry. Throws
|
|
877
|
+
* `KnowledgeRegistryFetchError`; never returns an unverified document.
|
|
549
878
|
*/
|
|
550
|
-
|
|
551
|
-
|
|
879
|
+
declare function fetchKnowledgeRegistry(url: string, opts: FetchKnowledgeRegistryOptions): Promise<{
|
|
880
|
+
registry: KnowledgeRegistryIndex;
|
|
552
881
|
keyId: string;
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
};
|
|
565
|
-
/**
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
};
|
|
572
|
-
/** The exact bytes the signature covers: JCS(manifest minus `signature`). */
|
|
573
|
-
declare function manifestSigningPayload(manifest: KnowledgeCatalogManifest): Buffer;
|
|
574
|
-
declare function signManifest(manifest: KnowledgeCatalogManifest, privateKeyPem: string): KnowledgeCatalogManifest;
|
|
882
|
+
}>;
|
|
883
|
+
/**
|
|
884
|
+
* Locate the row for an exact trusted coordinate. `contentDigest`, when
|
|
885
|
+
* given, must match the row — a registry that renames a digest under an
|
|
886
|
+
* existing (catalogId, version) simply fails to locate.
|
|
887
|
+
*/
|
|
888
|
+
declare function findRegistryEntry(registry: KnowledgeRegistryIndex, ref: {
|
|
889
|
+
publisherId: string;
|
|
890
|
+
catalogId: string;
|
|
891
|
+
version: string;
|
|
892
|
+
contentDigest?: string;
|
|
893
|
+
}): KnowledgeRegistryEntry | null;
|
|
894
|
+
/**
|
|
895
|
+
* Compare two catalog versions (CalVer-style dotted numerics, e.g.
|
|
896
|
+
* `2026.08.1`). Non-numeric segments fall back to lexicographic comparison
|
|
897
|
+
* so unconventional versions still order deterministically.
|
|
898
|
+
*/
|
|
899
|
+
declare function compareCatalogVersions(a: string, b: string): number;
|
|
575
900
|
/**
|
|
576
|
-
*
|
|
577
|
-
*
|
|
578
|
-
*
|
|
901
|
+
* Rows offering a strictly newer version of an installed catalog, newest
|
|
902
|
+
* first. Rows whose version does not exceed the installed one are ignored —
|
|
903
|
+
* a registry cannot "update" an install to an older or renamed release.
|
|
579
904
|
*/
|
|
580
|
-
declare function
|
|
905
|
+
declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installed: {
|
|
906
|
+
catalogId: string;
|
|
907
|
+
version: string;
|
|
908
|
+
}): KnowledgeRegistryEntry[];
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* Built-in trust anchors for signed knowledge registries and catalogs.
|
|
912
|
+
*
|
|
913
|
+
* Empty until the Qualla production keypair exists: the private key is
|
|
914
|
+
* generated OFFLINE on a workstation (`sign-knowledge-release --generate-key`)
|
|
915
|
+
* and its `.pub` lands here — pasted into a normal reviewed release, never
|
|
916
|
+
* fetched at runtime. Rotation = overlapping keyIds across an app release:
|
|
917
|
+
* add the new anchor, keep the old one until every registry consumers care
|
|
918
|
+
* about is re-signed, then remove it.
|
|
919
|
+
*
|
|
920
|
+
* Operators and tests extend the set without a release via
|
|
921
|
+
* `GEZEL_KNOWLEDGE_TRUST_ANCHORS` (a JSON file of KnowledgeTrustAnchor rows),
|
|
922
|
+
* which the service layers ON TOP of these — it can add trust, not veto it.
|
|
923
|
+
*/
|
|
924
|
+
|
|
925
|
+
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
581
926
|
|
|
582
|
-
export { type ArchiveEntry,
|
|
927
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TokenChunkerOptions, type TokenizerFn, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newerRegistryEntries, openCatalogDatabase, readGezkManifest, rerankK, resolveTransformersModelOptions, runDuckdbScript, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|