@bendyline/gezel-knowledge 1.0.7 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +28 -17
  2. package/dist/index.d.ts +376 -157
  3. package/dist/index.js +1061 -421
  4. package/package.json +5 -4
package/dist/index.d.ts CHANGED
@@ -1,21 +1,14 @@
1
- import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezel';
2
- export { CatalogDocument, KnowledgeCatalogManifest, KnowledgeCatalogRef, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex } from '@bendyline/gezel';
1
+ import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
2
+ export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
3
+ import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
4
+ export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
3
5
  import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
4
6
 
5
- /** `.gezk` v1 constants — frozen in docs/gezk-format-v1.md. */
6
- declare const GEZK_FORMAT_VERSION = 1;
7
- declare const GEZK_INDEX_SCHEMA_VERSION = 1;
8
- /** `PRAGMA application_id` for every database in a catalog: 'GEZK'. */
9
- declare const GEZK_APPLICATION_ID = 1195727435;
10
- declare const ROUTER_DB_PATH = "index/router.db";
11
- declare const MANIFEST_PATH = "manifest.json";
7
+ /** Toolchain knobs. Format-level constants come from @bendyline/gezk. */
8
+
12
9
  /** Compiler shard sizing (§3.1). */
13
10
  declare const SHARD_TARGET_CHUNKS = 200000;
14
11
  declare const SHARD_MAX_CHUNKS = 250000;
15
- /** Bodies below this ship uncompressed (`body_codec = 'none'`). */
16
- declare const BODY_CODEC_MIN_BYTES = 512;
17
- /** Maximum decompressed Markdown body returned from one catalog document. */
18
- declare const MAX_KNOWLEDGE_DOCUMENT_BYTES: number;
19
12
  /** Routing (§3.3–3.4). */
20
13
  declare const CENTROID_CHUNKS_PER = 12500;
21
14
  declare const CENTROID_MAX_PER_SHARD = 32;
@@ -26,58 +19,17 @@ declare const ROUTE_SHARDS_EXPLICIT = 6;
26
19
  /** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
27
20
  declare const RERANK_FINAL_K = 24;
28
21
  declare function rerankK(finalK?: number, chunkCount?: number): number;
29
- /** Fixed ZIP entry timestamp for deterministic archives (§11). */
30
- declare const ZIP_FIXED_MTIME: Date;
31
-
32
- /** Frozen v1 DDL (gezk-format-v1.md §6). Readers never migrate these. */
33
- declare const ROUTER_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE topics(\n id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,\n description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL\n);\n\nCREATE TABLE documents(\n id TEXT PRIMARY KEY,\n title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,\n language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),\n shard_id INTEGER NOT NULL,\n chunk_count INTEGER NOT NULL,\n source_url TEXT, source_revision TEXT, source_updated_at TEXT,\n attribution_json TEXT,\n body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),\n body_blob BLOB NOT NULL\n);\nCREATE INDEX documents_topic ON documents(topic_id, slug);\nCREATE INDEX documents_shard ON documents(shard_id);\n\nCREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,\n PRIMARY KEY (alias, document_id)) WITHOUT ROWID;\n\nCREATE TABLE shards(\n id INTEGER PRIMARY KEY,\n path TEXT NOT NULL,\n chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,\n topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,\n bytes INTEGER NOT NULL\n);\n\nCREATE TABLE route_centroids(\n id INTEGER PRIMARY KEY,\n shard_id INTEGER NOT NULL REFERENCES shards(id),\n embedding BLOB NOT NULL,\n weight INTEGER NOT NULL\n);\nCREATE INDEX route_centroids_shard ON route_centroids(shard_id);\n\nCREATE VIRTUAL TABLE fts_documents USING fts5(\n title, summary, aliases, document_id UNINDEXED,\n tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'\n);\n";
34
- /** Per-shard tables. The vec0 table is created separately (needs sqlite-vec). */
35
- declare const SHARD_DDL = "\nCREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;\n\nCREATE TABLE chunks(\n id INTEGER PRIMARY KEY,\n chunk_uid TEXT NOT NULL,\n document_id TEXT NOT NULL,\n ordinal INTEGER NOT NULL,\n title TEXT NOT NULL,\n heading_path TEXT NOT NULL,\n heading_text TEXT NOT NULL,\n line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,\n token_count INTEGER NOT NULL,\n content_hash TEXT NOT NULL,\n text TEXT NOT NULL\n);\nCREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);\nCREATE INDEX chunks_document ON chunks(document_id, ordinal);\n\nCREATE VIRTUAL TABLE fts_chunks USING fts5(\n title, heading_text, text,\n content = 'chunks', content_rowid = 'id',\n tokenize = 'unicode61 remove_diacritics 2'\n);\n\nCREATE TABLE chunk_vectors_int8(\n chunk_id INTEGER PRIMARY KEY,\n v BLOB NOT NULL\n);\n";
36
- /** vec0 table for the hamming stage; `dim` comes from the embedding profile. */
37
- declare function vecChunksDdl(dim: number): string;
38
-
39
- /**
40
- * Stable chunk id (gezk-format-v1.md §5): lowercase hex of the FIRST 16
41
- * bytes of SHA-256(docId || 0x00 || decimal-ordinal || 0x00 ||
42
- * SHA-256(chunkText) raw 32 bytes). Content-derived so identical inputs on
43
- * any toolchain produce identical ids — the anchor of both reproducibility
44
- * and the `#chunk=` citation fragment.
45
- */
46
- declare function chunkUid(documentId: string, ordinal: number, chunkText: string): string;
47
- /** Full SHA-256 hex of a chunk's text (`chunks.content_hash`). */
48
- declare function chunkContentHash(chunkText: string): string;
49
- /** URL-ish slug for a document title (display/dedup aid, not an identity). */
50
- declare function documentSlug(title: string): string;
51
-
52
- /**
53
- * Frozen quantization formulas (gezk-format-v1.md §2). Deliberately computed
54
- * here in TypeScript rather than via sqlite-vec's `vec_quantize_*` scalar
55
- * functions, so extension-internal semantics can never drift the format:
56
- * both the compiler (packing) and the reader (query packing + rerank) call
57
- * exactly these functions.
58
- */
59
- /** int8: symmetric linear, scale 127, −128 never produced. */
60
- declare function quantizeInt8(unitVector: ArrayLike<number>): Int8Array;
61
- /** binary: sign threshold 0 (exact 0.0 → 0), packed LSB-first. */
62
- declare function quantizeBinary(unitVector: ArrayLike<number>): Uint8Array;
63
- /**
64
- * Rerank score: dot(float32 unit query, dequantized int8 passage). Passages
65
- * were unit vectors, so this approximates cosine; the query is not quantized.
66
- */
67
- declare function rerankScore(query: ArrayLike<number>, passageInt8: Int8Array): number;
68
- /** L2-normalize in place-free fashion; throws on a degenerate vector. */
69
- declare function l2Normalize(vector: ArrayLike<number>): Float32Array;
70
22
 
71
23
  /**
72
24
  * The canonical markdown chunker, parameterized over the two frozen
73
- * profiles (docs/gezk-format-v1.md §5):
25
+ * profiles (the gezk spec §7):
74
26
  *
75
- * - `gezel-markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
27
+ * - `markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
76
28
  * algorithm lifted from the project indexer
77
29
  * (packages/service/src/index-store/docs.ts `chunkMarkdown` /
78
30
  * `chunkLineRange`), so the service can later re-point here with zero
79
31
  * behavior change.
80
- * - `gezel-markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
32
+ * - `markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
81
33
  * profile. Token counting is injected (`countTokens` — the embedding
82
34
  * profile's own tokenizer), packing is per-line with sentence-level
83
35
  * splitting for oversized lines, and every chunk carries a structural
@@ -116,7 +68,7 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
116
68
  declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
117
69
 
118
70
  /**
119
- * `.gezk` compiler (gezk-format-v1.md). One owner of normalization, chunk
71
+ * `.gezk` compiler (gezk 0.5; spec in bendyline/gezk). One owner of normalization, chunk
120
72
  * ids, embedding-profile conformance, SQLite schema, shard assignment,
121
73
  * centroid routing, and deterministic archive layout — the CLI's Markdown
122
74
  * adapter and Qualla's Wikipedia adapter both feed this API.
@@ -157,6 +109,7 @@ interface CompileKnowledgeCatalogOptions {
157
109
  createdAt: string;
158
110
  license: {
159
111
  name: string;
112
+ spdx?: string;
160
113
  noticePath?: string;
161
114
  attributionRequired: boolean;
162
115
  };
@@ -202,8 +155,13 @@ interface CompileKnowledgeCatalogOptions {
202
155
  * know in advance which titles win the BM25 rank race.
203
156
  */
204
157
  smokeQueryPolicy?: 'require' | 'select';
158
+ /** Provenance; defaults to this package's identity. */
205
159
  toolchain?: KnowledgeCatalogManifest['toolchain'];
206
- /** Extra archive files (README.md, LICENSES/…): path → utf8 content. */
160
+ /**
161
+ * Extra archive files (path → utf8 content). `README.md` and
162
+ * `LICENSES/catalog.txt` are always shipped: minimal ones are generated
163
+ * from the catalog block unless supplied here.
164
+ */
207
165
  extraFiles?: Record<string, string>;
208
166
  /**
209
167
  * Last touch before the manifest is archived — the signing seam
@@ -219,7 +177,7 @@ interface CompileKnowledgeCatalogOptions {
219
177
  }) => void;
220
178
  embedBatchSize?: number;
221
179
  /**
222
- * TEST/EVAL override of the 200k shard target (gezk-format-v1.md §3.1) —
180
+ * TEST/EVAL override of the 200k shard target (the gezk spec §8.1) —
223
181
  * lets a small corpus build multi-shard so routing recall can be measured
224
182
  * against a scan-all control without a 2M-document corpus. Production
225
183
  * builds must not set it: the constant IS the format's scale design.
@@ -236,11 +194,13 @@ interface CompileReport {
236
194
  declare function compileKnowledgeCatalog(opts: CompileKnowledgeCatalogOptions): Promise<CompileReport>;
237
195
 
238
196
  /**
239
- * Deterministic `.gezk` ZIP writer (gezk-format-v1.md §11): entries sorted
240
- * by path bytes, fixed timestamp (2000-01-01T00:00:00Z), mode 0644, UTF-8
241
- * names, ZIP64 as needed (yazl handles the transition). Compression is
242
- * zlib's default deflate — deterministic per toolchain, which is exactly the
243
- * determinism promise (the manifest records the toolchain fingerprint).
197
+ * Deterministic `.gezk` ZIP writer: the stored `mimetype` entry first (the
198
+ * container's magic), then entries sorted by path bytes, fixed timestamp
199
+ * (2000-01-01T00:00:00Z), mode 0644, UTF-8 names, ZIP64 as needed (yazl
200
+ * handles the transition). Every entry is STORED, not deflated: SQLite pages
201
+ * carrying brotli bodies compress little, byte identity then no longer
202
+ * depends on the zlib build, and content-addressed hosting (Hugging Face
203
+ * Xet) can deduplicate unchanged pages across releases.
244
204
  */
245
205
  interface ArchiveEntry {
246
206
  /** Forward-slashed archive path. */
@@ -270,8 +230,8 @@ declare const GEZK_ARCHIVE_LIMITS: {
270
230
  readonly maxCompressionRatio: 400;
271
231
  };
272
232
  declare class GezkArchiveError extends Error {
273
- readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io';
274
- constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'io');
233
+ readonly reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io';
234
+ constructor(message: string, reason: 'unsafe-entry' | 'limits' | 'manifest' | 'undeclared-file' | 'missing-file' | 'hash-mismatch' | 'mimetype' | 'format-version' | 'io');
275
235
  }
276
236
  interface GezkArchiveInspection {
277
237
  manifest: KnowledgeCatalogManifest;
@@ -308,32 +268,37 @@ type DatabaseSync = DatabaseSync$1;
308
268
  * service's `openIndexDatabase`, whose behaviors are all wrong for a signed
309
269
  * publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
310
270
  * and quarantine-RENAMES a corrupt database. A catalog reader must fail
311
- * with a typed reason and never touch the bytes.
271
+ * with a typed reason and never touch the bytes. No extension is loaded:
272
+ * every table in a 0.5 catalog is plain SQLite (plus FTS5).
312
273
  */
313
274
 
314
275
  declare class CatalogOpenError extends Error {
315
- readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'vec-unavailable' | 'corrupt';
316
- constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'vec-unavailable' | 'corrupt');
276
+ readonly reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt';
277
+ constructor(message: string, reason: 'not-found' | 'not-a-catalog' | 'schema-version' | 'corrupt');
317
278
  }
318
279
  interface CatalogDb {
319
280
  db: DatabaseSync;
320
281
  close(): void;
321
282
  }
322
283
  /**
323
- * Open one catalog SQLite file read-only + immutable, load sqlite-vec, and
324
- * verify the format stamps. `immutable=1` promises SQLite the file cannot
325
- * change while open — no locking, no journal probing — which is true for a
326
- * published catalog version and is what makes concurrent readers free.
284
+ * Open one catalog SQLite file read-only + immutable and verify the format
285
+ * stamps. `immutable=1` promises SQLite the file cannot change while open —
286
+ * no locking, no journal probing — which is true for a published catalog
287
+ * version and is what makes concurrent readers free.
327
288
  */
328
289
  declare function openCatalogDatabase(absPath: string): CatalogDb;
329
290
 
330
291
  /**
331
292
  * One mounted catalog: the router database plus lazily opened shards, and
332
- * the two-stage query flow (gezk-format-v1.md §3, §9):
293
+ * the two-stage query flow (the gezk spec §8 and §9):
333
294
  *
334
295
  * embed query (caller) → centroid routing → per shard: bit-hamming KNN
335
296
  * top-K → int8 rerank → fuse.
336
297
  *
298
+ * Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
299
+ * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
300
+ * popcount, so no vector extension is needed to read a catalog.
301
+ *
337
302
  * The handle is synchronous (node:sqlite) and single-threaded by design —
338
303
  * the daemon confines every handle to its knowledge worker thread; the CLI
339
304
  * and tests call it directly.
@@ -380,6 +345,7 @@ declare class CatalogHandle {
380
345
  readonly rootDir: string;
381
346
  private readonly router;
382
347
  private readonly connections;
348
+ private readonly bitIndexes;
383
349
  readonly meta: Record<string, string>;
384
350
  readonly shards: ShardInfo[];
385
351
  private constructor();
@@ -395,6 +361,14 @@ declare class CatalogHandle {
395
361
  unknownCodecs: number;
396
362
  };
397
363
  private shardDb;
364
+ /**
365
+ * A shard's sign-bit rows as one contiguous array, loaded on first use and
366
+ * kept while the resident budget allows. Chunk ids must be dense from 1 —
367
+ * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
368
+ */
369
+ private shardBits;
370
+ /** The embedding dimension from the router's profile echo. */
371
+ private dimensions;
398
372
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
399
373
  resolveCatalogPath(path: string): string;
400
374
  topics(): CatalogTopic[];
@@ -452,15 +426,15 @@ declare class CatalogHandle {
452
426
  /** Chunk-body FTS over the routed shards (explicit search only). */
453
427
  searchChunksFts(query: string, shardIds: number[], limitPerShard?: number): CatalogChunkHit[];
454
428
  /**
455
- * Embedder-free integrity smoke (§6.3): chunk 1's own bit vector must be
456
- * its own nearest neighbor — proves the extension parses this file's vec
457
- * blobs. Returns false instead of throwing (callers report a reason).
429
+ * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
430
+ * nearest neighbor — proves the shard's vectors load and scan. Returns
431
+ * false instead of throwing (callers report a reason).
458
432
  */
459
433
  selfKnnSmoke(shardId: number): boolean;
460
434
  }
461
435
 
462
436
  /**
463
- * Catalog validation (gezk-format-v1.md §6.3) over an EXTRACTED catalog
437
+ * Catalog validation (the gezk spec §3.3 and §5) over an EXTRACTED catalog
464
438
  * directory: manifest parse, per-file hash reconciliation, read-only opens,
465
439
  * count reconciliation, and — in deep mode — SQLite quick_check, per-shard
466
440
  * vector-table alignment, the embedder-free self-KNN smoke, and the
@@ -485,32 +459,298 @@ interface ValidateCatalogOptions {
485
459
  declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
486
460
 
487
461
  /**
488
- * The registered embedding profiles (gezk-format-v1.md §4) — the single
489
- * source both the compiler and the reader import. A profile is the FULL
490
- * vector-space identity: readers refuse to mix vectors across profile ids,
491
- * and any change to model, instructions, or encoding is a NEW id.
462
+ * Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
463
+ * 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
464
+ * popcount plus a histogram selection of the top-K is a few milliseconds and
465
+ * needs nothing beyond plain SQLite to load — which is what lets any reader
466
+ * implement the format without a vector extension.
467
+ */
468
+ interface ShardBitIndex {
469
+ /** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
470
+ bits: Uint8Array;
471
+ bytesPerRow: number;
472
+ rows: number;
473
+ }
474
+ interface HammingHit {
475
+ chunkId: number;
476
+ distance: number;
477
+ }
478
+ /**
479
+ * The K nearest rows by hamming distance, ascending, ties broken by chunk
480
+ * id. Selection is a histogram over the (small, integer) distance range, so
481
+ * the whole query is two linear passes and no heap.
482
+ */
483
+ declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
484
+
485
+ /**
486
+ * Provenance stamped into every manifest this package compiles. Only the
487
+ * package identity: node/platform details would make two builds of the same
488
+ * inputs differ by machine, and the format promises byte-identical archives
489
+ * for identical inputs and toolchain.
490
+ */
491
+ declare const KNOWLEDGE_TOOLCHAIN: NonNullable<KnowledgeCatalogManifest['toolchain']>;
492
+
493
+ /**
494
+ * The exporter's DuckDB seam: one short-lived CLI child per script, SQL on
495
+ * stdin (never in argv), stdout captured. The binary is caller-supplied — the
496
+ * toolchain package never downloads engines — and its reported version is
497
+ * checked against the one the caller expects, because Parquet bytes are only
498
+ * reproducible against a pinned writer.
499
+ */
500
+ interface DuckdbCli {
501
+ binaryPath: string;
502
+ /** Refuse to run a different release (e.g. `1.5.5`); omit to accept any. */
503
+ expectedVersion?: string;
504
+ }
505
+ declare class DuckdbError extends Error {
506
+ readonly stderr: string;
507
+ constructor(message: string, stderr: string);
508
+ }
509
+ /** `duckdb --version` prints `v1.5.5 (Variegata) d8cdaa33fd`; the tag without its `v`. */
510
+ declare function duckdbVersion(binaryPath: string): Promise<string>;
511
+ declare function assertDuckdbCli(cli: DuckdbCli): Promise<string>;
512
+ /** Run a SQL script against an in-memory database; throws with DuckDB's own text on failure. */
513
+ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
514
+
515
+ /**
516
+ * The Parquet companion of a catalog: `documents-NNN.parquet` and
517
+ * `chunks-NNN.parquet` per shard plus `topics.parquet`, derived from the
518
+ * sealed archive so anyone can load a catalog with DuckDB, pandas, Polars or
519
+ * the `datasets` library without a gezk reader. Column names are the gezk DDL
520
+ * names; embeddings are the int8 and bit vectors EXACTLY as the archive
521
+ * stores them (provably identical, one third of float32's size), with the
522
+ * dequantisation rule recorded in the file metadata (gezk spec §6).
492
523
  *
493
- * Revisions are the HF commit hashes captured at Phase-0 freeze
494
- * (2026-08-20). The onnx/tokenizer digests are recorded when the release
495
- * pipeline first pulls the artifacts (Phase 5/6 hardening); their absence
496
- * does not weaken catalog verification, which pins vectors by content.
524
+ * Determinism: rows are read in primary-key order and written single-
525
+ * threaded with insertion order preserved, a fixed codec and level, a fixed
526
+ * row-group size, and no wall-clock metadata — so two exports of one archive
527
+ * with one DuckDB release produce byte-identical files. The pinned DuckDB
528
+ * version is part of the report because the writer's framing is part of the
529
+ * bytes.
497
530
  */
498
531
 
499
- /** The public Qualla profile (user decision: multilingual for world catalogs). */
500
- declare const GEZEL_MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
532
+ declare const PARQUET_EXPORT_VERSION = 1;
533
+ type ParquetExportSource = {
534
+ archivePath: string;
535
+ } | {
536
+ rootDir: string;
537
+ };
538
+ interface ParquetExportOptions {
539
+ source: ParquetExportSource;
540
+ outDir: string;
541
+ duckdb: DuckdbCli;
542
+ rowGroupSize?: number;
543
+ onProgress?: (event: ParquetExportProgress) => void;
544
+ }
545
+ type ParquetExportProgress = {
546
+ phase: 'extract';
547
+ } | {
548
+ phase: 'stage';
549
+ table: ParquetTable;
550
+ shardId?: number;
551
+ rows: number;
552
+ } | {
553
+ phase: 'write';
554
+ file: string;
555
+ };
556
+ type ParquetTable = 'documents' | 'chunks' | 'topics';
557
+ interface ParquetExportFile {
558
+ /** Relative to `outDir`, forward slashes. */
559
+ path: string;
560
+ table: ParquetTable;
561
+ shardId?: number;
562
+ rows: number;
563
+ sizeBytes: number;
564
+ sha256: string;
565
+ }
566
+ interface ParquetExportReport {
567
+ exportVersion: number;
568
+ catalogId: string;
569
+ version: string;
570
+ publisherId: string;
571
+ formatVersion: string;
572
+ embeddingProfileId: string;
573
+ dimensions: number;
574
+ files: ParquetExportFile[];
575
+ duckdbVersion: string;
576
+ exporter: {
577
+ name: string;
578
+ version: string;
579
+ };
580
+ }
581
+ declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
582
+ declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
583
+ /** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
584
+ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
585
+
586
+ /**
587
+ * The registered embedding profiles — the single source both the compiler
588
+ * and the reader import. A profile is the FULL vector-space identity:
589
+ * readers refuse to mix vectors across profile ids, and any change to model,
590
+ * instructions, or encoding is a NEW id.
591
+ *
592
+ * Revisions are the Hugging Face commit hashes the profiles were frozen at
593
+ * (2026-08-20). The artifact digests (recorded 2026-09-03) are the sha256 of
594
+ * the exact files at those revisions — the Hub's LFS object ids for the ONNX
595
+ * graphs and the e5 tokenizer, a hashed download for the bge tokenizer — and
596
+ * name the full-precision graph explicitly, so the precision the vectors were
597
+ * produced at is a pinned fact rather than a runtime default. Every embedder
598
+ * built from a profile verifies the files it loaded against these digests.
599
+ */
600
+
601
+ /** The public profile: multilingual, so world catalogs serve every reader. */
602
+ declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
501
603
  /** The local-build default — matches gezel's shipped project embedder. */
502
- declare const GEZEL_BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
604
+ declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
503
605
  declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
504
606
  declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
505
- /** The knowledge chunking profile (gezk-format-v1.md §5). */
506
- declare const GEZEL_MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
607
+ /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
608
+ declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
609
+ /** @deprecated gezk 0.4 names; removed with the next minor. */
610
+ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
611
+ id: string;
612
+ model: {
613
+ repo: string;
614
+ revision: string;
615
+ onnxFile?: string | undefined;
616
+ onnxDigest?: string | undefined;
617
+ };
618
+ tokenizer: {
619
+ kind: string;
620
+ file?: string | undefined;
621
+ digest?: string | undefined;
622
+ };
623
+ pooling: "mean" | "cls" | "last";
624
+ normalized: boolean;
625
+ dimensions: number;
626
+ maxTokens: number;
627
+ queryInstruction: string;
628
+ passageInstruction: string;
629
+ vectorEncoding: "bit+int8";
630
+ distance: {
631
+ stage1: "hamming";
632
+ stage2: "cosine";
633
+ };
634
+ quantization: {
635
+ int8: {
636
+ method: "symmetric-linear";
637
+ scale: 127;
638
+ };
639
+ binary: {
640
+ method: "sign";
641
+ threshold: 0;
642
+ packing: "lsb-first";
643
+ };
644
+ };
645
+ };
646
+ /** @deprecated gezk 0.4 names; removed with the next minor. */
647
+ declare const GEZEL_BGE_SMALL_EN_V15_1: {
648
+ id: string;
649
+ model: {
650
+ repo: string;
651
+ revision: string;
652
+ onnxFile?: string | undefined;
653
+ onnxDigest?: string | undefined;
654
+ };
655
+ tokenizer: {
656
+ kind: string;
657
+ file?: string | undefined;
658
+ digest?: string | undefined;
659
+ };
660
+ pooling: "mean" | "cls" | "last";
661
+ normalized: boolean;
662
+ dimensions: number;
663
+ maxTokens: number;
664
+ queryInstruction: string;
665
+ passageInstruction: string;
666
+ vectorEncoding: "bit+int8";
667
+ distance: {
668
+ stage1: "hamming";
669
+ stage2: "cosine";
670
+ };
671
+ quantization: {
672
+ int8: {
673
+ method: "symmetric-linear";
674
+ scale: 127;
675
+ };
676
+ binary: {
677
+ method: "sign";
678
+ threshold: 0;
679
+ packing: "lsb-first";
680
+ };
681
+ };
682
+ };
683
+ /** @deprecated gezk 0.4 names; removed with the next minor. */
684
+ declare const GEZEL_MARKDOWN_CHUNKS_2: {
685
+ id: string;
686
+ unit: "chars" | "tokens";
687
+ tokenizer: "profile" | "none";
688
+ target: number;
689
+ overlap: number;
690
+ contextHeader: {
691
+ max: number;
692
+ };
693
+ };
694
+
695
+ /**
696
+ * Verifies the model files an embedder loaded against the digests its
697
+ * profile pins. transformers.js caches every fetched file under a key that
698
+ * mirrors the repo path — `<repo>/<file>` for `main`, `<repo>/<revision>/<file>`
699
+ * for a pinned revision — so after a load the exact bytes are on disk and
700
+ * can be hashed. That turns "same repo and revision" from a convention into
701
+ * a checked fact: a profile that declares digests is served only by weights
702
+ * whose sha256 matches them, and a mismatch is reported as a different
703
+ * vector space rather than silently producing slightly different vectors.
704
+ */
705
+
706
+ type ArtifactRole = 'onnx' | 'tokenizer';
707
+ interface ArtifactCheck {
708
+ role: ArtifactRole;
709
+ /** Repo-relative file the profile pins. */
710
+ file: string;
711
+ /** Where the loaded copy was looked for. */
712
+ path: string;
713
+ expected: string;
714
+ /** `null` when no file was found at `path`. */
715
+ actual: string | null;
716
+ }
717
+ interface VerifiedArtifacts {
718
+ /** `unpinned` when the profile declares no digests, so nothing was checked. */
719
+ status: 'verified' | 'unpinned';
720
+ checks: ArtifactCheck[];
721
+ }
722
+ declare class EmbedderArtifactError extends Error {
723
+ readonly reason: 'missing' | 'mismatch';
724
+ readonly profileId: string;
725
+ readonly checks: ArtifactCheck[];
726
+ readonly isActionable = true;
727
+ constructor(reason: 'missing' | 'mismatch', profileId: string, checks: ArtifactCheck[]);
728
+ }
729
+ /** The transformers.js FileCache key for one repo file. */
730
+ declare function transformersCachePath(cacheDir: string, repo: string, revision: string, file: string): string;
731
+ /**
732
+ * Hash the profile's pinned files in a transformers.js cache and compare.
733
+ * `revision` names the cache key the files were fetched under; it defaults
734
+ * to the profile's own revision and is `main` for a pipeline loaded
735
+ * unpinned. Throws `EmbedderArtifactError` on a missing or differing file.
736
+ */
737
+ declare function verifyProfileArtifacts(profile: KnowledgeEmbeddingProfile, opts: {
738
+ cacheDir: string;
739
+ revision?: string;
740
+ }): Promise<VerifiedArtifacts>;
507
741
 
508
742
  /**
509
743
  * Profile-conformant embedder + tokenizer, driven entirely by a declarative
510
- * KnowledgeEmbeddingProfile — model repo, pinned revision, pooling, and
511
- * instruction prefixes all come from the profile object, never from env-var
512
- * regexes. This is what the offline CLI wires into the compiler and the
513
- * explicit-search path; the daemon wires its own pipeline pool later.
744
+ * KnowledgeEmbeddingProfile — model repo, pinned revision, the exact ONNX
745
+ * graph, pooling, and instruction prefixes all come from the profile object,
746
+ * never from env-var regexes. This is what the offline CLI wires into the
747
+ * compiler and the explicit-search path; the daemon wires its own pipeline
748
+ * pool later.
749
+ *
750
+ * After loading, the files transformers.js fetched are hashed and compared
751
+ * with the digests the profile pins (artifact-verify.ts). A profile that
752
+ * declares digests is therefore served only by the very bytes that produced
753
+ * its catalogs; anything else is refused as a different vector space.
514
754
  *
515
755
  * `@huggingface/transformers` stays a dynamic import (mirroring the
516
756
  * service's embed-core): the CLI must start instantly without it, and a
@@ -521,6 +761,8 @@ declare const GEZEL_MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
521
761
 
522
762
  interface ProfileEmbedder {
523
763
  readonly profile: KnowledgeEmbeddingProfile;
764
+ /** What the loaded model files were checked against after loading. */
765
+ readonly verification: VerifiedArtifacts;
524
766
  /** Raw passage embed — takes ALREADY-PREFIXED texts (the compiler's contract). */
525
767
  embed(texts: string[]): Promise<number[][]>;
526
768
  /** Query embed — applies the profile's queryInstruction itself. */
@@ -533,6 +775,37 @@ declare class EmbedderUnavailableError extends Error {
533
775
  readonly isActionable = true;
534
776
  constructor(message: string);
535
777
  }
778
+ /** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
779
+ interface TransformersModule {
780
+ pipeline: (task: string, model: string, options?: Record<string, unknown>) => Promise<PipelineFn>;
781
+ AutoTokenizer: {
782
+ from_pretrained: (model: string, options?: Record<string, unknown>) => Promise<TokenizerFn>;
783
+ };
784
+ env: {
785
+ cacheDir?: string;
786
+ useFSCache?: boolean;
787
+ allowRemoteModels?: boolean;
788
+ };
789
+ }
790
+ type PipelineFn = ((texts: string[], options: {
791
+ pooling: string;
792
+ normalize: boolean;
793
+ }) => Promise<{
794
+ tolist(): number[][];
795
+ }>) & {
796
+ dispose?: () => Promise<void>;
797
+ };
798
+ interface TokenizerFn {
799
+ encode(text: string): number[];
800
+ }
801
+ /** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
802
+ interface TransformersModelOptions {
803
+ revision: string;
804
+ dtype: string;
805
+ subfolder: string;
806
+ model_file_name: string;
807
+ }
808
+ declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
536
809
  declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
537
810
  cacheDir?: string;
538
811
  /**
@@ -543,12 +816,14 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
543
816
  * what affects values, and that stays with the caller.
544
817
  */
545
818
  sessionOptions?: Record<string, unknown>;
819
+ /** Test seam: a stand-in for `@huggingface/transformers`. */
820
+ transformers?: TransformersModule;
546
821
  }): Promise<ProfileEmbedder>;
547
822
 
548
823
  /**
549
824
  * Markdown-folder catalog source — the `gezel knowledge build` input adapter.
550
825
  * Folders become the shipped table of contents (a format requirement:
551
- * gezk-format-v1.md §6.1), files become documents. A flat corpus — files at
826
+ * the gezk spec §5.2), files become documents. A flat corpus — files at
552
827
  * the root with no subfolders — gets the single root topic the compiler
553
828
  * demands. Deterministic: sorted walk, stable topic-id collision suffixes.
554
829
  */
@@ -570,54 +845,6 @@ interface LoadMarkdownCatalogOptions {
570
845
  */
571
846
  declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
572
847
 
573
- /**
574
- * Manifest signing (gezk-format-v1.md §7): Ed25519 over the RFC 8785
575
- * canonical form of the manifest WITHOUT its `signature` field. Key
576
- * encodings follow the repo's identity conventions (remotes/identity.ts):
577
- * SPKI/PKCS#8 PEM, base64 signatures, node:crypto one-shot sign/verify.
578
- * The private key never appears in CI — signing is a manual workstation
579
- * step (the plan's key-custody stance); verification is what ships.
580
- */
581
-
582
- interface KnowledgeTrustAnchor {
583
- keyId: string;
584
- publicKeyPem: string;
585
- /** Display-only: whose key this is (e.g. `qualla`). */
586
- publisherId?: string;
587
- }
588
- type ManifestSignatureVerdict = {
589
- ok: true;
590
- keyId: string;
591
- } | {
592
- ok: false;
593
- reason: 'unsigned' | 'unknown-key' | 'bad-signature' | 'error';
594
- detail?: string;
595
- };
596
- /** Stable key identity: first 16 hex chars of SHA-256 over the SPKI DER. */
597
- declare function knowledgeKeyId(publicKeyPem: string): string;
598
- declare function generateKnowledgeSigningKeyPair(): {
599
- publicKeyPem: string;
600
- privateKeyPem: string;
601
- keyId: string;
602
- };
603
- /** The exact bytes the signature covers: JCS(manifest minus `signature`). */
604
- declare function manifestSigningPayload(manifest: KnowledgeCatalogManifest): Buffer;
605
- declare function signManifest(manifest: KnowledgeCatalogManifest, privateKeyPem: string): KnowledgeCatalogManifest;
606
- /**
607
- * Verify against keyId-indexed trust anchors. Key rotation = overlapping
608
- * anchors for one publisher; an unknown keyId is a hard failure, never a
609
- * fallback scan across anchors.
610
- */
611
- declare function verifyManifestSignature(manifest: KnowledgeCatalogManifest, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
612
- /**
613
- * The publisher registry (`_knowledge/registry/index.json`) signs with the
614
- * same discipline as a catalog manifest: Ed25519 over JCS(document minus
615
- * `signature`). The signing side lives in Qualla's release command; readers
616
- * (the machine broker's resolver, Settings → available catalogs) verify.
617
- */
618
- declare function signRegistryIndex(index: KnowledgeRegistryIndex, privateKeyPem: string): KnowledgeRegistryIndex;
619
- declare function verifyRegistryIndex(index: KnowledgeRegistryIndex, anchors: readonly KnowledgeTrustAnchor[]): ManifestSignatureVerdict;
620
-
621
848
  /**
622
849
  * Signed-registry consumption (knowledge-catalogs.md Phase 6): fetch a
623
850
  * publisher's `_knowledge/registry/index.json`, verify its Ed25519 signature
@@ -639,8 +866,10 @@ declare class KnowledgeRegistryFetchError extends Error {
639
866
  interface FetchKnowledgeRegistryOptions {
640
867
  anchors: readonly KnowledgeTrustAnchor[];
641
868
  maxBytes?: number;
642
- /** Awake-time budget (suspend-aware) for the whole fetch. */
869
+ /** Budget for the whole fetch when no `signal` is supplied. */
643
870
  timeoutMs?: number;
871
+ /** Caller-owned deadline (the daemon passes a suspend-aware signal). */
872
+ signal?: AbortSignal;
644
873
  fetchImpl?: typeof fetch;
645
874
  }
646
875
  /**
@@ -695,14 +924,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
695
924
 
696
925
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
697
926
 
698
- /**
699
- * RFC 8785 (JSON Canonicalization Scheme) serializer — the byte-exact input
700
- * to the manifest's Ed25519 signature (gezk-format-v1.md §7). ECMAScript's
701
- * own JSON.stringify already implements the RFC's number and string
702
- * serialization; canonicalization adds only (a) object keys sorted by UTF-16
703
- * code units and (b) a hard rejection of anything without a JSON identity
704
- * (undefined, functions, non-finite numbers, BigInt).
705
- */
706
- declare function canonicalizeJson(value: unknown): string;
707
-
708
- export { type ArchiveEntry, BODY_CODEC_MIN_BYTES, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_APPLICATION_ID, GEZK_ARCHIVE_LIMITS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_EMBEDDING_PROFILES, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type KnowledgeTrustAnchor, type LoadMarkdownCatalogOptions, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, type ManifestSignatureVerdict, type MarkdownCatalogSource, type MarkdownChunk, type ProfileEmbedder, RERANK_FINAL_K, ROUTER_DB_PATH, ROUTER_DDL, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_DDL, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type TokenChunkerOptions, type ValidateCatalogOptions, ZIP_FIXED_MTIME, canonicalizeJson, chunkContentHash, chunkMarkdownProfile, chunkUid, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, documentSlug, extractGezkVerified, fetchKnowledgeRegistry, findRegistryEntry, generateKnowledgeSigningKeyPair, inspectGezkArchive, knowledgeEmbeddingProfile, knowledgeKeyId, l2Normalize, loadMarkdownCatalog, manifestSigningPayload, newerRegistryEntries, openCatalogDatabase, quantizeBinary, quantizeInt8, readGezkManifest, rerankK, rerankScore, signManifest, signRegistryIndex, validateExtractedCatalog, vecChunksDdl, verifyManifestSignature, verifyRegistryIndex, writeGezkArchive };
927
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TokenChunkerOptions, type TokenizerFn, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newerRegistryEntries, openCatalogDatabase, readGezkManifest, rerankK, resolveTransformersModelOptions, runDuckdbScript, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };