@bendyline/gezel-knowledge 1.1.0 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,7 +12,7 @@ npm install @bendyline/gezel-knowledge
12
12
 
13
13
  ## What a `.gezk` is
14
14
 
15
- An open format (gezk 0.5, preliminary until 1.0): a ZIP whose first entry is
15
+ An open format (gezk 0.6, preliminary until 1.0; 0.5 archives still open): a ZIP whose first entry is
16
16
  the stored `mimetype` magic (`application/vnd.gezk+zip`), then
17
17
  `manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
18
18
  directory, brotli bodies, routing centroids) and shard SQLite databases with
@@ -81,8 +81,9 @@ Extraction verifies every file against the manifest (both directions —
81
81
  undeclared and missing files are equally fatal) and rejects unsafe archive
82
82
  entries. Catalog databases are opened strictly read-only and immutable.
83
83
  Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
84
- full 200,000-chunk shard) and scans them with a popcount; the int8 rerank
85
- reads the candidates by rowid.
84
+ full 200,000-chunk shard) and scores the float query against them through a
85
+ per-byte lookup table (the query is centered first when the catalog's profile
86
+ is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
86
87
 
87
88
  ## Stability
88
89
 
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
- import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
2
- export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
1
+ import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
2
+ export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
3
3
  import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
4
4
  export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
5
5
  import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
16
16
  declare const CENTROID_SAMPLE_MAX = 65536;
17
17
  declare const ROUTE_SHARDS_PROACTIVE = 3;
18
18
  declare const ROUTE_SHARDS_EXPLICIT = 6;
19
- /** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
19
+ /**
20
+ * Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
21
+ * candidates. K is deliberately generous. Measured on a 43,859-chunk
22
+ * multilingual-e5-small catalog with natural-language questions, recall of
23
+ * the exact top-24 among the stage-1 candidates was 22% at the former
24
+ * K = 192 with raw sign bits, and with centered bits (`centered-sign`
25
+ * profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
26
+ * reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
27
+ * shard. A small K is a silent recall cliff; a large one is cheap.
28
+ */
20
29
  declare const RERANK_FINAL_K = 24;
30
+ declare const RERANK_CANDIDATES_MIN = 1024;
31
+ declare const RERANK_CANDIDATES_MAX = 4096;
21
32
  declare function rerankK(finalK?: number, chunkCount?: number): number;
22
33
 
23
34
  /**
@@ -68,7 +79,7 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
68
79
  declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
69
80
 
70
81
  /**
71
- * `.gezk` compiler (gezk 0.5; spec in bendyline/gezk). One owner of normalization, chunk
82
+ * `.gezk` compiler (gezk 0.6; spec in bendyline/gezk). One owner of normalization, chunk
72
83
  * ids, embedding-profile conformance, SQLite schema, shard assignment,
73
84
  * centroid routing, and deterministic archive layout — the CLI's Markdown
74
85
  * adapter and Qualla's Wikipedia adapter both feed this API.
@@ -93,6 +104,13 @@ interface CompileTopic {
93
104
  description?: string;
94
105
  sortKey?: string;
95
106
  }
107
+ interface CompileAsset {
108
+ /** Archive path, `assets/…`, matching the format's asset path grammar. */
109
+ path: string;
110
+ /** Exactly one of `absPath` / `content`. */
111
+ absPath?: string;
112
+ content?: Buffer;
113
+ }
96
114
  interface CompileKnowledgeCatalogOptions {
97
115
  catalog: {
98
116
  id: string;
@@ -160,9 +178,17 @@ interface CompileKnowledgeCatalogOptions {
160
178
  /**
161
179
  * Extra archive files (path → utf8 content). `README.md` and
162
180
  * `LICENSES/catalog.txt` are always shipped: minimal ones are generated
163
- * from the catalog block unless supplied here.
181
+ * from the catalog block unless supplied here. Paths under `assets/` are
182
+ * refused here — images go through `assets`.
164
183
  */
165
184
  extraFiles?: Record<string, string>;
185
+ /**
186
+ * Images shipped under `assets/` for document bodies to reference by
187
+ * archive path. Each is validated against the format's asset rules
188
+ * (path grammar, size, leading bytes matching the extension, inert SVG)
189
+ * and declared in `manifest.files` like every other entry.
190
+ */
191
+ assets?: CompileAsset[];
166
192
  /**
167
193
  * Last touch before the manifest is archived — the signing seam
168
194
  * (signatures/signing.ts `signManifest`). Must only add/replace the
@@ -269,7 +295,7 @@ type DatabaseSync = DatabaseSync$1;
269
295
  * publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
270
296
  * and quarantine-RENAMES a corrupt database. A catalog reader must fail
271
297
  * with a typed reason and never touch the bytes. No extension is loaded:
272
- * every table in a 0.5 catalog is plain SQLite (plus FTS5).
298
+ * every table in a catalog is plain SQLite (plus FTS5).
273
299
  */
274
300
 
275
301
  declare class CatalogOpenError extends Error {
@@ -278,6 +304,8 @@ declare class CatalogOpenError extends Error {
278
304
  }
279
305
  interface CatalogDb {
280
306
  db: DatabaseSync;
307
+ /** The `PRAGMA user_version` generation the file was written with. */
308
+ schemaVersion: GezkIndexSchemaVersion;
281
309
  close(): void;
282
310
  }
283
311
  /**
@@ -292,24 +320,32 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
292
320
  * One mounted catalog: the router database plus lazily opened shards, and
293
321
  * the two-stage query flow (the gezk spec §8 and §9):
294
322
  *
295
- * embed query (caller) → centroid routing → per shard: bit-hamming KNN
323
+ * embed query (caller) → centroid routing → per shard: sign-bit scan
296
324
  * top-K → int8 rerank → fuse.
297
325
  *
298
326
  * Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
299
- * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
300
- * popcount, so no vector extension is needed to read a catalog.
327
+ * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
328
+ * the float query (`asymmetricTopK`), so no vector extension is needed to
329
+ * read a catalog. The catalog's own profile echo says how its bits were
330
+ * derived: a `centered-sign` profile's bits come from `vector − center`, so
331
+ * the query is centered the same way before the scan. The int8 rerank is
332
+ * identical for both kinds of bits.
301
333
  *
302
334
  * The handle is synchronous (node:sqlite) and single-threaded by design —
303
335
  * the daemon confines every handle to its knowledge worker thread; the CLI
304
336
  * and tests call it directly.
305
337
  */
338
+
306
339
  interface CatalogTopic {
307
340
  id: string;
308
341
  parentId: string | null;
309
342
  name: string;
310
343
  description: string | null;
311
344
  sortKey: string;
345
+ /** Documents filed directly at this topic (`topics.document_count`). */
312
346
  documentCount: number;
347
+ /** Direct plus every descendant topic's documents (computed at read time). */
348
+ totalDocumentCount: number;
313
349
  }
314
350
  interface CatalogDocumentMeta {
315
351
  id: string;
@@ -317,11 +353,26 @@ interface CatalogDocumentMeta {
317
353
  slug: string;
318
354
  summary: string | null;
319
355
  language: string;
356
+ /** The topic the document is filed at — the leaf of its path on 0.6, the root on 0.5. */
320
357
  topicId: string;
358
+ /** Listing position among the documents of its topic; null when unordered. */
359
+ ordinal: number | null;
321
360
  sourceUrl: string | null;
322
361
  sourceRevision: string | null;
323
362
  sourceUpdatedAt: string | null;
324
363
  attribution: Record<string, string> | null;
364
+ /** Producer-defined metadata (`meta_json`), opaque to the reader. */
365
+ meta: Record<string, unknown> | null;
366
+ }
367
+ interface CatalogAssetInfo {
368
+ /** Archive path under `assets/`. */
369
+ path: string;
370
+ contentType: string;
371
+ sizeBytes: number;
372
+ sha256: string;
373
+ }
374
+ interface CatalogAssetRead extends CatalogAssetInfo {
375
+ bytes: Uint8Array;
325
376
  }
326
377
  interface CatalogChunkHit {
327
378
  chunkUid: string;
@@ -346,8 +397,13 @@ declare class CatalogHandle {
346
397
  private readonly router;
347
398
  private readonly connections;
348
399
  private readonly bitIndexes;
400
+ private assetsByPath;
401
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
402
+ private binaryCenterCache;
349
403
  readonly meta: Record<string, string>;
350
404
  readonly shards: ShardInfo[];
405
+ /** The router's `PRAGMA user_version` generation; decides which columns exist. */
406
+ readonly schemaVersion: GezkIndexSchemaVersion;
351
407
  private constructor();
352
408
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
353
409
  static open(rootDir: string): CatalogHandle;
@@ -367,15 +423,35 @@ declare class CatalogHandle {
367
423
  * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
368
424
  */
369
425
  private shardBits;
426
+ /**
427
+ * The stage-1 center the catalog's bits were derived with (a
428
+ * `centered-sign` profile), or null for plain sign bits. Read from the
429
+ * router's profile echo, never from this reader's registry: the echo is
430
+ * the statement of how THIS catalog was built.
431
+ */
432
+ private binaryCenter;
370
433
  /** The embedding dimension from the router's profile echo. */
371
434
  private dimensions;
372
435
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
373
436
  resolveCatalogPath(path: string): string;
437
+ /** The format version the router was written with (`meta.format_version`). */
438
+ get formatVersion(): string;
439
+ private documentColumns;
440
+ private documentOrder;
441
+ /**
442
+ * Every topic with its direct count and a rollup over its subtree. On a
443
+ * 0.5 catalog every document sits at a root, so both counts agree there.
444
+ */
374
445
  topics(): CatalogTopic[];
446
+ /**
447
+ * A page of documents, by default including those filed under the topic's
448
+ * descendants (`descendants: false` lists only the topic's own).
449
+ */
375
450
  documentsPage(opts?: {
376
451
  topicId?: string;
377
452
  offset?: number;
378
453
  limit?: number;
454
+ descendants?: boolean;
379
455
  }): {
380
456
  documents: CatalogDocumentMeta[];
381
457
  total: number;
@@ -383,6 +459,32 @@ declare class CatalogHandle {
383
459
  getDocument(id: string): (CatalogDocumentMeta & {
384
460
  markdown: string;
385
461
  }) | null;
462
+ /** Documents whose `topic_id` names no declared topic (a validator check). */
463
+ documentsWithUndeclaredTopic(): number;
464
+ /**
465
+ * Validator support: every `meta_json` must be a JSON object within the
466
+ * byte limit. Returns the offending ids (capped) rather than throwing.
467
+ */
468
+ checkDocumentMeta(maxBytes: number, cap?: number): {
469
+ invalid: string[];
470
+ oversize: string[];
471
+ };
472
+ /** Validator support: decoded bodies, streamed one at a time. */
473
+ documentBodies(): IterableIterator<{
474
+ id: string;
475
+ markdown: string;
476
+ }>;
477
+ private decodeBody;
478
+ /**
479
+ * The manifest's `files` entries under `assets/`, read lazily: the
480
+ * declaration in the manifest is what authorizes serving a file, since
481
+ * extraction reconciled and hashed every declared entry.
482
+ */
483
+ private assetIndex;
484
+ /** Declared assets, sorted by path. */
485
+ assets(): CatalogAssetInfo[];
486
+ /** One declared asset's bytes, or null when the catalog declares no such asset. */
487
+ readAsset(path: string): CatalogAssetRead | null;
386
488
  /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
387
489
  searchDocumentsFts(query: string, limit?: number): Array<{
388
490
  documentId: string;
@@ -411,7 +513,7 @@ declare class CatalogHandle {
411
513
  /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
412
514
  routeShards(queryVector: Float32Array, shardBudget: number): number[];
413
515
  /**
414
- * Two-stage semantic search over the routed shards: bit-hamming KNN
516
+ * Two-stage semantic search over the routed shards: sign-bit scan
415
517
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
416
518
  */
417
519
  searchSemantic(queryVector: Float32Array, opts?: {
@@ -457,13 +559,26 @@ interface ValidateCatalogOptions {
457
559
  deep?: boolean;
458
560
  }
459
561
  declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
562
+ /** `[…](assets/…)` and `![…](assets/…)` targets in a body, deduplicated. */
563
+ declare function assetReferences(markdown: string): string[];
460
564
 
461
565
  /**
462
566
  * Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
463
- * 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
464
- * popcount plus a histogram selection of the top-K is a few milliseconds and
465
- * needs nothing beyond plain SQLite to load — which is what lets any reader
466
- * implement the format without a vector extension.
567
+ * 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
568
+ * pass over them, a few milliseconds, needing nothing beyond plain SQLite
569
+ * to load — which is what lets any reader implement the format without a
570
+ * vector extension.
571
+ *
572
+ * Two scans share the bits:
573
+ * - `hammingTopK` — symmetric: the query is binarized too and rows are
574
+ * ranked by popcount(xor). The format's baseline (§2), and what the
575
+ * self-KNN smoke and the conformance kit exercise.
576
+ * - `asymmetricTopK` — the float query against each row's bits, ranked by
577
+ * Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
578
+ * embeddings roughly doubles the share of true neighbours a small
579
+ * candidate pool retains (measured on multilingual-e5-small: 22% → 39%
580
+ * of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
581
+ * bits). This is what `CatalogHandle` uses.
467
582
  */
468
583
  interface ShardBitIndex {
469
584
  /** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
@@ -481,6 +596,23 @@ interface HammingHit {
481
596
  * the whole query is two linear passes and no heap.
482
597
  */
483
598
  declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
599
+ interface AsymmetricHit {
600
+ chunkId: number;
601
+ score: number;
602
+ }
603
+ /**
604
+ * The K best rows by asymmetric score, descending, ties broken by chunk id.
605
+ *
606
+ * The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
607
+ * of 256 partial sums is built for each byte position, so a row costs
608
+ * `bytesPerRow` table reads and adds — about the popcount scan's cost. The
609
+ * top-K is kept in a bounded min-heap whose root is the worst kept row.
610
+ *
611
+ * For a `centered-sign` profile the caller passes `query − center`; for a
612
+ * plain `sign` profile the unit query. The bits are whatever the catalog
613
+ * stored; this function does not know or care which.
614
+ */
615
+ declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
484
616
 
485
617
  /**
486
618
  * Provenance stamped into every manifest this package compiles. Only the
@@ -529,7 +661,7 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
529
661
  * bytes.
530
662
  */
531
663
 
532
- declare const PARQUET_EXPORT_VERSION = 1;
664
+ declare const PARQUET_EXPORT_VERSION = 2;
533
665
  type ParquetExportSource = {
534
666
  archivePath: string;
535
667
  } | {
@@ -598,10 +730,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
598
730
  * built from a profile verifies the files it loaded against these digests.
599
731
  */
600
732
 
601
- /** The public profile: multilingual, so world catalogs serve every reader. */
733
+ /**
734
+ * The public profile's first revision: multilingual, so world catalogs serve
735
+ * every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
736
+ * vectors, centered stage-1 bits); catalogs built with this revision stay
737
+ * readable and searchable, at the lower stage-1 recall of raw sign bits.
738
+ */
602
739
  declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
740
+ /**
741
+ * The public profile, revision 2: the model, files, instructions and int8
742
+ * rerank of revision 1, with the stage-1 sign bits taken from
743
+ * `vector − center` (`centered-sign`). e5-small vectors share one dominant
744
+ * direction, so raw sign bits mostly encode that direction rather than the
745
+ * passage and the hamming pre-filter loses most true neighbours; centering
746
+ * on the pinned corpus mean restores them (see e5-small-center.ts for the
747
+ * provenance and the measurements). A query vector for revision 1 is a
748
+ * valid query vector for revision 2 — `sameVectorSpace` says so — and the
749
+ * reader centers it itself from the catalog's profile echo.
750
+ */
751
+ declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
603
752
  /** The local-build default — matches gezel's shipped project embedder. */
604
753
  declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
754
+ /** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
605
755
  declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
606
756
  declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
607
757
  /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
@@ -637,9 +787,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
637
787
  scale: 127;
638
788
  };
639
789
  binary: {
640
- method: "sign";
790
+ method: "sign" | "centered-sign";
641
791
  threshold: 0;
642
792
  packing: "lsb-first";
793
+ center?: number[] | undefined;
643
794
  };
644
795
  };
645
796
  };
@@ -674,9 +825,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
674
825
  scale: 127;
675
826
  };
676
827
  binary: {
677
- method: "sign";
828
+ method: "sign" | "centered-sign";
678
829
  threshold: 0;
679
830
  packing: "lsb-first";
831
+ center?: number[] | undefined;
680
832
  };
681
833
  };
682
834
  };
@@ -820,28 +972,174 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
820
972
  transformers?: TransformersModule;
821
973
  }): Promise<ProfileEmbedder>;
822
974
 
975
+ /**
976
+ * Tables of contents that documentation trees already carry. Each parser
977
+ * turns one tool's outline into the same shape — a tree of topics holding
978
+ * ordered document entries — which the Markdown adapter lays over its
979
+ * folder-derived tree: a file the outline names is filed where the outline
980
+ * puts it, with the outline's title and position, and a file it omits stays
981
+ * in its folder. Parsers are pure: they see text and the list of Markdown
982
+ * files, never the disk, so an outline can be checked in a test without a
983
+ * tree on disk.
984
+ */
985
+ type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book';
986
+ type OutlineFormat = Exclude<TableOfContentsFormat, 'folders' | 'hugo'>;
987
+ interface OutlineEntry {
988
+ /** Root-relative POSIX path of the Markdown file, extension included. */
989
+ file: string;
990
+ /** The title the outline gives the page; wins over the file's own. */
991
+ title?: string;
992
+ /** Listing position among the documents of the containing topic. */
993
+ order?: number;
994
+ }
995
+ interface OutlineTopic {
996
+ name: string;
997
+ description?: string;
998
+ /** Position among sibling topics. */
999
+ order?: number;
1000
+ /** When set, the topic takes the resolved title of this document (a root-relative file). */
1001
+ titleFromFile?: string;
1002
+ children: OutlineTopic[];
1003
+ entries: OutlineEntry[];
1004
+ }
1005
+ interface Outline {
1006
+ format: OutlineFormat;
1007
+ /** Top-level documents and topics; `root.name` is unused. */
1008
+ root: OutlineTopic;
1009
+ /** Root-relative files that are the outline itself and never documents. */
1010
+ consumed: string[];
1011
+ warnings: string[];
1012
+ }
1013
+ declare function newOutlineTopic(name: string): OutlineTopic;
1014
+ /**
1015
+ * A link target relative to `baseDir` (root-relative POSIX, `''` for the
1016
+ * root) as a root-relative POSIX path, or null when it is empty, absolute
1017
+ * or leaves the root. Fragments and queries are dropped.
1018
+ */
1019
+ declare function resolveOutlinePath(baseDir: string, target: string): string | null;
1020
+ /**
1021
+ * Positions become orders wherever the outline gave none. Entries and
1022
+ * child topics are ranked separately, each 1-based, because the catalog
1023
+ * orders documents by ordinal and topics by sort key.
1024
+ */
1025
+ declare function assignOutlineOrders(topic: OutlineTopic): void;
1026
+ /**
1027
+ * `getting-started` → `Getting Started`. Separators become spaces and each
1028
+ * all-lowercase word is capitalized; a word that already carries a capital
1029
+ * (`API`, `iPhone`) is left as written.
1030
+ */
1031
+ declare function titleFromFolderName(name: string): string;
1032
+ /**
1033
+ * GitBook's `SUMMARY.md`: a nested list of `[Title](page.md)` items, where a
1034
+ * `## Heading` opens a part. An item with children becomes a topic; when
1035
+ * it links a page, that page leads the topic. Plain-text items are groups.
1036
+ */
1037
+ declare function parseGitbookSummary(text: string, summaryFile: string): Outline;
1038
+ /**
1039
+ * The `nav` list of an `mkdocs.yml`: `- page.md`, `- Title: page.md`, and
1040
+ * `- Section: [ … ]`. Returns null when the config has no `nav`, which in
1041
+ * MkDocs means the folder tree is the outline — the adapter's default too.
1042
+ * `docsDir` is the docs directory relative to the content root.
1043
+ */
1044
+ declare function parseMkdocsNav(config: unknown, docsDir: string, configFile?: string): Outline | null;
1045
+ /**
1046
+ * Jupyter Book's `_toc.yml`: `format: jb-book` (`root`, `parts` with a
1047
+ * `caption`, `chapters`, nested `sections`) or `jb-article` (`root`,
1048
+ * `sections`); the pre-1.0 list form is read too. Files are named without
1049
+ * extension and resolved against the Markdown files of the tree, so a
1050
+ * notebook entry is skipped with a warning. A chapter with sections becomes
1051
+ * a topic, led by its own page; `glob` entries expand in sorted order;
1052
+ * `url` entries are links, not pages.
1053
+ */
1054
+ declare function parseJupyterBookToc(doc: unknown, tocDir: string, files: readonly string[], tocFile?: string): Outline;
1055
+
823
1056
  /**
824
1057
  * Markdown-folder catalog source — the `gezel knowledge build` input adapter.
825
1058
  * Folders become the shipped table of contents (a format requirement:
826
- * the gezk spec §5.2), files become documents. A flat corpus — files at
827
- * the root with no subfolders — gets the single root topic the compiler
828
- * demands. Deterministic: sorted walk, stable topic-id collision suffixes.
1059
+ * the gezk spec §5.2), files become documents, images the bodies reference
1060
+ * become assets. A flat corpus — files at the root with no subfolders —
1061
+ * gets the single root topic the compiler demands.
1062
+ *
1063
+ * A tree that already carries an outline keeps it: GitBook's `SUMMARY.md`,
1064
+ * an `mkdocs.yml` nav and Jupyter Book's `_toc.yml` are read into the same
1065
+ * topic tree (`outline.ts`), and Hugo's conventions — `_index.md` section
1066
+ * pages, `weight`, `draft` — are honored on top of the folders. A file an
1067
+ * outline omits stays in its folder, with a warning, so nothing is lost.
1068
+ *
1069
+ * Front matter carries what a documentation tree knows about itself: the
1070
+ * title and summary, an explicit `id`, an `order` (the listing ordinal), a
1071
+ * `subcategory` shelf below the folder topic, and anything else as opaque
1072
+ * metadata. Two passes keep it deterministic: the first walks files in
1073
+ * sorted order and fixes every id; the second rewrites relative image and
1074
+ * article links now that every target is known.
829
1075
  */
830
1076
 
831
1077
  interface MarkdownCatalogSource {
832
1078
  topics: CompileTopic[];
833
1079
  documents: CatalogDocument[];
1080
+ /** Images referenced by the bodies, resolved to files under the root. */
1081
+ assets: CompileAsset[];
1082
+ /** The outline that shaped the topics, with its file when it came from one. */
1083
+ toc: {
1084
+ format: TableOfContentsFormat;
1085
+ path?: string;
1086
+ };
1087
+ }
1088
+ interface TopicOverride {
1089
+ name?: string;
1090
+ /** Listing position among sibling topics; encoded into the sort key. */
1091
+ order?: number;
1092
+ description?: string;
1093
+ }
1094
+ interface TableOfContentsOptions {
1095
+ format: TableOfContentsFormat;
1096
+ /**
1097
+ * The outline file for `gitbook` (`SUMMARY.md`), `mkdocs` (`mkdocs.yml`)
1098
+ * and `jupyter-book` (`_toc.yml`), absolute or relative to the root. By
1099
+ * default the root is searched, and for mkdocs its parent as well.
1100
+ */
1101
+ path?: string;
834
1102
  }
835
1103
  interface LoadMarkdownCatalogOptions {
836
1104
  language: string;
837
1105
  /** Topic id used for root-level files (default `general`). */
838
1106
  rootTopicId?: string;
839
1107
  rootTopicName?: string;
1108
+ /**
1109
+ * Root-relative POSIX directory path → overrides. An explicit entry wins
1110
+ * over a `_topic.yaml` sidecar in that directory.
1111
+ */
1112
+ topics?: Record<string, TopicOverride>;
1113
+ /** Root-relative files (POSIX) to leave out, e.g. `['README.md']`. */
1114
+ ignore?: string[];
1115
+ /** Where the table of contents comes from (default: the folders). */
1116
+ toc?: TableOfContentsOptions;
1117
+ /**
1118
+ * When set, relative links to other Markdown files in the tree are
1119
+ * rewritten to `knowledge://` references so a viewer can follow them.
1120
+ */
1121
+ uri?: {
1122
+ publisherId: string;
1123
+ catalogId: string;
1124
+ };
1125
+ /** What to do with an image link whose file is missing (default `error`). */
1126
+ missingAssets?: 'error' | 'warn';
1127
+ onWarning?: (message: string) => void;
840
1128
  }
1129
+ /** The docs directory an `mkdocs.yml` names (`docs_dir`, default `docs`), absolute. */
1130
+ declare function readMkdocsDocsDir(mkdocsPath: string): Promise<string>;
1131
+ /**
1132
+ * Which outline a tree carries: GitBook's `SUMMARY.md` or Jupyter Book's
1133
+ * `_toc.yml` at the root, an `mkdocs.yml` at the root, the project folder
1134
+ * or the root's parent, Hugo's `_index.md` section pages anywhere in the
1135
+ * tree — else the folders themselves.
1136
+ */
1137
+ declare function detectTableOfContents(rootDir: string, projectDir?: string): Promise<TableOfContentsOptions>;
841
1138
  /**
842
1139
  * Load a folder of Markdown into compiler inputs. Directory chain → topic
843
- * chain; the relative path (without extension, `/`-separated) is the stable
844
- * document id.
1140
+ * chain unless an outline places the file; the relative path (without
1141
+ * extension, `/`-separated) is the stable document id unless the front
1142
+ * matter names one.
845
1143
  */
846
1144
  declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
847
1145
 
@@ -924,4 +1222,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
924
1222
 
925
1223
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
926
1224
 
927
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TokenChunkerOptions, type TokenizerFn, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newerRegistryEntries, openCatalogDatabase, readGezkManifest, rerankK, resolveTransformersModelOptions, runDuckdbScript, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
1225
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };