@bendyline/gezel-knowledge 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/index.d.ts +322 -24
- package/dist/index.js +1866 -101
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ npm install @bendyline/gezel-knowledge
|
|
|
12
12
|
|
|
13
13
|
## What a `.gezk` is
|
|
14
14
|
|
|
15
|
-
An open format (gezk 0.
|
|
15
|
+
An open format (gezk 0.6, preliminary until 1.0; 0.5 archives still open): a ZIP whose first entry is
|
|
16
16
|
the stored `mimetype` magic (`application/vnd.gezk+zip`), then
|
|
17
17
|
`manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
|
|
18
18
|
directory, brotli bodies, routing centroids) and shard SQLite databases with
|
|
@@ -81,8 +81,9 @@ Extraction verifies every file against the manifest (both directions —
|
|
|
81
81
|
undeclared and missing files are equally fatal) and rejects unsafe archive
|
|
82
82
|
entries. Catalog databases are opened strictly read-only and immutable.
|
|
83
83
|
Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
|
|
84
|
-
full 200,000-chunk shard) and
|
|
85
|
-
|
|
84
|
+
full 200,000-chunk shard) and scores the float query against them through a
|
|
85
|
+
per-byte lookup table (the query is centered first when the catalog's profile
|
|
86
|
+
is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
|
|
86
87
|
|
|
87
88
|
## Stability
|
|
88
89
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
-
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
1
|
+
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
3
3
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
4
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
5
5
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
|
|
|
16
16
|
declare const CENTROID_SAMPLE_MAX = 65536;
|
|
17
17
|
declare const ROUTE_SHARDS_PROACTIVE = 3;
|
|
18
18
|
declare const ROUTE_SHARDS_EXPLICIT = 6;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
|
|
21
|
+
* candidates. K is deliberately generous. Measured on a 43,859-chunk
|
|
22
|
+
* multilingual-e5-small catalog with natural-language questions, recall of
|
|
23
|
+
* the exact top-24 among the stage-1 candidates was 22% at the former
|
|
24
|
+
* K = 192 with raw sign bits, and with centered bits (`centered-sign`
|
|
25
|
+
* profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
|
|
26
|
+
* reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
|
|
27
|
+
* shard. A small K is a silent recall cliff; a large one is cheap.
|
|
28
|
+
*/
|
|
20
29
|
declare const RERANK_FINAL_K = 24;
|
|
30
|
+
declare const RERANK_CANDIDATES_MIN = 1024;
|
|
31
|
+
declare const RERANK_CANDIDATES_MAX = 4096;
|
|
21
32
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
22
33
|
|
|
23
34
|
/**
|
|
@@ -68,7 +79,7 @@ type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
|
|
|
68
79
|
declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
|
|
69
80
|
|
|
70
81
|
/**
|
|
71
|
-
* `.gezk` compiler (gezk 0.
|
|
82
|
+
* `.gezk` compiler (gezk 0.6; spec in bendyline/gezk). One owner of normalization, chunk
|
|
72
83
|
* ids, embedding-profile conformance, SQLite schema, shard assignment,
|
|
73
84
|
* centroid routing, and deterministic archive layout — the CLI's Markdown
|
|
74
85
|
* adapter and Qualla's Wikipedia adapter both feed this API.
|
|
@@ -93,6 +104,13 @@ interface CompileTopic {
|
|
|
93
104
|
description?: string;
|
|
94
105
|
sortKey?: string;
|
|
95
106
|
}
|
|
107
|
+
interface CompileAsset {
|
|
108
|
+
/** Archive path, `assets/…`, matching the format's asset path grammar. */
|
|
109
|
+
path: string;
|
|
110
|
+
/** Exactly one of `absPath` / `content`. */
|
|
111
|
+
absPath?: string;
|
|
112
|
+
content?: Buffer;
|
|
113
|
+
}
|
|
96
114
|
interface CompileKnowledgeCatalogOptions {
|
|
97
115
|
catalog: {
|
|
98
116
|
id: string;
|
|
@@ -160,9 +178,17 @@ interface CompileKnowledgeCatalogOptions {
|
|
|
160
178
|
/**
|
|
161
179
|
* Extra archive files (path → utf8 content). `README.md` and
|
|
162
180
|
* `LICENSES/catalog.txt` are always shipped: minimal ones are generated
|
|
163
|
-
* from the catalog block unless supplied here.
|
|
181
|
+
* from the catalog block unless supplied here. Paths under `assets/` are
|
|
182
|
+
* refused here — images go through `assets`.
|
|
164
183
|
*/
|
|
165
184
|
extraFiles?: Record<string, string>;
|
|
185
|
+
/**
|
|
186
|
+
* Images shipped under `assets/` for document bodies to reference by
|
|
187
|
+
* archive path. Each is validated against the format's asset rules
|
|
188
|
+
* (path grammar, size, leading bytes matching the extension, inert SVG)
|
|
189
|
+
* and declared in `manifest.files` like every other entry.
|
|
190
|
+
*/
|
|
191
|
+
assets?: CompileAsset[];
|
|
166
192
|
/**
|
|
167
193
|
* Last touch before the manifest is archived — the signing seam
|
|
168
194
|
* (signatures/signing.ts `signManifest`). Must only add/replace the
|
|
@@ -269,7 +295,7 @@ type DatabaseSync = DatabaseSync$1;
|
|
|
269
295
|
* publisher artifact: it mkdirs the parent, switches on WAL (sidecar files),
|
|
270
296
|
* and quarantine-RENAMES a corrupt database. A catalog reader must fail
|
|
271
297
|
* with a typed reason and never touch the bytes. No extension is loaded:
|
|
272
|
-
* every table in a
|
|
298
|
+
* every table in a catalog is plain SQLite (plus FTS5).
|
|
273
299
|
*/
|
|
274
300
|
|
|
275
301
|
declare class CatalogOpenError extends Error {
|
|
@@ -278,6 +304,8 @@ declare class CatalogOpenError extends Error {
|
|
|
278
304
|
}
|
|
279
305
|
interface CatalogDb {
|
|
280
306
|
db: DatabaseSync;
|
|
307
|
+
/** The `PRAGMA user_version` generation the file was written with. */
|
|
308
|
+
schemaVersion: GezkIndexSchemaVersion;
|
|
281
309
|
close(): void;
|
|
282
310
|
}
|
|
283
311
|
/**
|
|
@@ -292,24 +320,32 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
|
292
320
|
* One mounted catalog: the router database plus lazily opened shards, and
|
|
293
321
|
* the two-stage query flow (the gezk spec §8 and §9):
|
|
294
322
|
*
|
|
295
|
-
* embed query (caller) → centroid routing → per shard: bit
|
|
323
|
+
* embed query (caller) → centroid routing → per shard: sign-bit scan
|
|
296
324
|
* top-K → int8 rerank → fuse.
|
|
297
325
|
*
|
|
298
326
|
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
299
|
-
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
300
|
-
*
|
|
327
|
+
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
328
|
+
* the float query (`asymmetricTopK`), so no vector extension is needed to
|
|
329
|
+
* read a catalog. The catalog's own profile echo says how its bits were
|
|
330
|
+
* derived: a `centered-sign` profile's bits come from `vector − center`, so
|
|
331
|
+
* the query is centered the same way before the scan. The int8 rerank is
|
|
332
|
+
* identical for both kinds of bits.
|
|
301
333
|
*
|
|
302
334
|
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
303
335
|
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
304
336
|
* and tests call it directly.
|
|
305
337
|
*/
|
|
338
|
+
|
|
306
339
|
interface CatalogTopic {
|
|
307
340
|
id: string;
|
|
308
341
|
parentId: string | null;
|
|
309
342
|
name: string;
|
|
310
343
|
description: string | null;
|
|
311
344
|
sortKey: string;
|
|
345
|
+
/** Documents filed directly at this topic (`topics.document_count`). */
|
|
312
346
|
documentCount: number;
|
|
347
|
+
/** Direct plus every descendant topic's documents (computed at read time). */
|
|
348
|
+
totalDocumentCount: number;
|
|
313
349
|
}
|
|
314
350
|
interface CatalogDocumentMeta {
|
|
315
351
|
id: string;
|
|
@@ -317,11 +353,26 @@ interface CatalogDocumentMeta {
|
|
|
317
353
|
slug: string;
|
|
318
354
|
summary: string | null;
|
|
319
355
|
language: string;
|
|
356
|
+
/** The topic the document is filed at — the leaf of its path on 0.6, the root on 0.5. */
|
|
320
357
|
topicId: string;
|
|
358
|
+
/** Listing position among the documents of its topic; null when unordered. */
|
|
359
|
+
ordinal: number | null;
|
|
321
360
|
sourceUrl: string | null;
|
|
322
361
|
sourceRevision: string | null;
|
|
323
362
|
sourceUpdatedAt: string | null;
|
|
324
363
|
attribution: Record<string, string> | null;
|
|
364
|
+
/** Producer-defined metadata (`meta_json`), opaque to the reader. */
|
|
365
|
+
meta: Record<string, unknown> | null;
|
|
366
|
+
}
|
|
367
|
+
interface CatalogAssetInfo {
|
|
368
|
+
/** Archive path under `assets/`. */
|
|
369
|
+
path: string;
|
|
370
|
+
contentType: string;
|
|
371
|
+
sizeBytes: number;
|
|
372
|
+
sha256: string;
|
|
373
|
+
}
|
|
374
|
+
interface CatalogAssetRead extends CatalogAssetInfo {
|
|
375
|
+
bytes: Uint8Array;
|
|
325
376
|
}
|
|
326
377
|
interface CatalogChunkHit {
|
|
327
378
|
chunkUid: string;
|
|
@@ -346,8 +397,13 @@ declare class CatalogHandle {
|
|
|
346
397
|
private readonly router;
|
|
347
398
|
private readonly connections;
|
|
348
399
|
private readonly bitIndexes;
|
|
400
|
+
private assetsByPath;
|
|
401
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
402
|
+
private binaryCenterCache;
|
|
349
403
|
readonly meta: Record<string, string>;
|
|
350
404
|
readonly shards: ShardInfo[];
|
|
405
|
+
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
406
|
+
readonly schemaVersion: GezkIndexSchemaVersion;
|
|
351
407
|
private constructor();
|
|
352
408
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
353
409
|
static open(rootDir: string): CatalogHandle;
|
|
@@ -367,15 +423,35 @@ declare class CatalogHandle {
|
|
|
367
423
|
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
368
424
|
*/
|
|
369
425
|
private shardBits;
|
|
426
|
+
/**
|
|
427
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
428
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
429
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
430
|
+
* the statement of how THIS catalog was built.
|
|
431
|
+
*/
|
|
432
|
+
private binaryCenter;
|
|
370
433
|
/** The embedding dimension from the router's profile echo. */
|
|
371
434
|
private dimensions;
|
|
372
435
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
373
436
|
resolveCatalogPath(path: string): string;
|
|
437
|
+
/** The format version the router was written with (`meta.format_version`). */
|
|
438
|
+
get formatVersion(): string;
|
|
439
|
+
private documentColumns;
|
|
440
|
+
private documentOrder;
|
|
441
|
+
/**
|
|
442
|
+
* Every topic with its direct count and a rollup over its subtree. On a
|
|
443
|
+
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
444
|
+
*/
|
|
374
445
|
topics(): CatalogTopic[];
|
|
446
|
+
/**
|
|
447
|
+
* A page of documents, by default including those filed under the topic's
|
|
448
|
+
* descendants (`descendants: false` lists only the topic's own).
|
|
449
|
+
*/
|
|
375
450
|
documentsPage(opts?: {
|
|
376
451
|
topicId?: string;
|
|
377
452
|
offset?: number;
|
|
378
453
|
limit?: number;
|
|
454
|
+
descendants?: boolean;
|
|
379
455
|
}): {
|
|
380
456
|
documents: CatalogDocumentMeta[];
|
|
381
457
|
total: number;
|
|
@@ -383,6 +459,32 @@ declare class CatalogHandle {
|
|
|
383
459
|
getDocument(id: string): (CatalogDocumentMeta & {
|
|
384
460
|
markdown: string;
|
|
385
461
|
}) | null;
|
|
462
|
+
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
463
|
+
documentsWithUndeclaredTopic(): number;
|
|
464
|
+
/**
|
|
465
|
+
* Validator support: every `meta_json` must be a JSON object within the
|
|
466
|
+
* byte limit. Returns the offending ids (capped) rather than throwing.
|
|
467
|
+
*/
|
|
468
|
+
checkDocumentMeta(maxBytes: number, cap?: number): {
|
|
469
|
+
invalid: string[];
|
|
470
|
+
oversize: string[];
|
|
471
|
+
};
|
|
472
|
+
/** Validator support: decoded bodies, streamed one at a time. */
|
|
473
|
+
documentBodies(): IterableIterator<{
|
|
474
|
+
id: string;
|
|
475
|
+
markdown: string;
|
|
476
|
+
}>;
|
|
477
|
+
private decodeBody;
|
|
478
|
+
/**
|
|
479
|
+
* The manifest's `files` entries under `assets/`, read lazily: the
|
|
480
|
+
* declaration in the manifest is what authorizes serving a file, since
|
|
481
|
+
* extraction reconciled and hashed every declared entry.
|
|
482
|
+
*/
|
|
483
|
+
private assetIndex;
|
|
484
|
+
/** Declared assets, sorted by path. */
|
|
485
|
+
assets(): CatalogAssetInfo[];
|
|
486
|
+
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
487
|
+
readAsset(path: string): CatalogAssetRead | null;
|
|
386
488
|
/** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
|
|
387
489
|
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
388
490
|
documentId: string;
|
|
@@ -411,7 +513,7 @@ declare class CatalogHandle {
|
|
|
411
513
|
/** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
|
|
412
514
|
routeShards(queryVector: Float32Array, shardBudget: number): number[];
|
|
413
515
|
/**
|
|
414
|
-
* Two-stage semantic search over the routed shards: bit
|
|
516
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
415
517
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
416
518
|
*/
|
|
417
519
|
searchSemantic(queryVector: Float32Array, opts?: {
|
|
@@ -457,13 +559,26 @@ interface ValidateCatalogOptions {
|
|
|
457
559
|
deep?: boolean;
|
|
458
560
|
}
|
|
459
561
|
declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalogOptions): Promise<CatalogValidationReport>;
|
|
562
|
+
/** `[…](assets/…)` and `` targets in a body, deduplicated. */
|
|
563
|
+
declare function assetReferences(markdown: string): string[];
|
|
460
564
|
|
|
461
565
|
/**
|
|
462
566
|
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
463
|
-
* 200k-chunk shard is 9.6 MB of bits
|
|
464
|
-
*
|
|
465
|
-
*
|
|
466
|
-
*
|
|
567
|
+
* 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
|
|
568
|
+
* pass over them, a few milliseconds, needing nothing beyond plain SQLite
|
|
569
|
+
* to load — which is what lets any reader implement the format without a
|
|
570
|
+
* vector extension.
|
|
571
|
+
*
|
|
572
|
+
* Two scans share the bits:
|
|
573
|
+
* - `hammingTopK` — symmetric: the query is binarized too and rows are
|
|
574
|
+
* ranked by popcount(xor). The format's baseline (§2), and what the
|
|
575
|
+
* self-KNN smoke and the conformance kit exercise.
|
|
576
|
+
* - `asymmetricTopK` — the float query against each row's bits, ranked by
|
|
577
|
+
* Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
|
|
578
|
+
* embeddings roughly doubles the share of true neighbours a small
|
|
579
|
+
* candidate pool retains (measured on multilingual-e5-small: 22% → 39%
|
|
580
|
+
* of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
|
|
581
|
+
* bits). This is what `CatalogHandle` uses.
|
|
467
582
|
*/
|
|
468
583
|
interface ShardBitIndex {
|
|
469
584
|
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
@@ -481,6 +596,23 @@ interface HammingHit {
|
|
|
481
596
|
* the whole query is two linear passes and no heap.
|
|
482
597
|
*/
|
|
483
598
|
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
599
|
+
interface AsymmetricHit {
|
|
600
|
+
chunkId: number;
|
|
601
|
+
score: number;
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* The K best rows by asymmetric score, descending, ties broken by chunk id.
|
|
605
|
+
*
|
|
606
|
+
* The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
|
|
607
|
+
* of 256 partial sums is built for each byte position, so a row costs
|
|
608
|
+
* `bytesPerRow` table reads and adds — about the popcount scan's cost. The
|
|
609
|
+
* top-K is kept in a bounded min-heap whose root is the worst kept row.
|
|
610
|
+
*
|
|
611
|
+
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
612
|
+
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
613
|
+
* stored; this function does not know or care which.
|
|
614
|
+
*/
|
|
615
|
+
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
|
|
484
616
|
|
|
485
617
|
/**
|
|
486
618
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
@@ -529,7 +661,7 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
529
661
|
* bytes.
|
|
530
662
|
*/
|
|
531
663
|
|
|
532
|
-
declare const PARQUET_EXPORT_VERSION =
|
|
664
|
+
declare const PARQUET_EXPORT_VERSION = 2;
|
|
533
665
|
type ParquetExportSource = {
|
|
534
666
|
archivePath: string;
|
|
535
667
|
} | {
|
|
@@ -598,10 +730,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
|
|
|
598
730
|
* built from a profile verifies the files it loaded against these digests.
|
|
599
731
|
*/
|
|
600
732
|
|
|
601
|
-
/**
|
|
733
|
+
/**
|
|
734
|
+
* The public profile's first revision: multilingual, so world catalogs serve
|
|
735
|
+
* every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
|
|
736
|
+
* vectors, centered stage-1 bits); catalogs built with this revision stay
|
|
737
|
+
* readable and searchable, at the lower stage-1 recall of raw sign bits.
|
|
738
|
+
*/
|
|
602
739
|
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
740
|
+
/**
|
|
741
|
+
* The public profile, revision 2: the model, files, instructions and int8
|
|
742
|
+
* rerank of revision 1, with the stage-1 sign bits taken from
|
|
743
|
+
* `vector − center` (`centered-sign`). e5-small vectors share one dominant
|
|
744
|
+
* direction, so raw sign bits mostly encode that direction rather than the
|
|
745
|
+
* passage and the hamming pre-filter loses most true neighbours; centering
|
|
746
|
+
* on the pinned corpus mean restores them (see e5-small-center.ts for the
|
|
747
|
+
* provenance and the measurements). A query vector for revision 1 is a
|
|
748
|
+
* valid query vector for revision 2 — `sameVectorSpace` says so — and the
|
|
749
|
+
* reader centers it itself from the catalog's profile echo.
|
|
750
|
+
*/
|
|
751
|
+
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
603
752
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
604
753
|
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
754
|
+
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
605
755
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
606
756
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
607
757
|
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
@@ -637,9 +787,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
637
787
|
scale: 127;
|
|
638
788
|
};
|
|
639
789
|
binary: {
|
|
640
|
-
method: "sign";
|
|
790
|
+
method: "sign" | "centered-sign";
|
|
641
791
|
threshold: 0;
|
|
642
792
|
packing: "lsb-first";
|
|
793
|
+
center?: number[] | undefined;
|
|
643
794
|
};
|
|
644
795
|
};
|
|
645
796
|
};
|
|
@@ -674,9 +825,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
674
825
|
scale: 127;
|
|
675
826
|
};
|
|
676
827
|
binary: {
|
|
677
|
-
method: "sign";
|
|
828
|
+
method: "sign" | "centered-sign";
|
|
678
829
|
threshold: 0;
|
|
679
830
|
packing: "lsb-first";
|
|
831
|
+
center?: number[] | undefined;
|
|
680
832
|
};
|
|
681
833
|
};
|
|
682
834
|
};
|
|
@@ -820,28 +972,174 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
820
972
|
transformers?: TransformersModule;
|
|
821
973
|
}): Promise<ProfileEmbedder>;
|
|
822
974
|
|
|
975
|
+
/**
|
|
976
|
+
* Tables of contents that documentation trees already carry. Each parser
|
|
977
|
+
* turns one tool's outline into the same shape — a tree of topics holding
|
|
978
|
+
* ordered document entries — which the Markdown adapter lays over its
|
|
979
|
+
* folder-derived tree: a file the outline names is filed where the outline
|
|
980
|
+
* puts it, with the outline's title and position, and a file it omits stays
|
|
981
|
+
* in its folder. Parsers are pure: they see text and the list of Markdown
|
|
982
|
+
* files, never the disk, so an outline can be checked in a test without a
|
|
983
|
+
* tree on disk.
|
|
984
|
+
*/
|
|
985
|
+
type TableOfContentsFormat = 'folders' | 'hugo' | 'gitbook' | 'mkdocs' | 'jupyter-book';
|
|
986
|
+
type OutlineFormat = Exclude<TableOfContentsFormat, 'folders' | 'hugo'>;
|
|
987
|
+
interface OutlineEntry {
|
|
988
|
+
/** Root-relative POSIX path of the Markdown file, extension included. */
|
|
989
|
+
file: string;
|
|
990
|
+
/** The title the outline gives the page; wins over the file's own. */
|
|
991
|
+
title?: string;
|
|
992
|
+
/** Listing position among the documents of the containing topic. */
|
|
993
|
+
order?: number;
|
|
994
|
+
}
|
|
995
|
+
interface OutlineTopic {
|
|
996
|
+
name: string;
|
|
997
|
+
description?: string;
|
|
998
|
+
/** Position among sibling topics. */
|
|
999
|
+
order?: number;
|
|
1000
|
+
/** When set, the topic takes the resolved title of this document (a root-relative file). */
|
|
1001
|
+
titleFromFile?: string;
|
|
1002
|
+
children: OutlineTopic[];
|
|
1003
|
+
entries: OutlineEntry[];
|
|
1004
|
+
}
|
|
1005
|
+
interface Outline {
|
|
1006
|
+
format: OutlineFormat;
|
|
1007
|
+
/** Top-level documents and topics; `root.name` is unused. */
|
|
1008
|
+
root: OutlineTopic;
|
|
1009
|
+
/** Root-relative files that are the outline itself and never documents. */
|
|
1010
|
+
consumed: string[];
|
|
1011
|
+
warnings: string[];
|
|
1012
|
+
}
|
|
1013
|
+
declare function newOutlineTopic(name: string): OutlineTopic;
|
|
1014
|
+
/**
|
|
1015
|
+
* A link target relative to `baseDir` (root-relative POSIX, `''` for the
|
|
1016
|
+
* root) as a root-relative POSIX path, or null when it is empty, absolute
|
|
1017
|
+
* or leaves the root. Fragments and queries are dropped.
|
|
1018
|
+
*/
|
|
1019
|
+
declare function resolveOutlinePath(baseDir: string, target: string): string | null;
|
|
1020
|
+
/**
|
|
1021
|
+
* Positions become orders wherever the outline gave none. Entries and
|
|
1022
|
+
* child topics are ranked separately, each 1-based, because the catalog
|
|
1023
|
+
* orders documents by ordinal and topics by sort key.
|
|
1024
|
+
*/
|
|
1025
|
+
declare function assignOutlineOrders(topic: OutlineTopic): void;
|
|
1026
|
+
/**
|
|
1027
|
+
* `getting-started` → `Getting Started`. Separators become spaces and each
|
|
1028
|
+
* all-lowercase word is capitalized; a word that already carries a capital
|
|
1029
|
+
* (`API`, `iPhone`) is left as written.
|
|
1030
|
+
*/
|
|
1031
|
+
declare function titleFromFolderName(name: string): string;
|
|
1032
|
+
/**
|
|
1033
|
+
* GitBook's `SUMMARY.md`: a nested list of `[Title](page.md)` items, where a
|
|
1034
|
+
* `## Heading` opens a part. An item with children becomes a topic; when
|
|
1035
|
+
* it links a page, that page leads the topic. Plain-text items are groups.
|
|
1036
|
+
*/
|
|
1037
|
+
declare function parseGitbookSummary(text: string, summaryFile: string): Outline;
|
|
1038
|
+
/**
|
|
1039
|
+
* The `nav` list of an `mkdocs.yml`: `- page.md`, `- Title: page.md`, and
|
|
1040
|
+
* `- Section: [ … ]`. Returns null when the config has no `nav`, which in
|
|
1041
|
+
* MkDocs means the folder tree is the outline — the adapter's default too.
|
|
1042
|
+
* `docsDir` is the docs directory relative to the content root.
|
|
1043
|
+
*/
|
|
1044
|
+
declare function parseMkdocsNav(config: unknown, docsDir: string, configFile?: string): Outline | null;
|
|
1045
|
+
/**
|
|
1046
|
+
* Jupyter Book's `_toc.yml`: `format: jb-book` (`root`, `parts` with a
|
|
1047
|
+
* `caption`, `chapters`, nested `sections`) or `jb-article` (`root`,
|
|
1048
|
+
* `sections`); the pre-1.0 list form is read too. Files are named without
|
|
1049
|
+
* extension and resolved against the Markdown files of the tree, so a
|
|
1050
|
+
* notebook entry is skipped with a warning. A chapter with sections becomes
|
|
1051
|
+
* a topic, led by its own page; `glob` entries expand in sorted order;
|
|
1052
|
+
* `url` entries are links, not pages.
|
|
1053
|
+
*/
|
|
1054
|
+
declare function parseJupyterBookToc(doc: unknown, tocDir: string, files: readonly string[], tocFile?: string): Outline;
|
|
1055
|
+
|
|
823
1056
|
/**
|
|
824
1057
|
* Markdown-folder catalog source — the `gezel knowledge build` input adapter.
|
|
825
1058
|
* Folders become the shipped table of contents (a format requirement:
|
|
826
|
-
* the gezk spec §5.2), files become documents
|
|
827
|
-
*
|
|
828
|
-
*
|
|
1059
|
+
* the gezk spec §5.2), files become documents, images the bodies reference
|
|
1060
|
+
* become assets. A flat corpus — files at the root with no subfolders —
|
|
1061
|
+
* gets the single root topic the compiler demands.
|
|
1062
|
+
*
|
|
1063
|
+
* A tree that already carries an outline keeps it: GitBook's `SUMMARY.md`,
|
|
1064
|
+
* an `mkdocs.yml` nav and Jupyter Book's `_toc.yml` are read into the same
|
|
1065
|
+
* topic tree (`outline.ts`), and Hugo's conventions — `_index.md` section
|
|
1066
|
+
* pages, `weight`, `draft` — are honored on top of the folders. A file an
|
|
1067
|
+
* outline omits stays in its folder, with a warning, so nothing is lost.
|
|
1068
|
+
*
|
|
1069
|
+
* Front matter carries what a documentation tree knows about itself: the
|
|
1070
|
+
* title and summary, an explicit `id`, an `order` (the listing ordinal), a
|
|
1071
|
+
* `subcategory` shelf below the folder topic, and anything else as opaque
|
|
1072
|
+
* metadata. Two passes keep it deterministic: the first walks files in
|
|
1073
|
+
* sorted order and fixes every id; the second rewrites relative image and
|
|
1074
|
+
* article links now that every target is known.
|
|
829
1075
|
*/
|
|
830
1076
|
|
|
831
1077
|
interface MarkdownCatalogSource {
|
|
832
1078
|
topics: CompileTopic[];
|
|
833
1079
|
documents: CatalogDocument[];
|
|
1080
|
+
/** Images referenced by the bodies, resolved to files under the root. */
|
|
1081
|
+
assets: CompileAsset[];
|
|
1082
|
+
/** The outline that shaped the topics, with its file when it came from one. */
|
|
1083
|
+
toc: {
|
|
1084
|
+
format: TableOfContentsFormat;
|
|
1085
|
+
path?: string;
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
interface TopicOverride {
|
|
1089
|
+
name?: string;
|
|
1090
|
+
/** Listing position among sibling topics; encoded into the sort key. */
|
|
1091
|
+
order?: number;
|
|
1092
|
+
description?: string;
|
|
1093
|
+
}
|
|
1094
|
+
interface TableOfContentsOptions {
|
|
1095
|
+
format: TableOfContentsFormat;
|
|
1096
|
+
/**
|
|
1097
|
+
* The outline file for `gitbook` (`SUMMARY.md`), `mkdocs` (`mkdocs.yml`)
|
|
1098
|
+
* and `jupyter-book` (`_toc.yml`), absolute or relative to the root. By
|
|
1099
|
+
* default the root is searched, and for mkdocs its parent as well.
|
|
1100
|
+
*/
|
|
1101
|
+
path?: string;
|
|
834
1102
|
}
|
|
835
1103
|
interface LoadMarkdownCatalogOptions {
|
|
836
1104
|
language: string;
|
|
837
1105
|
/** Topic id used for root-level files (default `general`). */
|
|
838
1106
|
rootTopicId?: string;
|
|
839
1107
|
rootTopicName?: string;
|
|
1108
|
+
/**
|
|
1109
|
+
* Root-relative POSIX directory path → overrides. An explicit entry wins
|
|
1110
|
+
* over a `_topic.yaml` sidecar in that directory.
|
|
1111
|
+
*/
|
|
1112
|
+
topics?: Record<string, TopicOverride>;
|
|
1113
|
+
/** Root-relative files (POSIX) to leave out, e.g. `['README.md']`. */
|
|
1114
|
+
ignore?: string[];
|
|
1115
|
+
/** Where the table of contents comes from (default: the folders). */
|
|
1116
|
+
toc?: TableOfContentsOptions;
|
|
1117
|
+
/**
|
|
1118
|
+
* When set, relative links to other Markdown files in the tree are
|
|
1119
|
+
* rewritten to `knowledge://` references so a viewer can follow them.
|
|
1120
|
+
*/
|
|
1121
|
+
uri?: {
|
|
1122
|
+
publisherId: string;
|
|
1123
|
+
catalogId: string;
|
|
1124
|
+
};
|
|
1125
|
+
/** What to do with an image link whose file is missing (default `error`). */
|
|
1126
|
+
missingAssets?: 'error' | 'warn';
|
|
1127
|
+
onWarning?: (message: string) => void;
|
|
840
1128
|
}
|
|
1129
|
+
/** The docs directory an `mkdocs.yml` names (`docs_dir`, default `docs`), absolute. */
|
|
1130
|
+
declare function readMkdocsDocsDir(mkdocsPath: string): Promise<string>;
|
|
1131
|
+
/**
|
|
1132
|
+
* Which outline a tree carries: GitBook's `SUMMARY.md` or Jupyter Book's
|
|
1133
|
+
* `_toc.yml` at the root, an `mkdocs.yml` at the root, the project folder
|
|
1134
|
+
* or the root's parent, Hugo's `_index.md` section pages anywhere in the
|
|
1135
|
+
* tree — else the folders themselves.
|
|
1136
|
+
*/
|
|
1137
|
+
declare function detectTableOfContents(rootDir: string, projectDir?: string): Promise<TableOfContentsOptions>;
|
|
841
1138
|
/**
|
|
842
1139
|
* Load a folder of Markdown into compiler inputs. Directory chain → topic
|
|
843
|
-
* chain; the relative path (without
|
|
844
|
-
* document id
|
|
1140
|
+
* chain unless an outline places the file; the relative path (without
|
|
1141
|
+
* extension, `/`-separated) is the stable document id unless the front
|
|
1142
|
+
* matter names one.
|
|
845
1143
|
*/
|
|
846
1144
|
declare function loadMarkdownCatalog(rootDir: string, opts: LoadMarkdownCatalogOptions): Promise<MarkdownCatalogSource>;
|
|
847
1145
|
|
|
@@ -924,4 +1222,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
924
1222
|
|
|
925
1223
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
926
1224
|
|
|
927
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TokenChunkerOptions, type TokenizerFn, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newerRegistryEntries, openCatalogDatabase, readGezkManifest, rerankK, resolveTransformersModelOptions, runDuckdbScript, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1225
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|