@bendyline/gezel-knowledge 1.1.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,13 +12,15 @@ npm install @bendyline/gezel-knowledge
12
12
 
13
13
  ## What a `.gezk` is
14
14
 
15
- An open format (gezk 0.6, preliminary until 1.0; 0.5 archives still open): a ZIP whose first entry is
15
+ An open format (format version 0.6, preliminary until format 1.0; 0.5
16
+ archives still open): a ZIP whose first entry is
16
17
  the stored `mimetype` magic (`application/vnd.gezk+zip`), then
17
18
  `manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
18
19
  directory, brotli bodies, routing centroids) and shard SQLite databases with
19
20
  FTS5 and quantized vectors in plain BLOB tables. Nothing beyond stock SQLite
20
21
  is needed to read one. The format definitions are the separate
21
- [`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package;
22
+ [`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package,
23
+ whose npm version is independent of the format version it writes;
22
24
  the specification, JSON Schemas, conformance fixtures and a Python reference
23
25
  reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
24
26
 
@@ -81,8 +83,9 @@ Extraction verifies every file against the manifest (both directions —
81
83
  undeclared and missing files are equally fatal) and rejects unsafe archive
82
84
  entries. Catalog databases are opened strictly read-only and immutable.
83
85
  Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
84
- full 200,000-chunk shard) and scans them with a popcount; the int8 rerank
85
- reads the candidates by rowid.
86
+ full 200,000-chunk shard) and scores the float query against them through a
87
+ per-byte lookup table (the query is centered first when the catalog's profile
88
+ is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
86
89
 
87
90
  ## Stability
88
91
 
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
2
- export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
2
+ export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
3
3
  import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
4
4
  export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
5
5
  import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
16
16
  declare const CENTROID_SAMPLE_MAX = 65536;
17
17
  declare const ROUTE_SHARDS_PROACTIVE = 3;
18
18
  declare const ROUTE_SHARDS_EXPLICIT = 6;
19
- /** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
19
+ /**
20
+ * Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
21
+ * candidates. K is deliberately generous. Measured on a 43,859-chunk
22
+ * multilingual-e5-small catalog with natural-language questions, recall of
23
+ * the exact top-24 among the stage-1 candidates was 22% at the former
24
+ * K = 192 with raw sign bits, and with centered bits (`centered-sign`
25
+ * profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
26
+ * reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
27
+ * shard. A small K is a silent recall cliff; a large one is cheap.
28
+ */
20
29
  declare const RERANK_FINAL_K = 24;
30
+ declare const RERANK_CANDIDATES_MIN = 1024;
31
+ declare const RERANK_CANDIDATES_MAX = 4096;
21
32
  declare function rerankK(finalK?: number, chunkCount?: number): number;
22
33
 
23
34
  /**
@@ -309,12 +320,16 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
309
320
  * One mounted catalog: the router database plus lazily opened shards, and
310
321
  * the two-stage query flow (the gezk spec §8 and §9):
311
322
  *
312
- * embed query (caller) → centroid routing → per shard: bit-hamming KNN
323
+ * embed query (caller) → centroid routing → per shard: sign-bit scan
313
324
  * top-K → int8 rerank → fuse.
314
325
  *
315
326
  * Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
316
- * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
317
- * popcount, so no vector extension is needed to read a catalog.
327
+ * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
328
+ * the float query (`asymmetricTopK`), so no vector extension is needed to
329
+ * read a catalog. The catalog's own profile echo says how its bits were
330
+ * derived: a `centered-sign` profile's bits come from `vector − center`, so
331
+ * the query is centered the same way before the scan. The int8 rerank is
332
+ * identical for both kinds of bits.
318
333
  *
319
334
  * The handle is synchronous (node:sqlite) and single-threaded by design —
320
335
  * the daemon confines every handle to its knowledge worker thread; the CLI
@@ -383,6 +398,8 @@ declare class CatalogHandle {
383
398
  private readonly connections;
384
399
  private readonly bitIndexes;
385
400
  private assetsByPath;
401
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
402
+ private binaryCenterCache;
386
403
  readonly meta: Record<string, string>;
387
404
  readonly shards: ShardInfo[];
388
405
  /** The router's `PRAGMA user_version` generation; decides which columns exist. */
@@ -406,6 +423,13 @@ declare class CatalogHandle {
406
423
  * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
407
424
  */
408
425
  private shardBits;
426
+ /**
427
+ * The stage-1 center the catalog's bits were derived with (a
428
+ * `centered-sign` profile), or null for plain sign bits. Read from the
429
+ * router's profile echo, never from this reader's registry: the echo is
430
+ * the statement of how THIS catalog was built.
431
+ */
432
+ private binaryCenter;
409
433
  /** The embedding dimension from the router's profile echo. */
410
434
  private dimensions;
411
435
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
@@ -461,7 +485,10 @@ declare class CatalogHandle {
461
485
  assets(): CatalogAssetInfo[];
462
486
  /** One declared asset's bytes, or null when the catalog declares no such asset. */
463
487
  readAsset(path: string): CatalogAssetRead | null;
464
- /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
488
+ /**
489
+ * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
490
+ * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
491
+ */
465
492
  searchDocumentsFts(query: string, limit?: number): Array<{
466
493
  documentId: string;
467
494
  rank: number;
@@ -489,7 +516,7 @@ declare class CatalogHandle {
489
516
  /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
490
517
  routeShards(queryVector: Float32Array, shardBudget: number): number[];
491
518
  /**
492
- * Two-stage semantic search over the routed shards: bit-hamming KNN
519
+ * Two-stage semantic search over the routed shards: sign-bit scan
493
520
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
494
521
  */
495
522
  searchSemantic(queryVector: Float32Array, opts?: {
@@ -540,10 +567,21 @@ declare function assetReferences(markdown: string): string[];
540
567
 
541
568
  /**
542
569
  * Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
543
- * 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
544
- * popcount plus a histogram selection of the top-K is a few milliseconds and
545
- * needs nothing beyond plain SQLite to load — which is what lets any reader
546
- * implement the format without a vector extension.
570
+ * 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
571
+ * pass over them, a few milliseconds, needing nothing beyond plain SQLite
572
+ * to load — which is what lets any reader implement the format without a
573
+ * vector extension.
574
+ *
575
+ * Two scans share the bits:
576
+ * - `hammingTopK` — symmetric: the query is binarized too and rows are
577
+ * ranked by popcount(xor). The format's baseline (§2), and what the
578
+ * self-KNN smoke and the conformance kit exercise.
579
+ * - `asymmetricTopK` — the float query against each row's bits, ranked by
580
+ * Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
581
+ * embeddings roughly doubles the share of true neighbours a small
582
+ * candidate pool retains (measured on multilingual-e5-small: 22% → 39%
583
+ * of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
584
+ * bits). This is what `CatalogHandle` uses.
547
585
  */
548
586
  interface ShardBitIndex {
549
587
  /** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
@@ -561,6 +599,23 @@ interface HammingHit {
561
599
  * the whole query is two linear passes and no heap.
562
600
  */
563
601
  declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
602
+ interface AsymmetricHit {
603
+ chunkId: number;
604
+ score: number;
605
+ }
606
+ /**
607
+ * The K best rows by asymmetric score, descending, ties broken by chunk id.
608
+ *
609
+ * The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
610
+ * of 256 partial sums is built for each byte position, so a row costs
611
+ * `bytesPerRow` table reads and adds — about the popcount scan's cost. The
612
+ * top-K is kept in a bounded min-heap whose root is the worst kept row.
613
+ *
614
+ * For a `centered-sign` profile the caller passes `query − center`; for a
615
+ * plain `sign` profile the unit query. The bits are whatever the catalog
616
+ * stored; this function does not know or care which.
617
+ */
618
+ declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
564
619
 
565
620
  /**
566
621
  * Provenance stamped into every manifest this package compiles. Only the
@@ -601,6 +656,13 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
601
656
  * stores them (provably identical, one third of float32's size), with the
602
657
  * dequantisation rule recorded in the file metadata (gezk spec §6).
603
658
  *
659
+ * The bit rule in that metadata follows the catalog's profile: a `sign`
660
+ * profile stores sign(x), a `centered-sign` profile sign(x − c) with the
661
+ * profile's center c — which the metadata then carries too, so the Parquet
662
+ * files alone are enough to binarise a query the way the archive did. (Export
663
+ * version 2 wrote the plain-sign rule for every profile, which misdescribed
664
+ * centered-sign catalogs; version 3 fixed that.)
665
+ *
604
666
  * Determinism: rows are read in primary-key order and written single-
605
667
  * threaded with insertion order preserved, a fixed codec and level, a fixed
606
668
  * row-group size, and no wall-clock metadata — so two exports of one archive
@@ -609,7 +671,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
609
671
  * bytes.
610
672
  */
611
673
 
612
- declare const PARQUET_EXPORT_VERSION = 2;
674
+ /** 3: the bit-rule metadata follows the profile's binary method (was always plain sign). */
675
+ declare const PARQUET_EXPORT_VERSION = 3;
613
676
  type ParquetExportSource = {
614
677
  archivePath: string;
615
678
  } | {
@@ -660,6 +723,16 @@ interface ParquetExportReport {
660
723
  }
661
724
  declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
662
725
  declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
726
+ /**
727
+ * The key/value metadata every exported file carries. The bit rule is read
728
+ * from the profile — never assumed — because a `centered-sign` profile's bits
729
+ * are meaningless to a reader who binarises a query with plain sign(x). `x`
730
+ * is the float unit vector the bits were taken from, before int8 rounding, so
731
+ * recomputing them from `q / 127` can flip the few dimensions within rounding
732
+ * error of the threshold. Insertion order is the file's key order, so it is
733
+ * part of the deterministic bytes.
734
+ */
735
+ declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<string, string>;
663
736
  /** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
664
737
  declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
665
738
 
@@ -678,10 +751,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
678
751
  * built from a profile verifies the files it loaded against these digests.
679
752
  */
680
753
 
681
- /** The public profile: multilingual, so world catalogs serve every reader. */
754
+ /**
755
+ * The public profile's first revision: multilingual, so world catalogs serve
756
+ * every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
757
+ * vectors, centered stage-1 bits); catalogs built with this revision stay
758
+ * readable and searchable, at the lower stage-1 recall of raw sign bits.
759
+ */
682
760
  declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
761
+ /**
762
+ * The public profile, revision 2: the model, files, instructions and int8
763
+ * rerank of revision 1, with the stage-1 sign bits taken from
764
+ * `vector − center` (`centered-sign`). e5-small vectors share one dominant
765
+ * direction, so raw sign bits mostly encode that direction rather than the
766
+ * passage and the hamming pre-filter loses most true neighbours; centering
767
+ * on the pinned corpus mean restores them (see e5-small-center.ts for the
768
+ * provenance and the measurements). A query vector for revision 1 is a
769
+ * valid query vector for revision 2 — `sameVectorSpace` says so — and the
770
+ * reader centers it itself from the catalog's profile echo.
771
+ */
772
+ declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
683
773
  /** The local-build default — matches gezel's shipped project embedder. */
684
774
  declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
775
+ /** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
685
776
  declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
686
777
  declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
687
778
  /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
@@ -717,9 +808,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
717
808
  scale: 127;
718
809
  };
719
810
  binary: {
720
- method: "sign";
811
+ method: "sign" | "centered-sign";
721
812
  threshold: 0;
722
813
  packing: "lsb-first";
814
+ center?: number[] | undefined;
723
815
  };
724
816
  };
725
817
  };
@@ -754,9 +846,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
754
846
  scale: 127;
755
847
  };
756
848
  binary: {
757
- method: "sign";
849
+ method: "sign" | "centered-sign";
758
850
  threshold: 0;
759
851
  packing: "lsb-first";
852
+ center?: number[] | undefined;
760
853
  };
761
854
  };
762
855
  };
@@ -878,6 +971,11 @@ type PipelineFn = ((texts: string[], options: {
878
971
  interface TokenizerFn {
879
972
  encode(text: string): number[];
880
973
  }
974
+ /** Bytes fetched so far across every file a profile load pulls, summed. */
975
+ interface ModelDownloadProgress {
976
+ bytesDone: number;
977
+ bytesTotal: number;
978
+ }
881
979
  /** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
882
980
  interface TransformersModelOptions {
883
981
  revision: string;
@@ -886,6 +984,18 @@ interface TransformersModelOptions {
886
984
  model_file_name: string;
887
985
  }
888
986
  declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
987
+ /**
988
+ * The transformers.js options that select exactly one pinned ONNX graph:
989
+ * its folder, its base name, and the dtype whose suffix transformers.js
990
+ * appends to that name. A file with no recognized suffix loads as fp32 under
991
+ * its full name, which selects the same file. Null for a path that is not a
992
+ * named `.onnx` graph.
993
+ */
994
+ declare function transformersGraphOptions(onnxFile: string): {
995
+ dtype: string;
996
+ subfolder: string;
997
+ model_file_name: string;
998
+ } | null;
889
999
  declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
890
1000
  cacheDir?: string;
891
1001
  /**
@@ -896,6 +1006,8 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
896
1006
  * what affects values, and that stays with the caller.
897
1007
  */
898
1008
  sessionOptions?: Record<string, unknown>;
1009
+ /** Called as the model and tokenizer files download. */
1010
+ onDownloadProgress?: (progress: ModelDownloadProgress) => void;
899
1011
  /** Test seam: a stand-in for `@huggingface/transformers`. */
900
1012
  transformers?: TransformersModule;
901
1013
  }): Promise<ProfileEmbedder>;
@@ -1150,4 +1262,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1150
1262
 
1151
1263
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1152
1264
 
1153
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
1265
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -30,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
30
30
  var ROUTE_SHARDS_PROACTIVE = 3;
31
31
  var ROUTE_SHARDS_EXPLICIT = 6;
32
32
  var RERANK_FINAL_K = 24;
33
+ var RERANK_CANDIDATES_MIN = 1024;
34
+ var RERANK_CANDIDATES_MAX = 4096;
33
35
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
34
- return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
36
+ return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
35
37
  }
36
38
 
37
39
  // src/format/ddl.ts
@@ -42,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
42
44
  import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
43
45
 
44
46
  // src/format/quantize.ts
45
- import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
47
+ import {
48
+ centerVector,
49
+ l2Normalize,
50
+ quantizeBinary,
51
+ quantizeBinaryForProfile,
52
+ quantizeInt8,
53
+ rerankScore
54
+ } from "@bendyline/gezk";
46
55
 
47
56
  // src/chunking/markdown-chunker.ts
48
57
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -241,16 +250,20 @@ function sanitizeFtsQuery(query) {
241
250
  if (tokens.length === 0) return null;
242
251
  return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
243
252
  }
244
- function documentFtsTopIds(db, match, limit) {
253
+ var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
254
+ var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
255
+ function documentFtsTopIds(db, match, limit, exactTitle) {
256
+ const title = exactTitle?.normalize("NFKC").trim() ?? "";
245
257
  return db.prepare(
246
- "SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
247
- ).all(match, limit).map((row) => row.document_id);
258
+ `SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
259
+ ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
260
+ ).all(match, title, limit).map((row) => row.document_id);
248
261
  }
249
262
  var SMOKE_QUERY_TOP_N = 10;
250
263
  function documentSmokeQueryMisses(db, smoke) {
251
264
  const match = sanitizeFtsQuery(smoke.query);
252
265
  if (!match) return [...smoke.expectedDocumentIds];
253
- const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
266
+ const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
254
267
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
255
268
  }
256
269
 
@@ -469,7 +482,7 @@ async function compileKnowledgeCatalog(opts) {
469
482
  }
470
483
  const row = pendingRows[i];
471
484
  const stmts = vecStmtsFor(shardDb(row.shardId));
472
- stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinary(unit)));
485
+ stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
473
486
  stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
474
487
  const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
475
488
  void shardTotal;
@@ -1403,8 +1416,10 @@ import { readFileSync as readFileSync2 } from "fs";
1403
1416
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1404
1417
  import { brotliDecompressSync } from "zlib";
1405
1418
  import {
1419
+ KnowledgeEmbeddingProfileSchema,
1406
1420
  MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1407
1421
  assetContentType,
1422
+ embeddingProfileCenter,
1408
1423
  isKnowledgeAssetPath
1409
1424
  } from "@bendyline/gezk";
1410
1425
 
@@ -1478,6 +1493,86 @@ function hammingTopK(index, query, k) {
1478
1493
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1479
1494
  return hits;
1480
1495
  }
1496
+ function asymmetricTopK(index, query, k) {
1497
+ const { bits, bytesPerRow, rows } = index;
1498
+ if (Math.ceil(query.length / 8) !== bytesPerRow) {
1499
+ throw new Error(
1500
+ `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1501
+ );
1502
+ }
1503
+ const limit = Math.min(k, rows);
1504
+ if (limit <= 0) return [];
1505
+ const lut = new Float32Array(bytesPerRow * 256);
1506
+ for (let p = 0; p < bytesPerRow; p++) {
1507
+ for (let b = 0; b < 256; b++) {
1508
+ let s = 0;
1509
+ for (let i = 0; i < 8; i++) {
1510
+ const d = p * 8 + i;
1511
+ if (d >= query.length) break;
1512
+ const v = query[d];
1513
+ s += b >> i & 1 ? v : -v;
1514
+ }
1515
+ lut[p * 256 + b] = s;
1516
+ }
1517
+ }
1518
+ const heapScore = new Float64Array(limit);
1519
+ const heapId = new Int32Array(limit);
1520
+ let size = 0;
1521
+ const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
1522
+ const swap = (i, j) => {
1523
+ const s = heapScore[i];
1524
+ const id = heapId[i];
1525
+ heapScore[i] = heapScore[j];
1526
+ heapId[i] = heapId[j];
1527
+ heapScore[j] = s;
1528
+ heapId[j] = id;
1529
+ };
1530
+ const siftUp = (start) => {
1531
+ let i = start;
1532
+ while (i > 0) {
1533
+ const parent = i - 1 >> 1;
1534
+ if (!worse(i, parent)) break;
1535
+ swap(i, parent);
1536
+ i = parent;
1537
+ }
1538
+ };
1539
+ const siftDown = (start) => {
1540
+ let i = start;
1541
+ for (; ; ) {
1542
+ const left = 2 * i + 1;
1543
+ if (left >= size) break;
1544
+ const right = left + 1;
1545
+ const child = right < size && worse(right, left) ? right : left;
1546
+ if (!worse(child, i)) break;
1547
+ swap(child, i);
1548
+ i = child;
1549
+ }
1550
+ };
1551
+ let offset = 0;
1552
+ for (let r = 0; r < rows; r++) {
1553
+ let score = 0;
1554
+ for (let p = 0; p < bytesPerRow; p++) {
1555
+ score += lut[p * 256 + bits[offset + p]];
1556
+ }
1557
+ offset += bytesPerRow;
1558
+ if (size < limit) {
1559
+ heapScore[size] = score;
1560
+ heapId[size] = r + 1;
1561
+ size++;
1562
+ siftUp(size - 1);
1563
+ } else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
1564
+ heapScore[0] = score;
1565
+ heapId[0] = r + 1;
1566
+ siftDown(0);
1567
+ }
1568
+ }
1569
+ const hits = [];
1570
+ for (let i = 0; i < size; i++) {
1571
+ hits.push({ chunkId: heapId[i], score: heapScore[i] });
1572
+ }
1573
+ hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
1574
+ return hits;
1575
+ }
1481
1576
 
1482
1577
  // src/reader/catalog-handle.ts
1483
1578
  var TOPIC_WALK_MAX_DEPTH = 32;
@@ -1499,6 +1594,8 @@ var CatalogHandle = class _CatalogHandle {
1499
1594
  connections = /* @__PURE__ */ new Map();
1500
1595
  bitIndexes = /* @__PURE__ */ new Map();
1501
1596
  assetsByPath = null;
1597
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
1598
+ binaryCenterCache;
1502
1599
  meta;
1503
1600
  shards;
1504
1601
  /** The router's `PRAGMA user_version` generation; decides which columns exist. */
@@ -1581,6 +1678,28 @@ var CatalogHandle = class _CatalogHandle {
1581
1678
  this.bitIndexes.set(shard.path, index);
1582
1679
  return index;
1583
1680
  }
1681
+ /**
1682
+ * The stage-1 center the catalog's bits were derived with (a
1683
+ * `centered-sign` profile), or null for plain sign bits. Read from the
1684
+ * router's profile echo, never from this reader's registry: the echo is
1685
+ * the statement of how THIS catalog was built.
1686
+ */
1687
+ binaryCenter() {
1688
+ if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
1689
+ const raw = this.meta.embedding_profile_json;
1690
+ let parsed = null;
1691
+ try {
1692
+ parsed = raw ? JSON.parse(raw) : null;
1693
+ } catch {
1694
+ parsed = null;
1695
+ }
1696
+ const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
1697
+ if (!profile.success) {
1698
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1699
+ }
1700
+ this.binaryCenterCache = embeddingProfileCenter(profile.data);
1701
+ return this.binaryCenterCache;
1702
+ }
1584
1703
  /** The embedding dimension from the router's profile echo. */
1585
1704
  dimensions() {
1586
1705
  const raw = this.meta.embedding_profile_json;
@@ -1809,12 +1928,15 @@ var CatalogHandle = class _CatalogHandle {
1809
1928
  return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
1810
1929
  }
1811
1930
  // ── search ────────────────────────────────────────────────────────────────
1812
- /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
1931
+ /**
1932
+ * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
1933
+ * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
1934
+ */
1813
1935
  searchDocumentsFts(query, limit = 10) {
1814
1936
  const match = ftsQuery(query);
1815
1937
  if (!match) return [];
1816
1938
  try {
1817
- return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
1939
+ return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
1818
1940
  documentId,
1819
1941
  rank: i
1820
1942
  }));
@@ -1863,7 +1985,7 @@ var CatalogHandle = class _CatalogHandle {
1863
1985
  return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
1864
1986
  }
1865
1987
  /**
1866
- * Two-stage semantic search over the routed shards: bit-hamming KNN
1988
+ * Two-stage semantic search over the routed shards: sign-bit scan
1867
1989
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
1868
1990
  */
1869
1991
  searchSemantic(queryVector, opts = {}) {
@@ -1875,14 +1997,15 @@ var CatalogHandle = class _CatalogHandle {
1875
1997
  * after global cross-catalog routing has already spent the S budget.
1876
1998
  */
1877
1999
  searchShards(queryVector, shardIds, finalK) {
1878
- const queryBits = quantizeBinary(queryVector);
2000
+ const center = this.binaryCenter();
2001
+ const scanQuery = center ? centerVector(queryVector, center) : queryVector;
1879
2002
  const hits = [];
1880
2003
  for (const shardId of shardIds) {
1881
2004
  const shard = this.shards.find((s) => s.id === shardId);
1882
2005
  if (!shard) continue;
1883
2006
  const db = this.shardDb(shard);
1884
2007
  const k = rerankK(finalK, shard.chunkCount);
1885
- const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
2008
+ const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
1886
2009
  chunk_id: hit.chunkId
1887
2010
  }));
1888
2011
  if (candidates.length === 0) continue;
@@ -2372,8 +2495,8 @@ import { createWriteStream as createWriteStream3 } from "fs";
2372
2495
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2373
2496
  import { join as join4, resolve as resolve3 } from "path";
2374
2497
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2375
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
2376
- var PARQUET_EXPORT_VERSION = 2;
2498
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
2499
+ var PARQUET_EXPORT_VERSION = 3;
2377
2500
  var DEFAULT_ROW_GROUP_SIZE = 32768;
2378
2501
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2379
2502
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2398,7 +2521,7 @@ async function exportCatalogParquet(opts) {
2398
2521
  const staging = join4(work, "ndjson");
2399
2522
  await mkdir3(staging, { recursive: true });
2400
2523
  const files = [];
2401
- const metadata = kvMetadata(manifest);
2524
+ const metadata = parquetKvMetadata(manifest);
2402
2525
  const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
2403
2526
  const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
2404
2527
  try {
@@ -2666,7 +2789,9 @@ function parseJsonStrings(raw) {
2666
2789
  function sqlString(value) {
2667
2790
  return `'${value.replace(/'/g, "''")}'`;
2668
2791
  }
2669
- function kvMetadata(manifest) {
2792
+ function parquetKvMetadata(manifest) {
2793
+ const binary = manifest.embedding.quantization.binary;
2794
+ const centered = embeddingProfileCenter2(manifest.embedding) !== null;
2670
2795
  return {
2671
2796
  gezk_catalog_id: manifest.id,
2672
2797
  gezk_catalog_version: manifest.version,
@@ -2676,7 +2801,9 @@ function kvMetadata(manifest) {
2676
2801
  gezk_embedding_model: manifest.embedding.model.repo,
2677
2802
  gezk_embedding_dimensions: String(manifest.embedding.dimensions),
2678
2803
  gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
2679
- gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
2804
+ gezk_embedding_bit: centered ? "centered-sign bits, LSB-first, ceil(dimensions/8) bytes: bit = (x - c) > 0, x the float unit vector, c = gezk_embedding_bit_center" : "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0, x the float unit vector",
2805
+ gezk_embedding_bit_method: binary.method,
2806
+ ...centered ? { gezk_embedding_bit_center: JSON.stringify(binary.center) } : {},
2680
2807
  gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
2681
2808
  gezk_license: manifest.license.name
2682
2809
  };
@@ -2787,6 +2914,394 @@ async function findDuckdbBinary(env = process.env) {
2787
2914
  return null;
2788
2915
  }
2789
2916
 
2917
+ // src/profiles/e5-small-center.ts
2918
+ var MULTILINGUAL_E5_SMALL_CENTER = [
2919
+ 0.046697,
2920
+ -0.013211,
2921
+ -0.042276,
2922
+ -0.049082,
2923
+ 0.060627,
2924
+ -0.032643,
2925
+ 0.029874,
2926
+ 0.033307,
2927
+ 0.058447,
2928
+ -0.012016,
2929
+ 0.046472,
2930
+ 5991e-6,
2931
+ 0.04884,
2932
+ -0.028836,
2933
+ -0.03244,
2934
+ 0.045633,
2935
+ 0.056798,
2936
+ -0.043557,
2937
+ -0.034472,
2938
+ -0.05355,
2939
+ 0.026363,
2940
+ -0.010318,
2941
+ -0.030487,
2942
+ 0.047083,
2943
+ 0.057205,
2944
+ 0.035654,
2945
+ -0.017461,
2946
+ 0.010881,
2947
+ 0.047985,
2948
+ -0.048514,
2949
+ -0.048956,
2950
+ -0.027643,
2951
+ 0.053259,
2952
+ -0.051057,
2953
+ 0.045055,
2954
+ 0.017888,
2955
+ -0.055755,
2956
+ -0.046578,
2957
+ 0.05214,
2958
+ -0.037085,
2959
+ -0.016268,
2960
+ 0.027508,
2961
+ 0.035793,
2962
+ 0.052082,
2963
+ 0.034885,
2964
+ 0.047174,
2965
+ -0.036778,
2966
+ 0.031394,
2967
+ -9156e-6,
2968
+ -0.02486,
2969
+ -0.048897,
2970
+ 0.052576,
2971
+ 0.017289,
2972
+ 0.047794,
2973
+ 0.051939,
2974
+ -0.051261,
2975
+ -0.061345,
2976
+ -0.056267,
2977
+ -0.049305,
2978
+ 0.047241,
2979
+ 0.043763,
2980
+ -0.04506,
2981
+ 0.015752,
2982
+ 0.017543,
2983
+ 0.067856,
2984
+ 0.065763,
2985
+ 0.037218,
2986
+ 0.044877,
2987
+ -0.057149,
2988
+ -0.028072,
2989
+ -0.060566,
2990
+ 0.064004,
2991
+ 6095e-6,
2992
+ -0.039693,
2993
+ 0.021968,
2994
+ 0.02654,
2995
+ 0.025546,
2996
+ -0.048954,
2997
+ 0.026967,
2998
+ -0.045002,
2999
+ -0.055556,
3000
+ -0.034963,
3001
+ -0.03466,
3002
+ 0.032486,
3003
+ -0.049597,
3004
+ 0.058718,
3005
+ 0.048021,
3006
+ -0.066783,
3007
+ 0.059049,
3008
+ -0.04665,
3009
+ 0.052902,
3010
+ 0.031438,
3011
+ -0.061515,
3012
+ -0.05342,
3013
+ -0.068253,
3014
+ -0.06666,
3015
+ -0.04553,
3016
+ 0.059252,
3017
+ 0.050728,
3018
+ -0.030429,
3019
+ 0.067182,
3020
+ -0.038819,
3021
+ 0.049358,
3022
+ -0.055247,
3023
+ -0.032933,
3024
+ 0.041239,
3025
+ 0.035164,
3026
+ -0.066574,
3027
+ 0.05348,
3028
+ -0.052658,
3029
+ -0.034758,
3030
+ 0.037219,
3031
+ 0.064111,
3032
+ 0.047861,
3033
+ -0.04608,
3034
+ -0.041338,
3035
+ -0.016072,
3036
+ -0.046026,
3037
+ 0.0255,
3038
+ -0.041581,
3039
+ 0.059684,
3040
+ -0.019611,
3041
+ -0.050363,
3042
+ -0.061268,
3043
+ -0.057556,
3044
+ -0.03184,
3045
+ 0.018522,
3046
+ 0.046887,
3047
+ 0.054774,
3048
+ 0.012536,
3049
+ 0.033019,
3050
+ 0.033211,
3051
+ 0.029609,
3052
+ 0.050215,
3053
+ 0.035425,
3054
+ 0.076419,
3055
+ -0.055196,
3056
+ -0.010016,
3057
+ -936e-6,
3058
+ -0.023764,
3059
+ -0.059648,
3060
+ 0.054857,
3061
+ -0.030456,
3062
+ 0.041281,
3063
+ 0.063894,
3064
+ 0.035466,
3065
+ 0.069754,
3066
+ -0.026593,
3067
+ 0.066305,
3068
+ -0.028609,
3069
+ 0.05324,
3070
+ -0.022879,
3071
+ 0.04882,
3072
+ 0.032385,
3073
+ 0.042921,
3074
+ -0.043319,
3075
+ -0.065022,
3076
+ -0.044018,
3077
+ 0.04237,
3078
+ 0.051018,
3079
+ -0.048907,
3080
+ -0.053045,
3081
+ -0.057301,
3082
+ -0.019731,
3083
+ -0.060812,
3084
+ -0.043114,
3085
+ 0.021207,
3086
+ 0.052498,
3087
+ -0.038307,
3088
+ -0.037082,
3089
+ -0.043732,
3090
+ 0.045959,
3091
+ -0.044296,
3092
+ 0.057199,
3093
+ -0.011835,
3094
+ 0.054221,
3095
+ -0.046354,
3096
+ 0.043442,
3097
+ 0.063337,
3098
+ 0.041964,
3099
+ -0.032836,
3100
+ -0.020089,
3101
+ -0.056327,
3102
+ -0.05089,
3103
+ -0.053933,
3104
+ -0.035486,
3105
+ -0.046215,
3106
+ 0.025754,
3107
+ 0.032986,
3108
+ -0.025048,
3109
+ -9903e-6,
3110
+ 0.031162,
3111
+ -0.050969,
3112
+ -0.049401,
3113
+ -0.046206,
3114
+ 0.023197,
3115
+ -0.033441,
3116
+ 0.045075,
3117
+ 0.043318,
3118
+ 0.043766,
3119
+ -6601e-6,
3120
+ -0.023077,
3121
+ 0.033917,
3122
+ 0.032279,
3123
+ 0.034874,
3124
+ 5185e-6,
3125
+ -0.057329,
3126
+ 0.048437,
3127
+ -0.033964,
3128
+ 0.044195,
3129
+ 0.051107,
3130
+ -0.056642,
3131
+ -0.070856,
3132
+ 0.045431,
3133
+ -0.047205,
3134
+ -0.021457,
3135
+ 3103e-6,
3136
+ 0.062536,
3137
+ -0.037804,
3138
+ 0.031278,
3139
+ 0.041619,
3140
+ -0.015547,
3141
+ 0.052298,
3142
+ -0.078327,
3143
+ -0.052303,
3144
+ 0.043099,
3145
+ 0.05375,
3146
+ -0.041854,
3147
+ -0.038314,
3148
+ 0.039399,
3149
+ -0.047769,
3150
+ -0.038469,
3151
+ -0.056793,
3152
+ -0.064312,
3153
+ -0.047884,
3154
+ -0.060067,
3155
+ -0.030687,
3156
+ 0.030053,
3157
+ 0.033569,
3158
+ -0.038014,
3159
+ -0.032396,
3160
+ -0.041395,
3161
+ 0.038163,
3162
+ -0.056206,
3163
+ 0.047115,
3164
+ -0.038099,
3165
+ -0.031927,
3166
+ 0.033857,
3167
+ -0.011227,
3168
+ 0.032883,
3169
+ 0.032516,
3170
+ -0.045622,
3171
+ -0.044387,
3172
+ -0.034374,
3173
+ -0.030433,
3174
+ 0.055097,
3175
+ 0.055568,
3176
+ 0.058726,
3177
+ -0.043569,
3178
+ 0.031854,
3179
+ 775e-5,
3180
+ -0.022533,
3181
+ 0.096133,
3182
+ 0.052876,
3183
+ 0.045582,
3184
+ 0.041868,
3185
+ -0.039268,
3186
+ -0.037865,
3187
+ -0.054658,
3188
+ -0.031297,
3189
+ -0.058104,
3190
+ 0.018705,
3191
+ 0.031934,
3192
+ -0.052746,
3193
+ -0.044438,
3194
+ -0.029991,
3195
+ 0.011917,
3196
+ 0.070103,
3197
+ -0.046682,
3198
+ -0.048965,
3199
+ 0.046255,
3200
+ 0.027877,
3201
+ 0.0475,
3202
+ 0.044287,
3203
+ 0.04251,
3204
+ -0.017647,
3205
+ -0.017812,
3206
+ 0.055672,
3207
+ -0.042207,
3208
+ -0.035516,
3209
+ -0.025558,
3210
+ -0.062443,
3211
+ 0.037519,
3212
+ -0.031697,
3213
+ 0.064998,
3214
+ 0.045949,
3215
+ 302e-6,
3216
+ 0.065205,
3217
+ -0.052016,
3218
+ 0.036721,
3219
+ -0.034769,
3220
+ -0.049074,
3221
+ 0.040413,
3222
+ 0.045425,
3223
+ -0.068063,
3224
+ 0.021485,
3225
+ 8549e-6,
3226
+ 0.028179,
3227
+ 0.026914,
3228
+ 0.020493,
3229
+ 0.056155,
3230
+ 0.047202,
3231
+ -0.049085,
3232
+ -0.022624,
3233
+ 0.063734,
3234
+ 0.05471,
3235
+ -0.032258,
3236
+ 0.047761,
3237
+ -0.065033,
3238
+ -0.047818,
3239
+ -0.034254,
3240
+ -0.062863,
3241
+ -0.01362,
3242
+ -0.053904,
3243
+ 0.024415,
3244
+ 0.044367,
3245
+ -0.064056,
3246
+ -0.038419,
3247
+ 0.038997,
3248
+ 1015e-6,
3249
+ 0.038928,
3250
+ -0.045794,
3251
+ -0.038313,
3252
+ 0.043922,
3253
+ -0.05873,
3254
+ -0.020314,
3255
+ -0.020457,
3256
+ 0.045522,
3257
+ -0.057731,
3258
+ -0.062966,
3259
+ 0.023159,
3260
+ 0.020283,
3261
+ -0.057352,
3262
+ 0.044701,
3263
+ -0.03585,
3264
+ -0.052798,
3265
+ 0.056931,
3266
+ -0.035609,
3267
+ -0.035113,
3268
+ 0.026919,
3269
+ 0.059404,
3270
+ -0.084434,
3271
+ 0.014558,
3272
+ 0.06226,
3273
+ -0.044173,
3274
+ 0.044172,
3275
+ -0.046482,
3276
+ -0.034933,
3277
+ 0.052513,
3278
+ 0.044018,
3279
+ -0.057677,
3280
+ -0.033961,
3281
+ 0.046334,
3282
+ 0.065767,
3283
+ 0.044779,
3284
+ 0.063658,
3285
+ -0.022574,
3286
+ -4821e-6,
3287
+ 0.037388,
3288
+ -0.043197,
3289
+ 0.063,
3290
+ 0.048943,
3291
+ -0.049495,
3292
+ 0.014798,
3293
+ -0.018176,
3294
+ -0.03824,
3295
+ -0.045486,
3296
+ 0.053828,
3297
+ -0.056118,
3298
+ -0.067025,
3299
+ 0.027818,
3300
+ 0.050305,
3301
+ 0.037134,
3302
+ 0.049673
3303
+ ];
3304
+
2790
3305
  // src/profiles/registry.ts
2791
3306
  var SHARED_ENCODING = {
2792
3307
  pooling: "mean",
@@ -2817,6 +3332,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
2817
3332
  passageInstruction: "passage: ",
2818
3333
  ...SHARED_ENCODING
2819
3334
  };
3335
+ var MULTILINGUAL_E5_SMALL_2 = {
3336
+ ...MULTILINGUAL_E5_SMALL_1,
3337
+ id: "multilingual-e5-small@2",
3338
+ quantization: {
3339
+ int8: { method: "symmetric-linear", scale: 127 },
3340
+ binary: {
3341
+ method: "centered-sign",
3342
+ threshold: 0,
3343
+ packing: "lsb-first",
3344
+ center: [...MULTILINGUAL_E5_SMALL_CENTER]
3345
+ }
3346
+ }
3347
+ };
2820
3348
  var BGE_SMALL_EN_V15_1 = {
2821
3349
  id: "bge-small-en-v1.5@1",
2822
3350
  model: {
@@ -2835,6 +3363,7 @@ var BGE_SMALL_EN_V15_1 = {
2835
3363
  ...SHARED_ENCODING
2836
3364
  };
2837
3365
  var KNOWLEDGE_EMBEDDING_PROFILES = [
3366
+ MULTILINGUAL_E5_SMALL_2,
2838
3367
  MULTILINGUAL_E5_SMALL_1,
2839
3368
  BGE_SMALL_EN_V15_1
2840
3369
  ];
@@ -2950,6 +3479,25 @@ var EmbedderUnavailableError = class extends Error {
2950
3479
  this.name = "EmbedderUnavailableError";
2951
3480
  }
2952
3481
  };
3482
+ function aggregateDownloadProgress(onProgress) {
3483
+ const files = /* @__PURE__ */ new Map();
3484
+ return (info) => {
3485
+ if (info.status !== "progress") return;
3486
+ const loaded = typeof info.loaded === "number" ? info.loaded : 0;
3487
+ const total = typeof info.total === "number" ? info.total : 0;
3488
+ files.set(`${String(info.name)}/${String(info.file)}`, {
3489
+ loaded,
3490
+ total: Math.max(total, loaded)
3491
+ });
3492
+ let bytesDone = 0;
3493
+ let bytesTotal = 0;
3494
+ for (const file of files.values()) {
3495
+ bytesDone += file.loaded;
3496
+ bytesTotal += file.total;
3497
+ }
3498
+ onProgress({ bytesDone, bytesTotal });
3499
+ };
3500
+ }
2953
3501
  var ONNX_DTYPE_BY_SUFFIX = [
2954
3502
  ["_q4f16", "q4f16"],
2955
3503
  ["_quantized", "q8"],
@@ -2970,21 +3518,24 @@ function resolveTransformersModelOptions(profile) {
2970
3518
  `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
2971
3519
  );
2972
3520
  }
2973
- if (!onnxFile.endsWith(".onnx")) {
3521
+ const graph = transformersGraphOptions(onnxFile);
3522
+ if (!graph) {
2974
3523
  throw new EmbedderUnavailableError(
2975
- `profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
3524
+ `profile ${profile.id} pins ${onnxFile}; this runtime loads named ONNX graphs only`
2976
3525
  );
2977
3526
  }
3527
+ return { revision: profile.model.revision, ...graph };
3528
+ }
3529
+ function transformersGraphOptions(onnxFile) {
3530
+ if (!onnxFile.endsWith(".onnx")) return null;
2978
3531
  const slash = onnxFile.lastIndexOf("/");
2979
3532
  const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
2980
3533
  const base = onnxFile.slice(slash + 1, -".onnx".length);
2981
3534
  const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
2982
3535
  const [suffix, dtype] = variant ?? ["", "fp32"];
2983
3536
  const modelFileName = base.slice(0, base.length - suffix.length);
2984
- if (modelFileName === "") {
2985
- throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
2986
- }
2987
- return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
3537
+ if (modelFileName === "") return null;
3538
+ return { dtype, subfolder, model_file_name: modelFileName };
2988
3539
  }
2989
3540
  async function loadTransformers() {
2990
3541
  try {
@@ -3015,13 +3566,16 @@ async function createProfileEmbedder(profile, opts = {}) {
3015
3566
  transformers.env.useFSCache = true;
3016
3567
  transformers.env.allowRemoteModels = true;
3017
3568
  }
3569
+ const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
3018
3570
  const [pipe, tokenizer] = await Promise.all([
3019
3571
  transformers.pipeline("feature-extraction", profile.model.repo, {
3020
3572
  ...modelOptions,
3021
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
3573
+ ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
3574
+ ...progress
3022
3575
  }),
3023
3576
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3024
- revision: modelOptions.revision
3577
+ revision: modelOptions.revision,
3578
+ ...progress
3025
3579
  })
3026
3580
  ]);
3027
3581
  let verification;
@@ -4122,9 +4676,12 @@ export {
4122
4676
  MAX_KNOWLEDGE_TOPIC_DEPTH,
4123
4677
  MIMETYPE_PATH,
4124
4678
  MULTILINGUAL_E5_SMALL_1,
4679
+ MULTILINGUAL_E5_SMALL_2,
4125
4680
  PARQUET_EXPORT_VERSION,
4126
4681
  PARQUET_REPORT_PATH,
4127
4682
  README_PATH,
4683
+ RERANK_CANDIDATES_MAX,
4684
+ RERANK_CANDIDATES_MIN,
4128
4685
  RERANK_FINAL_K,
4129
4686
  ROUTER_DB_PATH,
4130
4687
  ROUTER_DDL,
@@ -4138,7 +4695,9 @@ export {
4138
4695
  assertDuckdbCli,
4139
4696
  assetReferences,
4140
4697
  assignOutlineOrders,
4698
+ asymmetricTopK,
4141
4699
  canonicalizeJson2 as canonicalizeJson,
4700
+ centerVector,
4142
4701
  chunkContentHash,
4143
4702
  chunkMarkdownProfile,
4144
4703
  chunkUid,
@@ -4166,10 +4725,12 @@ export {
4166
4725
  newOutlineTopic,
4167
4726
  newerRegistryEntries,
4168
4727
  openCatalogDatabase,
4728
+ parquetKvMetadata,
4169
4729
  parseGitbookSummary,
4170
4730
  parseJupyterBookToc,
4171
4731
  parseMkdocsNav,
4172
4732
  quantizeBinary,
4733
+ quantizeBinaryForProfile,
4173
4734
  quantizeInt8,
4174
4735
  readGezkManifest,
4175
4736
  readMkdocsDocsDir,
@@ -4182,6 +4743,7 @@ export {
4182
4743
  signRegistryIndex,
4183
4744
  titleFromFolderName,
4184
4745
  transformersCachePath,
4746
+ transformersGraphOptions,
4185
4747
  validateExtractedCatalog,
4186
4748
  verifyManifestSignature,
4187
4749
  verifyProfileArtifacts,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.1.1",
3
+ "version": "1.2.0",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [
@@ -47,7 +47,7 @@
47
47
  "!dist/**/*.map"
48
48
  ],
49
49
  "dependencies": {
50
- "@bendyline/gezk": "1.0.1",
50
+ "@bendyline/gezk": "1.0.2",
51
51
  "yaml": "^2.9.0",
52
52
  "yauzl": "^3.4.0",
53
53
  "yazl": "^3.3.1",