@bendyline/gezel-knowledge 1.1.1 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -81,8 +81,9 @@ Extraction verifies every file against the manifest (both directions —
81
81
  undeclared and missing files are equally fatal) and rejects unsafe archive
82
82
  entries. Catalog databases are opened strictly read-only and immutable.
83
83
  Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
84
- full 200,000-chunk shard) and scans them with a popcount; the int8 rerank
85
- reads the candidates by rowid.
84
+ full 200,000-chunk shard) and scores the float query against them through a
85
+ per-byte lookup table (the query is centered first when the catalog's profile
86
+ is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
86
87
 
87
88
  ## Stability
88
89
 
package/dist/index.d.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
2
- export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
2
+ export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
3
3
  import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
4
4
  export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
5
5
  import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
16
16
  declare const CENTROID_SAMPLE_MAX = 65536;
17
17
  declare const ROUTE_SHARDS_PROACTIVE = 3;
18
18
  declare const ROUTE_SHARDS_EXPLICIT = 6;
19
- /** Two-stage retrieval (§2): hamming top-K → int8 rerank. */
19
+ /**
20
+ * Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
21
+ * candidates. K is deliberately generous. Measured on a 43,859-chunk
22
+ * multilingual-e5-small catalog with natural-language questions, recall of
23
+ * the exact top-24 among the stage-1 candidates was 22% at the former
24
+ * K = 192 with raw sign bits, and with centered bits (`centered-sign`
25
+ * profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
26
+ * reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
27
+ * shard. A small K is a silent recall cliff; a large one is cheap.
28
+ */
20
29
  declare const RERANK_FINAL_K = 24;
30
+ declare const RERANK_CANDIDATES_MIN = 1024;
31
+ declare const RERANK_CANDIDATES_MAX = 4096;
21
32
  declare function rerankK(finalK?: number, chunkCount?: number): number;
22
33
 
23
34
  /**
@@ -309,12 +320,16 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
309
320
  * One mounted catalog: the router database plus lazily opened shards, and
310
321
  * the two-stage query flow (the gezk spec §8 and §9):
311
322
  *
312
- * embed query (caller) → centroid routing → per shard: bit-hamming KNN
323
+ * embed query (caller) → centroid routing → per shard: sign-bit scan
313
324
  * top-K → int8 rerank → fuse.
314
325
  *
315
326
  * Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
316
- * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with a
317
- * popcount, so no vector extension is needed to read a catalog.
327
+ * contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
328
+ * the float query (`asymmetricTopK`), so no vector extension is needed to
329
+ * read a catalog. The catalog's own profile echo says how its bits were
330
+ * derived: a `centered-sign` profile's bits come from `vector − center`, so
331
+ * the query is centered the same way before the scan. The int8 rerank is
332
+ * identical for both kinds of bits.
318
333
  *
319
334
  * The handle is synchronous (node:sqlite) and single-threaded by design —
320
335
  * the daemon confines every handle to its knowledge worker thread; the CLI
@@ -383,6 +398,8 @@ declare class CatalogHandle {
383
398
  private readonly connections;
384
399
  private readonly bitIndexes;
385
400
  private assetsByPath;
401
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
402
+ private binaryCenterCache;
386
403
  readonly meta: Record<string, string>;
387
404
  readonly shards: ShardInfo[];
388
405
  /** The router's `PRAGMA user_version` generation; decides which columns exist. */
@@ -406,6 +423,13 @@ declare class CatalogHandle {
406
423
  * the row index IS the id — so a gap is a corrupt shard, not a sparse one.
407
424
  */
408
425
  private shardBits;
426
+ /**
427
+ * The stage-1 center the catalog's bits were derived with (a
428
+ * `centered-sign` profile), or null for plain sign bits. Read from the
429
+ * router's profile echo, never from this reader's registry: the echo is
430
+ * the statement of how THIS catalog was built.
431
+ */
432
+ private binaryCenter;
409
433
  /** The embedding dimension from the router's profile echo. */
410
434
  private dimensions;
411
435
  /** Resolve a manifest/router path without allowing it to escape the catalog. */
@@ -489,7 +513,7 @@ declare class CatalogHandle {
489
513
  /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
490
514
  routeShards(queryVector: Float32Array, shardBudget: number): number[];
491
515
  /**
492
- * Two-stage semantic search over the routed shards: bit-hamming KNN
516
+ * Two-stage semantic search over the routed shards: sign-bit scan
493
517
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
494
518
  */
495
519
  searchSemantic(queryVector: Float32Array, opts?: {
@@ -540,10 +564,21 @@ declare function assetReferences(markdown: string): string[];
540
564
 
541
565
  /**
542
566
  * Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
543
- * 200k-chunk shard is 9.6 MB of bits; one linear hamming pass with a SWAR
544
- * popcount plus a histogram selection of the top-K is a few milliseconds and
545
- * needs nothing beyond plain SQLite to load — which is what lets any reader
546
- * implement the format without a vector extension.
567
+ * 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
568
+ * pass over them, a few milliseconds, needing nothing beyond plain SQLite
569
+ * to load — which is what lets any reader implement the format without a
570
+ * vector extension.
571
+ *
572
+ * Two scans share the bits:
573
+ * - `hammingTopK` — symmetric: the query is binarized too and rows are
574
+ * ranked by popcount(xor). The format's baseline (§2), and what the
575
+ * self-KNN smoke and the conformance kit exercise.
576
+ * - `asymmetricTopK` — the float query against each row's bits, ranked by
577
+ * Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
578
+ * embeddings roughly doubles the share of true neighbours a small
579
+ * candidate pool retains (measured on multilingual-e5-small: 22% → 39%
580
+ * of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
581
+ * bits). This is what `CatalogHandle` uses.
547
582
  */
548
583
  interface ShardBitIndex {
549
584
  /** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
@@ -561,6 +596,23 @@ interface HammingHit {
561
596
  * the whole query is two linear passes and no heap.
562
597
  */
563
598
  declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
599
+ interface AsymmetricHit {
600
+ chunkId: number;
601
+ score: number;
602
+ }
603
+ /**
604
+ * The K best rows by asymmetric score, descending, ties broken by chunk id.
605
+ *
606
+ * The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
607
+ * of 256 partial sums is built for each byte position, so a row costs
608
+ * `bytesPerRow` table reads and adds — about the popcount scan's cost. The
609
+ * top-K is kept in a bounded min-heap whose root is the worst kept row.
610
+ *
611
+ * For a `centered-sign` profile the caller passes `query − center`; for a
612
+ * plain `sign` profile the unit query. The bits are whatever the catalog
613
+ * stored; this function does not know or care which.
614
+ */
615
+ declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
564
616
 
565
617
  /**
566
618
  * Provenance stamped into every manifest this package compiles. Only the
@@ -678,10 +730,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
678
730
  * built from a profile verifies the files it loaded against these digests.
679
731
  */
680
732
 
681
- /** The public profile: multilingual, so world catalogs serve every reader. */
733
+ /**
734
+ * The public profile's first revision: multilingual, so world catalogs serve
735
+ * every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
736
+ * vectors, centered stage-1 bits); catalogs built with this revision stay
737
+ * readable and searchable, at the lower stage-1 recall of raw sign bits.
738
+ */
682
739
  declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
740
+ /**
741
+ * The public profile, revision 2: the model, files, instructions and int8
742
+ * rerank of revision 1, with the stage-1 sign bits taken from
743
+ * `vector − center` (`centered-sign`). e5-small vectors share one dominant
744
+ * direction, so raw sign bits mostly encode that direction rather than the
745
+ * passage and the hamming pre-filter loses most true neighbours; centering
746
+ * on the pinned corpus mean restores them (see e5-small-center.ts for the
747
+ * provenance and the measurements). A query vector for revision 1 is a
748
+ * valid query vector for revision 2 — `sameVectorSpace` says so — and the
749
+ * reader centers it itself from the catalog's profile echo.
750
+ */
751
+ declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
683
752
  /** The local-build default — matches gezel's shipped project embedder. */
684
753
  declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
754
+ /** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
685
755
  declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
686
756
  declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
687
757
  /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
@@ -717,9 +787,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
717
787
  scale: 127;
718
788
  };
719
789
  binary: {
720
- method: "sign";
790
+ method: "sign" | "centered-sign";
721
791
  threshold: 0;
722
792
  packing: "lsb-first";
793
+ center?: number[] | undefined;
723
794
  };
724
795
  };
725
796
  };
@@ -754,9 +825,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
754
825
  scale: 127;
755
826
  };
756
827
  binary: {
757
- method: "sign";
828
+ method: "sign" | "centered-sign";
758
829
  threshold: 0;
759
830
  packing: "lsb-first";
831
+ center?: number[] | undefined;
760
832
  };
761
833
  };
762
834
  };
@@ -1150,4 +1222,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1150
1222
 
1151
1223
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1152
1224
 
1153
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
1225
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -30,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
30
30
  var ROUTE_SHARDS_PROACTIVE = 3;
31
31
  var ROUTE_SHARDS_EXPLICIT = 6;
32
32
  var RERANK_FINAL_K = 24;
33
+ var RERANK_CANDIDATES_MIN = 1024;
34
+ var RERANK_CANDIDATES_MAX = 4096;
33
35
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
34
- return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
36
+ return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
35
37
  }
36
38
 
37
39
  // src/format/ddl.ts
@@ -42,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
42
44
  import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
43
45
 
44
46
  // src/format/quantize.ts
45
- import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
47
+ import {
48
+ centerVector,
49
+ l2Normalize,
50
+ quantizeBinary,
51
+ quantizeBinaryForProfile,
52
+ quantizeInt8,
53
+ rerankScore
54
+ } from "@bendyline/gezk";
46
55
 
47
56
  // src/chunking/markdown-chunker.ts
48
57
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -469,7 +478,7 @@ async function compileKnowledgeCatalog(opts) {
469
478
  }
470
479
  const row = pendingRows[i];
471
480
  const stmts = vecStmtsFor(shardDb(row.shardId));
472
- stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinary(unit)));
481
+ stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
473
482
  stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
474
483
  const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
475
484
  void shardTotal;
@@ -1403,8 +1412,10 @@ import { readFileSync as readFileSync2 } from "fs";
1403
1412
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1404
1413
  import { brotliDecompressSync } from "zlib";
1405
1414
  import {
1415
+ KnowledgeEmbeddingProfileSchema,
1406
1416
  MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1407
1417
  assetContentType,
1418
+ embeddingProfileCenter,
1408
1419
  isKnowledgeAssetPath
1409
1420
  } from "@bendyline/gezk";
1410
1421
 
@@ -1478,6 +1489,86 @@ function hammingTopK(index, query, k) {
1478
1489
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1479
1490
  return hits;
1480
1491
  }
1492
+ function asymmetricTopK(index, query, k) {
1493
+ const { bits, bytesPerRow, rows } = index;
1494
+ if (Math.ceil(query.length / 8) !== bytesPerRow) {
1495
+ throw new Error(
1496
+ `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1497
+ );
1498
+ }
1499
+ const limit = Math.min(k, rows);
1500
+ if (limit <= 0) return [];
1501
+ const lut = new Float32Array(bytesPerRow * 256);
1502
+ for (let p = 0; p < bytesPerRow; p++) {
1503
+ for (let b = 0; b < 256; b++) {
1504
+ let s = 0;
1505
+ for (let i = 0; i < 8; i++) {
1506
+ const d = p * 8 + i;
1507
+ if (d >= query.length) break;
1508
+ const v = query[d];
1509
+ s += b >> i & 1 ? v : -v;
1510
+ }
1511
+ lut[p * 256 + b] = s;
1512
+ }
1513
+ }
1514
+ const heapScore = new Float64Array(limit);
1515
+ const heapId = new Int32Array(limit);
1516
+ let size = 0;
1517
+ const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
1518
+ const swap = (i, j) => {
1519
+ const s = heapScore[i];
1520
+ const id = heapId[i];
1521
+ heapScore[i] = heapScore[j];
1522
+ heapId[i] = heapId[j];
1523
+ heapScore[j] = s;
1524
+ heapId[j] = id;
1525
+ };
1526
+ const siftUp = (start) => {
1527
+ let i = start;
1528
+ while (i > 0) {
1529
+ const parent = i - 1 >> 1;
1530
+ if (!worse(i, parent)) break;
1531
+ swap(i, parent);
1532
+ i = parent;
1533
+ }
1534
+ };
1535
+ const siftDown = (start) => {
1536
+ let i = start;
1537
+ for (; ; ) {
1538
+ const left = 2 * i + 1;
1539
+ if (left >= size) break;
1540
+ const right = left + 1;
1541
+ const child = right < size && worse(right, left) ? right : left;
1542
+ if (!worse(child, i)) break;
1543
+ swap(child, i);
1544
+ i = child;
1545
+ }
1546
+ };
1547
+ let offset = 0;
1548
+ for (let r = 0; r < rows; r++) {
1549
+ let score = 0;
1550
+ for (let p = 0; p < bytesPerRow; p++) {
1551
+ score += lut[p * 256 + bits[offset + p]];
1552
+ }
1553
+ offset += bytesPerRow;
1554
+ if (size < limit) {
1555
+ heapScore[size] = score;
1556
+ heapId[size] = r + 1;
1557
+ size++;
1558
+ siftUp(size - 1);
1559
+ } else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
1560
+ heapScore[0] = score;
1561
+ heapId[0] = r + 1;
1562
+ siftDown(0);
1563
+ }
1564
+ }
1565
+ const hits = [];
1566
+ for (let i = 0; i < size; i++) {
1567
+ hits.push({ chunkId: heapId[i], score: heapScore[i] });
1568
+ }
1569
+ hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
1570
+ return hits;
1571
+ }
1481
1572
 
1482
1573
  // src/reader/catalog-handle.ts
1483
1574
  var TOPIC_WALK_MAX_DEPTH = 32;
@@ -1499,6 +1590,8 @@ var CatalogHandle = class _CatalogHandle {
1499
1590
  connections = /* @__PURE__ */ new Map();
1500
1591
  bitIndexes = /* @__PURE__ */ new Map();
1501
1592
  assetsByPath = null;
1593
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
1594
+ binaryCenterCache;
1502
1595
  meta;
1503
1596
  shards;
1504
1597
  /** The router's `PRAGMA user_version` generation; decides which columns exist. */
@@ -1581,6 +1674,28 @@ var CatalogHandle = class _CatalogHandle {
1581
1674
  this.bitIndexes.set(shard.path, index);
1582
1675
  return index;
1583
1676
  }
1677
+ /**
1678
+ * The stage-1 center the catalog's bits were derived with (a
1679
+ * `centered-sign` profile), or null for plain sign bits. Read from the
1680
+ * router's profile echo, never from this reader's registry: the echo is
1681
+ * the statement of how THIS catalog was built.
1682
+ */
1683
+ binaryCenter() {
1684
+ if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
1685
+ const raw = this.meta.embedding_profile_json;
1686
+ let parsed = null;
1687
+ try {
1688
+ parsed = raw ? JSON.parse(raw) : null;
1689
+ } catch {
1690
+ parsed = null;
1691
+ }
1692
+ const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
1693
+ if (!profile.success) {
1694
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1695
+ }
1696
+ this.binaryCenterCache = embeddingProfileCenter(profile.data);
1697
+ return this.binaryCenterCache;
1698
+ }
1584
1699
  /** The embedding dimension from the router's profile echo. */
1585
1700
  dimensions() {
1586
1701
  const raw = this.meta.embedding_profile_json;
@@ -1863,7 +1978,7 @@ var CatalogHandle = class _CatalogHandle {
1863
1978
  return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
1864
1979
  }
1865
1980
  /**
1866
- * Two-stage semantic search over the routed shards: bit-hamming KNN
1981
+ * Two-stage semantic search over the routed shards: sign-bit scan
1867
1982
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
1868
1983
  */
1869
1984
  searchSemantic(queryVector, opts = {}) {
@@ -1875,14 +1990,15 @@ var CatalogHandle = class _CatalogHandle {
1875
1990
  * after global cross-catalog routing has already spent the S budget.
1876
1991
  */
1877
1992
  searchShards(queryVector, shardIds, finalK) {
1878
- const queryBits = quantizeBinary(queryVector);
1993
+ const center = this.binaryCenter();
1994
+ const scanQuery = center ? centerVector(queryVector, center) : queryVector;
1879
1995
  const hits = [];
1880
1996
  for (const shardId of shardIds) {
1881
1997
  const shard = this.shards.find((s) => s.id === shardId);
1882
1998
  if (!shard) continue;
1883
1999
  const db = this.shardDb(shard);
1884
2000
  const k = rerankK(finalK, shard.chunkCount);
1885
- const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
2001
+ const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
1886
2002
  chunk_id: hit.chunkId
1887
2003
  }));
1888
2004
  if (candidates.length === 0) continue;
@@ -2787,6 +2903,394 @@ async function findDuckdbBinary(env = process.env) {
2787
2903
  return null;
2788
2904
  }
2789
2905
 
2906
+ // src/profiles/e5-small-center.ts
2907
+ var MULTILINGUAL_E5_SMALL_CENTER = [
2908
+ 0.046697,
2909
+ -0.013211,
2910
+ -0.042276,
2911
+ -0.049082,
2912
+ 0.060627,
2913
+ -0.032643,
2914
+ 0.029874,
2915
+ 0.033307,
2916
+ 0.058447,
2917
+ -0.012016,
2918
+ 0.046472,
2919
+ 5991e-6,
2920
+ 0.04884,
2921
+ -0.028836,
2922
+ -0.03244,
2923
+ 0.045633,
2924
+ 0.056798,
2925
+ -0.043557,
2926
+ -0.034472,
2927
+ -0.05355,
2928
+ 0.026363,
2929
+ -0.010318,
2930
+ -0.030487,
2931
+ 0.047083,
2932
+ 0.057205,
2933
+ 0.035654,
2934
+ -0.017461,
2935
+ 0.010881,
2936
+ 0.047985,
2937
+ -0.048514,
2938
+ -0.048956,
2939
+ -0.027643,
2940
+ 0.053259,
2941
+ -0.051057,
2942
+ 0.045055,
2943
+ 0.017888,
2944
+ -0.055755,
2945
+ -0.046578,
2946
+ 0.05214,
2947
+ -0.037085,
2948
+ -0.016268,
2949
+ 0.027508,
2950
+ 0.035793,
2951
+ 0.052082,
2952
+ 0.034885,
2953
+ 0.047174,
2954
+ -0.036778,
2955
+ 0.031394,
2956
+ -9156e-6,
2957
+ -0.02486,
2958
+ -0.048897,
2959
+ 0.052576,
2960
+ 0.017289,
2961
+ 0.047794,
2962
+ 0.051939,
2963
+ -0.051261,
2964
+ -0.061345,
2965
+ -0.056267,
2966
+ -0.049305,
2967
+ 0.047241,
2968
+ 0.043763,
2969
+ -0.04506,
2970
+ 0.015752,
2971
+ 0.017543,
2972
+ 0.067856,
2973
+ 0.065763,
2974
+ 0.037218,
2975
+ 0.044877,
2976
+ -0.057149,
2977
+ -0.028072,
2978
+ -0.060566,
2979
+ 0.064004,
2980
+ 6095e-6,
2981
+ -0.039693,
2982
+ 0.021968,
2983
+ 0.02654,
2984
+ 0.025546,
2985
+ -0.048954,
2986
+ 0.026967,
2987
+ -0.045002,
2988
+ -0.055556,
2989
+ -0.034963,
2990
+ -0.03466,
2991
+ 0.032486,
2992
+ -0.049597,
2993
+ 0.058718,
2994
+ 0.048021,
2995
+ -0.066783,
2996
+ 0.059049,
2997
+ -0.04665,
2998
+ 0.052902,
2999
+ 0.031438,
3000
+ -0.061515,
3001
+ -0.05342,
3002
+ -0.068253,
3003
+ -0.06666,
3004
+ -0.04553,
3005
+ 0.059252,
3006
+ 0.050728,
3007
+ -0.030429,
3008
+ 0.067182,
3009
+ -0.038819,
3010
+ 0.049358,
3011
+ -0.055247,
3012
+ -0.032933,
3013
+ 0.041239,
3014
+ 0.035164,
3015
+ -0.066574,
3016
+ 0.05348,
3017
+ -0.052658,
3018
+ -0.034758,
3019
+ 0.037219,
3020
+ 0.064111,
3021
+ 0.047861,
3022
+ -0.04608,
3023
+ -0.041338,
3024
+ -0.016072,
3025
+ -0.046026,
3026
+ 0.0255,
3027
+ -0.041581,
3028
+ 0.059684,
3029
+ -0.019611,
3030
+ -0.050363,
3031
+ -0.061268,
3032
+ -0.057556,
3033
+ -0.03184,
3034
+ 0.018522,
3035
+ 0.046887,
3036
+ 0.054774,
3037
+ 0.012536,
3038
+ 0.033019,
3039
+ 0.033211,
3040
+ 0.029609,
3041
+ 0.050215,
3042
+ 0.035425,
3043
+ 0.076419,
3044
+ -0.055196,
3045
+ -0.010016,
3046
+ -936e-6,
3047
+ -0.023764,
3048
+ -0.059648,
3049
+ 0.054857,
3050
+ -0.030456,
3051
+ 0.041281,
3052
+ 0.063894,
3053
+ 0.035466,
3054
+ 0.069754,
3055
+ -0.026593,
3056
+ 0.066305,
3057
+ -0.028609,
3058
+ 0.05324,
3059
+ -0.022879,
3060
+ 0.04882,
3061
+ 0.032385,
3062
+ 0.042921,
3063
+ -0.043319,
3064
+ -0.065022,
3065
+ -0.044018,
3066
+ 0.04237,
3067
+ 0.051018,
3068
+ -0.048907,
3069
+ -0.053045,
3070
+ -0.057301,
3071
+ -0.019731,
3072
+ -0.060812,
3073
+ -0.043114,
3074
+ 0.021207,
3075
+ 0.052498,
3076
+ -0.038307,
3077
+ -0.037082,
3078
+ -0.043732,
3079
+ 0.045959,
3080
+ -0.044296,
3081
+ 0.057199,
3082
+ -0.011835,
3083
+ 0.054221,
3084
+ -0.046354,
3085
+ 0.043442,
3086
+ 0.063337,
3087
+ 0.041964,
3088
+ -0.032836,
3089
+ -0.020089,
3090
+ -0.056327,
3091
+ -0.05089,
3092
+ -0.053933,
3093
+ -0.035486,
3094
+ -0.046215,
3095
+ 0.025754,
3096
+ 0.032986,
3097
+ -0.025048,
3098
+ -9903e-6,
3099
+ 0.031162,
3100
+ -0.050969,
3101
+ -0.049401,
3102
+ -0.046206,
3103
+ 0.023197,
3104
+ -0.033441,
3105
+ 0.045075,
3106
+ 0.043318,
3107
+ 0.043766,
3108
+ -6601e-6,
3109
+ -0.023077,
3110
+ 0.033917,
3111
+ 0.032279,
3112
+ 0.034874,
3113
+ 5185e-6,
3114
+ -0.057329,
3115
+ 0.048437,
3116
+ -0.033964,
3117
+ 0.044195,
3118
+ 0.051107,
3119
+ -0.056642,
3120
+ -0.070856,
3121
+ 0.045431,
3122
+ -0.047205,
3123
+ -0.021457,
3124
+ 3103e-6,
3125
+ 0.062536,
3126
+ -0.037804,
3127
+ 0.031278,
3128
+ 0.041619,
3129
+ -0.015547,
3130
+ 0.052298,
3131
+ -0.078327,
3132
+ -0.052303,
3133
+ 0.043099,
3134
+ 0.05375,
3135
+ -0.041854,
3136
+ -0.038314,
3137
+ 0.039399,
3138
+ -0.047769,
3139
+ -0.038469,
3140
+ -0.056793,
3141
+ -0.064312,
3142
+ -0.047884,
3143
+ -0.060067,
3144
+ -0.030687,
3145
+ 0.030053,
3146
+ 0.033569,
3147
+ -0.038014,
3148
+ -0.032396,
3149
+ -0.041395,
3150
+ 0.038163,
3151
+ -0.056206,
3152
+ 0.047115,
3153
+ -0.038099,
3154
+ -0.031927,
3155
+ 0.033857,
3156
+ -0.011227,
3157
+ 0.032883,
3158
+ 0.032516,
3159
+ -0.045622,
3160
+ -0.044387,
3161
+ -0.034374,
3162
+ -0.030433,
3163
+ 0.055097,
3164
+ 0.055568,
3165
+ 0.058726,
3166
+ -0.043569,
3167
+ 0.031854,
3168
+ 775e-5,
3169
+ -0.022533,
3170
+ 0.096133,
3171
+ 0.052876,
3172
+ 0.045582,
3173
+ 0.041868,
3174
+ -0.039268,
3175
+ -0.037865,
3176
+ -0.054658,
3177
+ -0.031297,
3178
+ -0.058104,
3179
+ 0.018705,
3180
+ 0.031934,
3181
+ -0.052746,
3182
+ -0.044438,
3183
+ -0.029991,
3184
+ 0.011917,
3185
+ 0.070103,
3186
+ -0.046682,
3187
+ -0.048965,
3188
+ 0.046255,
3189
+ 0.027877,
3190
+ 0.0475,
3191
+ 0.044287,
3192
+ 0.04251,
3193
+ -0.017647,
3194
+ -0.017812,
3195
+ 0.055672,
3196
+ -0.042207,
3197
+ -0.035516,
3198
+ -0.025558,
3199
+ -0.062443,
3200
+ 0.037519,
3201
+ -0.031697,
3202
+ 0.064998,
3203
+ 0.045949,
3204
+ 302e-6,
3205
+ 0.065205,
3206
+ -0.052016,
3207
+ 0.036721,
3208
+ -0.034769,
3209
+ -0.049074,
3210
+ 0.040413,
3211
+ 0.045425,
3212
+ -0.068063,
3213
+ 0.021485,
3214
+ 8549e-6,
3215
+ 0.028179,
3216
+ 0.026914,
3217
+ 0.020493,
3218
+ 0.056155,
3219
+ 0.047202,
3220
+ -0.049085,
3221
+ -0.022624,
3222
+ 0.063734,
3223
+ 0.05471,
3224
+ -0.032258,
3225
+ 0.047761,
3226
+ -0.065033,
3227
+ -0.047818,
3228
+ -0.034254,
3229
+ -0.062863,
3230
+ -0.01362,
3231
+ -0.053904,
3232
+ 0.024415,
3233
+ 0.044367,
3234
+ -0.064056,
3235
+ -0.038419,
3236
+ 0.038997,
3237
+ 1015e-6,
3238
+ 0.038928,
3239
+ -0.045794,
3240
+ -0.038313,
3241
+ 0.043922,
3242
+ -0.05873,
3243
+ -0.020314,
3244
+ -0.020457,
3245
+ 0.045522,
3246
+ -0.057731,
3247
+ -0.062966,
3248
+ 0.023159,
3249
+ 0.020283,
3250
+ -0.057352,
3251
+ 0.044701,
3252
+ -0.03585,
3253
+ -0.052798,
3254
+ 0.056931,
3255
+ -0.035609,
3256
+ -0.035113,
3257
+ 0.026919,
3258
+ 0.059404,
3259
+ -0.084434,
3260
+ 0.014558,
3261
+ 0.06226,
3262
+ -0.044173,
3263
+ 0.044172,
3264
+ -0.046482,
3265
+ -0.034933,
3266
+ 0.052513,
3267
+ 0.044018,
3268
+ -0.057677,
3269
+ -0.033961,
3270
+ 0.046334,
3271
+ 0.065767,
3272
+ 0.044779,
3273
+ 0.063658,
3274
+ -0.022574,
3275
+ -4821e-6,
3276
+ 0.037388,
3277
+ -0.043197,
3278
+ 0.063,
3279
+ 0.048943,
3280
+ -0.049495,
3281
+ 0.014798,
3282
+ -0.018176,
3283
+ -0.03824,
3284
+ -0.045486,
3285
+ 0.053828,
3286
+ -0.056118,
3287
+ -0.067025,
3288
+ 0.027818,
3289
+ 0.050305,
3290
+ 0.037134,
3291
+ 0.049673
3292
+ ];
3293
+
2790
3294
  // src/profiles/registry.ts
2791
3295
  var SHARED_ENCODING = {
2792
3296
  pooling: "mean",
@@ -2817,6 +3321,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
2817
3321
  passageInstruction: "passage: ",
2818
3322
  ...SHARED_ENCODING
2819
3323
  };
3324
+ var MULTILINGUAL_E5_SMALL_2 = {
3325
+ ...MULTILINGUAL_E5_SMALL_1,
3326
+ id: "multilingual-e5-small@2",
3327
+ quantization: {
3328
+ int8: { method: "symmetric-linear", scale: 127 },
3329
+ binary: {
3330
+ method: "centered-sign",
3331
+ threshold: 0,
3332
+ packing: "lsb-first",
3333
+ center: [...MULTILINGUAL_E5_SMALL_CENTER]
3334
+ }
3335
+ }
3336
+ };
2820
3337
  var BGE_SMALL_EN_V15_1 = {
2821
3338
  id: "bge-small-en-v1.5@1",
2822
3339
  model: {
@@ -2835,6 +3352,7 @@ var BGE_SMALL_EN_V15_1 = {
2835
3352
  ...SHARED_ENCODING
2836
3353
  };
2837
3354
  var KNOWLEDGE_EMBEDDING_PROFILES = [
3355
+ MULTILINGUAL_E5_SMALL_2,
2838
3356
  MULTILINGUAL_E5_SMALL_1,
2839
3357
  BGE_SMALL_EN_V15_1
2840
3358
  ];
@@ -4122,9 +4640,12 @@ export {
4122
4640
  MAX_KNOWLEDGE_TOPIC_DEPTH,
4123
4641
  MIMETYPE_PATH,
4124
4642
  MULTILINGUAL_E5_SMALL_1,
4643
+ MULTILINGUAL_E5_SMALL_2,
4125
4644
  PARQUET_EXPORT_VERSION,
4126
4645
  PARQUET_REPORT_PATH,
4127
4646
  README_PATH,
4647
+ RERANK_CANDIDATES_MAX,
4648
+ RERANK_CANDIDATES_MIN,
4128
4649
  RERANK_FINAL_K,
4129
4650
  ROUTER_DB_PATH,
4130
4651
  ROUTER_DDL,
@@ -4138,7 +4659,9 @@ export {
4138
4659
  assertDuckdbCli,
4139
4660
  assetReferences,
4140
4661
  assignOutlineOrders,
4662
+ asymmetricTopK,
4141
4663
  canonicalizeJson2 as canonicalizeJson,
4664
+ centerVector,
4142
4665
  chunkContentHash,
4143
4666
  chunkMarkdownProfile,
4144
4667
  chunkUid,
@@ -4170,6 +4693,7 @@ export {
4170
4693
  parseJupyterBookToc,
4171
4694
  parseMkdocsNav,
4172
4695
  quantizeBinary,
4696
+ quantizeBinaryForProfile,
4173
4697
  quantizeInt8,
4174
4698
  readGezkManifest,
4175
4699
  readMkdocsDocsDir,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.1.1",
3
+ "version": "1.1.2",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [
@@ -47,7 +47,7 @@
47
47
  "!dist/**/*.map"
48
48
  ],
49
49
  "dependencies": {
50
- "@bendyline/gezk": "1.0.1",
50
+ "@bendyline/gezk": "1.0.2",
51
51
  "yaml": "^2.9.0",
52
52
  "yauzl": "^3.4.0",
53
53
  "yazl": "^3.3.1",