@bendyline/gezel-knowledge 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/index.d.ts +86 -14
- package/dist/index.js +530 -6
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -81,8 +81,9 @@ Extraction verifies every file against the manifest (both directions —
|
|
|
81
81
|
undeclared and missing files are equally fatal) and rejects unsafe archive
|
|
82
82
|
entries. Catalog databases are opened strictly read-only and immutable.
|
|
83
83
|
Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
|
|
84
|
-
full 200,000-chunk shard) and
|
|
85
|
-
|
|
84
|
+
full 200,000-chunk shard) and scores the float query against them through a
|
|
85
|
+
per-byte lookup table (the query is centered first when the catalog's profile
|
|
86
|
+
is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
|
|
86
87
|
|
|
87
88
|
## Stability
|
|
88
89
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
-
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
2
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
3
3
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
4
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
5
5
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
|
|
|
16
16
|
declare const CENTROID_SAMPLE_MAX = 65536;
|
|
17
17
|
declare const ROUTE_SHARDS_PROACTIVE = 3;
|
|
18
18
|
declare const ROUTE_SHARDS_EXPLICIT = 6;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
|
|
21
|
+
* candidates. K is deliberately generous. Measured on a 43,859-chunk
|
|
22
|
+
* multilingual-e5-small catalog with natural-language questions, recall of
|
|
23
|
+
* the exact top-24 among the stage-1 candidates was 22% at the former
|
|
24
|
+
* K = 192 with raw sign bits, and with centered bits (`centered-sign`
|
|
25
|
+
* profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
|
|
26
|
+
* reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
|
|
27
|
+
* shard. A small K is a silent recall cliff; a large one is cheap.
|
|
28
|
+
*/
|
|
20
29
|
declare const RERANK_FINAL_K = 24;
|
|
30
|
+
declare const RERANK_CANDIDATES_MIN = 1024;
|
|
31
|
+
declare const RERANK_CANDIDATES_MAX = 4096;
|
|
21
32
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
22
33
|
|
|
23
34
|
/**
|
|
@@ -309,12 +320,16 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
|
309
320
|
* One mounted catalog: the router database plus lazily opened shards, and
|
|
310
321
|
* the two-stage query flow (the gezk spec §8 and §9):
|
|
311
322
|
*
|
|
312
|
-
* embed query (caller) → centroid routing → per shard: bit
|
|
323
|
+
* embed query (caller) → centroid routing → per shard: sign-bit scan
|
|
313
324
|
* top-K → int8 rerank → fuse.
|
|
314
325
|
*
|
|
315
326
|
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
316
|
-
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
317
|
-
*
|
|
327
|
+
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
328
|
+
* the float query (`asymmetricTopK`), so no vector extension is needed to
|
|
329
|
+
* read a catalog. The catalog's own profile echo says how its bits were
|
|
330
|
+
* derived: a `centered-sign` profile's bits come from `vector − center`, so
|
|
331
|
+
* the query is centered the same way before the scan. The int8 rerank is
|
|
332
|
+
* identical for both kinds of bits.
|
|
318
333
|
*
|
|
319
334
|
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
320
335
|
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
@@ -383,6 +398,8 @@ declare class CatalogHandle {
|
|
|
383
398
|
private readonly connections;
|
|
384
399
|
private readonly bitIndexes;
|
|
385
400
|
private assetsByPath;
|
|
401
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
402
|
+
private binaryCenterCache;
|
|
386
403
|
readonly meta: Record<string, string>;
|
|
387
404
|
readonly shards: ShardInfo[];
|
|
388
405
|
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
@@ -406,6 +423,13 @@ declare class CatalogHandle {
|
|
|
406
423
|
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
407
424
|
*/
|
|
408
425
|
private shardBits;
|
|
426
|
+
/**
|
|
427
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
428
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
429
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
430
|
+
* the statement of how THIS catalog was built.
|
|
431
|
+
*/
|
|
432
|
+
private binaryCenter;
|
|
409
433
|
/** The embedding dimension from the router's profile echo. */
|
|
410
434
|
private dimensions;
|
|
411
435
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
@@ -489,7 +513,7 @@ declare class CatalogHandle {
|
|
|
489
513
|
/** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
|
|
490
514
|
routeShards(queryVector: Float32Array, shardBudget: number): number[];
|
|
491
515
|
/**
|
|
492
|
-
* Two-stage semantic search over the routed shards: bit
|
|
516
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
493
517
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
494
518
|
*/
|
|
495
519
|
searchSemantic(queryVector: Float32Array, opts?: {
|
|
@@ -540,10 +564,21 @@ declare function assetReferences(markdown: string): string[];
|
|
|
540
564
|
|
|
541
565
|
/**
|
|
542
566
|
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
543
|
-
* 200k-chunk shard is 9.6 MB of bits
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
567
|
+
* 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
|
|
568
|
+
* pass over them, a few milliseconds, needing nothing beyond plain SQLite
|
|
569
|
+
* to load — which is what lets any reader implement the format without a
|
|
570
|
+
* vector extension.
|
|
571
|
+
*
|
|
572
|
+
* Two scans share the bits:
|
|
573
|
+
* - `hammingTopK` — symmetric: the query is binarized too and rows are
|
|
574
|
+
* ranked by popcount(xor). The format's baseline (§2), and what the
|
|
575
|
+
* self-KNN smoke and the conformance kit exercise.
|
|
576
|
+
* - `asymmetricTopK` — the float query against each row's bits, ranked by
|
|
577
|
+
* Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
|
|
578
|
+
* embeddings roughly doubles the share of true neighbours a small
|
|
579
|
+
* candidate pool retains (measured on multilingual-e5-small: 22% → 39%
|
|
580
|
+
* of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
|
|
581
|
+
* bits). This is what `CatalogHandle` uses.
|
|
547
582
|
*/
|
|
548
583
|
interface ShardBitIndex {
|
|
549
584
|
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
@@ -561,6 +596,23 @@ interface HammingHit {
|
|
|
561
596
|
* the whole query is two linear passes and no heap.
|
|
562
597
|
*/
|
|
563
598
|
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
599
|
+
interface AsymmetricHit {
|
|
600
|
+
chunkId: number;
|
|
601
|
+
score: number;
|
|
602
|
+
}
|
|
603
|
+
/**
|
|
604
|
+
* The K best rows by asymmetric score, descending, ties broken by chunk id.
|
|
605
|
+
*
|
|
606
|
+
* The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
|
|
607
|
+
* of 256 partial sums is built for each byte position, so a row costs
|
|
608
|
+
* `bytesPerRow` table reads and adds — about the popcount scan's cost. The
|
|
609
|
+
* top-K is kept in a bounded min-heap whose root is the worst kept row.
|
|
610
|
+
*
|
|
611
|
+
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
612
|
+
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
613
|
+
* stored; this function does not know or care which.
|
|
614
|
+
*/
|
|
615
|
+
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
|
|
564
616
|
|
|
565
617
|
/**
|
|
566
618
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
@@ -678,10 +730,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
|
|
|
678
730
|
* built from a profile verifies the files it loaded against these digests.
|
|
679
731
|
*/
|
|
680
732
|
|
|
681
|
-
/**
|
|
733
|
+
/**
|
|
734
|
+
* The public profile's first revision: multilingual, so world catalogs serve
|
|
735
|
+
* every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
|
|
736
|
+
* vectors, centered stage-1 bits); catalogs built with this revision stay
|
|
737
|
+
* readable and searchable, at the lower stage-1 recall of raw sign bits.
|
|
738
|
+
*/
|
|
682
739
|
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
740
|
+
/**
|
|
741
|
+
* The public profile, revision 2: the model, files, instructions and int8
|
|
742
|
+
* rerank of revision 1, with the stage-1 sign bits taken from
|
|
743
|
+
* `vector − center` (`centered-sign`). e5-small vectors share one dominant
|
|
744
|
+
* direction, so raw sign bits mostly encode that direction rather than the
|
|
745
|
+
* passage and the hamming pre-filter loses most true neighbours; centering
|
|
746
|
+
* on the pinned corpus mean restores them (see e5-small-center.ts for the
|
|
747
|
+
* provenance and the measurements). A query vector for revision 1 is a
|
|
748
|
+
* valid query vector for revision 2 — `sameVectorSpace` says so — and the
|
|
749
|
+
* reader centers it itself from the catalog's profile echo.
|
|
750
|
+
*/
|
|
751
|
+
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
683
752
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
684
753
|
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
754
|
+
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
685
755
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
686
756
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
687
757
|
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
@@ -717,9 +787,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
717
787
|
scale: 127;
|
|
718
788
|
};
|
|
719
789
|
binary: {
|
|
720
|
-
method: "sign";
|
|
790
|
+
method: "sign" | "centered-sign";
|
|
721
791
|
threshold: 0;
|
|
722
792
|
packing: "lsb-first";
|
|
793
|
+
center?: number[] | undefined;
|
|
723
794
|
};
|
|
724
795
|
};
|
|
725
796
|
};
|
|
@@ -754,9 +825,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
754
825
|
scale: 127;
|
|
755
826
|
};
|
|
756
827
|
binary: {
|
|
757
|
-
method: "sign";
|
|
828
|
+
method: "sign" | "centered-sign";
|
|
758
829
|
threshold: 0;
|
|
759
830
|
packing: "lsb-first";
|
|
831
|
+
center?: number[] | undefined;
|
|
760
832
|
};
|
|
761
833
|
};
|
|
762
834
|
};
|
|
@@ -1150,4 +1222,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1150
1222
|
|
|
1151
1223
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1152
1224
|
|
|
1153
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1225
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -30,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
|
|
|
30
30
|
var ROUTE_SHARDS_PROACTIVE = 3;
|
|
31
31
|
var ROUTE_SHARDS_EXPLICIT = 6;
|
|
32
32
|
var RERANK_FINAL_K = 24;
|
|
33
|
+
var RERANK_CANDIDATES_MIN = 1024;
|
|
34
|
+
var RERANK_CANDIDATES_MAX = 4096;
|
|
33
35
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
34
|
-
return Math.min(
|
|
36
|
+
return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
// src/format/ddl.ts
|
|
@@ -42,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
|
|
|
42
44
|
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
43
45
|
|
|
44
46
|
// src/format/quantize.ts
|
|
45
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
centerVector,
|
|
49
|
+
l2Normalize,
|
|
50
|
+
quantizeBinary,
|
|
51
|
+
quantizeBinaryForProfile,
|
|
52
|
+
quantizeInt8,
|
|
53
|
+
rerankScore
|
|
54
|
+
} from "@bendyline/gezk";
|
|
46
55
|
|
|
47
56
|
// src/chunking/markdown-chunker.ts
|
|
48
57
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -469,7 +478,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
469
478
|
}
|
|
470
479
|
const row = pendingRows[i];
|
|
471
480
|
const stmts = vecStmtsFor(shardDb(row.shardId));
|
|
472
|
-
stmts.vec.run(BigInt(row.chunkId), Buffer.from(
|
|
481
|
+
stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
|
|
473
482
|
stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
|
|
474
483
|
const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
|
|
475
484
|
void shardTotal;
|
|
@@ -1403,8 +1412,10 @@ import { readFileSync as readFileSync2 } from "fs";
|
|
|
1403
1412
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1404
1413
|
import { brotliDecompressSync } from "zlib";
|
|
1405
1414
|
import {
|
|
1415
|
+
KnowledgeEmbeddingProfileSchema,
|
|
1406
1416
|
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
|
|
1407
1417
|
assetContentType,
|
|
1418
|
+
embeddingProfileCenter,
|
|
1408
1419
|
isKnowledgeAssetPath
|
|
1409
1420
|
} from "@bendyline/gezk";
|
|
1410
1421
|
|
|
@@ -1478,6 +1489,86 @@ function hammingTopK(index, query, k) {
|
|
|
1478
1489
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1479
1490
|
return hits;
|
|
1480
1491
|
}
|
|
1492
|
+
function asymmetricTopK(index, query, k) {
|
|
1493
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1494
|
+
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1495
|
+
throw new Error(
|
|
1496
|
+
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1497
|
+
);
|
|
1498
|
+
}
|
|
1499
|
+
const limit = Math.min(k, rows);
|
|
1500
|
+
if (limit <= 0) return [];
|
|
1501
|
+
const lut = new Float32Array(bytesPerRow * 256);
|
|
1502
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1503
|
+
for (let b = 0; b < 256; b++) {
|
|
1504
|
+
let s = 0;
|
|
1505
|
+
for (let i = 0; i < 8; i++) {
|
|
1506
|
+
const d = p * 8 + i;
|
|
1507
|
+
if (d >= query.length) break;
|
|
1508
|
+
const v = query[d];
|
|
1509
|
+
s += b >> i & 1 ? v : -v;
|
|
1510
|
+
}
|
|
1511
|
+
lut[p * 256 + b] = s;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
const heapScore = new Float64Array(limit);
|
|
1515
|
+
const heapId = new Int32Array(limit);
|
|
1516
|
+
let size = 0;
|
|
1517
|
+
const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
|
|
1518
|
+
const swap = (i, j) => {
|
|
1519
|
+
const s = heapScore[i];
|
|
1520
|
+
const id = heapId[i];
|
|
1521
|
+
heapScore[i] = heapScore[j];
|
|
1522
|
+
heapId[i] = heapId[j];
|
|
1523
|
+
heapScore[j] = s;
|
|
1524
|
+
heapId[j] = id;
|
|
1525
|
+
};
|
|
1526
|
+
const siftUp = (start) => {
|
|
1527
|
+
let i = start;
|
|
1528
|
+
while (i > 0) {
|
|
1529
|
+
const parent = i - 1 >> 1;
|
|
1530
|
+
if (!worse(i, parent)) break;
|
|
1531
|
+
swap(i, parent);
|
|
1532
|
+
i = parent;
|
|
1533
|
+
}
|
|
1534
|
+
};
|
|
1535
|
+
const siftDown = (start) => {
|
|
1536
|
+
let i = start;
|
|
1537
|
+
for (; ; ) {
|
|
1538
|
+
const left = 2 * i + 1;
|
|
1539
|
+
if (left >= size) break;
|
|
1540
|
+
const right = left + 1;
|
|
1541
|
+
const child = right < size && worse(right, left) ? right : left;
|
|
1542
|
+
if (!worse(child, i)) break;
|
|
1543
|
+
swap(child, i);
|
|
1544
|
+
i = child;
|
|
1545
|
+
}
|
|
1546
|
+
};
|
|
1547
|
+
let offset = 0;
|
|
1548
|
+
for (let r = 0; r < rows; r++) {
|
|
1549
|
+
let score = 0;
|
|
1550
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1551
|
+
score += lut[p * 256 + bits[offset + p]];
|
|
1552
|
+
}
|
|
1553
|
+
offset += bytesPerRow;
|
|
1554
|
+
if (size < limit) {
|
|
1555
|
+
heapScore[size] = score;
|
|
1556
|
+
heapId[size] = r + 1;
|
|
1557
|
+
size++;
|
|
1558
|
+
siftUp(size - 1);
|
|
1559
|
+
} else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
|
|
1560
|
+
heapScore[0] = score;
|
|
1561
|
+
heapId[0] = r + 1;
|
|
1562
|
+
siftDown(0);
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
const hits = [];
|
|
1566
|
+
for (let i = 0; i < size; i++) {
|
|
1567
|
+
hits.push({ chunkId: heapId[i], score: heapScore[i] });
|
|
1568
|
+
}
|
|
1569
|
+
hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
|
|
1570
|
+
return hits;
|
|
1571
|
+
}
|
|
1481
1572
|
|
|
1482
1573
|
// src/reader/catalog-handle.ts
|
|
1483
1574
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
@@ -1499,6 +1590,8 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1499
1590
|
connections = /* @__PURE__ */ new Map();
|
|
1500
1591
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
1501
1592
|
assetsByPath = null;
|
|
1593
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1594
|
+
binaryCenterCache;
|
|
1502
1595
|
meta;
|
|
1503
1596
|
shards;
|
|
1504
1597
|
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
@@ -1581,6 +1674,28 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1581
1674
|
this.bitIndexes.set(shard.path, index);
|
|
1582
1675
|
return index;
|
|
1583
1676
|
}
|
|
1677
|
+
/**
|
|
1678
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
1679
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
1680
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
1681
|
+
* the statement of how THIS catalog was built.
|
|
1682
|
+
*/
|
|
1683
|
+
binaryCenter() {
|
|
1684
|
+
if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
|
|
1685
|
+
const raw = this.meta.embedding_profile_json;
|
|
1686
|
+
let parsed = null;
|
|
1687
|
+
try {
|
|
1688
|
+
parsed = raw ? JSON.parse(raw) : null;
|
|
1689
|
+
} catch {
|
|
1690
|
+
parsed = null;
|
|
1691
|
+
}
|
|
1692
|
+
const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
|
|
1693
|
+
if (!profile.success) {
|
|
1694
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1695
|
+
}
|
|
1696
|
+
this.binaryCenterCache = embeddingProfileCenter(profile.data);
|
|
1697
|
+
return this.binaryCenterCache;
|
|
1698
|
+
}
|
|
1584
1699
|
/** The embedding dimension from the router's profile echo. */
|
|
1585
1700
|
dimensions() {
|
|
1586
1701
|
const raw = this.meta.embedding_profile_json;
|
|
@@ -1863,7 +1978,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1863
1978
|
return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
1864
1979
|
}
|
|
1865
1980
|
/**
|
|
1866
|
-
* Two-stage semantic search over the routed shards: bit
|
|
1981
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
1867
1982
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
1868
1983
|
*/
|
|
1869
1984
|
searchSemantic(queryVector, opts = {}) {
|
|
@@ -1875,14 +1990,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1875
1990
|
* after global cross-catalog routing has already spent the S budget.
|
|
1876
1991
|
*/
|
|
1877
1992
|
searchShards(queryVector, shardIds, finalK) {
|
|
1878
|
-
const
|
|
1993
|
+
const center = this.binaryCenter();
|
|
1994
|
+
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
1879
1995
|
const hits = [];
|
|
1880
1996
|
for (const shardId of shardIds) {
|
|
1881
1997
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1882
1998
|
if (!shard) continue;
|
|
1883
1999
|
const db = this.shardDb(shard);
|
|
1884
2000
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1885
|
-
const candidates =
|
|
2001
|
+
const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
|
|
1886
2002
|
chunk_id: hit.chunkId
|
|
1887
2003
|
}));
|
|
1888
2004
|
if (candidates.length === 0) continue;
|
|
@@ -2787,6 +2903,394 @@ async function findDuckdbBinary(env = process.env) {
|
|
|
2787
2903
|
return null;
|
|
2788
2904
|
}
|
|
2789
2905
|
|
|
2906
|
+
// src/profiles/e5-small-center.ts
|
|
2907
|
+
var MULTILINGUAL_E5_SMALL_CENTER = [
|
|
2908
|
+
0.046697,
|
|
2909
|
+
-0.013211,
|
|
2910
|
+
-0.042276,
|
|
2911
|
+
-0.049082,
|
|
2912
|
+
0.060627,
|
|
2913
|
+
-0.032643,
|
|
2914
|
+
0.029874,
|
|
2915
|
+
0.033307,
|
|
2916
|
+
0.058447,
|
|
2917
|
+
-0.012016,
|
|
2918
|
+
0.046472,
|
|
2919
|
+
5991e-6,
|
|
2920
|
+
0.04884,
|
|
2921
|
+
-0.028836,
|
|
2922
|
+
-0.03244,
|
|
2923
|
+
0.045633,
|
|
2924
|
+
0.056798,
|
|
2925
|
+
-0.043557,
|
|
2926
|
+
-0.034472,
|
|
2927
|
+
-0.05355,
|
|
2928
|
+
0.026363,
|
|
2929
|
+
-0.010318,
|
|
2930
|
+
-0.030487,
|
|
2931
|
+
0.047083,
|
|
2932
|
+
0.057205,
|
|
2933
|
+
0.035654,
|
|
2934
|
+
-0.017461,
|
|
2935
|
+
0.010881,
|
|
2936
|
+
0.047985,
|
|
2937
|
+
-0.048514,
|
|
2938
|
+
-0.048956,
|
|
2939
|
+
-0.027643,
|
|
2940
|
+
0.053259,
|
|
2941
|
+
-0.051057,
|
|
2942
|
+
0.045055,
|
|
2943
|
+
0.017888,
|
|
2944
|
+
-0.055755,
|
|
2945
|
+
-0.046578,
|
|
2946
|
+
0.05214,
|
|
2947
|
+
-0.037085,
|
|
2948
|
+
-0.016268,
|
|
2949
|
+
0.027508,
|
|
2950
|
+
0.035793,
|
|
2951
|
+
0.052082,
|
|
2952
|
+
0.034885,
|
|
2953
|
+
0.047174,
|
|
2954
|
+
-0.036778,
|
|
2955
|
+
0.031394,
|
|
2956
|
+
-9156e-6,
|
|
2957
|
+
-0.02486,
|
|
2958
|
+
-0.048897,
|
|
2959
|
+
0.052576,
|
|
2960
|
+
0.017289,
|
|
2961
|
+
0.047794,
|
|
2962
|
+
0.051939,
|
|
2963
|
+
-0.051261,
|
|
2964
|
+
-0.061345,
|
|
2965
|
+
-0.056267,
|
|
2966
|
+
-0.049305,
|
|
2967
|
+
0.047241,
|
|
2968
|
+
0.043763,
|
|
2969
|
+
-0.04506,
|
|
2970
|
+
0.015752,
|
|
2971
|
+
0.017543,
|
|
2972
|
+
0.067856,
|
|
2973
|
+
0.065763,
|
|
2974
|
+
0.037218,
|
|
2975
|
+
0.044877,
|
|
2976
|
+
-0.057149,
|
|
2977
|
+
-0.028072,
|
|
2978
|
+
-0.060566,
|
|
2979
|
+
0.064004,
|
|
2980
|
+
6095e-6,
|
|
2981
|
+
-0.039693,
|
|
2982
|
+
0.021968,
|
|
2983
|
+
0.02654,
|
|
2984
|
+
0.025546,
|
|
2985
|
+
-0.048954,
|
|
2986
|
+
0.026967,
|
|
2987
|
+
-0.045002,
|
|
2988
|
+
-0.055556,
|
|
2989
|
+
-0.034963,
|
|
2990
|
+
-0.03466,
|
|
2991
|
+
0.032486,
|
|
2992
|
+
-0.049597,
|
|
2993
|
+
0.058718,
|
|
2994
|
+
0.048021,
|
|
2995
|
+
-0.066783,
|
|
2996
|
+
0.059049,
|
|
2997
|
+
-0.04665,
|
|
2998
|
+
0.052902,
|
|
2999
|
+
0.031438,
|
|
3000
|
+
-0.061515,
|
|
3001
|
+
-0.05342,
|
|
3002
|
+
-0.068253,
|
|
3003
|
+
-0.06666,
|
|
3004
|
+
-0.04553,
|
|
3005
|
+
0.059252,
|
|
3006
|
+
0.050728,
|
|
3007
|
+
-0.030429,
|
|
3008
|
+
0.067182,
|
|
3009
|
+
-0.038819,
|
|
3010
|
+
0.049358,
|
|
3011
|
+
-0.055247,
|
|
3012
|
+
-0.032933,
|
|
3013
|
+
0.041239,
|
|
3014
|
+
0.035164,
|
|
3015
|
+
-0.066574,
|
|
3016
|
+
0.05348,
|
|
3017
|
+
-0.052658,
|
|
3018
|
+
-0.034758,
|
|
3019
|
+
0.037219,
|
|
3020
|
+
0.064111,
|
|
3021
|
+
0.047861,
|
|
3022
|
+
-0.04608,
|
|
3023
|
+
-0.041338,
|
|
3024
|
+
-0.016072,
|
|
3025
|
+
-0.046026,
|
|
3026
|
+
0.0255,
|
|
3027
|
+
-0.041581,
|
|
3028
|
+
0.059684,
|
|
3029
|
+
-0.019611,
|
|
3030
|
+
-0.050363,
|
|
3031
|
+
-0.061268,
|
|
3032
|
+
-0.057556,
|
|
3033
|
+
-0.03184,
|
|
3034
|
+
0.018522,
|
|
3035
|
+
0.046887,
|
|
3036
|
+
0.054774,
|
|
3037
|
+
0.012536,
|
|
3038
|
+
0.033019,
|
|
3039
|
+
0.033211,
|
|
3040
|
+
0.029609,
|
|
3041
|
+
0.050215,
|
|
3042
|
+
0.035425,
|
|
3043
|
+
0.076419,
|
|
3044
|
+
-0.055196,
|
|
3045
|
+
-0.010016,
|
|
3046
|
+
-936e-6,
|
|
3047
|
+
-0.023764,
|
|
3048
|
+
-0.059648,
|
|
3049
|
+
0.054857,
|
|
3050
|
+
-0.030456,
|
|
3051
|
+
0.041281,
|
|
3052
|
+
0.063894,
|
|
3053
|
+
0.035466,
|
|
3054
|
+
0.069754,
|
|
3055
|
+
-0.026593,
|
|
3056
|
+
0.066305,
|
|
3057
|
+
-0.028609,
|
|
3058
|
+
0.05324,
|
|
3059
|
+
-0.022879,
|
|
3060
|
+
0.04882,
|
|
3061
|
+
0.032385,
|
|
3062
|
+
0.042921,
|
|
3063
|
+
-0.043319,
|
|
3064
|
+
-0.065022,
|
|
3065
|
+
-0.044018,
|
|
3066
|
+
0.04237,
|
|
3067
|
+
0.051018,
|
|
3068
|
+
-0.048907,
|
|
3069
|
+
-0.053045,
|
|
3070
|
+
-0.057301,
|
|
3071
|
+
-0.019731,
|
|
3072
|
+
-0.060812,
|
|
3073
|
+
-0.043114,
|
|
3074
|
+
0.021207,
|
|
3075
|
+
0.052498,
|
|
3076
|
+
-0.038307,
|
|
3077
|
+
-0.037082,
|
|
3078
|
+
-0.043732,
|
|
3079
|
+
0.045959,
|
|
3080
|
+
-0.044296,
|
|
3081
|
+
0.057199,
|
|
3082
|
+
-0.011835,
|
|
3083
|
+
0.054221,
|
|
3084
|
+
-0.046354,
|
|
3085
|
+
0.043442,
|
|
3086
|
+
0.063337,
|
|
3087
|
+
0.041964,
|
|
3088
|
+
-0.032836,
|
|
3089
|
+
-0.020089,
|
|
3090
|
+
-0.056327,
|
|
3091
|
+
-0.05089,
|
|
3092
|
+
-0.053933,
|
|
3093
|
+
-0.035486,
|
|
3094
|
+
-0.046215,
|
|
3095
|
+
0.025754,
|
|
3096
|
+
0.032986,
|
|
3097
|
+
-0.025048,
|
|
3098
|
+
-9903e-6,
|
|
3099
|
+
0.031162,
|
|
3100
|
+
-0.050969,
|
|
3101
|
+
-0.049401,
|
|
3102
|
+
-0.046206,
|
|
3103
|
+
0.023197,
|
|
3104
|
+
-0.033441,
|
|
3105
|
+
0.045075,
|
|
3106
|
+
0.043318,
|
|
3107
|
+
0.043766,
|
|
3108
|
+
-6601e-6,
|
|
3109
|
+
-0.023077,
|
|
3110
|
+
0.033917,
|
|
3111
|
+
0.032279,
|
|
3112
|
+
0.034874,
|
|
3113
|
+
5185e-6,
|
|
3114
|
+
-0.057329,
|
|
3115
|
+
0.048437,
|
|
3116
|
+
-0.033964,
|
|
3117
|
+
0.044195,
|
|
3118
|
+
0.051107,
|
|
3119
|
+
-0.056642,
|
|
3120
|
+
-0.070856,
|
|
3121
|
+
0.045431,
|
|
3122
|
+
-0.047205,
|
|
3123
|
+
-0.021457,
|
|
3124
|
+
3103e-6,
|
|
3125
|
+
0.062536,
|
|
3126
|
+
-0.037804,
|
|
3127
|
+
0.031278,
|
|
3128
|
+
0.041619,
|
|
3129
|
+
-0.015547,
|
|
3130
|
+
0.052298,
|
|
3131
|
+
-0.078327,
|
|
3132
|
+
-0.052303,
|
|
3133
|
+
0.043099,
|
|
3134
|
+
0.05375,
|
|
3135
|
+
-0.041854,
|
|
3136
|
+
-0.038314,
|
|
3137
|
+
0.039399,
|
|
3138
|
+
-0.047769,
|
|
3139
|
+
-0.038469,
|
|
3140
|
+
-0.056793,
|
|
3141
|
+
-0.064312,
|
|
3142
|
+
-0.047884,
|
|
3143
|
+
-0.060067,
|
|
3144
|
+
-0.030687,
|
|
3145
|
+
0.030053,
|
|
3146
|
+
0.033569,
|
|
3147
|
+
-0.038014,
|
|
3148
|
+
-0.032396,
|
|
3149
|
+
-0.041395,
|
|
3150
|
+
0.038163,
|
|
3151
|
+
-0.056206,
|
|
3152
|
+
0.047115,
|
|
3153
|
+
-0.038099,
|
|
3154
|
+
-0.031927,
|
|
3155
|
+
0.033857,
|
|
3156
|
+
-0.011227,
|
|
3157
|
+
0.032883,
|
|
3158
|
+
0.032516,
|
|
3159
|
+
-0.045622,
|
|
3160
|
+
-0.044387,
|
|
3161
|
+
-0.034374,
|
|
3162
|
+
-0.030433,
|
|
3163
|
+
0.055097,
|
|
3164
|
+
0.055568,
|
|
3165
|
+
0.058726,
|
|
3166
|
+
-0.043569,
|
|
3167
|
+
0.031854,
|
|
3168
|
+
775e-5,
|
|
3169
|
+
-0.022533,
|
|
3170
|
+
0.096133,
|
|
3171
|
+
0.052876,
|
|
3172
|
+
0.045582,
|
|
3173
|
+
0.041868,
|
|
3174
|
+
-0.039268,
|
|
3175
|
+
-0.037865,
|
|
3176
|
+
-0.054658,
|
|
3177
|
+
-0.031297,
|
|
3178
|
+
-0.058104,
|
|
3179
|
+
0.018705,
|
|
3180
|
+
0.031934,
|
|
3181
|
+
-0.052746,
|
|
3182
|
+
-0.044438,
|
|
3183
|
+
-0.029991,
|
|
3184
|
+
0.011917,
|
|
3185
|
+
0.070103,
|
|
3186
|
+
-0.046682,
|
|
3187
|
+
-0.048965,
|
|
3188
|
+
0.046255,
|
|
3189
|
+
0.027877,
|
|
3190
|
+
0.0475,
|
|
3191
|
+
0.044287,
|
|
3192
|
+
0.04251,
|
|
3193
|
+
-0.017647,
|
|
3194
|
+
-0.017812,
|
|
3195
|
+
0.055672,
|
|
3196
|
+
-0.042207,
|
|
3197
|
+
-0.035516,
|
|
3198
|
+
-0.025558,
|
|
3199
|
+
-0.062443,
|
|
3200
|
+
0.037519,
|
|
3201
|
+
-0.031697,
|
|
3202
|
+
0.064998,
|
|
3203
|
+
0.045949,
|
|
3204
|
+
302e-6,
|
|
3205
|
+
0.065205,
|
|
3206
|
+
-0.052016,
|
|
3207
|
+
0.036721,
|
|
3208
|
+
-0.034769,
|
|
3209
|
+
-0.049074,
|
|
3210
|
+
0.040413,
|
|
3211
|
+
0.045425,
|
|
3212
|
+
-0.068063,
|
|
3213
|
+
0.021485,
|
|
3214
|
+
8549e-6,
|
|
3215
|
+
0.028179,
|
|
3216
|
+
0.026914,
|
|
3217
|
+
0.020493,
|
|
3218
|
+
0.056155,
|
|
3219
|
+
0.047202,
|
|
3220
|
+
-0.049085,
|
|
3221
|
+
-0.022624,
|
|
3222
|
+
0.063734,
|
|
3223
|
+
0.05471,
|
|
3224
|
+
-0.032258,
|
|
3225
|
+
0.047761,
|
|
3226
|
+
-0.065033,
|
|
3227
|
+
-0.047818,
|
|
3228
|
+
-0.034254,
|
|
3229
|
+
-0.062863,
|
|
3230
|
+
-0.01362,
|
|
3231
|
+
-0.053904,
|
|
3232
|
+
0.024415,
|
|
3233
|
+
0.044367,
|
|
3234
|
+
-0.064056,
|
|
3235
|
+
-0.038419,
|
|
3236
|
+
0.038997,
|
|
3237
|
+
1015e-6,
|
|
3238
|
+
0.038928,
|
|
3239
|
+
-0.045794,
|
|
3240
|
+
-0.038313,
|
|
3241
|
+
0.043922,
|
|
3242
|
+
-0.05873,
|
|
3243
|
+
-0.020314,
|
|
3244
|
+
-0.020457,
|
|
3245
|
+
0.045522,
|
|
3246
|
+
-0.057731,
|
|
3247
|
+
-0.062966,
|
|
3248
|
+
0.023159,
|
|
3249
|
+
0.020283,
|
|
3250
|
+
-0.057352,
|
|
3251
|
+
0.044701,
|
|
3252
|
+
-0.03585,
|
|
3253
|
+
-0.052798,
|
|
3254
|
+
0.056931,
|
|
3255
|
+
-0.035609,
|
|
3256
|
+
-0.035113,
|
|
3257
|
+
0.026919,
|
|
3258
|
+
0.059404,
|
|
3259
|
+
-0.084434,
|
|
3260
|
+
0.014558,
|
|
3261
|
+
0.06226,
|
|
3262
|
+
-0.044173,
|
|
3263
|
+
0.044172,
|
|
3264
|
+
-0.046482,
|
|
3265
|
+
-0.034933,
|
|
3266
|
+
0.052513,
|
|
3267
|
+
0.044018,
|
|
3268
|
+
-0.057677,
|
|
3269
|
+
-0.033961,
|
|
3270
|
+
0.046334,
|
|
3271
|
+
0.065767,
|
|
3272
|
+
0.044779,
|
|
3273
|
+
0.063658,
|
|
3274
|
+
-0.022574,
|
|
3275
|
+
-4821e-6,
|
|
3276
|
+
0.037388,
|
|
3277
|
+
-0.043197,
|
|
3278
|
+
0.063,
|
|
3279
|
+
0.048943,
|
|
3280
|
+
-0.049495,
|
|
3281
|
+
0.014798,
|
|
3282
|
+
-0.018176,
|
|
3283
|
+
-0.03824,
|
|
3284
|
+
-0.045486,
|
|
3285
|
+
0.053828,
|
|
3286
|
+
-0.056118,
|
|
3287
|
+
-0.067025,
|
|
3288
|
+
0.027818,
|
|
3289
|
+
0.050305,
|
|
3290
|
+
0.037134,
|
|
3291
|
+
0.049673
|
|
3292
|
+
];
|
|
3293
|
+
|
|
2790
3294
|
// src/profiles/registry.ts
|
|
2791
3295
|
var SHARED_ENCODING = {
|
|
2792
3296
|
pooling: "mean",
|
|
@@ -2817,6 +3321,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
|
|
|
2817
3321
|
passageInstruction: "passage: ",
|
|
2818
3322
|
...SHARED_ENCODING
|
|
2819
3323
|
};
|
|
3324
|
+
var MULTILINGUAL_E5_SMALL_2 = {
|
|
3325
|
+
...MULTILINGUAL_E5_SMALL_1,
|
|
3326
|
+
id: "multilingual-e5-small@2",
|
|
3327
|
+
quantization: {
|
|
3328
|
+
int8: { method: "symmetric-linear", scale: 127 },
|
|
3329
|
+
binary: {
|
|
3330
|
+
method: "centered-sign",
|
|
3331
|
+
threshold: 0,
|
|
3332
|
+
packing: "lsb-first",
|
|
3333
|
+
center: [...MULTILINGUAL_E5_SMALL_CENTER]
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
};
|
|
2820
3337
|
var BGE_SMALL_EN_V15_1 = {
|
|
2821
3338
|
id: "bge-small-en-v1.5@1",
|
|
2822
3339
|
model: {
|
|
@@ -2835,6 +3352,7 @@ var BGE_SMALL_EN_V15_1 = {
|
|
|
2835
3352
|
...SHARED_ENCODING
|
|
2836
3353
|
};
|
|
2837
3354
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
3355
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
2838
3356
|
MULTILINGUAL_E5_SMALL_1,
|
|
2839
3357
|
BGE_SMALL_EN_V15_1
|
|
2840
3358
|
];
|
|
@@ -4122,9 +4640,12 @@ export {
|
|
|
4122
4640
|
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
4123
4641
|
MIMETYPE_PATH,
|
|
4124
4642
|
MULTILINGUAL_E5_SMALL_1,
|
|
4643
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
4125
4644
|
PARQUET_EXPORT_VERSION,
|
|
4126
4645
|
PARQUET_REPORT_PATH,
|
|
4127
4646
|
README_PATH,
|
|
4647
|
+
RERANK_CANDIDATES_MAX,
|
|
4648
|
+
RERANK_CANDIDATES_MIN,
|
|
4128
4649
|
RERANK_FINAL_K,
|
|
4129
4650
|
ROUTER_DB_PATH,
|
|
4130
4651
|
ROUTER_DDL,
|
|
@@ -4138,7 +4659,9 @@ export {
|
|
|
4138
4659
|
assertDuckdbCli,
|
|
4139
4660
|
assetReferences,
|
|
4140
4661
|
assignOutlineOrders,
|
|
4662
|
+
asymmetricTopK,
|
|
4141
4663
|
canonicalizeJson2 as canonicalizeJson,
|
|
4664
|
+
centerVector,
|
|
4142
4665
|
chunkContentHash,
|
|
4143
4666
|
chunkMarkdownProfile,
|
|
4144
4667
|
chunkUid,
|
|
@@ -4170,6 +4693,7 @@ export {
|
|
|
4170
4693
|
parseJupyterBookToc,
|
|
4171
4694
|
parseMkdocsNav,
|
|
4172
4695
|
quantizeBinary,
|
|
4696
|
+
quantizeBinaryForProfile,
|
|
4173
4697
|
quantizeInt8,
|
|
4174
4698
|
readGezkManifest,
|
|
4175
4699
|
readMkdocsDocsDir,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-knowledge",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.2",
|
|
4
4
|
"description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"!dist/**/*.map"
|
|
48
48
|
],
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@bendyline/gezk": "1.0.
|
|
50
|
+
"@bendyline/gezk": "1.0.2",
|
|
51
51
|
"yaml": "^2.9.0",
|
|
52
52
|
"yauzl": "^3.4.0",
|
|
53
53
|
"yazl": "^3.3.1",
|