@bendyline/gezel-knowledge 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/dist/index.d.ts +128 -16
- package/dist/index.js +587 -25
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -12,13 +12,15 @@ npm install @bendyline/gezel-knowledge
|
|
|
12
12
|
|
|
13
13
|
## What a `.gezk` is
|
|
14
14
|
|
|
15
|
-
An open format (
|
|
15
|
+
An open format (format version 0.6, preliminary until format 1.0; 0.5
|
|
16
|
+
archives still open): a ZIP whose first entry is
|
|
16
17
|
the stored `mimetype` magic (`application/vnd.gezk+zip`), then
|
|
17
18
|
`manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
|
|
18
19
|
directory, brotli bodies, routing centroids) and shard SQLite databases with
|
|
19
20
|
FTS5 and quantized vectors in plain BLOB tables. Nothing beyond stock SQLite
|
|
20
21
|
is needed to read one. The format definitions are the separate
|
|
21
|
-
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package
|
|
22
|
+
[`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package,
|
|
23
|
+
whose npm version is independent of the format version it writes;
|
|
22
24
|
the specification, JSON Schemas, conformance fixtures and a Python reference
|
|
23
25
|
reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
|
|
24
26
|
|
|
@@ -81,8 +83,9 @@ Extraction verifies every file against the manifest (both directions —
|
|
|
81
83
|
undeclared and missing files are equally fatal) and rejects unsafe archive
|
|
82
84
|
entries. Catalog databases are opened strictly read-only and immutable.
|
|
83
85
|
Semantic search loads a shard's sign-bit rows into memory once (9.6 MB for a
|
|
84
|
-
full 200,000-chunk shard) and
|
|
85
|
-
|
|
86
|
+
full 200,000-chunk shard) and scores the float query against them through a
|
|
87
|
+
per-byte lookup table (the query is centered first when the catalog's profile
|
|
88
|
+
is `centered-sign`); the int8 rerank reads the top 1,024 candidates by rowid.
|
|
86
89
|
|
|
87
90
|
## Stability
|
|
88
91
|
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CatalogDocument, KnowledgeEmbeddingProfile, KnowledgeChunkingProfile, KnowledgeCatalogManifest, GezkIndexSchemaVersion, KnowledgeRegistryIndex, KnowledgeRegistryEntry } from '@bendyline/gezk';
|
|
2
|
-
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
2
|
+
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore } from '@bendyline/gezk';
|
|
3
3
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
4
4
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
5
5
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
@@ -16,8 +16,19 @@ declare const CENTROID_KMEANS_MAX_ITER = 25;
|
|
|
16
16
|
declare const CENTROID_SAMPLE_MAX = 65536;
|
|
17
17
|
declare const ROUTE_SHARDS_PROACTIVE = 3;
|
|
18
18
|
declare const ROUTE_SHARDS_EXPLICIT = 6;
|
|
19
|
-
/**
|
|
19
|
+
/**
|
|
20
|
+
* Two-stage retrieval (§2): stage-1 sign-bit scan → int8 rerank of K
|
|
21
|
+
* candidates. K is deliberately generous. Measured on a 43,859-chunk
|
|
22
|
+
* multilingual-e5-small catalog with natural-language questions, recall of
|
|
23
|
+
* the exact top-24 among the stage-1 candidates was 22% at the former
|
|
24
|
+
* K = 192 with raw sign bits, and with centered bits (`centered-sign`
|
|
25
|
+
* profiles) 88% at 192, 97% at 512, 99% at 1,024 and 100% at 2,048 — while
|
|
26
|
+
* reranking 1,024 int8 rows costs about 4 ms of SQLite point reads per
|
|
27
|
+
* shard. A small K is a silent recall cliff; a large one is cheap.
|
|
28
|
+
*/
|
|
20
29
|
declare const RERANK_FINAL_K = 24;
|
|
30
|
+
declare const RERANK_CANDIDATES_MIN = 1024;
|
|
31
|
+
declare const RERANK_CANDIDATES_MAX = 4096;
|
|
21
32
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
22
33
|
|
|
23
34
|
/**
|
|
@@ -309,12 +320,16 @@ declare function openCatalogDatabase(absPath: string): CatalogDb;
|
|
|
309
320
|
* One mounted catalog: the router database plus lazily opened shards, and
|
|
310
321
|
* the two-stage query flow (the gezk spec §8 and §9):
|
|
311
322
|
*
|
|
312
|
-
* embed query (caller) → centroid routing → per shard: bit
|
|
323
|
+
* embed query (caller) → centroid routing → per shard: sign-bit scan
|
|
313
324
|
* top-K → int8 rerank → fuse.
|
|
314
325
|
*
|
|
315
326
|
* Stage 1 runs in memory: a shard's sign-bit rows are loaded once into a
|
|
316
|
-
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
317
|
-
*
|
|
327
|
+
* contiguous array (9.6 MB for a full 200k-chunk shard) and scanned with
|
|
328
|
+
* the float query (`asymmetricTopK`), so no vector extension is needed to
|
|
329
|
+
* read a catalog. The catalog's own profile echo says how its bits were
|
|
330
|
+
* derived: a `centered-sign` profile's bits come from `vector − center`, so
|
|
331
|
+
* the query is centered the same way before the scan. The int8 rerank is
|
|
332
|
+
* identical for both kinds of bits.
|
|
318
333
|
*
|
|
319
334
|
* The handle is synchronous (node:sqlite) and single-threaded by design —
|
|
320
335
|
* the daemon confines every handle to its knowledge worker thread; the CLI
|
|
@@ -383,6 +398,8 @@ declare class CatalogHandle {
|
|
|
383
398
|
private readonly connections;
|
|
384
399
|
private readonly bitIndexes;
|
|
385
400
|
private assetsByPath;
|
|
401
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
402
|
+
private binaryCenterCache;
|
|
386
403
|
readonly meta: Record<string, string>;
|
|
387
404
|
readonly shards: ShardInfo[];
|
|
388
405
|
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
@@ -406,6 +423,13 @@ declare class CatalogHandle {
|
|
|
406
423
|
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
407
424
|
*/
|
|
408
425
|
private shardBits;
|
|
426
|
+
/**
|
|
427
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
428
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
429
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
430
|
+
* the statement of how THIS catalog was built.
|
|
431
|
+
*/
|
|
432
|
+
private binaryCenter;
|
|
409
433
|
/** The embedding dimension from the router's profile echo. */
|
|
410
434
|
private dimensions;
|
|
411
435
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
@@ -461,7 +485,10 @@ declare class CatalogHandle {
|
|
|
461
485
|
assets(): CatalogAssetInfo[];
|
|
462
486
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
463
487
|
readAsset(path: string): CatalogAssetRead | null;
|
|
464
|
-
/**
|
|
488
|
+
/**
|
|
489
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
490
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
491
|
+
*/
|
|
465
492
|
searchDocumentsFts(query: string, limit?: number): Array<{
|
|
466
493
|
documentId: string;
|
|
467
494
|
rank: number;
|
|
@@ -489,7 +516,7 @@ declare class CatalogHandle {
|
|
|
489
516
|
/** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
|
|
490
517
|
routeShards(queryVector: Float32Array, shardBudget: number): number[];
|
|
491
518
|
/**
|
|
492
|
-
* Two-stage semantic search over the routed shards: bit
|
|
519
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
493
520
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
494
521
|
*/
|
|
495
522
|
searchSemantic(queryVector: Float32Array, opts?: {
|
|
@@ -540,10 +567,21 @@ declare function assetReferences(markdown: string): string[];
|
|
|
540
567
|
|
|
541
568
|
/**
|
|
542
569
|
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
543
|
-
* 200k-chunk shard is 9.6 MB of bits
|
|
544
|
-
*
|
|
545
|
-
*
|
|
546
|
-
*
|
|
570
|
+
* 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
|
|
571
|
+
* pass over them, a few milliseconds, needing nothing beyond plain SQLite
|
|
572
|
+
* to load — which is what lets any reader implement the format without a
|
|
573
|
+
* vector extension.
|
|
574
|
+
*
|
|
575
|
+
* Two scans share the bits:
|
|
576
|
+
* - `hammingTopK` — symmetric: the query is binarized too and rows are
|
|
577
|
+
* ranked by popcount(xor). The format's baseline (§2), and what the
|
|
578
|
+
* self-KNN smoke and the conformance kit exercise.
|
|
579
|
+
* - `asymmetricTopK` — the float query against each row's bits, ranked by
|
|
580
|
+
* Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
|
|
581
|
+
* embeddings roughly doubles the share of true neighbours a small
|
|
582
|
+
* candidate pool retains (measured on multilingual-e5-small: 22% → 39%
|
|
583
|
+
* of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
|
|
584
|
+
* bits). This is what `CatalogHandle` uses.
|
|
547
585
|
*/
|
|
548
586
|
interface ShardBitIndex {
|
|
549
587
|
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
@@ -561,6 +599,23 @@ interface HammingHit {
|
|
|
561
599
|
* the whole query is two linear passes and no heap.
|
|
562
600
|
*/
|
|
563
601
|
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
602
|
+
interface AsymmetricHit {
|
|
603
|
+
chunkId: number;
|
|
604
|
+
score: number;
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* The K best rows by asymmetric score, descending, ties broken by chunk id.
|
|
608
|
+
*
|
|
609
|
+
* The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
|
|
610
|
+
* of 256 partial sums is built for each byte position, so a row costs
|
|
611
|
+
* `bytesPerRow` table reads and adds — about the popcount scan's cost. The
|
|
612
|
+
* top-K is kept in a bounded min-heap whose root is the worst kept row.
|
|
613
|
+
*
|
|
614
|
+
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
615
|
+
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
616
|
+
* stored; this function does not know or care which.
|
|
617
|
+
*/
|
|
618
|
+
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number): AsymmetricHit[];
|
|
564
619
|
|
|
565
620
|
/**
|
|
566
621
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
@@ -601,6 +656,13 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
601
656
|
* stores them (provably identical, one third of float32's size), with the
|
|
602
657
|
* dequantisation rule recorded in the file metadata (gezk spec §6).
|
|
603
658
|
*
|
|
659
|
+
* The bit rule in that metadata follows the catalog's profile: a `sign`
|
|
660
|
+
* profile stores sign(x), a `centered-sign` profile sign(x − c) with the
|
|
661
|
+
* profile's center c — which the metadata then carries too, so the Parquet
|
|
662
|
+
* files alone are enough to binarise a query the way the archive did. (Export
|
|
663
|
+
* version 2 wrote the plain-sign rule for every profile, which misdescribed
|
|
664
|
+
* centered-sign catalogs; version 3 fixed that.)
|
|
665
|
+
*
|
|
604
666
|
* Determinism: rows are read in primary-key order and written single-
|
|
605
667
|
* threaded with insertion order preserved, a fixed codec and level, a fixed
|
|
606
668
|
* row-group size, and no wall-clock metadata — so two exports of one archive
|
|
@@ -609,7 +671,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
|
|
|
609
671
|
* bytes.
|
|
610
672
|
*/
|
|
611
673
|
|
|
612
|
-
|
|
674
|
+
/** 3: the bit-rule metadata follows the profile's binary method (was always plain sign). */
|
|
675
|
+
declare const PARQUET_EXPORT_VERSION = 3;
|
|
613
676
|
type ParquetExportSource = {
|
|
614
677
|
archivePath: string;
|
|
615
678
|
} | {
|
|
@@ -660,6 +723,16 @@ interface ParquetExportReport {
|
|
|
660
723
|
}
|
|
661
724
|
declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
662
725
|
declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
|
|
726
|
+
/**
|
|
727
|
+
* The key/value metadata every exported file carries. The bit rule is read
|
|
728
|
+
* from the profile — never assumed — because a `centered-sign` profile's bits
|
|
729
|
+
* are meaningless to a reader who binarises a query with plain sign(x). `x`
|
|
730
|
+
* is the float unit vector the bits were taken from, before int8 rounding, so
|
|
731
|
+
* recomputing them from `q / 127` can flip the few dimensions within rounding
|
|
732
|
+
* error of the threshold. Insertion order is the file's key order, so it is
|
|
733
|
+
* part of the deterministic bytes.
|
|
734
|
+
*/
|
|
735
|
+
declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<string, string>;
|
|
663
736
|
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
664
737
|
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
665
738
|
|
|
@@ -678,10 +751,28 @@ declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | nul
|
|
|
678
751
|
* built from a profile verifies the files it loaded against these digests.
|
|
679
752
|
*/
|
|
680
753
|
|
|
681
|
-
/**
|
|
754
|
+
/**
|
|
755
|
+
* The public profile's first revision: multilingual, so world catalogs serve
|
|
756
|
+
* every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
|
|
757
|
+
* vectors, centered stage-1 bits); catalogs built with this revision stay
|
|
758
|
+
* readable and searchable, at the lower stage-1 recall of raw sign bits.
|
|
759
|
+
*/
|
|
682
760
|
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
761
|
+
/**
|
|
762
|
+
* The public profile, revision 2: the model, files, instructions and int8
|
|
763
|
+
* rerank of revision 1, with the stage-1 sign bits taken from
|
|
764
|
+
* `vector − center` (`centered-sign`). e5-small vectors share one dominant
|
|
765
|
+
* direction, so raw sign bits mostly encode that direction rather than the
|
|
766
|
+
* passage and the hamming pre-filter loses most true neighbours; centering
|
|
767
|
+
* on the pinned corpus mean restores them (see e5-small-center.ts for the
|
|
768
|
+
* provenance and the measurements). A query vector for revision 1 is a
|
|
769
|
+
* valid query vector for revision 2 — `sameVectorSpace` says so — and the
|
|
770
|
+
* reader centers it itself from the catalog's profile echo.
|
|
771
|
+
*/
|
|
772
|
+
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
683
773
|
/** The local-build default — matches gezel's shipped project embedder. */
|
|
684
774
|
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
775
|
+
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
685
776
|
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
686
777
|
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
687
778
|
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
@@ -717,9 +808,10 @@ declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
|
717
808
|
scale: 127;
|
|
718
809
|
};
|
|
719
810
|
binary: {
|
|
720
|
-
method: "sign";
|
|
811
|
+
method: "sign" | "centered-sign";
|
|
721
812
|
threshold: 0;
|
|
722
813
|
packing: "lsb-first";
|
|
814
|
+
center?: number[] | undefined;
|
|
723
815
|
};
|
|
724
816
|
};
|
|
725
817
|
};
|
|
@@ -754,9 +846,10 @@ declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
|
754
846
|
scale: 127;
|
|
755
847
|
};
|
|
756
848
|
binary: {
|
|
757
|
-
method: "sign";
|
|
849
|
+
method: "sign" | "centered-sign";
|
|
758
850
|
threshold: 0;
|
|
759
851
|
packing: "lsb-first";
|
|
852
|
+
center?: number[] | undefined;
|
|
760
853
|
};
|
|
761
854
|
};
|
|
762
855
|
};
|
|
@@ -878,6 +971,11 @@ type PipelineFn = ((texts: string[], options: {
|
|
|
878
971
|
interface TokenizerFn {
|
|
879
972
|
encode(text: string): number[];
|
|
880
973
|
}
|
|
974
|
+
/** Bytes fetched so far across every file a profile load pulls, summed. */
|
|
975
|
+
interface ModelDownloadProgress {
|
|
976
|
+
bytesDone: number;
|
|
977
|
+
bytesTotal: number;
|
|
978
|
+
}
|
|
881
979
|
/** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
|
|
882
980
|
interface TransformersModelOptions {
|
|
883
981
|
revision: string;
|
|
@@ -886,6 +984,18 @@ interface TransformersModelOptions {
|
|
|
886
984
|
model_file_name: string;
|
|
887
985
|
}
|
|
888
986
|
declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
|
|
987
|
+
/**
|
|
988
|
+
* The transformers.js options that select exactly one pinned ONNX graph:
|
|
989
|
+
* its folder, its base name, and the dtype whose suffix transformers.js
|
|
990
|
+
* appends to that name. A file with no recognized suffix loads as fp32 under
|
|
991
|
+
* its full name, which selects the same file. Null for a path that is not a
|
|
992
|
+
* named `.onnx` graph.
|
|
993
|
+
*/
|
|
994
|
+
declare function transformersGraphOptions(onnxFile: string): {
|
|
995
|
+
dtype: string;
|
|
996
|
+
subfolder: string;
|
|
997
|
+
model_file_name: string;
|
|
998
|
+
} | null;
|
|
889
999
|
declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
|
|
890
1000
|
cacheDir?: string;
|
|
891
1001
|
/**
|
|
@@ -896,6 +1006,8 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
|
|
|
896
1006
|
* what affects values, and that stays with the caller.
|
|
897
1007
|
*/
|
|
898
1008
|
sessionOptions?: Record<string, unknown>;
|
|
1009
|
+
/** Called as the model and tokenizer files download. */
|
|
1010
|
+
onDownloadProgress?: (progress: ModelDownloadProgress) => void;
|
|
899
1011
|
/** Test seam: a stand-in for `@huggingface/transformers`. */
|
|
900
1012
|
transformers?: TransformersModule;
|
|
901
1013
|
}): Promise<ProfileEmbedder>;
|
|
@@ -1150,4 +1262,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1150
1262
|
|
|
1151
1263
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1152
1264
|
|
|
1153
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
|
1265
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -30,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
|
|
|
30
30
|
var ROUTE_SHARDS_PROACTIVE = 3;
|
|
31
31
|
var ROUTE_SHARDS_EXPLICIT = 6;
|
|
32
32
|
var RERANK_FINAL_K = 24;
|
|
33
|
+
var RERANK_CANDIDATES_MIN = 1024;
|
|
34
|
+
var RERANK_CANDIDATES_MAX = 4096;
|
|
33
35
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
34
|
-
return Math.min(
|
|
36
|
+
return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
|
|
35
37
|
}
|
|
36
38
|
|
|
37
39
|
// src/format/ddl.ts
|
|
@@ -42,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
|
|
|
42
44
|
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
43
45
|
|
|
44
46
|
// src/format/quantize.ts
|
|
45
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
centerVector,
|
|
49
|
+
l2Normalize,
|
|
50
|
+
quantizeBinary,
|
|
51
|
+
quantizeBinaryForProfile,
|
|
52
|
+
quantizeInt8,
|
|
53
|
+
rerankScore
|
|
54
|
+
} from "@bendyline/gezk";
|
|
46
55
|
|
|
47
56
|
// src/chunking/markdown-chunker.ts
|
|
48
57
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -241,16 +250,20 @@ function sanitizeFtsQuery(query) {
|
|
|
241
250
|
if (tokens.length === 0) return null;
|
|
242
251
|
return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
243
252
|
}
|
|
244
|
-
|
|
253
|
+
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
254
|
+
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
255
|
+
function documentFtsTopIds(db, match, limit, exactTitle) {
|
|
256
|
+
const title = exactTitle?.normalize("NFKC").trim() ?? "";
|
|
245
257
|
return db.prepare(
|
|
246
|
-
|
|
247
|
-
|
|
258
|
+
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
259
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
260
|
+
).all(match, title, limit).map((row) => row.document_id);
|
|
248
261
|
}
|
|
249
262
|
var SMOKE_QUERY_TOP_N = 10;
|
|
250
263
|
function documentSmokeQueryMisses(db, smoke) {
|
|
251
264
|
const match = sanitizeFtsQuery(smoke.query);
|
|
252
265
|
if (!match) return [...smoke.expectedDocumentIds];
|
|
253
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
|
|
266
|
+
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
254
267
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
255
268
|
}
|
|
256
269
|
|
|
@@ -469,7 +482,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
469
482
|
}
|
|
470
483
|
const row = pendingRows[i];
|
|
471
484
|
const stmts = vecStmtsFor(shardDb(row.shardId));
|
|
472
|
-
stmts.vec.run(BigInt(row.chunkId), Buffer.from(
|
|
485
|
+
stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
|
|
473
486
|
stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
|
|
474
487
|
const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
|
|
475
488
|
void shardTotal;
|
|
@@ -1403,8 +1416,10 @@ import { readFileSync as readFileSync2 } from "fs";
|
|
|
1403
1416
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1404
1417
|
import { brotliDecompressSync } from "zlib";
|
|
1405
1418
|
import {
|
|
1419
|
+
KnowledgeEmbeddingProfileSchema,
|
|
1406
1420
|
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
|
|
1407
1421
|
assetContentType,
|
|
1422
|
+
embeddingProfileCenter,
|
|
1408
1423
|
isKnowledgeAssetPath
|
|
1409
1424
|
} from "@bendyline/gezk";
|
|
1410
1425
|
|
|
@@ -1478,6 +1493,86 @@ function hammingTopK(index, query, k) {
|
|
|
1478
1493
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1479
1494
|
return hits;
|
|
1480
1495
|
}
|
|
1496
|
+
function asymmetricTopK(index, query, k) {
|
|
1497
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1498
|
+
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1499
|
+
throw new Error(
|
|
1500
|
+
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1501
|
+
);
|
|
1502
|
+
}
|
|
1503
|
+
const limit = Math.min(k, rows);
|
|
1504
|
+
if (limit <= 0) return [];
|
|
1505
|
+
const lut = new Float32Array(bytesPerRow * 256);
|
|
1506
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1507
|
+
for (let b = 0; b < 256; b++) {
|
|
1508
|
+
let s = 0;
|
|
1509
|
+
for (let i = 0; i < 8; i++) {
|
|
1510
|
+
const d = p * 8 + i;
|
|
1511
|
+
if (d >= query.length) break;
|
|
1512
|
+
const v = query[d];
|
|
1513
|
+
s += b >> i & 1 ? v : -v;
|
|
1514
|
+
}
|
|
1515
|
+
lut[p * 256 + b] = s;
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
const heapScore = new Float64Array(limit);
|
|
1519
|
+
const heapId = new Int32Array(limit);
|
|
1520
|
+
let size = 0;
|
|
1521
|
+
const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
|
|
1522
|
+
const swap = (i, j) => {
|
|
1523
|
+
const s = heapScore[i];
|
|
1524
|
+
const id = heapId[i];
|
|
1525
|
+
heapScore[i] = heapScore[j];
|
|
1526
|
+
heapId[i] = heapId[j];
|
|
1527
|
+
heapScore[j] = s;
|
|
1528
|
+
heapId[j] = id;
|
|
1529
|
+
};
|
|
1530
|
+
const siftUp = (start) => {
|
|
1531
|
+
let i = start;
|
|
1532
|
+
while (i > 0) {
|
|
1533
|
+
const parent = i - 1 >> 1;
|
|
1534
|
+
if (!worse(i, parent)) break;
|
|
1535
|
+
swap(i, parent);
|
|
1536
|
+
i = parent;
|
|
1537
|
+
}
|
|
1538
|
+
};
|
|
1539
|
+
const siftDown = (start) => {
|
|
1540
|
+
let i = start;
|
|
1541
|
+
for (; ; ) {
|
|
1542
|
+
const left = 2 * i + 1;
|
|
1543
|
+
if (left >= size) break;
|
|
1544
|
+
const right = left + 1;
|
|
1545
|
+
const child = right < size && worse(right, left) ? right : left;
|
|
1546
|
+
if (!worse(child, i)) break;
|
|
1547
|
+
swap(child, i);
|
|
1548
|
+
i = child;
|
|
1549
|
+
}
|
|
1550
|
+
};
|
|
1551
|
+
let offset = 0;
|
|
1552
|
+
for (let r = 0; r < rows; r++) {
|
|
1553
|
+
let score = 0;
|
|
1554
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1555
|
+
score += lut[p * 256 + bits[offset + p]];
|
|
1556
|
+
}
|
|
1557
|
+
offset += bytesPerRow;
|
|
1558
|
+
if (size < limit) {
|
|
1559
|
+
heapScore[size] = score;
|
|
1560
|
+
heapId[size] = r + 1;
|
|
1561
|
+
size++;
|
|
1562
|
+
siftUp(size - 1);
|
|
1563
|
+
} else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
|
|
1564
|
+
heapScore[0] = score;
|
|
1565
|
+
heapId[0] = r + 1;
|
|
1566
|
+
siftDown(0);
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
const hits = [];
|
|
1570
|
+
for (let i = 0; i < size; i++) {
|
|
1571
|
+
hits.push({ chunkId: heapId[i], score: heapScore[i] });
|
|
1572
|
+
}
|
|
1573
|
+
hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
|
|
1574
|
+
return hits;
|
|
1575
|
+
}
|
|
1481
1576
|
|
|
1482
1577
|
// src/reader/catalog-handle.ts
|
|
1483
1578
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
@@ -1499,6 +1594,8 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1499
1594
|
connections = /* @__PURE__ */ new Map();
|
|
1500
1595
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
1501
1596
|
assetsByPath = null;
|
|
1597
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1598
|
+
binaryCenterCache;
|
|
1502
1599
|
meta;
|
|
1503
1600
|
shards;
|
|
1504
1601
|
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
@@ -1581,6 +1678,28 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1581
1678
|
this.bitIndexes.set(shard.path, index);
|
|
1582
1679
|
return index;
|
|
1583
1680
|
}
|
|
1681
|
+
/**
|
|
1682
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
1683
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
1684
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
1685
|
+
* the statement of how THIS catalog was built.
|
|
1686
|
+
*/
|
|
1687
|
+
binaryCenter() {
|
|
1688
|
+
if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
|
|
1689
|
+
const raw = this.meta.embedding_profile_json;
|
|
1690
|
+
let parsed = null;
|
|
1691
|
+
try {
|
|
1692
|
+
parsed = raw ? JSON.parse(raw) : null;
|
|
1693
|
+
} catch {
|
|
1694
|
+
parsed = null;
|
|
1695
|
+
}
|
|
1696
|
+
const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
|
|
1697
|
+
if (!profile.success) {
|
|
1698
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1699
|
+
}
|
|
1700
|
+
this.binaryCenterCache = embeddingProfileCenter(profile.data);
|
|
1701
|
+
return this.binaryCenterCache;
|
|
1702
|
+
}
|
|
1584
1703
|
/** The embedding dimension from the router's profile echo. */
|
|
1585
1704
|
dimensions() {
|
|
1586
1705
|
const raw = this.meta.embedding_profile_json;
|
|
@@ -1809,12 +1928,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1809
1928
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1810
1929
|
}
|
|
1811
1930
|
// ── search ────────────────────────────────────────────────────────────────
|
|
1812
|
-
/**
|
|
1931
|
+
/**
|
|
1932
|
+
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
1933
|
+
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
1934
|
+
*/
|
|
1813
1935
|
searchDocumentsFts(query, limit = 10) {
|
|
1814
1936
|
const match = ftsQuery(query);
|
|
1815
1937
|
if (!match) return [];
|
|
1816
1938
|
try {
|
|
1817
|
-
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
1939
|
+
return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
|
|
1818
1940
|
documentId,
|
|
1819
1941
|
rank: i
|
|
1820
1942
|
}));
|
|
@@ -1863,7 +1985,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1863
1985
|
return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
1864
1986
|
}
|
|
1865
1987
|
/**
|
|
1866
|
-
* Two-stage semantic search over the routed shards: bit
|
|
1988
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
1867
1989
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
1868
1990
|
*/
|
|
1869
1991
|
searchSemantic(queryVector, opts = {}) {
|
|
@@ -1875,14 +1997,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1875
1997
|
* after global cross-catalog routing has already spent the S budget.
|
|
1876
1998
|
*/
|
|
1877
1999
|
searchShards(queryVector, shardIds, finalK) {
|
|
1878
|
-
const
|
|
2000
|
+
const center = this.binaryCenter();
|
|
2001
|
+
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
1879
2002
|
const hits = [];
|
|
1880
2003
|
for (const shardId of shardIds) {
|
|
1881
2004
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1882
2005
|
if (!shard) continue;
|
|
1883
2006
|
const db = this.shardDb(shard);
|
|
1884
2007
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1885
|
-
const candidates =
|
|
2008
|
+
const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
|
|
1886
2009
|
chunk_id: hit.chunkId
|
|
1887
2010
|
}));
|
|
1888
2011
|
if (candidates.length === 0) continue;
|
|
@@ -2372,8 +2495,8 @@ import { createWriteStream as createWriteStream3 } from "fs";
|
|
|
2372
2495
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2373
2496
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2374
2497
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2375
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
2376
|
-
var PARQUET_EXPORT_VERSION =
|
|
2498
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2499
|
+
var PARQUET_EXPORT_VERSION = 3;
|
|
2377
2500
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2378
2501
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2379
2502
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2398,7 +2521,7 @@ async function exportCatalogParquet(opts) {
|
|
|
2398
2521
|
const staging = join4(work, "ndjson");
|
|
2399
2522
|
await mkdir3(staging, { recursive: true });
|
|
2400
2523
|
const files = [];
|
|
2401
|
-
const metadata =
|
|
2524
|
+
const metadata = parquetKvMetadata(manifest);
|
|
2402
2525
|
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
2403
2526
|
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
2404
2527
|
try {
|
|
@@ -2666,7 +2789,9 @@ function parseJsonStrings(raw) {
|
|
|
2666
2789
|
function sqlString(value) {
|
|
2667
2790
|
return `'${value.replace(/'/g, "''")}'`;
|
|
2668
2791
|
}
|
|
2669
|
-
function
|
|
2792
|
+
function parquetKvMetadata(manifest) {
|
|
2793
|
+
const binary = manifest.embedding.quantization.binary;
|
|
2794
|
+
const centered = embeddingProfileCenter2(manifest.embedding) !== null;
|
|
2670
2795
|
return {
|
|
2671
2796
|
gezk_catalog_id: manifest.id,
|
|
2672
2797
|
gezk_catalog_version: manifest.version,
|
|
@@ -2676,7 +2801,9 @@ function kvMetadata(manifest) {
|
|
|
2676
2801
|
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2677
2802
|
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2678
2803
|
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2679
|
-
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
2804
|
+
gezk_embedding_bit: centered ? "centered-sign bits, LSB-first, ceil(dimensions/8) bytes: bit = (x - c) > 0, x the float unit vector, c = gezk_embedding_bit_center" : "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0, x the float unit vector",
|
|
2805
|
+
gezk_embedding_bit_method: binary.method,
|
|
2806
|
+
...centered ? { gezk_embedding_bit_center: JSON.stringify(binary.center) } : {},
|
|
2680
2807
|
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2681
2808
|
gezk_license: manifest.license.name
|
|
2682
2809
|
};
|
|
@@ -2787,6 +2914,394 @@ async function findDuckdbBinary(env = process.env) {
|
|
|
2787
2914
|
return null;
|
|
2788
2915
|
}
|
|
2789
2916
|
|
|
2917
|
+
// src/profiles/e5-small-center.ts
|
|
2918
|
+
var MULTILINGUAL_E5_SMALL_CENTER = [
|
|
2919
|
+
0.046697,
|
|
2920
|
+
-0.013211,
|
|
2921
|
+
-0.042276,
|
|
2922
|
+
-0.049082,
|
|
2923
|
+
0.060627,
|
|
2924
|
+
-0.032643,
|
|
2925
|
+
0.029874,
|
|
2926
|
+
0.033307,
|
|
2927
|
+
0.058447,
|
|
2928
|
+
-0.012016,
|
|
2929
|
+
0.046472,
|
|
2930
|
+
5991e-6,
|
|
2931
|
+
0.04884,
|
|
2932
|
+
-0.028836,
|
|
2933
|
+
-0.03244,
|
|
2934
|
+
0.045633,
|
|
2935
|
+
0.056798,
|
|
2936
|
+
-0.043557,
|
|
2937
|
+
-0.034472,
|
|
2938
|
+
-0.05355,
|
|
2939
|
+
0.026363,
|
|
2940
|
+
-0.010318,
|
|
2941
|
+
-0.030487,
|
|
2942
|
+
0.047083,
|
|
2943
|
+
0.057205,
|
|
2944
|
+
0.035654,
|
|
2945
|
+
-0.017461,
|
|
2946
|
+
0.010881,
|
|
2947
|
+
0.047985,
|
|
2948
|
+
-0.048514,
|
|
2949
|
+
-0.048956,
|
|
2950
|
+
-0.027643,
|
|
2951
|
+
0.053259,
|
|
2952
|
+
-0.051057,
|
|
2953
|
+
0.045055,
|
|
2954
|
+
0.017888,
|
|
2955
|
+
-0.055755,
|
|
2956
|
+
-0.046578,
|
|
2957
|
+
0.05214,
|
|
2958
|
+
-0.037085,
|
|
2959
|
+
-0.016268,
|
|
2960
|
+
0.027508,
|
|
2961
|
+
0.035793,
|
|
2962
|
+
0.052082,
|
|
2963
|
+
0.034885,
|
|
2964
|
+
0.047174,
|
|
2965
|
+
-0.036778,
|
|
2966
|
+
0.031394,
|
|
2967
|
+
-9156e-6,
|
|
2968
|
+
-0.02486,
|
|
2969
|
+
-0.048897,
|
|
2970
|
+
0.052576,
|
|
2971
|
+
0.017289,
|
|
2972
|
+
0.047794,
|
|
2973
|
+
0.051939,
|
|
2974
|
+
-0.051261,
|
|
2975
|
+
-0.061345,
|
|
2976
|
+
-0.056267,
|
|
2977
|
+
-0.049305,
|
|
2978
|
+
0.047241,
|
|
2979
|
+
0.043763,
|
|
2980
|
+
-0.04506,
|
|
2981
|
+
0.015752,
|
|
2982
|
+
0.017543,
|
|
2983
|
+
0.067856,
|
|
2984
|
+
0.065763,
|
|
2985
|
+
0.037218,
|
|
2986
|
+
0.044877,
|
|
2987
|
+
-0.057149,
|
|
2988
|
+
-0.028072,
|
|
2989
|
+
-0.060566,
|
|
2990
|
+
0.064004,
|
|
2991
|
+
6095e-6,
|
|
2992
|
+
-0.039693,
|
|
2993
|
+
0.021968,
|
|
2994
|
+
0.02654,
|
|
2995
|
+
0.025546,
|
|
2996
|
+
-0.048954,
|
|
2997
|
+
0.026967,
|
|
2998
|
+
-0.045002,
|
|
2999
|
+
-0.055556,
|
|
3000
|
+
-0.034963,
|
|
3001
|
+
-0.03466,
|
|
3002
|
+
0.032486,
|
|
3003
|
+
-0.049597,
|
|
3004
|
+
0.058718,
|
|
3005
|
+
0.048021,
|
|
3006
|
+
-0.066783,
|
|
3007
|
+
0.059049,
|
|
3008
|
+
-0.04665,
|
|
3009
|
+
0.052902,
|
|
3010
|
+
0.031438,
|
|
3011
|
+
-0.061515,
|
|
3012
|
+
-0.05342,
|
|
3013
|
+
-0.068253,
|
|
3014
|
+
-0.06666,
|
|
3015
|
+
-0.04553,
|
|
3016
|
+
0.059252,
|
|
3017
|
+
0.050728,
|
|
3018
|
+
-0.030429,
|
|
3019
|
+
0.067182,
|
|
3020
|
+
-0.038819,
|
|
3021
|
+
0.049358,
|
|
3022
|
+
-0.055247,
|
|
3023
|
+
-0.032933,
|
|
3024
|
+
0.041239,
|
|
3025
|
+
0.035164,
|
|
3026
|
+
-0.066574,
|
|
3027
|
+
0.05348,
|
|
3028
|
+
-0.052658,
|
|
3029
|
+
-0.034758,
|
|
3030
|
+
0.037219,
|
|
3031
|
+
0.064111,
|
|
3032
|
+
0.047861,
|
|
3033
|
+
-0.04608,
|
|
3034
|
+
-0.041338,
|
|
3035
|
+
-0.016072,
|
|
3036
|
+
-0.046026,
|
|
3037
|
+
0.0255,
|
|
3038
|
+
-0.041581,
|
|
3039
|
+
0.059684,
|
|
3040
|
+
-0.019611,
|
|
3041
|
+
-0.050363,
|
|
3042
|
+
-0.061268,
|
|
3043
|
+
-0.057556,
|
|
3044
|
+
-0.03184,
|
|
3045
|
+
0.018522,
|
|
3046
|
+
0.046887,
|
|
3047
|
+
0.054774,
|
|
3048
|
+
0.012536,
|
|
3049
|
+
0.033019,
|
|
3050
|
+
0.033211,
|
|
3051
|
+
0.029609,
|
|
3052
|
+
0.050215,
|
|
3053
|
+
0.035425,
|
|
3054
|
+
0.076419,
|
|
3055
|
+
-0.055196,
|
|
3056
|
+
-0.010016,
|
|
3057
|
+
-936e-6,
|
|
3058
|
+
-0.023764,
|
|
3059
|
+
-0.059648,
|
|
3060
|
+
0.054857,
|
|
3061
|
+
-0.030456,
|
|
3062
|
+
0.041281,
|
|
3063
|
+
0.063894,
|
|
3064
|
+
0.035466,
|
|
3065
|
+
0.069754,
|
|
3066
|
+
-0.026593,
|
|
3067
|
+
0.066305,
|
|
3068
|
+
-0.028609,
|
|
3069
|
+
0.05324,
|
|
3070
|
+
-0.022879,
|
|
3071
|
+
0.04882,
|
|
3072
|
+
0.032385,
|
|
3073
|
+
0.042921,
|
|
3074
|
+
-0.043319,
|
|
3075
|
+
-0.065022,
|
|
3076
|
+
-0.044018,
|
|
3077
|
+
0.04237,
|
|
3078
|
+
0.051018,
|
|
3079
|
+
-0.048907,
|
|
3080
|
+
-0.053045,
|
|
3081
|
+
-0.057301,
|
|
3082
|
+
-0.019731,
|
|
3083
|
+
-0.060812,
|
|
3084
|
+
-0.043114,
|
|
3085
|
+
0.021207,
|
|
3086
|
+
0.052498,
|
|
3087
|
+
-0.038307,
|
|
3088
|
+
-0.037082,
|
|
3089
|
+
-0.043732,
|
|
3090
|
+
0.045959,
|
|
3091
|
+
-0.044296,
|
|
3092
|
+
0.057199,
|
|
3093
|
+
-0.011835,
|
|
3094
|
+
0.054221,
|
|
3095
|
+
-0.046354,
|
|
3096
|
+
0.043442,
|
|
3097
|
+
0.063337,
|
|
3098
|
+
0.041964,
|
|
3099
|
+
-0.032836,
|
|
3100
|
+
-0.020089,
|
|
3101
|
+
-0.056327,
|
|
3102
|
+
-0.05089,
|
|
3103
|
+
-0.053933,
|
|
3104
|
+
-0.035486,
|
|
3105
|
+
-0.046215,
|
|
3106
|
+
0.025754,
|
|
3107
|
+
0.032986,
|
|
3108
|
+
-0.025048,
|
|
3109
|
+
-9903e-6,
|
|
3110
|
+
0.031162,
|
|
3111
|
+
-0.050969,
|
|
3112
|
+
-0.049401,
|
|
3113
|
+
-0.046206,
|
|
3114
|
+
0.023197,
|
|
3115
|
+
-0.033441,
|
|
3116
|
+
0.045075,
|
|
3117
|
+
0.043318,
|
|
3118
|
+
0.043766,
|
|
3119
|
+
-6601e-6,
|
|
3120
|
+
-0.023077,
|
|
3121
|
+
0.033917,
|
|
3122
|
+
0.032279,
|
|
3123
|
+
0.034874,
|
|
3124
|
+
5185e-6,
|
|
3125
|
+
-0.057329,
|
|
3126
|
+
0.048437,
|
|
3127
|
+
-0.033964,
|
|
3128
|
+
0.044195,
|
|
3129
|
+
0.051107,
|
|
3130
|
+
-0.056642,
|
|
3131
|
+
-0.070856,
|
|
3132
|
+
0.045431,
|
|
3133
|
+
-0.047205,
|
|
3134
|
+
-0.021457,
|
|
3135
|
+
3103e-6,
|
|
3136
|
+
0.062536,
|
|
3137
|
+
-0.037804,
|
|
3138
|
+
0.031278,
|
|
3139
|
+
0.041619,
|
|
3140
|
+
-0.015547,
|
|
3141
|
+
0.052298,
|
|
3142
|
+
-0.078327,
|
|
3143
|
+
-0.052303,
|
|
3144
|
+
0.043099,
|
|
3145
|
+
0.05375,
|
|
3146
|
+
-0.041854,
|
|
3147
|
+
-0.038314,
|
|
3148
|
+
0.039399,
|
|
3149
|
+
-0.047769,
|
|
3150
|
+
-0.038469,
|
|
3151
|
+
-0.056793,
|
|
3152
|
+
-0.064312,
|
|
3153
|
+
-0.047884,
|
|
3154
|
+
-0.060067,
|
|
3155
|
+
-0.030687,
|
|
3156
|
+
0.030053,
|
|
3157
|
+
0.033569,
|
|
3158
|
+
-0.038014,
|
|
3159
|
+
-0.032396,
|
|
3160
|
+
-0.041395,
|
|
3161
|
+
0.038163,
|
|
3162
|
+
-0.056206,
|
|
3163
|
+
0.047115,
|
|
3164
|
+
-0.038099,
|
|
3165
|
+
-0.031927,
|
|
3166
|
+
0.033857,
|
|
3167
|
+
-0.011227,
|
|
3168
|
+
0.032883,
|
|
3169
|
+
0.032516,
|
|
3170
|
+
-0.045622,
|
|
3171
|
+
-0.044387,
|
|
3172
|
+
-0.034374,
|
|
3173
|
+
-0.030433,
|
|
3174
|
+
0.055097,
|
|
3175
|
+
0.055568,
|
|
3176
|
+
0.058726,
|
|
3177
|
+
-0.043569,
|
|
3178
|
+
0.031854,
|
|
3179
|
+
775e-5,
|
|
3180
|
+
-0.022533,
|
|
3181
|
+
0.096133,
|
|
3182
|
+
0.052876,
|
|
3183
|
+
0.045582,
|
|
3184
|
+
0.041868,
|
|
3185
|
+
-0.039268,
|
|
3186
|
+
-0.037865,
|
|
3187
|
+
-0.054658,
|
|
3188
|
+
-0.031297,
|
|
3189
|
+
-0.058104,
|
|
3190
|
+
0.018705,
|
|
3191
|
+
0.031934,
|
|
3192
|
+
-0.052746,
|
|
3193
|
+
-0.044438,
|
|
3194
|
+
-0.029991,
|
|
3195
|
+
0.011917,
|
|
3196
|
+
0.070103,
|
|
3197
|
+
-0.046682,
|
|
3198
|
+
-0.048965,
|
|
3199
|
+
0.046255,
|
|
3200
|
+
0.027877,
|
|
3201
|
+
0.0475,
|
|
3202
|
+
0.044287,
|
|
3203
|
+
0.04251,
|
|
3204
|
+
-0.017647,
|
|
3205
|
+
-0.017812,
|
|
3206
|
+
0.055672,
|
|
3207
|
+
-0.042207,
|
|
3208
|
+
-0.035516,
|
|
3209
|
+
-0.025558,
|
|
3210
|
+
-0.062443,
|
|
3211
|
+
0.037519,
|
|
3212
|
+
-0.031697,
|
|
3213
|
+
0.064998,
|
|
3214
|
+
0.045949,
|
|
3215
|
+
302e-6,
|
|
3216
|
+
0.065205,
|
|
3217
|
+
-0.052016,
|
|
3218
|
+
0.036721,
|
|
3219
|
+
-0.034769,
|
|
3220
|
+
-0.049074,
|
|
3221
|
+
0.040413,
|
|
3222
|
+
0.045425,
|
|
3223
|
+
-0.068063,
|
|
3224
|
+
0.021485,
|
|
3225
|
+
8549e-6,
|
|
3226
|
+
0.028179,
|
|
3227
|
+
0.026914,
|
|
3228
|
+
0.020493,
|
|
3229
|
+
0.056155,
|
|
3230
|
+
0.047202,
|
|
3231
|
+
-0.049085,
|
|
3232
|
+
-0.022624,
|
|
3233
|
+
0.063734,
|
|
3234
|
+
0.05471,
|
|
3235
|
+
-0.032258,
|
|
3236
|
+
0.047761,
|
|
3237
|
+
-0.065033,
|
|
3238
|
+
-0.047818,
|
|
3239
|
+
-0.034254,
|
|
3240
|
+
-0.062863,
|
|
3241
|
+
-0.01362,
|
|
3242
|
+
-0.053904,
|
|
3243
|
+
0.024415,
|
|
3244
|
+
0.044367,
|
|
3245
|
+
-0.064056,
|
|
3246
|
+
-0.038419,
|
|
3247
|
+
0.038997,
|
|
3248
|
+
1015e-6,
|
|
3249
|
+
0.038928,
|
|
3250
|
+
-0.045794,
|
|
3251
|
+
-0.038313,
|
|
3252
|
+
0.043922,
|
|
3253
|
+
-0.05873,
|
|
3254
|
+
-0.020314,
|
|
3255
|
+
-0.020457,
|
|
3256
|
+
0.045522,
|
|
3257
|
+
-0.057731,
|
|
3258
|
+
-0.062966,
|
|
3259
|
+
0.023159,
|
|
3260
|
+
0.020283,
|
|
3261
|
+
-0.057352,
|
|
3262
|
+
0.044701,
|
|
3263
|
+
-0.03585,
|
|
3264
|
+
-0.052798,
|
|
3265
|
+
0.056931,
|
|
3266
|
+
-0.035609,
|
|
3267
|
+
-0.035113,
|
|
3268
|
+
0.026919,
|
|
3269
|
+
0.059404,
|
|
3270
|
+
-0.084434,
|
|
3271
|
+
0.014558,
|
|
3272
|
+
0.06226,
|
|
3273
|
+
-0.044173,
|
|
3274
|
+
0.044172,
|
|
3275
|
+
-0.046482,
|
|
3276
|
+
-0.034933,
|
|
3277
|
+
0.052513,
|
|
3278
|
+
0.044018,
|
|
3279
|
+
-0.057677,
|
|
3280
|
+
-0.033961,
|
|
3281
|
+
0.046334,
|
|
3282
|
+
0.065767,
|
|
3283
|
+
0.044779,
|
|
3284
|
+
0.063658,
|
|
3285
|
+
-0.022574,
|
|
3286
|
+
-4821e-6,
|
|
3287
|
+
0.037388,
|
|
3288
|
+
-0.043197,
|
|
3289
|
+
0.063,
|
|
3290
|
+
0.048943,
|
|
3291
|
+
-0.049495,
|
|
3292
|
+
0.014798,
|
|
3293
|
+
-0.018176,
|
|
3294
|
+
-0.03824,
|
|
3295
|
+
-0.045486,
|
|
3296
|
+
0.053828,
|
|
3297
|
+
-0.056118,
|
|
3298
|
+
-0.067025,
|
|
3299
|
+
0.027818,
|
|
3300
|
+
0.050305,
|
|
3301
|
+
0.037134,
|
|
3302
|
+
0.049673
|
|
3303
|
+
];
|
|
3304
|
+
|
|
2790
3305
|
// src/profiles/registry.ts
|
|
2791
3306
|
var SHARED_ENCODING = {
|
|
2792
3307
|
pooling: "mean",
|
|
@@ -2817,6 +3332,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
|
|
|
2817
3332
|
passageInstruction: "passage: ",
|
|
2818
3333
|
...SHARED_ENCODING
|
|
2819
3334
|
};
|
|
3335
|
+
var MULTILINGUAL_E5_SMALL_2 = {
|
|
3336
|
+
...MULTILINGUAL_E5_SMALL_1,
|
|
3337
|
+
id: "multilingual-e5-small@2",
|
|
3338
|
+
quantization: {
|
|
3339
|
+
int8: { method: "symmetric-linear", scale: 127 },
|
|
3340
|
+
binary: {
|
|
3341
|
+
method: "centered-sign",
|
|
3342
|
+
threshold: 0,
|
|
3343
|
+
packing: "lsb-first",
|
|
3344
|
+
center: [...MULTILINGUAL_E5_SMALL_CENTER]
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
3347
|
+
};
|
|
2820
3348
|
var BGE_SMALL_EN_V15_1 = {
|
|
2821
3349
|
id: "bge-small-en-v1.5@1",
|
|
2822
3350
|
model: {
|
|
@@ -2835,6 +3363,7 @@ var BGE_SMALL_EN_V15_1 = {
|
|
|
2835
3363
|
...SHARED_ENCODING
|
|
2836
3364
|
};
|
|
2837
3365
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
3366
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
2838
3367
|
MULTILINGUAL_E5_SMALL_1,
|
|
2839
3368
|
BGE_SMALL_EN_V15_1
|
|
2840
3369
|
];
|
|
@@ -2950,6 +3479,25 @@ var EmbedderUnavailableError = class extends Error {
|
|
|
2950
3479
|
this.name = "EmbedderUnavailableError";
|
|
2951
3480
|
}
|
|
2952
3481
|
};
|
|
3482
|
+
function aggregateDownloadProgress(onProgress) {
|
|
3483
|
+
const files = /* @__PURE__ */ new Map();
|
|
3484
|
+
return (info) => {
|
|
3485
|
+
if (info.status !== "progress") return;
|
|
3486
|
+
const loaded = typeof info.loaded === "number" ? info.loaded : 0;
|
|
3487
|
+
const total = typeof info.total === "number" ? info.total : 0;
|
|
3488
|
+
files.set(`${String(info.name)}/${String(info.file)}`, {
|
|
3489
|
+
loaded,
|
|
3490
|
+
total: Math.max(total, loaded)
|
|
3491
|
+
});
|
|
3492
|
+
let bytesDone = 0;
|
|
3493
|
+
let bytesTotal = 0;
|
|
3494
|
+
for (const file of files.values()) {
|
|
3495
|
+
bytesDone += file.loaded;
|
|
3496
|
+
bytesTotal += file.total;
|
|
3497
|
+
}
|
|
3498
|
+
onProgress({ bytesDone, bytesTotal });
|
|
3499
|
+
};
|
|
3500
|
+
}
|
|
2953
3501
|
var ONNX_DTYPE_BY_SUFFIX = [
|
|
2954
3502
|
["_q4f16", "q4f16"],
|
|
2955
3503
|
["_quantized", "q8"],
|
|
@@ -2970,21 +3518,24 @@ function resolveTransformersModelOptions(profile) {
|
|
|
2970
3518
|
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
2971
3519
|
);
|
|
2972
3520
|
}
|
|
2973
|
-
|
|
3521
|
+
const graph = transformersGraphOptions(onnxFile);
|
|
3522
|
+
if (!graph) {
|
|
2974
3523
|
throw new EmbedderUnavailableError(
|
|
2975
|
-
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
3524
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads named ONNX graphs only`
|
|
2976
3525
|
);
|
|
2977
3526
|
}
|
|
3527
|
+
return { revision: profile.model.revision, ...graph };
|
|
3528
|
+
}
|
|
3529
|
+
function transformersGraphOptions(onnxFile) {
|
|
3530
|
+
if (!onnxFile.endsWith(".onnx")) return null;
|
|
2978
3531
|
const slash = onnxFile.lastIndexOf("/");
|
|
2979
3532
|
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
2980
3533
|
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
2981
3534
|
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
2982
3535
|
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
2983
3536
|
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
2984
|
-
if (modelFileName === "")
|
|
2985
|
-
|
|
2986
|
-
}
|
|
2987
|
-
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
3537
|
+
if (modelFileName === "") return null;
|
|
3538
|
+
return { dtype, subfolder, model_file_name: modelFileName };
|
|
2988
3539
|
}
|
|
2989
3540
|
async function loadTransformers() {
|
|
2990
3541
|
try {
|
|
@@ -3015,13 +3566,16 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3015
3566
|
transformers.env.useFSCache = true;
|
|
3016
3567
|
transformers.env.allowRemoteModels = true;
|
|
3017
3568
|
}
|
|
3569
|
+
const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
|
|
3018
3570
|
const [pipe, tokenizer] = await Promise.all([
|
|
3019
3571
|
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3020
3572
|
...modelOptions,
|
|
3021
|
-
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
3573
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
|
|
3574
|
+
...progress
|
|
3022
3575
|
}),
|
|
3023
3576
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3024
|
-
revision: modelOptions.revision
|
|
3577
|
+
revision: modelOptions.revision,
|
|
3578
|
+
...progress
|
|
3025
3579
|
})
|
|
3026
3580
|
]);
|
|
3027
3581
|
let verification;
|
|
@@ -4122,9 +4676,12 @@ export {
|
|
|
4122
4676
|
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
4123
4677
|
MIMETYPE_PATH,
|
|
4124
4678
|
MULTILINGUAL_E5_SMALL_1,
|
|
4679
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
4125
4680
|
PARQUET_EXPORT_VERSION,
|
|
4126
4681
|
PARQUET_REPORT_PATH,
|
|
4127
4682
|
README_PATH,
|
|
4683
|
+
RERANK_CANDIDATES_MAX,
|
|
4684
|
+
RERANK_CANDIDATES_MIN,
|
|
4128
4685
|
RERANK_FINAL_K,
|
|
4129
4686
|
ROUTER_DB_PATH,
|
|
4130
4687
|
ROUTER_DDL,
|
|
@@ -4138,7 +4695,9 @@ export {
|
|
|
4138
4695
|
assertDuckdbCli,
|
|
4139
4696
|
assetReferences,
|
|
4140
4697
|
assignOutlineOrders,
|
|
4698
|
+
asymmetricTopK,
|
|
4141
4699
|
canonicalizeJson2 as canonicalizeJson,
|
|
4700
|
+
centerVector,
|
|
4142
4701
|
chunkContentHash,
|
|
4143
4702
|
chunkMarkdownProfile,
|
|
4144
4703
|
chunkUid,
|
|
@@ -4166,10 +4725,12 @@ export {
|
|
|
4166
4725
|
newOutlineTopic,
|
|
4167
4726
|
newerRegistryEntries,
|
|
4168
4727
|
openCatalogDatabase,
|
|
4728
|
+
parquetKvMetadata,
|
|
4169
4729
|
parseGitbookSummary,
|
|
4170
4730
|
parseJupyterBookToc,
|
|
4171
4731
|
parseMkdocsNav,
|
|
4172
4732
|
quantizeBinary,
|
|
4733
|
+
quantizeBinaryForProfile,
|
|
4173
4734
|
quantizeInt8,
|
|
4174
4735
|
readGezkManifest,
|
|
4175
4736
|
readMkdocsDocsDir,
|
|
@@ -4182,6 +4743,7 @@ export {
|
|
|
4182
4743
|
signRegistryIndex,
|
|
4183
4744
|
titleFromFolderName,
|
|
4184
4745
|
transformersCachePath,
|
|
4746
|
+
transformersGraphOptions,
|
|
4185
4747
|
validateExtractedCatalog,
|
|
4186
4748
|
verifyManifestSignature,
|
|
4187
4749
|
verifyProfileArtifacts,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bendyline/gezel-knowledge",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"keywords": [
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"!dist/**/*.map"
|
|
48
48
|
],
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@bendyline/gezk": "1.0.
|
|
50
|
+
"@bendyline/gezk": "1.0.2",
|
|
51
51
|
"yaml": "^2.9.0",
|
|
52
52
|
"yauzl": "^3.4.0",
|
|
53
53
|
"yazl": "^3.3.1",
|