@bendyline/gezel-knowledge 1.2.11 → 1.2.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +77 -0
- package/dist/browser.js +1410 -0
- package/dist/index.d.ts +16 -330
- package/dist/index.js +66 -19
- package/dist/registry-xi2R42Av.d.ts +372 -0
- package/package.json +6 -2
package/dist/index.d.ts
CHANGED
|
@@ -3,8 +3,23 @@ import { KnowledgeAssetModality, KnowledgeEmbeddingProfile, CatalogDocument, Kno
|
|
|
3
3
|
export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, KnowledgeTocReference, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, QUERY_STOP_WORDS, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, formatAtLeast, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, profileUnitVector, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, shardDdlFor } from '@bendyline/gezk';
|
|
4
4
|
import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
|
|
5
5
|
export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
|
|
6
|
+
import { g as MarkdownChunk } from './registry-xi2R42Av.js';
|
|
7
|
+
export { A as AsymmetricHit, B as BGE_SMALL_EN_V15_1, C as CharChunkerOptions, a as ChunkerOptions, E as EMBEDDINGGEMMA_2_512_1, G as GEZEL_BGE_SMALL_EN_V15_1, b as GEZEL_MARKDOWN_CHUNKS_2, c as GEZEL_MULTILINGUAL_E5_SMALL_1, H as HammingHit, K as KNOWLEDGE_EMBEDDING_PROFILES, d as KNOWLEDGE_VECTOR_FLOORS, M as MARKDOWN_CHUNKS_2, e as MULTILINGUAL_E5_SMALL_1, f as MULTILINGUAL_E5_SMALL_2, P as PassageEmbedInput, S as ShardBitIndex, T as TokenChunkerOptions, h as asymmetricTopK, i as buildContextHeader, j as chunkMarkdownProfile, m as hammingTopK, k as knowledgeEmbeddingProfile, l as knowledgeProfileQueryModelBytes, p as passageEmbedInput } from './registry-xi2R42Av.js';
|
|
6
8
|
import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
|
|
7
9
|
|
|
10
|
+
/**
|
|
11
|
+
* node:sqlite loader shim. The repo's tsup/esbuild pipeline rewrites static
|
|
12
|
+
* `node:`-prefixed imports to their bare legacy names, which is harmless for
|
|
13
|
+
* `fs`/`crypto` but fatal for `sqlite` — the builtin exists ONLY under the
|
|
14
|
+
* `node:` prefix, so the bundled import resolves to a nonexistent npm
|
|
15
|
+
* package. The service hit the same wall and loads it by string at runtime
|
|
16
|
+
* (sqlite-driver.ts `nodeRequire('node:sqlite')`); this shim is that pattern
|
|
17
|
+
* with the types re-exported so call sites stay typed.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
declare const DatabaseSync: typeof DatabaseSync$1;
|
|
21
|
+
type DatabaseSync = DatabaseSync$1;
|
|
22
|
+
|
|
8
23
|
/** Toolchain knobs. Format-level constants come from @bendyline/gezk. */
|
|
9
24
|
|
|
10
25
|
/** Compiler shard sizing (§3.1). */
|
|
@@ -32,53 +47,6 @@ declare const RERANK_CANDIDATES_MIN = 1024;
|
|
|
32
47
|
declare const RERANK_CANDIDATES_MAX = 4096;
|
|
33
48
|
declare function rerankK(finalK?: number, chunkCount?: number): number;
|
|
34
49
|
|
|
35
|
-
/**
|
|
36
|
-
* The canonical markdown chunker, parameterized over the two frozen
|
|
37
|
-
* profiles (the gezk spec §7):
|
|
38
|
-
*
|
|
39
|
-
* - `markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
|
|
40
|
-
* algorithm lifted from the project indexer
|
|
41
|
-
* (packages/service/src/index-store/docs.ts `chunkMarkdown` /
|
|
42
|
-
* `chunkLineRange`), so the service can later re-point here with zero
|
|
43
|
-
* behavior change.
|
|
44
|
-
* - `markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
|
|
45
|
-
* profile. Token counting is injected (`countTokens` — the embedding
|
|
46
|
-
* profile's own tokenizer), packing is per-line with sentence-level
|
|
47
|
-
* splitting for oversized lines, and every chunk carries a structural
|
|
48
|
-
* `headingPath` (root→nearest, the section's own heading last).
|
|
49
|
-
*
|
|
50
|
-
* Pure functions, no I/O, no dependencies — importable by the compiler, the
|
|
51
|
-
* service indexer, and tests alike.
|
|
52
|
-
*/
|
|
53
|
-
interface MarkdownChunk {
|
|
54
|
-
kind: 'doc' | 'preamble' | 'section';
|
|
55
|
-
/** 1-based inclusive source line span. */
|
|
56
|
-
lineStart: number;
|
|
57
|
-
lineEnd: number;
|
|
58
|
-
text: string;
|
|
59
|
-
/** Raw heading texts, root → nearest (empty for preamble/doc chunks). */
|
|
60
|
-
headingPath: string[];
|
|
61
|
-
}
|
|
62
|
-
interface CharChunkerOptions {
|
|
63
|
-
unit: 'chars';
|
|
64
|
-
/** Max chunk size in characters (profile @1: 4000). */
|
|
65
|
-
target: number;
|
|
66
|
-
/** Overlap replayed between adjacent chunks, in characters (@1: 400). */
|
|
67
|
-
overlap: number;
|
|
68
|
-
}
|
|
69
|
-
interface TokenChunkerOptions {
|
|
70
|
-
unit: 'tokens';
|
|
71
|
-
/** Target chunk size in tokens (profile @2: 420). */
|
|
72
|
-
target: number;
|
|
73
|
-
/** Overlap in tokens (@2: 64). */
|
|
74
|
-
overlap: number;
|
|
75
|
-
/** The embedding profile's tokenizer — deterministic given its pinned digest. */
|
|
76
|
-
countTokens: (text: string) => number;
|
|
77
|
-
}
|
|
78
|
-
type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
|
|
79
|
-
/** Split markdown into heading-aware, bounded, overlapping chunks. */
|
|
80
|
-
declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
|
|
81
|
-
|
|
82
50
|
interface CompileAsset {
|
|
83
51
|
/** Archive path, assets/…, matching the format's asset path grammar. */
|
|
84
52
|
path: string;
|
|
@@ -421,19 +389,6 @@ declare function readGezkManifest(archivePath: string): Promise<KnowledgeCatalog
|
|
|
421
389
|
*/
|
|
422
390
|
declare function extractGezkVerified(archivePath: string, destDir: string): Promise<KnowledgeCatalogManifest>;
|
|
423
391
|
|
|
424
|
-
/**
|
|
425
|
-
* node:sqlite loader shim. The repo's tsup/esbuild pipeline rewrites static
|
|
426
|
-
* `node:`-prefixed imports to their bare legacy names, which is harmless for
|
|
427
|
-
* `fs`/`crypto` but fatal for `sqlite` — the builtin exists ONLY under the
|
|
428
|
-
* `node:` prefix, so the bundled import resolves to a nonexistent npm
|
|
429
|
-
* package. The service hit the same wall and loads it by string at runtime
|
|
430
|
-
* (sqlite-driver.ts `nodeRequire('node:sqlite')`); this shim is that pattern
|
|
431
|
-
* with the types re-exported so call sites stay typed.
|
|
432
|
-
*/
|
|
433
|
-
|
|
434
|
-
declare const DatabaseSync: typeof DatabaseSync$1;
|
|
435
|
-
type DatabaseSync = DatabaseSync$1;
|
|
436
|
-
|
|
437
392
|
/**
|
|
438
393
|
* Read-only, immutable catalog database opener. Deliberately NOT the
|
|
439
394
|
* service's `openIndexDatabase`, whose behaviors are all wrong for a signed
|
|
@@ -876,61 +831,6 @@ declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalo
|
|
|
876
831
|
/** `[…](assets/…)` and `` targets in a body, deduplicated. */
|
|
877
832
|
declare function assetReferences(markdown: string): string[];
|
|
878
833
|
|
|
879
|
-
/**
|
|
880
|
-
* Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
|
|
881
|
-
* 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
|
|
882
|
-
* pass over them, a few milliseconds, needing nothing beyond plain SQLite
|
|
883
|
-
* to load — which is what lets any reader implement the format without a
|
|
884
|
-
* vector extension.
|
|
885
|
-
*
|
|
886
|
-
* Two scans share the bits:
|
|
887
|
-
* - `hammingTopK` — symmetric: the query is binarized too and rows are
|
|
888
|
-
* ranked by popcount(xor). The format's baseline (§2), and what the
|
|
889
|
-
* self-KNN smoke and the conformance kit exercise.
|
|
890
|
-
* - `asymmetricTopK` — the float query against each row's bits, ranked by
|
|
891
|
-
* Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
|
|
892
|
-
* embeddings roughly doubles the share of true neighbours a small
|
|
893
|
-
* candidate pool retains (measured on multilingual-e5-small: 22% → 39%
|
|
894
|
-
* of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
|
|
895
|
-
* bits). This is what `CatalogHandle` uses.
|
|
896
|
-
*/
|
|
897
|
-
interface ShardBitIndex {
|
|
898
|
-
/** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
|
|
899
|
-
bits: Uint8Array;
|
|
900
|
-
bytesPerRow: number;
|
|
901
|
-
rows: number;
|
|
902
|
-
}
|
|
903
|
-
interface HammingHit {
|
|
904
|
-
chunkId: number;
|
|
905
|
-
distance: number;
|
|
906
|
-
}
|
|
907
|
-
/**
|
|
908
|
-
* The K nearest rows by hamming distance, ascending, ties broken by chunk
|
|
909
|
-
* id. Selection is a histogram over the (small, integer) distance range, so
|
|
910
|
-
* the whole query is two linear passes and no heap.
|
|
911
|
-
*/
|
|
912
|
-
declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
|
|
913
|
-
interface AsymmetricHit {
|
|
914
|
-
chunkId: number;
|
|
915
|
-
score: number;
|
|
916
|
-
}
|
|
917
|
-
/**
|
|
918
|
-
* The K best rows by asymmetric score, descending, ties broken by chunk id.
|
|
919
|
-
*
|
|
920
|
-
* The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
|
|
921
|
-
* of 256 partial sums is built for each byte position, so a row costs
|
|
922
|
-
* `bytesPerRow` table reads and adds — about the popcount scan's cost. The
|
|
923
|
-
* top-K is kept in a bounded min-heap whose root is the worst kept row.
|
|
924
|
-
*
|
|
925
|
-
* For a `centered-sign` profile the caller passes `query − center`; for a
|
|
926
|
-
* plain `sign` profile the unit query. The bits are whatever the catalog
|
|
927
|
-
* stored; this function does not know or care which.
|
|
928
|
-
*
|
|
929
|
-
* `excludedChunkIds` drops rows outright — media rows, which the text lane
|
|
930
|
-
* leaves to its own exact scan rather than letting them take candidate slots.
|
|
931
|
-
*/
|
|
932
|
-
declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number, eligibleChunkIds?: ReadonlySet<number>, excludedChunkIds?: ReadonlySet<number>): AsymmetricHit[];
|
|
933
|
-
|
|
934
834
|
/**
|
|
935
835
|
* Provenance stamped into every manifest this package compiles. Only the
|
|
936
836
|
* package identity: node/platform details would make two builds of the same
|
|
@@ -1050,220 +950,6 @@ declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<s
|
|
|
1050
950
|
/** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
|
|
1051
951
|
declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
|
|
1052
952
|
|
|
1053
|
-
/**
|
|
1054
|
-
* The registered embedding profiles — the single source both the compiler
|
|
1055
|
-
* and the reader import. A profile is the FULL vector-space identity:
|
|
1056
|
-
* readers refuse to mix vectors across profile ids, and any change to model,
|
|
1057
|
-
* instructions, or encoding is a NEW id.
|
|
1058
|
-
*
|
|
1059
|
-
* Revisions are the Hugging Face commit hashes the profiles were frozen at
|
|
1060
|
-
* (2026-08-20). The artifact digests (recorded 2026-09-03) are the sha256 of
|
|
1061
|
-
* the exact files at those revisions — the Hub's LFS object ids for the ONNX
|
|
1062
|
-
* graphs and the e5 tokenizer, a hashed download for the bge tokenizer — and
|
|
1063
|
-
* name the full-precision graph explicitly, so the precision the vectors were
|
|
1064
|
-
* produced at is a pinned fact rather than a runtime default. Every embedder
|
|
1065
|
-
* built from a profile verifies the files it loaded against these digests.
|
|
1066
|
-
*/
|
|
1067
|
-
|
|
1068
|
-
/**
|
|
1069
|
-
* The public profile's first revision: multilingual, so world catalogs serve
|
|
1070
|
-
* every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
|
|
1071
|
-
* vectors, centered stage-1 bits); catalogs built with this revision stay
|
|
1072
|
-
* readable and searchable, at the lower stage-1 recall of raw sign bits.
|
|
1073
|
-
*/
|
|
1074
|
-
declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
|
|
1075
|
-
/**
|
|
1076
|
-
* The public profile, revision 2: the model, files, instructions and int8
|
|
1077
|
-
* rerank of revision 1, with the stage-1 sign bits taken from
|
|
1078
|
-
* `vector − center` (`centered-sign`). e5-small vectors share one dominant
|
|
1079
|
-
* direction, so raw sign bits mostly encode that direction rather than the
|
|
1080
|
-
* passage and the hamming pre-filter loses most true neighbours; centering
|
|
1081
|
-
* on the pinned corpus mean restores them (see e5-small-center.ts for the
|
|
1082
|
-
* provenance and the measurements). A query vector for revision 1 is a
|
|
1083
|
-
* valid query vector for revision 2 — `sameVectorSpace` says so — and the
|
|
1084
|
-
* reader centers it itself from the catalog's profile echo.
|
|
1085
|
-
*/
|
|
1086
|
-
declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
|
|
1087
|
-
/** The local-build default — matches gezel's shipped project embedder. */
|
|
1088
|
-
declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
|
|
1089
|
-
declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
|
|
1090
|
-
/** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
|
|
1091
|
-
declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
|
|
1092
|
-
declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
|
|
1093
|
-
/** What a profile's query model costs to download, or null for an unregistered profile. */
|
|
1094
|
-
declare function knowledgeProfileQueryModelBytes(id: string): number | null;
|
|
1095
|
-
/** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
|
|
1096
|
-
declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
|
|
1097
|
-
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
1098
|
-
declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
|
|
1099
|
-
id: string;
|
|
1100
|
-
model: {
|
|
1101
|
-
repo: string;
|
|
1102
|
-
revision: string;
|
|
1103
|
-
onnxFile?: string | undefined;
|
|
1104
|
-
onnxDigest?: string | undefined;
|
|
1105
|
-
files?: {
|
|
1106
|
-
path: string;
|
|
1107
|
-
digest?: string | undefined;
|
|
1108
|
-
}[] | undefined;
|
|
1109
|
-
};
|
|
1110
|
-
tokenizer: {
|
|
1111
|
-
kind: string;
|
|
1112
|
-
file?: string | undefined;
|
|
1113
|
-
digest?: string | undefined;
|
|
1114
|
-
};
|
|
1115
|
-
pooling: "mean" | "cls" | "last";
|
|
1116
|
-
normalized: boolean;
|
|
1117
|
-
dimensions: number;
|
|
1118
|
-
maxTokens: number;
|
|
1119
|
-
queryInstruction: string;
|
|
1120
|
-
passageInstruction: string;
|
|
1121
|
-
vectorEncoding: "bit+int8";
|
|
1122
|
-
distance: {
|
|
1123
|
-
stage1: "hamming";
|
|
1124
|
-
stage2: "cosine";
|
|
1125
|
-
};
|
|
1126
|
-
quantization: {
|
|
1127
|
-
int8: {
|
|
1128
|
-
method: "symmetric-linear";
|
|
1129
|
-
scale: 127;
|
|
1130
|
-
};
|
|
1131
|
-
binary: {
|
|
1132
|
-
method: "sign" | "centered-sign";
|
|
1133
|
-
threshold: 0;
|
|
1134
|
-
packing: "lsb-first";
|
|
1135
|
-
center?: number[] | undefined;
|
|
1136
|
-
};
|
|
1137
|
-
};
|
|
1138
|
-
truncation?: {
|
|
1139
|
-
method: "prefix";
|
|
1140
|
-
sourceDimensions: number;
|
|
1141
|
-
} | undefined;
|
|
1142
|
-
media?: {
|
|
1143
|
-
image?: {
|
|
1144
|
-
encoder: {
|
|
1145
|
-
onnxFile: string;
|
|
1146
|
-
onnxDigest?: string | undefined;
|
|
1147
|
-
files?: {
|
|
1148
|
-
path: string;
|
|
1149
|
-
digest?: string | undefined;
|
|
1150
|
-
}[] | undefined;
|
|
1151
|
-
};
|
|
1152
|
-
tokenBudget: number;
|
|
1153
|
-
resample: "bilinear" | "bicubic";
|
|
1154
|
-
alpha: "composite-white";
|
|
1155
|
-
} | undefined;
|
|
1156
|
-
video?: {
|
|
1157
|
-
framesPerSecond: number;
|
|
1158
|
-
maxFrames: number;
|
|
1159
|
-
tokenBudgetPerFrame: number;
|
|
1160
|
-
} | undefined;
|
|
1161
|
-
audio?: {
|
|
1162
|
-
encoder: {
|
|
1163
|
-
onnxFile: string;
|
|
1164
|
-
onnxDigest?: string | undefined;
|
|
1165
|
-
files?: {
|
|
1166
|
-
path: string;
|
|
1167
|
-
digest?: string | undefined;
|
|
1168
|
-
}[] | undefined;
|
|
1169
|
-
};
|
|
1170
|
-
sampleRate: number;
|
|
1171
|
-
channels: 1;
|
|
1172
|
-
maxWindowMs: number;
|
|
1173
|
-
} | undefined;
|
|
1174
|
-
} | undefined;
|
|
1175
|
-
};
|
|
1176
|
-
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
1177
|
-
declare const GEZEL_BGE_SMALL_EN_V15_1: {
|
|
1178
|
-
id: string;
|
|
1179
|
-
model: {
|
|
1180
|
-
repo: string;
|
|
1181
|
-
revision: string;
|
|
1182
|
-
onnxFile?: string | undefined;
|
|
1183
|
-
onnxDigest?: string | undefined;
|
|
1184
|
-
files?: {
|
|
1185
|
-
path: string;
|
|
1186
|
-
digest?: string | undefined;
|
|
1187
|
-
}[] | undefined;
|
|
1188
|
-
};
|
|
1189
|
-
tokenizer: {
|
|
1190
|
-
kind: string;
|
|
1191
|
-
file?: string | undefined;
|
|
1192
|
-
digest?: string | undefined;
|
|
1193
|
-
};
|
|
1194
|
-
pooling: "mean" | "cls" | "last";
|
|
1195
|
-
normalized: boolean;
|
|
1196
|
-
dimensions: number;
|
|
1197
|
-
maxTokens: number;
|
|
1198
|
-
queryInstruction: string;
|
|
1199
|
-
passageInstruction: string;
|
|
1200
|
-
vectorEncoding: "bit+int8";
|
|
1201
|
-
distance: {
|
|
1202
|
-
stage1: "hamming";
|
|
1203
|
-
stage2: "cosine";
|
|
1204
|
-
};
|
|
1205
|
-
quantization: {
|
|
1206
|
-
int8: {
|
|
1207
|
-
method: "symmetric-linear";
|
|
1208
|
-
scale: 127;
|
|
1209
|
-
};
|
|
1210
|
-
binary: {
|
|
1211
|
-
method: "sign" | "centered-sign";
|
|
1212
|
-
threshold: 0;
|
|
1213
|
-
packing: "lsb-first";
|
|
1214
|
-
center?: number[] | undefined;
|
|
1215
|
-
};
|
|
1216
|
-
};
|
|
1217
|
-
truncation?: {
|
|
1218
|
-
method: "prefix";
|
|
1219
|
-
sourceDimensions: number;
|
|
1220
|
-
} | undefined;
|
|
1221
|
-
media?: {
|
|
1222
|
-
image?: {
|
|
1223
|
-
encoder: {
|
|
1224
|
-
onnxFile: string;
|
|
1225
|
-
onnxDigest?: string | undefined;
|
|
1226
|
-
files?: {
|
|
1227
|
-
path: string;
|
|
1228
|
-
digest?: string | undefined;
|
|
1229
|
-
}[] | undefined;
|
|
1230
|
-
};
|
|
1231
|
-
tokenBudget: number;
|
|
1232
|
-
resample: "bilinear" | "bicubic";
|
|
1233
|
-
alpha: "composite-white";
|
|
1234
|
-
} | undefined;
|
|
1235
|
-
video?: {
|
|
1236
|
-
framesPerSecond: number;
|
|
1237
|
-
maxFrames: number;
|
|
1238
|
-
tokenBudgetPerFrame: number;
|
|
1239
|
-
} | undefined;
|
|
1240
|
-
audio?: {
|
|
1241
|
-
encoder: {
|
|
1242
|
-
onnxFile: string;
|
|
1243
|
-
onnxDigest?: string | undefined;
|
|
1244
|
-
files?: {
|
|
1245
|
-
path: string;
|
|
1246
|
-
digest?: string | undefined;
|
|
1247
|
-
}[] | undefined;
|
|
1248
|
-
};
|
|
1249
|
-
sampleRate: number;
|
|
1250
|
-
channels: 1;
|
|
1251
|
-
maxWindowMs: number;
|
|
1252
|
-
} | undefined;
|
|
1253
|
-
} | undefined;
|
|
1254
|
-
};
|
|
1255
|
-
/** @deprecated gezk 0.4 names; removed with the next minor. */
|
|
1256
|
-
declare const GEZEL_MARKDOWN_CHUNKS_2: {
|
|
1257
|
-
id: string;
|
|
1258
|
-
unit: "chars" | "tokens";
|
|
1259
|
-
tokenizer: "profile" | "none";
|
|
1260
|
-
target: number;
|
|
1261
|
-
overlap: number;
|
|
1262
|
-
contextHeader: {
|
|
1263
|
-
max: number;
|
|
1264
|
-
};
|
|
1265
|
-
};
|
|
1266
|
-
|
|
1267
953
|
/**
|
|
1268
954
|
* Verifies the model files an embedder loaded against the digests its
|
|
1269
955
|
* profile pins. transformers.js caches every fetched file under a key that
|
|
@@ -1808,4 +1494,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
|
|
|
1808
1494
|
|
|
1809
1495
|
declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
|
|
1810
1496
|
|
|
1811
|
-
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole,
|
|
1497
|
+
export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, type MarkdownCatalogSource, MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, catalogFormatVersion, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
|
package/dist/index.js
CHANGED
|
@@ -181,6 +181,29 @@ function splitOversizeLine(line, opts) {
|
|
|
181
181
|
return out;
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
+
// src/chunking/embed-input.ts
|
|
185
|
+
function buildContextHeader(title, headingPath, maxTokens, countTokens) {
|
|
186
|
+
const path = headingPath.filter((h) => h !== title);
|
|
187
|
+
let header = path.length > 0 ? `${title}
|
|
188
|
+
${path.join(" > ")}
|
|
189
|
+
` : `${title}
|
|
190
|
+
`;
|
|
191
|
+
if (countTokens(header) > maxTokens) {
|
|
192
|
+
header = `${title}
|
|
193
|
+
`;
|
|
194
|
+
if (countTokens(header) > maxTokens) header = "";
|
|
195
|
+
}
|
|
196
|
+
return header;
|
|
197
|
+
}
|
|
198
|
+
function passageEmbedInput(input) {
|
|
199
|
+
return `${input.passageInstruction}${buildContextHeader(
|
|
200
|
+
input.title,
|
|
201
|
+
input.headingPath,
|
|
202
|
+
input.headerMaxTokens,
|
|
203
|
+
input.countTokens
|
|
204
|
+
)}${input.text}`;
|
|
205
|
+
}
|
|
206
|
+
|
|
184
207
|
// src/compiler/compile.ts
|
|
185
208
|
import { createHash as createHash3 } from "crypto";
|
|
186
209
|
import {
|
|
@@ -695,12 +718,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
695
718
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
696
719
|
const embedded = totalRows <= shardTarget;
|
|
697
720
|
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
698
|
-
const embedInput = (p, chunk) =>
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
721
|
+
const embedInput = (p, chunk) => passageEmbedInput({
|
|
722
|
+
passageInstruction: profile.passageInstruction,
|
|
723
|
+
title: p.doc.title,
|
|
724
|
+
headingPath: chunk.headingPath,
|
|
725
|
+
text: chunk.text,
|
|
726
|
+
headerMaxTokens: opts.chunkingProfile.contextHeader.max,
|
|
727
|
+
countTokens: opts.countTokens
|
|
728
|
+
});
|
|
704
729
|
rmSync(opts.workDir, { recursive: true, force: true });
|
|
705
730
|
mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
|
|
706
731
|
const stagedPath = join(opts.workDir, "staged-vectors.f32");
|
|
@@ -1351,19 +1376,6 @@ function defaultLicenseNotice(opts) {
|
|
|
1351
1376
|
function normalizeMarkdown(md) {
|
|
1352
1377
|
return md.replace(/\r\n/g, "\n").normalize("NFC");
|
|
1353
1378
|
}
|
|
1354
|
-
function buildContextHeader(title, headingPath, maxTokens, countTokens) {
|
|
1355
|
-
const path = headingPath.filter((h) => h !== title);
|
|
1356
|
-
let header = path.length > 0 ? `${title}
|
|
1357
|
-
${path.join(" > ")}
|
|
1358
|
-
` : `${title}
|
|
1359
|
-
`;
|
|
1360
|
-
if (countTokens(header) > maxTokens) {
|
|
1361
|
-
header = `${title}
|
|
1362
|
-
`;
|
|
1363
|
-
if (countTokens(header) > maxTokens) header = "";
|
|
1364
|
-
}
|
|
1365
|
-
return header;
|
|
1366
|
-
}
|
|
1367
1379
|
async function stageVectors(args) {
|
|
1368
1380
|
const { prepared, profile } = args;
|
|
1369
1381
|
const { dimensions } = profile;
|
|
@@ -5230,6 +5242,38 @@ var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
|
|
|
5230
5242
|
var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
5231
5243
|
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
5232
5244
|
|
|
5245
|
+
// src/profiles/vector-floors.ts
|
|
5246
|
+
var KNOWLEDGE_VECTOR_FLOORS = {
|
|
5247
|
+
// Handboek 1.1.2: answers ≥ 0.687, off-topic ≤ 0.624.
|
|
5248
|
+
"bendyline/handboek": 0.65,
|
|
5249
|
+
// The project index's measured floor for the daemon's own embedder
|
|
5250
|
+
// (index-store VECTOR_ARM_MIN_SIMILARITY) — what a folder-built catalog gets.
|
|
5251
|
+
"bge-small-en-v1.5@1": 0.55,
|
|
5252
|
+
// Wikipedia Food & Drink 2026.4.3: answers ≥ 0.869; the scale is compressed
|
|
5253
|
+
// and 2 of 25 off-topic prompts still reach it, so grounding and the
|
|
5254
|
+
// relevance model do the rest.
|
|
5255
|
+
"multilingual-e5-small@2": 0.865,
|
|
5256
|
+
// Revision 1 has the same model, instructions and int8 vectors — only its
|
|
5257
|
+
// stage-1 sign bits differ — so its rerank cosine is on the same scale.
|
|
5258
|
+
"multilingual-e5-small@1": 0.865,
|
|
5259
|
+
// Handboek 1.2.2 + a Food & Drink sample re-embedded in this profile (the
|
|
5260
|
+
// labelled calibration queries): answers ≥ 0.762 in both, off-topic ≤ 0.706,
|
|
5261
|
+
// a query's best chunk in the OTHER catalog ≤ 0.715. A wide gap; the scale
|
|
5262
|
+
// is far less compressed than e5's.
|
|
5263
|
+
"embeddinggemma-2-512@1": 0.735,
|
|
5264
|
+
// Media rows of the same profile, workspace and catalogs alike
|
|
5265
|
+
// (evals/src/retrieval-bench/MEDIA-BENCH-2026-10-07.md). Each sits a step
|
|
5266
|
+
// above the lowest floor no off-topic prompt reached on the bench corpus,
|
|
5267
|
+
// because a real library is larger and its off-topic maximum higher.
|
|
5268
|
+
// COCO photos: no off-topic hit from 0.66; 293 of 300 answers keep their photo.
|
|
5269
|
+
"embeddinggemma-2-512@1#image": 0.67,
|
|
5270
|
+
// MSR-VTT clips: no off-topic hit from 0.65; 54 of 60 answers clear 0.65 and 0.66.
|
|
5271
|
+
"embeddinggemma-2-512@1#video": 0.66,
|
|
5272
|
+
// ESC-50 sounds: scores bunch (a wrong clip clears 0.66 for most queries),
|
|
5273
|
+
// so the floor does less here than ranking; 47 of 50 answers clear 0.68.
|
|
5274
|
+
"embeddinggemma-2-512@1#audio": 0.68
|
|
5275
|
+
};
|
|
5276
|
+
|
|
5233
5277
|
// src/embedding/profile-embedder.ts
|
|
5234
5278
|
import {
|
|
5235
5279
|
embeddingProfileArtifacts as embeddingProfileArtifacts3,
|
|
@@ -7283,6 +7327,7 @@ export {
|
|
|
7283
7327
|
GezkArchiveError,
|
|
7284
7328
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
7285
7329
|
KNOWLEDGE_TOOLCHAIN,
|
|
7330
|
+
KNOWLEDGE_VECTOR_FLOORS,
|
|
7286
7331
|
KnowledgeRegistryFetchError,
|
|
7287
7332
|
LICENSE_NOTICE_PATH,
|
|
7288
7333
|
MANIFEST_PATH,
|
|
@@ -7319,6 +7364,7 @@ export {
|
|
|
7319
7364
|
assetReferences,
|
|
7320
7365
|
assignOutlineOrders,
|
|
7321
7366
|
asymmetricTopK,
|
|
7367
|
+
buildContextHeader,
|
|
7322
7368
|
canonicalizeJson3 as canonicalizeJson,
|
|
7323
7369
|
catalogFormatVersion,
|
|
7324
7370
|
centerVector,
|
|
@@ -7363,6 +7409,7 @@ export {
|
|
|
7363
7409
|
parseGitbookSummary,
|
|
7364
7410
|
parseJupyterBookToc,
|
|
7365
7411
|
parseMkdocsNav,
|
|
7412
|
+
passageEmbedInput,
|
|
7366
7413
|
pinnedExternalDataChunks,
|
|
7367
7414
|
pinnedLoadFiles,
|
|
7368
7415
|
prefetchExternalData,
|