@bendyline/gezel-knowledge 1.2.11 → 1.2.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -3,8 +3,23 @@ import { KnowledgeAssetModality, KnowledgeEmbeddingProfile, CatalogDocument, Kno
3
3
  export { BODY_CODEC_MIN_BYTES, CatalogDocument, GEZK_APPLICATION_ID, GEZK_FORMAT_GENERATIONS, GEZK_FORMAT_VERSION, GEZK_INDEX_SCHEMA_VERSION, GEZK_MIME_TYPE, GEZK_SUPPORTED_FORMAT_VERSIONS, GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS, GezkFormatVersion, GezkIndexSchemaVersion, KnowledgeCatalogManifest, KnowledgeChunkingProfile, KnowledgeEmbeddingProfile, KnowledgeRegistryEntry, KnowledgeRegistryIndex, KnowledgeTocReference, LICENSE_NOTICE_PATH, MANIFEST_PATH, MAX_KNOWLEDGE_DOCUMENT_BYTES, MAX_KNOWLEDGE_DOCUMENT_META_BYTES, MAX_KNOWLEDGE_TOPIC_DEPTH, MIMETYPE_PATH, QUERY_STOP_WORDS, README_PATH, ROUTER_DB_PATH, ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, SOURCE_NOTICES_PATH, ZIP_FIXED_MTIME, canonicalizeJson, centerVector, documentSlug, formatAtLeast, isSupportedFormatVersion, isSupportedIndexSchemaVersion, l2Normalize, profileUnitVector, quantizeBinary, quantizeBinaryForProfile, quantizeInt8, rerankScore, shardDdlFor } from '@bendyline/gezk';
4
4
  import { KnowledgeTrustAnchor } from '@bendyline/gezk/node';
5
5
  export { KnowledgeTrustAnchor, ManifestSignatureVerdict, chunkContentHash, chunkUid, generateKnowledgeSigningKeyPair, knowledgeKeyId, manifestSigningPayload, signManifest, signRegistryIndex, verifyManifestSignature, verifyRegistryIndex } from '@bendyline/gezk/node';
6
+ import { g as MarkdownChunk } from './registry-xi2R42Av.js';
7
+ export { A as AsymmetricHit, B as BGE_SMALL_EN_V15_1, C as CharChunkerOptions, a as ChunkerOptions, E as EMBEDDINGGEMMA_2_512_1, G as GEZEL_BGE_SMALL_EN_V15_1, b as GEZEL_MARKDOWN_CHUNKS_2, c as GEZEL_MULTILINGUAL_E5_SMALL_1, H as HammingHit, K as KNOWLEDGE_EMBEDDING_PROFILES, d as KNOWLEDGE_VECTOR_FLOORS, M as MARKDOWN_CHUNKS_2, e as MULTILINGUAL_E5_SMALL_1, f as MULTILINGUAL_E5_SMALL_2, P as PassageEmbedInput, S as ShardBitIndex, T as TokenChunkerOptions, h as asymmetricTopK, i as buildContextHeader, j as chunkMarkdownProfile, m as hammingTopK, k as knowledgeEmbeddingProfile, l as knowledgeProfileQueryModelBytes, p as passageEmbedInput } from './registry-xi2R42Av.js';
6
8
  import { DatabaseSync as DatabaseSync$1 } from 'node:sqlite';
7
9
 
10
+ /**
11
+ * node:sqlite loader shim. The repo's tsup/esbuild pipeline rewrites static
12
+ * `node:`-prefixed imports to their bare legacy names, which is harmless for
13
+ * `fs`/`crypto` but fatal for `sqlite` — the builtin exists ONLY under the
14
+ * `node:` prefix, so the bundled import resolves to a nonexistent npm
15
+ * package. The service hit the same wall and loads it by string at runtime
16
+ * (sqlite-driver.ts `nodeRequire('node:sqlite')`); this shim is that pattern
17
+ * with the types re-exported so call sites stay typed.
18
+ */
19
+
20
+ declare const DatabaseSync: typeof DatabaseSync$1;
21
+ type DatabaseSync = DatabaseSync$1;
22
+
8
23
  /** Toolchain knobs. Format-level constants come from @bendyline/gezk. */
9
24
 
10
25
  /** Compiler shard sizing (§3.1). */
@@ -32,53 +47,6 @@ declare const RERANK_CANDIDATES_MIN = 1024;
32
47
  declare const RERANK_CANDIDATES_MAX = 4096;
33
48
  declare function rerankK(finalK?: number, chunkCount?: number): number;
34
49
 
35
- /**
36
- * The canonical markdown chunker, parameterized over the two frozen
37
- * profiles (the gezk spec §7):
38
- *
39
- * - `markdown-chunks@1` — unit 'chars', 4000/400: byte-for-byte the
40
- * algorithm lifted from the project indexer
41
- * (packages/service/src/index-store/docs.ts `chunkMarkdown` /
42
- * `chunkLineRange`), so the service can later re-point here with zero
43
- * behavior change.
44
- * - `markdown-chunks@2` — unit 'tokens', 420/64: the knowledge
45
- * profile. Token counting is injected (`countTokens` — the embedding
46
- * profile's own tokenizer), packing is per-line with sentence-level
47
- * splitting for oversized lines, and every chunk carries a structural
48
- * `headingPath` (root→nearest, the section's own heading last).
49
- *
50
- * Pure functions, no I/O, no dependencies — importable by the compiler, the
51
- * service indexer, and tests alike.
52
- */
53
- interface MarkdownChunk {
54
- kind: 'doc' | 'preamble' | 'section';
55
- /** 1-based inclusive source line span. */
56
- lineStart: number;
57
- lineEnd: number;
58
- text: string;
59
- /** Raw heading texts, root → nearest (empty for preamble/doc chunks). */
60
- headingPath: string[];
61
- }
62
- interface CharChunkerOptions {
63
- unit: 'chars';
64
- /** Max chunk size in characters (profile @1: 4000). */
65
- target: number;
66
- /** Overlap replayed between adjacent chunks, in characters (@1: 400). */
67
- overlap: number;
68
- }
69
- interface TokenChunkerOptions {
70
- unit: 'tokens';
71
- /** Target chunk size in tokens (profile @2: 420). */
72
- target: number;
73
- /** Overlap in tokens (@2: 64). */
74
- overlap: number;
75
- /** The embedding profile's tokenizer — deterministic given its pinned digest. */
76
- countTokens: (text: string) => number;
77
- }
78
- type ChunkerOptions = CharChunkerOptions | TokenChunkerOptions;
79
- /** Split markdown into heading-aware, bounded, overlapping chunks. */
80
- declare function chunkMarkdownProfile(md: string, opts: ChunkerOptions): MarkdownChunk[];
81
-
82
50
  interface CompileAsset {
83
51
  /** Archive path, assets/…, matching the format's asset path grammar. */
84
52
  path: string;
@@ -421,19 +389,6 @@ declare function readGezkManifest(archivePath: string): Promise<KnowledgeCatalog
421
389
  */
422
390
  declare function extractGezkVerified(archivePath: string, destDir: string): Promise<KnowledgeCatalogManifest>;
423
391
 
424
- /**
425
- * node:sqlite loader shim. The repo's tsup/esbuild pipeline rewrites static
426
- * `node:`-prefixed imports to their bare legacy names, which is harmless for
427
- * `fs`/`crypto` but fatal for `sqlite` — the builtin exists ONLY under the
428
- * `node:` prefix, so the bundled import resolves to a nonexistent npm
429
- * package. The service hit the same wall and loads it by string at runtime
430
- * (sqlite-driver.ts `nodeRequire('node:sqlite')`); this shim is that pattern
431
- * with the types re-exported so call sites stay typed.
432
- */
433
-
434
- declare const DatabaseSync: typeof DatabaseSync$1;
435
- type DatabaseSync = DatabaseSync$1;
436
-
437
392
  /**
438
393
  * Read-only, immutable catalog database opener. Deliberately NOT the
439
394
  * service's `openIndexDatabase`, whose behaviors are all wrong for a signed
@@ -876,61 +831,6 @@ declare function validateExtractedCatalog(rootDir: string, opts?: ValidateCatalo
876
831
  /** `[…](assets/…)` and `![…](assets/…)` targets in a body, deduplicated. */
877
832
  declare function assetReferences(markdown: string): string[];
878
833
 
879
- /**
880
- * Stage-1 retrieval over a shard's sign-bit vectors, in memory. A full
881
- * 200k-chunk shard is 9.6 MB of bits, and either scan below is one linear
882
- * pass over them, a few milliseconds, needing nothing beyond plain SQLite
883
- * to load — which is what lets any reader implement the format without a
884
- * vector extension.
885
- *
886
- * Two scans share the bits:
887
- * - `hammingTopK` — symmetric: the query is binarized too and rows are
888
- * ranked by popcount(xor). The format's baseline (§2), and what the
889
- * self-KNN smoke and the conformance kit exercise.
890
- * - `asymmetricTopK` — the float query against each row's bits, ranked by
891
- * Σ q[d]·(bit ? +1 : −1). Keeps the query's magnitudes, which on real
892
- * embeddings roughly doubles the share of true neighbours a small
893
- * candidate pool retains (measured on multilingual-e5-small: 22% → 39%
894
- * of the exact top-24 at K = 192 with raw bits, 73% → 88% with centered
895
- * bits). This is what `CatalogHandle` uses.
896
- */
897
- interface ShardBitIndex {
898
- /** Row-major sign-bit rows, `bytesPerRow` each; row i holds chunk_id i + 1. */
899
- bits: Uint8Array;
900
- bytesPerRow: number;
901
- rows: number;
902
- }
903
- interface HammingHit {
904
- chunkId: number;
905
- distance: number;
906
- }
907
- /**
908
- * The K nearest rows by hamming distance, ascending, ties broken by chunk
909
- * id. Selection is a histogram over the (small, integer) distance range, so
910
- * the whole query is two linear passes and no heap.
911
- */
912
- declare function hammingTopK(index: ShardBitIndex, query: Uint8Array, k: number): HammingHit[];
913
- interface AsymmetricHit {
914
- chunkId: number;
915
- score: number;
916
- }
917
- /**
918
- * The K best rows by asymmetric score, descending, ties broken by chunk id.
919
- *
920
- * The score of a row is Σ_d query[d] · (bit_d ? +1 : −1). Per query a table
921
- * of 256 partial sums is built for each byte position, so a row costs
922
- * `bytesPerRow` table reads and adds — about the popcount scan's cost. The
923
- * top-K is kept in a bounded min-heap whose root is the worst kept row.
924
- *
925
- * For a `centered-sign` profile the caller passes `query − center`; for a
926
- * plain `sign` profile the unit query. The bits are whatever the catalog
927
- * stored; this function does not know or care which.
928
- *
929
- * `excludedChunkIds` drops rows outright — media rows, which the text lane
930
- * leaves to its own exact scan rather than letting them take candidate slots.
931
- */
932
- declare function asymmetricTopK(index: ShardBitIndex, query: ArrayLike<number>, k: number, eligibleChunkIds?: ReadonlySet<number>, excludedChunkIds?: ReadonlySet<number>): AsymmetricHit[];
933
-
934
834
  /**
935
835
  * Provenance stamped into every manifest this package compiles. Only the
936
836
  * package identity: node/platform details would make two builds of the same
@@ -1050,220 +950,6 @@ declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<s
1050
950
  /** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
1051
951
  declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
1052
952
 
1053
- /**
1054
- * The registered embedding profiles — the single source both the compiler
1055
- * and the reader import. A profile is the FULL vector-space identity:
1056
- * readers refuse to mix vectors across profile ids, and any change to model,
1057
- * instructions, or encoding is a NEW id.
1058
- *
1059
- * Revisions are the Hugging Face commit hashes the profiles were frozen at
1060
- * (2026-08-20). The artifact digests (recorded 2026-09-03) are the sha256 of
1061
- * the exact files at those revisions — the Hub's LFS object ids for the ONNX
1062
- * graphs and the e5 tokenizer, a hashed download for the bge tokenizer — and
1063
- * name the full-precision graph explicitly, so the precision the vectors were
1064
- * produced at is a pinned fact rather than a runtime default. Every embedder
1065
- * built from a profile verifies the files it loaded against these digests.
1066
- */
1067
-
1068
- /**
1069
- * The public profile's first revision: multilingual, so world catalogs serve
1070
- * every reader. Superseded for new builds by MULTILINGUAL_E5_SMALL_2 (same
1071
- * vectors, centered stage-1 bits); catalogs built with this revision stay
1072
- * readable and searchable, at the lower stage-1 recall of raw sign bits.
1073
- */
1074
- declare const MULTILINGUAL_E5_SMALL_1: KnowledgeEmbeddingProfile;
1075
- /**
1076
- * The public profile, revision 2: the model, files, instructions and int8
1077
- * rerank of revision 1, with the stage-1 sign bits taken from
1078
- * `vector − center` (`centered-sign`). e5-small vectors share one dominant
1079
- * direction, so raw sign bits mostly encode that direction rather than the
1080
- * passage and the hamming pre-filter loses most true neighbours; centering
1081
- * on the pinned corpus mean restores them (see e5-small-center.ts for the
1082
- * provenance and the measurements). A query vector for revision 1 is a
1083
- * valid query vector for revision 2 — `sameVectorSpace` says so — and the
1084
- * reader centers it itself from the catalog's profile echo.
1085
- */
1086
- declare const MULTILINGUAL_E5_SMALL_2: KnowledgeEmbeddingProfile;
1087
- /** The local-build default — matches gezel's shipped project embedder. */
1088
- declare const BGE_SMALL_EN_V15_1: KnowledgeEmbeddingProfile;
1089
- declare const EMBEDDINGGEMMA_2_512_1: KnowledgeEmbeddingProfile;
1090
- /** Newest revision of a model first: `daemonEmbedderPin` picks the first match by repo. */
1091
- declare const KNOWLEDGE_EMBEDDING_PROFILES: readonly KnowledgeEmbeddingProfile[];
1092
- declare function knowledgeEmbeddingProfile(id: string): KnowledgeEmbeddingProfile | null;
1093
- /** What a profile's query model costs to download, or null for an unregistered profile. */
1094
- declare function knowledgeProfileQueryModelBytes(id: string): number | null;
1095
- /** The knowledge chunking profile: token-bounded, heading-aware Markdown sections. */
1096
- declare const MARKDOWN_CHUNKS_2: KnowledgeChunkingProfile;
1097
- /** @deprecated gezk 0.4 names; removed with the next minor. */
1098
- declare const GEZEL_MULTILINGUAL_E5_SMALL_1: {
1099
- id: string;
1100
- model: {
1101
- repo: string;
1102
- revision: string;
1103
- onnxFile?: string | undefined;
1104
- onnxDigest?: string | undefined;
1105
- files?: {
1106
- path: string;
1107
- digest?: string | undefined;
1108
- }[] | undefined;
1109
- };
1110
- tokenizer: {
1111
- kind: string;
1112
- file?: string | undefined;
1113
- digest?: string | undefined;
1114
- };
1115
- pooling: "mean" | "cls" | "last";
1116
- normalized: boolean;
1117
- dimensions: number;
1118
- maxTokens: number;
1119
- queryInstruction: string;
1120
- passageInstruction: string;
1121
- vectorEncoding: "bit+int8";
1122
- distance: {
1123
- stage1: "hamming";
1124
- stage2: "cosine";
1125
- };
1126
- quantization: {
1127
- int8: {
1128
- method: "symmetric-linear";
1129
- scale: 127;
1130
- };
1131
- binary: {
1132
- method: "sign" | "centered-sign";
1133
- threshold: 0;
1134
- packing: "lsb-first";
1135
- center?: number[] | undefined;
1136
- };
1137
- };
1138
- truncation?: {
1139
- method: "prefix";
1140
- sourceDimensions: number;
1141
- } | undefined;
1142
- media?: {
1143
- image?: {
1144
- encoder: {
1145
- onnxFile: string;
1146
- onnxDigest?: string | undefined;
1147
- files?: {
1148
- path: string;
1149
- digest?: string | undefined;
1150
- }[] | undefined;
1151
- };
1152
- tokenBudget: number;
1153
- resample: "bilinear" | "bicubic";
1154
- alpha: "composite-white";
1155
- } | undefined;
1156
- video?: {
1157
- framesPerSecond: number;
1158
- maxFrames: number;
1159
- tokenBudgetPerFrame: number;
1160
- } | undefined;
1161
- audio?: {
1162
- encoder: {
1163
- onnxFile: string;
1164
- onnxDigest?: string | undefined;
1165
- files?: {
1166
- path: string;
1167
- digest?: string | undefined;
1168
- }[] | undefined;
1169
- };
1170
- sampleRate: number;
1171
- channels: 1;
1172
- maxWindowMs: number;
1173
- } | undefined;
1174
- } | undefined;
1175
- };
1176
- /** @deprecated gezk 0.4 names; removed with the next minor. */
1177
- declare const GEZEL_BGE_SMALL_EN_V15_1: {
1178
- id: string;
1179
- model: {
1180
- repo: string;
1181
- revision: string;
1182
- onnxFile?: string | undefined;
1183
- onnxDigest?: string | undefined;
1184
- files?: {
1185
- path: string;
1186
- digest?: string | undefined;
1187
- }[] | undefined;
1188
- };
1189
- tokenizer: {
1190
- kind: string;
1191
- file?: string | undefined;
1192
- digest?: string | undefined;
1193
- };
1194
- pooling: "mean" | "cls" | "last";
1195
- normalized: boolean;
1196
- dimensions: number;
1197
- maxTokens: number;
1198
- queryInstruction: string;
1199
- passageInstruction: string;
1200
- vectorEncoding: "bit+int8";
1201
- distance: {
1202
- stage1: "hamming";
1203
- stage2: "cosine";
1204
- };
1205
- quantization: {
1206
- int8: {
1207
- method: "symmetric-linear";
1208
- scale: 127;
1209
- };
1210
- binary: {
1211
- method: "sign" | "centered-sign";
1212
- threshold: 0;
1213
- packing: "lsb-first";
1214
- center?: number[] | undefined;
1215
- };
1216
- };
1217
- truncation?: {
1218
- method: "prefix";
1219
- sourceDimensions: number;
1220
- } | undefined;
1221
- media?: {
1222
- image?: {
1223
- encoder: {
1224
- onnxFile: string;
1225
- onnxDigest?: string | undefined;
1226
- files?: {
1227
- path: string;
1228
- digest?: string | undefined;
1229
- }[] | undefined;
1230
- };
1231
- tokenBudget: number;
1232
- resample: "bilinear" | "bicubic";
1233
- alpha: "composite-white";
1234
- } | undefined;
1235
- video?: {
1236
- framesPerSecond: number;
1237
- maxFrames: number;
1238
- tokenBudgetPerFrame: number;
1239
- } | undefined;
1240
- audio?: {
1241
- encoder: {
1242
- onnxFile: string;
1243
- onnxDigest?: string | undefined;
1244
- files?: {
1245
- path: string;
1246
- digest?: string | undefined;
1247
- }[] | undefined;
1248
- };
1249
- sampleRate: number;
1250
- channels: 1;
1251
- maxWindowMs: number;
1252
- } | undefined;
1253
- } | undefined;
1254
- };
1255
- /** @deprecated gezk 0.4 names; removed with the next minor. */
1256
- declare const GEZEL_MARKDOWN_CHUNKS_2: {
1257
- id: string;
1258
- unit: "chars" | "tokens";
1259
- tokenizer: "profile" | "none";
1260
- target: number;
1261
- overlap: number;
1262
- contextHeader: {
1263
- max: number;
1264
- };
1265
- };
1266
-
1267
953
  /**
1268
954
  * Verifies the model files an embedder loaded against the digests its
1269
955
  * profile pins. transformers.js caches every fetched file under a key that
@@ -1808,4 +1494,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1808
1494
 
1809
1495
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1810
1496
 
1811
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EMBEDDINGGEMMA_2_512_1, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, catalogFormatVersion, chunkMarkdownProfile, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, hammingTopK, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, knowledgeEmbeddingProfile, knowledgeProfileQueryModelBytes, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
1497
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, type BatchTokenizerFn, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetFile, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogChunkMedia, type CatalogDb, type CatalogDocumentInitial, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, CatalogQueryError, type CatalogTopic, type CatalogValidationReport, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DocumentLocationRow, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type EncoderModelFn, type FetchKnowledgeRegistryOptions, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MAX_MEDIA_ROWS_PER_DOCUMENT, MEDIA_ROW_TEXT_MAX_TOKENS, type MarkdownCatalogSource, MarkdownChunk, type MediaEmbedRequest, type MediaEmbedResult, type MediaEmbedder, type MediaRow, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PinnedLoadFile, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type SpatialMatch, TITLE_ROUTE_SHARDS, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TensorLike, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, catalogFormatVersion, collectMediaRows, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, fileDigest, findDuckdbBinary, findRegistryEntry, humanizeAssetName, inspectGezkArchive, isExternalDataFile, isMultimodalEncoderConfig, loadMarkdownCatalog, mediaEncoderPins, mediaReferences, missingPinnedFiles, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, pinnedExternalDataChunks, pinnedLoadFiles, prefetchExternalData, preparePinnedLoad, readGezkManifest, readMkdocsDocsDir, removeMismatchedPinnedFiles, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyMediaArtifacts, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -181,6 +181,29 @@ function splitOversizeLine(line, opts) {
181
181
  return out;
182
182
  }
183
183
 
184
+ // src/chunking/embed-input.ts
185
+ function buildContextHeader(title, headingPath, maxTokens, countTokens) {
186
+ const path = headingPath.filter((h) => h !== title);
187
+ let header = path.length > 0 ? `${title}
188
+ ${path.join(" > ")}
189
+ ` : `${title}
190
+ `;
191
+ if (countTokens(header) > maxTokens) {
192
+ header = `${title}
193
+ `;
194
+ if (countTokens(header) > maxTokens) header = "";
195
+ }
196
+ return header;
197
+ }
198
+ function passageEmbedInput(input) {
199
+ return `${input.passageInstruction}${buildContextHeader(
200
+ input.title,
201
+ input.headingPath,
202
+ input.headerMaxTokens,
203
+ input.countTokens
204
+ )}${input.text}`;
205
+ }
206
+
184
207
  // src/compiler/compile.ts
185
208
  import { createHash as createHash3 } from "crypto";
186
209
  import {
@@ -695,12 +718,14 @@ async function compileKnowledgeCatalog(opts) {
695
718
  const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
696
719
  const embedded = totalRows <= shardTarget;
697
720
  const semanticFill = opts.shardFill === "semantic" && !embedded;
698
- const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
699
- p.doc.title,
700
- chunk.headingPath,
701
- opts.chunkingProfile.contextHeader.max,
702
- opts.countTokens
703
- )}${chunk.text}`;
721
+ const embedInput = (p, chunk) => passageEmbedInput({
722
+ passageInstruction: profile.passageInstruction,
723
+ title: p.doc.title,
724
+ headingPath: chunk.headingPath,
725
+ text: chunk.text,
726
+ headerMaxTokens: opts.chunkingProfile.contextHeader.max,
727
+ countTokens: opts.countTokens
728
+ });
704
729
  rmSync(opts.workDir, { recursive: true, force: true });
705
730
  mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
706
731
  const stagedPath = join(opts.workDir, "staged-vectors.f32");
@@ -1351,19 +1376,6 @@ function defaultLicenseNotice(opts) {
1351
1376
  function normalizeMarkdown(md) {
1352
1377
  return md.replace(/\r\n/g, "\n").normalize("NFC");
1353
1378
  }
1354
- function buildContextHeader(title, headingPath, maxTokens, countTokens) {
1355
- const path = headingPath.filter((h) => h !== title);
1356
- let header = path.length > 0 ? `${title}
1357
- ${path.join(" > ")}
1358
- ` : `${title}
1359
- `;
1360
- if (countTokens(header) > maxTokens) {
1361
- header = `${title}
1362
- `;
1363
- if (countTokens(header) > maxTokens) header = "";
1364
- }
1365
- return header;
1366
- }
1367
1379
  async function stageVectors(args) {
1368
1380
  const { prepared, profile } = args;
1369
1381
  const { dimensions } = profile;
@@ -5230,6 +5242,38 @@ var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
5230
5242
  var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
5231
5243
  var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
5232
5244
 
5245
+ // src/profiles/vector-floors.ts
5246
+ var KNOWLEDGE_VECTOR_FLOORS = {
5247
+ // Handboek 1.1.2: answers ≥ 0.687, off-topic ≤ 0.624.
5248
+ "bendyline/handboek": 0.65,
5249
+ // The project index's measured floor for the daemon's own embedder
5250
+ // (index-store VECTOR_ARM_MIN_SIMILARITY) — what a folder-built catalog gets.
5251
+ "bge-small-en-v1.5@1": 0.55,
5252
+ // Wikipedia Food & Drink 2026.4.3: answers ≥ 0.869; the scale is compressed
5253
+ // and 2 of 25 off-topic prompts still reach it, so grounding and the
5254
+ // relevance model do the rest.
5255
+ "multilingual-e5-small@2": 0.865,
5256
+ // Revision 1 has the same model, instructions and int8 vectors — only its
5257
+ // stage-1 sign bits differ — so its rerank cosine is on the same scale.
5258
+ "multilingual-e5-small@1": 0.865,
5259
+ // Handboek 1.2.2 + a Food & Drink sample re-embedded in this profile (the
5260
+ // labelled calibration queries): answers ≥ 0.762 in both, off-topic ≤ 0.706,
5261
+ // a query's best chunk in the OTHER catalog ≤ 0.715. A wide gap; the scale
5262
+ // is far less compressed than e5's.
5263
+ "embeddinggemma-2-512@1": 0.735,
5264
+ // Media rows of the same profile, workspace and catalogs alike
5265
+ // (evals/src/retrieval-bench/MEDIA-BENCH-2026-10-07.md). Each sits a step
5266
+ // above the lowest floor no off-topic prompt reached on the bench corpus,
5267
+ // because a real library is larger and its off-topic maximum higher.
5268
+ // COCO photos: no off-topic hit from 0.66; 293 of 300 answers keep their photo.
5269
+ "embeddinggemma-2-512@1#image": 0.67,
5270
+ // MSR-VTT clips: no off-topic hit from 0.65; 54 of 60 answers clear 0.65 and 0.66.
5271
+ "embeddinggemma-2-512@1#video": 0.66,
5272
+ // ESC-50 sounds: scores bunch (a wrong clip clears 0.66 for most queries),
5273
+ // so the floor does less here than ranking; 47 of 50 answers clear 0.68.
5274
+ "embeddinggemma-2-512@1#audio": 0.68
5275
+ };
5276
+
5233
5277
  // src/embedding/profile-embedder.ts
5234
5278
  import {
5235
5279
  embeddingProfileArtifacts as embeddingProfileArtifacts3,
@@ -7283,6 +7327,7 @@ export {
7283
7327
  GezkArchiveError,
7284
7328
  KNOWLEDGE_EMBEDDING_PROFILES,
7285
7329
  KNOWLEDGE_TOOLCHAIN,
7330
+ KNOWLEDGE_VECTOR_FLOORS,
7286
7331
  KnowledgeRegistryFetchError,
7287
7332
  LICENSE_NOTICE_PATH,
7288
7333
  MANIFEST_PATH,
@@ -7319,6 +7364,7 @@ export {
7319
7364
  assetReferences,
7320
7365
  assignOutlineOrders,
7321
7366
  asymmetricTopK,
7367
+ buildContextHeader,
7322
7368
  canonicalizeJson3 as canonicalizeJson,
7323
7369
  catalogFormatVersion,
7324
7370
  centerVector,
@@ -7363,6 +7409,7 @@ export {
7363
7409
  parseGitbookSummary,
7364
7410
  parseJupyterBookToc,
7365
7411
  parseMkdocsNav,
7412
+ passageEmbedInput,
7366
7413
  pinnedExternalDataChunks,
7367
7414
  pinnedLoadFiles,
7368
7415
  prefetchExternalData,