@bendyline/gezel-knowledge 1.1.2 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -12,13 +12,15 @@ npm install @bendyline/gezel-knowledge
12
12
 
13
13
  ## What a `.gezk` is
14
14
 
15
- An open format (gezk 0.6, preliminary until 1.0; 0.5 archives still open): a ZIP whose first entry is
15
+ An open format (format version 0.6, preliminary until format 1.0; 0.5
16
+ archives still open): a ZIP whose first entry is
16
17
  the stored `mimetype` magic (`application/vnd.gezk+zip`), then
17
18
  `manifest.json`, `README.md`, `LICENSES/`, a `router.db` (topics, document
18
19
  directory, brotli bodies, routing centroids) and shard SQLite databases with
19
20
  FTS5 and quantized vectors in plain BLOB tables. Nothing beyond stock SQLite
20
21
  is needed to read one. The format definitions are the separate
21
- [`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package;
22
+ [`@bendyline/gezk`](https://www.npmjs.com/package/@bendyline/gezk) package,
23
+ whose npm version is independent of the format version it writes;
22
24
  the specification, JSON Schemas, conformance fixtures and a Python reference
23
25
  reader live in [bendyline/gezk](https://github.com/bendyline/gezk).
24
26
 
package/dist/index.d.ts CHANGED
@@ -209,6 +209,26 @@ interface CompileKnowledgeCatalogOptions {
209
209
  * builds must not set it: the constant IS the format's scale design.
210
210
  */
211
211
  shardTargetChunks?: number;
212
+ /**
213
+ * How a multi-shard catalog's documents are assigned to shards.
214
+ *
215
+ * - `'topic'` (default; the gezk spec §8.1): greedy fill in
216
+ * `(topicPathKey, documentId)` order, so shards are slices of the table
217
+ * of contents.
218
+ * - `'semantic'`: embed every chunk first, then fill shards by balanced
219
+ * k-means over each document's mean chunk vector, so documents that
220
+ * answer the same queries share a shard and routing (§8.4) reaches them
221
+ * within its budget. Topic slices put an artist, their albums and their
222
+ * songs in different shards. Within a shard documents keep
223
+ * `(topicPathKey, documentId)` order. Every chunk is still embedded
224
+ * exactly once: vectors are staged in `workDir` between the passes.
225
+ *
226
+ * Measured on a 1.85M-chunk Wikipedia music catalog (155 queries,
227
+ * routed recall@8 vs an exact scan): 10 shards, S=6 82.6% → 85.7% and
228
+ * S=3 52.1% → 63.9%; recut to 37 shards, S=6 36.0% → 55.5%. The gain
229
+ * grows with the shard count.
230
+ */
231
+ shardFill?: 'topic' | 'semantic';
212
232
  }
213
233
  interface CompileReport {
214
234
  manifest: KnowledgeCatalogManifest;
@@ -485,7 +505,10 @@ declare class CatalogHandle {
485
505
  assets(): CatalogAssetInfo[];
486
506
  /** One declared asset's bytes, or null when the catalog declares no such asset. */
487
507
  readAsset(path: string): CatalogAssetRead | null;
488
- /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
508
+ /**
509
+ * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
510
+ * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
511
+ */
489
512
  searchDocumentsFts(query: string, limit?: number): Array<{
490
513
  documentId: string;
491
514
  rank: number;
@@ -653,6 +676,13 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
653
676
  * stores them (provably identical, one third of float32's size), with the
654
677
  * dequantisation rule recorded in the file metadata (gezk spec §6).
655
678
  *
679
+ * The bit rule in that metadata follows the catalog's profile: a `sign`
680
+ * profile stores sign(x), a `centered-sign` profile sign(x − c) with the
681
+ * profile's center c — which the metadata then carries too, so the Parquet
682
+ * files alone are enough to binarise a query the way the archive did. (Export
683
+ * version 2 wrote the plain-sign rule for every profile, which misdescribed
684
+ * centered-sign catalogs; version 3 fixed that.)
685
+ *
656
686
  * Determinism: rows are read in primary-key order and written single-
657
687
  * threaded with insertion order preserved, a fixed codec and level, a fixed
658
688
  * row-group size, and no wall-clock metadata — so two exports of one archive
@@ -661,7 +691,8 @@ declare function runDuckdbScript(cli: DuckdbCli, sql: string): Promise<string>;
661
691
  * bytes.
662
692
  */
663
693
 
664
- declare const PARQUET_EXPORT_VERSION = 2;
694
+ /** 3: the bit-rule metadata follows the profile's binary method (was always plain sign). */
695
+ declare const PARQUET_EXPORT_VERSION = 3;
665
696
  type ParquetExportSource = {
666
697
  archivePath: string;
667
698
  } | {
@@ -712,6 +743,16 @@ interface ParquetExportReport {
712
743
  }
713
744
  declare const PARQUET_REPORT_PATH = "parquet-manifest.json";
714
745
  declare function exportCatalogParquet(opts: ParquetExportOptions): Promise<ParquetExportReport>;
746
+ /**
747
+ * The key/value metadata every exported file carries. The bit rule is read
748
+ * from the profile — never assumed — because a `centered-sign` profile's bits
749
+ * are meaningless to a reader who binarises a query with plain sign(x). `x`
750
+ * is the float unit vector the bits were taken from, before int8 rounding, so
751
+ * recomputing them from `q / 127` can flip the few dimensions within rounding
752
+ * error of the threshold. Insertion order is the file's key order, so it is
753
+ * part of the deterministic bytes.
754
+ */
755
+ declare function parquetKvMetadata(manifest: KnowledgeCatalogManifest): Record<string, string>;
715
756
  /** Where a test or CLI may find a DuckDB CLI without configuration: env, then a gezel engine cache. */
716
757
  declare function findDuckdbBinary(env?: NodeJS.ProcessEnv): Promise<string | null>;
717
758
 
@@ -923,9 +964,19 @@ interface ProfileEmbedder {
923
964
  countTokens(text: string): number;
924
965
  dispose(): Promise<void>;
925
966
  }
967
+ /**
968
+ * The `@huggingface/transformers` range this package supports, the same range
969
+ * as its optional peer dependency (a test holds the two together). Error
970
+ * messages quote it so the install command they print is exact.
971
+ */
972
+ declare const TRANSFORMERS_PEER_RANGE = "^3.8.1";
926
973
  declare class EmbedderUnavailableError extends Error {
927
974
  readonly isActionable = true;
928
- constructor(message: string);
975
+ /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
976
+ readonly runtimeMissing: boolean;
977
+ constructor(message: string, options?: {
978
+ runtimeMissing?: boolean;
979
+ });
929
980
  }
930
981
  /** The slice of `@huggingface/transformers` this module drives; injectable for tests. */
931
982
  interface TransformersModule {
@@ -950,6 +1001,11 @@ type PipelineFn = ((texts: string[], options: {
950
1001
  interface TokenizerFn {
951
1002
  encode(text: string): number[];
952
1003
  }
1004
+ /** Bytes fetched so far across every file a profile load pulls, summed. */
1005
+ interface ModelDownloadProgress {
1006
+ bytesDone: number;
1007
+ bytesTotal: number;
1008
+ }
953
1009
  /** The load options that make transformers.js fetch exactly the profile's ONNX graph. */
954
1010
  interface TransformersModelOptions {
955
1011
  revision: string;
@@ -958,6 +1014,18 @@ interface TransformersModelOptions {
958
1014
  model_file_name: string;
959
1015
  }
960
1016
  declare function resolveTransformersModelOptions(profile: KnowledgeEmbeddingProfile): TransformersModelOptions;
1017
+ /**
1018
+ * The transformers.js options that select exactly one pinned ONNX graph:
1019
+ * its folder, its base name, and the dtype whose suffix transformers.js
1020
+ * appends to that name. A file with no recognized suffix loads as fp32 under
1021
+ * its full name, which selects the same file. Null for a path that is not a
1022
+ * named `.onnx` graph.
1023
+ */
1024
+ declare function transformersGraphOptions(onnxFile: string): {
1025
+ dtype: string;
1026
+ subfolder: string;
1027
+ model_file_name: string;
1028
+ } | null;
961
1029
  declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?: {
962
1030
  cacheDir?: string;
963
1031
  /**
@@ -968,6 +1036,8 @@ declare function createProfileEmbedder(profile: KnowledgeEmbeddingProfile, opts?
968
1036
  * what affects values, and that stays with the caller.
969
1037
  */
970
1038
  sessionOptions?: Record<string, unknown>;
1039
+ /** Called as the model and tokenizer files download. */
1040
+ onDownloadProgress?: (progress: ModelDownloadProgress) => void;
971
1041
  /** Test seam: a stand-in for `@huggingface/transformers`. */
972
1042
  transformers?: TransformersModule;
973
1043
  }): Promise<ProfileEmbedder>;
@@ -1222,4 +1292,4 @@ declare function newerRegistryEntries(registry: KnowledgeRegistryIndex, installe
1222
1292
 
1223
1293
  declare const BUILTIN_KNOWLEDGE_TRUST_ANCHORS: readonly KnowledgeTrustAnchor[];
1224
1294
 
1225
- export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
1295
+ export { type ArchiveEntry, type ArtifactCheck, type ArtifactRole, type AsymmetricHit, BGE_SMALL_EN_V15_1, BUILTIN_KNOWLEDGE_TRUST_ANCHORS, CENTROID_CHUNKS_PER, CENTROID_KMEANS_MAX_ITER, CENTROID_MAX_PER_SHARD, CENTROID_SAMPLE_MAX, type CatalogAssetInfo, type CatalogAssetRead, type CatalogCheck, type CatalogChunkHit, type CatalogDb, type CatalogDocumentMeta, CatalogHandle, CatalogOpenError, type CatalogTopic, type CatalogValidationReport, type CharChunkerOptions, type ChunkerOptions, type CompileAsset, type CompileKnowledgeCatalogOptions, type CompileReport, type CompileTopic, type DuckdbCli, DuckdbError, EmbedderArtifactError, EmbedderUnavailableError, type FetchKnowledgeRegistryOptions, GEZEL_BGE_SMALL_EN_V15_1, GEZEL_MARKDOWN_CHUNKS_2, GEZEL_MULTILINGUAL_E5_SMALL_1, GEZK_ARCHIVE_LIMITS, GezkArchiveError, type GezkArchiveInspection, type HammingHit, KNOWLEDGE_EMBEDDING_PROFILES, KNOWLEDGE_TOOLCHAIN, type KnowledgeRegistryFetchCode, KnowledgeRegistryFetchError, type LoadMarkdownCatalogOptions, MARKDOWN_CHUNKS_2, MULTILINGUAL_E5_SMALL_1, MULTILINGUAL_E5_SMALL_2, type MarkdownCatalogSource, type MarkdownChunk, type ModelDownloadProgress, type Outline, type OutlineEntry, type OutlineFormat, type OutlineTopic, PARQUET_EXPORT_VERSION, PARQUET_REPORT_PATH, type ParquetExportFile, type ParquetExportOptions, type ParquetExportProgress, type ParquetExportReport, type ParquetExportSource, type ParquetTable, type PipelineFn, type ProfileEmbedder, RERANK_CANDIDATES_MAX, RERANK_CANDIDATES_MIN, RERANK_FINAL_K, ROUTE_SHARDS_EXPLICIT, ROUTE_SHARDS_PROACTIVE, SHARD_MAX_CHUNKS, SHARD_TARGET_CHUNKS, type ShardBitIndex, TRANSFORMERS_PEER_RANGE, type TableOfContentsFormat, type TableOfContentsOptions, type TokenChunkerOptions, type TokenizerFn, type TopicOverride, type TransformersModelOptions, type TransformersModule, type ValidateCatalogOptions, type VerifiedArtifacts, assertDuckdbCli, assetReferences, assignOutlineOrders, asymmetricTopK, chunkMarkdownProfile, compareCatalogVersions, compileKnowledgeCatalog, createProfileEmbedder, detectTableOfContents, duckdbVersion, exportCatalogParquet, extractGezkVerified, fetchKnowledgeRegistry, findDuckdbBinary, findRegistryEntry, hammingTopK, inspectGezkArchive, knowledgeEmbeddingProfile, loadMarkdownCatalog, newOutlineTopic, newerRegistryEntries, openCatalogDatabase, parquetKvMetadata, parseGitbookSummary, parseJupyterBookToc, parseMkdocsNav, readGezkManifest, readMkdocsDocsDir, rerankK, resolveOutlinePath, resolveTransformersModelOptions, runDuckdbScript, titleFromFolderName, transformersCachePath, transformersGraphOptions, validateExtractedCatalog, verifyProfileArtifacts, writeGezkArchive };
package/dist/index.js CHANGED
@@ -181,7 +181,16 @@ function splitOversizeLine(line, opts) {
181
181
 
182
182
  // src/compiler/compile.ts
183
183
  import { createHash } from "crypto";
184
- import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
184
+ import {
185
+ closeSync,
186
+ mkdirSync,
187
+ openSync,
188
+ readFileSync,
189
+ readSync,
190
+ rmSync,
191
+ statSync,
192
+ writeSync
193
+ } from "fs";
185
194
  import { stat } from "fs/promises";
186
195
  import { join } from "path";
187
196
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
@@ -250,16 +259,20 @@ function sanitizeFtsQuery(query) {
250
259
  if (tokens.length === 0) return null;
251
260
  return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
252
261
  }
253
- function documentFtsTopIds(db, match, limit) {
262
+ var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
263
+ var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
264
+ function documentFtsTopIds(db, match, limit, exactTitle) {
265
+ const title = exactTitle?.normalize("NFKC").trim() ?? "";
254
266
  return db.prepare(
255
- "SELECT document_id FROM fts_documents WHERE fts_documents MATCH ? ORDER BY rank LIMIT ?"
256
- ).all(match, limit).map((row) => row.document_id);
267
+ `SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
268
+ ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
269
+ ).all(match, title, limit).map((row) => row.document_id);
257
270
  }
258
271
  var SMOKE_QUERY_TOP_N = 10;
259
272
  function documentSmokeQueryMisses(db, smoke) {
260
273
  const match = sanitizeFtsQuery(smoke.query);
261
274
  if (!match) return [...smoke.expectedDocumentIds];
262
- const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N);
275
+ const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
263
276
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
264
277
  }
265
278
 
@@ -306,10 +319,47 @@ async function compileKnowledgeCatalog(opts) {
306
319
  const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
307
320
  const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
308
321
  const embedded = totalChunks <= shardTarget;
322
+ const semanticFill = opts.shardFill === "semantic" && !embedded;
323
+ const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
324
+ p.doc.title,
325
+ chunk.headingPath,
326
+ opts.chunkingProfile.contextHeader.max,
327
+ opts.countTokens
328
+ )}${chunk.text}`;
329
+ rmSync(opts.workDir, { recursive: true, force: true });
330
+ mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
331
+ const stagedPath = join(opts.workDir, "staged-vectors.f32");
332
+ let staged = null;
333
+ let docVectors = [];
334
+ if (semanticFill) {
335
+ docVectors = await stageVectors({
336
+ prepared,
337
+ stagedPath,
338
+ dimensions: profile.dimensions,
339
+ batchSize: opts.embedBatchSize ?? 32,
340
+ embed: opts.embed,
341
+ embedInput,
342
+ onProgress: (done) => opts.onProgress?.({ phase: "embed", done, total: totalChunks })
343
+ });
344
+ staged = new StagedVectorReader(stagedPath, profile.dimensions);
345
+ }
309
346
  const shardOf = /* @__PURE__ */ new Map();
310
347
  let shardCount = 1;
311
348
  if (embedded) {
312
349
  for (const p of prepared) shardOf.set(p.doc.id, 0);
350
+ } else if (semanticFill) {
351
+ shardCount = Math.ceil(totalChunks / shardTarget);
352
+ const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
353
+ const assignment = semanticShards(
354
+ docVectors,
355
+ prepared.map((p) => p.chunks.length),
356
+ shardCount,
357
+ capacity,
358
+ seedFor(`${opts.catalog.id}#shard-fill`, 0)
359
+ );
360
+ prepared.forEach((p, i) => shardOf.set(p.doc.id, assignment[i]));
361
+ shardCount = new Set(assignment).size;
362
+ docVectors = [];
313
363
  } else {
314
364
  let current = 0;
315
365
  let filled = 0;
@@ -323,8 +373,6 @@ async function compileKnowledgeCatalog(opts) {
323
373
  }
324
374
  shardCount = current + 1;
325
375
  }
326
- rmSync(opts.workDir, { recursive: true, force: true });
327
- mkdirSync(join(opts.workDir, "index", "shards"), { recursive: true });
328
376
  const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
329
377
  const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
330
378
  const shardDbs = /* @__PURE__ */ new Map();
@@ -463,7 +511,7 @@ async function compileKnowledgeCatalog(opts) {
463
511
  let processedChunks = 0;
464
512
  const flushEmbeds = async () => {
465
513
  if (pendingTexts.length === 0) return;
466
- const vectors = await opts.embed(pendingTexts);
514
+ const vectors = staged ? staged.next(pendingTexts.length) : await opts.embed(pendingTexts);
467
515
  if (vectors.length !== pendingTexts.length) {
468
516
  throw new Error(
469
517
  `embedder returned ${vectors.length} vectors for ${pendingTexts.length} texts`
@@ -488,7 +536,11 @@ async function compileKnowledgeCatalog(opts) {
488
536
  if ((row.chunkId - 1) % stride === 0) samples.push(unit);
489
537
  }
490
538
  processedChunks += pendingTexts.length;
491
- opts.onProgress?.({ phase: "embed", done: processedChunks, total: totalChunks });
539
+ opts.onProgress?.({
540
+ phase: staged ? "write" : "embed",
541
+ done: processedChunks,
542
+ total: totalChunks
543
+ });
492
544
  pendingTexts = [];
493
545
  pendingRows = [];
494
546
  };
@@ -526,18 +578,14 @@ async function compileKnowledgeCatalog(opts) {
526
578
  const chunkId = Number(row.id);
527
579
  insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, chunk.text);
528
580
  shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
529
- const header = buildContextHeader(
530
- p.doc.title,
531
- chunk.headingPath,
532
- opts.chunkingProfile.contextHeader.max,
533
- opts.countTokens
534
- );
535
- pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
581
+ pendingTexts.push(embedInput(p, chunk));
536
582
  pendingRows.push({ shardId, chunkId });
537
583
  if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
538
584
  }
539
585
  }
540
586
  await flushEmbeds();
587
+ staged?.close();
588
+ rmSync(stagedPath, { force: true });
541
589
  for (let id = 0; id < shardCount; id++) {
542
590
  if (!embedded)
543
591
  putMeta(shardDb(id), {
@@ -909,6 +957,158 @@ ${path.join(" > ")}
909
957
  }
910
958
  return header;
911
959
  }
960
+ async function stageVectors(args) {
961
+ const { prepared, dimensions } = args;
962
+ const docVectors = prepared.map(() => new Float32Array(dimensions));
963
+ const fd = openSync(args.stagedPath, "w");
964
+ let texts = [];
965
+ let owners = [];
966
+ let done = 0;
967
+ const flush = async () => {
968
+ if (texts.length === 0) return;
969
+ const vectors = await args.embed(texts);
970
+ if (vectors.length !== texts.length) {
971
+ throw new Error(`embedder returned ${vectors.length} vectors for ${texts.length} texts`);
972
+ }
973
+ const block = new Float32Array(texts.length * dimensions);
974
+ for (let i = 0; i < vectors.length; i++) {
975
+ const unit = l2Normalize(vectors[i]);
976
+ if (unit.length !== dimensions) {
977
+ throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
978
+ }
979
+ block.set(unit, i * dimensions);
980
+ const sum = docVectors[owners[i]];
981
+ for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
982
+ }
983
+ writeSync(fd, new Uint8Array(block.buffer));
984
+ done += texts.length;
985
+ args.onProgress(done);
986
+ texts = [];
987
+ owners = [];
988
+ };
989
+ try {
990
+ for (let i = 0; i < prepared.length; i++) {
991
+ const p = prepared[i];
992
+ for (const chunk of p.chunks) {
993
+ texts.push(args.embedInput(p, chunk));
994
+ owners.push(i);
995
+ if (texts.length >= args.batchSize) await flush();
996
+ }
997
+ }
998
+ await flush();
999
+ } finally {
1000
+ closeSync(fd);
1001
+ }
1002
+ return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
1003
+ }
1004
+ var StagedVectorReader = class {
1005
+ constructor(path, dimensions) {
1006
+ this.dimensions = dimensions;
1007
+ this.fd = openSync(path, "r");
1008
+ }
1009
+ dimensions;
1010
+ fd;
1011
+ position = 0;
1012
+ next(count) {
1013
+ const bytes = count * this.dimensions * 4;
1014
+ const buffer = Buffer.alloc(bytes);
1015
+ let read = 0;
1016
+ while (read < bytes) {
1017
+ const n = readSync(this.fd, buffer, read, bytes - read, this.position + read);
1018
+ if (n === 0)
1019
+ throw new Error("staged vectors ended early: the write pass and pass 1 disagree");
1020
+ read += n;
1021
+ }
1022
+ this.position += bytes;
1023
+ const floats = new Float32Array(buffer.buffer, buffer.byteOffset, count * this.dimensions);
1024
+ const out = [];
1025
+ for (let i = 0; i < count; i++) {
1026
+ out.push(Array.from(floats.subarray(i * this.dimensions, (i + 1) * this.dimensions)));
1027
+ }
1028
+ return out;
1029
+ }
1030
+ close() {
1031
+ closeSync(this.fd);
1032
+ }
1033
+ };
1034
+ var SEMANTIC_FILL_ROUNDS = 6;
1035
+ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1036
+ const n = docVectors.length;
1037
+ const dim = docVectors[0].length;
1038
+ const dot = (a, b) => {
1039
+ let sum = 0;
1040
+ for (let d = 0; d < dim; d++) sum += a[d] * b[d];
1041
+ return sum;
1042
+ };
1043
+ const stride = Math.max(1, Math.ceil(n / 2e4));
1044
+ const sample = docVectors.filter((_, i) => i % stride === 0);
1045
+ const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
1046
+ for (let i = 0; centers.length < shardCount; i++) {
1047
+ centers.push(
1048
+ Float32Array.from(docVectors[Math.floor(i * n / shardCount) % n])
1049
+ );
1050
+ }
1051
+ let assignment = new Array(n).fill(-1);
1052
+ for (let round = 0; round < SEMANTIC_FILL_ROUNDS; round++) {
1053
+ const best = new Int32Array(n);
1054
+ const margin = new Float64Array(n);
1055
+ for (let i = 0; i < n; i++) {
1056
+ let first = Number.NEGATIVE_INFINITY;
1057
+ let second = Number.NEGATIVE_INFINITY;
1058
+ let arg = 0;
1059
+ for (let c = 0; c < shardCount; c++) {
1060
+ const s = dot(docVectors[i], centers[c]);
1061
+ if (s > first) {
1062
+ second = first;
1063
+ first = s;
1064
+ arg = c;
1065
+ } else if (s > second) {
1066
+ second = s;
1067
+ }
1068
+ }
1069
+ best[i] = arg;
1070
+ margin[i] = first - (Number.isFinite(second) ? second : first);
1071
+ }
1072
+ const order = Array.from({ length: n }, (_, i) => i).sort(
1073
+ (a, b) => margin[b] - margin[a] || a - b
1074
+ );
1075
+ const load = new Array(shardCount).fill(0);
1076
+ const next = new Array(n).fill(-1);
1077
+ for (const i of order) {
1078
+ const size = docChunks[i];
1079
+ let shard = best[i];
1080
+ if (load[shard] + size > capacity) {
1081
+ const ranked = centers.map((c, id) => ({ id, s: dot(docVectors[i], c) })).sort((a, b) => b.s - a.s || a.id - b.id);
1082
+ const fit = ranked.find((r) => load[r.id] + size <= capacity);
1083
+ shard = fit ? fit.id : load.reduce((min, l, id) => l < load[min] ? id : min, 0);
1084
+ }
1085
+ next[i] = shard;
1086
+ load[shard] = load[shard] + size;
1087
+ }
1088
+ const moved = next.some((s, i) => s !== assignment[i]);
1089
+ assignment = next;
1090
+ if (!moved) break;
1091
+ const sums = centers.map(() => new Float64Array(dim));
1092
+ for (let i = 0; i < n; i++) {
1093
+ const sum = sums[assignment[i]];
1094
+ const v = docVectors[i];
1095
+ const w = docChunks[i];
1096
+ for (let d = 0; d < dim; d++) sum[d] = sum[d] + v[d] * w;
1097
+ }
1098
+ for (let c = 0; c < shardCount; c++) {
1099
+ const sum = sums[c];
1100
+ let norm = 0;
1101
+ for (let d = 0; d < dim; d++) norm += sum[d] ** 2;
1102
+ if (norm === 0) continue;
1103
+ norm = Math.sqrt(norm);
1104
+ const center = centers[c];
1105
+ for (let d = 0; d < dim; d++) center[d] = sum[d] / norm;
1106
+ }
1107
+ }
1108
+ const renumber = /* @__PURE__ */ new Map();
1109
+ for (const s of assignment) if (!renumber.has(s)) renumber.set(s, renumber.size);
1110
+ return assignment.map((s) => renumber.get(s));
1111
+ }
912
1112
  function seedFor(catalogId, shardId) {
913
1113
  const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
914
1114
  return digest.readBigUInt64LE(24);
@@ -1924,12 +2124,15 @@ var CatalogHandle = class _CatalogHandle {
1924
2124
  return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
1925
2125
  }
1926
2126
  // ── search ────────────────────────────────────────────────────────────────
1927
- /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
2127
+ /**
2128
+ * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
2129
+ * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
2130
+ */
1928
2131
  searchDocumentsFts(query, limit = 10) {
1929
2132
  const match = ftsQuery(query);
1930
2133
  if (!match) return [];
1931
2134
  try {
1932
- return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
2135
+ return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
1933
2136
  documentId,
1934
2137
  rank: i
1935
2138
  }));
@@ -2488,8 +2691,8 @@ import { createWriteStream as createWriteStream3 } from "fs";
2488
2691
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2489
2692
  import { join as join4, resolve as resolve3 } from "path";
2490
2693
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2491
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
2492
- var PARQUET_EXPORT_VERSION = 2;
2694
+ import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
2695
+ var PARQUET_EXPORT_VERSION = 3;
2493
2696
  var DEFAULT_ROW_GROUP_SIZE = 32768;
2494
2697
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2495
2698
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2514,7 +2717,7 @@ async function exportCatalogParquet(opts) {
2514
2717
  const staging = join4(work, "ndjson");
2515
2718
  await mkdir3(staging, { recursive: true });
2516
2719
  const files = [];
2517
- const metadata = kvMetadata(manifest);
2720
+ const metadata = parquetKvMetadata(manifest);
2518
2721
  const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
2519
2722
  const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
2520
2723
  try {
@@ -2782,7 +2985,9 @@ function parseJsonStrings(raw) {
2782
2985
  function sqlString(value) {
2783
2986
  return `'${value.replace(/'/g, "''")}'`;
2784
2987
  }
2785
- function kvMetadata(manifest) {
2988
+ function parquetKvMetadata(manifest) {
2989
+ const binary = manifest.embedding.quantization.binary;
2990
+ const centered = embeddingProfileCenter2(manifest.embedding) !== null;
2786
2991
  return {
2787
2992
  gezk_catalog_id: manifest.id,
2788
2993
  gezk_catalog_version: manifest.version,
@@ -2792,7 +2997,9 @@ function kvMetadata(manifest) {
2792
2997
  gezk_embedding_model: manifest.embedding.model.repo,
2793
2998
  gezk_embedding_dimensions: String(manifest.embedding.dimensions),
2794
2999
  gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
2795
- gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
3000
+ gezk_embedding_bit: centered ? "centered-sign bits, LSB-first, ceil(dimensions/8) bytes: bit = (x - c) > 0, x the float unit vector, c = gezk_embedding_bit_center" : "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0, x the float unit vector",
3001
+ gezk_embedding_bit_method: binary.method,
3002
+ ...centered ? { gezk_embedding_bit_center: JSON.stringify(binary.center) } : {},
2796
3003
  gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
2797
3004
  gezk_license: manifest.license.name
2798
3005
  };
@@ -3461,13 +3668,36 @@ async function verifyProfileArtifacts(profile, opts) {
3461
3668
  // src/embedding/profile-embedder.ts
3462
3669
  var MAX_BATCH = 8;
3463
3670
  var MAX_CHARS = 8e3;
3671
+ var TRANSFORMERS_PEER_RANGE = "^3.8.1";
3464
3672
  var EmbedderUnavailableError = class extends Error {
3465
3673
  isActionable = true;
3466
- constructor(message) {
3674
+ /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
3675
+ runtimeMissing;
3676
+ constructor(message, options = {}) {
3467
3677
  super(message);
3468
3678
  this.name = "EmbedderUnavailableError";
3679
+ this.runtimeMissing = options.runtimeMissing ?? false;
3469
3680
  }
3470
3681
  };
3682
+ function aggregateDownloadProgress(onProgress) {
3683
+ const files = /* @__PURE__ */ new Map();
3684
+ return (info) => {
3685
+ if (info.status !== "progress") return;
3686
+ const loaded = typeof info.loaded === "number" ? info.loaded : 0;
3687
+ const total = typeof info.total === "number" ? info.total : 0;
3688
+ files.set(`${String(info.name)}/${String(info.file)}`, {
3689
+ loaded,
3690
+ total: Math.max(total, loaded)
3691
+ });
3692
+ let bytesDone = 0;
3693
+ let bytesTotal = 0;
3694
+ for (const file of files.values()) {
3695
+ bytesDone += file.loaded;
3696
+ bytesTotal += file.total;
3697
+ }
3698
+ onProgress({ bytesDone, bytesTotal });
3699
+ };
3700
+ }
3471
3701
  var ONNX_DTYPE_BY_SUFFIX = [
3472
3702
  ["_q4f16", "q4f16"],
3473
3703
  ["_quantized", "q8"],
@@ -3488,21 +3718,24 @@ function resolveTransformersModelOptions(profile) {
3488
3718
  `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
3489
3719
  );
3490
3720
  }
3491
- if (!onnxFile.endsWith(".onnx")) {
3721
+ const graph = transformersGraphOptions(onnxFile);
3722
+ if (!graph) {
3492
3723
  throw new EmbedderUnavailableError(
3493
- `profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
3724
+ `profile ${profile.id} pins ${onnxFile}; this runtime loads named ONNX graphs only`
3494
3725
  );
3495
3726
  }
3727
+ return { revision: profile.model.revision, ...graph };
3728
+ }
3729
+ function transformersGraphOptions(onnxFile) {
3730
+ if (!onnxFile.endsWith(".onnx")) return null;
3496
3731
  const slash = onnxFile.lastIndexOf("/");
3497
3732
  const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
3498
3733
  const base = onnxFile.slice(slash + 1, -".onnx".length);
3499
3734
  const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
3500
3735
  const [suffix, dtype] = variant ?? ["", "fp32"];
3501
3736
  const modelFileName = base.slice(0, base.length - suffix.length);
3502
- if (modelFileName === "") {
3503
- throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
3504
- }
3505
- return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
3737
+ if (modelFileName === "") return null;
3738
+ return { dtype, subfolder, model_file_name: modelFileName };
3506
3739
  }
3507
3740
  async function loadTransformers() {
3508
3741
  try {
@@ -3512,7 +3745,8 @@ async function loadTransformers() {
3512
3745
  const message = err instanceof Error ? err.message : String(err);
3513
3746
  if (/Cannot find (module|package)/i.test(message)) {
3514
3747
  throw new EmbedderUnavailableError(
3515
- "the embedding runtime (@huggingface/transformers) is not installed \u2014 knowledge builds and semantic search need it; install it or run inside a gezel install that ships it"
3748
+ `the embedding runtime is not installed \u2014 knowledge builds and semantic search need the optional peer @huggingface/transformers@${TRANSFORMERS_PEER_RANGE}; install it alongside @bendyline/gezel-knowledge`,
3749
+ { runtimeMissing: true }
3516
3750
  );
3517
3751
  }
3518
3752
  throw err;
@@ -3533,13 +3767,16 @@ async function createProfileEmbedder(profile, opts = {}) {
3533
3767
  transformers.env.useFSCache = true;
3534
3768
  transformers.env.allowRemoteModels = true;
3535
3769
  }
3770
+ const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
3536
3771
  const [pipe, tokenizer] = await Promise.all([
3537
3772
  transformers.pipeline("feature-extraction", profile.model.repo, {
3538
3773
  ...modelOptions,
3539
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
3774
+ ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
3775
+ ...progress
3540
3776
  }),
3541
3777
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3542
- revision: modelOptions.revision
3778
+ revision: modelOptions.revision,
3779
+ ...progress
3543
3780
  })
3544
3781
  ]);
3545
3782
  let verification;
@@ -4655,6 +4892,7 @@ export {
4655
4892
  SHARD_MAX_CHUNKS,
4656
4893
  SHARD_TARGET_CHUNKS,
4657
4894
  SOURCE_NOTICES_PATH,
4895
+ TRANSFORMERS_PEER_RANGE,
4658
4896
  ZIP_FIXED_MTIME,
4659
4897
  assertDuckdbCli,
4660
4898
  assetReferences,
@@ -4689,6 +4927,7 @@ export {
4689
4927
  newOutlineTopic,
4690
4928
  newerRegistryEntries,
4691
4929
  openCatalogDatabase,
4930
+ parquetKvMetadata,
4692
4931
  parseGitbookSummary,
4693
4932
  parseJupyterBookToc,
4694
4933
  parseMkdocsNav,
@@ -4706,6 +4945,7 @@ export {
4706
4945
  signRegistryIndex,
4707
4946
  titleFromFolderName,
4708
4947
  transformersCachePath,
4948
+ transformersGraphOptions,
4709
4949
  validateExtractedCatalog,
4710
4950
  verifyManifestSignature,
4711
4951
  verifyProfileArtifacts,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@bendyline/gezel-knowledge",
3
- "version": "1.1.2",
3
+ "version": "1.2.1",
4
4
  "description": "Gezel knowledge catalogs: the .gezk compiler, archive safety, read-only reader, routing, and signatures.",
5
5
  "license": "MIT",
6
6
  "keywords": [