@bendyline/gezel-knowledge 1.2.3 → 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  ROUTER_DB_PATH,
19
19
  SOURCE_NOTICES_PATH,
20
20
  ZIP_FIXED_MTIME,
21
+ formatAtLeast,
21
22
  isSupportedFormatVersion,
22
23
  isSupportedIndexSchemaVersion
23
24
  } from "@bendyline/gezk";
@@ -37,7 +38,7 @@ function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY)
37
38
  }
38
39
 
39
40
  // src/format/ddl.ts
40
- import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
41
+ import { ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, shardDdlFor } from "@bendyline/gezk";
41
42
 
42
43
  // src/format/ids.ts
43
44
  import { documentSlug } from "@bendyline/gezk";
@@ -47,6 +48,7 @@ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
47
48
  import {
48
49
  centerVector,
49
50
  l2Normalize,
51
+ profileUnitVector,
50
52
  quantizeBinary,
51
53
  quantizeBinaryForProfile,
52
54
  quantizeInt8,
@@ -180,7 +182,7 @@ function splitOversizeLine(line, opts) {
180
182
  }
181
183
 
182
184
  // src/compiler/compile.ts
183
- import { createHash as createHash2 } from "crypto";
185
+ import { createHash as createHash3 } from "crypto";
184
186
  import {
185
187
  closeSync,
186
188
  mkdirSync,
@@ -200,6 +202,8 @@ import {
200
202
  KnowledgeOrdinalSchema,
201
203
  KnowledgeTocReferenceSchema,
202
204
  canonicalizeJson,
205
+ embeddingProfileMinimumFormat,
206
+ isKnowledgeMediaAssetPath,
203
207
  normalizeLongitude,
204
208
  spatialManifest
205
209
  } from "@bendyline/gezk";
@@ -248,176 +252,7 @@ var sqlite = nodeRequire2("node:sqlite");
248
252
  var DatabaseSync = sqlite.DatabaseSync;
249
253
 
250
254
  // src/query-stopwords.ts
251
- var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
252
- // articles + determiners
253
- "a",
254
- "an",
255
- "the",
256
- "this",
257
- "that",
258
- "these",
259
- "those",
260
- "some",
261
- "any",
262
- "all",
263
- "each",
264
- "both",
265
- "every",
266
- "other",
267
- "another",
268
- "same",
269
- "such",
270
- // pronouns
271
- "i",
272
- "me",
273
- "my",
274
- "we",
275
- "us",
276
- "our",
277
- "you",
278
- "your",
279
- "it",
280
- "its",
281
- "he",
282
- "him",
283
- "his",
284
- "she",
285
- "her",
286
- "they",
287
- "them",
288
- "their",
289
- // be / have / do
290
- "am",
291
- "is",
292
- "are",
293
- "was",
294
- "were",
295
- "be",
296
- "been",
297
- "being",
298
- "have",
299
- "has",
300
- "had",
301
- "do",
302
- "does",
303
- "did",
304
- "done",
305
- // modals
306
- "can",
307
- "could",
308
- "will",
309
- "would",
310
- "shall",
311
- "should",
312
- "may",
313
- "might",
314
- "must",
315
- // question words
316
- "how",
317
- "what",
318
- "when",
319
- "where",
320
- "which",
321
- "who",
322
- "whom",
323
- "why",
324
- // prepositions + conjunctions
325
- "about",
326
- "and",
327
- "as",
328
- "at",
329
- "but",
330
- "by",
331
- "for",
332
- "from",
333
- "if",
334
- "in",
335
- "into",
336
- "of",
337
- "on",
338
- "or",
339
- "so",
340
- "than",
341
- "then",
342
- "there",
343
- "to",
344
- "with",
345
- // request scaffolding + degree words
346
- "please",
347
- "help",
348
- "let",
349
- "tell",
350
- "want",
351
- "need",
352
- "create",
353
- "make",
354
- "give",
355
- "get",
356
- "got",
357
- "use",
358
- "using",
359
- "just",
360
- "like",
361
- "also",
362
- "very",
363
- "only",
364
- "new",
365
- "more",
366
- "most",
367
- "much",
368
- "many",
369
- "few",
370
- "own",
371
- // conversational scaffolding
372
- "hey",
373
- "hi",
374
- "hiya",
375
- "hello",
376
- "thanks",
377
- "thank",
378
- "okay",
379
- "ok",
380
- "yep",
381
- "yeah",
382
- "yes",
383
- "nope",
384
- "good",
385
- "great",
386
- "fine",
387
- "nice",
388
- "cool",
389
- "ready",
390
- "going",
391
- "doing",
392
- "sounds",
393
- "morning",
394
- "afternoon",
395
- "evening",
396
- "night",
397
- // Apostrophes are token boundaries, so contractions otherwise leave these
398
- // fragments looking like rare, highly distinctive query terms.
399
- "s",
400
- "t",
401
- "re",
402
- "ve",
403
- "ll",
404
- "d",
405
- "m",
406
- "isn",
407
- "aren",
408
- "wasn",
409
- "weren",
410
- "don",
411
- "doesn",
412
- "didn",
413
- "hasn",
414
- "haven",
415
- "hadn",
416
- "won",
417
- "wouldn",
418
- "shouldn",
419
- "couldn"
420
- ]);
255
+ import { QUERY_STOP_WORDS } from "@bendyline/gezk";
421
256
 
422
257
  // src/reader/fts-query.ts
423
258
  function contentTokens(query) {
@@ -563,11 +398,11 @@ import { createHash } from "crypto";
563
398
  import { readFileSync } from "fs";
564
399
  import {
565
400
  KnowledgeAssetPathSchema,
566
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
567
- MAX_KNOWLEDGE_ASSET_BYTES,
568
401
  MAX_KNOWLEDGE_ASSET_COUNT,
569
402
  assetExtension,
570
403
  assetKindForExtension,
404
+ knowledgeAssetLimitsProblem,
405
+ maxKnowledgeAssetBytes,
571
406
  sniffAssetType,
572
407
  svgInertnessProblem
573
408
  } from "@bendyline/gezk";
@@ -588,7 +423,6 @@ function prepareAssets(assets, extraFiles, opts) {
588
423
  skippedPaths.add(path);
589
424
  opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
590
425
  };
591
- let total = 0;
592
426
  for (const asset of assets) {
593
427
  const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
594
428
  if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
@@ -599,10 +433,11 @@ function prepareAssets(assets, extraFiles, opts) {
599
433
  throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
600
434
  }
601
435
  const bytes = asset.content ?? readFileSync(asset.absPath);
602
- if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
436
+ const limit = maxKnowledgeAssetBytes(asset.path);
437
+ if (bytes.byteLength > limit) {
603
438
  rejectContent(
604
439
  asset.path,
605
- `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
440
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${limit}`
606
441
  );
607
442
  continue;
608
443
  }
@@ -623,17 +458,16 @@ function prepareAssets(assets, extraFiles, opts) {
623
458
  continue;
624
459
  }
625
460
  }
626
- total += bytes.byteLength;
627
- if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
628
- throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
629
- }
630
461
  prepared.push({
631
462
  path: asset.path,
632
463
  bytes,
633
464
  sizeBytes: bytes.byteLength,
634
- sha256: createHash("sha256").update(bytes).digest("hex")
465
+ sha256: createHash("sha256").update(bytes).digest("hex"),
466
+ ...asset.attribution ? { attribution: asset.attribution } : {}
635
467
  });
636
468
  }
469
+ const totals = knowledgeAssetLimitsProblem(prepared);
470
+ if (totals) throw new Error(totals);
637
471
  return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
638
472
  }
639
473
  function omitSkippedAssetReferences(markdown, skippedPaths) {
@@ -644,6 +478,131 @@ function omitSkippedAssetReferences(markdown, skippedPaths) {
644
478
  );
645
479
  }
646
480
 
481
+ // src/compiler/media-rows.ts
482
+ import { createHash as createHash2 } from "crypto";
483
+ import {
484
+ assetContentType,
485
+ assetModality
486
+ } from "@bendyline/gezk";
487
+ var MEDIA_ROW_TEXT_MAX_TOKENS = 128;
488
+ var MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
489
+ var MEDIA_REFERENCE = /!\[([^\]\n]*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+"[^"\n]*")?\s*\)/g;
490
+ function mediaReferences(markdown) {
491
+ const out = [];
492
+ markdown.split("\n").forEach((text, index) => {
493
+ for (const match of text.matchAll(MEDIA_REFERENCE)) {
494
+ out.push({
495
+ alt: (match[1] ?? "").trim(),
496
+ path: match[2],
497
+ line: index + 1,
498
+ caption: text.replace(MEDIA_REFERENCE, " ").replace(/\s+/g, " ").trim()
499
+ });
500
+ }
501
+ });
502
+ return out;
503
+ }
504
+ function humanizeAssetName(path) {
505
+ const base = path.slice(path.lastIndexOf("/") + 1);
506
+ const stem2 = base.includes(".") ? base.slice(0, base.lastIndexOf(".")) : base;
507
+ return stem2.replace(/[-_.]+/g, " ").replace(/\s+/g, " ").trim();
508
+ }
509
+ function boundedText(parts, countTokens) {
510
+ const unique = [];
511
+ for (const part of parts.map((p) => p.trim()).filter(Boolean)) {
512
+ if (!unique.some((u) => u.toLowerCase() === part.toLowerCase())) unique.push(part);
513
+ }
514
+ let text = unique.join("\n").normalize("NFC");
515
+ if (countTokens(text) <= MEDIA_ROW_TEXT_MAX_TOKENS) return text;
516
+ const words = text.split(/(\s+)/);
517
+ let lo = 0;
518
+ let hi = words.length;
519
+ while (lo < hi) {
520
+ const mid = Math.ceil((lo + hi) / 2);
521
+ if (countTokens(words.slice(0, mid).join("")) <= MEDIA_ROW_TEXT_MAX_TOKENS) lo = mid;
522
+ else hi = mid - 1;
523
+ }
524
+ text = words.slice(0, lo).join("").trim();
525
+ return text;
526
+ }
527
+ function headingAt(chunks, line) {
528
+ const inside = chunks.find((c) => c.lineStart <= line && line <= c.lineEnd);
529
+ if (inside) return inside.headingPath;
530
+ let before;
531
+ for (const c of chunks) if (c.lineEnd < line) before = c;
532
+ return before?.headingPath ?? [];
533
+ }
534
+ function windowHash(assetSha256, startMs, endMs) {
535
+ if (startMs === null || endMs === null) return assetSha256;
536
+ return createHash2("sha256").update(`${assetSha256}\0${startMs}\0${endMs}`).digest("hex");
537
+ }
538
+ async function collectMediaRows(args) {
539
+ const byPath = new Map(args.assets.map((a) => [a.path, a]));
540
+ const embedded = /* @__PURE__ */ new Map();
541
+ const warned = /* @__PURE__ */ new Set();
542
+ const out = /* @__PURE__ */ new Map();
543
+ for (const doc of args.documents) {
544
+ const rows = [];
545
+ const seen = /* @__PURE__ */ new Set();
546
+ for (const ref of mediaReferences(doc.markdown)) {
547
+ if (seen.has(ref.path)) continue;
548
+ seen.add(ref.path);
549
+ const asset = byPath.get(ref.path);
550
+ const modality = assetModality(ref.path);
551
+ const mimeType = assetContentType(ref.path);
552
+ if (!asset || !modality || !mimeType || mimeType === "image/svg+xml") continue;
553
+ const block = modality === "audio" ? args.profile.media?.audio : modality === "video" ? args.profile.media?.video : args.profile.media?.image;
554
+ if (!block) {
555
+ if (!warned.has(modality)) {
556
+ warned.add(modality);
557
+ args.onWarning?.(
558
+ `profile ${args.profile.id} describes no ${modality} encoder; ${modality} references get no media rows`
559
+ );
560
+ }
561
+ continue;
562
+ }
563
+ let pending = embedded.get(ref.path);
564
+ if (!pending) {
565
+ pending = args.embedMedia({ modality, path: ref.path, bytes: asset.bytes });
566
+ embedded.set(ref.path, pending);
567
+ }
568
+ const results = [...await pending].sort((a, b) => (a.startMs ?? 0) - (b.startMs ?? 0));
569
+ const headingPath = headingAt(doc.chunks, ref.line);
570
+ for (const result of results) {
571
+ const startMs = modality === "image" ? null : result.startMs ?? 0;
572
+ const endMs = modality === "image" ? null : result.endMs ?? null;
573
+ if (startMs !== null && (endMs === null || endMs <= startMs)) {
574
+ throw new Error(`${ref.path}: a ${modality} window needs startMs < endMs`);
575
+ }
576
+ rows.push({
577
+ modality,
578
+ assetPath: ref.path,
579
+ mimeType,
580
+ width: result.width ?? null,
581
+ height: result.height ?? null,
582
+ startMs,
583
+ endMs,
584
+ attributionJson: asset.attribution ? JSON.stringify(asset.attribution) : null,
585
+ headingPath,
586
+ line: ref.line,
587
+ text: boundedText(
588
+ [ref.alt, ref.caption, humanizeAssetName(ref.path), result.transcript ?? ""],
589
+ args.countTokens
590
+ ),
591
+ contentHash: windowHash(asset.sha256, startMs, endMs),
592
+ vector: result.vector
593
+ });
594
+ }
595
+ }
596
+ if (rows.length > MAX_MEDIA_ROWS_PER_DOCUMENT) {
597
+ throw new Error(
598
+ `document ${doc.id} has ${rows.length} media rows; the limit is ${MAX_MEDIA_ROWS_PER_DOCUMENT}`
599
+ );
600
+ }
601
+ if (rows.length > 0) out.set(doc.id, rows);
602
+ }
603
+ return out;
604
+ }
605
+
647
606
  // src/compiler/compile.ts
648
607
  async function compileKnowledgeCatalog(opts) {
649
608
  if (opts.topics.length === 0) {
@@ -701,15 +660,40 @@ async function compileKnowledgeCatalog(opts) {
701
660
  leafTopicId,
702
661
  placements,
703
662
  metaJson: encodeDocumentMeta(doc),
704
- chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
663
+ chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts),
664
+ media: []
705
665
  });
706
666
  }
707
667
  prepared.sort(
708
668
  (a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
709
669
  );
710
670
  const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
671
+ if (opts.embedMedia) {
672
+ const rows = await collectMediaRows({
673
+ documents: prepared.map((p) => ({
674
+ id: p.doc.id,
675
+ title: p.doc.title,
676
+ markdown: normalizeMarkdown(p.doc.markdown),
677
+ chunks: p.chunks
678
+ })),
679
+ assets: preparedAssets,
680
+ profile,
681
+ embedMedia: opts.embedMedia,
682
+ countTokens: opts.countTokens,
683
+ onWarning: opts.onWarning
684
+ });
685
+ for (const p of prepared) p.media = rows.get(p.doc.id) ?? [];
686
+ }
687
+ const mediaCounts = { image: 0, video: 0, audio: 0 };
688
+ for (const p of prepared) for (const row of p.media) mediaCounts[row.modality] += 1;
689
+ const totalMedia = mediaCounts.image + mediaCounts.video + mediaCounts.audio;
690
+ const totalRows = totalChunks + totalMedia;
691
+ const rowsOf = (p) => p.chunks.length + p.media.length;
692
+ const formatVersion = catalogFormatVersion(profile, preparedAssets, totalMedia);
693
+ const indexSchemaVersion = GEZK_FORMAT_GENERATIONS[formatVersion];
694
+ const shardDdl = shardDdlFor(indexSchemaVersion);
711
695
  const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
712
- const embedded = totalChunks <= shardTarget;
696
+ const embedded = totalRows <= shardTarget;
713
697
  const semanticFill = opts.shardFill === "semantic" && !embedded;
714
698
  const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
715
699
  p.doc.title,
@@ -726,7 +710,7 @@ async function compileKnowledgeCatalog(opts) {
726
710
  docVectors = await stageVectors({
727
711
  prepared,
728
712
  stagedPath,
729
- dimensions: profile.dimensions,
713
+ profile,
730
714
  batchSize: opts.embedBatchSize ?? 32,
731
715
  embed: opts.embed,
732
716
  embedInput,
@@ -739,11 +723,11 @@ async function compileKnowledgeCatalog(opts) {
739
723
  if (embedded) {
740
724
  for (const p of prepared) shardOf.set(p.doc.id, 0);
741
725
  } else if (semanticFill) {
742
- shardCount = Math.ceil(totalChunks / shardTarget);
743
- const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
726
+ shardCount = Math.ceil(totalRows / shardTarget);
727
+ const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalRows / shardCount * 1.02));
744
728
  const assignment = semanticShards(
745
729
  docVectors,
746
- prepared.map((p) => p.chunks.length),
730
+ prepared.map(rowsOf),
747
731
  shardCount,
748
732
  capacity,
749
733
  seedFor(`${opts.catalog.id}#shard-fill`, 0)
@@ -755,19 +739,19 @@ async function compileKnowledgeCatalog(opts) {
755
739
  let current = 0;
756
740
  let filled = 0;
757
741
  for (const p of prepared) {
758
- if (filled > 0 && filled + p.chunks.length > shardTarget) {
742
+ if (filled > 0 && filled + rowsOf(p) > shardTarget) {
759
743
  current++;
760
744
  filled = 0;
761
745
  }
762
746
  shardOf.set(p.doc.id, current);
763
- filled += p.chunks.length;
747
+ filled += rowsOf(p);
764
748
  }
765
749
  shardCount = current + 1;
766
750
  }
767
751
  const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
768
752
  const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
769
753
  const shardDbs = /* @__PURE__ */ new Map();
770
- const router = createCatalogDb(routerStaged);
754
+ const router = createCatalogDb(routerStaged, indexSchemaVersion);
771
755
  const closeAll = () => {
772
756
  for (const db of [router, ...shardDbs.values()]) {
773
757
  try {
@@ -780,11 +764,11 @@ async function compileKnowledgeCatalog(opts) {
780
764
  router.exec(ROUTER_DDL);
781
765
  router.exec("BEGIN");
782
766
  if (embedded) {
783
- router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
767
+ router.exec(shardDdl.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
784
768
  }
785
769
  const commonMeta = {
786
- format_version: String(GEZK_FORMAT_VERSION),
787
- index_schema_version: String(GEZK_INDEX_SCHEMA_VERSION),
770
+ format_version: String(formatVersion),
771
+ index_schema_version: String(indexSchemaVersion),
788
772
  catalog_id: opts.catalog.id,
789
773
  catalog_version: opts.catalog.version,
790
774
  embedding_profile_id: profile.id,
@@ -864,7 +848,7 @@ async function compileKnowledgeCatalog(opts) {
864
848
  p.leafTopicId,
865
849
  p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
866
850
  BigInt(shardOf.get(p.doc.id)),
867
- BigInt(p.chunks.length),
851
+ BigInt(rowsOf(p)),
868
852
  p.doc.sourceUrl ?? null,
869
853
  p.doc.sourceRevision ?? null,
870
854
  p.doc.sourceUpdatedAt ?? null,
@@ -909,8 +893,8 @@ async function compileKnowledgeCatalog(opts) {
909
893
  if (embedded) return router;
910
894
  let db = shardDbs.get(id);
911
895
  if (!db) {
912
- db = createCatalogDb(join(opts.workDir, shardPath(id)));
913
- db.exec(SHARD_DDL);
896
+ db = createCatalogDb(join(opts.workDir, shardPath(id)), indexSchemaVersion);
897
+ db.exec(shardDdl);
914
898
  db.exec("BEGIN");
915
899
  shardDbs.set(id, db);
916
900
  }
@@ -941,7 +925,8 @@ async function compileKnowledgeCatalog(opts) {
941
925
  );
942
926
  }
943
927
  for (let i = 0; i < vectors.length; i++) {
944
- const unit = l2Normalize(vectors[i]);
928
+ const raw = vectors[i];
929
+ const unit = staged ? l2Normalize(raw) : profileUnitVector(profile, raw);
945
930
  if (unit.length !== profile.dimensions) {
946
931
  throw new Error(
947
932
  `embedder returned dim ${unit.length}, profile expects ${profile.dimensions}`
@@ -1005,6 +990,46 @@ async function compileKnowledgeCatalog(opts) {
1005
990
  pendingRows.push({ shardId, chunkId });
1006
991
  if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
1007
992
  }
993
+ if (p.media.length > 0) {
994
+ const insertMedia = db.prepare(
995
+ `INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
996
+ line_start, line_end, token_count, content_hash, text, modality, asset_path, mime_type,
997
+ width, height, start_ms, end_ms, attribution_json)
998
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
999
+ );
1000
+ const stmts = vecStmtsFor(db);
1001
+ p.media.forEach((media, index) => {
1002
+ const ordinal = p.chunks.length + index;
1003
+ const headingText = media.headingPath.join(" > ");
1004
+ const row = insertMedia.get(
1005
+ chunkUid(p.doc.id, ordinal, `media:${media.modality}:${media.contentHash}`),
1006
+ p.doc.id,
1007
+ BigInt(ordinal),
1008
+ p.doc.title,
1009
+ JSON.stringify(media.headingPath),
1010
+ headingText,
1011
+ BigInt(media.line),
1012
+ BigInt(media.line),
1013
+ BigInt(opts.countTokens(media.text)),
1014
+ media.contentHash,
1015
+ media.text,
1016
+ media.modality,
1017
+ media.assetPath,
1018
+ media.mimeType,
1019
+ media.width === null ? null : BigInt(media.width),
1020
+ media.height === null ? null : BigInt(media.height),
1021
+ media.startMs === null ? null : BigInt(media.startMs),
1022
+ media.endMs === null ? null : BigInt(media.endMs),
1023
+ media.attributionJson
1024
+ );
1025
+ const chunkId = Number(row.id);
1026
+ insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, media.text);
1027
+ shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
1028
+ const unit = profileUnitVector(profile, media.vector);
1029
+ stmts.vec.run(BigInt(chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
1030
+ stmts.int8.run(BigInt(chunkId), Buffer.from(quantizeInt8(unit).buffer));
1031
+ });
1032
+ }
1008
1033
  }
1009
1034
  await flushEmbeds();
1010
1035
  staged?.close();
@@ -1016,7 +1041,7 @@ async function compileKnowledgeCatalog(opts) {
1016
1041
  chunk_count: String(shardChunkCounts.get(id) ?? 0)
1017
1042
  });
1018
1043
  }
1019
- if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
1044
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalRows) });
1020
1045
  const finalDir = join(opts.workDir, "final");
1021
1046
  mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
1022
1047
  const vacuumed = [];
@@ -1128,7 +1153,7 @@ async function compileKnowledgeCatalog(opts) {
1128
1153
  files.push({
1129
1154
  path,
1130
1155
  sizeBytes: bytes.length,
1131
- sha256: createHash2("sha256").update(bytes).digest("hex")
1156
+ sha256: createHash3("sha256").update(bytes).digest("hex")
1132
1157
  });
1133
1158
  }
1134
1159
  for (const asset of preparedAssets) {
@@ -1137,8 +1162,8 @@ async function compileKnowledgeCatalog(opts) {
1137
1162
  assertAssetReferencesDeclared(prepared, preparedAssets);
1138
1163
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
1139
1164
  kind: GEZK_MANIFEST_KIND,
1140
- formatVersion: GEZK_FORMAT_VERSION,
1141
- indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
1165
+ formatVersion,
1166
+ indexSchemaVersion,
1142
1167
  id: opts.catalog.id,
1143
1168
  version: opts.catalog.version,
1144
1169
  name: opts.catalog.name,
@@ -1167,9 +1192,10 @@ async function compileKnowledgeCatalog(opts) {
1167
1192
  },
1168
1193
  counts: {
1169
1194
  documents: prepared.length,
1170
- chunks: totalChunks,
1195
+ chunks: totalRows,
1171
1196
  shards: shardCount,
1172
- assets: preparedAssets.length
1197
+ assets: preparedAssets.length,
1198
+ ...formatVersion === "0.8" ? { media: mediaCounts } : {}
1173
1199
  },
1174
1200
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
1175
1201
  spatial: spatialManifest(
@@ -1177,7 +1203,7 @@ async function compileKnowledgeCatalog(opts) {
1177
1203
  (p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
1178
1204
  )
1179
1205
  ),
1180
- requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
1206
+ requires: { formatVersion, features: [] },
1181
1207
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
1182
1208
  toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
1183
1209
  });
@@ -1201,7 +1227,8 @@ async function compileKnowledgeCatalog(opts) {
1201
1227
  return {
1202
1228
  manifest,
1203
1229
  documents: prepared.length,
1204
- chunks: totalChunks,
1230
+ chunks: totalRows,
1231
+ media: mediaCounts,
1205
1232
  shards: shardCount,
1206
1233
  archiveBytes
1207
1234
  };
@@ -1285,11 +1312,15 @@ function assertAssetReferencesDeclared(prepared, assets) {
1285
1312
  }
1286
1313
  }
1287
1314
  }
1288
- function createCatalogDb(absPath) {
1315
+ function catalogFormatVersion(profile, assets, mediaRows = 0) {
1316
+ const needsMedia = mediaRows > 0 || assets.some((asset) => isKnowledgeMediaAssetPath(asset.path));
1317
+ return needsMedia || embeddingProfileMinimumFormat(profile) === "0.8" ? "0.8" : "0.7";
1318
+ }
1319
+ function createCatalogDb(absPath, indexSchemaVersion) {
1289
1320
  const db = new DatabaseSync(absPath);
1290
1321
  db.exec("PRAGMA page_size=8192");
1291
1322
  db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
1292
- db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
1323
+ db.exec(`PRAGMA user_version=${indexSchemaVersion}`);
1293
1324
  db.exec("PRAGMA journal_mode=DELETE");
1294
1325
  return db;
1295
1326
  }
@@ -1334,7 +1365,8 @@ ${path.join(" > ")}
1334
1365
  return header;
1335
1366
  }
1336
1367
  async function stageVectors(args) {
1337
- const { prepared, dimensions } = args;
1368
+ const { prepared, profile } = args;
1369
+ const { dimensions } = profile;
1338
1370
  const docVectors = prepared.map(() => new Float32Array(dimensions));
1339
1371
  const fd = openSync(args.stagedPath, "w");
1340
1372
  let texts = [];
@@ -1348,10 +1380,7 @@ async function stageVectors(args) {
1348
1380
  }
1349
1381
  const block = new Float32Array(texts.length * dimensions);
1350
1382
  for (let i = 0; i < vectors.length; i++) {
1351
- const unit = l2Normalize(vectors[i]);
1352
- if (unit.length !== dimensions) {
1353
- throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
1354
- }
1383
+ const unit = profileUnitVector(profile, vectors[i]);
1355
1384
  block.set(unit, i * dimensions);
1356
1385
  const sum = docVectors[owners[i]];
1357
1386
  for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
@@ -1375,7 +1404,9 @@ async function stageVectors(args) {
1375
1404
  } finally {
1376
1405
  closeSync(fd);
1377
1406
  }
1378
- return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
1407
+ return docVectors.map(
1408
+ (v) => v.some((x) => x !== 0) ? Float32Array.from(l2Normalize(Array.from(v))) : v
1409
+ );
1379
1410
  }
1380
1411
  var StagedVectorReader = class {
1381
1412
  constructor(path, dimensions) {
@@ -1417,7 +1448,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1417
1448
  return sum;
1418
1449
  };
1419
1450
  const stride = Math.max(1, Math.ceil(n / 2e4));
1420
- const sample = docVectors.filter((_, i) => i % stride === 0);
1451
+ const sample = docVectors.filter((v, i) => i % stride === 0 && v.some((x) => x !== 0));
1421
1452
  const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
1422
1453
  for (let i = 0; centers.length < shardCount; i++) {
1423
1454
  centers.push(
@@ -1486,7 +1517,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1486
1517
  return assignment.map((s) => renumber.get(s));
1487
1518
  }
1488
1519
  function seedFor(catalogId, shardId) {
1489
- const digest = createHash2("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1520
+ const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1490
1521
  return digest.readBigUInt64LE(24);
1491
1522
  }
1492
1523
  function makePrng(seed) {
@@ -1571,7 +1602,7 @@ function kMeans(samples, k, seed) {
1571
1602
  }
1572
1603
 
1573
1604
  // src/archive/read.ts
1574
- import { createHash as createHash3 } from "crypto";
1605
+ import { createHash as createHash4 } from "crypto";
1575
1606
  import { createWriteStream as createWriteStream2 } from "fs";
1576
1607
  import { mkdir as mkdir2 } from "fs/promises";
1577
1608
  import { createRequire as createRequire4 } from "module";
@@ -1861,7 +1892,7 @@ async function extractGezkVerified(archivePath, destDir) {
1861
1892
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1862
1893
  return;
1863
1894
  }
1864
- const hasher = createHash3("sha256");
1895
+ const hasher = createHash4("sha256");
1865
1896
  let sizeBytes = 0;
1866
1897
  const verify = new Transform({
1867
1898
  transform(chunk, _encoding, callback) {
@@ -1931,6 +1962,14 @@ var CatalogOpenError = class extends Error {
1931
1962
  }
1932
1963
  reason;
1933
1964
  };
1965
+ var CatalogQueryError = class extends Error {
1966
+ constructor(message, reason) {
1967
+ super(message);
1968
+ this.reason = reason;
1969
+ this.name = "CatalogQueryError";
1970
+ }
1971
+ reason;
1972
+ };
1934
1973
  function openCatalogDatabase(absPath) {
1935
1974
  const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1936
1975
  let db;
@@ -1984,15 +2023,15 @@ function readPragmaNumber(db, pragma) {
1984
2023
  }
1985
2024
 
1986
2025
  // src/reader/catalog-handle.ts
1987
- import { readFileSync as readFileSync3 } from "fs";
2026
+ import { readFileSync as readFileSync3, statSync as statSync2 } from "fs";
1988
2027
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1989
2028
  import { brotliDecompressSync } from "zlib";
1990
2029
  import {
1991
2030
  KnowledgeEmbeddingProfileSchema,
1992
- MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1993
- assetContentType,
2031
+ assetContentType as assetContentType2,
1994
2032
  embeddingProfileCenter,
1995
- isKnowledgeAssetPath
2033
+ isKnowledgeAssetPath,
2034
+ maxKnowledgeAssetBytes as maxKnowledgeAssetBytes2
1996
2035
  } from "@bendyline/gezk";
1997
2036
 
1998
2037
  // src/reader/bit-scan.ts
@@ -2065,7 +2104,7 @@ function hammingTopK(index, query, k) {
2065
2104
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
2066
2105
  return hits;
2067
2106
  }
2068
- function asymmetricTopK(index, query, k, eligibleChunkIds) {
2107
+ function asymmetricTopK(index, query, k, eligibleChunkIds, excludedChunkIds) {
2069
2108
  const { bits, bytesPerRow, rows } = index;
2070
2109
  if (Math.ceil(query.length / 8) !== bytesPerRow) {
2071
2110
  throw new Error(
@@ -2122,7 +2161,7 @@ function asymmetricTopK(index, query, k, eligibleChunkIds) {
2122
2161
  };
2123
2162
  let offset = 0;
2124
2163
  for (let r = 0; r < rows; r++) {
2125
- if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
2164
+ if (eligibleChunkIds && !eligibleChunkIds.has(r + 1) || excludedChunkIds?.has(r + 1)) {
2126
2165
  offset += bytesPerRow;
2127
2166
  continue;
2128
2167
  }
@@ -2261,6 +2300,40 @@ var DocumentSpatialIndex = class {
2261
2300
 
2262
2301
  // src/reader/catalog-handle.ts
2263
2302
  var TOPIC_WALK_MAX_DEPTH = 32;
2303
+ var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
2304
+ var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
2305
+ function mediaOf(row) {
2306
+ const modality = row.modality;
2307
+ if (!modality || modality === "text") return void 0;
2308
+ const num = (v) => v === null || v === void 0 ? void 0 : Number(v);
2309
+ let attribution;
2310
+ if (typeof row.attribution_json === "string") {
2311
+ try {
2312
+ const parsed = JSON.parse(row.attribution_json);
2313
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
2314
+ attribution = parsed;
2315
+ }
2316
+ } catch {
2317
+ attribution = void 0;
2318
+ }
2319
+ }
2320
+ const media = {
2321
+ modality,
2322
+ assetPath: row.asset_path,
2323
+ mimeType: row.mime_type
2324
+ };
2325
+ const width = num(row.width);
2326
+ const height = num(row.height);
2327
+ const startMs = num(row.start_ms);
2328
+ const endMs = num(row.end_ms);
2329
+ if (width !== void 0) media.width = width;
2330
+ if (height !== void 0) media.height = height;
2331
+ if (startMs !== void 0) media.startMs = startMs;
2332
+ if (endMs !== void 0) media.endMs = endMs;
2333
+ if (typeof row.thumbnail_path === "string") media.thumbnailAssetPath = row.thumbnail_path;
2334
+ if (attribution) media.attribution = attribution;
2335
+ return media;
2336
+ }
2264
2337
  var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
2265
2338
  var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
2266
2339
  var TITLE_ROUTE_SHARDS = 1;
@@ -2281,6 +2354,7 @@ var CatalogHandle = class _CatalogHandle {
2281
2354
  router;
2282
2355
  connections = /* @__PURE__ */ new Map();
2283
2356
  bitIndexes = /* @__PURE__ */ new Map();
2357
+ mediaIndexes = /* @__PURE__ */ new Map();
2284
2358
  /** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
2285
2359
  chunkTermRows = /* @__PURE__ */ new Map();
2286
2360
  spatial;
@@ -2414,6 +2488,127 @@ var CatalogHandle = class _CatalogHandle {
2414
2488
  this.binaryCenterCache = embeddingProfileCenter(profile.data);
2415
2489
  return this.binaryCenterCache;
2416
2490
  }
2491
+ /** The chunk columns this catalog's schema has. */
2492
+ chunkColumns() {
2493
+ return this.schemaVersion >= 5 ? `${CHUNK_COLUMNS}, ${MEDIA_COLUMNS}` : CHUNK_COLUMNS;
2494
+ }
2495
+ /** A shard's media rows with their int8 vectors (empty before index schema 5). */
2496
+ shardMedia(shard) {
2497
+ const cached = this.mediaIndexes.get(shard.path);
2498
+ if (cached) return cached;
2499
+ const empty = {
2500
+ ids: [],
2501
+ modalities: [],
2502
+ int8: new Int8Array(0),
2503
+ idSet: /* @__PURE__ */ new Set()
2504
+ };
2505
+ if (this.schemaVersion < 5) {
2506
+ this.mediaIndexes.set(shard.path, empty);
2507
+ return empty;
2508
+ }
2509
+ const dims = this.dimensions();
2510
+ const rows = this.shardDb(shard).prepare(
2511
+ `SELECT c.id, c.modality, v.v FROM chunks c JOIN chunk_vectors_int8 v ON v.chunk_id = c.id
2512
+ WHERE c.modality <> 'text' ORDER BY c.id`
2513
+ ).all();
2514
+ const int8 = new Int8Array(rows.length * dims);
2515
+ rows.forEach((row, i) => {
2516
+ if (row.v.byteLength !== dims) {
2517
+ throw new CatalogOpenError(`int8 vector width ${row.v.byteLength} != ${dims}`, "corrupt");
2518
+ }
2519
+ int8.set(new Int8Array(row.v.buffer, row.v.byteOffset, dims), i * dims);
2520
+ });
2521
+ const ids = rows.map((r) => Number(r.id));
2522
+ const index = {
2523
+ ids,
2524
+ modalities: rows.map((r) => r.modality),
2525
+ int8,
2526
+ idSet: new Set(ids)
2527
+ };
2528
+ this.mediaIndexes.set(shard.path, index);
2529
+ return index;
2530
+ }
2531
+ /** Media rows this catalog holds, by modality (all zero before index schema 5). */
2532
+ mediaCounts() {
2533
+ const counts = { image: 0, video: 0, audio: 0 };
2534
+ for (const shard of this.shards) {
2535
+ for (const modality of this.shardMedia(shard).modalities) counts[modality] += 1;
2536
+ }
2537
+ return counts;
2538
+ }
2539
+ /**
2540
+ * The media lane: every media row of every shard scored exactly against
2541
+ * the query (int8 rerank), keeping the best `perModality` of each kind.
2542
+ * Images sit at a lower cosine than text in a shared space, so in the text
2543
+ * lane's sign-bit pre-filter they would rarely survive the cut; media rows
2544
+ * are few enough (capped per catalog) to score them all.
2545
+ */
2546
+ searchMedia(queryVector, opts = {}) {
2547
+ this.assertQueryWidth(queryVector);
2548
+ const perModality = opts.perModality ?? 4;
2549
+ const dims = this.dimensions();
2550
+ const best = /* @__PURE__ */ new Map();
2551
+ for (const shard of this.shards) {
2552
+ const index = this.shardMedia(shard);
2553
+ for (let i = 0; i < index.ids.length; i++) {
2554
+ let dot = 0;
2555
+ const base = i * dims;
2556
+ for (let d = 0; d < dims; d++)
2557
+ dot += queryVector[d] * (index.int8[base + d] / 127);
2558
+ const modality = index.modalities[i];
2559
+ const list = best.get(modality) ?? [];
2560
+ list.push({ shard, id: index.ids[i], cosine: dot });
2561
+ best.set(modality, list);
2562
+ }
2563
+ }
2564
+ const hits = [];
2565
+ for (const list of best.values()) {
2566
+ list.sort((a, b) => b.cosine - a.cosine || a.id - b.id);
2567
+ let kept = 0;
2568
+ for (const candidate of list) {
2569
+ if (kept >= perModality) break;
2570
+ const row = this.shardDb(candidate.shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`).get(BigInt(candidate.id));
2571
+ if (!row) continue;
2572
+ if (opts.allowedDocumentIds && !opts.allowedDocumentIds.has(row.document_id))
2573
+ continue;
2574
+ kept++;
2575
+ hits.push({
2576
+ ...this.hitFromRow(row, candidate.shard.id),
2577
+ cosine: candidate.cosine,
2578
+ source: "vector"
2579
+ });
2580
+ }
2581
+ }
2582
+ return hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
2583
+ }
2584
+ hitFromRow(row, shardId) {
2585
+ const media = mediaOf(row);
2586
+ return {
2587
+ chunkUid: row.chunk_uid,
2588
+ documentId: row.document_id,
2589
+ title: row.title,
2590
+ headingPath: parseHeadingPath(row.heading_path),
2591
+ lineStart: Number(row.line_start),
2592
+ lineEnd: Number(row.line_end),
2593
+ text: row.text,
2594
+ shardId,
2595
+ ...media ? { media } : {}
2596
+ };
2597
+ }
2598
+ /**
2599
+ * Refuse a query vector whose width is not the catalog's stored width,
2600
+ * before any scan: routing and rerank would otherwise score a silent
2601
+ * prefix or fail deep inside a shard with an untyped error.
2602
+ */
2603
+ assertQueryWidth(queryVector) {
2604
+ const dims = this.dimensions();
2605
+ if (queryVector.length !== dims) {
2606
+ throw new CatalogQueryError(
2607
+ `query vector has ${queryVector.length} dimensions, the catalog stores ${dims}`,
2608
+ "dimension"
2609
+ );
2610
+ }
2611
+ }
2417
2612
  /** The embedding dimension from the router's profile echo. */
2418
2613
  dimensions() {
2419
2614
  const raw = this.meta.embedding_profile_json;
@@ -2597,21 +2792,9 @@ var CatalogHandle = class _CatalogHandle {
2597
2792
  const shardId = Number(doc.shard_id);
2598
2793
  const shard = this.shards.find((s) => s.id === shardId);
2599
2794
  if (!shard) return null;
2600
- const row = this.shardDb(shard).prepare(
2601
- `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
2602
- FROM chunks WHERE chunk_uid = ? AND document_id = ?`
2603
- ).get(chunkUid2, documentId);
2795
+ const row = this.shardDb(shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE chunk_uid = ? AND document_id = ?`).get(chunkUid2, documentId);
2604
2796
  if (!row) return null;
2605
- return {
2606
- chunkUid: row.chunk_uid,
2607
- documentId: row.document_id,
2608
- title: row.title,
2609
- headingPath: parseHeadingPath(row.heading_path),
2610
- lineStart: Number(row.line_start),
2611
- lineEnd: Number(row.line_end),
2612
- text: row.text,
2613
- shardId
2614
- };
2797
+ return this.hitFromRow(row, shardId);
2615
2798
  }
2616
2799
  /** Documents whose `topic_id` names no declared topic (a validator check). */
2617
2800
  documentsWithUndeclaredTopic() {
@@ -2700,7 +2883,7 @@ var CatalogHandle = class _CatalogHandle {
2700
2883
  for (const file of files) {
2701
2884
  const path = file?.path;
2702
2885
  if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
2703
- const contentType = assetContentType(path);
2886
+ const contentType = assetContentType2(path);
2704
2887
  if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
2705
2888
  continue;
2706
2889
  }
@@ -2714,11 +2897,37 @@ var CatalogHandle = class _CatalogHandle {
2714
2897
  assets() {
2715
2898
  return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
2716
2899
  }
2900
+ /**
2901
+ * Where a declared asset lives on disk, so a large one (video, audio) can be
2902
+ * streamed with byte ranges instead of read whole. Checks containment and
2903
+ * the manifest size; the content hash was verified once, at extraction.
2904
+ */
2905
+ assetFile(path) {
2906
+ const info = this.assetIndex().get(path);
2907
+ if (!info) return null;
2908
+ if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
2909
+ throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
2910
+ }
2911
+ const absPath = this.resolveCatalogPath(path);
2912
+ let size;
2913
+ try {
2914
+ size = statSync2(absPath).size;
2915
+ } catch (error) {
2916
+ throw new CatalogOpenError(
2917
+ `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
2918
+ "corrupt"
2919
+ );
2920
+ }
2921
+ if (size !== info.sizeBytes) {
2922
+ throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
2923
+ }
2924
+ return { ...info, absPath };
2925
+ }
2717
2926
  /** One declared asset's bytes, or null when the catalog declares no such asset. */
2718
2927
  readAsset(path) {
2719
2928
  const info = this.assetIndex().get(path);
2720
2929
  if (!info) return null;
2721
- if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
2930
+ if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
2722
2931
  throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
2723
2932
  }
2724
2933
  let bytes;
@@ -2811,6 +3020,7 @@ var CatalogHandle = class _CatalogHandle {
2811
3020
  * takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
2812
3021
  */
2813
3022
  scoreShards(queryVector) {
3023
+ this.assertQueryWidth(queryVector);
2814
3024
  const rows = this.router.db.prepare("SELECT shard_id, embedding FROM route_centroids").all();
2815
3025
  const best = /* @__PURE__ */ new Map();
2816
3026
  for (const row of rows) {
@@ -2820,8 +3030,7 @@ var CatalogHandle = class _CatalogHandle {
2820
3030
  Math.floor(row.embedding.byteLength / 4)
2821
3031
  );
2822
3032
  let dot = 0;
2823
- const n = Math.min(centroid.length, queryVector.length);
2824
- for (let i = 0; i < n; i++) dot += centroid[i] * queryVector[i];
3033
+ for (let i = 0; i < queryVector.length; i++) dot += centroid[i] * queryVector[i];
2825
3034
  const id = Number(row.shard_id);
2826
3035
  if (dot > (best.get(id) ?? Number.NEGATIVE_INFINITY)) best.set(id, dot);
2827
3036
  }
@@ -2894,6 +3103,7 @@ var CatalogHandle = class _CatalogHandle {
2894
3103
  * after global cross-catalog routing has already spent the S budget.
2895
3104
  */
2896
3105
  searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
3106
+ this.assertQueryWidth(queryVector);
2897
3107
  const center = this.binaryCenter();
2898
3108
  const scanQuery = center ? centerVector(queryVector, center) : queryVector;
2899
3109
  const hits = [];
@@ -2906,12 +3116,18 @@ var CatalogHandle = class _CatalogHandle {
2906
3116
  "SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
2907
3117
  ).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
2908
3118
  ) : void 0;
2909
- const k = rerankK(finalK, eligible?.size ?? shard.chunkCount);
2910
- const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k, eligible).map(
2911
- (hit) => ({
2912
- chunk_id: hit.chunkId
2913
- })
2914
- );
3119
+ const media = this.shardMedia(shard);
3120
+ const k = rerankK(finalK, eligible?.size ?? shard.chunkCount - media.ids.length);
3121
+ const excluded = media.ids.length > 0 ? media.idSet : void 0;
3122
+ const candidates = asymmetricTopK(
3123
+ this.shardBits(shard),
3124
+ scanQuery,
3125
+ k,
3126
+ eligible,
3127
+ excluded
3128
+ ).map((hit) => ({
3129
+ chunk_id: hit.chunkId
3130
+ }));
2915
3131
  if (candidates.length === 0) continue;
2916
3132
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
2917
3133
  const reranked = candidates.map((c) => {
@@ -2920,25 +3136,11 @@ var CatalogHandle = class _CatalogHandle {
2920
3136
  const int8 = new Int8Array(row.v.buffer, row.v.byteOffset, row.v.byteLength);
2921
3137
  return { chunkId: Number(c.chunk_id), cosine: rerankScore(queryVector, int8) };
2922
3138
  }).filter((r) => r !== null).sort((a, b) => b.cosine - a.cosine).slice(0, finalK);
2923
- const getChunk = db.prepare(
2924
- `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
2925
- FROM chunks WHERE id = ?`
2926
- );
3139
+ const getChunk = db.prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`);
2927
3140
  for (const r of reranked) {
2928
3141
  const row = getChunk.get(BigInt(r.chunkId));
2929
3142
  if (!row) continue;
2930
- hits.push({
2931
- chunkUid: row.chunk_uid,
2932
- documentId: row.document_id,
2933
- title: row.title,
2934
- headingPath: parseHeadingPath(row.heading_path),
2935
- lineStart: Number(row.line_start),
2936
- lineEnd: Number(row.line_end),
2937
- text: row.text,
2938
- cosine: r.cosine,
2939
- source: "vector",
2940
- shardId
2941
- });
3143
+ hits.push({ ...this.hitFromRow(row, shardId), cosine: r.cosine, source: "vector" });
2942
3144
  }
2943
3145
  }
2944
3146
  hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
@@ -2959,24 +3161,14 @@ var CatalogHandle = class _CatalogHandle {
2959
3161
  );
2960
3162
  if (!match) return hits;
2961
3163
  const rows = db.prepare(
2962
- `SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
3164
+ `SELECT ${this.chunkColumns().split(", ").map((column) => `c.${column}`).join(", ")}
2963
3165
  FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
2964
3166
  WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
2965
3167
  ).all(
2966
3168
  ...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
2967
3169
  );
2968
3170
  for (const row of rows) {
2969
- hits.push({
2970
- chunkUid: row.chunk_uid,
2971
- documentId: row.document_id,
2972
- title: row.title,
2973
- headingPath: parseHeadingPath(row.heading_path),
2974
- lineStart: Number(row.line_start),
2975
- lineEnd: Number(row.line_end),
2976
- text: row.text,
2977
- source: "fts",
2978
- shardId
2979
- });
3171
+ hits.push({ ...this.hitFromRow(row, shardId), source: "fts" });
2980
3172
  }
2981
3173
  } catch {
2982
3174
  }
@@ -3050,17 +3242,18 @@ function parseHeadingPath(raw) {
3050
3242
  }
3051
3243
 
3052
3244
  // src/reader/validate.ts
3053
- import { readFile } from "fs/promises";
3245
+ import { open, readFile } from "fs/promises";
3054
3246
  import { join as join3 } from "path";
3055
3247
  import {
3056
3248
  KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
3057
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
3058
- MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
3059
- MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
3060
3249
  assetExtension as assetExtension2,
3061
3250
  assetKindForExtension as assetKindForExtension2,
3251
+ assetModality as assetModality2,
3062
3252
  canonicalizeJson as canonicalizeJson2,
3253
+ formatAtLeast as formatAtLeast2,
3063
3254
  isKnowledgeAssetPath as isKnowledgeAssetPath2,
3255
+ isKnowledgeMediaAssetPath as isKnowledgeMediaAssetPath2,
3256
+ knowledgeAssetLimitsProblem as knowledgeAssetLimitsProblem2,
3064
3257
  sniffAssetType as sniffAssetType2,
3065
3258
  svgInertnessProblem as svgInertnessProblem2
3066
3259
  } from "@bendyline/gezk";
@@ -3135,7 +3328,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3135
3328
  handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
3136
3329
  `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
3137
3330
  );
3138
- if (manifest.formatVersion === "0.7") {
3331
+ if (formatAtLeast2(manifest.formatVersion, "0.7")) {
3139
3332
  try {
3140
3333
  const spatial = handle.spatialIntegrity();
3141
3334
  check("document-locations-table", spatial.hasTable);
@@ -3171,13 +3364,16 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3171
3364
  } else {
3172
3365
  const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
3173
3366
  check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
3174
- const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
3175
- const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
3176
- check(
3177
- "assets-limits",
3178
- assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
3179
- `${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
3180
- );
3367
+ if (!formatAtLeast2(manifest.formatVersion, "0.8")) {
3368
+ const media = assetFiles.filter((f) => isKnowledgeMediaAssetPath2(f.path));
3369
+ check(
3370
+ "assets-media-in-0.8",
3371
+ media.length === 0,
3372
+ `audio/video assets need format 0.8: ${media.map((f) => f.path).join(", ")}`
3373
+ );
3374
+ }
3375
+ const limits = knowledgeAssetLimitsProblem2(assetFiles);
3376
+ check("assets-limits", limits === null, limits ?? "");
3181
3377
  check(
3182
3378
  "counts-assets",
3183
3379
  (manifest.counts.assets ?? 0) === assetFiles.length,
@@ -3236,6 +3432,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3236
3432
  bodies.maxCompressedBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024,
3237
3433
  `largest compressed body is ${bodies.maxCompressedBytes} bytes`
3238
3434
  );
3435
+ const mediaRowCounts = { image: 0, video: 0, audio: 0 };
3239
3436
  for (const shard of handle.shards) {
3240
3437
  const conn = openCatalogDatabase(handle.resolveCatalogPath(shard.path));
3241
3438
  const db = conn.db;
@@ -3272,6 +3469,21 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3272
3469
  badBit === 0 && badInt8 === 0,
3273
3470
  `${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
3274
3471
  );
3472
+ if (handle.schemaVersion >= 5) {
3473
+ const rows = db.prepare(
3474
+ `SELECT modality, asset_path, thumbnail_path FROM chunks WHERE modality <> 'text'`
3475
+ ).all();
3476
+ const declaredAssets = new Set(assetFiles.map((f) => f.path));
3477
+ const wrong = rows.filter(
3478
+ (r) => !declaredAssets.has(r.asset_path) || assetModality2(r.asset_path) !== r.modality || r.thumbnail_path !== null && (!declaredAssets.has(r.thumbnail_path) || assetModality2(r.thumbnail_path) !== "image")
3479
+ ).slice(0, 5).map((r) => `${r.modality} ${r.asset_path}`);
3480
+ check(
3481
+ `media-rows:${shard.path}`,
3482
+ wrong.length === 0,
3483
+ `media rows naming an undeclared or mismatched asset: ${wrong.join(", ")}`
3484
+ );
3485
+ for (const r of rows) mediaRowCounts[r.modality] += 1;
3486
+ }
3275
3487
  const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
3276
3488
  check(
3277
3489
  `chunk-ids-dense:${shard.path}`,
@@ -3283,6 +3495,14 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3283
3495
  }
3284
3496
  check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
3285
3497
  }
3498
+ if (handle.schemaVersion >= 5) {
3499
+ const counted = manifest.counts.media ?? { image: 0, video: 0, audio: 0 };
3500
+ check(
3501
+ "counts-media",
3502
+ counted.image === mediaRowCounts.image && counted.video === mediaRowCounts.video && counted.audio === mediaRowCounts.audio,
3503
+ `manifest counts ${JSON.stringify(counted)}, shards hold ${JSON.stringify(mediaRowCounts)}`
3504
+ );
3505
+ }
3286
3506
  if (handle.schemaVersion >= 3) {
3287
3507
  const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
3288
3508
  check(
@@ -3294,7 +3514,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
3294
3514
  for (const file of assetFiles) {
3295
3515
  const ext = assetExtension2(file.path);
3296
3516
  if (!ext) continue;
3297
- const bytes = await readFile(join3(rootDir, file.path));
3517
+ const bytes = isKnowledgeMediaAssetPath2(file.path) ? await readLeadingBytes(join3(rootDir, file.path), 64) : await readFile(join3(rootDir, file.path));
3298
3518
  const kind = sniffAssetType2(bytes);
3299
3519
  check(
3300
3520
  `asset-type:${file.path}`,
@@ -3372,6 +3592,16 @@ function topicTreeProblem(topics) {
3372
3592
  }
3373
3593
  return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
3374
3594
  }
3595
+ async function readLeadingBytes(path, length) {
3596
+ const handle = await open(path, "r");
3597
+ try {
3598
+ const buffer = new Uint8Array(length);
3599
+ const { bytesRead } = await handle.read(buffer, 0, length, 0);
3600
+ return buffer.subarray(0, bytesRead);
3601
+ } finally {
3602
+ await handle.close();
3603
+ }
3604
+ }
3375
3605
 
3376
3606
  // src/export/duckdb.ts
3377
3607
  import { spawn } from "child_process";
@@ -3442,7 +3672,7 @@ function runDuckdbProcess(binaryPath, args, stdin) {
3442
3672
  }
3443
3673
 
3444
3674
  // src/export/parquet.ts
3445
- import { createHash as createHash4 } from "crypto";
3675
+ import { createHash as createHash5 } from "crypto";
3446
3676
  import { createWriteStream as createWriteStream3 } from "fs";
3447
3677
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
3448
3678
  import { join as join4, resolve as resolve3 } from "path";
@@ -3865,7 +4095,7 @@ async function writeParquet(cli, spec) {
3865
4095
  ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
3866
4096
  rows: spec.rows,
3867
4097
  sizeBytes: (await stat2(target)).size,
3868
- sha256: createHash4("sha256").update(bytes).digest("hex")
4098
+ sha256: createHash5("sha256").update(bytes).digest("hex")
3869
4099
  };
3870
4100
  }
3871
4101
  async function findDuckdbBinary(env = process.env) {
@@ -4282,6 +4512,522 @@ var MULTILINGUAL_E5_SMALL_CENTER = [
4282
4512
  0.049673
4283
4513
  ];
4284
4514
 
4515
+ // src/profiles/embeddinggemma-2-center.ts
4516
+ var EMBEDDINGGEMMA_2_512_CENTER = [
4517
+ -3987e-6,
4518
+ 0.017212,
4519
+ 0.023034,
4520
+ -0.020831,
4521
+ 0.013136,
4522
+ 0.018836,
4523
+ 7334e-6,
4524
+ -0.022294,
4525
+ 5809e-6,
4526
+ -0.147176,
4527
+ -5646e-6,
4528
+ -0.022884,
4529
+ -0.011497,
4530
+ 1292e-6,
4531
+ 0.014257,
4532
+ -1878e-6,
4533
+ 0.051834,
4534
+ -0.012991,
4535
+ 933e-6,
4536
+ 0.022558,
4537
+ -0.034902,
4538
+ -0.030019,
4539
+ 0.017879,
4540
+ 0.015619,
4541
+ -0.014837,
4542
+ 0.028657,
4543
+ 0.017575,
4544
+ -0.045617,
4545
+ -0.03244,
4546
+ 0.23277,
4547
+ -0.021971,
4548
+ -0.026583,
4549
+ 0.011485,
4550
+ -0.01468,
4551
+ 0.044649,
4552
+ 0.018019,
4553
+ -0.011519,
4554
+ 0.035574,
4555
+ 512e-5,
4556
+ -0.030136,
4557
+ 4583e-6,
4558
+ 0.018314,
4559
+ -0.028859,
4560
+ 0.027199,
4561
+ 0.020695,
4562
+ -849e-6,
4563
+ 0.015867,
4564
+ -0.024724,
4565
+ -8979e-6,
4566
+ 0.023625,
4567
+ 0.028948,
4568
+ -0.025744,
4569
+ -1547e-6,
4570
+ 0.0281,
4571
+ 0.024258,
4572
+ 0.02262,
4573
+ 1386e-6,
4574
+ 0.015002,
4575
+ 3764e-6,
4576
+ 0.016978,
4577
+ 0.019178,
4578
+ 0.012229,
4579
+ -193e-6,
4580
+ 0.032654,
4581
+ -0.030602,
4582
+ -1749e-6,
4583
+ 0.027403,
4584
+ -4047e-6,
4585
+ 0.031605,
4586
+ 0.023884,
4587
+ 0.015804,
4588
+ 3971e-6,
4589
+ 0.021826,
4590
+ 0.02457,
4591
+ -0.023111,
4592
+ -0.032593,
4593
+ -0.090226,
4594
+ 0.020798,
4595
+ 6402e-6,
4596
+ 0.015607,
4597
+ -8457e-6,
4598
+ -0.010067,
4599
+ 6362e-6,
4600
+ 9973e-6,
4601
+ 0.0122,
4602
+ -0.020958,
4603
+ -0.031666,
4604
+ 0.034654,
4605
+ -0.012847,
4606
+ -1119e-6,
4607
+ 0.024259,
4608
+ 0.018097,
4609
+ 9105e-6,
4610
+ -9523e-6,
4611
+ -0.022682,
4612
+ -5167e-6,
4613
+ -0.022959,
4614
+ 0.012175,
4615
+ -0.020933,
4616
+ 0.014024,
4617
+ -4898e-6,
4618
+ -2018e-6,
4619
+ -7178e-6,
4620
+ 0.038098,
4621
+ 0.016307,
4622
+ 5668e-6,
4623
+ -0.014475,
4624
+ 661e-6,
4625
+ -1287e-6,
4626
+ -0.025024,
4627
+ 5704e-6,
4628
+ -0.32188,
4629
+ -0.026611,
4630
+ 0.032111,
4631
+ -9511e-6,
4632
+ 0.260031,
4633
+ 0.027274,
4634
+ -909e-5,
4635
+ 0.055461,
4636
+ 0.024395,
4637
+ -0.017894,
4638
+ 0.061775,
4639
+ -8529e-6,
4640
+ -1795e-6,
4641
+ -0.017545,
4642
+ 0.011492,
4643
+ 0.044594,
4644
+ 38e-4,
4645
+ -0.02231,
4646
+ 0.029977,
4647
+ 0.010629,
4648
+ -489e-5,
4649
+ 2646e-6,
4650
+ 6408e-6,
4651
+ -0.014453,
4652
+ -0.020764,
4653
+ -0.018043,
4654
+ -0.015625,
4655
+ 0.074213,
4656
+ -0.012319,
4657
+ -8038e-6,
4658
+ -6265e-6,
4659
+ 0.012548,
4660
+ -621e-6,
4661
+ -0.021893,
4662
+ 0.039889,
4663
+ 3623e-6,
4664
+ -0.029173,
4665
+ -716e-6,
4666
+ -0.013168,
4667
+ -6674e-6,
4668
+ -0.010069,
4669
+ -0.02451,
4670
+ 0.015114,
4671
+ -0.012945,
4672
+ 1561e-6,
4673
+ 0.013299,
4674
+ -0.029098,
4675
+ 0.01304,
4676
+ 0.013519,
4677
+ -0.014282,
4678
+ 0.026849,
4679
+ -2462e-6,
4680
+ 0.019473,
4681
+ -8227e-6,
4682
+ -822e-6,
4683
+ -7241e-6,
4684
+ 4259e-6,
4685
+ -0.016026,
4686
+ 7987e-6,
4687
+ -7065e-6,
4688
+ -1229e-6,
4689
+ -0.01583,
4690
+ 0.021277,
4691
+ 0.013837,
4692
+ 0.024282,
4693
+ -7589e-6,
4694
+ -0.016577,
4695
+ 7774e-6,
4696
+ 0.010479,
4697
+ -0.012539,
4698
+ -0.020799,
4699
+ 0.026506,
4700
+ 8733e-6,
4701
+ -0.015001,
4702
+ 2151e-6,
4703
+ -0.011922,
4704
+ -0.02785,
4705
+ -0.060631,
4706
+ 3372e-6,
4707
+ 0.022441,
4708
+ -0.012448,
4709
+ 6827e-6,
4710
+ -7047e-6,
4711
+ 1249e-6,
4712
+ -2934e-6,
4713
+ 4559e-6,
4714
+ -0.011409,
4715
+ -0.012564,
4716
+ 5788e-6,
4717
+ -0.020554,
4718
+ -4292e-6,
4719
+ 0.01927,
4720
+ 0.013487,
4721
+ -0.015484,
4722
+ 9704e-6,
4723
+ 5617e-6,
4724
+ 0.02447,
4725
+ 2056e-6,
4726
+ 0.011381,
4727
+ 0.039831,
4728
+ -7878e-6,
4729
+ -0.011366,
4730
+ -814e-5,
4731
+ 8099e-6,
4732
+ 6637e-6,
4733
+ 154e-6,
4734
+ 0.072273,
4735
+ 7195e-6,
4736
+ -0.011899,
4737
+ 0.016669,
4738
+ 3331e-6,
4739
+ 0.032787,
4740
+ -0.012251,
4741
+ -6049e-6,
4742
+ -0.017814,
4743
+ 0.055136,
4744
+ 0.028157,
4745
+ -0.028898,
4746
+ 0.035646,
4747
+ 0.036174,
4748
+ -5621e-6,
4749
+ 0.050018,
4750
+ -0.040307,
4751
+ 0.060677,
4752
+ 58e-6,
4753
+ -0.045134,
4754
+ 4752e-6,
4755
+ 7161e-6,
4756
+ 0.040398,
4757
+ -0.051202,
4758
+ -572e-6,
4759
+ 6186e-6,
4760
+ -0.010613,
4761
+ 0.039894,
4762
+ -0.014924,
4763
+ 49e-4,
4764
+ -0.011771,
4765
+ 9397e-6,
4766
+ 9457e-6,
4767
+ 0.025576,
4768
+ 2251e-6,
4769
+ 0.020638,
4770
+ -0.029722,
4771
+ 7545e-6,
4772
+ -533e-6,
4773
+ 0.026445,
4774
+ 921e-6,
4775
+ 9681e-6,
4776
+ -959e-6,
4777
+ -0.018021,
4778
+ 807e-5,
4779
+ 8041e-6,
4780
+ -8004e-6,
4781
+ 4422e-6,
4782
+ -2478e-6,
4783
+ 7376e-6,
4784
+ -828e-5,
4785
+ 8006e-6,
4786
+ -3182e-6,
4787
+ 5771e-6,
4788
+ -526e-5,
4789
+ 0.01422,
4790
+ -3677e-6,
4791
+ -0.010823,
4792
+ 0.019873,
4793
+ 3328e-6,
4794
+ 4427e-6,
4795
+ 0.02227,
4796
+ -0.014147,
4797
+ 0.014306,
4798
+ -5089e-6,
4799
+ -0.010849,
4800
+ 322e-5,
4801
+ -8249e-6,
4802
+ 0.019141,
4803
+ -0.012199,
4804
+ 0.015451,
4805
+ 0.026615,
4806
+ 0.02352,
4807
+ 0.011117,
4808
+ -399e-6,
4809
+ 0.016821,
4810
+ 0.023447,
4811
+ 6638e-6,
4812
+ 8598e-6,
4813
+ -0.02239,
4814
+ 0.017325,
4815
+ 0.022729,
4816
+ 9922e-6,
4817
+ -4099e-6,
4818
+ 9124e-6,
4819
+ -8221e-6,
4820
+ -7554e-6,
4821
+ 0.046752,
4822
+ 0.027253,
4823
+ 437e-6,
4824
+ 4693e-6,
4825
+ 7093e-6,
4826
+ -0.014217,
4827
+ -0.024195,
4828
+ 2768e-6,
4829
+ -0.012022,
4830
+ -0.012142,
4831
+ -4326e-6,
4832
+ -0.01054,
4833
+ 0.01168,
4834
+ 4821e-6,
4835
+ -5609e-6,
4836
+ -0.096205,
4837
+ -0.018675,
4838
+ -0.017776,
4839
+ -0.010107,
4840
+ -0.012825,
4841
+ -644e-6,
4842
+ 0.011283,
4843
+ 3759e-6,
4844
+ -0.025286,
4845
+ 2306e-6,
4846
+ 9735e-6,
4847
+ -158e-5,
4848
+ 0.012131,
4849
+ -0.018451,
4850
+ 0.014135,
4851
+ 6421e-6,
4852
+ -7006e-6,
4853
+ -6122e-6,
4854
+ 0.013727,
4855
+ 8605e-6,
4856
+ -6385e-6,
4857
+ -0.016703,
4858
+ -0.184351,
4859
+ 4162e-6,
4860
+ 0.022412,
4861
+ -0.021583,
4862
+ -5397e-6,
4863
+ -0.011593,
4864
+ -1118e-6,
4865
+ 0.013629,
4866
+ 4367e-6,
4867
+ 238e-5,
4868
+ 0.030207,
4869
+ 2765e-6,
4870
+ 0.015452,
4871
+ -0.023998,
4872
+ 0.016889,
4873
+ 5828e-6,
4874
+ 0.01799,
4875
+ -0.019521,
4876
+ 0.012821,
4877
+ -0.021567,
4878
+ 5999e-6,
4879
+ 0.017122,
4880
+ -9796e-6,
4881
+ 6654e-6,
4882
+ 291e-5,
4883
+ -0.017359,
4884
+ 9342e-6,
4885
+ -929e-6,
4886
+ -0.012544,
4887
+ -8234e-6,
4888
+ 0.016922,
4889
+ 0.018215,
4890
+ -758e-6,
4891
+ 5614e-6,
4892
+ 0.02137,
4893
+ 7615e-6,
4894
+ 0.010542,
4895
+ 635e-6,
4896
+ 0.023702,
4897
+ 0.020834,
4898
+ 0.013032,
4899
+ 0.011873,
4900
+ -2792e-6,
4901
+ 0.04144,
4902
+ -0.02252,
4903
+ 0.027235,
4904
+ -0.012274,
4905
+ -4002e-6,
4906
+ 0.022356,
4907
+ -0.020197,
4908
+ 0.270597,
4909
+ -3966e-6,
4910
+ -4187e-6,
4911
+ -0.014922,
4912
+ -0.013515,
4913
+ -0.01073,
4914
+ 5186e-6,
4915
+ -0.01013,
4916
+ -0.024027,
4917
+ -897e-6,
4918
+ 0.023069,
4919
+ 0.021741,
4920
+ 573e-6,
4921
+ -7467e-6,
4922
+ 823e-5,
4923
+ 0.037651,
4924
+ -0.017797,
4925
+ -0.014499,
4926
+ 2699e-6,
4927
+ -0.010982,
4928
+ -0.027587,
4929
+ 0.028585,
4930
+ -6217e-6,
4931
+ -9743e-6,
4932
+ 7437e-6,
4933
+ -0.023614,
4934
+ -0.023206,
4935
+ -0.029976,
4936
+ -903e-6,
4937
+ -5667e-6,
4938
+ -0.011557,
4939
+ 0.011335,
4940
+ -0.019149,
4941
+ 0.027405,
4942
+ -0.032284,
4943
+ -0.010479,
4944
+ -0.013504,
4945
+ 0.013693,
4946
+ -6407e-6,
4947
+ 8791e-6,
4948
+ -4461e-6,
4949
+ -2256e-6,
4950
+ -0.015897,
4951
+ 8837e-6,
4952
+ -1417e-6,
4953
+ 0.02557,
4954
+ -0.012118,
4955
+ -0.021653,
4956
+ 3051e-6,
4957
+ -4168e-6,
4958
+ -0.01862,
4959
+ -9461e-6,
4960
+ -0.01088,
4961
+ -982e-5,
4962
+ -0.011997,
4963
+ -0.015663,
4964
+ -0.021258,
4965
+ 5679e-6,
4966
+ 0.012566,
4967
+ 1521e-6,
4968
+ -8604e-6,
4969
+ 0.012445,
4970
+ 8421e-6,
4971
+ -0.033856,
4972
+ -0.019957,
4973
+ 146e-6,
4974
+ -185e-6,
4975
+ 781e-6,
4976
+ 0.018164,
4977
+ 4102e-6,
4978
+ -0.022985,
4979
+ -8788e-6,
4980
+ -4097e-6,
4981
+ 0.02217,
4982
+ -0.019959,
4983
+ -0.011669,
4984
+ 5313e-6,
4985
+ 381e-6,
4986
+ 3747e-6,
4987
+ -0.036681,
4988
+ -6177e-6,
4989
+ -0.014834,
4990
+ -0.031706,
4991
+ 0.010642,
4992
+ 1258e-6,
4993
+ 0.01296,
4994
+ 0.016435,
4995
+ -8722e-6,
4996
+ -0.010652,
4997
+ 9787e-6,
4998
+ -474e-5,
4999
+ 0.011086,
5000
+ -0.020615,
5001
+ 0.021049,
5002
+ -468e-5,
5003
+ 8645e-6,
5004
+ -9937e-6,
5005
+ 216e-5,
5006
+ 0.010686,
5007
+ -6216e-6,
5008
+ -0.011397,
5009
+ -5593e-6,
5010
+ 9687e-6,
5011
+ 0.017522,
5012
+ -6329e-6,
5013
+ 9519e-6,
5014
+ -0.022554,
5015
+ -0.02169,
5016
+ 3307e-6,
5017
+ -4782e-6,
5018
+ 1506e-6,
5019
+ 1607e-6,
5020
+ -3791e-6,
5021
+ -3601e-6,
5022
+ 0.030061,
5023
+ -468e-5,
5024
+ -0.186431,
5025
+ -1492e-6,
5026
+ 0.02242,
5027
+ -8878e-6,
5028
+ 0.01108
5029
+ ];
5030
+
4285
5031
  // src/profiles/registry.ts
4286
5032
  var SHARED_ENCODING = {
4287
5033
  pooling: "mean",
@@ -4342,7 +5088,94 @@ var BGE_SMALL_EN_V15_1 = {
4342
5088
  passageInstruction: "",
4343
5089
  ...SHARED_ENCODING
4344
5090
  };
5091
+ var GEMMA2_REPO = "onnx-community/embeddinggemma-2-ONNX";
5092
+ var GEMMA2_PROCESSOR_CONFIG = {
5093
+ path: "processor_config.json",
5094
+ digest: "sha256:168f6a08522f3ce5dea596d94d003af2fd691742d4f41fe1f9d8cce76bfbf69c"
5095
+ };
5096
+ var EMBEDDINGGEMMA_2_512_1 = {
5097
+ id: "embeddinggemma-2-512@1",
5098
+ model: {
5099
+ repo: GEMMA2_REPO,
5100
+ revision: "daa72c51243991dfcaf9f9137d2c573d8f7790c0",
5101
+ onnxFile: "onnx/model_quantized.onnx",
5102
+ onnxDigest: "sha256:d06edd601f851c633a2519304cbeb8dc6170d7ceb61b436625c17fb9b6e74953",
5103
+ files: [
5104
+ {
5105
+ path: "onnx/model_quantized.onnx_data",
5106
+ digest: "sha256:278a7ff1248c3618e4bd11a607fc54f7bdc7778854230f3956d3f86bd9db4f3b"
5107
+ },
5108
+ {
5109
+ path: "config.json",
5110
+ digest: "sha256:8d011bfe08b5e345bbe0b81e5c6fd02c381920b345b986047bc2a33ce7b90d1d"
5111
+ },
5112
+ {
5113
+ path: "tokenizer_config.json",
5114
+ digest: "sha256:17bd5d6e9364ca49a534e1502076593317c298d4a663623091ed45388f004874"
5115
+ }
5116
+ ]
5117
+ },
5118
+ tokenizer: {
5119
+ kind: "gemma-bpe",
5120
+ file: "tokenizer.json",
5121
+ digest: "sha256:4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4"
5122
+ },
5123
+ pooling: "mean",
5124
+ normalized: true,
5125
+ dimensions: 512,
5126
+ truncation: { method: "prefix", sourceDimensions: 768 },
5127
+ maxTokens: 512,
5128
+ queryInstruction: "task: search result | query: ",
5129
+ passageInstruction: "title: none | text: ",
5130
+ vectorEncoding: "bit+int8",
5131
+ distance: { stage1: "hamming", stage2: "cosine" },
5132
+ quantization: {
5133
+ int8: { method: "symmetric-linear", scale: 127 },
5134
+ binary: {
5135
+ method: "centered-sign",
5136
+ threshold: 0,
5137
+ packing: "lsb-first",
5138
+ center: [...EMBEDDINGGEMMA_2_512_CENTER]
5139
+ }
5140
+ },
5141
+ media: {
5142
+ image: {
5143
+ encoder: {
5144
+ onnxFile: "onnx/vision_encoder_quantized.onnx",
5145
+ onnxDigest: "sha256:bb0de2df53a2448a32dc7908a187c168c8afd514d4d6f674f7f46024875fa4e3",
5146
+ files: [
5147
+ {
5148
+ path: "onnx/vision_encoder_quantized.onnx_data",
5149
+ digest: "sha256:3dabd69c0a36e9a8771ad82030dde74daa5a0e02b7047a5d3f3382b1137bab89"
5150
+ },
5151
+ GEMMA2_PROCESSOR_CONFIG
5152
+ ]
5153
+ },
5154
+ tokenBudget: 280,
5155
+ resample: "bicubic",
5156
+ alpha: "composite-white"
5157
+ },
5158
+ video: { framesPerSecond: 1, maxFrames: 32, tokenBudgetPerFrame: 140 },
5159
+ audio: {
5160
+ encoder: {
5161
+ onnxFile: "onnx/audio_encoder_quantized.onnx",
5162
+ onnxDigest: "sha256:04a9a9094ba76fb169e4c69be45a4b621580188654d6be59eab860eadcd42d3c",
5163
+ files: [
5164
+ {
5165
+ path: "onnx/audio_encoder_quantized.onnx_data",
5166
+ digest: "sha256:aa6361d898e1f1d6303f8dd2b5fabf4fd3629e15fd709e9cb06ac5cb9416a030"
5167
+ },
5168
+ GEMMA2_PROCESSOR_CONFIG
5169
+ ]
5170
+ },
5171
+ sampleRate: 16e3,
5172
+ channels: 1,
5173
+ maxWindowMs: 3e4
5174
+ }
5175
+ }
5176
+ };
4345
5177
  var KNOWLEDGE_EMBEDDING_PROFILES = [
5178
+ EMBEDDINGGEMMA_2_512_1,
4346
5179
  MULTILINGUAL_E5_SMALL_2,
4347
5180
  MULTILINGUAL_E5_SMALL_1,
4348
5181
  BGE_SMALL_EN_V15_1
@@ -4363,10 +5196,13 @@ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
4363
5196
  var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
4364
5197
 
4365
5198
  // src/embedding/profile-embedder.ts
4366
- import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
5199
+ import {
5200
+ embeddingProfileArtifacts as embeddingProfileArtifacts3,
5201
+ profileUnitVector as profileUnitVector2
5202
+ } from "@bendyline/gezk";
4367
5203
 
4368
5204
  // src/embedding/artifact-verify.ts
4369
- import { createHash as createHash5 } from "crypto";
5205
+ import { createHash as createHash6 } from "crypto";
4370
5206
  import { createReadStream } from "fs";
4371
5207
  import { stat as stat3 } from "fs/promises";
4372
5208
  import { join as join5 } from "path";
@@ -4410,7 +5246,7 @@ async function fileDigest(path) {
4410
5246
  const memo = digestMemo.get(key);
4411
5247
  if (memo) return memo;
4412
5248
  const digest = await new Promise((resolve5, reject) => {
4413
- const hash = createHash5("sha256");
5249
+ const hash = createHash6("sha256");
4414
5250
  createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
4415
5251
  });
4416
5252
  if (digestMemo.size >= DIGEST_MEMO_MAX) {
@@ -4420,6 +5256,45 @@ async function fileDigest(path) {
4420
5256
  digestMemo.set(key, digest);
4421
5257
  return digest;
4422
5258
  }
5259
+ function mediaEncoderPins(profile, modalities) {
5260
+ const encoders = /* @__PURE__ */ new Set();
5261
+ const media = profile.media;
5262
+ for (const modality of modalities) {
5263
+ const encoder = modality === "audio" ? media?.audio?.encoder : media?.image?.encoder;
5264
+ if (encoder) encoders.add(encoder);
5265
+ }
5266
+ const pins = [];
5267
+ const seen = /* @__PURE__ */ new Set();
5268
+ for (const encoder of encoders) {
5269
+ const files = [
5270
+ { path: encoder.onnxFile, digest: encoder.onnxDigest },
5271
+ ...encoder.files ?? []
5272
+ ];
5273
+ for (const { path, digest } of files) {
5274
+ if (!digest || seen.has(path)) continue;
5275
+ seen.add(path);
5276
+ pins.push({ file: path, expected: digest });
5277
+ }
5278
+ }
5279
+ return pins;
5280
+ }
5281
+ async function verifyMediaArtifacts(profile, modalities, opts) {
5282
+ const pins = mediaEncoderPins(profile, modalities);
5283
+ if (pins.length === 0) return { status: "unpinned", checks: [] };
5284
+ const revision = opts.revision ?? profile.model.revision;
5285
+ const checks = [];
5286
+ for (const { file, expected } of pins) {
5287
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
5288
+ checks.push({ role: "media", file, path, expected, actual: await fileDigest(path) });
5289
+ }
5290
+ if (checks.some((c) => c.actual === null)) {
5291
+ throw new EmbedderArtifactError("missing", profile.id, checks);
5292
+ }
5293
+ if (checks.some((c) => c.actual !== c.expected)) {
5294
+ throw new EmbedderArtifactError("mismatch", profile.id, checks);
5295
+ }
5296
+ return { status: "verified", checks };
5297
+ }
4423
5298
  async function verifyProfileArtifacts(profile, opts) {
4424
5299
  const artifacts = embeddingProfileArtifacts(profile);
4425
5300
  const revision = opts.revision ?? profile.model.revision;
@@ -4434,6 +5309,9 @@ async function verifyProfileArtifacts(profile, opts) {
4434
5309
  expected: artifacts.tokenizerDigest
4435
5310
  });
4436
5311
  }
5312
+ for (const extra of artifacts.files) {
5313
+ if (extra.digest) pinned.push({ role: "file", file: extra.path, expected: extra.digest });
5314
+ }
4437
5315
  if (pinned.length === 0) return { status: "unpinned", checks: [] };
4438
5316
  const checks = [];
4439
5317
  for (const { role, file, expected } of pinned) {
@@ -4449,10 +5327,89 @@ async function verifyProfileArtifacts(profile, opts) {
4449
5327
  return { status: "verified", checks };
4450
5328
  }
4451
5329
 
5330
+ // src/embedding/pinned-files.ts
5331
+ import { createHash as createHash7 } from "crypto";
5332
+ import { createWriteStream as createWriteStream4 } from "fs";
5333
+ import { mkdir as mkdir4, rename, rm as rm2, stat as stat4 } from "fs/promises";
5334
+ import { dirname as dirname3 } from "path";
5335
+ import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
5336
+ function isExternalDataFile(path) {
5337
+ return /\.onnx_data(_\d+)?$/.test(path);
5338
+ }
5339
+ function pinnedLoadFiles(profile, modalities = []) {
5340
+ const artifacts = embeddingProfileArtifacts2(profile);
5341
+ const files = [
5342
+ {
5343
+ path: artifacts.onnxFile,
5344
+ ...artifacts.onnxDigest ? { digest: artifacts.onnxDigest } : {}
5345
+ },
5346
+ {
5347
+ path: artifacts.tokenizerFile,
5348
+ ...artifacts.tokenizerDigest ? { digest: artifacts.tokenizerDigest } : {}
5349
+ },
5350
+ ...artifacts.files.map((f) => ({ path: f.path, ...f.digest ? { digest: f.digest } : {} })),
5351
+ ...mediaEncoderPins(profile, modalities).map((p) => ({ path: p.file, digest: p.expected }))
5352
+ ];
5353
+ const seen = /* @__PURE__ */ new Set();
5354
+ return files.filter((f) => !seen.has(f.path) && seen.add(f.path));
5355
+ }
5356
+ async function isFile(path) {
5357
+ return stat4(path).then((s) => s.isFile()).catch(() => false);
5358
+ }
5359
+ async function missingPinnedFiles(profile, opts) {
5360
+ const revision = opts.revision ?? profile.model.revision;
5361
+ const missing = [];
5362
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5363
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5364
+ if (!await isFile(path)) missing.push(file.path);
5365
+ }
5366
+ return missing;
5367
+ }
5368
+ async function prefetchExternalData(profile, opts) {
5369
+ const revision = opts.revision ?? profile.model.revision;
5370
+ const fetchImpl = opts.fetchImpl ?? fetch;
5371
+ const fetched = [];
5372
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5373
+ if (!isExternalDataFile(file.path)) continue;
5374
+ const dest = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5375
+ if (await isFile(dest)) continue;
5376
+ const url = `https://huggingface.co/${profile.model.repo}/resolve/${revision}/${file.path}`;
5377
+ const res = await fetchImpl(url);
5378
+ if (!res.ok || !res.body) throw new Error(`${file.path}: download failed (HTTP ${res.status})`);
5379
+ await mkdir4(dirname3(dest), { recursive: true });
5380
+ const partial = `${dest}.partial`;
5381
+ const hash = createHash7("sha256");
5382
+ try {
5383
+ const out = createWriteStream4(partial);
5384
+ const reader = res.body.getReader();
5385
+ for (; ; ) {
5386
+ const { done, value } = await reader.read();
5387
+ if (done) break;
5388
+ hash.update(value);
5389
+ if (!out.write(value)) await new Promise((r) => out.once("drain", () => r()));
5390
+ }
5391
+ await new Promise((resolve5, reject) => {
5392
+ out.on("error", reject);
5393
+ out.end(() => resolve5());
5394
+ });
5395
+ const digest = `sha256:${hash.digest("hex")}`;
5396
+ if (file.digest && digest !== file.digest) {
5397
+ throw new Error(`${file.path}: downloaded ${digest}, profile pins ${file.digest}`);
5398
+ }
5399
+ await rename(partial, dest);
5400
+ } catch (err) {
5401
+ await rm2(partial, { force: true });
5402
+ throw err;
5403
+ }
5404
+ fetched.push(file.path);
5405
+ }
5406
+ return fetched;
5407
+ }
5408
+
4452
5409
  // src/embedding/profile-embedder.ts
4453
5410
  var MAX_BATCH = 8;
4454
5411
  var MAX_CHARS = 8e3;
4455
- var TRANSFORMERS_PEER_RANGE = "^3.8.1";
5412
+ var TRANSFORMERS_PEER_RANGE = "^4.3.1";
4456
5413
  var EmbedderUnavailableError = class extends Error {
4457
5414
  isActionable = true;
4458
5415
  /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
@@ -4496,7 +5453,7 @@ var POOLING_BY_PROFILE = {
4496
5453
  cls: "cls"
4497
5454
  };
4498
5455
  function resolveTransformersModelOptions(profile) {
4499
- const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
5456
+ const { onnxFile, tokenizerFile } = embeddingProfileArtifacts3(profile);
4500
5457
  if (tokenizerFile !== "tokenizer.json") {
4501
5458
  throw new EmbedderUnavailableError(
4502
5459
  `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
@@ -4521,6 +5478,41 @@ function transformersGraphOptions(onnxFile) {
4521
5478
  if (modelFileName === "") return null;
4522
5479
  return { dtype, subfolder, model_file_name: modelFileName };
4523
5480
  }
5481
+ function isMultimodalEncoderConfig(config) {
5482
+ return Boolean(config.vision_config || config.audio_config);
5483
+ }
5484
+ function pinnedExternalDataChunks(profile) {
5485
+ const { onnxFile, files } = embeddingProfileArtifacts3(profile);
5486
+ const prefix = `${onnxFile}_data`;
5487
+ return files.filter((f) => f.path === prefix || /^_\d+$/.test(f.path.slice(prefix.length))).length;
5488
+ }
5489
+ function encoderRows(outputs, attentionMask) {
5490
+ const pooled = outputs.sentence_embedding;
5491
+ if (pooled) {
5492
+ const [batch2 = 0, width2 = 0] = pooled.dims;
5493
+ return Array.from(
5494
+ { length: batch2 },
5495
+ (_, b) => Array.from({ length: width2 }, (_2, d) => pooled.data[b * width2 + d])
5496
+ );
5497
+ }
5498
+ const hidden = outputs.last_hidden_state;
5499
+ if (!hidden || !attentionMask) {
5500
+ throw new Error("the encoder returned neither sentence_embedding nor last_hidden_state");
5501
+ }
5502
+ const [batch = 0, tokens = 0, width = 0] = hidden.dims;
5503
+ return Array.from({ length: batch }, (_, b) => {
5504
+ const sum = new Array(width).fill(0);
5505
+ let count = 0;
5506
+ for (let t = 0; t < tokens; t++) {
5507
+ if (!Number(attentionMask.data[b * tokens + t])) continue;
5508
+ count++;
5509
+ const base = (b * tokens + t) * width;
5510
+ for (let d = 0; d < width; d++)
5511
+ sum[d] = sum[d] + hidden.data[base + d];
5512
+ }
5513
+ return sum.map((v) => v / Math.max(1, count));
5514
+ });
5515
+ }
4524
5516
  async function loadTransformers() {
4525
5517
  try {
4526
5518
  const specifier = "@huggingface/transformers";
@@ -4536,6 +5528,20 @@ async function loadTransformers() {
4536
5528
  throw err;
4537
5529
  }
4538
5530
  }
5531
+ async function preparePinnedLoad(profile, opts) {
5532
+ const files = pinnedLoadFiles(profile, opts.modalities);
5533
+ if (!files.some((f) => isExternalDataFile(f.path))) return;
5534
+ if (opts.localFilesOnly) {
5535
+ const missing = await missingPinnedFiles(profile, opts);
5536
+ if (missing.length > 0) {
5537
+ throw new EmbedderUnavailableError(
5538
+ `the model files for profile ${profile.id} are not all installed (missing: ${missing.join(", ")})`
5539
+ );
5540
+ }
5541
+ return;
5542
+ }
5543
+ await prefetchExternalData(profile, opts);
5544
+ }
4539
5545
  async function createProfileEmbedder(profile, opts = {}) {
4540
5546
  const pooling = POOLING_BY_PROFILE[profile.pooling];
4541
5547
  if (!pooling) {
@@ -4552,16 +5558,47 @@ async function createProfileEmbedder(profile, opts = {}) {
4552
5558
  transformers.env.allowRemoteModels = true;
4553
5559
  }
4554
5560
  const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
4555
- const [pipe, tokenizer] = await Promise.all([
4556
- transformers.pipeline("feature-extraction", profile.model.repo, {
5561
+ const localOnly = opts.localFilesOnly ? { local_files_only: true } : {};
5562
+ if (!opts.transformers) {
5563
+ await preparePinnedLoad(profile, {
5564
+ cacheDir: cacheDir ?? transformers.env.cacheDir ?? "",
5565
+ ...opts.localFilesOnly ? { localFilesOnly: true } : {}
5566
+ });
5567
+ }
5568
+ const sessionOptions = opts.sessionOptions ? { session_options: opts.sessionOptions } : {};
5569
+ const config = transformers.AutoConfig ? await transformers.AutoConfig.from_pretrained(profile.model.repo, {
5570
+ revision: modelOptions.revision,
5571
+ ...localOnly,
5572
+ ...progress
5573
+ }) : null;
5574
+ const multimodal = config !== null && isMultimodalEncoderConfig(config);
5575
+ if (multimodal && !transformers.AutoModel) {
5576
+ throw new EmbedderUnavailableError(
5577
+ `profile ${profile.id} is a multimodal encoder; this runtime cannot load its text session`
5578
+ );
5579
+ }
5580
+ const externalData = pinnedExternalDataChunks(profile);
5581
+ const [runner, tokenizer] = await Promise.all([
5582
+ multimodal ? transformers.AutoModel.from_pretrained(
5583
+ profile.model.repo,
5584
+ {
5585
+ ...modelOptions,
5586
+ config: { ...config, vision_config: null, audio_config: null },
5587
+ ...externalData > 0 ? { use_external_data_format: externalData } : {},
5588
+ ...localOnly,
5589
+ ...sessionOptions,
5590
+ ...progress
5591
+ }
5592
+ ) : transformers.pipeline("feature-extraction", profile.model.repo, {
4557
5593
  ...modelOptions,
4558
- ...opts.localFilesOnly ? { local_files_only: true } : {},
4559
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
5594
+ ...externalData > 0 ? { use_external_data_format: externalData } : {},
5595
+ ...localOnly,
5596
+ ...sessionOptions,
4560
5597
  ...progress
4561
5598
  }),
4562
5599
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
4563
5600
  revision: modelOptions.revision,
4564
- ...opts.localFilesOnly ? { local_files_only: true } : {},
5601
+ ...localOnly,
4565
5602
  ...progress
4566
5603
  })
4567
5604
  ]);
@@ -4571,17 +5608,23 @@ async function createProfileEmbedder(profile, opts = {}) {
4571
5608
  cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
4572
5609
  });
4573
5610
  } catch (err) {
4574
- await pipe.dispose?.().catch(() => {
5611
+ await runner.dispose?.().catch(() => {
4575
5612
  });
4576
5613
  throw err;
4577
5614
  }
4578
5615
  const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
5616
+ const runBatch = multimodal ? async (slice) => {
5617
+ const inputs = tokenizer(slice, {
5618
+ padding: true,
5619
+ truncation: true,
5620
+ max_length: profile.maxTokens
5621
+ });
5622
+ return encoderRows(await runner(inputs), inputs.attention_mask);
5623
+ } : async (slice) => (await runner(slice, { pooling, normalize: true })).tolist();
4579
5624
  const embed = async (texts) => {
4580
5625
  const out = [];
4581
5626
  for (let i = 0; i < texts.length; i += MAX_BATCH) {
4582
- const slice = texts.slice(i, i + MAX_BATCH).map(cap);
4583
- const result = await pipe(slice, { pooling, normalize: true });
4584
- out.push(...result.tolist());
5627
+ out.push(...await runBatch(texts.slice(i, i + MAX_BATCH).map(cap)));
4585
5628
  }
4586
5629
  return out;
4587
5630
  };
@@ -4591,11 +5634,12 @@ async function createProfileEmbedder(profile, opts = {}) {
4591
5634
  embed,
4592
5635
  embedQuery: async (text) => {
4593
5636
  const [vector] = await embed([`${profile.queryInstruction}${text}`]);
4594
- return Float32Array.from(vector ?? []);
5637
+ if (!vector) throw new Error(`profile ${profile.id} returned no query vector`);
5638
+ return profileUnitVector2(profile, vector);
4595
5639
  },
4596
5640
  countTokens: (text) => tokenizer.encode(text).length,
4597
5641
  dispose: async () => {
4598
- await pipe.dispose?.();
5642
+ await runner.dispose?.();
4599
5643
  }
4600
5644
  };
4601
5645
  }
@@ -4603,7 +5647,7 @@ async function createProfileEmbedder(profile, opts = {}) {
4603
5647
  // src/markdown-adapter/load.ts
4604
5648
  import { existsSync } from "fs";
4605
5649
  import { readFile as readFile3, readdir } from "fs/promises";
4606
- import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
5650
+ import { basename, dirname as dirname4, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
4607
5651
  import {
4608
5652
  CatalogDocumentSchema,
4609
5653
  KnowledgeDocumentIdSchema,
@@ -5281,9 +6325,9 @@ function parseYaml(text, opts) {
5281
6325
  var OPEN = /^---[ \t]*\r?\n/;
5282
6326
  var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
5283
6327
  function parseMarkdownFrontMatter(source) {
5284
- const open = OPEN.exec(source);
5285
- if (!open) return { data: {}, body: source };
5286
- const lines = source.slice(open[0].length).split(/\r?\n/);
6328
+ const open2 = OPEN.exec(source);
6329
+ if (!open2) return { data: {}, body: source };
6330
+ const lines = source.slice(open2[0].length).split(/\r?\n/);
5287
6331
  const closeAt = lines.findIndex((line) => CLOSE.test(line));
5288
6332
  if (closeAt === -1) throw new Error("front matter is not closed with ---");
5289
6333
  const yamlText = lines.slice(0, closeAt).join("\n");
@@ -5482,7 +6526,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
5482
6526
  maxAliasCount: 100
5483
6527
  });
5484
6528
  const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
5485
- return resolve4(dirname3(mkdocsPath), docsDir);
6529
+ return resolve4(dirname4(mkdocsPath), docsDir);
5486
6530
  }
5487
6531
  async function detectTableOfContents(rootDir, projectDir = rootDir) {
5488
6532
  const summary = join6(rootDir, "SUMMARY.md");
@@ -5490,7 +6534,7 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
5490
6534
  const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
5491
6535
  if (jupyter) return { format: "jupyter-book", path: jupyter };
5492
6536
  const mkdocs = firstExisting(
5493
- [rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
6537
+ [rootDir, projectDir, dirname4(rootDir)].flatMap((dir) => [
5494
6538
  join6(dir, "mkdocs.yml"),
5495
6539
  join6(dir, "mkdocs.yaml")
5496
6540
  ])
@@ -5530,7 +6574,7 @@ async function loadOutline(rootDir, toc, files) {
5530
6574
  }
5531
6575
  if (toc.format === "mkdocs") {
5532
6576
  const path2 = explicit ?? firstExisting(
5533
- [rootDir, dirname3(rootDir)].flatMap((dir) => [
6577
+ [rootDir, dirname4(rootDir)].flatMap((dir) => [
5534
6578
  join6(dir, "mkdocs.yml"),
5535
6579
  join6(dir, "mkdocs.yaml")
5536
6580
  ])
@@ -5553,7 +6597,7 @@ async function loadOutline(rootDir, toc, files) {
5553
6597
  }
5554
6598
  const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
5555
6599
  if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
5556
- const tocDir = insideRoot(rootDir, dirname3(path));
6600
+ const tocDir = insideRoot(rootDir, dirname4(path));
5557
6601
  if (tocDir === null)
5558
6602
  throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
5559
6603
  const doc = parseYaml(await readOutlineText(path), {
@@ -6110,7 +7154,9 @@ export {
6110
7154
  CENTROID_SAMPLE_MAX,
6111
7155
  CatalogHandle,
6112
7156
  CatalogOpenError,
7157
+ CatalogQueryError,
6113
7158
  DuckdbError,
7159
+ EMBEDDINGGEMMA_2_512_1,
6114
7160
  EmbedderArtifactError,
6115
7161
  EmbedderUnavailableError,
6116
7162
  GEZEL_BGE_SMALL_EN_V15_1,
@@ -6134,6 +7180,8 @@ export {
6134
7180
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
6135
7181
  MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
6136
7182
  MAX_KNOWLEDGE_TOPIC_DEPTH,
7183
+ MAX_MEDIA_ROWS_PER_DOCUMENT,
7184
+ MEDIA_ROW_TEXT_MAX_TOKENS,
6137
7185
  MIMETYPE_PATH,
6138
7186
  MULTILINGUAL_E5_SMALL_1,
6139
7187
  MULTILINGUAL_E5_SMALL_2,
@@ -6149,6 +7197,8 @@ export {
6149
7197
  ROUTE_SHARDS_EXPLICIT,
6150
7198
  ROUTE_SHARDS_PROACTIVE,
6151
7199
  SHARD_DDL,
7200
+ SHARD_DDL_V4,
7201
+ SHARD_DDL_V5,
6152
7202
  SHARD_MAX_CHUNKS,
6153
7203
  SHARD_TARGET_CHUNKS,
6154
7204
  SOURCE_NOTICES_PATH,
@@ -6160,10 +7210,12 @@ export {
6160
7210
  assignOutlineOrders,
6161
7211
  asymmetricTopK,
6162
7212
  canonicalizeJson3 as canonicalizeJson,
7213
+ catalogFormatVersion,
6163
7214
  centerVector,
6164
7215
  chunkContentHash,
6165
7216
  chunkMarkdownProfile,
6166
7217
  chunkUid,
7218
+ collectMediaRows,
6167
7219
  compareCatalogVersions,
6168
7220
  compileKnowledgeCatalog,
6169
7221
  createProfileEmbedder,
@@ -6175,9 +7227,13 @@ export {
6175
7227
  fetchKnowledgeRegistry,
6176
7228
  findDuckdbBinary,
6177
7229
  findRegistryEntry,
7230
+ formatAtLeast,
6178
7231
  generateKnowledgeSigningKeyPair,
6179
7232
  hammingTopK,
7233
+ humanizeAssetName,
6180
7234
  inspectGezkArchive,
7235
+ isExternalDataFile,
7236
+ isMultimodalEncoderConfig,
6181
7237
  isSupportedFormatVersion,
6182
7238
  isSupportedIndexSchemaVersion,
6183
7239
  knowledgeEmbeddingProfile,
@@ -6185,6 +7241,9 @@ export {
6185
7241
  l2Normalize,
6186
7242
  loadMarkdownCatalog,
6187
7243
  manifestSigningPayload,
7244
+ mediaEncoderPins,
7245
+ mediaReferences,
7246
+ missingPinnedFiles,
6188
7247
  newOutlineTopic,
6189
7248
  newerRegistryEntries,
6190
7249
  openCatalogDatabase,
@@ -6192,6 +7251,11 @@ export {
6192
7251
  parseGitbookSummary,
6193
7252
  parseJupyterBookToc,
6194
7253
  parseMkdocsNav,
7254
+ pinnedExternalDataChunks,
7255
+ pinnedLoadFiles,
7256
+ prefetchExternalData,
7257
+ preparePinnedLoad,
7258
+ profileUnitVector,
6195
7259
  quantizeBinary,
6196
7260
  quantizeBinaryForProfile,
6197
7261
  quantizeInt8,
@@ -6202,6 +7266,7 @@ export {
6202
7266
  resolveOutlinePath,
6203
7267
  resolveTransformersModelOptions,
6204
7268
  runDuckdbScript,
7269
+ shardDdlFor,
6205
7270
  signManifest,
6206
7271
  signRegistryIndex,
6207
7272
  titleFromFolderName,
@@ -6209,6 +7274,7 @@ export {
6209
7274
  transformersGraphOptions,
6210
7275
  validateExtractedCatalog,
6211
7276
  verifyManifestSignature,
7277
+ verifyMediaArtifacts,
6212
7278
  verifyProfileArtifacts,
6213
7279
  verifyRegistryIndex,
6214
7280
  writeGezkArchive