@bendyline/gezel-knowledge 1.2.2 → 1.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -18,6 +18,7 @@ import {
18
18
  ROUTER_DB_PATH,
19
19
  SOURCE_NOTICES_PATH,
20
20
  ZIP_FIXED_MTIME,
21
+ formatAtLeast,
21
22
  isSupportedFormatVersion,
22
23
  isSupportedIndexSchemaVersion
23
24
  } from "@bendyline/gezk";
@@ -37,7 +38,7 @@ function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY)
37
38
  }
38
39
 
39
40
  // src/format/ddl.ts
40
- import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
41
+ import { ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, shardDdlFor } from "@bendyline/gezk";
41
42
 
42
43
  // src/format/ids.ts
43
44
  import { documentSlug } from "@bendyline/gezk";
@@ -47,6 +48,7 @@ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
47
48
  import {
48
49
  centerVector,
49
50
  l2Normalize,
51
+ profileUnitVector,
50
52
  quantizeBinary,
51
53
  quantizeBinaryForProfile,
52
54
  quantizeInt8,
@@ -180,12 +182,11 @@ function splitOversizeLine(line, opts) {
180
182
  }
181
183
 
182
184
  // src/compiler/compile.ts
183
- import { createHash } from "crypto";
185
+ import { createHash as createHash3 } from "crypto";
184
186
  import {
185
187
  closeSync,
186
188
  mkdirSync,
187
189
  openSync,
188
- readFileSync,
189
190
  readSync,
190
191
  rmSync,
191
192
  statSync,
@@ -196,16 +197,15 @@ import { join } from "path";
196
197
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
197
198
  import {
198
199
  GEZK_MANIFEST_KIND,
199
- KnowledgeAssetPathSchema,
200
200
  KnowledgeCatalogManifestSchema,
201
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
202
- MAX_KNOWLEDGE_ASSET_BYTES,
203
- MAX_KNOWLEDGE_ASSET_COUNT,
204
- assetExtension,
205
- assetKindForExtension,
201
+ KnowledgeLocationsSchema,
202
+ KnowledgeOrdinalSchema,
203
+ KnowledgeTocReferenceSchema,
206
204
  canonicalizeJson,
207
- sniffAssetType,
208
- svgInertnessProblem
205
+ embeddingProfileMinimumFormat,
206
+ isKnowledgeMediaAssetPath,
207
+ normalizeLongitude,
208
+ spatialManifest
209
209
  } from "@bendyline/gezk";
210
210
 
211
211
  // src/archive/write.ts
@@ -251,28 +251,137 @@ var nodeRequire2 = createRequire2(import.meta.url);
251
251
  var sqlite = nodeRequire2("node:sqlite");
252
252
  var DatabaseSync = sqlite.DatabaseSync;
253
253
 
254
+ // src/query-stopwords.ts
255
+ import { QUERY_STOP_WORDS } from "@bendyline/gezk";
256
+
254
257
  // src/reader/fts-query.ts
255
- function sanitizeFtsQuery(query) {
258
+ function contentTokens(query) {
256
259
  const tokens = [
257
260
  ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
258
261
  ];
262
+ const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
263
+ return content.length > 0 ? content : tokens;
264
+ }
265
+ function contentFtsQuery(query) {
266
+ const tokens = contentTokens(query);
267
+ return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
268
+ }
269
+ function selectiveFtsQuery(query, rows, rowsMatching) {
270
+ const tokens = contentTokens(query);
259
271
  if (tokens.length === 0) return null;
260
- return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
272
+ const counted = tokens.map((token) => {
273
+ const phrase = quoteToken(token);
274
+ return { phrase, hits: rowsMatching(phrase) };
275
+ });
276
+ const kept = counted.filter((c) => c.hits * 2 <= rows);
277
+ const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
278
+ return chosen.map((c) => c.phrase).join(" OR ");
261
279
  }
262
280
  var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
263
281
  var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
264
- function documentFtsTopIds(db, match, limit, exactTitle) {
265
- const title = exactTitle?.normalize("NFKC").trim() ?? "";
266
- return db.prepare(
282
+ function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
283
+ const match = contentFtsQuery(query);
284
+ if (!match || allowedDocumentIds?.size === 0) return [];
285
+ const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
286
+ const title = query.normalize("NFKC").trim();
287
+ const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
288
+ const rest = db.prepare(
267
289
  `SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
268
- ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
269
- ).all(match, title, limit).map((row) => row.document_id);
290
+ ${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
291
+ ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
292
+ ).all(
293
+ ...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
294
+ );
295
+ for (const row of rest) {
296
+ if (ids.length >= limit) break;
297
+ if (!ids.includes(row.document_id)) ids.push(row.document_id);
298
+ }
299
+ return ids;
300
+ }
301
+ var NAME_STOPWORDS = new Set(
302
+ "a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
303
+ );
304
+ var NAMED_TITLE_CANDIDATES = 256;
305
+ var TITLE_DF_CACHE_MAX = 5e4;
306
+ var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
307
+ var SAME_WORDS_SCORE = Number.MAX_VALUE;
308
+ function nameWords(text) {
309
+ return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
310
+ }
311
+ function quoteToken(token) {
312
+ return `"${token.replace(/"/g, '""')}"`;
313
+ }
314
+ var titleStatsByDb = /* @__PURE__ */ new WeakMap();
315
+ function titleStats(db) {
316
+ let stats = titleStatsByDb.get(db);
317
+ if (!stats) {
318
+ const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
319
+ stats = {
320
+ documents: Math.max(1, Number(row.c)),
321
+ df: /* @__PURE__ */ new Map(),
322
+ dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
323
+ candidates: db.prepare(
324
+ `SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
325
+ ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
326
+ )
327
+ };
328
+ titleStatsByDb.set(db, stats);
329
+ }
330
+ return stats;
331
+ }
332
+ function titleDf(stats, word) {
333
+ let df = stats.df.get(word);
334
+ if (df === void 0) {
335
+ if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
336
+ const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
337
+ df = Number(row.c);
338
+ stats.df.set(word, df);
339
+ }
340
+ return df;
341
+ }
342
+ function namedTitleMatches(db, query, limit, allowedDocumentIds) {
343
+ if (allowedDocumentIds?.size === 0) return [];
344
+ const raw = query.normalize("NFKC").trim();
345
+ const words = [...new Set(nameWords(raw))].slice(0, 16);
346
+ const content = words.filter((w) => !NAME_STOPWORDS.has(w));
347
+ if (content.length === 0 || limit <= 0) return [];
348
+ const asked = new Set(words);
349
+ const capitalized = new Set(
350
+ (raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
351
+ );
352
+ const queryWords = nameWords(raw).join(" ");
353
+ const stats = titleStats(db);
354
+ const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
355
+ const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
356
+ AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
357
+ const named = [];
358
+ for (const row of rows) {
359
+ const titleWords = nameWords(row.title);
360
+ if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
361
+ const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
362
+ if (distinct.length === 0) continue;
363
+ let score;
364
+ if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
365
+ score = EXACT_TITLE_SCORE;
366
+ } else if (titleWords.join(" ") === queryWords) {
367
+ score = SAME_WORDS_SCORE;
368
+ } else {
369
+ score = 0;
370
+ for (const w of distinct) {
371
+ score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
372
+ }
373
+ }
374
+ named.push({ documentId: row.document_id, score, order: named.length });
375
+ }
376
+ named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
377
+ return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
378
+ }
379
+ function compareScores(a, b) {
380
+ return a === b ? 0 : a > b ? 1 : -1;
270
381
  }
271
382
  var SMOKE_QUERY_TOP_N = 10;
272
383
  function documentSmokeQueryMisses(db, smoke) {
273
- const match = sanitizeFtsQuery(smoke.query);
274
- if (!match) return [...smoke.expectedDocumentIds];
275
- const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
384
+ const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
276
385
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
277
386
  }
278
387
 
@@ -284,6 +393,216 @@ var KNOWLEDGE_TOOLCHAIN = {
284
393
  version: nodeRequire3("../package.json").version
285
394
  };
286
395
 
396
+ // src/compiler/assets.ts
397
+ import { createHash } from "crypto";
398
+ import { readFileSync } from "fs";
399
+ import {
400
+ KnowledgeAssetPathSchema,
401
+ MAX_KNOWLEDGE_ASSET_COUNT,
402
+ assetExtension,
403
+ assetKindForExtension,
404
+ knowledgeAssetLimitsProblem,
405
+ maxKnowledgeAssetBytes,
406
+ sniffAssetType,
407
+ svgInertnessProblem
408
+ } from "@bendyline/gezk";
409
+ function prepareAssets(assets, extraFiles, opts) {
410
+ for (const path of Object.keys(extraFiles)) {
411
+ if (path.startsWith("assets/")) {
412
+ throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
413
+ }
414
+ }
415
+ if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
416
+ throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
417
+ }
418
+ const seen = /* @__PURE__ */ new Set();
419
+ const prepared = [];
420
+ const skippedPaths = /* @__PURE__ */ new Set();
421
+ const rejectContent = (path, message) => {
422
+ if (opts.invalidAssets !== "warn") throw new Error(message);
423
+ skippedPaths.add(path);
424
+ opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
425
+ };
426
+ for (const asset of assets) {
427
+ const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
428
+ if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
429
+ const key = asset.path.toLowerCase();
430
+ if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
431
+ seen.add(key);
432
+ if (asset.content === void 0 === (asset.absPath === void 0)) {
433
+ throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
434
+ }
435
+ const bytes = asset.content ?? readFileSync(asset.absPath);
436
+ const limit = maxKnowledgeAssetBytes(asset.path);
437
+ if (bytes.byteLength > limit) {
438
+ rejectContent(
439
+ asset.path,
440
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${limit}`
441
+ );
442
+ continue;
443
+ }
444
+ const ext = assetExtension(asset.path);
445
+ const expected = ext ? assetKindForExtension(ext) : null;
446
+ const actual = sniffAssetType(bytes);
447
+ if (expected === null || actual !== expected) {
448
+ rejectContent(
449
+ asset.path,
450
+ `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
451
+ );
452
+ continue;
453
+ }
454
+ if (ext === "svg") {
455
+ const problem = svgInertnessProblem(bytes);
456
+ if (problem) {
457
+ rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
458
+ continue;
459
+ }
460
+ }
461
+ prepared.push({
462
+ path: asset.path,
463
+ bytes,
464
+ sizeBytes: bytes.byteLength,
465
+ sha256: createHash("sha256").update(bytes).digest("hex"),
466
+ ...asset.attribution ? { attribution: asset.attribution } : {}
467
+ });
468
+ }
469
+ const totals = knowledgeAssetLimitsProblem(prepared);
470
+ if (totals) throw new Error(totals);
471
+ return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
472
+ }
473
+ function omitSkippedAssetReferences(markdown, skippedPaths) {
474
+ if (skippedPaths.size === 0) return markdown;
475
+ return markdown.replace(
476
+ /!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
477
+ (whole, text, target) => skippedPaths.has(target) ? text : whole
478
+ );
479
+ }
480
+
481
+ // src/compiler/media-rows.ts
482
+ import { createHash as createHash2 } from "crypto";
483
+ import {
484
+ assetContentType,
485
+ assetModality
486
+ } from "@bendyline/gezk";
487
+ var MEDIA_ROW_TEXT_MAX_TOKENS = 128;
488
+ var MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
489
+ var MEDIA_REFERENCE = /!\[([^\]\n]*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+"[^"\n]*")?\s*\)/g;
490
+ function mediaReferences(markdown) {
491
+ const out = [];
492
+ markdown.split("\n").forEach((text, index) => {
493
+ for (const match of text.matchAll(MEDIA_REFERENCE)) {
494
+ out.push({
495
+ alt: (match[1] ?? "").trim(),
496
+ path: match[2],
497
+ line: index + 1,
498
+ caption: text.replace(MEDIA_REFERENCE, " ").replace(/\s+/g, " ").trim()
499
+ });
500
+ }
501
+ });
502
+ return out;
503
+ }
504
+ function humanizeAssetName(path) {
505
+ const base = path.slice(path.lastIndexOf("/") + 1);
506
+ const stem2 = base.includes(".") ? base.slice(0, base.lastIndexOf(".")) : base;
507
+ return stem2.replace(/[-_.]+/g, " ").replace(/\s+/g, " ").trim();
508
+ }
509
+ function boundedText(parts, countTokens) {
510
+ const unique = [];
511
+ for (const part of parts.map((p) => p.trim()).filter(Boolean)) {
512
+ if (!unique.some((u) => u.toLowerCase() === part.toLowerCase())) unique.push(part);
513
+ }
514
+ let text = unique.join("\n").normalize("NFC");
515
+ if (countTokens(text) <= MEDIA_ROW_TEXT_MAX_TOKENS) return text;
516
+ const words = text.split(/(\s+)/);
517
+ let lo = 0;
518
+ let hi = words.length;
519
+ while (lo < hi) {
520
+ const mid = Math.ceil((lo + hi) / 2);
521
+ if (countTokens(words.slice(0, mid).join("")) <= MEDIA_ROW_TEXT_MAX_TOKENS) lo = mid;
522
+ else hi = mid - 1;
523
+ }
524
+ text = words.slice(0, lo).join("").trim();
525
+ return text;
526
+ }
527
+ function headingAt(chunks, line) {
528
+ const inside = chunks.find((c) => c.lineStart <= line && line <= c.lineEnd);
529
+ if (inside) return inside.headingPath;
530
+ let before;
531
+ for (const c of chunks) if (c.lineEnd < line) before = c;
532
+ return before?.headingPath ?? [];
533
+ }
534
+ function windowHash(assetSha256, startMs, endMs) {
535
+ if (startMs === null || endMs === null) return assetSha256;
536
+ return createHash2("sha256").update(`${assetSha256}\0${startMs}\0${endMs}`).digest("hex");
537
+ }
538
+ async function collectMediaRows(args) {
539
+ const byPath = new Map(args.assets.map((a) => [a.path, a]));
540
+ const embedded = /* @__PURE__ */ new Map();
541
+ const warned = /* @__PURE__ */ new Set();
542
+ const out = /* @__PURE__ */ new Map();
543
+ for (const doc of args.documents) {
544
+ const rows = [];
545
+ const seen = /* @__PURE__ */ new Set();
546
+ for (const ref of mediaReferences(doc.markdown)) {
547
+ if (seen.has(ref.path)) continue;
548
+ seen.add(ref.path);
549
+ const asset = byPath.get(ref.path);
550
+ const modality = assetModality(ref.path);
551
+ const mimeType = assetContentType(ref.path);
552
+ if (!asset || !modality || !mimeType || mimeType === "image/svg+xml") continue;
553
+ const block = modality === "audio" ? args.profile.media?.audio : modality === "video" ? args.profile.media?.video : args.profile.media?.image;
554
+ if (!block) {
555
+ if (!warned.has(modality)) {
556
+ warned.add(modality);
557
+ args.onWarning?.(
558
+ `profile ${args.profile.id} describes no ${modality} encoder; ${modality} references get no media rows`
559
+ );
560
+ }
561
+ continue;
562
+ }
563
+ let pending = embedded.get(ref.path);
564
+ if (!pending) {
565
+ pending = args.embedMedia({ modality, path: ref.path, bytes: asset.bytes });
566
+ embedded.set(ref.path, pending);
567
+ }
568
+ const results = [...await pending].sort((a, b) => (a.startMs ?? 0) - (b.startMs ?? 0));
569
+ const headingPath = headingAt(doc.chunks, ref.line);
570
+ for (const result of results) {
571
+ const startMs = modality === "image" ? null : result.startMs ?? 0;
572
+ const endMs = modality === "image" ? null : result.endMs ?? null;
573
+ if (startMs !== null && (endMs === null || endMs <= startMs)) {
574
+ throw new Error(`${ref.path}: a ${modality} window needs startMs < endMs`);
575
+ }
576
+ rows.push({
577
+ modality,
578
+ assetPath: ref.path,
579
+ mimeType,
580
+ width: result.width ?? null,
581
+ height: result.height ?? null,
582
+ startMs,
583
+ endMs,
584
+ attributionJson: asset.attribution ? JSON.stringify(asset.attribution) : null,
585
+ headingPath,
586
+ line: ref.line,
587
+ text: boundedText(
588
+ [ref.alt, ref.caption, humanizeAssetName(ref.path), result.transcript ?? ""],
589
+ args.countTokens
590
+ ),
591
+ contentHash: windowHash(asset.sha256, startMs, endMs),
592
+ vector: result.vector
593
+ });
594
+ }
595
+ }
596
+ if (rows.length > MAX_MEDIA_ROWS_PER_DOCUMENT) {
597
+ throw new Error(
598
+ `document ${doc.id} has ${rows.length} media rows; the limit is ${MAX_MEDIA_ROWS_PER_DOCUMENT}`
599
+ );
600
+ }
601
+ if (rows.length > 0) out.set(doc.id, rows);
602
+ }
603
+ return out;
604
+ }
605
+
287
606
  // src/compiler/compile.ts
288
607
  async function compileKnowledgeCatalog(opts) {
289
608
  if (opts.topics.length === 0) {
@@ -292,7 +611,11 @@ async function compileKnowledgeCatalog(opts) {
292
611
  );
293
612
  }
294
613
  const topicById = assertTopicForest(opts.topics);
295
- const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
614
+ const { assets: preparedAssets, skippedPaths } = prepareAssets(
615
+ opts.assets ?? [],
616
+ opts.extraFiles ?? {},
617
+ opts
618
+ );
296
619
  const profile = opts.embeddingProfile;
297
620
  const chunkerOpts = {
298
621
  unit: opts.chunkingProfile.unit,
@@ -302,23 +625,75 @@ async function compileKnowledgeCatalog(opts) {
302
625
  };
303
626
  const prepared = [];
304
627
  const seenIds = /* @__PURE__ */ new Set();
305
- for await (const doc of opts.documents) {
628
+ for await (const input of opts.documents) {
629
+ const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
630
+ ...location,
631
+ longitude: normalizeLongitude(location.longitude)
632
+ })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
633
+ const doc = {
634
+ ...input,
635
+ locations,
636
+ markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
637
+ };
306
638
  if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
307
639
  seenIds.add(doc.id);
640
+ const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
641
+ const placements = [
642
+ {
643
+ topicId: leafTopicId,
644
+ ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
645
+ }
646
+ ];
647
+ const leaves = /* @__PURE__ */ new Set([leafTopicId]);
648
+ for (const raw of doc.tocReferences ?? []) {
649
+ const reference = KnowledgeTocReferenceSchema.parse(raw);
650
+ const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
651
+ if (leaves.has(topicId))
652
+ throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
653
+ leaves.add(topicId);
654
+ placements.push({ topicId, ordinal: reference.ordinal ?? null });
655
+ }
656
+ placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
308
657
  prepared.push({
309
658
  doc,
310
659
  topicPathKey: doc.topicPath.join("/"),
311
- leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
660
+ leafTopicId,
661
+ placements,
312
662
  metaJson: encodeDocumentMeta(doc),
313
- chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
663
+ chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts),
664
+ media: []
314
665
  });
315
666
  }
316
667
  prepared.sort(
317
668
  (a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
318
669
  );
319
670
  const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
671
+ if (opts.embedMedia) {
672
+ const rows = await collectMediaRows({
673
+ documents: prepared.map((p) => ({
674
+ id: p.doc.id,
675
+ title: p.doc.title,
676
+ markdown: normalizeMarkdown(p.doc.markdown),
677
+ chunks: p.chunks
678
+ })),
679
+ assets: preparedAssets,
680
+ profile,
681
+ embedMedia: opts.embedMedia,
682
+ countTokens: opts.countTokens,
683
+ onWarning: opts.onWarning
684
+ });
685
+ for (const p of prepared) p.media = rows.get(p.doc.id) ?? [];
686
+ }
687
+ const mediaCounts = { image: 0, video: 0, audio: 0 };
688
+ for (const p of prepared) for (const row of p.media) mediaCounts[row.modality] += 1;
689
+ const totalMedia = mediaCounts.image + mediaCounts.video + mediaCounts.audio;
690
+ const totalRows = totalChunks + totalMedia;
691
+ const rowsOf = (p) => p.chunks.length + p.media.length;
692
+ const formatVersion = catalogFormatVersion(profile, preparedAssets, totalMedia);
693
+ const indexSchemaVersion = GEZK_FORMAT_GENERATIONS[formatVersion];
694
+ const shardDdl = shardDdlFor(indexSchemaVersion);
320
695
  const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
321
- const embedded = totalChunks <= shardTarget;
696
+ const embedded = totalRows <= shardTarget;
322
697
  const semanticFill = opts.shardFill === "semantic" && !embedded;
323
698
  const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
324
699
  p.doc.title,
@@ -335,7 +710,7 @@ async function compileKnowledgeCatalog(opts) {
335
710
  docVectors = await stageVectors({
336
711
  prepared,
337
712
  stagedPath,
338
- dimensions: profile.dimensions,
713
+ profile,
339
714
  batchSize: opts.embedBatchSize ?? 32,
340
715
  embed: opts.embed,
341
716
  embedInput,
@@ -348,11 +723,11 @@ async function compileKnowledgeCatalog(opts) {
348
723
  if (embedded) {
349
724
  for (const p of prepared) shardOf.set(p.doc.id, 0);
350
725
  } else if (semanticFill) {
351
- shardCount = Math.ceil(totalChunks / shardTarget);
352
- const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalChunks / shardCount * 1.02));
726
+ shardCount = Math.ceil(totalRows / shardTarget);
727
+ const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalRows / shardCount * 1.02));
353
728
  const assignment = semanticShards(
354
729
  docVectors,
355
- prepared.map((p) => p.chunks.length),
730
+ prepared.map(rowsOf),
356
731
  shardCount,
357
732
  capacity,
358
733
  seedFor(`${opts.catalog.id}#shard-fill`, 0)
@@ -364,19 +739,19 @@ async function compileKnowledgeCatalog(opts) {
364
739
  let current = 0;
365
740
  let filled = 0;
366
741
  for (const p of prepared) {
367
- if (filled > 0 && filled + p.chunks.length > shardTarget) {
742
+ if (filled > 0 && filled + rowsOf(p) > shardTarget) {
368
743
  current++;
369
744
  filled = 0;
370
745
  }
371
746
  shardOf.set(p.doc.id, current);
372
- filled += p.chunks.length;
747
+ filled += rowsOf(p);
373
748
  }
374
749
  shardCount = current + 1;
375
750
  }
376
751
  const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
377
752
  const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
378
753
  const shardDbs = /* @__PURE__ */ new Map();
379
- const router = createCatalogDb(routerStaged);
754
+ const router = createCatalogDb(routerStaged, indexSchemaVersion);
380
755
  const closeAll = () => {
381
756
  for (const db of [router, ...shardDbs.values()]) {
382
757
  try {
@@ -389,11 +764,11 @@ async function compileKnowledgeCatalog(opts) {
389
764
  router.exec(ROUTER_DDL);
390
765
  router.exec("BEGIN");
391
766
  if (embedded) {
392
- router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
767
+ router.exec(shardDdl.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
393
768
  }
394
769
  const commonMeta = {
395
- format_version: String(GEZK_FORMAT_VERSION),
396
- index_schema_version: String(GEZK_INDEX_SCHEMA_VERSION),
770
+ format_version: String(formatVersion),
771
+ index_schema_version: String(indexSchemaVersion),
397
772
  catalog_id: opts.catalog.id,
398
773
  catalog_version: opts.catalog.version,
399
774
  embedding_profile_id: profile.id,
@@ -411,7 +786,9 @@ async function compileKnowledgeCatalog(opts) {
411
786
  });
412
787
  const topicDocCount = /* @__PURE__ */ new Map();
413
788
  for (const p of prepared) {
414
- topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
789
+ for (const placement of p.placements) {
790
+ topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
791
+ }
415
792
  }
416
793
  {
417
794
  const stmt = router.prepare(
@@ -435,6 +812,13 @@ async function compileKnowledgeCatalog(opts) {
435
812
  body_codec, body_blob)
436
813
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
437
814
  );
815
+ const insertLocation = router.prepare(
816
+ `INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
817
+ VALUES (?, ?, ?, ?, ?, ?)`
818
+ );
819
+ const insertPlacement = router.prepare(
820
+ "INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
821
+ );
438
822
  const insertAlias = router.prepare(
439
823
  "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
440
824
  );
@@ -464,7 +848,7 @@ async function compileKnowledgeCatalog(opts) {
464
848
  p.leafTopicId,
465
849
  p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
466
850
  BigInt(shardOf.get(p.doc.id)),
467
- BigInt(p.chunks.length),
851
+ BigInt(rowsOf(p)),
468
852
  p.doc.sourceUrl ?? null,
469
853
  p.doc.sourceRevision ?? null,
470
854
  p.doc.sourceUpdatedAt ?? null,
@@ -473,6 +857,29 @@ async function compileKnowledgeCatalog(opts) {
473
857
  useBrotli ? "br" : "none",
474
858
  blob
475
859
  );
860
+ for (const location of p.doc.locations ?? []) {
861
+ const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
862
+ if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
863
+ throw new Error(
864
+ `document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
865
+ );
866
+ }
867
+ insertLocation.run(
868
+ p.doc.id,
869
+ location.id,
870
+ location.latitude,
871
+ location.longitude,
872
+ location.role,
873
+ provenance
874
+ );
875
+ }
876
+ for (const placement of p.placements) {
877
+ insertPlacement.run(
878
+ placement.topicId,
879
+ p.doc.id,
880
+ placement.ordinal === null ? null : BigInt(placement.ordinal)
881
+ );
882
+ }
476
883
  const aliases = [...new Set(p.doc.aliases ?? [])].sort();
477
884
  for (const alias of aliases) insertAlias.run(alias, p.doc.id);
478
885
  insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
@@ -486,8 +893,8 @@ async function compileKnowledgeCatalog(opts) {
486
893
  if (embedded) return router;
487
894
  let db = shardDbs.get(id);
488
895
  if (!db) {
489
- db = createCatalogDb(join(opts.workDir, shardPath(id)));
490
- db.exec(SHARD_DDL);
896
+ db = createCatalogDb(join(opts.workDir, shardPath(id)), indexSchemaVersion);
897
+ db.exec(shardDdl);
491
898
  db.exec("BEGIN");
492
899
  shardDbs.set(id, db);
493
900
  }
@@ -518,7 +925,8 @@ async function compileKnowledgeCatalog(opts) {
518
925
  );
519
926
  }
520
927
  for (let i = 0; i < vectors.length; i++) {
521
- const unit = l2Normalize(vectors[i]);
928
+ const raw = vectors[i];
929
+ const unit = staged ? l2Normalize(raw) : profileUnitVector(profile, raw);
522
930
  if (unit.length !== profile.dimensions) {
523
931
  throw new Error(
524
932
  `embedder returned dim ${unit.length}, profile expects ${profile.dimensions}`
@@ -582,6 +990,46 @@ async function compileKnowledgeCatalog(opts) {
582
990
  pendingRows.push({ shardId, chunkId });
583
991
  if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
584
992
  }
993
+ if (p.media.length > 0) {
994
+ const insertMedia = db.prepare(
995
+ `INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
996
+ line_start, line_end, token_count, content_hash, text, modality, asset_path, mime_type,
997
+ width, height, start_ms, end_ms, attribution_json)
998
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
999
+ );
1000
+ const stmts = vecStmtsFor(db);
1001
+ p.media.forEach((media, index) => {
1002
+ const ordinal = p.chunks.length + index;
1003
+ const headingText = media.headingPath.join(" > ");
1004
+ const row = insertMedia.get(
1005
+ chunkUid(p.doc.id, ordinal, `media:${media.modality}:${media.contentHash}`),
1006
+ p.doc.id,
1007
+ BigInt(ordinal),
1008
+ p.doc.title,
1009
+ JSON.stringify(media.headingPath),
1010
+ headingText,
1011
+ BigInt(media.line),
1012
+ BigInt(media.line),
1013
+ BigInt(opts.countTokens(media.text)),
1014
+ media.contentHash,
1015
+ media.text,
1016
+ media.modality,
1017
+ media.assetPath,
1018
+ media.mimeType,
1019
+ media.width === null ? null : BigInt(media.width),
1020
+ media.height === null ? null : BigInt(media.height),
1021
+ media.startMs === null ? null : BigInt(media.startMs),
1022
+ media.endMs === null ? null : BigInt(media.endMs),
1023
+ media.attributionJson
1024
+ );
1025
+ const chunkId = Number(row.id);
1026
+ insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, media.text);
1027
+ shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
1028
+ const unit = profileUnitVector(profile, media.vector);
1029
+ stmts.vec.run(BigInt(chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
1030
+ stmts.int8.run(BigInt(chunkId), Buffer.from(quantizeInt8(unit).buffer));
1031
+ });
1032
+ }
585
1033
  }
586
1034
  await flushEmbeds();
587
1035
  staged?.close();
@@ -593,7 +1041,7 @@ async function compileKnowledgeCatalog(opts) {
593
1041
  chunk_count: String(shardChunkCounts.get(id) ?? 0)
594
1042
  });
595
1043
  }
596
- if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
1044
+ if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalRows) });
597
1045
  const finalDir = join(opts.workDir, "final");
598
1046
  mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
599
1047
  const vacuumed = [];
@@ -705,7 +1153,7 @@ async function compileKnowledgeCatalog(opts) {
705
1153
  files.push({
706
1154
  path,
707
1155
  sizeBytes: bytes.length,
708
- sha256: createHash("sha256").update(bytes).digest("hex")
1156
+ sha256: createHash3("sha256").update(bytes).digest("hex")
709
1157
  });
710
1158
  }
711
1159
  for (const asset of preparedAssets) {
@@ -714,8 +1162,8 @@ async function compileKnowledgeCatalog(opts) {
714
1162
  assertAssetReferencesDeclared(prepared, preparedAssets);
715
1163
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
716
1164
  kind: GEZK_MANIFEST_KIND,
717
- formatVersion: GEZK_FORMAT_VERSION,
718
- indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
1165
+ formatVersion,
1166
+ indexSchemaVersion,
719
1167
  id: opts.catalog.id,
720
1168
  version: opts.catalog.version,
721
1169
  name: opts.catalog.name,
@@ -744,12 +1192,18 @@ async function compileKnowledgeCatalog(opts) {
744
1192
  },
745
1193
  counts: {
746
1194
  documents: prepared.length,
747
- chunks: totalChunks,
1195
+ chunks: totalRows,
748
1196
  shards: shardCount,
749
- assets: preparedAssets.length
1197
+ assets: preparedAssets.length,
1198
+ ...formatVersion === "0.8" ? { media: mediaCounts } : {}
750
1199
  },
751
1200
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
752
- requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
1201
+ spatial: spatialManifest(
1202
+ prepared.flatMap(
1203
+ (p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
1204
+ )
1205
+ ),
1206
+ requires: { formatVersion, features: [] },
753
1207
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
754
1208
  toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
755
1209
  });
@@ -773,7 +1227,8 @@ async function compileKnowledgeCatalog(opts) {
773
1227
  return {
774
1228
  manifest,
775
1229
  documents: prepared.length,
776
- chunks: totalChunks,
1230
+ chunks: totalRows,
1231
+ media: mediaCounts,
777
1232
  shards: shardCount,
778
1233
  archiveBytes
779
1234
  };
@@ -844,58 +1299,6 @@ function encodeDocumentMeta(doc) {
844
1299
  }
845
1300
  return json;
846
1301
  }
847
- function prepareAssets(assets, extraFiles) {
848
- for (const path of Object.keys(extraFiles)) {
849
- if (path.startsWith("assets/")) {
850
- throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
851
- }
852
- }
853
- if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
854
- throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
855
- }
856
- const seen = /* @__PURE__ */ new Set();
857
- const prepared = [];
858
- let total = 0;
859
- for (const asset of assets) {
860
- const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
861
- if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
862
- const key = asset.path.toLowerCase();
863
- if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
864
- seen.add(key);
865
- if (asset.content === void 0 === (asset.absPath === void 0)) {
866
- throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
867
- }
868
- const bytes = asset.content ?? readFileSync(asset.absPath);
869
- if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
870
- throw new Error(
871
- `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
872
- );
873
- }
874
- total += bytes.byteLength;
875
- if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
876
- throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
877
- }
878
- const ext = assetExtension(asset.path);
879
- const expected = ext ? assetKindForExtension(ext) : null;
880
- const actual = sniffAssetType(bytes);
881
- if (expected === null || actual !== expected) {
882
- throw new Error(
883
- `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
884
- );
885
- }
886
- if (ext === "svg") {
887
- const problem = svgInertnessProblem(bytes);
888
- if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
889
- }
890
- prepared.push({
891
- path: asset.path,
892
- bytes,
893
- sizeBytes: bytes.byteLength,
894
- sha256: createHash("sha256").update(bytes).digest("hex")
895
- });
896
- }
897
- return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
898
- }
899
1302
  function assertAssetReferencesDeclared(prepared, assets) {
900
1303
  const declared = new Set(assets.map((a) => a.path));
901
1304
  for (const p of prepared) {
@@ -909,11 +1312,15 @@ function assertAssetReferencesDeclared(prepared, assets) {
909
1312
  }
910
1313
  }
911
1314
  }
912
- function createCatalogDb(absPath) {
1315
+ function catalogFormatVersion(profile, assets, mediaRows = 0) {
1316
+ const needsMedia = mediaRows > 0 || assets.some((asset) => isKnowledgeMediaAssetPath(asset.path));
1317
+ return needsMedia || embeddingProfileMinimumFormat(profile) === "0.8" ? "0.8" : "0.7";
1318
+ }
1319
+ function createCatalogDb(absPath, indexSchemaVersion) {
913
1320
  const db = new DatabaseSync(absPath);
914
1321
  db.exec("PRAGMA page_size=8192");
915
1322
  db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
916
- db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
1323
+ db.exec(`PRAGMA user_version=${indexSchemaVersion}`);
917
1324
  db.exec("PRAGMA journal_mode=DELETE");
918
1325
  return db;
919
1326
  }
@@ -958,7 +1365,8 @@ ${path.join(" > ")}
958
1365
  return header;
959
1366
  }
960
1367
  async function stageVectors(args) {
961
- const { prepared, dimensions } = args;
1368
+ const { prepared, profile } = args;
1369
+ const { dimensions } = profile;
962
1370
  const docVectors = prepared.map(() => new Float32Array(dimensions));
963
1371
  const fd = openSync(args.stagedPath, "w");
964
1372
  let texts = [];
@@ -972,10 +1380,7 @@ async function stageVectors(args) {
972
1380
  }
973
1381
  const block = new Float32Array(texts.length * dimensions);
974
1382
  for (let i = 0; i < vectors.length; i++) {
975
- const unit = l2Normalize(vectors[i]);
976
- if (unit.length !== dimensions) {
977
- throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
978
- }
1383
+ const unit = profileUnitVector(profile, vectors[i]);
979
1384
  block.set(unit, i * dimensions);
980
1385
  const sum = docVectors[owners[i]];
981
1386
  for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
@@ -999,7 +1404,9 @@ async function stageVectors(args) {
999
1404
  } finally {
1000
1405
  closeSync(fd);
1001
1406
  }
1002
- return docVectors.map((v) => Float32Array.from(l2Normalize(Array.from(v))));
1407
+ return docVectors.map(
1408
+ (v) => v.some((x) => x !== 0) ? Float32Array.from(l2Normalize(Array.from(v))) : v
1409
+ );
1003
1410
  }
1004
1411
  var StagedVectorReader = class {
1005
1412
  constructor(path, dimensions) {
@@ -1041,7 +1448,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1041
1448
  return sum;
1042
1449
  };
1043
1450
  const stride = Math.max(1, Math.ceil(n / 2e4));
1044
- const sample = docVectors.filter((_, i) => i % stride === 0);
1451
+ const sample = docVectors.filter((v, i) => i % stride === 0 && v.some((x) => x !== 0));
1045
1452
  const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
1046
1453
  for (let i = 0; centers.length < shardCount; i++) {
1047
1454
  centers.push(
@@ -1110,7 +1517,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1110
1517
  return assignment.map((s) => renumber.get(s));
1111
1518
  }
1112
1519
  function seedFor(catalogId, shardId) {
1113
- const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1520
+ const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1114
1521
  return digest.readBigUInt64LE(24);
1115
1522
  }
1116
1523
  function makePrng(seed) {
@@ -1195,7 +1602,7 @@ function kMeans(samples, k, seed) {
1195
1602
  }
1196
1603
 
1197
1604
  // src/archive/read.ts
1198
- import { createHash as createHash2 } from "crypto";
1605
+ import { createHash as createHash4 } from "crypto";
1199
1606
  import { createWriteStream as createWriteStream2 } from "fs";
1200
1607
  import { mkdir as mkdir2 } from "fs/promises";
1201
1608
  import { createRequire as createRequire4 } from "module";
@@ -1485,7 +1892,7 @@ async function extractGezkVerified(archivePath, destDir) {
1485
1892
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1486
1893
  return;
1487
1894
  }
1488
- const hasher = createHash2("sha256");
1895
+ const hasher = createHash4("sha256");
1489
1896
  let sizeBytes = 0;
1490
1897
  const verify = new Transform({
1491
1898
  transform(chunk, _encoding, callback) {
@@ -1555,6 +1962,14 @@ var CatalogOpenError = class extends Error {
1555
1962
  }
1556
1963
  reason;
1557
1964
  };
1965
+ var CatalogQueryError = class extends Error {
1966
+ constructor(message, reason) {
1967
+ super(message);
1968
+ this.reason = reason;
1969
+ this.name = "CatalogQueryError";
1970
+ }
1971
+ reason;
1972
+ };
1558
1973
  function openCatalogDatabase(absPath) {
1559
1974
  const uri = `${pathToFileURL(absPath).href}?immutable=1`;
1560
1975
  let db;
@@ -1608,15 +2023,15 @@ function readPragmaNumber(db, pragma) {
1608
2023
  }
1609
2024
 
1610
2025
  // src/reader/catalog-handle.ts
1611
- import { readFileSync as readFileSync2 } from "fs";
2026
+ import { readFileSync as readFileSync3, statSync as statSync2 } from "fs";
1612
2027
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1613
2028
  import { brotliDecompressSync } from "zlib";
1614
2029
  import {
1615
2030
  KnowledgeEmbeddingProfileSchema,
1616
- MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1617
- assetContentType,
2031
+ assetContentType as assetContentType2,
1618
2032
  embeddingProfileCenter,
1619
- isKnowledgeAssetPath
2033
+ isKnowledgeAssetPath,
2034
+ maxKnowledgeAssetBytes as maxKnowledgeAssetBytes2
1620
2035
  } from "@bendyline/gezk";
1621
2036
 
1622
2037
  // src/reader/bit-scan.ts
@@ -1689,14 +2104,14 @@ function hammingTopK(index, query, k) {
1689
2104
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1690
2105
  return hits;
1691
2106
  }
1692
- function asymmetricTopK(index, query, k) {
2107
+ function asymmetricTopK(index, query, k, eligibleChunkIds, excludedChunkIds) {
1693
2108
  const { bits, bytesPerRow, rows } = index;
1694
2109
  if (Math.ceil(query.length / 8) !== bytesPerRow) {
1695
2110
  throw new Error(
1696
2111
  `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1697
2112
  );
1698
2113
  }
1699
- const limit = Math.min(k, rows);
2114
+ const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
1700
2115
  if (limit <= 0) return [];
1701
2116
  const lut = new Float32Array(bytesPerRow * 256);
1702
2117
  for (let p = 0; p < bytesPerRow; p++) {
@@ -1746,6 +2161,10 @@ function asymmetricTopK(index, query, k) {
1746
2161
  };
1747
2162
  let offset = 0;
1748
2163
  for (let r = 0; r < rows; r++) {
2164
+ if (eligibleChunkIds && !eligibleChunkIds.has(r + 1) || excludedChunkIds?.has(r + 1)) {
2165
+ offset += bytesPerRow;
2166
+ continue;
2167
+ }
1749
2168
  let score = 0;
1750
2169
  for (let p = 0; p < bytesPerRow; p++) {
1751
2170
  score += lut[p * 256 + bits[offset + p]];
@@ -1770,14 +2189,160 @@ function asymmetricTopK(index, query, k) {
1770
2189
  return hits;
1771
2190
  }
1772
2191
 
2192
+ // src/reader/spatial-index.ts
2193
+ import {
2194
+ KnowledgeLocationSchema,
2195
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
2196
+ KnowledgeRadiusSchema,
2197
+ locationDistanceMeters,
2198
+ locationInBounds,
2199
+ normalizeLongitude as normalizeLongitude2,
2200
+ radiusBounds,
2201
+ spatialManifest as spatialManifest2
2202
+ } from "@bendyline/gezk";
2203
+ var DocumentSpatialIndex = class {
2204
+ constructor(db, allowLegacy) {
2205
+ this.db = db;
2206
+ this.allowLegacy = allowLegacy;
2207
+ this.hasTable = Boolean(
2208
+ db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
2209
+ );
2210
+ }
2211
+ db;
2212
+ allowLegacy;
2213
+ hasTable;
2214
+ legacy;
2215
+ decode(row) {
2216
+ const provenance = row.provenance_json;
2217
+ if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
2218
+ throw new Error("location provenance exceeds 16384 bytes");
2219
+ return {
2220
+ documentId: String(row.document_id),
2221
+ location: KnowledgeLocationSchema.parse({
2222
+ id: row.location_id,
2223
+ latitude: row.latitude,
2224
+ longitude: row.longitude,
2225
+ role: row.role,
2226
+ ...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
2227
+ })
2228
+ };
2229
+ }
2230
+ legacyRows() {
2231
+ if (this.legacy) return this.legacy;
2232
+ this.legacy = [];
2233
+ if (!this.allowLegacy) return this.legacy;
2234
+ for (const row of this.db.prepare(
2235
+ "SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
2236
+ ).all()) {
2237
+ try {
2238
+ const meta = JSON.parse(String(row.meta_json));
2239
+ if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
2240
+ continue;
2241
+ const point = KnowledgeLocationSchema.safeParse({
2242
+ id: "qualla-coordinate",
2243
+ role: "subject",
2244
+ latitude: meta.coordinates.lat,
2245
+ longitude: meta.coordinates.lng,
2246
+ provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
2247
+ });
2248
+ if (point.success)
2249
+ this.legacy.push({
2250
+ documentId: String(row.id),
2251
+ location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
2252
+ });
2253
+ } catch {
2254
+ }
2255
+ }
2256
+ return this.legacy;
2257
+ }
2258
+ rows() {
2259
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
2260
+ }
2261
+ locations(documentId) {
2262
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
2263
+ }
2264
+ integrity() {
2265
+ const rows = this.rows();
2266
+ const grouped = /* @__PURE__ */ new Map();
2267
+ for (const row of rows) {
2268
+ if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
2269
+ throw new Error("location longitude must be in [-180, 180)");
2270
+ const points = grouped.get(row.documentId) ?? [];
2271
+ points.push(row.location);
2272
+ grouped.set(row.documentId, points);
2273
+ if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
2274
+ throw new Error("location references missing document");
2275
+ }
2276
+ for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
2277
+ return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
2278
+ }
2279
+ matches(raw) {
2280
+ const radius = KnowledgeRadiusSchema.parse(raw);
2281
+ const boxes = radiusBounds(radius);
2282
+ const candidates = this.hasTable ? boxes.flatMap(
2283
+ (box) => this.db.prepare(`SELECT * FROM document_locations
2284
+ WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
2285
+ ) : this.legacyRows().filter(
2286
+ (row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
2287
+ );
2288
+ const matches = /* @__PURE__ */ new Map();
2289
+ for (const { documentId, location } of candidates) {
2290
+ const distanceMeters = locationDistanceMeters(radius, location);
2291
+ if (distanceMeters > radius.radiusMeters) continue;
2292
+ const previous = matches.get(documentId);
2293
+ if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
2294
+ matches.set(documentId, { distanceMeters, matchedLocation: location });
2295
+ }
2296
+ }
2297
+ return matches;
2298
+ }
2299
+ };
2300
+
1773
2301
  // src/reader/catalog-handle.ts
1774
2302
  var TOPIC_WALK_MAX_DEPTH = 32;
2303
+ var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
2304
+ var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
2305
+ function mediaOf(row) {
2306
+ const modality = row.modality;
2307
+ if (!modality || modality === "text") return void 0;
2308
+ const num = (v) => v === null || v === void 0 ? void 0 : Number(v);
2309
+ let attribution;
2310
+ if (typeof row.attribution_json === "string") {
2311
+ try {
2312
+ const parsed = JSON.parse(row.attribution_json);
2313
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
2314
+ attribution = parsed;
2315
+ }
2316
+ } catch {
2317
+ attribution = void 0;
2318
+ }
2319
+ }
2320
+ const media = {
2321
+ modality,
2322
+ assetPath: row.asset_path,
2323
+ mimeType: row.mime_type
2324
+ };
2325
+ const width = num(row.width);
2326
+ const height = num(row.height);
2327
+ const startMs = num(row.start_ms);
2328
+ const endMs = num(row.end_ms);
2329
+ if (width !== void 0) media.width = width;
2330
+ if (height !== void 0) media.height = height;
2331
+ if (startMs !== void 0) media.startMs = startMs;
2332
+ if (endMs !== void 0) media.endMs = endMs;
2333
+ if (typeof row.thumbnail_path === "string") media.thumbnailAssetPath = row.thumbnail_path;
2334
+ if (attribution) media.attribution = attribution;
2335
+ return media;
2336
+ }
1775
2337
  var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
2338
+ var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
2339
+ var TITLE_ROUTE_SHARDS = 1;
1776
2340
  var CatalogHandle = class _CatalogHandle {
1777
2341
  constructor(rootDir, router) {
1778
2342
  this.rootDir = rootDir;
1779
2343
  this.router = router;
1780
2344
  this.schemaVersion = router.schemaVersion;
2345
+ this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
1781
2346
  this.meta = {};
1782
2347
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1783
2348
  this.meta[row.key] = row.value;
@@ -1789,6 +2354,10 @@ var CatalogHandle = class _CatalogHandle {
1789
2354
  router;
1790
2355
  connections = /* @__PURE__ */ new Map();
1791
2356
  bitIndexes = /* @__PURE__ */ new Map();
2357
+ mediaIndexes = /* @__PURE__ */ new Map();
2358
+ /** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
2359
+ chunkTermRows = /* @__PURE__ */ new Map();
2360
+ spatial;
1792
2361
  assetsByPath = null;
1793
2362
  /** Lazily parsed from the profile echo; `undefined` until first needed. */
1794
2363
  binaryCenterCache;
@@ -1799,7 +2368,12 @@ var CatalogHandle = class _CatalogHandle {
1799
2368
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1800
2369
  static open(rootDir) {
1801
2370
  const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
1802
- return new _CatalogHandle(rootDir, router);
2371
+ try {
2372
+ return new _CatalogHandle(rootDir, router);
2373
+ } catch (error) {
2374
+ router.close();
2375
+ throw error;
2376
+ }
1803
2377
  }
1804
2378
  close() {
1805
2379
  for (const conn of this.connections.values()) conn.close();
@@ -1874,6 +2448,24 @@ var CatalogHandle = class _CatalogHandle {
1874
2448
  this.bitIndexes.set(shard.path, index);
1875
2449
  return index;
1876
2450
  }
2451
+ /**
2452
+ * Load the next shard's sign bits that are not resident yet, if they fit
2453
+ * the resident budget. False when nothing is left to load. One shard per
2454
+ * call, so a host can keep answering queries between shards while it warms
2455
+ * a large catalog, and never evicts a shard it loaded a moment ago.
2456
+ */
2457
+ prewarmNextShard() {
2458
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
2459
+ let resident = 0;
2460
+ for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
2461
+ for (const shard of this.shards) {
2462
+ if (this.bitIndexes.has(shard.path)) continue;
2463
+ if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
2464
+ this.shardBits(shard);
2465
+ return true;
2466
+ }
2467
+ return false;
2468
+ }
1877
2469
  /**
1878
2470
  * The stage-1 center the catalog's bits were derived with (a
1879
2471
  * `centered-sign` profile), or null for plain sign bits. Read from the
@@ -1896,6 +2488,127 @@ var CatalogHandle = class _CatalogHandle {
1896
2488
  this.binaryCenterCache = embeddingProfileCenter(profile.data);
1897
2489
  return this.binaryCenterCache;
1898
2490
  }
2491
+ /** The chunk columns this catalog's schema has. */
2492
+ chunkColumns() {
2493
+ return this.schemaVersion >= 5 ? `${CHUNK_COLUMNS}, ${MEDIA_COLUMNS}` : CHUNK_COLUMNS;
2494
+ }
2495
+ /** A shard's media rows with their int8 vectors (empty before index schema 5). */
2496
+ shardMedia(shard) {
2497
+ const cached = this.mediaIndexes.get(shard.path);
2498
+ if (cached) return cached;
2499
+ const empty = {
2500
+ ids: [],
2501
+ modalities: [],
2502
+ int8: new Int8Array(0),
2503
+ idSet: /* @__PURE__ */ new Set()
2504
+ };
2505
+ if (this.schemaVersion < 5) {
2506
+ this.mediaIndexes.set(shard.path, empty);
2507
+ return empty;
2508
+ }
2509
+ const dims = this.dimensions();
2510
+ const rows = this.shardDb(shard).prepare(
2511
+ `SELECT c.id, c.modality, v.v FROM chunks c JOIN chunk_vectors_int8 v ON v.chunk_id = c.id
2512
+ WHERE c.modality <> 'text' ORDER BY c.id`
2513
+ ).all();
2514
+ const int8 = new Int8Array(rows.length * dims);
2515
+ rows.forEach((row, i) => {
2516
+ if (row.v.byteLength !== dims) {
2517
+ throw new CatalogOpenError(`int8 vector width ${row.v.byteLength} != ${dims}`, "corrupt");
2518
+ }
2519
+ int8.set(new Int8Array(row.v.buffer, row.v.byteOffset, dims), i * dims);
2520
+ });
2521
+ const ids = rows.map((r) => Number(r.id));
2522
+ const index = {
2523
+ ids,
2524
+ modalities: rows.map((r) => r.modality),
2525
+ int8,
2526
+ idSet: new Set(ids)
2527
+ };
2528
+ this.mediaIndexes.set(shard.path, index);
2529
+ return index;
2530
+ }
2531
+ /** Media rows this catalog holds, by modality (all zero before index schema 5). */
2532
+ mediaCounts() {
2533
+ const counts = { image: 0, video: 0, audio: 0 };
2534
+ for (const shard of this.shards) {
2535
+ for (const modality of this.shardMedia(shard).modalities) counts[modality] += 1;
2536
+ }
2537
+ return counts;
2538
+ }
2539
+ /**
2540
+ * The media lane: every media row of every shard scored exactly against
2541
+ * the query (int8 rerank), keeping the best `perModality` of each kind.
2542
+ * Images sit at a lower cosine than text in a shared space, so in the text
2543
+ * lane's sign-bit pre-filter they would rarely survive the cut; media rows
2544
+ * are few enough (capped per catalog) to score them all.
2545
+ */
2546
+ searchMedia(queryVector, opts = {}) {
2547
+ this.assertQueryWidth(queryVector);
2548
+ const perModality = opts.perModality ?? 4;
2549
+ const dims = this.dimensions();
2550
+ const best = /* @__PURE__ */ new Map();
2551
+ for (const shard of this.shards) {
2552
+ const index = this.shardMedia(shard);
2553
+ for (let i = 0; i < index.ids.length; i++) {
2554
+ let dot = 0;
2555
+ const base = i * dims;
2556
+ for (let d = 0; d < dims; d++)
2557
+ dot += queryVector[d] * (index.int8[base + d] / 127);
2558
+ const modality = index.modalities[i];
2559
+ const list = best.get(modality) ?? [];
2560
+ list.push({ shard, id: index.ids[i], cosine: dot });
2561
+ best.set(modality, list);
2562
+ }
2563
+ }
2564
+ const hits = [];
2565
+ for (const list of best.values()) {
2566
+ list.sort((a, b) => b.cosine - a.cosine || a.id - b.id);
2567
+ let kept = 0;
2568
+ for (const candidate of list) {
2569
+ if (kept >= perModality) break;
2570
+ const row = this.shardDb(candidate.shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`).get(BigInt(candidate.id));
2571
+ if (!row) continue;
2572
+ if (opts.allowedDocumentIds && !opts.allowedDocumentIds.has(row.document_id))
2573
+ continue;
2574
+ kept++;
2575
+ hits.push({
2576
+ ...this.hitFromRow(row, candidate.shard.id),
2577
+ cosine: candidate.cosine,
2578
+ source: "vector"
2579
+ });
2580
+ }
2581
+ }
2582
+ return hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
2583
+ }
2584
+ hitFromRow(row, shardId) {
2585
+ const media = mediaOf(row);
2586
+ return {
2587
+ chunkUid: row.chunk_uid,
2588
+ documentId: row.document_id,
2589
+ title: row.title,
2590
+ headingPath: parseHeadingPath(row.heading_path),
2591
+ lineStart: Number(row.line_start),
2592
+ lineEnd: Number(row.line_end),
2593
+ text: row.text,
2594
+ shardId,
2595
+ ...media ? { media } : {}
2596
+ };
2597
+ }
2598
+ /**
2599
+ * Refuse a query vector whose width is not the catalog's stored width,
2600
+ * before any scan: routing and rerank would otherwise score a silent
2601
+ * prefix or fail deep inside a shard with an untyped error.
2602
+ */
2603
+ assertQueryWidth(queryVector) {
2604
+ const dims = this.dimensions();
2605
+ if (queryVector.length !== dims) {
2606
+ throw new CatalogQueryError(
2607
+ `query vector has ${queryVector.length} dimensions, the catalog stores ${dims}`,
2608
+ "dimension"
2609
+ );
2610
+ }
2611
+ }
1899
2612
  /** The embedding dimension from the router's profile echo. */
1900
2613
  dimensions() {
1901
2614
  const raw = this.meta.embedding_profile_json;
@@ -1923,9 +2636,13 @@ var CatalogHandle = class _CatalogHandle {
1923
2636
  get formatVersion() {
1924
2637
  return this.meta.format_version ?? "";
1925
2638
  }
1926
- documentColumns() {
2639
+ documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
1927
2640
  const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
1928
- return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2641
+ const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2642
+ return columns.split(", ").map((column) => {
2643
+ const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
2644
+ return prefix ? `${prefix}.${column} AS ${column}` : column;
2645
+ }).join(", ");
1929
2646
  }
1930
2647
  documentOrder() {
1931
2648
  return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
@@ -1935,6 +2652,9 @@ var CatalogHandle = class _CatalogHandle {
1935
2652
  * 0.5 catalog every document sits at a root, so both counts agree there.
1936
2653
  */
1937
2654
  topics() {
2655
+ const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
2656
+ FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
2657
+ FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
1938
2658
  return this.router.db.prepare(
1939
2659
  `WITH RECURSIVE sub(root, id, depth) AS (
1940
2660
  SELECT id, id, 0 FROM topics
@@ -1943,8 +2663,7 @@ var CatalogHandle = class _CatalogHandle {
1943
2663
  WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1944
2664
  ),
1945
2665
  rollup AS (
1946
- SELECT sub.root AS id, SUM(t.document_count) AS total
1947
- FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
2666
+ ${rollup}
1948
2667
  )
1949
2668
  SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
1950
2669
  rollup.total AS total_document_count
@@ -1968,6 +2687,9 @@ var CatalogHandle = class _CatalogHandle {
1968
2687
  const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1969
2688
  const offset = Math.max(0, opts.offset ?? 0);
1970
2689
  const descendants = opts.descendants ?? true;
2690
+ if (this.schemaVersion >= 4 && opts.topicId) {
2691
+ return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
2692
+ }
1971
2693
  let scope = "";
1972
2694
  let where = "";
1973
2695
  const params = [];
@@ -1990,16 +2712,89 @@ var CatalogHandle = class _CatalogHandle {
1990
2712
  const documents = this.router.db.prepare(
1991
2713
  `${scope} SELECT ${this.documentColumns()}
1992
2714
  FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
1993
- ).all(...params, limit, offset).map(rowToDocumentMeta);
2715
+ ).all(...params, limit, offset).map((row) => this.documentMeta(row));
2716
+ return { documents, total };
2717
+ }
2718
+ /** A reference is a placement, not a duplicate result or a redirect page. */
2719
+ sharedDocumentsPage(topicId, descendants, limit, offset) {
2720
+ const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
2721
+ JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
2722
+ const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
2723
+ SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
2724
+ FROM topic_documents td JOIN topics t ON t.id = td.topic_id
2725
+ WHERE td.topic_id IN (SELECT id FROM sub)
2726
+ )`;
2727
+ const total = Number(
2728
+ this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
2729
+ );
2730
+ const documents = this.router.db.prepare(
2731
+ `${scope}, ranked AS (
2732
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
2733
+ ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
2734
+ ) SELECT ${this.documentColumns("d", "p")}
2735
+ FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
2736
+ ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
2737
+ ).all(topicId, limit, offset).map((row) => this.documentMeta(row));
1994
2738
  return { documents, total };
1995
2739
  }
2740
+ /** Integrity facts kept separate from browse counts, which include shared references. */
2741
+ tocIntegrity() {
2742
+ const count = (sql) => Number(this.router.db.prepare(sql).get().n);
2743
+ const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
2744
+ const incorrectCounts = count(
2745
+ `SELECT COUNT(*) AS n FROM topics t LEFT JOIN
2746
+ (SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
2747
+ WHERE t.document_count != COALESCE(r.n, 0)`
2748
+ );
2749
+ if (this.schemaVersion < 4)
2750
+ return {
2751
+ placements: count("SELECT COUNT(*) AS n FROM documents"),
2752
+ invalidReferences: 0,
2753
+ missingPrimary: 0,
2754
+ incorrectCounts,
2755
+ duplicateReferences: 0
2756
+ };
2757
+ return {
2758
+ placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
2759
+ invalidReferences: count(
2760
+ `SELECT COUNT(*) AS n FROM topic_documents r
2761
+ LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
2762
+ WHERE d.id IS NULL OR t.id IS NULL OR
2763
+ (r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
2764
+ ),
2765
+ missingPrimary: count(
2766
+ `SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
2767
+ SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
2768
+ ),
2769
+ incorrectCounts,
2770
+ duplicateReferences: count(
2771
+ "SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
2772
+ )
2773
+ };
2774
+ }
1996
2775
  getDocument(id) {
1997
2776
  const row = this.router.db.prepare(
1998
2777
  `SELECT ${this.documentColumns()}, body_codec, body_blob
1999
2778
  FROM documents WHERE id = ?`
2000
2779
  ).get(id);
2001
2780
  if (!row) return null;
2002
- return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
2781
+ return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
2782
+ }
2783
+ /**
2784
+ * One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
2785
+ * shard the document's chunks live in. Null when the document or the
2786
+ * passage is not in this catalog, which is what a fabricated or stale
2787
+ * citation looks like.
2788
+ */
2789
+ getChunk(documentId, chunkUid2) {
2790
+ const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
2791
+ if (!doc) return null;
2792
+ const shardId = Number(doc.shard_id);
2793
+ const shard = this.shards.find((s) => s.id === shardId);
2794
+ if (!shard) return null;
2795
+ const row = this.shardDb(shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE chunk_uid = ? AND document_id = ?`).get(chunkUid2, documentId);
2796
+ if (!row) return null;
2797
+ return this.hitFromRow(row, shardId);
2003
2798
  }
2004
2799
  /** Documents whose `topic_id` names no declared topic (a validator check). */
2005
2800
  documentsWithUndeclaredTopic() {
@@ -2072,7 +2867,7 @@ var CatalogHandle = class _CatalogHandle {
2072
2867
  const index = /* @__PURE__ */ new Map();
2073
2868
  let raw;
2074
2869
  try {
2075
- raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2870
+ raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2076
2871
  } catch (error) {
2077
2872
  if (error.code === "ENOENT") {
2078
2873
  this.assetsByPath = index;
@@ -2088,7 +2883,7 @@ var CatalogHandle = class _CatalogHandle {
2088
2883
  for (const file of files) {
2089
2884
  const path = file?.path;
2090
2885
  if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
2091
- const contentType = assetContentType(path);
2886
+ const contentType = assetContentType2(path);
2092
2887
  if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
2093
2888
  continue;
2094
2889
  }
@@ -2102,16 +2897,42 @@ var CatalogHandle = class _CatalogHandle {
2102
2897
  assets() {
2103
2898
  return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
2104
2899
  }
2900
+ /**
2901
+ * Where a declared asset lives on disk, so a large one (video, audio) can be
2902
+ * streamed with byte ranges instead of read whole. Checks containment and
2903
+ * the manifest size; the content hash was verified once, at extraction.
2904
+ */
2905
+ assetFile(path) {
2906
+ const info = this.assetIndex().get(path);
2907
+ if (!info) return null;
2908
+ if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
2909
+ throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
2910
+ }
2911
+ const absPath = this.resolveCatalogPath(path);
2912
+ let size;
2913
+ try {
2914
+ size = statSync2(absPath).size;
2915
+ } catch (error) {
2916
+ throw new CatalogOpenError(
2917
+ `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
2918
+ "corrupt"
2919
+ );
2920
+ }
2921
+ if (size !== info.sizeBytes) {
2922
+ throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
2923
+ }
2924
+ return { ...info, absPath };
2925
+ }
2105
2926
  /** One declared asset's bytes, or null when the catalog declares no such asset. */
2106
2927
  readAsset(path) {
2107
2928
  const info = this.assetIndex().get(path);
2108
2929
  if (!info) return null;
2109
- if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
2930
+ if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
2110
2931
  throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
2111
2932
  }
2112
2933
  let bytes;
2113
2934
  try {
2114
- bytes = readFileSync2(this.resolveCatalogPath(path));
2935
+ bytes = readFileSync3(this.resolveCatalogPath(path));
2115
2936
  } catch (error) {
2116
2937
  throw new CatalogOpenError(
2117
2938
  `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
@@ -2123,19 +2944,59 @@ var CatalogHandle = class _CatalogHandle {
2123
2944
  }
2124
2945
  return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
2125
2946
  }
2947
+ documentMeta(row) {
2948
+ const document = rowToDocumentMeta(row);
2949
+ const locations = this.spatial.locations(document.id);
2950
+ return locations.length ? { ...document, locations } : document;
2951
+ }
2952
+ documentLocations(documentId) {
2953
+ return this.spatial.locations(documentId);
2954
+ }
2955
+ documentLocationRows() {
2956
+ return this.spatial.rows();
2957
+ }
2958
+ spatialIntegrity() {
2959
+ return this.spatial.integrity();
2960
+ }
2961
+ spatialMatches(radius) {
2962
+ return this.spatial.matches(radius);
2963
+ }
2964
+ nearbyDocuments(radius, opts = {}) {
2965
+ const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
2966
+ const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
2967
+ const ordered = [...this.spatialMatches(radius)].sort(
2968
+ (a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
2969
+ );
2970
+ const get = this.router.db.prepare(
2971
+ `SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
2972
+ );
2973
+ return {
2974
+ total: ordered.length,
2975
+ documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
2976
+ ...this.documentMeta(get.get(id)),
2977
+ ...match
2978
+ }))
2979
+ };
2980
+ }
2981
+ shardsForDocuments(ids) {
2982
+ if (!ids.size) return [];
2983
+ return this.router.db.prepare(
2984
+ "SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
2985
+ ).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
2986
+ }
2126
2987
  // ── search ────────────────────────────────────────────────────────────────
2127
2988
  /**
2128
2989
  * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
2129
2990
  * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
2130
2991
  */
2131
- searchDocumentsFts(query, limit = 10) {
2132
- const match = ftsQuery(query);
2133
- if (!match) return [];
2992
+ searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
2134
2993
  try {
2135
- return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
2136
- documentId,
2137
- rank: i
2138
- }));
2994
+ return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
2995
+ (documentId, i) => ({
2996
+ documentId,
2997
+ rank: i
2998
+ })
2999
+ );
2139
3000
  } catch {
2140
3001
  return [];
2141
3002
  }
@@ -2159,6 +3020,7 @@ var CatalogHandle = class _CatalogHandle {
2159
3020
  * takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
2160
3021
  */
2161
3022
  scoreShards(queryVector) {
3023
+ this.assertQueryWidth(queryVector);
2162
3024
  const rows = this.router.db.prepare("SELECT shard_id, embedding FROM route_centroids").all();
2163
3025
  const best = /* @__PURE__ */ new Map();
2164
3026
  for (const row of rows) {
@@ -2168,31 +3030,80 @@ var CatalogHandle = class _CatalogHandle {
2168
3030
  Math.floor(row.embedding.byteLength / 4)
2169
3031
  );
2170
3032
  let dot = 0;
2171
- const n = Math.min(centroid.length, queryVector.length);
2172
- for (let i = 0; i < n; i++) dot += centroid[i] * queryVector[i];
3033
+ for (let i = 0; i < queryVector.length; i++) dot += centroid[i] * queryVector[i];
2173
3034
  const id = Number(row.shard_id);
2174
3035
  if (dot > (best.get(id) ?? Number.NEGATIVE_INFINITY)) best.set(id, dot);
2175
3036
  }
2176
3037
  return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
2177
3038
  }
2178
- /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
2179
- routeShards(queryVector, shardBudget) {
3039
+ /**
3040
+ * Top-S shard ids within THIS catalog (single-catalog callers, tests).
3041
+ * With the raw `query`, also the shard of the page it names by title
3042
+ * (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
3043
+ */
3044
+ routeShards(queryVector, shardBudget, query) {
2180
3045
  if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
2181
- return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
3046
+ const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
3047
+ if (query) {
3048
+ for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
3049
+ if (!routed.includes(named.shardId)) routed.push(named.shardId);
3050
+ }
3051
+ }
3052
+ return routed;
3053
+ }
3054
+ /**
3055
+ * Title-assisted routing: the shards holding the pages `query` names by
3056
+ * title (`namedTitleMatches`), most specific name first, at most `limit`
3057
+ * distinct shards. Centroid routing compares a question with averages of
3058
+ * whole shards, and e5 scores those in a band a few hundredths wide, so
3059
+ * the shard holding the very page a question names can rank 8th of 10:
3060
+ * "Who were the members of ABBA?" never reached the ABBA article in the
3061
+ * Wikipedia music catalog. Its title does — the vector arm scans that
3062
+ * shard too. Measured on the 10-shard music and film-tv catalogs, one such
3063
+ * shard lifted end-to-end recall@8 on "What is <title>?" questions from
3064
+ * 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
3065
+ *
3066
+ * `score` is the named match's specificity (fts-query.ts), comparable
3067
+ * across catalogs closely enough for the daemon to choose which catalogs'
3068
+ * shards to add when several name a page. Empty on any FTS error.
3069
+ */
3070
+ titleRouteShards(query, limit) {
3071
+ if (limit <= 0) return [];
3072
+ try {
3073
+ const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
3074
+ const out = [];
3075
+ for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
3076
+ const row = shardOf.get(named.documentId);
3077
+ if (!row) continue;
3078
+ const shardId = Number(row.shard_id);
3079
+ if (out.some((o) => o.shardId === shardId)) continue;
3080
+ out.push({ shardId, score: named.score });
3081
+ if (out.length >= limit) break;
3082
+ }
3083
+ return out.sort((a, b) => compareScores(b.score, a.score));
3084
+ } catch {
3085
+ return [];
3086
+ }
2182
3087
  }
2183
3088
  /**
2184
3089
  * Two-stage semantic search over the routed shards: sign-bit scan
2185
3090
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
2186
3091
  */
2187
3092
  searchSemantic(queryVector, opts = {}) {
2188
- const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
2189
- return this.searchShards(queryVector, shardIds, opts.finalK ?? RERANK_FINAL_K);
3093
+ const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
3094
+ return this.searchShards(
3095
+ queryVector,
3096
+ shardIds,
3097
+ opts.finalK ?? RERANK_FINAL_K,
3098
+ opts.allowedDocumentIds
3099
+ );
2190
3100
  }
2191
3101
  /**
2192
3102
  * The scan stage against an EXPLICIT shard list — what the daemon calls
2193
3103
  * after global cross-catalog routing has already spent the S budget.
2194
3104
  */
2195
- searchShards(queryVector, shardIds, finalK) {
3105
+ searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
3106
+ this.assertQueryWidth(queryVector);
2196
3107
  const center = this.binaryCenter();
2197
3108
  const scanQuery = center ? centerVector(queryVector, center) : queryVector;
2198
3109
  const hits = [];
@@ -2200,8 +3111,21 @@ var CatalogHandle = class _CatalogHandle {
2200
3111
  const shard = this.shards.find((s) => s.id === shardId);
2201
3112
  if (!shard) continue;
2202
3113
  const db = this.shardDb(shard);
2203
- const k = rerankK(finalK, shard.chunkCount);
2204
- const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
3114
+ const eligible = allowedDocumentIds ? new Set(
3115
+ db.prepare(
3116
+ "SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
3117
+ ).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
3118
+ ) : void 0;
3119
+ const media = this.shardMedia(shard);
3120
+ const k = rerankK(finalK, eligible?.size ?? shard.chunkCount - media.ids.length);
3121
+ const excluded = media.ids.length > 0 ? media.idSet : void 0;
3122
+ const candidates = asymmetricTopK(
3123
+ this.shardBits(shard),
3124
+ scanQuery,
3125
+ k,
3126
+ eligible,
3127
+ excluded
3128
+ ).map((hit) => ({
2205
3129
  chunk_id: hit.chunkId
2206
3130
  }));
2207
3131
  if (candidates.length === 0) continue;
@@ -2212,63 +3136,57 @@ var CatalogHandle = class _CatalogHandle {
2212
3136
  const int8 = new Int8Array(row.v.buffer, row.v.byteOffset, row.v.byteLength);
2213
3137
  return { chunkId: Number(c.chunk_id), cosine: rerankScore(queryVector, int8) };
2214
3138
  }).filter((r) => r !== null).sort((a, b) => b.cosine - a.cosine).slice(0, finalK);
2215
- const getChunk = db.prepare(
2216
- `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
2217
- FROM chunks WHERE id = ?`
2218
- );
3139
+ const getChunk = db.prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`);
2219
3140
  for (const r of reranked) {
2220
3141
  const row = getChunk.get(BigInt(r.chunkId));
2221
3142
  if (!row) continue;
2222
- hits.push({
2223
- chunkUid: row.chunk_uid,
2224
- documentId: row.document_id,
2225
- title: row.title,
2226
- headingPath: parseHeadingPath(row.heading_path),
2227
- lineStart: Number(row.line_start),
2228
- lineEnd: Number(row.line_end),
2229
- text: row.text,
2230
- cosine: r.cosine,
2231
- source: "vector",
2232
- shardId
2233
- });
3143
+ hits.push({ ...this.hitFromRow(row, shardId), cosine: r.cosine, source: "vector" });
2234
3144
  }
2235
3145
  }
2236
3146
  hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
2237
3147
  return hits;
2238
3148
  }
2239
3149
  /** Chunk-body FTS over the routed shards (explicit search only). */
2240
- searchChunksFts(query, shardIds, limitPerShard = 12) {
2241
- const match = ftsQuery(query);
2242
- if (!match) return [];
3150
+ searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
2243
3151
  const hits = [];
2244
3152
  for (const shardId of shardIds) {
2245
3153
  const shard = this.shards.find((s) => s.id === shardId);
2246
3154
  if (!shard) continue;
2247
3155
  const db = this.shardDb(shard);
2248
3156
  try {
3157
+ const match = selectiveFtsQuery(
3158
+ query,
3159
+ shard.chunkCount,
3160
+ (phrase) => this.chunkFtsRows(shard, db, phrase)
3161
+ );
3162
+ if (!match) return hits;
2249
3163
  const rows = db.prepare(
2250
- `SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
3164
+ `SELECT ${this.chunkColumns().split(", ").map((column) => `c.${column}`).join(", ")}
2251
3165
  FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
2252
- WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
2253
- ).all(match, limitPerShard);
3166
+ WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
3167
+ ).all(
3168
+ ...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
3169
+ );
2254
3170
  for (const row of rows) {
2255
- hits.push({
2256
- chunkUid: row.chunk_uid,
2257
- documentId: row.document_id,
2258
- title: row.title,
2259
- headingPath: parseHeadingPath(row.heading_path),
2260
- lineStart: Number(row.line_start),
2261
- lineEnd: Number(row.line_end),
2262
- text: row.text,
2263
- source: "fts",
2264
- shardId
2265
- });
3171
+ hits.push({ ...this.hitFromRow(row, shardId), source: "fts" });
2266
3172
  }
2267
3173
  } catch {
2268
3174
  }
2269
3175
  }
2270
3176
  return hits;
2271
3177
  }
3178
+ /** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
3179
+ chunkFtsRows(shard, db, phrase) {
3180
+ const key = `${shard.path}\0${phrase}`;
3181
+ let rows = this.chunkTermRows.get(key);
3182
+ if (rows === void 0) {
3183
+ if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
3184
+ const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
3185
+ rows = Number(row.c);
3186
+ this.chunkTermRows.set(key, rows);
3187
+ }
3188
+ return rows;
3189
+ }
2272
3190
  /**
2273
3191
  * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
2274
3192
  * nearest neighbor — proves the shard's vectors load and scan. Returns
@@ -2322,19 +3240,20 @@ function parseHeadingPath(raw) {
2322
3240
  return [];
2323
3241
  }
2324
3242
  }
2325
- var ftsQuery = sanitizeFtsQuery;
2326
3243
 
2327
3244
  // src/reader/validate.ts
2328
- import { readFile } from "fs/promises";
3245
+ import { open, readFile } from "fs/promises";
2329
3246
  import { join as join3 } from "path";
2330
3247
  import {
2331
3248
  KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
2332
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2333
- MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
2334
- MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
2335
3249
  assetExtension as assetExtension2,
2336
3250
  assetKindForExtension as assetKindForExtension2,
3251
+ assetModality as assetModality2,
3252
+ canonicalizeJson as canonicalizeJson2,
3253
+ formatAtLeast as formatAtLeast2,
2337
3254
  isKnowledgeAssetPath as isKnowledgeAssetPath2,
3255
+ isKnowledgeMediaAssetPath as isKnowledgeMediaAssetPath2,
3256
+ knowledgeAssetLimitsProblem as knowledgeAssetLimitsProblem2,
2338
3257
  sniffAssetType as sniffAssetType2,
2339
3258
  svgInertnessProblem as svgInertnessProblem2
2340
3259
  } from "@bendyline/gezk";
@@ -2409,6 +3328,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2409
3328
  handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
2410
3329
  `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
2411
3330
  );
3331
+ if (formatAtLeast2(manifest.formatVersion, "0.7")) {
3332
+ try {
3333
+ const spatial = handle.spatialIntegrity();
3334
+ check("document-locations-table", spatial.hasTable);
3335
+ check(
3336
+ "document-locations-summary",
3337
+ Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
3338
+ "spatial counts and coverage must match router points"
3339
+ );
3340
+ } catch (error) {
3341
+ check(
3342
+ "document-locations-integrity",
3343
+ false,
3344
+ error instanceof Error ? error.message : String(error)
3345
+ );
3346
+ }
3347
+ }
2412
3348
  const topics = handle.topics();
2413
3349
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
2414
3350
  check("topics-tree", ...topicTreeProblem(topics));
@@ -2428,13 +3364,16 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2428
3364
  } else {
2429
3365
  const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
2430
3366
  check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
2431
- const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
2432
- const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
2433
- check(
2434
- "assets-limits",
2435
- assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2436
- `${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
2437
- );
3367
+ if (!formatAtLeast2(manifest.formatVersion, "0.8")) {
3368
+ const media = assetFiles.filter((f) => isKnowledgeMediaAssetPath2(f.path));
3369
+ check(
3370
+ "assets-media-in-0.8",
3371
+ media.length === 0,
3372
+ `audio/video assets need format 0.8: ${media.map((f) => f.path).join(", ")}`
3373
+ );
3374
+ }
3375
+ const limits = knowledgeAssetLimitsProblem2(assetFiles);
3376
+ check("assets-limits", limits === null, limits ?? "");
2438
3377
  check(
2439
3378
  "counts-assets",
2440
3379
  (manifest.counts.assets ?? 0) === assetFiles.length,
@@ -2446,11 +3385,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2446
3385
  manifest.files.some((f) => f.path === manifest.license.noticePath),
2447
3386
  `manifest.files lacks the declared notice ${manifest.license.noticePath}`
2448
3387
  );
3388
+ const toc = handle.tocIntegrity();
3389
+ check(
3390
+ "toc-references",
3391
+ toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
3392
+ `${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
3393
+ );
3394
+ check(
3395
+ "toc-counts",
3396
+ toc.incorrectCounts === 0,
3397
+ `${toc.incorrectCounts} incorrect per-topic counts`
3398
+ );
2449
3399
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
2450
3400
  check(
2451
3401
  "counts-documents",
2452
- handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === manifest.counts.documents,
2453
- `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
3402
+ handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
3403
+ `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
2454
3404
  );
2455
3405
  check(
2456
3406
  "counts-shards",
@@ -2482,6 +3432,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2482
3432
  bodies.maxCompressedBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024,
2483
3433
  `largest compressed body is ${bodies.maxCompressedBytes} bytes`
2484
3434
  );
3435
+ const mediaRowCounts = { image: 0, video: 0, audio: 0 };
2485
3436
  for (const shard of handle.shards) {
2486
3437
  const conn = openCatalogDatabase(handle.resolveCatalogPath(shard.path));
2487
3438
  const db = conn.db;
@@ -2518,6 +3469,21 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2518
3469
  badBit === 0 && badInt8 === 0,
2519
3470
  `${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
2520
3471
  );
3472
+ if (handle.schemaVersion >= 5) {
3473
+ const rows = db.prepare(
3474
+ `SELECT modality, asset_path, thumbnail_path FROM chunks WHERE modality <> 'text'`
3475
+ ).all();
3476
+ const declaredAssets = new Set(assetFiles.map((f) => f.path));
3477
+ const wrong = rows.filter(
3478
+ (r) => !declaredAssets.has(r.asset_path) || assetModality2(r.asset_path) !== r.modality || r.thumbnail_path !== null && (!declaredAssets.has(r.thumbnail_path) || assetModality2(r.thumbnail_path) !== "image")
3479
+ ).slice(0, 5).map((r) => `${r.modality} ${r.asset_path}`);
3480
+ check(
3481
+ `media-rows:${shard.path}`,
3482
+ wrong.length === 0,
3483
+ `media rows naming an undeclared or mismatched asset: ${wrong.join(", ")}`
3484
+ );
3485
+ for (const r of rows) mediaRowCounts[r.modality] += 1;
3486
+ }
2521
3487
  const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
2522
3488
  check(
2523
3489
  `chunk-ids-dense:${shard.path}`,
@@ -2529,6 +3495,14 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2529
3495
  }
2530
3496
  check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
2531
3497
  }
3498
+ if (handle.schemaVersion >= 5) {
3499
+ const counted = manifest.counts.media ?? { image: 0, video: 0, audio: 0 };
3500
+ check(
3501
+ "counts-media",
3502
+ counted.image === mediaRowCounts.image && counted.video === mediaRowCounts.video && counted.audio === mediaRowCounts.audio,
3503
+ `manifest counts ${JSON.stringify(counted)}, shards hold ${JSON.stringify(mediaRowCounts)}`
3504
+ );
3505
+ }
2532
3506
  if (handle.schemaVersion >= 3) {
2533
3507
  const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
2534
3508
  check(
@@ -2540,7 +3514,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2540
3514
  for (const file of assetFiles) {
2541
3515
  const ext = assetExtension2(file.path);
2542
3516
  if (!ext) continue;
2543
- const bytes = await readFile(join3(rootDir, file.path));
3517
+ const bytes = isKnowledgeMediaAssetPath2(file.path) ? await readLeadingBytes(join3(rootDir, file.path), 64) : await readFile(join3(rootDir, file.path));
2544
3518
  const kind = sniffAssetType2(bytes);
2545
3519
  check(
2546
3520
  `asset-type:${file.path}`,
@@ -2578,6 +3552,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2578
3552
  );
2579
3553
  }
2580
3554
  }
3555
+ } catch (error) {
3556
+ check("catalog-structure", false, error instanceof Error ? error.message : String(error));
2581
3557
  } finally {
2582
3558
  handle.close();
2583
3559
  }
@@ -2616,6 +3592,16 @@ function topicTreeProblem(topics) {
2616
3592
  }
2617
3593
  return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
2618
3594
  }
3595
+ async function readLeadingBytes(path, length) {
3596
+ const handle = await open(path, "r");
3597
+ try {
3598
+ const buffer = new Uint8Array(length);
3599
+ const { bytesRead } = await handle.read(buffer, 0, length, 0);
3600
+ return buffer.subarray(0, bytesRead);
3601
+ } finally {
3602
+ await handle.close();
3603
+ }
3604
+ }
2619
3605
 
2620
3606
  // src/export/duckdb.ts
2621
3607
  import { spawn } from "child_process";
@@ -2686,13 +3672,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
2686
3672
  }
2687
3673
 
2688
3674
  // src/export/parquet.ts
2689
- import { createHash as createHash3 } from "crypto";
3675
+ import { createHash as createHash5 } from "crypto";
2690
3676
  import { createWriteStream as createWriteStream3 } from "fs";
2691
3677
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2692
3678
  import { join as join4, resolve as resolve3 } from "path";
2693
3679
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2694
3680
  import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
2695
- var PARQUET_EXPORT_VERSION = 3;
3681
+ var PARQUET_EXPORT_VERSION = 4;
2696
3682
  var DEFAULT_ROW_GROUP_SIZE = 32768;
2697
3683
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2698
3684
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2739,6 +3725,34 @@ async function exportCatalogParquet(opts) {
2739
3725
  onProgress: opts.onProgress
2740
3726
  })
2741
3727
  );
3728
+ const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
3729
+ const locations = spatial.rows();
3730
+ if (spatial.hasTable || locations.length) {
3731
+ const path = join4(staging, "document-locations.ndjson");
3732
+ const out = ndjsonWriter(path);
3733
+ for (const { documentId, location } of locations)
3734
+ await out.write({
3735
+ document_id: documentId,
3736
+ location_id: location.id,
3737
+ latitude: location.latitude,
3738
+ longitude: location.longitude,
3739
+ role: location.role,
3740
+ provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
3741
+ });
3742
+ await out.close();
3743
+ const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
3744
+ const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
3745
+ files.push(
3746
+ await writeParquet(opts.duckdb, {
3747
+ outDir,
3748
+ file: "document-locations.parquet",
3749
+ table: "document_locations",
3750
+ rows: locations.length,
3751
+ sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
3752
+ onProgress: opts.onProgress
3753
+ })
3754
+ );
3755
+ }
2742
3756
  for (const shard of manifest.router.shards) {
2743
3757
  const tag = String(shard.id).padStart(3, "0");
2744
3758
  const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
@@ -3081,7 +4095,7 @@ async function writeParquet(cli, spec) {
3081
4095
  ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
3082
4096
  rows: spec.rows,
3083
4097
  sizeBytes: (await stat2(target)).size,
3084
- sha256: createHash3("sha256").update(bytes).digest("hex")
4098
+ sha256: createHash5("sha256").update(bytes).digest("hex")
3085
4099
  };
3086
4100
  }
3087
4101
  async function findDuckdbBinary(env = process.env) {
@@ -3498,6 +4512,522 @@ var MULTILINGUAL_E5_SMALL_CENTER = [
3498
4512
  0.049673
3499
4513
  ];
3500
4514
 
4515
+ // src/profiles/embeddinggemma-2-center.ts
4516
+ var EMBEDDINGGEMMA_2_512_CENTER = [
4517
+ -3987e-6,
4518
+ 0.017212,
4519
+ 0.023034,
4520
+ -0.020831,
4521
+ 0.013136,
4522
+ 0.018836,
4523
+ 7334e-6,
4524
+ -0.022294,
4525
+ 5809e-6,
4526
+ -0.147176,
4527
+ -5646e-6,
4528
+ -0.022884,
4529
+ -0.011497,
4530
+ 1292e-6,
4531
+ 0.014257,
4532
+ -1878e-6,
4533
+ 0.051834,
4534
+ -0.012991,
4535
+ 933e-6,
4536
+ 0.022558,
4537
+ -0.034902,
4538
+ -0.030019,
4539
+ 0.017879,
4540
+ 0.015619,
4541
+ -0.014837,
4542
+ 0.028657,
4543
+ 0.017575,
4544
+ -0.045617,
4545
+ -0.03244,
4546
+ 0.23277,
4547
+ -0.021971,
4548
+ -0.026583,
4549
+ 0.011485,
4550
+ -0.01468,
4551
+ 0.044649,
4552
+ 0.018019,
4553
+ -0.011519,
4554
+ 0.035574,
4555
+ 512e-5,
4556
+ -0.030136,
4557
+ 4583e-6,
4558
+ 0.018314,
4559
+ -0.028859,
4560
+ 0.027199,
4561
+ 0.020695,
4562
+ -849e-6,
4563
+ 0.015867,
4564
+ -0.024724,
4565
+ -8979e-6,
4566
+ 0.023625,
4567
+ 0.028948,
4568
+ -0.025744,
4569
+ -1547e-6,
4570
+ 0.0281,
4571
+ 0.024258,
4572
+ 0.02262,
4573
+ 1386e-6,
4574
+ 0.015002,
4575
+ 3764e-6,
4576
+ 0.016978,
4577
+ 0.019178,
4578
+ 0.012229,
4579
+ -193e-6,
4580
+ 0.032654,
4581
+ -0.030602,
4582
+ -1749e-6,
4583
+ 0.027403,
4584
+ -4047e-6,
4585
+ 0.031605,
4586
+ 0.023884,
4587
+ 0.015804,
4588
+ 3971e-6,
4589
+ 0.021826,
4590
+ 0.02457,
4591
+ -0.023111,
4592
+ -0.032593,
4593
+ -0.090226,
4594
+ 0.020798,
4595
+ 6402e-6,
4596
+ 0.015607,
4597
+ -8457e-6,
4598
+ -0.010067,
4599
+ 6362e-6,
4600
+ 9973e-6,
4601
+ 0.0122,
4602
+ -0.020958,
4603
+ -0.031666,
4604
+ 0.034654,
4605
+ -0.012847,
4606
+ -1119e-6,
4607
+ 0.024259,
4608
+ 0.018097,
4609
+ 9105e-6,
4610
+ -9523e-6,
4611
+ -0.022682,
4612
+ -5167e-6,
4613
+ -0.022959,
4614
+ 0.012175,
4615
+ -0.020933,
4616
+ 0.014024,
4617
+ -4898e-6,
4618
+ -2018e-6,
4619
+ -7178e-6,
4620
+ 0.038098,
4621
+ 0.016307,
4622
+ 5668e-6,
4623
+ -0.014475,
4624
+ 661e-6,
4625
+ -1287e-6,
4626
+ -0.025024,
4627
+ 5704e-6,
4628
+ -0.32188,
4629
+ -0.026611,
4630
+ 0.032111,
4631
+ -9511e-6,
4632
+ 0.260031,
4633
+ 0.027274,
4634
+ -909e-5,
4635
+ 0.055461,
4636
+ 0.024395,
4637
+ -0.017894,
4638
+ 0.061775,
4639
+ -8529e-6,
4640
+ -1795e-6,
4641
+ -0.017545,
4642
+ 0.011492,
4643
+ 0.044594,
4644
+ 38e-4,
4645
+ -0.02231,
4646
+ 0.029977,
4647
+ 0.010629,
4648
+ -489e-5,
4649
+ 2646e-6,
4650
+ 6408e-6,
4651
+ -0.014453,
4652
+ -0.020764,
4653
+ -0.018043,
4654
+ -0.015625,
4655
+ 0.074213,
4656
+ -0.012319,
4657
+ -8038e-6,
4658
+ -6265e-6,
4659
+ 0.012548,
4660
+ -621e-6,
4661
+ -0.021893,
4662
+ 0.039889,
4663
+ 3623e-6,
4664
+ -0.029173,
4665
+ -716e-6,
4666
+ -0.013168,
4667
+ -6674e-6,
4668
+ -0.010069,
4669
+ -0.02451,
4670
+ 0.015114,
4671
+ -0.012945,
4672
+ 1561e-6,
4673
+ 0.013299,
4674
+ -0.029098,
4675
+ 0.01304,
4676
+ 0.013519,
4677
+ -0.014282,
4678
+ 0.026849,
4679
+ -2462e-6,
4680
+ 0.019473,
4681
+ -8227e-6,
4682
+ -822e-6,
4683
+ -7241e-6,
4684
+ 4259e-6,
4685
+ -0.016026,
4686
+ 7987e-6,
4687
+ -7065e-6,
4688
+ -1229e-6,
4689
+ -0.01583,
4690
+ 0.021277,
4691
+ 0.013837,
4692
+ 0.024282,
4693
+ -7589e-6,
4694
+ -0.016577,
4695
+ 7774e-6,
4696
+ 0.010479,
4697
+ -0.012539,
4698
+ -0.020799,
4699
+ 0.026506,
4700
+ 8733e-6,
4701
+ -0.015001,
4702
+ 2151e-6,
4703
+ -0.011922,
4704
+ -0.02785,
4705
+ -0.060631,
4706
+ 3372e-6,
4707
+ 0.022441,
4708
+ -0.012448,
4709
+ 6827e-6,
4710
+ -7047e-6,
4711
+ 1249e-6,
4712
+ -2934e-6,
4713
+ 4559e-6,
4714
+ -0.011409,
4715
+ -0.012564,
4716
+ 5788e-6,
4717
+ -0.020554,
4718
+ -4292e-6,
4719
+ 0.01927,
4720
+ 0.013487,
4721
+ -0.015484,
4722
+ 9704e-6,
4723
+ 5617e-6,
4724
+ 0.02447,
4725
+ 2056e-6,
4726
+ 0.011381,
4727
+ 0.039831,
4728
+ -7878e-6,
4729
+ -0.011366,
4730
+ -814e-5,
4731
+ 8099e-6,
4732
+ 6637e-6,
4733
+ 154e-6,
4734
+ 0.072273,
4735
+ 7195e-6,
4736
+ -0.011899,
4737
+ 0.016669,
4738
+ 3331e-6,
4739
+ 0.032787,
4740
+ -0.012251,
4741
+ -6049e-6,
4742
+ -0.017814,
4743
+ 0.055136,
4744
+ 0.028157,
4745
+ -0.028898,
4746
+ 0.035646,
4747
+ 0.036174,
4748
+ -5621e-6,
4749
+ 0.050018,
4750
+ -0.040307,
4751
+ 0.060677,
4752
+ 58e-6,
4753
+ -0.045134,
4754
+ 4752e-6,
4755
+ 7161e-6,
4756
+ 0.040398,
4757
+ -0.051202,
4758
+ -572e-6,
4759
+ 6186e-6,
4760
+ -0.010613,
4761
+ 0.039894,
4762
+ -0.014924,
4763
+ 49e-4,
4764
+ -0.011771,
4765
+ 9397e-6,
4766
+ 9457e-6,
4767
+ 0.025576,
4768
+ 2251e-6,
4769
+ 0.020638,
4770
+ -0.029722,
4771
+ 7545e-6,
4772
+ -533e-6,
4773
+ 0.026445,
4774
+ 921e-6,
4775
+ 9681e-6,
4776
+ -959e-6,
4777
+ -0.018021,
4778
+ 807e-5,
4779
+ 8041e-6,
4780
+ -8004e-6,
4781
+ 4422e-6,
4782
+ -2478e-6,
4783
+ 7376e-6,
4784
+ -828e-5,
4785
+ 8006e-6,
4786
+ -3182e-6,
4787
+ 5771e-6,
4788
+ -526e-5,
4789
+ 0.01422,
4790
+ -3677e-6,
4791
+ -0.010823,
4792
+ 0.019873,
4793
+ 3328e-6,
4794
+ 4427e-6,
4795
+ 0.02227,
4796
+ -0.014147,
4797
+ 0.014306,
4798
+ -5089e-6,
4799
+ -0.010849,
4800
+ 322e-5,
4801
+ -8249e-6,
4802
+ 0.019141,
4803
+ -0.012199,
4804
+ 0.015451,
4805
+ 0.026615,
4806
+ 0.02352,
4807
+ 0.011117,
4808
+ -399e-6,
4809
+ 0.016821,
4810
+ 0.023447,
4811
+ 6638e-6,
4812
+ 8598e-6,
4813
+ -0.02239,
4814
+ 0.017325,
4815
+ 0.022729,
4816
+ 9922e-6,
4817
+ -4099e-6,
4818
+ 9124e-6,
4819
+ -8221e-6,
4820
+ -7554e-6,
4821
+ 0.046752,
4822
+ 0.027253,
4823
+ 437e-6,
4824
+ 4693e-6,
4825
+ 7093e-6,
4826
+ -0.014217,
4827
+ -0.024195,
4828
+ 2768e-6,
4829
+ -0.012022,
4830
+ -0.012142,
4831
+ -4326e-6,
4832
+ -0.01054,
4833
+ 0.01168,
4834
+ 4821e-6,
4835
+ -5609e-6,
4836
+ -0.096205,
4837
+ -0.018675,
4838
+ -0.017776,
4839
+ -0.010107,
4840
+ -0.012825,
4841
+ -644e-6,
4842
+ 0.011283,
4843
+ 3759e-6,
4844
+ -0.025286,
4845
+ 2306e-6,
4846
+ 9735e-6,
4847
+ -158e-5,
4848
+ 0.012131,
4849
+ -0.018451,
4850
+ 0.014135,
4851
+ 6421e-6,
4852
+ -7006e-6,
4853
+ -6122e-6,
4854
+ 0.013727,
4855
+ 8605e-6,
4856
+ -6385e-6,
4857
+ -0.016703,
4858
+ -0.184351,
4859
+ 4162e-6,
4860
+ 0.022412,
4861
+ -0.021583,
4862
+ -5397e-6,
4863
+ -0.011593,
4864
+ -1118e-6,
4865
+ 0.013629,
4866
+ 4367e-6,
4867
+ 238e-5,
4868
+ 0.030207,
4869
+ 2765e-6,
4870
+ 0.015452,
4871
+ -0.023998,
4872
+ 0.016889,
4873
+ 5828e-6,
4874
+ 0.01799,
4875
+ -0.019521,
4876
+ 0.012821,
4877
+ -0.021567,
4878
+ 5999e-6,
4879
+ 0.017122,
4880
+ -9796e-6,
4881
+ 6654e-6,
4882
+ 291e-5,
4883
+ -0.017359,
4884
+ 9342e-6,
4885
+ -929e-6,
4886
+ -0.012544,
4887
+ -8234e-6,
4888
+ 0.016922,
4889
+ 0.018215,
4890
+ -758e-6,
4891
+ 5614e-6,
4892
+ 0.02137,
4893
+ 7615e-6,
4894
+ 0.010542,
4895
+ 635e-6,
4896
+ 0.023702,
4897
+ 0.020834,
4898
+ 0.013032,
4899
+ 0.011873,
4900
+ -2792e-6,
4901
+ 0.04144,
4902
+ -0.02252,
4903
+ 0.027235,
4904
+ -0.012274,
4905
+ -4002e-6,
4906
+ 0.022356,
4907
+ -0.020197,
4908
+ 0.270597,
4909
+ -3966e-6,
4910
+ -4187e-6,
4911
+ -0.014922,
4912
+ -0.013515,
4913
+ -0.01073,
4914
+ 5186e-6,
4915
+ -0.01013,
4916
+ -0.024027,
4917
+ -897e-6,
4918
+ 0.023069,
4919
+ 0.021741,
4920
+ 573e-6,
4921
+ -7467e-6,
4922
+ 823e-5,
4923
+ 0.037651,
4924
+ -0.017797,
4925
+ -0.014499,
4926
+ 2699e-6,
4927
+ -0.010982,
4928
+ -0.027587,
4929
+ 0.028585,
4930
+ -6217e-6,
4931
+ -9743e-6,
4932
+ 7437e-6,
4933
+ -0.023614,
4934
+ -0.023206,
4935
+ -0.029976,
4936
+ -903e-6,
4937
+ -5667e-6,
4938
+ -0.011557,
4939
+ 0.011335,
4940
+ -0.019149,
4941
+ 0.027405,
4942
+ -0.032284,
4943
+ -0.010479,
4944
+ -0.013504,
4945
+ 0.013693,
4946
+ -6407e-6,
4947
+ 8791e-6,
4948
+ -4461e-6,
4949
+ -2256e-6,
4950
+ -0.015897,
4951
+ 8837e-6,
4952
+ -1417e-6,
4953
+ 0.02557,
4954
+ -0.012118,
4955
+ -0.021653,
4956
+ 3051e-6,
4957
+ -4168e-6,
4958
+ -0.01862,
4959
+ -9461e-6,
4960
+ -0.01088,
4961
+ -982e-5,
4962
+ -0.011997,
4963
+ -0.015663,
4964
+ -0.021258,
4965
+ 5679e-6,
4966
+ 0.012566,
4967
+ 1521e-6,
4968
+ -8604e-6,
4969
+ 0.012445,
4970
+ 8421e-6,
4971
+ -0.033856,
4972
+ -0.019957,
4973
+ 146e-6,
4974
+ -185e-6,
4975
+ 781e-6,
4976
+ 0.018164,
4977
+ 4102e-6,
4978
+ -0.022985,
4979
+ -8788e-6,
4980
+ -4097e-6,
4981
+ 0.02217,
4982
+ -0.019959,
4983
+ -0.011669,
4984
+ 5313e-6,
4985
+ 381e-6,
4986
+ 3747e-6,
4987
+ -0.036681,
4988
+ -6177e-6,
4989
+ -0.014834,
4990
+ -0.031706,
4991
+ 0.010642,
4992
+ 1258e-6,
4993
+ 0.01296,
4994
+ 0.016435,
4995
+ -8722e-6,
4996
+ -0.010652,
4997
+ 9787e-6,
4998
+ -474e-5,
4999
+ 0.011086,
5000
+ -0.020615,
5001
+ 0.021049,
5002
+ -468e-5,
5003
+ 8645e-6,
5004
+ -9937e-6,
5005
+ 216e-5,
5006
+ 0.010686,
5007
+ -6216e-6,
5008
+ -0.011397,
5009
+ -5593e-6,
5010
+ 9687e-6,
5011
+ 0.017522,
5012
+ -6329e-6,
5013
+ 9519e-6,
5014
+ -0.022554,
5015
+ -0.02169,
5016
+ 3307e-6,
5017
+ -4782e-6,
5018
+ 1506e-6,
5019
+ 1607e-6,
5020
+ -3791e-6,
5021
+ -3601e-6,
5022
+ 0.030061,
5023
+ -468e-5,
5024
+ -0.186431,
5025
+ -1492e-6,
5026
+ 0.02242,
5027
+ -8878e-6,
5028
+ 0.01108
5029
+ ];
5030
+
3501
5031
  // src/profiles/registry.ts
3502
5032
  var SHARED_ENCODING = {
3503
5033
  pooling: "mean",
@@ -3558,7 +5088,94 @@ var BGE_SMALL_EN_V15_1 = {
3558
5088
  passageInstruction: "",
3559
5089
  ...SHARED_ENCODING
3560
5090
  };
5091
+ var GEMMA2_REPO = "onnx-community/embeddinggemma-2-ONNX";
5092
+ var GEMMA2_PROCESSOR_CONFIG = {
5093
+ path: "processor_config.json",
5094
+ digest: "sha256:168f6a08522f3ce5dea596d94d003af2fd691742d4f41fe1f9d8cce76bfbf69c"
5095
+ };
5096
+ var EMBEDDINGGEMMA_2_512_1 = {
5097
+ id: "embeddinggemma-2-512@1",
5098
+ model: {
5099
+ repo: GEMMA2_REPO,
5100
+ revision: "daa72c51243991dfcaf9f9137d2c573d8f7790c0",
5101
+ onnxFile: "onnx/model_quantized.onnx",
5102
+ onnxDigest: "sha256:d06edd601f851c633a2519304cbeb8dc6170d7ceb61b436625c17fb9b6e74953",
5103
+ files: [
5104
+ {
5105
+ path: "onnx/model_quantized.onnx_data",
5106
+ digest: "sha256:278a7ff1248c3618e4bd11a607fc54f7bdc7778854230f3956d3f86bd9db4f3b"
5107
+ },
5108
+ {
5109
+ path: "config.json",
5110
+ digest: "sha256:8d011bfe08b5e345bbe0b81e5c6fd02c381920b345b986047bc2a33ce7b90d1d"
5111
+ },
5112
+ {
5113
+ path: "tokenizer_config.json",
5114
+ digest: "sha256:17bd5d6e9364ca49a534e1502076593317c298d4a663623091ed45388f004874"
5115
+ }
5116
+ ]
5117
+ },
5118
+ tokenizer: {
5119
+ kind: "gemma-bpe",
5120
+ file: "tokenizer.json",
5121
+ digest: "sha256:4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4"
5122
+ },
5123
+ pooling: "mean",
5124
+ normalized: true,
5125
+ dimensions: 512,
5126
+ truncation: { method: "prefix", sourceDimensions: 768 },
5127
+ maxTokens: 512,
5128
+ queryInstruction: "task: search result | query: ",
5129
+ passageInstruction: "title: none | text: ",
5130
+ vectorEncoding: "bit+int8",
5131
+ distance: { stage1: "hamming", stage2: "cosine" },
5132
+ quantization: {
5133
+ int8: { method: "symmetric-linear", scale: 127 },
5134
+ binary: {
5135
+ method: "centered-sign",
5136
+ threshold: 0,
5137
+ packing: "lsb-first",
5138
+ center: [...EMBEDDINGGEMMA_2_512_CENTER]
5139
+ }
5140
+ },
5141
+ media: {
5142
+ image: {
5143
+ encoder: {
5144
+ onnxFile: "onnx/vision_encoder_quantized.onnx",
5145
+ onnxDigest: "sha256:bb0de2df53a2448a32dc7908a187c168c8afd514d4d6f674f7f46024875fa4e3",
5146
+ files: [
5147
+ {
5148
+ path: "onnx/vision_encoder_quantized.onnx_data",
5149
+ digest: "sha256:3dabd69c0a36e9a8771ad82030dde74daa5a0e02b7047a5d3f3382b1137bab89"
5150
+ },
5151
+ GEMMA2_PROCESSOR_CONFIG
5152
+ ]
5153
+ },
5154
+ tokenBudget: 280,
5155
+ resample: "bicubic",
5156
+ alpha: "composite-white"
5157
+ },
5158
+ video: { framesPerSecond: 1, maxFrames: 32, tokenBudgetPerFrame: 140 },
5159
+ audio: {
5160
+ encoder: {
5161
+ onnxFile: "onnx/audio_encoder_quantized.onnx",
5162
+ onnxDigest: "sha256:04a9a9094ba76fb169e4c69be45a4b621580188654d6be59eab860eadcd42d3c",
5163
+ files: [
5164
+ {
5165
+ path: "onnx/audio_encoder_quantized.onnx_data",
5166
+ digest: "sha256:aa6361d898e1f1d6303f8dd2b5fabf4fd3629e15fd709e9cb06ac5cb9416a030"
5167
+ },
5168
+ GEMMA2_PROCESSOR_CONFIG
5169
+ ]
5170
+ },
5171
+ sampleRate: 16e3,
5172
+ channels: 1,
5173
+ maxWindowMs: 3e4
5174
+ }
5175
+ }
5176
+ };
3561
5177
  var KNOWLEDGE_EMBEDDING_PROFILES = [
5178
+ EMBEDDINGGEMMA_2_512_1,
3562
5179
  MULTILINGUAL_E5_SMALL_2,
3563
5180
  MULTILINGUAL_E5_SMALL_1,
3564
5181
  BGE_SMALL_EN_V15_1
@@ -3579,10 +5196,13 @@ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
3579
5196
  var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
3580
5197
 
3581
5198
  // src/embedding/profile-embedder.ts
3582
- import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
5199
+ import {
5200
+ embeddingProfileArtifacts as embeddingProfileArtifacts3,
5201
+ profileUnitVector as profileUnitVector2
5202
+ } from "@bendyline/gezk";
3583
5203
 
3584
5204
  // src/embedding/artifact-verify.ts
3585
- import { createHash as createHash4 } from "crypto";
5205
+ import { createHash as createHash6 } from "crypto";
3586
5206
  import { createReadStream } from "fs";
3587
5207
  import { stat as stat3 } from "fs/promises";
3588
5208
  import { join as join5 } from "path";
@@ -3626,7 +5246,7 @@ async function fileDigest(path) {
3626
5246
  const memo = digestMemo.get(key);
3627
5247
  if (memo) return memo;
3628
5248
  const digest = await new Promise((resolve5, reject) => {
3629
- const hash = createHash4("sha256");
5249
+ const hash = createHash6("sha256");
3630
5250
  createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
3631
5251
  });
3632
5252
  if (digestMemo.size >= DIGEST_MEMO_MAX) {
@@ -3636,6 +5256,45 @@ async function fileDigest(path) {
3636
5256
  digestMemo.set(key, digest);
3637
5257
  return digest;
3638
5258
  }
5259
+ function mediaEncoderPins(profile, modalities) {
5260
+ const encoders = /* @__PURE__ */ new Set();
5261
+ const media = profile.media;
5262
+ for (const modality of modalities) {
5263
+ const encoder = modality === "audio" ? media?.audio?.encoder : media?.image?.encoder;
5264
+ if (encoder) encoders.add(encoder);
5265
+ }
5266
+ const pins = [];
5267
+ const seen = /* @__PURE__ */ new Set();
5268
+ for (const encoder of encoders) {
5269
+ const files = [
5270
+ { path: encoder.onnxFile, digest: encoder.onnxDigest },
5271
+ ...encoder.files ?? []
5272
+ ];
5273
+ for (const { path, digest } of files) {
5274
+ if (!digest || seen.has(path)) continue;
5275
+ seen.add(path);
5276
+ pins.push({ file: path, expected: digest });
5277
+ }
5278
+ }
5279
+ return pins;
5280
+ }
5281
+ async function verifyMediaArtifacts(profile, modalities, opts) {
5282
+ const pins = mediaEncoderPins(profile, modalities);
5283
+ if (pins.length === 0) return { status: "unpinned", checks: [] };
5284
+ const revision = opts.revision ?? profile.model.revision;
5285
+ const checks = [];
5286
+ for (const { file, expected } of pins) {
5287
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
5288
+ checks.push({ role: "media", file, path, expected, actual: await fileDigest(path) });
5289
+ }
5290
+ if (checks.some((c) => c.actual === null)) {
5291
+ throw new EmbedderArtifactError("missing", profile.id, checks);
5292
+ }
5293
+ if (checks.some((c) => c.actual !== c.expected)) {
5294
+ throw new EmbedderArtifactError("mismatch", profile.id, checks);
5295
+ }
5296
+ return { status: "verified", checks };
5297
+ }
3639
5298
  async function verifyProfileArtifacts(profile, opts) {
3640
5299
  const artifacts = embeddingProfileArtifacts(profile);
3641
5300
  const revision = opts.revision ?? profile.model.revision;
@@ -3650,6 +5309,9 @@ async function verifyProfileArtifacts(profile, opts) {
3650
5309
  expected: artifacts.tokenizerDigest
3651
5310
  });
3652
5311
  }
5312
+ for (const extra of artifacts.files) {
5313
+ if (extra.digest) pinned.push({ role: "file", file: extra.path, expected: extra.digest });
5314
+ }
3653
5315
  if (pinned.length === 0) return { status: "unpinned", checks: [] };
3654
5316
  const checks = [];
3655
5317
  for (const { role, file, expected } of pinned) {
@@ -3665,10 +5327,89 @@ async function verifyProfileArtifacts(profile, opts) {
3665
5327
  return { status: "verified", checks };
3666
5328
  }
3667
5329
 
5330
+ // src/embedding/pinned-files.ts
5331
+ import { createHash as createHash7 } from "crypto";
5332
+ import { createWriteStream as createWriteStream4 } from "fs";
5333
+ import { mkdir as mkdir4, rename, rm as rm2, stat as stat4 } from "fs/promises";
5334
+ import { dirname as dirname3 } from "path";
5335
+ import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
5336
+ function isExternalDataFile(path) {
5337
+ return /\.onnx_data(_\d+)?$/.test(path);
5338
+ }
5339
+ function pinnedLoadFiles(profile, modalities = []) {
5340
+ const artifacts = embeddingProfileArtifacts2(profile);
5341
+ const files = [
5342
+ {
5343
+ path: artifacts.onnxFile,
5344
+ ...artifacts.onnxDigest ? { digest: artifacts.onnxDigest } : {}
5345
+ },
5346
+ {
5347
+ path: artifacts.tokenizerFile,
5348
+ ...artifacts.tokenizerDigest ? { digest: artifacts.tokenizerDigest } : {}
5349
+ },
5350
+ ...artifacts.files.map((f) => ({ path: f.path, ...f.digest ? { digest: f.digest } : {} })),
5351
+ ...mediaEncoderPins(profile, modalities).map((p) => ({ path: p.file, digest: p.expected }))
5352
+ ];
5353
+ const seen = /* @__PURE__ */ new Set();
5354
+ return files.filter((f) => !seen.has(f.path) && seen.add(f.path));
5355
+ }
5356
+ async function isFile(path) {
5357
+ return stat4(path).then((s) => s.isFile()).catch(() => false);
5358
+ }
5359
+ async function missingPinnedFiles(profile, opts) {
5360
+ const revision = opts.revision ?? profile.model.revision;
5361
+ const missing = [];
5362
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5363
+ const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5364
+ if (!await isFile(path)) missing.push(file.path);
5365
+ }
5366
+ return missing;
5367
+ }
5368
+ async function prefetchExternalData(profile, opts) {
5369
+ const revision = opts.revision ?? profile.model.revision;
5370
+ const fetchImpl = opts.fetchImpl ?? fetch;
5371
+ const fetched = [];
5372
+ for (const file of pinnedLoadFiles(profile, opts.modalities)) {
5373
+ if (!isExternalDataFile(file.path)) continue;
5374
+ const dest = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
5375
+ if (await isFile(dest)) continue;
5376
+ const url = `https://huggingface.co/${profile.model.repo}/resolve/${revision}/${file.path}`;
5377
+ const res = await fetchImpl(url);
5378
+ if (!res.ok || !res.body) throw new Error(`${file.path}: download failed (HTTP ${res.status})`);
5379
+ await mkdir4(dirname3(dest), { recursive: true });
5380
+ const partial = `${dest}.partial`;
5381
+ const hash = createHash7("sha256");
5382
+ try {
5383
+ const out = createWriteStream4(partial);
5384
+ const reader = res.body.getReader();
5385
+ for (; ; ) {
5386
+ const { done, value } = await reader.read();
5387
+ if (done) break;
5388
+ hash.update(value);
5389
+ if (!out.write(value)) await new Promise((r) => out.once("drain", () => r()));
5390
+ }
5391
+ await new Promise((resolve5, reject) => {
5392
+ out.on("error", reject);
5393
+ out.end(() => resolve5());
5394
+ });
5395
+ const digest = `sha256:${hash.digest("hex")}`;
5396
+ if (file.digest && digest !== file.digest) {
5397
+ throw new Error(`${file.path}: downloaded ${digest}, profile pins ${file.digest}`);
5398
+ }
5399
+ await rename(partial, dest);
5400
+ } catch (err) {
5401
+ await rm2(partial, { force: true });
5402
+ throw err;
5403
+ }
5404
+ fetched.push(file.path);
5405
+ }
5406
+ return fetched;
5407
+ }
5408
+
3668
5409
  // src/embedding/profile-embedder.ts
3669
5410
  var MAX_BATCH = 8;
3670
5411
  var MAX_CHARS = 8e3;
3671
- var TRANSFORMERS_PEER_RANGE = "^3.8.1";
5412
+ var TRANSFORMERS_PEER_RANGE = "^4.3.1";
3672
5413
  var EmbedderUnavailableError = class extends Error {
3673
5414
  isActionable = true;
3674
5415
  /** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
@@ -3712,7 +5453,7 @@ var POOLING_BY_PROFILE = {
3712
5453
  cls: "cls"
3713
5454
  };
3714
5455
  function resolveTransformersModelOptions(profile) {
3715
- const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
5456
+ const { onnxFile, tokenizerFile } = embeddingProfileArtifacts3(profile);
3716
5457
  if (tokenizerFile !== "tokenizer.json") {
3717
5458
  throw new EmbedderUnavailableError(
3718
5459
  `profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
@@ -3737,6 +5478,41 @@ function transformersGraphOptions(onnxFile) {
3737
5478
  if (modelFileName === "") return null;
3738
5479
  return { dtype, subfolder, model_file_name: modelFileName };
3739
5480
  }
5481
+ function isMultimodalEncoderConfig(config) {
5482
+ return Boolean(config.vision_config || config.audio_config);
5483
+ }
5484
+ function pinnedExternalDataChunks(profile) {
5485
+ const { onnxFile, files } = embeddingProfileArtifacts3(profile);
5486
+ const prefix = `${onnxFile}_data`;
5487
+ return files.filter((f) => f.path === prefix || /^_\d+$/.test(f.path.slice(prefix.length))).length;
5488
+ }
5489
+ function encoderRows(outputs, attentionMask) {
5490
+ const pooled = outputs.sentence_embedding;
5491
+ if (pooled) {
5492
+ const [batch2 = 0, width2 = 0] = pooled.dims;
5493
+ return Array.from(
5494
+ { length: batch2 },
5495
+ (_, b) => Array.from({ length: width2 }, (_2, d) => pooled.data[b * width2 + d])
5496
+ );
5497
+ }
5498
+ const hidden = outputs.last_hidden_state;
5499
+ if (!hidden || !attentionMask) {
5500
+ throw new Error("the encoder returned neither sentence_embedding nor last_hidden_state");
5501
+ }
5502
+ const [batch = 0, tokens = 0, width = 0] = hidden.dims;
5503
+ return Array.from({ length: batch }, (_, b) => {
5504
+ const sum = new Array(width).fill(0);
5505
+ let count = 0;
5506
+ for (let t = 0; t < tokens; t++) {
5507
+ if (!Number(attentionMask.data[b * tokens + t])) continue;
5508
+ count++;
5509
+ const base = (b * tokens + t) * width;
5510
+ for (let d = 0; d < width; d++)
5511
+ sum[d] = sum[d] + hidden.data[base + d];
5512
+ }
5513
+ return sum.map((v) => v / Math.max(1, count));
5514
+ });
5515
+ }
3740
5516
  async function loadTransformers() {
3741
5517
  try {
3742
5518
  const specifier = "@huggingface/transformers";
@@ -3752,6 +5528,20 @@ async function loadTransformers() {
3752
5528
  throw err;
3753
5529
  }
3754
5530
  }
5531
+ async function preparePinnedLoad(profile, opts) {
5532
+ const files = pinnedLoadFiles(profile, opts.modalities);
5533
+ if (!files.some((f) => isExternalDataFile(f.path))) return;
5534
+ if (opts.localFilesOnly) {
5535
+ const missing = await missingPinnedFiles(profile, opts);
5536
+ if (missing.length > 0) {
5537
+ throw new EmbedderUnavailableError(
5538
+ `the model files for profile ${profile.id} are not all installed (missing: ${missing.join(", ")})`
5539
+ );
5540
+ }
5541
+ return;
5542
+ }
5543
+ await prefetchExternalData(profile, opts);
5544
+ }
3755
5545
  async function createProfileEmbedder(profile, opts = {}) {
3756
5546
  const pooling = POOLING_BY_PROFILE[profile.pooling];
3757
5547
  if (!pooling) {
@@ -3768,14 +5558,47 @@ async function createProfileEmbedder(profile, opts = {}) {
3768
5558
  transformers.env.allowRemoteModels = true;
3769
5559
  }
3770
5560
  const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
3771
- const [pipe, tokenizer] = await Promise.all([
3772
- transformers.pipeline("feature-extraction", profile.model.repo, {
5561
+ const localOnly = opts.localFilesOnly ? { local_files_only: true } : {};
5562
+ if (!opts.transformers) {
5563
+ await preparePinnedLoad(profile, {
5564
+ cacheDir: cacheDir ?? transformers.env.cacheDir ?? "",
5565
+ ...opts.localFilesOnly ? { localFilesOnly: true } : {}
5566
+ });
5567
+ }
5568
+ const sessionOptions = opts.sessionOptions ? { session_options: opts.sessionOptions } : {};
5569
+ const config = transformers.AutoConfig ? await transformers.AutoConfig.from_pretrained(profile.model.repo, {
5570
+ revision: modelOptions.revision,
5571
+ ...localOnly,
5572
+ ...progress
5573
+ }) : null;
5574
+ const multimodal = config !== null && isMultimodalEncoderConfig(config);
5575
+ if (multimodal && !transformers.AutoModel) {
5576
+ throw new EmbedderUnavailableError(
5577
+ `profile ${profile.id} is a multimodal encoder; this runtime cannot load its text session`
5578
+ );
5579
+ }
5580
+ const externalData = pinnedExternalDataChunks(profile);
5581
+ const [runner, tokenizer] = await Promise.all([
5582
+ multimodal ? transformers.AutoModel.from_pretrained(
5583
+ profile.model.repo,
5584
+ {
5585
+ ...modelOptions,
5586
+ config: { ...config, vision_config: null, audio_config: null },
5587
+ ...externalData > 0 ? { use_external_data_format: externalData } : {},
5588
+ ...localOnly,
5589
+ ...sessionOptions,
5590
+ ...progress
5591
+ }
5592
+ ) : transformers.pipeline("feature-extraction", profile.model.repo, {
3773
5593
  ...modelOptions,
3774
- ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
5594
+ ...externalData > 0 ? { use_external_data_format: externalData } : {},
5595
+ ...localOnly,
5596
+ ...sessionOptions,
3775
5597
  ...progress
3776
5598
  }),
3777
5599
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3778
5600
  revision: modelOptions.revision,
5601
+ ...localOnly,
3779
5602
  ...progress
3780
5603
  })
3781
5604
  ]);
@@ -3785,17 +5608,23 @@ async function createProfileEmbedder(profile, opts = {}) {
3785
5608
  cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
3786
5609
  });
3787
5610
  } catch (err) {
3788
- await pipe.dispose?.().catch(() => {
5611
+ await runner.dispose?.().catch(() => {
3789
5612
  });
3790
5613
  throw err;
3791
5614
  }
3792
5615
  const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
5616
+ const runBatch = multimodal ? async (slice) => {
5617
+ const inputs = tokenizer(slice, {
5618
+ padding: true,
5619
+ truncation: true,
5620
+ max_length: profile.maxTokens
5621
+ });
5622
+ return encoderRows(await runner(inputs), inputs.attention_mask);
5623
+ } : async (slice) => (await runner(slice, { pooling, normalize: true })).tolist();
3793
5624
  const embed = async (texts) => {
3794
5625
  const out = [];
3795
5626
  for (let i = 0; i < texts.length; i += MAX_BATCH) {
3796
- const slice = texts.slice(i, i + MAX_BATCH).map(cap);
3797
- const result = await pipe(slice, { pooling, normalize: true });
3798
- out.push(...result.tolist());
5627
+ out.push(...await runBatch(texts.slice(i, i + MAX_BATCH).map(cap)));
3799
5628
  }
3800
5629
  return out;
3801
5630
  };
@@ -3805,11 +5634,12 @@ async function createProfileEmbedder(profile, opts = {}) {
3805
5634
  embed,
3806
5635
  embedQuery: async (text) => {
3807
5636
  const [vector] = await embed([`${profile.queryInstruction}${text}`]);
3808
- return Float32Array.from(vector ?? []);
5637
+ if (!vector) throw new Error(`profile ${profile.id} returned no query vector`);
5638
+ return profileUnitVector2(profile, vector);
3809
5639
  },
3810
5640
  countTokens: (text) => tokenizer.encode(text).length,
3811
5641
  dispose: async () => {
3812
- await pipe.dispose?.();
5642
+ await runner.dispose?.();
3813
5643
  }
3814
5644
  };
3815
5645
  }
@@ -3817,11 +5647,12 @@ async function createProfileEmbedder(profile, opts = {}) {
3817
5647
  // src/markdown-adapter/load.ts
3818
5648
  import { existsSync } from "fs";
3819
5649
  import { readFile as readFile3, readdir } from "fs/promises";
3820
- import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
5650
+ import { basename, dirname as dirname4, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
3821
5651
  import {
3822
5652
  CatalogDocumentSchema,
3823
5653
  KnowledgeDocumentIdSchema,
3824
- KnowledgeOrdinalSchema,
5654
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
5655
+ KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
3825
5656
  MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
3826
5657
  assetExtension as assetExtension3,
3827
5658
  formatKnowledgeUri,
@@ -4494,9 +6325,9 @@ function parseYaml(text, opts) {
4494
6325
  var OPEN = /^---[ \t]*\r?\n/;
4495
6326
  var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
4496
6327
  function parseMarkdownFrontMatter(source) {
4497
- const open = OPEN.exec(source);
4498
- if (!open) return { data: {}, body: source };
4499
- const lines = source.slice(open[0].length).split(/\r?\n/);
6328
+ const open2 = OPEN.exec(source);
6329
+ if (!open2) return { data: {}, body: source };
6330
+ const lines = source.slice(open2[0].length).split(/\r?\n/);
4500
6331
  const closeAt = lines.findIndex((line) => CLOSE.test(line));
4501
6332
  if (closeAt === -1) throw new Error("front matter is not closed with ---");
4502
6333
  const yamlText = lines.slice(0, closeAt).join("\n");
@@ -4517,7 +6348,15 @@ var HUGO_SECTION_PAGE = "_index";
4517
6348
  var DOCFX_JSON = "docfx.json";
4518
6349
  var WARNING_EXAMPLES2 = 20;
4519
6350
  var SECTION_PAGE_ORDINAL = -2147483648;
4520
- var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
6351
+ var RESERVED_KEYS = /* @__PURE__ */ new Set([
6352
+ "title",
6353
+ "summary",
6354
+ "aliases",
6355
+ "id",
6356
+ "order",
6357
+ "subcategory",
6358
+ "locations"
6359
+ ]);
4521
6360
  function readFrontMatter(raw, file, conventions) {
4522
6361
  const { hugo } = conventions;
4523
6362
  let data;
@@ -4536,6 +6375,12 @@ function readFrontMatter(raw, file, conventions) {
4536
6375
  throw new Error(`${file}: front matter '${key}' must be a string`);
4537
6376
  return value.trim() || void 0;
4538
6377
  };
6378
+ if (data.locations !== void 0) {
6379
+ const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
6380
+ if (!locations.success)
6381
+ throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
6382
+ out.locations = locations.data;
6383
+ }
4539
6384
  out.title = str("title");
4540
6385
  out.summary = str("summary");
4541
6386
  out.id = str("id");
@@ -4548,7 +6393,7 @@ function readFrontMatter(raw, file, conventions) {
4548
6393
  throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
4549
6394
  }
4550
6395
  if (data.order !== void 0 && data.order !== null) {
4551
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
6396
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4552
6397
  if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
4553
6398
  out.order = order.data;
4554
6399
  }
@@ -4569,7 +6414,7 @@ function readFrontMatter(raw, file, conventions) {
4569
6414
  subcategory.title = record.title.trim();
4570
6415
  }
4571
6416
  if (record.order !== void 0 && record.order !== null) {
4572
- const order = KnowledgeOrdinalSchema.safeParse(record.order);
6417
+ const order = KnowledgeOrdinalSchema2.safeParse(record.order);
4573
6418
  if (!order.success)
4574
6419
  throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
4575
6420
  subcategory.order = order.data;
@@ -4580,7 +6425,7 @@ function readFrontMatter(raw, file, conventions) {
4580
6425
  if (data.draft === true) out.unpublished = "draft";
4581
6426
  else if (data.headless === true) out.unpublished = "headless";
4582
6427
  if (data.weight !== void 0 && data.weight !== null) {
4583
- const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
6428
+ const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
4584
6429
  if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
4585
6430
  if (weight.data > 0) out.weight = weight.data;
4586
6431
  }
@@ -4653,7 +6498,7 @@ async function readTopicSidecar(dir) {
4653
6498
  out.description = data.description.trim();
4654
6499
  }
4655
6500
  if (data.order !== void 0 && data.order !== null) {
4656
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
6501
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4657
6502
  if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
4658
6503
  out.order = order.data;
4659
6504
  }
@@ -4681,7 +6526,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
4681
6526
  maxAliasCount: 100
4682
6527
  });
4683
6528
  const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
4684
- return resolve4(dirname3(mkdocsPath), docsDir);
6529
+ return resolve4(dirname4(mkdocsPath), docsDir);
4685
6530
  }
4686
6531
  async function detectTableOfContents(rootDir, projectDir = rootDir) {
4687
6532
  const summary = join6(rootDir, "SUMMARY.md");
@@ -4689,7 +6534,7 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
4689
6534
  const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
4690
6535
  if (jupyter) return { format: "jupyter-book", path: jupyter };
4691
6536
  const mkdocs = firstExisting(
4692
- [rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
6537
+ [rootDir, projectDir, dirname4(rootDir)].flatMap((dir) => [
4693
6538
  join6(dir, "mkdocs.yml"),
4694
6539
  join6(dir, "mkdocs.yaml")
4695
6540
  ])
@@ -4729,7 +6574,7 @@ async function loadOutline(rootDir, toc, files) {
4729
6574
  }
4730
6575
  if (toc.format === "mkdocs") {
4731
6576
  const path2 = explicit ?? firstExisting(
4732
- [rootDir, dirname3(rootDir)].flatMap((dir) => [
6577
+ [rootDir, dirname4(rootDir)].flatMap((dir) => [
4733
6578
  join6(dir, "mkdocs.yml"),
4734
6579
  join6(dir, "mkdocs.yaml")
4735
6580
  ])
@@ -4752,7 +6597,7 @@ async function loadOutline(rootDir, toc, files) {
4752
6597
  }
4753
6598
  const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
4754
6599
  if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
4755
- const tocDir = insideRoot(rootDir, dirname3(path));
6600
+ const tocDir = insideRoot(rootDir, dirname4(path));
4756
6601
  if (tocDir === null)
4757
6602
  throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
4758
6603
  const doc = parseYaml(await readOutlineText(path), {
@@ -4885,6 +6730,7 @@ function rewriteLinks(ctx, markdown) {
4885
6730
  return line.replace(
4886
6731
  INLINE_LINK,
4887
6732
  (whole, bang, text, target, title) => {
6733
+ if (bang === "!" && ctx.skipImages) return text;
4888
6734
  const rewritten = rewriteTarget(ctx, bang === "!", target);
4889
6735
  return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
4890
6736
  }
@@ -4955,22 +6801,16 @@ async function loadMarkdownCatalog(rootDir, opts) {
4955
6801
  };
4956
6802
  const placements = /* @__PURE__ */ new Map();
4957
6803
  if (outline) {
4958
- const duplicates = [];
4959
6804
  const walk = (topic, chain) => {
4960
6805
  for (const entry of topic.entries) {
4961
6806
  const key = entry.file.normalize("NFC");
4962
- if (placements.has(key)) duplicates.push(entry.file);
4963
- else placements.set(key, { chain, entry });
6807
+ const existing = placements.get(key) ?? [];
6808
+ existing.push({ chain, entry });
6809
+ placements.set(key, existing);
4964
6810
  }
4965
6811
  for (const child of topic.children) walk(child, [...chain, child]);
4966
6812
  };
4967
6813
  walk(outline.root, []);
4968
- warnCapped(
4969
- warn,
4970
- duplicates,
4971
- (file) => `${file}: listed more than once in the table of contents; the first place wins`,
4972
- (more) => `${more} more files are listed more than once in the table of contents`
4973
- );
4974
6814
  }
4975
6815
  const outlineTopicIds = /* @__PURE__ */ new Map();
4976
6816
  let outlineTooDeep = false;
@@ -5045,7 +6885,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
5045
6885
  let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
5046
6886
  let ordinal = fm.order;
5047
6887
  let topicPath;
5048
- const placement = placements.get(rel.normalize("NFC"));
6888
+ const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
6889
+ const placement = documentPlacements[0];
5049
6890
  if (placement) {
5050
6891
  topicPath = ensureOutlineChain(placement.chain);
5051
6892
  if (placement.entry.title) title = placement.entry.title;
@@ -5075,6 +6916,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
5075
6916
  const parentId = topicPath[topicPath.length - 1];
5076
6917
  topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
5077
6918
  }
6919
+ const tocReferences = [];
6920
+ const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
6921
+ for (const extra of documentPlacements.slice(1)) {
6922
+ let path = ensureOutlineChain(extra.chain);
6923
+ if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
6924
+ const leaf = path.at(-1);
6925
+ if (placedLeaves.has(leaf)) continue;
6926
+ placedLeaves.add(leaf);
6927
+ const order = extra.entry.order ?? fm.order;
6928
+ tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
6929
+ }
5078
6930
  const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
5079
6931
  loaded.push({
5080
6932
  rel,
@@ -5086,6 +6938,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
5086
6938
  ...summary ? { summary } : {},
5087
6939
  language: opts.language,
5088
6940
  topicPath,
6941
+ ...tocReferences.length ? { tocReferences } : {},
6942
+ ...fm.locations ? { locations: fm.locations } : {},
5089
6943
  markdown: fm.body,
5090
6944
  ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
5091
6945
  ...ordinal !== void 0 ? { ordinal } : {},
@@ -5115,7 +6969,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
5115
6969
  );
5116
6970
  warnCapped(
5117
6971
  warn,
5118
- [...placements].filter(([file]) => !seenRel.has(file)).map(([, p]) => p.entry.file),
6972
+ [...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
5119
6973
  (file) => `${file}: named by the table of contents but not found among the Markdown files`,
5120
6974
  (more) => `${more} more files named by the table of contents are not among the Markdown files`
5121
6975
  );
@@ -5152,6 +7006,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
5152
7006
  // A docfx docset draws on dependent repositories (`~/reusable-content`)
5153
7007
  // that a checkout never holds, so a missing image is expected there.
5154
7008
  missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
7009
+ skipImages: opts.skipImages ?? false,
5155
7010
  tildeIsRoot: docfx,
5156
7011
  warn: (message) => linkWarnings.push(message)
5157
7012
  };
@@ -5288,7 +7143,7 @@ function newerRegistryEntries(registry, installed) {
5288
7143
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
5289
7144
 
5290
7145
  // src/signatures/jcs.ts
5291
- import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
7146
+ import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
5292
7147
  export {
5293
7148
  BGE_SMALL_EN_V15_1,
5294
7149
  BODY_CODEC_MIN_BYTES,
@@ -5299,7 +7154,9 @@ export {
5299
7154
  CENTROID_SAMPLE_MAX,
5300
7155
  CatalogHandle,
5301
7156
  CatalogOpenError,
7157
+ CatalogQueryError,
5302
7158
  DuckdbError,
7159
+ EMBEDDINGGEMMA_2_512_1,
5303
7160
  EmbedderArtifactError,
5304
7161
  EmbedderUnavailableError,
5305
7162
  GEZEL_BGE_SMALL_EN_V15_1,
@@ -5323,11 +7180,14 @@ export {
5323
7180
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
5324
7181
  MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
5325
7182
  MAX_KNOWLEDGE_TOPIC_DEPTH,
7183
+ MAX_MEDIA_ROWS_PER_DOCUMENT,
7184
+ MEDIA_ROW_TEXT_MAX_TOKENS,
5326
7185
  MIMETYPE_PATH,
5327
7186
  MULTILINGUAL_E5_SMALL_1,
5328
7187
  MULTILINGUAL_E5_SMALL_2,
5329
7188
  PARQUET_EXPORT_VERSION,
5330
7189
  PARQUET_REPORT_PATH,
7190
+ QUERY_STOP_WORDS,
5331
7191
  README_PATH,
5332
7192
  RERANK_CANDIDATES_MAX,
5333
7193
  RERANK_CANDIDATES_MIN,
@@ -5337,20 +7197,25 @@ export {
5337
7197
  ROUTE_SHARDS_EXPLICIT,
5338
7198
  ROUTE_SHARDS_PROACTIVE,
5339
7199
  SHARD_DDL,
7200
+ SHARD_DDL_V4,
7201
+ SHARD_DDL_V5,
5340
7202
  SHARD_MAX_CHUNKS,
5341
7203
  SHARD_TARGET_CHUNKS,
5342
7204
  SOURCE_NOTICES_PATH,
7205
+ TITLE_ROUTE_SHARDS,
5343
7206
  TRANSFORMERS_PEER_RANGE,
5344
7207
  ZIP_FIXED_MTIME,
5345
7208
  assertDuckdbCli,
5346
7209
  assetReferences,
5347
7210
  assignOutlineOrders,
5348
7211
  asymmetricTopK,
5349
- canonicalizeJson2 as canonicalizeJson,
7212
+ canonicalizeJson3 as canonicalizeJson,
7213
+ catalogFormatVersion,
5350
7214
  centerVector,
5351
7215
  chunkContentHash,
5352
7216
  chunkMarkdownProfile,
5353
7217
  chunkUid,
7218
+ collectMediaRows,
5354
7219
  compareCatalogVersions,
5355
7220
  compileKnowledgeCatalog,
5356
7221
  createProfileEmbedder,
@@ -5362,9 +7227,13 @@ export {
5362
7227
  fetchKnowledgeRegistry,
5363
7228
  findDuckdbBinary,
5364
7229
  findRegistryEntry,
7230
+ formatAtLeast,
5365
7231
  generateKnowledgeSigningKeyPair,
5366
7232
  hammingTopK,
7233
+ humanizeAssetName,
5367
7234
  inspectGezkArchive,
7235
+ isExternalDataFile,
7236
+ isMultimodalEncoderConfig,
5368
7237
  isSupportedFormatVersion,
5369
7238
  isSupportedIndexSchemaVersion,
5370
7239
  knowledgeEmbeddingProfile,
@@ -5372,6 +7241,9 @@ export {
5372
7241
  l2Normalize,
5373
7242
  loadMarkdownCatalog,
5374
7243
  manifestSigningPayload,
7244
+ mediaEncoderPins,
7245
+ mediaReferences,
7246
+ missingPinnedFiles,
5375
7247
  newOutlineTopic,
5376
7248
  newerRegistryEntries,
5377
7249
  openCatalogDatabase,
@@ -5379,6 +7251,11 @@ export {
5379
7251
  parseGitbookSummary,
5380
7252
  parseJupyterBookToc,
5381
7253
  parseMkdocsNav,
7254
+ pinnedExternalDataChunks,
7255
+ pinnedLoadFiles,
7256
+ prefetchExternalData,
7257
+ preparePinnedLoad,
7258
+ profileUnitVector,
5382
7259
  quantizeBinary,
5383
7260
  quantizeBinaryForProfile,
5384
7261
  quantizeInt8,
@@ -5389,6 +7266,7 @@ export {
5389
7266
  resolveOutlinePath,
5390
7267
  resolveTransformersModelOptions,
5391
7268
  runDuckdbScript,
7269
+ shardDdlFor,
5392
7270
  signManifest,
5393
7271
  signRegistryIndex,
5394
7272
  titleFromFolderName,
@@ -5396,6 +7274,7 @@ export {
5396
7274
  transformersGraphOptions,
5397
7275
  validateExtractedCatalog,
5398
7276
  verifyManifestSignature,
7277
+ verifyMediaArtifacts,
5399
7278
  verifyProfileArtifacts,
5400
7279
  verifyRegistryIndex,
5401
7280
  writeGezkArchive