@bendyline/gezel-knowledge 1.2.2 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.ts +161 -40
  2. package/dist/index.js +954 -141
  3. package/package.json +2 -2
package/dist/index.js CHANGED
@@ -180,12 +180,11 @@ function splitOversizeLine(line, opts) {
180
180
  }
181
181
 
182
182
  // src/compiler/compile.ts
183
- import { createHash } from "crypto";
183
+ import { createHash as createHash2 } from "crypto";
184
184
  import {
185
185
  closeSync,
186
186
  mkdirSync,
187
187
  openSync,
188
- readFileSync,
189
188
  readSync,
190
189
  rmSync,
191
190
  statSync,
@@ -196,16 +195,13 @@ import { join } from "path";
196
195
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
197
196
  import {
198
197
  GEZK_MANIFEST_KIND,
199
- KnowledgeAssetPathSchema,
200
198
  KnowledgeCatalogManifestSchema,
201
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
202
- MAX_KNOWLEDGE_ASSET_BYTES,
203
- MAX_KNOWLEDGE_ASSET_COUNT,
204
- assetExtension,
205
- assetKindForExtension,
199
+ KnowledgeLocationsSchema,
200
+ KnowledgeOrdinalSchema,
201
+ KnowledgeTocReferenceSchema,
206
202
  canonicalizeJson,
207
- sniffAssetType,
208
- svgInertnessProblem
203
+ normalizeLongitude,
204
+ spatialManifest
209
205
  } from "@bendyline/gezk";
210
206
 
211
207
  // src/archive/write.ts
@@ -251,28 +247,306 @@ var nodeRequire2 = createRequire2(import.meta.url);
251
247
  var sqlite = nodeRequire2("node:sqlite");
252
248
  var DatabaseSync = sqlite.DatabaseSync;
253
249
 
250
+ // src/query-stopwords.ts
251
+ var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
252
+ // articles + determiners
253
+ "a",
254
+ "an",
255
+ "the",
256
+ "this",
257
+ "that",
258
+ "these",
259
+ "those",
260
+ "some",
261
+ "any",
262
+ "all",
263
+ "each",
264
+ "both",
265
+ "every",
266
+ "other",
267
+ "another",
268
+ "same",
269
+ "such",
270
+ // pronouns
271
+ "i",
272
+ "me",
273
+ "my",
274
+ "we",
275
+ "us",
276
+ "our",
277
+ "you",
278
+ "your",
279
+ "it",
280
+ "its",
281
+ "he",
282
+ "him",
283
+ "his",
284
+ "she",
285
+ "her",
286
+ "they",
287
+ "them",
288
+ "their",
289
+ // be / have / do
290
+ "am",
291
+ "is",
292
+ "are",
293
+ "was",
294
+ "were",
295
+ "be",
296
+ "been",
297
+ "being",
298
+ "have",
299
+ "has",
300
+ "had",
301
+ "do",
302
+ "does",
303
+ "did",
304
+ "done",
305
+ // modals
306
+ "can",
307
+ "could",
308
+ "will",
309
+ "would",
310
+ "shall",
311
+ "should",
312
+ "may",
313
+ "might",
314
+ "must",
315
+ // question words
316
+ "how",
317
+ "what",
318
+ "when",
319
+ "where",
320
+ "which",
321
+ "who",
322
+ "whom",
323
+ "why",
324
+ // prepositions + conjunctions
325
+ "about",
326
+ "and",
327
+ "as",
328
+ "at",
329
+ "but",
330
+ "by",
331
+ "for",
332
+ "from",
333
+ "if",
334
+ "in",
335
+ "into",
336
+ "of",
337
+ "on",
338
+ "or",
339
+ "so",
340
+ "than",
341
+ "then",
342
+ "there",
343
+ "to",
344
+ "with",
345
+ // request scaffolding + degree words
346
+ "please",
347
+ "help",
348
+ "let",
349
+ "tell",
350
+ "want",
351
+ "need",
352
+ "create",
353
+ "make",
354
+ "give",
355
+ "get",
356
+ "got",
357
+ "use",
358
+ "using",
359
+ "just",
360
+ "like",
361
+ "also",
362
+ "very",
363
+ "only",
364
+ "new",
365
+ "more",
366
+ "most",
367
+ "much",
368
+ "many",
369
+ "few",
370
+ "own",
371
+ // conversational scaffolding
372
+ "hey",
373
+ "hi",
374
+ "hiya",
375
+ "hello",
376
+ "thanks",
377
+ "thank",
378
+ "okay",
379
+ "ok",
380
+ "yep",
381
+ "yeah",
382
+ "yes",
383
+ "nope",
384
+ "good",
385
+ "great",
386
+ "fine",
387
+ "nice",
388
+ "cool",
389
+ "ready",
390
+ "going",
391
+ "doing",
392
+ "sounds",
393
+ "morning",
394
+ "afternoon",
395
+ "evening",
396
+ "night",
397
+ // Apostrophes are token boundaries, so contractions otherwise leave these
398
+ // fragments looking like rare, highly distinctive query terms.
399
+ "s",
400
+ "t",
401
+ "re",
402
+ "ve",
403
+ "ll",
404
+ "d",
405
+ "m",
406
+ "isn",
407
+ "aren",
408
+ "wasn",
409
+ "weren",
410
+ "don",
411
+ "doesn",
412
+ "didn",
413
+ "hasn",
414
+ "haven",
415
+ "hadn",
416
+ "won",
417
+ "wouldn",
418
+ "shouldn",
419
+ "couldn"
420
+ ]);
421
+
254
422
  // src/reader/fts-query.ts
255
- function sanitizeFtsQuery(query) {
423
+ function contentTokens(query) {
256
424
  const tokens = [
257
425
  ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
258
426
  ];
427
+ const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
428
+ return content.length > 0 ? content : tokens;
429
+ }
430
+ function contentFtsQuery(query) {
431
+ const tokens = contentTokens(query);
432
+ return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
433
+ }
434
+ function selectiveFtsQuery(query, rows, rowsMatching) {
435
+ const tokens = contentTokens(query);
259
436
  if (tokens.length === 0) return null;
260
- return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
437
+ const counted = tokens.map((token) => {
438
+ const phrase = quoteToken(token);
439
+ return { phrase, hits: rowsMatching(phrase) };
440
+ });
441
+ const kept = counted.filter((c) => c.hits * 2 <= rows);
442
+ const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
443
+ return chosen.map((c) => c.phrase).join(" OR ");
261
444
  }
262
445
  var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
263
446
  var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
264
- function documentFtsTopIds(db, match, limit, exactTitle) {
265
- const title = exactTitle?.normalize("NFKC").trim() ?? "";
266
- return db.prepare(
447
+ function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
448
+ const match = contentFtsQuery(query);
449
+ if (!match || allowedDocumentIds?.size === 0) return [];
450
+ const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
451
+ const title = query.normalize("NFKC").trim();
452
+ const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
453
+ const rest = db.prepare(
267
454
  `SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
268
- ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
269
- ).all(match, title, limit).map((row) => row.document_id);
455
+ ${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
456
+ ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
457
+ ).all(
458
+ ...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
459
+ );
460
+ for (const row of rest) {
461
+ if (ids.length >= limit) break;
462
+ if (!ids.includes(row.document_id)) ids.push(row.document_id);
463
+ }
464
+ return ids;
465
+ }
466
+ var NAME_STOPWORDS = new Set(
467
+ "a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
468
+ );
469
+ var NAMED_TITLE_CANDIDATES = 256;
470
+ var TITLE_DF_CACHE_MAX = 5e4;
471
+ var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
472
+ var SAME_WORDS_SCORE = Number.MAX_VALUE;
473
+ function nameWords(text) {
474
+ return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
475
+ }
476
+ function quoteToken(token) {
477
+ return `"${token.replace(/"/g, '""')}"`;
478
+ }
479
+ var titleStatsByDb = /* @__PURE__ */ new WeakMap();
480
+ function titleStats(db) {
481
+ let stats = titleStatsByDb.get(db);
482
+ if (!stats) {
483
+ const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
484
+ stats = {
485
+ documents: Math.max(1, Number(row.c)),
486
+ df: /* @__PURE__ */ new Map(),
487
+ dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
488
+ candidates: db.prepare(
489
+ `SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
490
+ ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
491
+ )
492
+ };
493
+ titleStatsByDb.set(db, stats);
494
+ }
495
+ return stats;
496
+ }
497
+ function titleDf(stats, word) {
498
+ let df = stats.df.get(word);
499
+ if (df === void 0) {
500
+ if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
501
+ const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
502
+ df = Number(row.c);
503
+ stats.df.set(word, df);
504
+ }
505
+ return df;
506
+ }
507
+ function namedTitleMatches(db, query, limit, allowedDocumentIds) {
508
+ if (allowedDocumentIds?.size === 0) return [];
509
+ const raw = query.normalize("NFKC").trim();
510
+ const words = [...new Set(nameWords(raw))].slice(0, 16);
511
+ const content = words.filter((w) => !NAME_STOPWORDS.has(w));
512
+ if (content.length === 0 || limit <= 0) return [];
513
+ const asked = new Set(words);
514
+ const capitalized = new Set(
515
+ (raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
516
+ );
517
+ const queryWords = nameWords(raw).join(" ");
518
+ const stats = titleStats(db);
519
+ const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
520
+ const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
521
+ AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
522
+ const named = [];
523
+ for (const row of rows) {
524
+ const titleWords = nameWords(row.title);
525
+ if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
526
+ const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
527
+ if (distinct.length === 0) continue;
528
+ let score;
529
+ if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
530
+ score = EXACT_TITLE_SCORE;
531
+ } else if (titleWords.join(" ") === queryWords) {
532
+ score = SAME_WORDS_SCORE;
533
+ } else {
534
+ score = 0;
535
+ for (const w of distinct) {
536
+ score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
537
+ }
538
+ }
539
+ named.push({ documentId: row.document_id, score, order: named.length });
540
+ }
541
+ named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
542
+ return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
543
+ }
544
+ function compareScores(a, b) {
545
+ return a === b ? 0 : a > b ? 1 : -1;
270
546
  }
271
547
  var SMOKE_QUERY_TOP_N = 10;
272
548
  function documentSmokeQueryMisses(db, smoke) {
273
- const match = sanitizeFtsQuery(smoke.query);
274
- if (!match) return [...smoke.expectedDocumentIds];
275
- const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
549
+ const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
276
550
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
277
551
  }
278
552
 
@@ -284,6 +558,92 @@ var KNOWLEDGE_TOOLCHAIN = {
284
558
  version: nodeRequire3("../package.json").version
285
559
  };
286
560
 
561
+ // src/compiler/assets.ts
562
+ import { createHash } from "crypto";
563
+ import { readFileSync } from "fs";
564
+ import {
565
+ KnowledgeAssetPathSchema,
566
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
567
+ MAX_KNOWLEDGE_ASSET_BYTES,
568
+ MAX_KNOWLEDGE_ASSET_COUNT,
569
+ assetExtension,
570
+ assetKindForExtension,
571
+ sniffAssetType,
572
+ svgInertnessProblem
573
+ } from "@bendyline/gezk";
574
+ function prepareAssets(assets, extraFiles, opts) {
575
+ for (const path of Object.keys(extraFiles)) {
576
+ if (path.startsWith("assets/")) {
577
+ throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
578
+ }
579
+ }
580
+ if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
581
+ throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
582
+ }
583
+ const seen = /* @__PURE__ */ new Set();
584
+ const prepared = [];
585
+ const skippedPaths = /* @__PURE__ */ new Set();
586
+ const rejectContent = (path, message) => {
587
+ if (opts.invalidAssets !== "warn") throw new Error(message);
588
+ skippedPaths.add(path);
589
+ opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
590
+ };
591
+ let total = 0;
592
+ for (const asset of assets) {
593
+ const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
594
+ if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
595
+ const key = asset.path.toLowerCase();
596
+ if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
597
+ seen.add(key);
598
+ if (asset.content === void 0 === (asset.absPath === void 0)) {
599
+ throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
600
+ }
601
+ const bytes = asset.content ?? readFileSync(asset.absPath);
602
+ if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
603
+ rejectContent(
604
+ asset.path,
605
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
606
+ );
607
+ continue;
608
+ }
609
+ const ext = assetExtension(asset.path);
610
+ const expected = ext ? assetKindForExtension(ext) : null;
611
+ const actual = sniffAssetType(bytes);
612
+ if (expected === null || actual !== expected) {
613
+ rejectContent(
614
+ asset.path,
615
+ `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
616
+ );
617
+ continue;
618
+ }
619
+ if (ext === "svg") {
620
+ const problem = svgInertnessProblem(bytes);
621
+ if (problem) {
622
+ rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
623
+ continue;
624
+ }
625
+ }
626
+ total += bytes.byteLength;
627
+ if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
628
+ throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
629
+ }
630
+ prepared.push({
631
+ path: asset.path,
632
+ bytes,
633
+ sizeBytes: bytes.byteLength,
634
+ sha256: createHash("sha256").update(bytes).digest("hex")
635
+ });
636
+ }
637
+ return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
638
+ }
639
+ function omitSkippedAssetReferences(markdown, skippedPaths) {
640
+ if (skippedPaths.size === 0) return markdown;
641
+ return markdown.replace(
642
+ /!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
643
+ (whole, text, target) => skippedPaths.has(target) ? text : whole
644
+ );
645
+ }
646
+
287
647
  // src/compiler/compile.ts
288
648
  async function compileKnowledgeCatalog(opts) {
289
649
  if (opts.topics.length === 0) {
@@ -292,7 +652,11 @@ async function compileKnowledgeCatalog(opts) {
292
652
  );
293
653
  }
294
654
  const topicById = assertTopicForest(opts.topics);
295
- const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
655
+ const { assets: preparedAssets, skippedPaths } = prepareAssets(
656
+ opts.assets ?? [],
657
+ opts.extraFiles ?? {},
658
+ opts
659
+ );
296
660
  const profile = opts.embeddingProfile;
297
661
  const chunkerOpts = {
298
662
  unit: opts.chunkingProfile.unit,
@@ -302,13 +666,40 @@ async function compileKnowledgeCatalog(opts) {
302
666
  };
303
667
  const prepared = [];
304
668
  const seenIds = /* @__PURE__ */ new Set();
305
- for await (const doc of opts.documents) {
669
+ for await (const input of opts.documents) {
670
+ const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
671
+ ...location,
672
+ longitude: normalizeLongitude(location.longitude)
673
+ })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
674
+ const doc = {
675
+ ...input,
676
+ locations,
677
+ markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
678
+ };
306
679
  if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
307
680
  seenIds.add(doc.id);
681
+ const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
682
+ const placements = [
683
+ {
684
+ topicId: leafTopicId,
685
+ ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
686
+ }
687
+ ];
688
+ const leaves = /* @__PURE__ */ new Set([leafTopicId]);
689
+ for (const raw of doc.tocReferences ?? []) {
690
+ const reference = KnowledgeTocReferenceSchema.parse(raw);
691
+ const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
692
+ if (leaves.has(topicId))
693
+ throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
694
+ leaves.add(topicId);
695
+ placements.push({ topicId, ordinal: reference.ordinal ?? null });
696
+ }
697
+ placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
308
698
  prepared.push({
309
699
  doc,
310
700
  topicPathKey: doc.topicPath.join("/"),
311
- leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
701
+ leafTopicId,
702
+ placements,
312
703
  metaJson: encodeDocumentMeta(doc),
313
704
  chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
314
705
  });
@@ -411,7 +802,9 @@ async function compileKnowledgeCatalog(opts) {
411
802
  });
412
803
  const topicDocCount = /* @__PURE__ */ new Map();
413
804
  for (const p of prepared) {
414
- topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
805
+ for (const placement of p.placements) {
806
+ topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
807
+ }
415
808
  }
416
809
  {
417
810
  const stmt = router.prepare(
@@ -435,6 +828,13 @@ async function compileKnowledgeCatalog(opts) {
435
828
  body_codec, body_blob)
436
829
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
437
830
  );
831
+ const insertLocation = router.prepare(
832
+ `INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
833
+ VALUES (?, ?, ?, ?, ?, ?)`
834
+ );
835
+ const insertPlacement = router.prepare(
836
+ "INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
837
+ );
438
838
  const insertAlias = router.prepare(
439
839
  "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
440
840
  );
@@ -473,6 +873,29 @@ async function compileKnowledgeCatalog(opts) {
473
873
  useBrotli ? "br" : "none",
474
874
  blob
475
875
  );
876
+ for (const location of p.doc.locations ?? []) {
877
+ const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
878
+ if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
879
+ throw new Error(
880
+ `document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
881
+ );
882
+ }
883
+ insertLocation.run(
884
+ p.doc.id,
885
+ location.id,
886
+ location.latitude,
887
+ location.longitude,
888
+ location.role,
889
+ provenance
890
+ );
891
+ }
892
+ for (const placement of p.placements) {
893
+ insertPlacement.run(
894
+ placement.topicId,
895
+ p.doc.id,
896
+ placement.ordinal === null ? null : BigInt(placement.ordinal)
897
+ );
898
+ }
476
899
  const aliases = [...new Set(p.doc.aliases ?? [])].sort();
477
900
  for (const alias of aliases) insertAlias.run(alias, p.doc.id);
478
901
  insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
@@ -705,7 +1128,7 @@ async function compileKnowledgeCatalog(opts) {
705
1128
  files.push({
706
1129
  path,
707
1130
  sizeBytes: bytes.length,
708
- sha256: createHash("sha256").update(bytes).digest("hex")
1131
+ sha256: createHash2("sha256").update(bytes).digest("hex")
709
1132
  });
710
1133
  }
711
1134
  for (const asset of preparedAssets) {
@@ -749,6 +1172,11 @@ async function compileKnowledgeCatalog(opts) {
749
1172
  assets: preparedAssets.length
750
1173
  },
751
1174
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
1175
+ spatial: spatialManifest(
1176
+ prepared.flatMap(
1177
+ (p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
1178
+ )
1179
+ ),
752
1180
  requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
753
1181
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
754
1182
  toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
@@ -844,58 +1272,6 @@ function encodeDocumentMeta(doc) {
844
1272
  }
845
1273
  return json;
846
1274
  }
847
- function prepareAssets(assets, extraFiles) {
848
- for (const path of Object.keys(extraFiles)) {
849
- if (path.startsWith("assets/")) {
850
- throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
851
- }
852
- }
853
- if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
854
- throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
855
- }
856
- const seen = /* @__PURE__ */ new Set();
857
- const prepared = [];
858
- let total = 0;
859
- for (const asset of assets) {
860
- const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
861
- if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
862
- const key = asset.path.toLowerCase();
863
- if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
864
- seen.add(key);
865
- if (asset.content === void 0 === (asset.absPath === void 0)) {
866
- throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
867
- }
868
- const bytes = asset.content ?? readFileSync(asset.absPath);
869
- if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
870
- throw new Error(
871
- `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
872
- );
873
- }
874
- total += bytes.byteLength;
875
- if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
876
- throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
877
- }
878
- const ext = assetExtension(asset.path);
879
- const expected = ext ? assetKindForExtension(ext) : null;
880
- const actual = sniffAssetType(bytes);
881
- if (expected === null || actual !== expected) {
882
- throw new Error(
883
- `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
884
- );
885
- }
886
- if (ext === "svg") {
887
- const problem = svgInertnessProblem(bytes);
888
- if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
889
- }
890
- prepared.push({
891
- path: asset.path,
892
- bytes,
893
- sizeBytes: bytes.byteLength,
894
- sha256: createHash("sha256").update(bytes).digest("hex")
895
- });
896
- }
897
- return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
898
- }
899
1275
  function assertAssetReferencesDeclared(prepared, assets) {
900
1276
  const declared = new Set(assets.map((a) => a.path));
901
1277
  for (const p of prepared) {
@@ -1110,7 +1486,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1110
1486
  return assignment.map((s) => renumber.get(s));
1111
1487
  }
1112
1488
  function seedFor(catalogId, shardId) {
1113
- const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1489
+ const digest = createHash2("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1114
1490
  return digest.readBigUInt64LE(24);
1115
1491
  }
1116
1492
  function makePrng(seed) {
@@ -1195,7 +1571,7 @@ function kMeans(samples, k, seed) {
1195
1571
  }
1196
1572
 
1197
1573
  // src/archive/read.ts
1198
- import { createHash as createHash2 } from "crypto";
1574
+ import { createHash as createHash3 } from "crypto";
1199
1575
  import { createWriteStream as createWriteStream2 } from "fs";
1200
1576
  import { mkdir as mkdir2 } from "fs/promises";
1201
1577
  import { createRequire as createRequire4 } from "module";
@@ -1485,7 +1861,7 @@ async function extractGezkVerified(archivePath, destDir) {
1485
1861
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1486
1862
  return;
1487
1863
  }
1488
- const hasher = createHash2("sha256");
1864
+ const hasher = createHash3("sha256");
1489
1865
  let sizeBytes = 0;
1490
1866
  const verify = new Transform({
1491
1867
  transform(chunk, _encoding, callback) {
@@ -1608,7 +1984,7 @@ function readPragmaNumber(db, pragma) {
1608
1984
  }
1609
1985
 
1610
1986
  // src/reader/catalog-handle.ts
1611
- import { readFileSync as readFileSync2 } from "fs";
1987
+ import { readFileSync as readFileSync3 } from "fs";
1612
1988
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1613
1989
  import { brotliDecompressSync } from "zlib";
1614
1990
  import {
@@ -1689,14 +2065,14 @@ function hammingTopK(index, query, k) {
1689
2065
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1690
2066
  return hits;
1691
2067
  }
1692
- function asymmetricTopK(index, query, k) {
2068
+ function asymmetricTopK(index, query, k, eligibleChunkIds) {
1693
2069
  const { bits, bytesPerRow, rows } = index;
1694
2070
  if (Math.ceil(query.length / 8) !== bytesPerRow) {
1695
2071
  throw new Error(
1696
2072
  `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1697
2073
  );
1698
2074
  }
1699
- const limit = Math.min(k, rows);
2075
+ const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
1700
2076
  if (limit <= 0) return [];
1701
2077
  const lut = new Float32Array(bytesPerRow * 256);
1702
2078
  for (let p = 0; p < bytesPerRow; p++) {
@@ -1746,6 +2122,10 @@ function asymmetricTopK(index, query, k) {
1746
2122
  };
1747
2123
  let offset = 0;
1748
2124
  for (let r = 0; r < rows; r++) {
2125
+ if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
2126
+ offset += bytesPerRow;
2127
+ continue;
2128
+ }
1749
2129
  let score = 0;
1750
2130
  for (let p = 0; p < bytesPerRow; p++) {
1751
2131
  score += lut[p * 256 + bits[offset + p]];
@@ -1770,14 +2150,126 @@ function asymmetricTopK(index, query, k) {
1770
2150
  return hits;
1771
2151
  }
1772
2152
 
2153
+ // src/reader/spatial-index.ts
2154
+ import {
2155
+ KnowledgeLocationSchema,
2156
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
2157
+ KnowledgeRadiusSchema,
2158
+ locationDistanceMeters,
2159
+ locationInBounds,
2160
+ normalizeLongitude as normalizeLongitude2,
2161
+ radiusBounds,
2162
+ spatialManifest as spatialManifest2
2163
+ } from "@bendyline/gezk";
2164
+ var DocumentSpatialIndex = class {
2165
+ constructor(db, allowLegacy) {
2166
+ this.db = db;
2167
+ this.allowLegacy = allowLegacy;
2168
+ this.hasTable = Boolean(
2169
+ db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
2170
+ );
2171
+ }
2172
+ db;
2173
+ allowLegacy;
2174
+ hasTable;
2175
+ legacy;
2176
+ decode(row) {
2177
+ const provenance = row.provenance_json;
2178
+ if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
2179
+ throw new Error("location provenance exceeds 16384 bytes");
2180
+ return {
2181
+ documentId: String(row.document_id),
2182
+ location: KnowledgeLocationSchema.parse({
2183
+ id: row.location_id,
2184
+ latitude: row.latitude,
2185
+ longitude: row.longitude,
2186
+ role: row.role,
2187
+ ...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
2188
+ })
2189
+ };
2190
+ }
2191
+ legacyRows() {
2192
+ if (this.legacy) return this.legacy;
2193
+ this.legacy = [];
2194
+ if (!this.allowLegacy) return this.legacy;
2195
+ for (const row of this.db.prepare(
2196
+ "SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
2197
+ ).all()) {
2198
+ try {
2199
+ const meta = JSON.parse(String(row.meta_json));
2200
+ if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
2201
+ continue;
2202
+ const point = KnowledgeLocationSchema.safeParse({
2203
+ id: "qualla-coordinate",
2204
+ role: "subject",
2205
+ latitude: meta.coordinates.lat,
2206
+ longitude: meta.coordinates.lng,
2207
+ provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
2208
+ });
2209
+ if (point.success)
2210
+ this.legacy.push({
2211
+ documentId: String(row.id),
2212
+ location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
2213
+ });
2214
+ } catch {
2215
+ }
2216
+ }
2217
+ return this.legacy;
2218
+ }
2219
+ rows() {
2220
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
2221
+ }
2222
+ locations(documentId) {
2223
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
2224
+ }
2225
+ integrity() {
2226
+ const rows = this.rows();
2227
+ const grouped = /* @__PURE__ */ new Map();
2228
+ for (const row of rows) {
2229
+ if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
2230
+ throw new Error("location longitude must be in [-180, 180)");
2231
+ const points = grouped.get(row.documentId) ?? [];
2232
+ points.push(row.location);
2233
+ grouped.set(row.documentId, points);
2234
+ if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
2235
+ throw new Error("location references missing document");
2236
+ }
2237
+ for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
2238
+ return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
2239
+ }
2240
+ matches(raw) {
2241
+ const radius = KnowledgeRadiusSchema.parse(raw);
2242
+ const boxes = radiusBounds(radius);
2243
+ const candidates = this.hasTable ? boxes.flatMap(
2244
+ (box) => this.db.prepare(`SELECT * FROM document_locations
2245
+ WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
2246
+ ) : this.legacyRows().filter(
2247
+ (row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
2248
+ );
2249
+ const matches = /* @__PURE__ */ new Map();
2250
+ for (const { documentId, location } of candidates) {
2251
+ const distanceMeters = locationDistanceMeters(radius, location);
2252
+ if (distanceMeters > radius.radiusMeters) continue;
2253
+ const previous = matches.get(documentId);
2254
+ if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
2255
+ matches.set(documentId, { distanceMeters, matchedLocation: location });
2256
+ }
2257
+ }
2258
+ return matches;
2259
+ }
2260
+ };
2261
+
1773
2262
  // src/reader/catalog-handle.ts
1774
2263
  var TOPIC_WALK_MAX_DEPTH = 32;
1775
2264
  var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
2265
+ var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
2266
+ var TITLE_ROUTE_SHARDS = 1;
1776
2267
  var CatalogHandle = class _CatalogHandle {
1777
2268
  constructor(rootDir, router) {
1778
2269
  this.rootDir = rootDir;
1779
2270
  this.router = router;
1780
2271
  this.schemaVersion = router.schemaVersion;
2272
+ this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
1781
2273
  this.meta = {};
1782
2274
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1783
2275
  this.meta[row.key] = row.value;
@@ -1789,6 +2281,9 @@ var CatalogHandle = class _CatalogHandle {
1789
2281
  router;
1790
2282
  connections = /* @__PURE__ */ new Map();
1791
2283
  bitIndexes = /* @__PURE__ */ new Map();
2284
+ /** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
2285
+ chunkTermRows = /* @__PURE__ */ new Map();
2286
+ spatial;
1792
2287
  assetsByPath = null;
1793
2288
  /** Lazily parsed from the profile echo; `undefined` until first needed. */
1794
2289
  binaryCenterCache;
@@ -1799,7 +2294,12 @@ var CatalogHandle = class _CatalogHandle {
1799
2294
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1800
2295
  static open(rootDir) {
1801
2296
  const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
1802
- return new _CatalogHandle(rootDir, router);
2297
+ try {
2298
+ return new _CatalogHandle(rootDir, router);
2299
+ } catch (error) {
2300
+ router.close();
2301
+ throw error;
2302
+ }
1803
2303
  }
1804
2304
  close() {
1805
2305
  for (const conn of this.connections.values()) conn.close();
@@ -1874,6 +2374,24 @@ var CatalogHandle = class _CatalogHandle {
1874
2374
  this.bitIndexes.set(shard.path, index);
1875
2375
  return index;
1876
2376
  }
2377
+ /**
2378
+ * Load the next shard's sign bits that are not resident yet, if they fit
2379
+ * the resident budget. False when nothing is left to load. One shard per
2380
+ * call, so a host can keep answering queries between shards while it warms
2381
+ * a large catalog, and never evicts a shard it loaded a moment ago.
2382
+ */
2383
+ prewarmNextShard() {
2384
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
2385
+ let resident = 0;
2386
+ for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
2387
+ for (const shard of this.shards) {
2388
+ if (this.bitIndexes.has(shard.path)) continue;
2389
+ if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
2390
+ this.shardBits(shard);
2391
+ return true;
2392
+ }
2393
+ return false;
2394
+ }
1877
2395
  /**
1878
2396
  * The stage-1 center the catalog's bits were derived with (a
1879
2397
  * `centered-sign` profile), or null for plain sign bits. Read from the
@@ -1923,9 +2441,13 @@ var CatalogHandle = class _CatalogHandle {
1923
2441
  get formatVersion() {
1924
2442
  return this.meta.format_version ?? "";
1925
2443
  }
1926
- documentColumns() {
2444
+ documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
1927
2445
  const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
1928
- return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2446
+ const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2447
+ return columns.split(", ").map((column) => {
2448
+ const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
2449
+ return prefix ? `${prefix}.${column} AS ${column}` : column;
2450
+ }).join(", ");
1929
2451
  }
1930
2452
  documentOrder() {
1931
2453
  return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
@@ -1935,6 +2457,9 @@ var CatalogHandle = class _CatalogHandle {
1935
2457
  * 0.5 catalog every document sits at a root, so both counts agree there.
1936
2458
  */
1937
2459
  topics() {
2460
+ const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
2461
+ FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
2462
+ FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
1938
2463
  return this.router.db.prepare(
1939
2464
  `WITH RECURSIVE sub(root, id, depth) AS (
1940
2465
  SELECT id, id, 0 FROM topics
@@ -1943,8 +2468,7 @@ var CatalogHandle = class _CatalogHandle {
1943
2468
  WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1944
2469
  ),
1945
2470
  rollup AS (
1946
- SELECT sub.root AS id, SUM(t.document_count) AS total
1947
- FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
2471
+ ${rollup}
1948
2472
  )
1949
2473
  SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
1950
2474
  rollup.total AS total_document_count
@@ -1968,6 +2492,9 @@ var CatalogHandle = class _CatalogHandle {
1968
2492
  const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1969
2493
  const offset = Math.max(0, opts.offset ?? 0);
1970
2494
  const descendants = opts.descendants ?? true;
2495
+ if (this.schemaVersion >= 4 && opts.topicId) {
2496
+ return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
2497
+ }
1971
2498
  let scope = "";
1972
2499
  let where = "";
1973
2500
  const params = [];
@@ -1990,16 +2517,101 @@ var CatalogHandle = class _CatalogHandle {
1990
2517
  const documents = this.router.db.prepare(
1991
2518
  `${scope} SELECT ${this.documentColumns()}
1992
2519
  FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
1993
- ).all(...params, limit, offset).map(rowToDocumentMeta);
2520
+ ).all(...params, limit, offset).map((row) => this.documentMeta(row));
1994
2521
  return { documents, total };
1995
2522
  }
2523
+ /** A reference is a placement, not a duplicate result or a redirect page. */
2524
+ sharedDocumentsPage(topicId, descendants, limit, offset) {
2525
+ const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
2526
+ JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
2527
+ const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
2528
+ SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
2529
+ FROM topic_documents td JOIN topics t ON t.id = td.topic_id
2530
+ WHERE td.topic_id IN (SELECT id FROM sub)
2531
+ )`;
2532
+ const total = Number(
2533
+ this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
2534
+ );
2535
+ const documents = this.router.db.prepare(
2536
+ `${scope}, ranked AS (
2537
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
2538
+ ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
2539
+ ) SELECT ${this.documentColumns("d", "p")}
2540
+ FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
2541
+ ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
2542
+ ).all(topicId, limit, offset).map((row) => this.documentMeta(row));
2543
+ return { documents, total };
2544
+ }
2545
+ /** Integrity facts kept separate from browse counts, which include shared references. */
2546
+ tocIntegrity() {
2547
+ const count = (sql) => Number(this.router.db.prepare(sql).get().n);
2548
+ const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
2549
+ const incorrectCounts = count(
2550
+ `SELECT COUNT(*) AS n FROM topics t LEFT JOIN
2551
+ (SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
2552
+ WHERE t.document_count != COALESCE(r.n, 0)`
2553
+ );
2554
+ if (this.schemaVersion < 4)
2555
+ return {
2556
+ placements: count("SELECT COUNT(*) AS n FROM documents"),
2557
+ invalidReferences: 0,
2558
+ missingPrimary: 0,
2559
+ incorrectCounts,
2560
+ duplicateReferences: 0
2561
+ };
2562
+ return {
2563
+ placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
2564
+ invalidReferences: count(
2565
+ `SELECT COUNT(*) AS n FROM topic_documents r
2566
+ LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
2567
+ WHERE d.id IS NULL OR t.id IS NULL OR
2568
+ (r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
2569
+ ),
2570
+ missingPrimary: count(
2571
+ `SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
2572
+ SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
2573
+ ),
2574
+ incorrectCounts,
2575
+ duplicateReferences: count(
2576
+ "SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
2577
+ )
2578
+ };
2579
+ }
1996
2580
  getDocument(id) {
1997
2581
  const row = this.router.db.prepare(
1998
2582
  `SELECT ${this.documentColumns()}, body_codec, body_blob
1999
2583
  FROM documents WHERE id = ?`
2000
2584
  ).get(id);
2001
2585
  if (!row) return null;
2002
- return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
2586
+ return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
2587
+ }
2588
+ /**
2589
+ * One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
2590
+ * shard the document's chunks live in. Null when the document or the
2591
+ * passage is not in this catalog, which is what a fabricated or stale
2592
+ * citation looks like.
2593
+ */
2594
+ getChunk(documentId, chunkUid2) {
2595
+ const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
2596
+ if (!doc) return null;
2597
+ const shardId = Number(doc.shard_id);
2598
+ const shard = this.shards.find((s) => s.id === shardId);
2599
+ if (!shard) return null;
2600
+ const row = this.shardDb(shard).prepare(
2601
+ `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
2602
+ FROM chunks WHERE chunk_uid = ? AND document_id = ?`
2603
+ ).get(chunkUid2, documentId);
2604
+ if (!row) return null;
2605
+ return {
2606
+ chunkUid: row.chunk_uid,
2607
+ documentId: row.document_id,
2608
+ title: row.title,
2609
+ headingPath: parseHeadingPath(row.heading_path),
2610
+ lineStart: Number(row.line_start),
2611
+ lineEnd: Number(row.line_end),
2612
+ text: row.text,
2613
+ shardId
2614
+ };
2003
2615
  }
2004
2616
  /** Documents whose `topic_id` names no declared topic (a validator check). */
2005
2617
  documentsWithUndeclaredTopic() {
@@ -2072,7 +2684,7 @@ var CatalogHandle = class _CatalogHandle {
2072
2684
  const index = /* @__PURE__ */ new Map();
2073
2685
  let raw;
2074
2686
  try {
2075
- raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2687
+ raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2076
2688
  } catch (error) {
2077
2689
  if (error.code === "ENOENT") {
2078
2690
  this.assetsByPath = index;
@@ -2111,7 +2723,7 @@ var CatalogHandle = class _CatalogHandle {
2111
2723
  }
2112
2724
  let bytes;
2113
2725
  try {
2114
- bytes = readFileSync2(this.resolveCatalogPath(path));
2726
+ bytes = readFileSync3(this.resolveCatalogPath(path));
2115
2727
  } catch (error) {
2116
2728
  throw new CatalogOpenError(
2117
2729
  `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
@@ -2123,19 +2735,59 @@ var CatalogHandle = class _CatalogHandle {
2123
2735
  }
2124
2736
  return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
2125
2737
  }
2738
+ documentMeta(row) {
2739
+ const document = rowToDocumentMeta(row);
2740
+ const locations = this.spatial.locations(document.id);
2741
+ return locations.length ? { ...document, locations } : document;
2742
+ }
2743
+ documentLocations(documentId) {
2744
+ return this.spatial.locations(documentId);
2745
+ }
2746
+ documentLocationRows() {
2747
+ return this.spatial.rows();
2748
+ }
2749
+ spatialIntegrity() {
2750
+ return this.spatial.integrity();
2751
+ }
2752
+ spatialMatches(radius) {
2753
+ return this.spatial.matches(radius);
2754
+ }
2755
+ nearbyDocuments(radius, opts = {}) {
2756
+ const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
2757
+ const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
2758
+ const ordered = [...this.spatialMatches(radius)].sort(
2759
+ (a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
2760
+ );
2761
+ const get = this.router.db.prepare(
2762
+ `SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
2763
+ );
2764
+ return {
2765
+ total: ordered.length,
2766
+ documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
2767
+ ...this.documentMeta(get.get(id)),
2768
+ ...match
2769
+ }))
2770
+ };
2771
+ }
2772
+ shardsForDocuments(ids) {
2773
+ if (!ids.size) return [];
2774
+ return this.router.db.prepare(
2775
+ "SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
2776
+ ).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
2777
+ }
2126
2778
  // ── search ────────────────────────────────────────────────────────────────
2127
2779
  /**
2128
2780
  * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
2129
2781
  * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
2130
2782
  */
2131
- searchDocumentsFts(query, limit = 10) {
2132
- const match = ftsQuery(query);
2133
- if (!match) return [];
2783
+ searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
2134
2784
  try {
2135
- return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
2136
- documentId,
2137
- rank: i
2138
- }));
2785
+ return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
2786
+ (documentId, i) => ({
2787
+ documentId,
2788
+ rank: i
2789
+ })
2790
+ );
2139
2791
  } catch {
2140
2792
  return [];
2141
2793
  }
@@ -2175,24 +2827,73 @@ var CatalogHandle = class _CatalogHandle {
2175
2827
  }
2176
2828
  return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
2177
2829
  }
2178
- /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
2179
- routeShards(queryVector, shardBudget) {
2830
+ /**
2831
+ * Top-S shard ids within THIS catalog (single-catalog callers, tests).
2832
+ * With the raw `query`, also the shard of the page it names by title
2833
+ * (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
2834
+ */
2835
+ routeShards(queryVector, shardBudget, query) {
2180
2836
  if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
2181
- return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
2837
+ const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
2838
+ if (query) {
2839
+ for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
2840
+ if (!routed.includes(named.shardId)) routed.push(named.shardId);
2841
+ }
2842
+ }
2843
+ return routed;
2844
+ }
2845
+ /**
2846
+ * Title-assisted routing: the shards holding the pages `query` names by
2847
+ * title (`namedTitleMatches`), most specific name first, at most `limit`
2848
+ * distinct shards. Centroid routing compares a question with averages of
2849
+ * whole shards, and e5 scores those in a band a few hundredths wide, so
2850
+ * the shard holding the very page a question names can rank 8th of 10:
2851
+ * "Who were the members of ABBA?" never reached the ABBA article in the
2852
+ * Wikipedia music catalog. Its title does — the vector arm scans that
2853
+ * shard too. Measured on the 10-shard music and film-tv catalogs, one such
2854
+ * shard lifted end-to-end recall@8 on "What is <title>?" questions from
2855
+ * 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
2856
+ *
2857
+ * `score` is the named match's specificity (fts-query.ts), comparable
2858
+ * across catalogs closely enough for the daemon to choose which catalogs'
2859
+ * shards to add when several name a page. Empty on any FTS error.
2860
+ */
2861
+ titleRouteShards(query, limit) {
2862
+ if (limit <= 0) return [];
2863
+ try {
2864
+ const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
2865
+ const out = [];
2866
+ for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
2867
+ const row = shardOf.get(named.documentId);
2868
+ if (!row) continue;
2869
+ const shardId = Number(row.shard_id);
2870
+ if (out.some((o) => o.shardId === shardId)) continue;
2871
+ out.push({ shardId, score: named.score });
2872
+ if (out.length >= limit) break;
2873
+ }
2874
+ return out.sort((a, b) => compareScores(b.score, a.score));
2875
+ } catch {
2876
+ return [];
2877
+ }
2182
2878
  }
2183
2879
  /**
2184
2880
  * Two-stage semantic search over the routed shards: sign-bit scan
2185
2881
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
2186
2882
  */
2187
2883
  searchSemantic(queryVector, opts = {}) {
2188
- const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
2189
- return this.searchShards(queryVector, shardIds, opts.finalK ?? RERANK_FINAL_K);
2884
+ const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
2885
+ return this.searchShards(
2886
+ queryVector,
2887
+ shardIds,
2888
+ opts.finalK ?? RERANK_FINAL_K,
2889
+ opts.allowedDocumentIds
2890
+ );
2190
2891
  }
2191
2892
  /**
2192
2893
  * The scan stage against an EXPLICIT shard list — what the daemon calls
2193
2894
  * after global cross-catalog routing has already spent the S budget.
2194
2895
  */
2195
- searchShards(queryVector, shardIds, finalK) {
2896
+ searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
2196
2897
  const center = this.binaryCenter();
2197
2898
  const scanQuery = center ? centerVector(queryVector, center) : queryVector;
2198
2899
  const hits = [];
@@ -2200,10 +2901,17 @@ var CatalogHandle = class _CatalogHandle {
2200
2901
  const shard = this.shards.find((s) => s.id === shardId);
2201
2902
  if (!shard) continue;
2202
2903
  const db = this.shardDb(shard);
2203
- const k = rerankK(finalK, shard.chunkCount);
2204
- const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
2205
- chunk_id: hit.chunkId
2206
- }));
2904
+ const eligible = allowedDocumentIds ? new Set(
2905
+ db.prepare(
2906
+ "SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
2907
+ ).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
2908
+ ) : void 0;
2909
+ const k = rerankK(finalK, eligible?.size ?? shard.chunkCount);
2910
+ const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k, eligible).map(
2911
+ (hit) => ({
2912
+ chunk_id: hit.chunkId
2913
+ })
2914
+ );
2207
2915
  if (candidates.length === 0) continue;
2208
2916
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
2209
2917
  const reranked = candidates.map((c) => {
@@ -2237,20 +2945,26 @@ var CatalogHandle = class _CatalogHandle {
2237
2945
  return hits;
2238
2946
  }
2239
2947
  /** Chunk-body FTS over the routed shards (explicit search only). */
2240
- searchChunksFts(query, shardIds, limitPerShard = 12) {
2241
- const match = ftsQuery(query);
2242
- if (!match) return [];
2948
+ searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
2243
2949
  const hits = [];
2244
2950
  for (const shardId of shardIds) {
2245
2951
  const shard = this.shards.find((s) => s.id === shardId);
2246
2952
  if (!shard) continue;
2247
2953
  const db = this.shardDb(shard);
2248
2954
  try {
2955
+ const match = selectiveFtsQuery(
2956
+ query,
2957
+ shard.chunkCount,
2958
+ (phrase) => this.chunkFtsRows(shard, db, phrase)
2959
+ );
2960
+ if (!match) return hits;
2249
2961
  const rows = db.prepare(
2250
2962
  `SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
2251
2963
  FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
2252
- WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
2253
- ).all(match, limitPerShard);
2964
+ WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
2965
+ ).all(
2966
+ ...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
2967
+ );
2254
2968
  for (const row of rows) {
2255
2969
  hits.push({
2256
2970
  chunkUid: row.chunk_uid,
@@ -2269,6 +2983,18 @@ var CatalogHandle = class _CatalogHandle {
2269
2983
  }
2270
2984
  return hits;
2271
2985
  }
2986
+ /** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
2987
+ chunkFtsRows(shard, db, phrase) {
2988
+ const key = `${shard.path}\0${phrase}`;
2989
+ let rows = this.chunkTermRows.get(key);
2990
+ if (rows === void 0) {
2991
+ if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
2992
+ const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
2993
+ rows = Number(row.c);
2994
+ this.chunkTermRows.set(key, rows);
2995
+ }
2996
+ return rows;
2997
+ }
2272
2998
  /**
2273
2999
  * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
2274
3000
  * nearest neighbor — proves the shard's vectors load and scan. Returns
@@ -2322,7 +3048,6 @@ function parseHeadingPath(raw) {
2322
3048
  return [];
2323
3049
  }
2324
3050
  }
2325
- var ftsQuery = sanitizeFtsQuery;
2326
3051
 
2327
3052
  // src/reader/validate.ts
2328
3053
  import { readFile } from "fs/promises";
@@ -2334,6 +3059,7 @@ import {
2334
3059
  MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
2335
3060
  assetExtension as assetExtension2,
2336
3061
  assetKindForExtension as assetKindForExtension2,
3062
+ canonicalizeJson as canonicalizeJson2,
2337
3063
  isKnowledgeAssetPath as isKnowledgeAssetPath2,
2338
3064
  sniffAssetType as sniffAssetType2,
2339
3065
  svgInertnessProblem as svgInertnessProblem2
@@ -2409,6 +3135,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2409
3135
  handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
2410
3136
  `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
2411
3137
  );
3138
+ if (manifest.formatVersion === "0.7") {
3139
+ try {
3140
+ const spatial = handle.spatialIntegrity();
3141
+ check("document-locations-table", spatial.hasTable);
3142
+ check(
3143
+ "document-locations-summary",
3144
+ Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
3145
+ "spatial counts and coverage must match router points"
3146
+ );
3147
+ } catch (error) {
3148
+ check(
3149
+ "document-locations-integrity",
3150
+ false,
3151
+ error instanceof Error ? error.message : String(error)
3152
+ );
3153
+ }
3154
+ }
2412
3155
  const topics = handle.topics();
2413
3156
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
2414
3157
  check("topics-tree", ...topicTreeProblem(topics));
@@ -2446,11 +3189,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2446
3189
  manifest.files.some((f) => f.path === manifest.license.noticePath),
2447
3190
  `manifest.files lacks the declared notice ${manifest.license.noticePath}`
2448
3191
  );
3192
+ const toc = handle.tocIntegrity();
3193
+ check(
3194
+ "toc-references",
3195
+ toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
3196
+ `${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
3197
+ );
3198
+ check(
3199
+ "toc-counts",
3200
+ toc.incorrectCounts === 0,
3201
+ `${toc.incorrectCounts} incorrect per-topic counts`
3202
+ );
2449
3203
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
2450
3204
  check(
2451
3205
  "counts-documents",
2452
- handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === manifest.counts.documents,
2453
- `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
3206
+ handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
3207
+ `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
2454
3208
  );
2455
3209
  check(
2456
3210
  "counts-shards",
@@ -2578,6 +3332,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2578
3332
  );
2579
3333
  }
2580
3334
  }
3335
+ } catch (error) {
3336
+ check("catalog-structure", false, error instanceof Error ? error.message : String(error));
2581
3337
  } finally {
2582
3338
  handle.close();
2583
3339
  }
@@ -2686,13 +3442,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
2686
3442
  }
2687
3443
 
2688
3444
  // src/export/parquet.ts
2689
- import { createHash as createHash3 } from "crypto";
3445
+ import { createHash as createHash4 } from "crypto";
2690
3446
  import { createWriteStream as createWriteStream3 } from "fs";
2691
3447
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2692
3448
  import { join as join4, resolve as resolve3 } from "path";
2693
3449
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2694
3450
  import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
2695
- var PARQUET_EXPORT_VERSION = 3;
3451
+ var PARQUET_EXPORT_VERSION = 4;
2696
3452
  var DEFAULT_ROW_GROUP_SIZE = 32768;
2697
3453
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2698
3454
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2739,6 +3495,34 @@ async function exportCatalogParquet(opts) {
2739
3495
  onProgress: opts.onProgress
2740
3496
  })
2741
3497
  );
3498
+ const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
3499
+ const locations = spatial.rows();
3500
+ if (spatial.hasTable || locations.length) {
3501
+ const path = join4(staging, "document-locations.ndjson");
3502
+ const out = ndjsonWriter(path);
3503
+ for (const { documentId, location } of locations)
3504
+ await out.write({
3505
+ document_id: documentId,
3506
+ location_id: location.id,
3507
+ latitude: location.latitude,
3508
+ longitude: location.longitude,
3509
+ role: location.role,
3510
+ provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
3511
+ });
3512
+ await out.close();
3513
+ const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
3514
+ const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
3515
+ files.push(
3516
+ await writeParquet(opts.duckdb, {
3517
+ outDir,
3518
+ file: "document-locations.parquet",
3519
+ table: "document_locations",
3520
+ rows: locations.length,
3521
+ sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
3522
+ onProgress: opts.onProgress
3523
+ })
3524
+ );
3525
+ }
2742
3526
  for (const shard of manifest.router.shards) {
2743
3527
  const tag = String(shard.id).padStart(3, "0");
2744
3528
  const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
@@ -3081,7 +3865,7 @@ async function writeParquet(cli, spec) {
3081
3865
  ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
3082
3866
  rows: spec.rows,
3083
3867
  sizeBytes: (await stat2(target)).size,
3084
- sha256: createHash3("sha256").update(bytes).digest("hex")
3868
+ sha256: createHash4("sha256").update(bytes).digest("hex")
3085
3869
  };
3086
3870
  }
3087
3871
  async function findDuckdbBinary(env = process.env) {
@@ -3582,7 +4366,7 @@ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
3582
4366
  import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
3583
4367
 
3584
4368
  // src/embedding/artifact-verify.ts
3585
- import { createHash as createHash4 } from "crypto";
4369
+ import { createHash as createHash5 } from "crypto";
3586
4370
  import { createReadStream } from "fs";
3587
4371
  import { stat as stat3 } from "fs/promises";
3588
4372
  import { join as join5 } from "path";
@@ -3626,7 +4410,7 @@ async function fileDigest(path) {
3626
4410
  const memo = digestMemo.get(key);
3627
4411
  if (memo) return memo;
3628
4412
  const digest = await new Promise((resolve5, reject) => {
3629
- const hash = createHash4("sha256");
4413
+ const hash = createHash5("sha256");
3630
4414
  createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
3631
4415
  });
3632
4416
  if (digestMemo.size >= DIGEST_MEMO_MAX) {
@@ -3771,11 +4555,13 @@ async function createProfileEmbedder(profile, opts = {}) {
3771
4555
  const [pipe, tokenizer] = await Promise.all([
3772
4556
  transformers.pipeline("feature-extraction", profile.model.repo, {
3773
4557
  ...modelOptions,
4558
+ ...opts.localFilesOnly ? { local_files_only: true } : {},
3774
4559
  ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
3775
4560
  ...progress
3776
4561
  }),
3777
4562
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3778
4563
  revision: modelOptions.revision,
4564
+ ...opts.localFilesOnly ? { local_files_only: true } : {},
3779
4565
  ...progress
3780
4566
  })
3781
4567
  ]);
@@ -3821,7 +4607,8 @@ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6
3821
4607
  import {
3822
4608
  CatalogDocumentSchema,
3823
4609
  KnowledgeDocumentIdSchema,
3824
- KnowledgeOrdinalSchema,
4610
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
4611
+ KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
3825
4612
  MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
3826
4613
  assetExtension as assetExtension3,
3827
4614
  formatKnowledgeUri,
@@ -4517,7 +5304,15 @@ var HUGO_SECTION_PAGE = "_index";
4517
5304
  var DOCFX_JSON = "docfx.json";
4518
5305
  var WARNING_EXAMPLES2 = 20;
4519
5306
  var SECTION_PAGE_ORDINAL = -2147483648;
4520
- var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
5307
+ var RESERVED_KEYS = /* @__PURE__ */ new Set([
5308
+ "title",
5309
+ "summary",
5310
+ "aliases",
5311
+ "id",
5312
+ "order",
5313
+ "subcategory",
5314
+ "locations"
5315
+ ]);
4521
5316
  function readFrontMatter(raw, file, conventions) {
4522
5317
  const { hugo } = conventions;
4523
5318
  let data;
@@ -4536,6 +5331,12 @@ function readFrontMatter(raw, file, conventions) {
4536
5331
  throw new Error(`${file}: front matter '${key}' must be a string`);
4537
5332
  return value.trim() || void 0;
4538
5333
  };
5334
+ if (data.locations !== void 0) {
5335
+ const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
5336
+ if (!locations.success)
5337
+ throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
5338
+ out.locations = locations.data;
5339
+ }
4539
5340
  out.title = str("title");
4540
5341
  out.summary = str("summary");
4541
5342
  out.id = str("id");
@@ -4548,7 +5349,7 @@ function readFrontMatter(raw, file, conventions) {
4548
5349
  throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
4549
5350
  }
4550
5351
  if (data.order !== void 0 && data.order !== null) {
4551
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
5352
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4552
5353
  if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
4553
5354
  out.order = order.data;
4554
5355
  }
@@ -4569,7 +5370,7 @@ function readFrontMatter(raw, file, conventions) {
4569
5370
  subcategory.title = record.title.trim();
4570
5371
  }
4571
5372
  if (record.order !== void 0 && record.order !== null) {
4572
- const order = KnowledgeOrdinalSchema.safeParse(record.order);
5373
+ const order = KnowledgeOrdinalSchema2.safeParse(record.order);
4573
5374
  if (!order.success)
4574
5375
  throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
4575
5376
  subcategory.order = order.data;
@@ -4580,7 +5381,7 @@ function readFrontMatter(raw, file, conventions) {
4580
5381
  if (data.draft === true) out.unpublished = "draft";
4581
5382
  else if (data.headless === true) out.unpublished = "headless";
4582
5383
  if (data.weight !== void 0 && data.weight !== null) {
4583
- const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
5384
+ const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
4584
5385
  if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
4585
5386
  if (weight.data > 0) out.weight = weight.data;
4586
5387
  }
@@ -4653,7 +5454,7 @@ async function readTopicSidecar(dir) {
4653
5454
  out.description = data.description.trim();
4654
5455
  }
4655
5456
  if (data.order !== void 0 && data.order !== null) {
4656
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
5457
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4657
5458
  if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
4658
5459
  out.order = order.data;
4659
5460
  }
@@ -4885,6 +5686,7 @@ function rewriteLinks(ctx, markdown) {
4885
5686
  return line.replace(
4886
5687
  INLINE_LINK,
4887
5688
  (whole, bang, text, target, title) => {
5689
+ if (bang === "!" && ctx.skipImages) return text;
4888
5690
  const rewritten = rewriteTarget(ctx, bang === "!", target);
4889
5691
  return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
4890
5692
  }
@@ -4955,22 +5757,16 @@ async function loadMarkdownCatalog(rootDir, opts) {
4955
5757
  };
4956
5758
  const placements = /* @__PURE__ */ new Map();
4957
5759
  if (outline) {
4958
- const duplicates = [];
4959
5760
  const walk = (topic, chain) => {
4960
5761
  for (const entry of topic.entries) {
4961
5762
  const key = entry.file.normalize("NFC");
4962
- if (placements.has(key)) duplicates.push(entry.file);
4963
- else placements.set(key, { chain, entry });
5763
+ const existing = placements.get(key) ?? [];
5764
+ existing.push({ chain, entry });
5765
+ placements.set(key, existing);
4964
5766
  }
4965
5767
  for (const child of topic.children) walk(child, [...chain, child]);
4966
5768
  };
4967
5769
  walk(outline.root, []);
4968
- warnCapped(
4969
- warn,
4970
- duplicates,
4971
- (file) => `${file}: listed more than once in the table of contents; the first place wins`,
4972
- (more) => `${more} more files are listed more than once in the table of contents`
4973
- );
4974
5770
  }
4975
5771
  const outlineTopicIds = /* @__PURE__ */ new Map();
4976
5772
  let outlineTooDeep = false;
@@ -5045,7 +5841,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
5045
5841
  let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
5046
5842
  let ordinal = fm.order;
5047
5843
  let topicPath;
5048
- const placement = placements.get(rel.normalize("NFC"));
5844
+ const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
5845
+ const placement = documentPlacements[0];
5049
5846
  if (placement) {
5050
5847
  topicPath = ensureOutlineChain(placement.chain);
5051
5848
  if (placement.entry.title) title = placement.entry.title;
@@ -5075,6 +5872,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
5075
5872
  const parentId = topicPath[topicPath.length - 1];
5076
5873
  topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
5077
5874
  }
5875
+ const tocReferences = [];
5876
+ const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
5877
+ for (const extra of documentPlacements.slice(1)) {
5878
+ let path = ensureOutlineChain(extra.chain);
5879
+ if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
5880
+ const leaf = path.at(-1);
5881
+ if (placedLeaves.has(leaf)) continue;
5882
+ placedLeaves.add(leaf);
5883
+ const order = extra.entry.order ?? fm.order;
5884
+ tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
5885
+ }
5078
5886
  const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
5079
5887
  loaded.push({
5080
5888
  rel,
@@ -5086,6 +5894,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
5086
5894
  ...summary ? { summary } : {},
5087
5895
  language: opts.language,
5088
5896
  topicPath,
5897
+ ...tocReferences.length ? { tocReferences } : {},
5898
+ ...fm.locations ? { locations: fm.locations } : {},
5089
5899
  markdown: fm.body,
5090
5900
  ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
5091
5901
  ...ordinal !== void 0 ? { ordinal } : {},
@@ -5115,7 +5925,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
5115
5925
  );
5116
5926
  warnCapped(
5117
5927
  warn,
5118
- [...placements].filter(([file]) => !seenRel.has(file)).map(([, p]) => p.entry.file),
5928
+ [...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
5119
5929
  (file) => `${file}: named by the table of contents but not found among the Markdown files`,
5120
5930
  (more) => `${more} more files named by the table of contents are not among the Markdown files`
5121
5931
  );
@@ -5152,6 +5962,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
5152
5962
  // A docfx docset draws on dependent repositories (`~/reusable-content`)
5153
5963
  // that a checkout never holds, so a missing image is expected there.
5154
5964
  missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
5965
+ skipImages: opts.skipImages ?? false,
5155
5966
  tildeIsRoot: docfx,
5156
5967
  warn: (message) => linkWarnings.push(message)
5157
5968
  };
@@ -5288,7 +6099,7 @@ function newerRegistryEntries(registry, installed) {
5288
6099
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
5289
6100
 
5290
6101
  // src/signatures/jcs.ts
5291
- import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
6102
+ import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
5292
6103
  export {
5293
6104
  BGE_SMALL_EN_V15_1,
5294
6105
  BODY_CODEC_MIN_BYTES,
@@ -5328,6 +6139,7 @@ export {
5328
6139
  MULTILINGUAL_E5_SMALL_2,
5329
6140
  PARQUET_EXPORT_VERSION,
5330
6141
  PARQUET_REPORT_PATH,
6142
+ QUERY_STOP_WORDS,
5331
6143
  README_PATH,
5332
6144
  RERANK_CANDIDATES_MAX,
5333
6145
  RERANK_CANDIDATES_MIN,
@@ -5340,13 +6152,14 @@ export {
5340
6152
  SHARD_MAX_CHUNKS,
5341
6153
  SHARD_TARGET_CHUNKS,
5342
6154
  SOURCE_NOTICES_PATH,
6155
+ TITLE_ROUTE_SHARDS,
5343
6156
  TRANSFORMERS_PEER_RANGE,
5344
6157
  ZIP_FIXED_MTIME,
5345
6158
  assertDuckdbCli,
5346
6159
  assetReferences,
5347
6160
  assignOutlineOrders,
5348
6161
  asymmetricTopK,
5349
- canonicalizeJson2 as canonicalizeJson,
6162
+ canonicalizeJson3 as canonicalizeJson,
5350
6163
  centerVector,
5351
6164
  chunkContentHash,
5352
6165
  chunkMarkdownProfile,