@bendyline/gezel-knowledge 1.2.1 → 1.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.d.ts +175 -49
  2. package/dist/index.js +1462 -201
  3. package/package.json +2 -2
package/dist/index.js CHANGED
@@ -180,12 +180,11 @@ function splitOversizeLine(line, opts) {
180
180
  }
181
181
 
182
182
  // src/compiler/compile.ts
183
- import { createHash } from "crypto";
183
+ import { createHash as createHash2 } from "crypto";
184
184
  import {
185
185
  closeSync,
186
186
  mkdirSync,
187
187
  openSync,
188
- readFileSync,
189
188
  readSync,
190
189
  rmSync,
191
190
  statSync,
@@ -196,16 +195,13 @@ import { join } from "path";
196
195
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
197
196
  import {
198
197
  GEZK_MANIFEST_KIND,
199
- KnowledgeAssetPathSchema,
200
198
  KnowledgeCatalogManifestSchema,
201
- MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
202
- MAX_KNOWLEDGE_ASSET_BYTES,
203
- MAX_KNOWLEDGE_ASSET_COUNT,
204
- assetExtension,
205
- assetKindForExtension,
199
+ KnowledgeLocationsSchema,
200
+ KnowledgeOrdinalSchema,
201
+ KnowledgeTocReferenceSchema,
206
202
  canonicalizeJson,
207
- sniffAssetType,
208
- svgInertnessProblem
203
+ normalizeLongitude,
204
+ spatialManifest
209
205
  } from "@bendyline/gezk";
210
206
 
211
207
  // src/archive/write.ts
@@ -251,28 +247,306 @@ var nodeRequire2 = createRequire2(import.meta.url);
251
247
  var sqlite = nodeRequire2("node:sqlite");
252
248
  var DatabaseSync = sqlite.DatabaseSync;
253
249
 
250
+ // src/query-stopwords.ts
251
+ var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
252
+ // articles + determiners
253
+ "a",
254
+ "an",
255
+ "the",
256
+ "this",
257
+ "that",
258
+ "these",
259
+ "those",
260
+ "some",
261
+ "any",
262
+ "all",
263
+ "each",
264
+ "both",
265
+ "every",
266
+ "other",
267
+ "another",
268
+ "same",
269
+ "such",
270
+ // pronouns
271
+ "i",
272
+ "me",
273
+ "my",
274
+ "we",
275
+ "us",
276
+ "our",
277
+ "you",
278
+ "your",
279
+ "it",
280
+ "its",
281
+ "he",
282
+ "him",
283
+ "his",
284
+ "she",
285
+ "her",
286
+ "they",
287
+ "them",
288
+ "their",
289
+ // be / have / do
290
+ "am",
291
+ "is",
292
+ "are",
293
+ "was",
294
+ "were",
295
+ "be",
296
+ "been",
297
+ "being",
298
+ "have",
299
+ "has",
300
+ "had",
301
+ "do",
302
+ "does",
303
+ "did",
304
+ "done",
305
+ // modals
306
+ "can",
307
+ "could",
308
+ "will",
309
+ "would",
310
+ "shall",
311
+ "should",
312
+ "may",
313
+ "might",
314
+ "must",
315
+ // question words
316
+ "how",
317
+ "what",
318
+ "when",
319
+ "where",
320
+ "which",
321
+ "who",
322
+ "whom",
323
+ "why",
324
+ // prepositions + conjunctions
325
+ "about",
326
+ "and",
327
+ "as",
328
+ "at",
329
+ "but",
330
+ "by",
331
+ "for",
332
+ "from",
333
+ "if",
334
+ "in",
335
+ "into",
336
+ "of",
337
+ "on",
338
+ "or",
339
+ "so",
340
+ "than",
341
+ "then",
342
+ "there",
343
+ "to",
344
+ "with",
345
+ // request scaffolding + degree words
346
+ "please",
347
+ "help",
348
+ "let",
349
+ "tell",
350
+ "want",
351
+ "need",
352
+ "create",
353
+ "make",
354
+ "give",
355
+ "get",
356
+ "got",
357
+ "use",
358
+ "using",
359
+ "just",
360
+ "like",
361
+ "also",
362
+ "very",
363
+ "only",
364
+ "new",
365
+ "more",
366
+ "most",
367
+ "much",
368
+ "many",
369
+ "few",
370
+ "own",
371
+ // conversational scaffolding
372
+ "hey",
373
+ "hi",
374
+ "hiya",
375
+ "hello",
376
+ "thanks",
377
+ "thank",
378
+ "okay",
379
+ "ok",
380
+ "yep",
381
+ "yeah",
382
+ "yes",
383
+ "nope",
384
+ "good",
385
+ "great",
386
+ "fine",
387
+ "nice",
388
+ "cool",
389
+ "ready",
390
+ "going",
391
+ "doing",
392
+ "sounds",
393
+ "morning",
394
+ "afternoon",
395
+ "evening",
396
+ "night",
397
+ // Apostrophes are token boundaries, so contractions otherwise leave these
398
+ // fragments looking like rare, highly distinctive query terms.
399
+ "s",
400
+ "t",
401
+ "re",
402
+ "ve",
403
+ "ll",
404
+ "d",
405
+ "m",
406
+ "isn",
407
+ "aren",
408
+ "wasn",
409
+ "weren",
410
+ "don",
411
+ "doesn",
412
+ "didn",
413
+ "hasn",
414
+ "haven",
415
+ "hadn",
416
+ "won",
417
+ "wouldn",
418
+ "shouldn",
419
+ "couldn"
420
+ ]);
421
+
254
422
  // src/reader/fts-query.ts
255
- function sanitizeFtsQuery(query) {
423
+ function contentTokens(query) {
256
424
  const tokens = [
257
425
  ...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
258
426
  ];
427
+ const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
428
+ return content.length > 0 ? content : tokens;
429
+ }
430
+ function contentFtsQuery(query) {
431
+ const tokens = contentTokens(query);
432
+ return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
433
+ }
434
+ function selectiveFtsQuery(query, rows, rowsMatching) {
435
+ const tokens = contentTokens(query);
259
436
  if (tokens.length === 0) return null;
260
- return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
437
+ const counted = tokens.map((token) => {
438
+ const phrase = quoteToken(token);
439
+ return { phrase, hits: rowsMatching(phrase) };
440
+ });
441
+ const kept = counted.filter((c) => c.hits * 2 <= rows);
442
+ const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
443
+ return chosen.map((c) => c.phrase).join(" OR ");
261
444
  }
262
445
  var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
263
446
  var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
264
- function documentFtsTopIds(db, match, limit, exactTitle) {
265
- const title = exactTitle?.normalize("NFKC").trim() ?? "";
266
- return db.prepare(
447
+ function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
448
+ const match = contentFtsQuery(query);
449
+ if (!match || allowedDocumentIds?.size === 0) return [];
450
+ const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
451
+ const title = query.normalize("NFKC").trim();
452
+ const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
453
+ const rest = db.prepare(
267
454
  `SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
268
- ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
269
- ).all(match, title, limit).map((row) => row.document_id);
455
+ ${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
456
+ ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
457
+ ).all(
458
+ ...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
459
+ );
460
+ for (const row of rest) {
461
+ if (ids.length >= limit) break;
462
+ if (!ids.includes(row.document_id)) ids.push(row.document_id);
463
+ }
464
+ return ids;
465
+ }
466
+ var NAME_STOPWORDS = new Set(
467
+ "a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
468
+ );
469
+ var NAMED_TITLE_CANDIDATES = 256;
470
+ var TITLE_DF_CACHE_MAX = 5e4;
471
+ var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
472
+ var SAME_WORDS_SCORE = Number.MAX_VALUE;
473
+ function nameWords(text) {
474
+ return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
475
+ }
476
+ function quoteToken(token) {
477
+ return `"${token.replace(/"/g, '""')}"`;
478
+ }
479
+ var titleStatsByDb = /* @__PURE__ */ new WeakMap();
480
+ function titleStats(db) {
481
+ let stats = titleStatsByDb.get(db);
482
+ if (!stats) {
483
+ const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
484
+ stats = {
485
+ documents: Math.max(1, Number(row.c)),
486
+ df: /* @__PURE__ */ new Map(),
487
+ dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
488
+ candidates: db.prepare(
489
+ `SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
490
+ ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
491
+ )
492
+ };
493
+ titleStatsByDb.set(db, stats);
494
+ }
495
+ return stats;
496
+ }
497
+ function titleDf(stats, word) {
498
+ let df = stats.df.get(word);
499
+ if (df === void 0) {
500
+ if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
501
+ const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
502
+ df = Number(row.c);
503
+ stats.df.set(word, df);
504
+ }
505
+ return df;
506
+ }
507
+ function namedTitleMatches(db, query, limit, allowedDocumentIds) {
508
+ if (allowedDocumentIds?.size === 0) return [];
509
+ const raw = query.normalize("NFKC").trim();
510
+ const words = [...new Set(nameWords(raw))].slice(0, 16);
511
+ const content = words.filter((w) => !NAME_STOPWORDS.has(w));
512
+ if (content.length === 0 || limit <= 0) return [];
513
+ const asked = new Set(words);
514
+ const capitalized = new Set(
515
+ (raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
516
+ );
517
+ const queryWords = nameWords(raw).join(" ");
518
+ const stats = titleStats(db);
519
+ const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
520
+ const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
521
+ AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
522
+ const named = [];
523
+ for (const row of rows) {
524
+ const titleWords = nameWords(row.title);
525
+ if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
526
+ const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
527
+ if (distinct.length === 0) continue;
528
+ let score;
529
+ if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
530
+ score = EXACT_TITLE_SCORE;
531
+ } else if (titleWords.join(" ") === queryWords) {
532
+ score = SAME_WORDS_SCORE;
533
+ } else {
534
+ score = 0;
535
+ for (const w of distinct) {
536
+ score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
537
+ }
538
+ }
539
+ named.push({ documentId: row.document_id, score, order: named.length });
540
+ }
541
+ named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
542
+ return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
543
+ }
544
+ function compareScores(a, b) {
545
+ return a === b ? 0 : a > b ? 1 : -1;
270
546
  }
271
547
  var SMOKE_QUERY_TOP_N = 10;
272
548
  function documentSmokeQueryMisses(db, smoke) {
273
- const match = sanitizeFtsQuery(smoke.query);
274
- if (!match) return [...smoke.expectedDocumentIds];
275
- const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
549
+ const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
276
550
  return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
277
551
  }
278
552
 
@@ -284,6 +558,92 @@ var KNOWLEDGE_TOOLCHAIN = {
284
558
  version: nodeRequire3("../package.json").version
285
559
  };
286
560
 
561
+ // src/compiler/assets.ts
562
+ import { createHash } from "crypto";
563
+ import { readFileSync } from "fs";
564
+ import {
565
+ KnowledgeAssetPathSchema,
566
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
567
+ MAX_KNOWLEDGE_ASSET_BYTES,
568
+ MAX_KNOWLEDGE_ASSET_COUNT,
569
+ assetExtension,
570
+ assetKindForExtension,
571
+ sniffAssetType,
572
+ svgInertnessProblem
573
+ } from "@bendyline/gezk";
574
+ function prepareAssets(assets, extraFiles, opts) {
575
+ for (const path of Object.keys(extraFiles)) {
576
+ if (path.startsWith("assets/")) {
577
+ throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
578
+ }
579
+ }
580
+ if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
581
+ throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
582
+ }
583
+ const seen = /* @__PURE__ */ new Set();
584
+ const prepared = [];
585
+ const skippedPaths = /* @__PURE__ */ new Set();
586
+ const rejectContent = (path, message) => {
587
+ if (opts.invalidAssets !== "warn") throw new Error(message);
588
+ skippedPaths.add(path);
589
+ opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
590
+ };
591
+ let total = 0;
592
+ for (const asset of assets) {
593
+ const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
594
+ if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
595
+ const key = asset.path.toLowerCase();
596
+ if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
597
+ seen.add(key);
598
+ if (asset.content === void 0 === (asset.absPath === void 0)) {
599
+ throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
600
+ }
601
+ const bytes = asset.content ?? readFileSync(asset.absPath);
602
+ if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
603
+ rejectContent(
604
+ asset.path,
605
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
606
+ );
607
+ continue;
608
+ }
609
+ const ext = assetExtension(asset.path);
610
+ const expected = ext ? assetKindForExtension(ext) : null;
611
+ const actual = sniffAssetType(bytes);
612
+ if (expected === null || actual !== expected) {
613
+ rejectContent(
614
+ asset.path,
615
+ `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
616
+ );
617
+ continue;
618
+ }
619
+ if (ext === "svg") {
620
+ const problem = svgInertnessProblem(bytes);
621
+ if (problem) {
622
+ rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
623
+ continue;
624
+ }
625
+ }
626
+ total += bytes.byteLength;
627
+ if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
628
+ throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
629
+ }
630
+ prepared.push({
631
+ path: asset.path,
632
+ bytes,
633
+ sizeBytes: bytes.byteLength,
634
+ sha256: createHash("sha256").update(bytes).digest("hex")
635
+ });
636
+ }
637
+ return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
638
+ }
639
+ function omitSkippedAssetReferences(markdown, skippedPaths) {
640
+ if (skippedPaths.size === 0) return markdown;
641
+ return markdown.replace(
642
+ /!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
643
+ (whole, text, target) => skippedPaths.has(target) ? text : whole
644
+ );
645
+ }
646
+
287
647
  // src/compiler/compile.ts
288
648
  async function compileKnowledgeCatalog(opts) {
289
649
  if (opts.topics.length === 0) {
@@ -292,7 +652,11 @@ async function compileKnowledgeCatalog(opts) {
292
652
  );
293
653
  }
294
654
  const topicById = assertTopicForest(opts.topics);
295
- const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
655
+ const { assets: preparedAssets, skippedPaths } = prepareAssets(
656
+ opts.assets ?? [],
657
+ opts.extraFiles ?? {},
658
+ opts
659
+ );
296
660
  const profile = opts.embeddingProfile;
297
661
  const chunkerOpts = {
298
662
  unit: opts.chunkingProfile.unit,
@@ -302,13 +666,40 @@ async function compileKnowledgeCatalog(opts) {
302
666
  };
303
667
  const prepared = [];
304
668
  const seenIds = /* @__PURE__ */ new Set();
305
- for await (const doc of opts.documents) {
669
+ for await (const input of opts.documents) {
670
+ const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
671
+ ...location,
672
+ longitude: normalizeLongitude(location.longitude)
673
+ })).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
674
+ const doc = {
675
+ ...input,
676
+ locations,
677
+ markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
678
+ };
306
679
  if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
307
680
  seenIds.add(doc.id);
681
+ const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
682
+ const placements = [
683
+ {
684
+ topicId: leafTopicId,
685
+ ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
686
+ }
687
+ ];
688
+ const leaves = /* @__PURE__ */ new Set([leafTopicId]);
689
+ for (const raw of doc.tocReferences ?? []) {
690
+ const reference = KnowledgeTocReferenceSchema.parse(raw);
691
+ const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
692
+ if (leaves.has(topicId))
693
+ throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
694
+ leaves.add(topicId);
695
+ placements.push({ topicId, ordinal: reference.ordinal ?? null });
696
+ }
697
+ placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
308
698
  prepared.push({
309
699
  doc,
310
700
  topicPathKey: doc.topicPath.join("/"),
311
- leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
701
+ leafTopicId,
702
+ placements,
312
703
  metaJson: encodeDocumentMeta(doc),
313
704
  chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
314
705
  });
@@ -411,7 +802,9 @@ async function compileKnowledgeCatalog(opts) {
411
802
  });
412
803
  const topicDocCount = /* @__PURE__ */ new Map();
413
804
  for (const p of prepared) {
414
- topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
805
+ for (const placement of p.placements) {
806
+ topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
807
+ }
415
808
  }
416
809
  {
417
810
  const stmt = router.prepare(
@@ -435,6 +828,13 @@ async function compileKnowledgeCatalog(opts) {
435
828
  body_codec, body_blob)
436
829
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
437
830
  );
831
+ const insertLocation = router.prepare(
832
+ `INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
833
+ VALUES (?, ?, ?, ?, ?, ?)`
834
+ );
835
+ const insertPlacement = router.prepare(
836
+ "INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
837
+ );
438
838
  const insertAlias = router.prepare(
439
839
  "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
440
840
  );
@@ -473,6 +873,29 @@ async function compileKnowledgeCatalog(opts) {
473
873
  useBrotli ? "br" : "none",
474
874
  blob
475
875
  );
876
+ for (const location of p.doc.locations ?? []) {
877
+ const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
878
+ if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
879
+ throw new Error(
880
+ `document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
881
+ );
882
+ }
883
+ insertLocation.run(
884
+ p.doc.id,
885
+ location.id,
886
+ location.latitude,
887
+ location.longitude,
888
+ location.role,
889
+ provenance
890
+ );
891
+ }
892
+ for (const placement of p.placements) {
893
+ insertPlacement.run(
894
+ placement.topicId,
895
+ p.doc.id,
896
+ placement.ordinal === null ? null : BigInt(placement.ordinal)
897
+ );
898
+ }
476
899
  const aliases = [...new Set(p.doc.aliases ?? [])].sort();
477
900
  for (const alias of aliases) insertAlias.run(alias, p.doc.id);
478
901
  insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
@@ -705,7 +1128,7 @@ async function compileKnowledgeCatalog(opts) {
705
1128
  files.push({
706
1129
  path,
707
1130
  sizeBytes: bytes.length,
708
- sha256: createHash("sha256").update(bytes).digest("hex")
1131
+ sha256: createHash2("sha256").update(bytes).digest("hex")
709
1132
  });
710
1133
  }
711
1134
  for (const asset of preparedAssets) {
@@ -749,6 +1172,11 @@ async function compileKnowledgeCatalog(opts) {
749
1172
  assets: preparedAssets.length
750
1173
  },
751
1174
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
1175
+ spatial: spatialManifest(
1176
+ prepared.flatMap(
1177
+ (p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
1178
+ )
1179
+ ),
752
1180
  requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
753
1181
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
754
1182
  toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
@@ -844,58 +1272,6 @@ function encodeDocumentMeta(doc) {
844
1272
  }
845
1273
  return json;
846
1274
  }
847
- function prepareAssets(assets, extraFiles) {
848
- for (const path of Object.keys(extraFiles)) {
849
- if (path.startsWith("assets/")) {
850
- throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
851
- }
852
- }
853
- if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
854
- throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
855
- }
856
- const seen = /* @__PURE__ */ new Set();
857
- const prepared = [];
858
- let total = 0;
859
- for (const asset of assets) {
860
- const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
861
- if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
862
- const key = asset.path.toLowerCase();
863
- if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
864
- seen.add(key);
865
- if (asset.content === void 0 === (asset.absPath === void 0)) {
866
- throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
867
- }
868
- const bytes = asset.content ?? readFileSync(asset.absPath);
869
- if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
870
- throw new Error(
871
- `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
872
- );
873
- }
874
- total += bytes.byteLength;
875
- if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
876
- throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
877
- }
878
- const ext = assetExtension(asset.path);
879
- const expected = ext ? assetKindForExtension(ext) : null;
880
- const actual = sniffAssetType(bytes);
881
- if (expected === null || actual !== expected) {
882
- throw new Error(
883
- `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
884
- );
885
- }
886
- if (ext === "svg") {
887
- const problem = svgInertnessProblem(bytes);
888
- if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
889
- }
890
- prepared.push({
891
- path: asset.path,
892
- bytes,
893
- sizeBytes: bytes.byteLength,
894
- sha256: createHash("sha256").update(bytes).digest("hex")
895
- });
896
- }
897
- return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
898
- }
899
1275
  function assertAssetReferencesDeclared(prepared, assets) {
900
1276
  const declared = new Set(assets.map((a) => a.path));
901
1277
  for (const p of prepared) {
@@ -1110,7 +1486,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
1110
1486
  return assignment.map((s) => renumber.get(s));
1111
1487
  }
1112
1488
  function seedFor(catalogId, shardId) {
1113
- const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1489
+ const digest = createHash2("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
1114
1490
  return digest.readBigUInt64LE(24);
1115
1491
  }
1116
1492
  function makePrng(seed) {
@@ -1195,7 +1571,7 @@ function kMeans(samples, k, seed) {
1195
1571
  }
1196
1572
 
1197
1573
  // src/archive/read.ts
1198
- import { createHash as createHash2 } from "crypto";
1574
+ import { createHash as createHash3 } from "crypto";
1199
1575
  import { createWriteStream as createWriteStream2 } from "fs";
1200
1576
  import { mkdir as mkdir2 } from "fs/promises";
1201
1577
  import { createRequire as createRequire4 } from "module";
@@ -1485,7 +1861,7 @@ async function extractGezkVerified(archivePath, destDir) {
1485
1861
  pipeline(stream, createWriteStream2(target)).then(res, rej);
1486
1862
  return;
1487
1863
  }
1488
- const hasher = createHash2("sha256");
1864
+ const hasher = createHash3("sha256");
1489
1865
  let sizeBytes = 0;
1490
1866
  const verify = new Transform({
1491
1867
  transform(chunk, _encoding, callback) {
@@ -1608,7 +1984,7 @@ function readPragmaNumber(db, pragma) {
1608
1984
  }
1609
1985
 
1610
1986
  // src/reader/catalog-handle.ts
1611
- import { readFileSync as readFileSync2 } from "fs";
1987
+ import { readFileSync as readFileSync3 } from "fs";
1612
1988
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1613
1989
  import { brotliDecompressSync } from "zlib";
1614
1990
  import {
@@ -1689,14 +2065,14 @@ function hammingTopK(index, query, k) {
1689
2065
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1690
2066
  return hits;
1691
2067
  }
1692
- function asymmetricTopK(index, query, k) {
2068
+ function asymmetricTopK(index, query, k, eligibleChunkIds) {
1693
2069
  const { bits, bytesPerRow, rows } = index;
1694
2070
  if (Math.ceil(query.length / 8) !== bytesPerRow) {
1695
2071
  throw new Error(
1696
2072
  `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1697
2073
  );
1698
2074
  }
1699
- const limit = Math.min(k, rows);
2075
+ const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
1700
2076
  if (limit <= 0) return [];
1701
2077
  const lut = new Float32Array(bytesPerRow * 256);
1702
2078
  for (let p = 0; p < bytesPerRow; p++) {
@@ -1746,6 +2122,10 @@ function asymmetricTopK(index, query, k) {
1746
2122
  };
1747
2123
  let offset = 0;
1748
2124
  for (let r = 0; r < rows; r++) {
2125
+ if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
2126
+ offset += bytesPerRow;
2127
+ continue;
2128
+ }
1749
2129
  let score = 0;
1750
2130
  for (let p = 0; p < bytesPerRow; p++) {
1751
2131
  score += lut[p * 256 + bits[offset + p]];
@@ -1770,14 +2150,126 @@ function asymmetricTopK(index, query, k) {
1770
2150
  return hits;
1771
2151
  }
1772
2152
 
2153
+ // src/reader/spatial-index.ts
2154
+ import {
2155
+ KnowledgeLocationSchema,
2156
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
2157
+ KnowledgeRadiusSchema,
2158
+ locationDistanceMeters,
2159
+ locationInBounds,
2160
+ normalizeLongitude as normalizeLongitude2,
2161
+ radiusBounds,
2162
+ spatialManifest as spatialManifest2
2163
+ } from "@bendyline/gezk";
2164
+ var DocumentSpatialIndex = class {
2165
+ constructor(db, allowLegacy) {
2166
+ this.db = db;
2167
+ this.allowLegacy = allowLegacy;
2168
+ this.hasTable = Boolean(
2169
+ db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
2170
+ );
2171
+ }
2172
+ db;
2173
+ allowLegacy;
2174
+ hasTable;
2175
+ legacy;
2176
+ decode(row) {
2177
+ const provenance = row.provenance_json;
2178
+ if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
2179
+ throw new Error("location provenance exceeds 16384 bytes");
2180
+ return {
2181
+ documentId: String(row.document_id),
2182
+ location: KnowledgeLocationSchema.parse({
2183
+ id: row.location_id,
2184
+ latitude: row.latitude,
2185
+ longitude: row.longitude,
2186
+ role: row.role,
2187
+ ...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
2188
+ })
2189
+ };
2190
+ }
2191
+ legacyRows() {
2192
+ if (this.legacy) return this.legacy;
2193
+ this.legacy = [];
2194
+ if (!this.allowLegacy) return this.legacy;
2195
+ for (const row of this.db.prepare(
2196
+ "SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
2197
+ ).all()) {
2198
+ try {
2199
+ const meta = JSON.parse(String(row.meta_json));
2200
+ if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
2201
+ continue;
2202
+ const point = KnowledgeLocationSchema.safeParse({
2203
+ id: "qualla-coordinate",
2204
+ role: "subject",
2205
+ latitude: meta.coordinates.lat,
2206
+ longitude: meta.coordinates.lng,
2207
+ provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
2208
+ });
2209
+ if (point.success)
2210
+ this.legacy.push({
2211
+ documentId: String(row.id),
2212
+ location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
2213
+ });
2214
+ } catch {
2215
+ }
2216
+ }
2217
+ return this.legacy;
2218
+ }
2219
+ rows() {
2220
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
2221
+ }
2222
+ locations(documentId) {
2223
+ return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
2224
+ }
2225
+ integrity() {
2226
+ const rows = this.rows();
2227
+ const grouped = /* @__PURE__ */ new Map();
2228
+ for (const row of rows) {
2229
+ if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
2230
+ throw new Error("location longitude must be in [-180, 180)");
2231
+ const points = grouped.get(row.documentId) ?? [];
2232
+ points.push(row.location);
2233
+ grouped.set(row.documentId, points);
2234
+ if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
2235
+ throw new Error("location references missing document");
2236
+ }
2237
+ for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
2238
+ return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
2239
+ }
2240
+ matches(raw) {
2241
+ const radius = KnowledgeRadiusSchema.parse(raw);
2242
+ const boxes = radiusBounds(radius);
2243
+ const candidates = this.hasTable ? boxes.flatMap(
2244
+ (box) => this.db.prepare(`SELECT * FROM document_locations
2245
+ WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
2246
+ ) : this.legacyRows().filter(
2247
+ (row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
2248
+ );
2249
+ const matches = /* @__PURE__ */ new Map();
2250
+ for (const { documentId, location } of candidates) {
2251
+ const distanceMeters = locationDistanceMeters(radius, location);
2252
+ if (distanceMeters > radius.radiusMeters) continue;
2253
+ const previous = matches.get(documentId);
2254
+ if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
2255
+ matches.set(documentId, { distanceMeters, matchedLocation: location });
2256
+ }
2257
+ }
2258
+ return matches;
2259
+ }
2260
+ };
2261
+
1773
2262
  // src/reader/catalog-handle.ts
1774
2263
  var TOPIC_WALK_MAX_DEPTH = 32;
1775
2264
  var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
2265
+ var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
2266
+ var TITLE_ROUTE_SHARDS = 1;
1776
2267
  var CatalogHandle = class _CatalogHandle {
1777
2268
  constructor(rootDir, router) {
1778
2269
  this.rootDir = rootDir;
1779
2270
  this.router = router;
1780
2271
  this.schemaVersion = router.schemaVersion;
2272
+ this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
1781
2273
  this.meta = {};
1782
2274
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1783
2275
  this.meta[row.key] = row.value;
@@ -1789,6 +2281,9 @@ var CatalogHandle = class _CatalogHandle {
1789
2281
  router;
1790
2282
  connections = /* @__PURE__ */ new Map();
1791
2283
  bitIndexes = /* @__PURE__ */ new Map();
2284
+ /** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
2285
+ chunkTermRows = /* @__PURE__ */ new Map();
2286
+ spatial;
1792
2287
  assetsByPath = null;
1793
2288
  /** Lazily parsed from the profile echo; `undefined` until first needed. */
1794
2289
  binaryCenterCache;
@@ -1799,7 +2294,12 @@ var CatalogHandle = class _CatalogHandle {
1799
2294
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1800
2295
  static open(rootDir) {
1801
2296
  const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
1802
- return new _CatalogHandle(rootDir, router);
2297
+ try {
2298
+ return new _CatalogHandle(rootDir, router);
2299
+ } catch (error) {
2300
+ router.close();
2301
+ throw error;
2302
+ }
1803
2303
  }
1804
2304
  close() {
1805
2305
  for (const conn of this.connections.values()) conn.close();
@@ -1874,6 +2374,24 @@ var CatalogHandle = class _CatalogHandle {
1874
2374
  this.bitIndexes.set(shard.path, index);
1875
2375
  return index;
1876
2376
  }
2377
+ /**
2378
+ * Load the next shard's sign bits that are not resident yet, if they fit
2379
+ * the resident budget. False when nothing is left to load. One shard per
2380
+ * call, so a host can keep answering queries between shards while it warms
2381
+ * a large catalog, and never evicts a shard it loaded a moment ago.
2382
+ */
2383
+ prewarmNextShard() {
2384
+ const bytesPerRow = Math.ceil(this.dimensions() / 8);
2385
+ let resident = 0;
2386
+ for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
2387
+ for (const shard of this.shards) {
2388
+ if (this.bitIndexes.has(shard.path)) continue;
2389
+ if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
2390
+ this.shardBits(shard);
2391
+ return true;
2392
+ }
2393
+ return false;
2394
+ }
1877
2395
  /**
1878
2396
  * The stage-1 center the catalog's bits were derived with (a
1879
2397
  * `centered-sign` profile), or null for plain sign bits. Read from the
@@ -1923,9 +2441,13 @@ var CatalogHandle = class _CatalogHandle {
1923
2441
  get formatVersion() {
1924
2442
  return this.meta.format_version ?? "";
1925
2443
  }
1926
- documentColumns() {
2444
+ documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
1927
2445
  const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
1928
- return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2446
+ const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
2447
+ return columns.split(", ").map((column) => {
2448
+ const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
2449
+ return prefix ? `${prefix}.${column} AS ${column}` : column;
2450
+ }).join(", ");
1929
2451
  }
1930
2452
  documentOrder() {
1931
2453
  return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
@@ -1935,6 +2457,9 @@ var CatalogHandle = class _CatalogHandle {
1935
2457
  * 0.5 catalog every document sits at a root, so both counts agree there.
1936
2458
  */
1937
2459
  topics() {
2460
+ const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
2461
+ FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
2462
+ FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
1938
2463
  return this.router.db.prepare(
1939
2464
  `WITH RECURSIVE sub(root, id, depth) AS (
1940
2465
  SELECT id, id, 0 FROM topics
@@ -1943,8 +2468,7 @@ var CatalogHandle = class _CatalogHandle {
1943
2468
  WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1944
2469
  ),
1945
2470
  rollup AS (
1946
- SELECT sub.root AS id, SUM(t.document_count) AS total
1947
- FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
2471
+ ${rollup}
1948
2472
  )
1949
2473
  SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
1950
2474
  rollup.total AS total_document_count
@@ -1968,6 +2492,9 @@ var CatalogHandle = class _CatalogHandle {
1968
2492
  const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1969
2493
  const offset = Math.max(0, opts.offset ?? 0);
1970
2494
  const descendants = opts.descendants ?? true;
2495
+ if (this.schemaVersion >= 4 && opts.topicId) {
2496
+ return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
2497
+ }
1971
2498
  let scope = "";
1972
2499
  let where = "";
1973
2500
  const params = [];
@@ -1990,16 +2517,101 @@ var CatalogHandle = class _CatalogHandle {
1990
2517
  const documents = this.router.db.prepare(
1991
2518
  `${scope} SELECT ${this.documentColumns()}
1992
2519
  FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
1993
- ).all(...params, limit, offset).map(rowToDocumentMeta);
2520
+ ).all(...params, limit, offset).map((row) => this.documentMeta(row));
1994
2521
  return { documents, total };
1995
2522
  }
2523
+ /** A reference is a placement, not a duplicate result or a redirect page. */
2524
+ sharedDocumentsPage(topicId, descendants, limit, offset) {
2525
+ const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
2526
+ JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
2527
+ const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
2528
+ SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
2529
+ FROM topic_documents td JOIN topics t ON t.id = td.topic_id
2530
+ WHERE td.topic_id IN (SELECT id FROM sub)
2531
+ )`;
2532
+ const total = Number(
2533
+ this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
2534
+ );
2535
+ const documents = this.router.db.prepare(
2536
+ `${scope}, ranked AS (
2537
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
2538
+ ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
2539
+ ) SELECT ${this.documentColumns("d", "p")}
2540
+ FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
2541
+ ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
2542
+ ).all(topicId, limit, offset).map((row) => this.documentMeta(row));
2543
+ return { documents, total };
2544
+ }
2545
+ /** Integrity facts kept separate from browse counts, which include shared references. */
2546
+ tocIntegrity() {
2547
+ const count = (sql) => Number(this.router.db.prepare(sql).get().n);
2548
+ const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
2549
+ const incorrectCounts = count(
2550
+ `SELECT COUNT(*) AS n FROM topics t LEFT JOIN
2551
+ (SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
2552
+ WHERE t.document_count != COALESCE(r.n, 0)`
2553
+ );
2554
+ if (this.schemaVersion < 4)
2555
+ return {
2556
+ placements: count("SELECT COUNT(*) AS n FROM documents"),
2557
+ invalidReferences: 0,
2558
+ missingPrimary: 0,
2559
+ incorrectCounts,
2560
+ duplicateReferences: 0
2561
+ };
2562
+ return {
2563
+ placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
2564
+ invalidReferences: count(
2565
+ `SELECT COUNT(*) AS n FROM topic_documents r
2566
+ LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
2567
+ WHERE d.id IS NULL OR t.id IS NULL OR
2568
+ (r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
2569
+ ),
2570
+ missingPrimary: count(
2571
+ `SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
2572
+ SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
2573
+ ),
2574
+ incorrectCounts,
2575
+ duplicateReferences: count(
2576
+ "SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
2577
+ )
2578
+ };
2579
+ }
1996
2580
  getDocument(id) {
1997
2581
  const row = this.router.db.prepare(
1998
2582
  `SELECT ${this.documentColumns()}, body_codec, body_blob
1999
2583
  FROM documents WHERE id = ?`
2000
2584
  ).get(id);
2001
2585
  if (!row) return null;
2002
- return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
2586
+ return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
2587
+ }
2588
+ /**
2589
+ * One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
2590
+ * shard the document's chunks live in. Null when the document or the
2591
+ * passage is not in this catalog, which is what a fabricated or stale
2592
+ * citation looks like.
2593
+ */
2594
+ getChunk(documentId, chunkUid2) {
2595
+ const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
2596
+ if (!doc) return null;
2597
+ const shardId = Number(doc.shard_id);
2598
+ const shard = this.shards.find((s) => s.id === shardId);
2599
+ if (!shard) return null;
2600
+ const row = this.shardDb(shard).prepare(
2601
+ `SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
2602
+ FROM chunks WHERE chunk_uid = ? AND document_id = ?`
2603
+ ).get(chunkUid2, documentId);
2604
+ if (!row) return null;
2605
+ return {
2606
+ chunkUid: row.chunk_uid,
2607
+ documentId: row.document_id,
2608
+ title: row.title,
2609
+ headingPath: parseHeadingPath(row.heading_path),
2610
+ lineStart: Number(row.line_start),
2611
+ lineEnd: Number(row.line_end),
2612
+ text: row.text,
2613
+ shardId
2614
+ };
2003
2615
  }
2004
2616
  /** Documents whose `topic_id` names no declared topic (a validator check). */
2005
2617
  documentsWithUndeclaredTopic() {
@@ -2072,7 +2684,7 @@ var CatalogHandle = class _CatalogHandle {
2072
2684
  const index = /* @__PURE__ */ new Map();
2073
2685
  let raw;
2074
2686
  try {
2075
- raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2687
+ raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
2076
2688
  } catch (error) {
2077
2689
  if (error.code === "ENOENT") {
2078
2690
  this.assetsByPath = index;
@@ -2111,7 +2723,7 @@ var CatalogHandle = class _CatalogHandle {
2111
2723
  }
2112
2724
  let bytes;
2113
2725
  try {
2114
- bytes = readFileSync2(this.resolveCatalogPath(path));
2726
+ bytes = readFileSync3(this.resolveCatalogPath(path));
2115
2727
  } catch (error) {
2116
2728
  throw new CatalogOpenError(
2117
2729
  `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
@@ -2123,19 +2735,59 @@ var CatalogHandle = class _CatalogHandle {
2123
2735
  }
2124
2736
  return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
2125
2737
  }
2738
+ documentMeta(row) {
2739
+ const document = rowToDocumentMeta(row);
2740
+ const locations = this.spatial.locations(document.id);
2741
+ return locations.length ? { ...document, locations } : document;
2742
+ }
2743
+ documentLocations(documentId) {
2744
+ return this.spatial.locations(documentId);
2745
+ }
2746
+ documentLocationRows() {
2747
+ return this.spatial.rows();
2748
+ }
2749
+ spatialIntegrity() {
2750
+ return this.spatial.integrity();
2751
+ }
2752
+ spatialMatches(radius) {
2753
+ return this.spatial.matches(radius);
2754
+ }
2755
+ nearbyDocuments(radius, opts = {}) {
2756
+ const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
2757
+ const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
2758
+ const ordered = [...this.spatialMatches(radius)].sort(
2759
+ (a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
2760
+ );
2761
+ const get = this.router.db.prepare(
2762
+ `SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
2763
+ );
2764
+ return {
2765
+ total: ordered.length,
2766
+ documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
2767
+ ...this.documentMeta(get.get(id)),
2768
+ ...match
2769
+ }))
2770
+ };
2771
+ }
2772
+ shardsForDocuments(ids) {
2773
+ if (!ids.size) return [];
2774
+ return this.router.db.prepare(
2775
+ "SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
2776
+ ).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
2777
+ }
2126
2778
  // ── search ────────────────────────────────────────────────────────────────
2127
2779
  /**
2128
2780
  * Catalog-wide title/summary/alias FTS — always runs regardless of routing.
2129
2781
  * Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
2130
2782
  */
2131
- searchDocumentsFts(query, limit = 10) {
2132
- const match = ftsQuery(query);
2133
- if (!match) return [];
2783
+ searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
2134
2784
  try {
2135
- return documentFtsTopIds(this.router.db, match, limit, query).map((documentId, i) => ({
2136
- documentId,
2137
- rank: i
2138
- }));
2785
+ return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
2786
+ (documentId, i) => ({
2787
+ documentId,
2788
+ rank: i
2789
+ })
2790
+ );
2139
2791
  } catch {
2140
2792
  return [];
2141
2793
  }
@@ -2175,24 +2827,73 @@ var CatalogHandle = class _CatalogHandle {
2175
2827
  }
2176
2828
  return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
2177
2829
  }
2178
- /** Top-S shard ids within THIS catalog (single-catalog callers, tests). */
2179
- routeShards(queryVector, shardBudget) {
2830
+ /**
2831
+ * Top-S shard ids within THIS catalog (single-catalog callers, tests).
2832
+ * With the raw `query`, also the shard of the page it names by title
2833
+ * (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
2834
+ */
2835
+ routeShards(queryVector, shardBudget, query) {
2180
2836
  if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
2181
- return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
2837
+ const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
2838
+ if (query) {
2839
+ for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
2840
+ if (!routed.includes(named.shardId)) routed.push(named.shardId);
2841
+ }
2842
+ }
2843
+ return routed;
2844
+ }
2845
+ /**
2846
+ * Title-assisted routing: the shards holding the pages `query` names by
2847
+ * title (`namedTitleMatches`), most specific name first, at most `limit`
2848
+ * distinct shards. Centroid routing compares a question with averages of
2849
+ * whole shards, and e5 scores those in a band a few hundredths wide, so
2850
+ * the shard holding the very page a question names can rank 8th of 10:
2851
+ * "Who were the members of ABBA?" never reached the ABBA article in the
2852
+ * Wikipedia music catalog. Its title does — the vector arm scans that
2853
+ * shard too. Measured on the 10-shard music and film-tv catalogs, one such
2854
+ * shard lifted end-to-end recall@8 on "What is <title>?" questions from
2855
+ * 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
2856
+ *
2857
+ * `score` is the named match's specificity (fts-query.ts), comparable
2858
+ * across catalogs closely enough for the daemon to choose which catalogs'
2859
+ * shards to add when several name a page. Empty on any FTS error.
2860
+ */
2861
+ titleRouteShards(query, limit) {
2862
+ if (limit <= 0) return [];
2863
+ try {
2864
+ const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
2865
+ const out = [];
2866
+ for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
2867
+ const row = shardOf.get(named.documentId);
2868
+ if (!row) continue;
2869
+ const shardId = Number(row.shard_id);
2870
+ if (out.some((o) => o.shardId === shardId)) continue;
2871
+ out.push({ shardId, score: named.score });
2872
+ if (out.length >= limit) break;
2873
+ }
2874
+ return out.sort((a, b) => compareScores(b.score, a.score));
2875
+ } catch {
2876
+ return [];
2877
+ }
2182
2878
  }
2183
2879
  /**
2184
2880
  * Two-stage semantic search over the routed shards: sign-bit scan
2185
2881
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
2186
2882
  */
2187
2883
  searchSemantic(queryVector, opts = {}) {
2188
- const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
2189
- return this.searchShards(queryVector, shardIds, opts.finalK ?? RERANK_FINAL_K);
2884
+ const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
2885
+ return this.searchShards(
2886
+ queryVector,
2887
+ shardIds,
2888
+ opts.finalK ?? RERANK_FINAL_K,
2889
+ opts.allowedDocumentIds
2890
+ );
2190
2891
  }
2191
2892
  /**
2192
2893
  * The scan stage against an EXPLICIT shard list — what the daemon calls
2193
2894
  * after global cross-catalog routing has already spent the S budget.
2194
2895
  */
2195
- searchShards(queryVector, shardIds, finalK) {
2896
+ searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
2196
2897
  const center = this.binaryCenter();
2197
2898
  const scanQuery = center ? centerVector(queryVector, center) : queryVector;
2198
2899
  const hits = [];
@@ -2200,10 +2901,17 @@ var CatalogHandle = class _CatalogHandle {
2200
2901
  const shard = this.shards.find((s) => s.id === shardId);
2201
2902
  if (!shard) continue;
2202
2903
  const db = this.shardDb(shard);
2203
- const k = rerankK(finalK, shard.chunkCount);
2204
- const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
2205
- chunk_id: hit.chunkId
2206
- }));
2904
+ const eligible = allowedDocumentIds ? new Set(
2905
+ db.prepare(
2906
+ "SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
2907
+ ).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
2908
+ ) : void 0;
2909
+ const k = rerankK(finalK, eligible?.size ?? shard.chunkCount);
2910
+ const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k, eligible).map(
2911
+ (hit) => ({
2912
+ chunk_id: hit.chunkId
2913
+ })
2914
+ );
2207
2915
  if (candidates.length === 0) continue;
2208
2916
  const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
2209
2917
  const reranked = candidates.map((c) => {
@@ -2237,20 +2945,26 @@ var CatalogHandle = class _CatalogHandle {
2237
2945
  return hits;
2238
2946
  }
2239
2947
  /** Chunk-body FTS over the routed shards (explicit search only). */
2240
- searchChunksFts(query, shardIds, limitPerShard = 12) {
2241
- const match = ftsQuery(query);
2242
- if (!match) return [];
2948
+ searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
2243
2949
  const hits = [];
2244
2950
  for (const shardId of shardIds) {
2245
2951
  const shard = this.shards.find((s) => s.id === shardId);
2246
2952
  if (!shard) continue;
2247
2953
  const db = this.shardDb(shard);
2248
2954
  try {
2955
+ const match = selectiveFtsQuery(
2956
+ query,
2957
+ shard.chunkCount,
2958
+ (phrase) => this.chunkFtsRows(shard, db, phrase)
2959
+ );
2960
+ if (!match) return hits;
2249
2961
  const rows = db.prepare(
2250
2962
  `SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
2251
2963
  FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
2252
- WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
2253
- ).all(match, limitPerShard);
2964
+ WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
2965
+ ).all(
2966
+ ...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
2967
+ );
2254
2968
  for (const row of rows) {
2255
2969
  hits.push({
2256
2970
  chunkUid: row.chunk_uid,
@@ -2269,6 +2983,18 @@ var CatalogHandle = class _CatalogHandle {
2269
2983
  }
2270
2984
  return hits;
2271
2985
  }
2986
+ /** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
2987
+ chunkFtsRows(shard, db, phrase) {
2988
+ const key = `${shard.path}\0${phrase}`;
2989
+ let rows = this.chunkTermRows.get(key);
2990
+ if (rows === void 0) {
2991
+ if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
2992
+ const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
2993
+ rows = Number(row.c);
2994
+ this.chunkTermRows.set(key, rows);
2995
+ }
2996
+ return rows;
2997
+ }
2272
2998
  /**
2273
2999
  * Embedder-free integrity smoke: chunk 1's own bit vector must be its own
2274
3000
  * nearest neighbor — proves the shard's vectors load and scan. Returns
@@ -2322,7 +3048,6 @@ function parseHeadingPath(raw) {
2322
3048
  return [];
2323
3049
  }
2324
3050
  }
2325
- var ftsQuery = sanitizeFtsQuery;
2326
3051
 
2327
3052
  // src/reader/validate.ts
2328
3053
  import { readFile } from "fs/promises";
@@ -2334,6 +3059,7 @@ import {
2334
3059
  MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
2335
3060
  assetExtension as assetExtension2,
2336
3061
  assetKindForExtension as assetKindForExtension2,
3062
+ canonicalizeJson as canonicalizeJson2,
2337
3063
  isKnowledgeAssetPath as isKnowledgeAssetPath2,
2338
3064
  sniffAssetType as sniffAssetType2,
2339
3065
  svgInertnessProblem as svgInertnessProblem2
@@ -2409,6 +3135,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2409
3135
  handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
2410
3136
  `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
2411
3137
  );
3138
+ if (manifest.formatVersion === "0.7") {
3139
+ try {
3140
+ const spatial = handle.spatialIntegrity();
3141
+ check("document-locations-table", spatial.hasTable);
3142
+ check(
3143
+ "document-locations-summary",
3144
+ Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
3145
+ "spatial counts and coverage must match router points"
3146
+ );
3147
+ } catch (error) {
3148
+ check(
3149
+ "document-locations-integrity",
3150
+ false,
3151
+ error instanceof Error ? error.message : String(error)
3152
+ );
3153
+ }
3154
+ }
2412
3155
  const topics = handle.topics();
2413
3156
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
2414
3157
  check("topics-tree", ...topicTreeProblem(topics));
@@ -2446,11 +3189,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2446
3189
  manifest.files.some((f) => f.path === manifest.license.noticePath),
2447
3190
  `manifest.files lacks the declared notice ${manifest.license.noticePath}`
2448
3191
  );
3192
+ const toc = handle.tocIntegrity();
3193
+ check(
3194
+ "toc-references",
3195
+ toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
3196
+ `${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
3197
+ );
3198
+ check(
3199
+ "toc-counts",
3200
+ toc.incorrectCounts === 0,
3201
+ `${toc.incorrectCounts} incorrect per-topic counts`
3202
+ );
2449
3203
  const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
2450
3204
  check(
2451
3205
  "counts-documents",
2452
- handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === manifest.counts.documents,
2453
- `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
3206
+ handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
3207
+ `documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
2454
3208
  );
2455
3209
  check(
2456
3210
  "counts-shards",
@@ -2578,6 +3332,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
2578
3332
  );
2579
3333
  }
2580
3334
  }
3335
+ } catch (error) {
3336
+ check("catalog-structure", false, error instanceof Error ? error.message : String(error));
2581
3337
  } finally {
2582
3338
  handle.close();
2583
3339
  }
@@ -2686,13 +3442,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
2686
3442
  }
2687
3443
 
2688
3444
  // src/export/parquet.ts
2689
- import { createHash as createHash3 } from "crypto";
3445
+ import { createHash as createHash4 } from "crypto";
2690
3446
  import { createWriteStream as createWriteStream3 } from "fs";
2691
3447
  import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
2692
3448
  import { join as join4, resolve as resolve3 } from "path";
2693
3449
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
2694
3450
  import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
2695
- var PARQUET_EXPORT_VERSION = 3;
3451
+ var PARQUET_EXPORT_VERSION = 4;
2696
3452
  var DEFAULT_ROW_GROUP_SIZE = 32768;
2697
3453
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
2698
3454
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2739,6 +3495,34 @@ async function exportCatalogParquet(opts) {
2739
3495
  onProgress: opts.onProgress
2740
3496
  })
2741
3497
  );
3498
+ const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
3499
+ const locations = spatial.rows();
3500
+ if (spatial.hasTable || locations.length) {
3501
+ const path = join4(staging, "document-locations.ndjson");
3502
+ const out = ndjsonWriter(path);
3503
+ for (const { documentId, location } of locations)
3504
+ await out.write({
3505
+ document_id: documentId,
3506
+ location_id: location.id,
3507
+ latitude: location.latitude,
3508
+ longitude: location.longitude,
3509
+ role: location.role,
3510
+ provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
3511
+ });
3512
+ await out.close();
3513
+ const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
3514
+ const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
3515
+ files.push(
3516
+ await writeParquet(opts.duckdb, {
3517
+ outDir,
3518
+ file: "document-locations.parquet",
3519
+ table: "document_locations",
3520
+ rows: locations.length,
3521
+ sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
3522
+ onProgress: opts.onProgress
3523
+ })
3524
+ );
3525
+ }
2742
3526
  for (const shard of manifest.router.shards) {
2743
3527
  const tag = String(shard.id).padStart(3, "0");
2744
3528
  const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
@@ -3081,7 +3865,7 @@ async function writeParquet(cli, spec) {
3081
3865
  ...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
3082
3866
  rows: spec.rows,
3083
3867
  sizeBytes: (await stat2(target)).size,
3084
- sha256: createHash3("sha256").update(bytes).digest("hex")
3868
+ sha256: createHash4("sha256").update(bytes).digest("hex")
3085
3869
  };
3086
3870
  }
3087
3871
  async function findDuckdbBinary(env = process.env) {
@@ -3582,7 +4366,7 @@ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
3582
4366
  import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
3583
4367
 
3584
4368
  // src/embedding/artifact-verify.ts
3585
- import { createHash as createHash4 } from "crypto";
4369
+ import { createHash as createHash5 } from "crypto";
3586
4370
  import { createReadStream } from "fs";
3587
4371
  import { stat as stat3 } from "fs/promises";
3588
4372
  import { join as join5 } from "path";
@@ -3626,7 +4410,7 @@ async function fileDigest(path) {
3626
4410
  const memo = digestMemo.get(key);
3627
4411
  if (memo) return memo;
3628
4412
  const digest = await new Promise((resolve5, reject) => {
3629
- const hash = createHash4("sha256");
4413
+ const hash = createHash5("sha256");
3630
4414
  createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
3631
4415
  });
3632
4416
  if (digestMemo.size >= DIGEST_MEMO_MAX) {
@@ -3771,11 +4555,13 @@ async function createProfileEmbedder(profile, opts = {}) {
3771
4555
  const [pipe, tokenizer] = await Promise.all([
3772
4556
  transformers.pipeline("feature-extraction", profile.model.repo, {
3773
4557
  ...modelOptions,
4558
+ ...opts.localFilesOnly ? { local_files_only: true } : {},
3774
4559
  ...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
3775
4560
  ...progress
3776
4561
  }),
3777
4562
  transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
3778
4563
  revision: modelOptions.revision,
4564
+ ...opts.localFilesOnly ? { local_files_only: true } : {},
3779
4565
  ...progress
3780
4566
  })
3781
4567
  ]);
@@ -3817,52 +4603,20 @@ async function createProfileEmbedder(profile, opts = {}) {
3817
4603
  // src/markdown-adapter/load.ts
3818
4604
  import { existsSync } from "fs";
3819
4605
  import { readFile as readFile3, readdir } from "fs/promises";
3820
- import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
4606
+ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
3821
4607
  import {
3822
4608
  CatalogDocumentSchema,
3823
4609
  KnowledgeDocumentIdSchema,
3824
- KnowledgeOrdinalSchema,
4610
+ KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
4611
+ KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
4612
+ MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
3825
4613
  assetExtension as assetExtension3,
3826
4614
  formatKnowledgeUri,
3827
4615
  topicSortKeyForOrder
3828
4616
  } from "@bendyline/gezk";
3829
4617
 
3830
- // src/markdown-adapter/frontmatter.ts
3831
- import { parseDocument } from "yaml";
3832
- var FRONTMATTER_MAX_BYTES = 64 * 1024;
3833
- var OUTLINE_MAX_BYTES = 1024 * 1024;
3834
- function parseYaml(text, opts) {
3835
- const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
3836
- if (Buffer.byteLength(text, "utf8") > maxBytes) {
3837
- throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
3838
- }
3839
- const doc = parseDocument(text, {
3840
- version: "1.2",
3841
- schema: "core",
3842
- uniqueKeys: true,
3843
- logLevel: "silent"
3844
- });
3845
- const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
3846
- if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
3847
- return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
3848
- }
3849
- var OPEN = /^---[ \t]*\r?\n/;
3850
- var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
3851
- function parseMarkdownFrontMatter(source) {
3852
- const open = OPEN.exec(source);
3853
- if (!open) return { data: {}, body: source };
3854
- const lines = source.slice(open[0].length).split(/\r?\n/);
3855
- const closeAt = lines.findIndex((line) => CLOSE.test(line));
3856
- if (closeAt === -1) throw new Error("front matter is not closed with ---");
3857
- const yamlText = lines.slice(0, closeAt).join("\n");
3858
- const body = lines.slice(closeAt + 1).join("\n");
3859
- const parsed = parseYaml(yamlText, { what: "front matter" });
3860
- if (parsed === null || parsed === void 0) return { data: {}, body };
3861
- if (typeof parsed !== "object" || Array.isArray(parsed)) {
3862
- throw new Error("front matter must be a mapping");
3863
- }
3864
- return { data: parsed, body };
3865
- }
4618
+ // src/markdown-adapter/docfx.ts
4619
+ import { posix as posix2 } from "path";
3866
4620
 
3867
4621
  // src/markdown-adapter/outline.ts
3868
4622
  import { posix } from "path";
@@ -4159,14 +4913,408 @@ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
4159
4913
  return outline;
4160
4914
  }
4161
4915
 
4916
+ // src/markdown-adapter/docfx.ts
4917
+ var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
4918
+ var MARKDOWN_EXT2 = /\.(md|markdown)$/i;
4919
+ var TOC_FILE = /^toc\.(ya?ml|json|md)$/i;
4920
+ var LANDING_PAGE = /(^|\/)index\.(ya?ml|md)$/i;
4921
+ var WARNING_EXAMPLES = 5;
4922
+ function isRecord2(value) {
4923
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4924
+ }
4925
+ function normalizeFolder(value) {
4926
+ if (typeof value !== "string") return "";
4927
+ const cleaned = posix2.normalize(value.replace(/\\/g, "/")).replace(/^\.?\/+|\/+$/g, "");
4928
+ return cleaned === "." ? "" : cleaned;
4929
+ }
4930
+ function dirOf(file) {
4931
+ const dir = posix2.dirname(file);
4932
+ return dir === "." ? "" : dir;
4933
+ }
4934
+ function parentOf(dir) {
4935
+ return dir === "" ? null : dirOf(dir);
4936
+ }
4937
+ function isUnder(file, dir) {
4938
+ return dir === "" || file.startsWith(`${dir}/`);
4939
+ }
4940
+ function escapeRegExp(text) {
4941
+ return text.replace(/[.+^$()|[\]\\]/g, "\\$&");
4942
+ }
4943
+ function docfxGlob(pattern) {
4944
+ const p = pattern.replace(/\\/g, "/").replace(/^\.\//, "");
4945
+ let source = "";
4946
+ for (let i = 0; i < p.length; i++) {
4947
+ const ch = p[i];
4948
+ if (ch === "*") {
4949
+ if (p[i + 1] === "*") {
4950
+ if (p[i + 2] === "/") {
4951
+ source += "(?:.*/)?";
4952
+ i += 2;
4953
+ } else {
4954
+ source += ".*";
4955
+ i += 1;
4956
+ }
4957
+ } else source += "[^/]*";
4958
+ } else if (ch === "?") {
4959
+ source += "[^/]";
4960
+ } else if (ch === "{" && p.indexOf("}", i) > i) {
4961
+ const close = p.indexOf("}", i);
4962
+ source += `(?:${p.slice(i + 1, close).split(",").map(escapeRegExp).join("|")})`;
4963
+ i = close;
4964
+ } else {
4965
+ source += escapeRegExp(ch);
4966
+ }
4967
+ }
4968
+ return new RegExp(`^${source}$`, "i");
4969
+ }
4970
+ function globList(value) {
4971
+ const list = typeof value === "string" ? [value] : Array.isArray(value) ? value : [];
4972
+ return list.filter((g) => typeof g === "string" && g.trim() !== "").map(docfxGlob);
4973
+ }
4974
+ function parseDocfxJson(config, configFile = "docfx.json") {
4975
+ if (!isRecord2(config)) throw new Error(`${configFile}: not a JSON object`);
4976
+ const build = isRecord2(config.build) ? config.build : {};
4977
+ const content = [];
4978
+ for (const group of Array.isArray(build.content) ? build.content : []) {
4979
+ if (!isRecord2(group)) continue;
4980
+ const include = globList(group.files);
4981
+ if (include.length === 0) continue;
4982
+ content.push({
4983
+ src: normalizeFolder(group.src),
4984
+ dest: normalizeFolder(group.dest),
4985
+ include,
4986
+ exclude: globList(group.exclude)
4987
+ });
4988
+ }
4989
+ const meta = isRecord2(build.globalMetadata) ? build.globalMetadata : {};
4990
+ const breadcrumb = meta.breadcrumb_path;
4991
+ return {
4992
+ content,
4993
+ ...typeof breadcrumb === "string" && breadcrumb.trim() ? { breadcrumbPath: breadcrumb.trim() } : {}
4994
+ };
4995
+ }
4996
+ function docfxPublishes(project, file) {
4997
+ if (project.content.length === 0) return true;
4998
+ return project.content.some((group) => {
4999
+ if (!isUnder(file, group.src)) return false;
5000
+ const rel = group.src === "" ? file : file.slice(group.src.length + 1);
5001
+ return group.include.some((r) => r.test(rel)) && !group.exclude.some((r) => r.test(rel));
5002
+ });
5003
+ }
5004
+ function tocItems(doc) {
5005
+ if (Array.isArray(doc)) return doc;
5006
+ if (isRecord2(doc) && Array.isArray(doc.items)) return doc.items;
5007
+ return [];
5008
+ }
5009
+ function isBreadcrumbToc(doc) {
5010
+ let tocHref = false;
5011
+ const visit = (items) => {
5012
+ for (const item of items) {
5013
+ if (!isRecord2(item)) continue;
5014
+ if (typeof item.href === "string") return false;
5015
+ if (typeof item.tocHref === "string") tocHref = true;
5016
+ if (Array.isArray(item.items) && !visit(item.items)) return false;
5017
+ }
5018
+ return true;
5019
+ };
5020
+ return visit(tocItems(doc)) && tocHref;
5021
+ }
5022
+ function basePrefixes(url) {
5023
+ const segments = url.split("/").filter(Boolean);
5024
+ const out = ["/"];
5025
+ for (let i = 1; i < segments.length; i++) out.push(`/${segments.slice(0, i).join("/")}/`);
5026
+ return out;
5027
+ }
5028
+ function parseDocfxToc(input) {
5029
+ const outline = {
5030
+ format: "docfx",
5031
+ root: newOutlineTopic(""),
5032
+ consumed: input.files.filter((file) => TOC_FILE.test(posix2.basename(file))),
5033
+ warnings: []
5034
+ };
5035
+ const known = new Set(input.files.map((file) => file.normalize("NFC")));
5036
+ const groups = input.project?.content.length ? [...input.project.content].sort((a, b) => b.dest.length - a.dest.length) : [{ src: "", dest: "", include: [], exclude: [] }];
5037
+ const tocByDir = /* @__PURE__ */ new Map();
5038
+ for (const file of [...input.tocs.keys()].sort()) {
5039
+ if (/\.ya?ml$/i.test(file) && !tocByDir.has(dirOf(file))) tocByDir.set(dirOf(file), file);
5040
+ }
5041
+ const sourcesOf = (sitePath) => {
5042
+ const out = [];
5043
+ for (const group of groups) {
5044
+ if (group.dest !== "" && sitePath !== group.dest && !sitePath.startsWith(`${group.dest}/`))
5045
+ continue;
5046
+ const rest = sitePath.slice(group.dest.length).replace(/^\/+/, "");
5047
+ out.push(group.src && rest ? `${group.src}/${rest}` : group.src || rest);
5048
+ }
5049
+ return out;
5050
+ };
5051
+ const tocAt = (path) => {
5052
+ const trimmed = path.replace(/\/+$/, "");
5053
+ return TOC_FILE.test(posix2.basename(trimmed)) ? tocByDir.get(dirOf(trimmed)) : tocByDir.get(trimmed);
5054
+ };
5055
+ const pageAt = (path) => {
5056
+ const candidates = path === "" || path.endsWith("/") ? [`${path}index.md`] : MARKDOWN_EXT2.test(path) ? [path] : [`${path}.md`, `${path}/index.md`];
5057
+ return candidates.find((candidate) => known.has(candidate.normalize("NFC")));
5058
+ };
5059
+ const resolveSite = (url, base2) => {
5060
+ const path = url.replace(/[?#].*$/, "");
5061
+ let below;
5062
+ if (`${path}/` === base2) below = "";
5063
+ else if (path.startsWith(base2)) below = path.slice(base2.length);
5064
+ else return null;
5065
+ for (const source of sourcesOf(below)) {
5066
+ const page = pageAt(source);
5067
+ if (page) return { kind: "page", file: page };
5068
+ const toc = tocAt(source);
5069
+ if (toc) return { kind: "toc", file: toc };
5070
+ }
5071
+ return null;
5072
+ };
5073
+ let entry = input.entry;
5074
+ let base;
5075
+ const breadcrumb = input.project?.breadcrumbPath;
5076
+ if (!entry && breadcrumb) {
5077
+ for (const candidate of basePrefixes(breadcrumb)) {
5078
+ const target = resolveSite(breadcrumb, candidate);
5079
+ if (target?.kind === "toc") {
5080
+ entry = target.file;
5081
+ base = candidate;
5082
+ break;
5083
+ }
5084
+ }
5085
+ if (!entry) {
5086
+ outline.warnings.push(
5087
+ `docfx.json: breadcrumb_path '${breadcrumb}' names no toc.yml in the tree; ignored`
5088
+ );
5089
+ }
5090
+ }
5091
+ entry ??= tocByDir.get("");
5092
+ if (entry && base === void 0) {
5093
+ const urls = [];
5094
+ const collect = (items) => {
5095
+ for (const item of items) {
5096
+ if (!isRecord2(item)) continue;
5097
+ for (const key of ["tocHref", "topicHref", "href"]) {
5098
+ const value = item[key];
5099
+ if (typeof value === "string" && value.startsWith("/") && !value.startsWith("//")) {
5100
+ urls.push(value);
5101
+ }
5102
+ }
5103
+ if (Array.isArray(item.items)) collect(item.items);
5104
+ }
5105
+ };
5106
+ collect(tocItems(input.tocs.get(entry)));
5107
+ let best = { base: "/", hits: 0 };
5108
+ for (const candidate of [...new Set(urls.flatMap(basePrefixes))].sort(
5109
+ (a, b) => a.length - b.length
5110
+ )) {
5111
+ const hits = urls.filter((url) => resolveSite(url, candidate) !== null).length;
5112
+ if (hits > best.hits) best = { base: candidate, hits };
5113
+ }
5114
+ base = best.base;
5115
+ }
5116
+ const siteBase = base ?? "/";
5117
+ const classify = (href, tocDir) => {
5118
+ const value = href.trim();
5119
+ if (value === "" || value.startsWith("#") || HAS_SCHEME2.test(value) || value.startsWith("//"))
5120
+ return null;
5121
+ if (value.startsWith("/")) return resolveSite(value, siteBase);
5122
+ const path = resolveOutlinePath(tocDir, value);
5123
+ if (path === null) return null;
5124
+ if (MARKDOWN_EXT2.test(path)) return { kind: "page", file: path };
5125
+ const toc = tocAt(path);
5126
+ return toc ? { kind: "toc", file: toc } : null;
5127
+ };
5128
+ const crossLinks = /* @__PURE__ */ new WeakSet();
5129
+ const included = /* @__PURE__ */ new Set();
5130
+ const homes = /* @__PURE__ */ new Map();
5131
+ const pageEntry = (file, tocDir) => {
5132
+ const entry2 = { file };
5133
+ if (!isUnder(file, tocDir)) crossLinks.add(entry2);
5134
+ return entry2;
5135
+ };
5136
+ const walkItems = (items, tocFile, container, followTocHref) => {
5137
+ const tocDir = dirOf(tocFile);
5138
+ for (const item of items) {
5139
+ if (!isRecord2(item)) continue;
5140
+ const name = typeof item.name === "string" ? item.name.trim() : "";
5141
+ const href = typeof item.href === "string" ? classify(item.href, tocDir) : null;
5142
+ const topicHref = typeof item.topicHref === "string" ? classify(item.topicHref, tocDir) : null;
5143
+ const tocHref = followTocHref && typeof item.tocHref === "string" ? classify(item.tocHref, tocDir) : null;
5144
+ const page = [href, topicHref].find((target) => target?.kind === "page");
5145
+ const nested = [href, tocHref].find(
5146
+ (target) => target?.kind === "toc" && !included.has(target.file)
5147
+ );
5148
+ const children = Array.isArray(item.items) && item.items.length > 0 ? item.items : null;
5149
+ if (!nested && !children) {
5150
+ if (page) container.entries.push(pageEntry(page.file, tocDir));
5151
+ continue;
5152
+ }
5153
+ const topic = newOutlineTopic(
5154
+ name || (nested ? titleFromFolderName(posix2.basename(dirOf(nested.file))) : "Untitled")
5155
+ );
5156
+ if (!name && page) topic.titleFromFile = page.file;
5157
+ if (page) topic.entries.push(pageEntry(page.file, tocDir));
5158
+ if (nested) includeToc(nested.file, topic, false);
5159
+ if (children) walkItems(children, tocFile, topic, followTocHref);
5160
+ container.children.push(topic);
5161
+ }
5162
+ };
5163
+ const includeToc = (file, container, followTocHref) => {
5164
+ if (included.has(file)) return;
5165
+ included.add(file);
5166
+ const dir = dirOf(file);
5167
+ if (container !== outline.root && !homes.has(dir)) homes.set(dir, container);
5168
+ walkItems(tocItems(input.tocs.get(file)), file, container, followTocHref);
5169
+ };
5170
+ const homeOf = (dir) => {
5171
+ for (let d = dir; d !== null; d = parentOf(d)) {
5172
+ const home = homes.get(d);
5173
+ if (home) return home;
5174
+ }
5175
+ return void 0;
5176
+ };
5177
+ if (entry) {
5178
+ outline.entry = entry;
5179
+ includeToc(entry, outline.root, true);
5180
+ }
5181
+ const lone = outline.root.entries.length === 0 && outline.root.children.length === 1 ? outline.root.children[0] : void 0;
5182
+ const top = lone ?? outline.root;
5183
+ const orphans = [...input.tocs.keys()].filter((file) => !included.has(file) && /\.ya?ml$/i.test(file)).sort((a, b) => a.split("/").length - b.split("/").length || (a < b ? -1 : a > b ? 1 : 0));
5184
+ for (const file of orphans) {
5185
+ if (included.has(file)) continue;
5186
+ const doc = input.tocs.get(file);
5187
+ if (isBreadcrumbToc(doc)) continue;
5188
+ const parent = parentOf(dirOf(file));
5189
+ const topic = newOutlineTopic(orphanTopicName(file, doc));
5190
+ ((parent === null ? void 0 : homeOf(parent)) ?? top).children.push(topic);
5191
+ includeToc(file, topic, false);
5192
+ }
5193
+ const listed = /* @__PURE__ */ new Set();
5194
+ const collectListed = (topic) => {
5195
+ for (const entry2 of topic.entries) {
5196
+ if (!crossLinks.has(entry2)) listed.add(entry2.file.normalize("NFC"));
5197
+ }
5198
+ topic.children.forEach(collectListed);
5199
+ };
5200
+ collectListed(outline.root);
5201
+ const placed = /* @__PURE__ */ new Set();
5202
+ const missing = [];
5203
+ const duplicates = [];
5204
+ const settle = (topic) => {
5205
+ topic.entries = topic.entries.filter((entry2) => {
5206
+ const key = entry2.file.normalize("NFC");
5207
+ if (!known.has(key)) {
5208
+ missing.push(entry2.file);
5209
+ return false;
5210
+ }
5211
+ if (crossLinks.has(entry2) && listed.has(key)) return false;
5212
+ if (placed.has(key)) {
5213
+ if (!crossLinks.has(entry2)) duplicates.push(entry2.file);
5214
+ return false;
5215
+ }
5216
+ placed.add(key);
5217
+ return true;
5218
+ });
5219
+ topic.children.forEach(settle);
5220
+ };
5221
+ settle(outline.root);
5222
+ const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
5223
+ const homed = [];
5224
+ for (const file of input.files) {
5225
+ const key = file.normalize("NFC");
5226
+ if (placed.has(key) || consumed.has(key)) continue;
5227
+ const home = homeOf(dirOf(file));
5228
+ if (!home) continue;
5229
+ home.entries.push({ file });
5230
+ homed.push(file);
5231
+ }
5232
+ if (lone) {
5233
+ outline.root.entries = lone.entries;
5234
+ outline.root.children = lone.children;
5235
+ }
5236
+ assignOutlineOrders(outline.root);
5237
+ const report = (files, what) => {
5238
+ if (files.length === 0) return;
5239
+ const named = files.slice(0, WARNING_EXAMPLES).join(", ");
5240
+ const more = files.length > WARNING_EXAMPLES ? `, and ${files.length - WARNING_EXAMPLES} more` : "";
5241
+ outline.warnings.push(`${files.length} ${what}: ${named}${more}`);
5242
+ };
5243
+ report(
5244
+ [...new Set(missing)],
5245
+ "pages named by a toc.yml are not among the published Markdown files"
5246
+ );
5247
+ report([...new Set(duplicates)], "pages are listed more than once; the first place wins");
5248
+ report(homed, "pages are in no toc.yml; filed with their folder's section");
5249
+ return outline;
5250
+ }
5251
+ function orphanTopicName(file, doc) {
5252
+ for (const item of tocItems(doc)) {
5253
+ if (!isRecord2(item) || typeof item.name !== "string" || typeof item.href !== "string") continue;
5254
+ if (!LANDING_PAGE.test(item.href.replace(/[?#].*$/, ""))) continue;
5255
+ const name = item.name.trim().replace(/\s+(documentation|docs)$/i, "").trim();
5256
+ if (name) return name;
5257
+ }
5258
+ const folder = posix2.basename(dirOf(file));
5259
+ return folder ? titleFromFolderName(folder) : "Documentation";
5260
+ }
5261
+
5262
+ // src/markdown-adapter/frontmatter.ts
5263
+ import { parseDocument } from "yaml";
5264
+ var FRONTMATTER_MAX_BYTES = 64 * 1024;
5265
+ var OUTLINE_MAX_BYTES = 1024 * 1024;
5266
+ function parseYaml(text, opts) {
5267
+ const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
5268
+ if (Buffer.byteLength(text, "utf8") > maxBytes) {
5269
+ throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
5270
+ }
5271
+ const doc = parseDocument(text, {
5272
+ version: "1.2",
5273
+ schema: "core",
5274
+ uniqueKeys: true,
5275
+ logLevel: "silent"
5276
+ });
5277
+ const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
5278
+ if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
5279
+ return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
5280
+ }
5281
+ var OPEN = /^---[ \t]*\r?\n/;
5282
+ var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
5283
+ function parseMarkdownFrontMatter(source) {
5284
+ const open = OPEN.exec(source);
5285
+ if (!open) return { data: {}, body: source };
5286
+ const lines = source.slice(open[0].length).split(/\r?\n/);
5287
+ const closeAt = lines.findIndex((line) => CLOSE.test(line));
5288
+ if (closeAt === -1) throw new Error("front matter is not closed with ---");
5289
+ const yamlText = lines.slice(0, closeAt).join("\n");
5290
+ const body = lines.slice(closeAt + 1).join("\n");
5291
+ const parsed = parseYaml(yamlText, { what: "front matter" });
5292
+ if (parsed === null || parsed === void 0) return { data: {}, body };
5293
+ if (typeof parsed !== "object" || Array.isArray(parsed)) {
5294
+ throw new Error("front matter must be a mapping");
5295
+ }
5296
+ return { data: parsed, body };
5297
+ }
5298
+
4162
5299
  // src/markdown-adapter/load.ts
4163
5300
  var ROOT_TOPIC_ID = "general";
4164
5301
  var SUMMARY_MAX_CHARS = 280;
4165
5302
  var TOPIC_SIDECAR = "_topic.yaml";
4166
5303
  var HUGO_SECTION_PAGE = "_index";
5304
+ var DOCFX_JSON = "docfx.json";
5305
+ var WARNING_EXAMPLES2 = 20;
4167
5306
  var SECTION_PAGE_ORDINAL = -2147483648;
4168
- var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
4169
- function readFrontMatter(raw, file, hugo) {
5307
+ var RESERVED_KEYS = /* @__PURE__ */ new Set([
5308
+ "title",
5309
+ "summary",
5310
+ "aliases",
5311
+ "id",
5312
+ "order",
5313
+ "subcategory",
5314
+ "locations"
5315
+ ]);
5316
+ function readFrontMatter(raw, file, conventions) {
5317
+ const { hugo } = conventions;
4170
5318
  let data;
4171
5319
  let body;
4172
5320
  try {
@@ -4183,6 +5331,12 @@ function readFrontMatter(raw, file, hugo) {
4183
5331
  throw new Error(`${file}: front matter '${key}' must be a string`);
4184
5332
  return value.trim() || void 0;
4185
5333
  };
5334
+ if (data.locations !== void 0) {
5335
+ const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
5336
+ if (!locations.success)
5337
+ throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
5338
+ out.locations = locations.data;
5339
+ }
4186
5340
  out.title = str("title");
4187
5341
  out.summary = str("summary");
4188
5342
  out.id = str("id");
@@ -4195,7 +5349,7 @@ function readFrontMatter(raw, file, hugo) {
4195
5349
  throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
4196
5350
  }
4197
5351
  if (data.order !== void 0 && data.order !== null) {
4198
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
5352
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4199
5353
  if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
4200
5354
  out.order = order.data;
4201
5355
  }
@@ -4216,7 +5370,7 @@ function readFrontMatter(raw, file, hugo) {
4216
5370
  subcategory.title = record.title.trim();
4217
5371
  }
4218
5372
  if (record.order !== void 0 && record.order !== null) {
4219
- const order = KnowledgeOrdinalSchema.safeParse(record.order);
5373
+ const order = KnowledgeOrdinalSchema2.safeParse(record.order);
4220
5374
  if (!order.success)
4221
5375
  throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
4222
5376
  subcategory.order = order.data;
@@ -4227,12 +5381,12 @@ function readFrontMatter(raw, file, hugo) {
4227
5381
  if (data.draft === true) out.unpublished = "draft";
4228
5382
  else if (data.headless === true) out.unpublished = "headless";
4229
5383
  if (data.weight !== void 0 && data.weight !== null) {
4230
- const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
5384
+ const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
4231
5385
  if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
4232
5386
  if (weight.data > 0) out.weight = weight.data;
4233
5387
  }
4234
- out.description = str("description");
4235
5388
  }
5389
+ if (conventions.description) out.description = str("description");
4236
5390
  for (const [key, value] of Object.entries(data)) {
4237
5391
  if (RESERVED_KEYS.has(key) || value === void 0) continue;
4238
5392
  out.meta[key] = value;
@@ -4261,7 +5415,7 @@ function topicIdFor(name, taken) {
4261
5415
  taken.add(id);
4262
5416
  return id;
4263
5417
  }
4264
- async function walkMarkdownFiles(root) {
5418
+ async function walkFiles(root, matches) {
4265
5419
  const out = [];
4266
5420
  const visit = async (dir) => {
4267
5421
  const entries = (await readdir(dir, { withFileTypes: true })).sort(
@@ -4271,13 +5425,20 @@ async function walkMarkdownFiles(root) {
4271
5425
  if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
4272
5426
  const abs = join6(dir, entry.name);
4273
5427
  if (entry.isDirectory()) await visit(abs);
4274
- else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
5428
+ else if (entry.isFile() && matches(entry.name)) out.push(abs);
4275
5429
  }
4276
5430
  };
4277
5431
  await visit(root);
4278
5432
  return out;
4279
5433
  }
4280
- function isRecord2(value) {
5434
+ function walkMarkdownFiles(root) {
5435
+ return walkFiles(root, (name) => /\.(md|markdown)$/i.test(name));
5436
+ }
5437
+ function warnCapped(warn, items, each, rest) {
5438
+ for (const item of items.slice(0, WARNING_EXAMPLES2)) warn(each(item));
5439
+ if (items.length > WARNING_EXAMPLES2) warn(rest(items.length - WARNING_EXAMPLES2));
5440
+ }
5441
+ function isRecord3(value) {
4281
5442
  return typeof value === "object" && value !== null && !Array.isArray(value);
4282
5443
  }
4283
5444
  async function readTopicSidecar(dir) {
@@ -4286,14 +5447,14 @@ async function readTopicSidecar(dir) {
4286
5447
  const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
4287
5448
  const data = parseYaml(raw, { what: path });
4288
5449
  if (data === null || data === void 0) return {};
4289
- if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
5450
+ if (!isRecord3(data)) throw new Error(`${path}: must be a mapping`);
4290
5451
  const out = {};
4291
5452
  if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
4292
5453
  if (typeof data.description === "string" && data.description.trim()) {
4293
5454
  out.description = data.description.trim();
4294
5455
  }
4295
5456
  if (data.order !== void 0 && data.order !== null) {
4296
- const order = KnowledgeOrdinalSchema.safeParse(data.order);
5457
+ const order = KnowledgeOrdinalSchema2.safeParse(data.order);
4297
5458
  if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
4298
5459
  out.order = order.data;
4299
5460
  }
@@ -4320,7 +5481,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
4320
5481
  tolerateUnknownTags: true,
4321
5482
  maxAliasCount: 100
4322
5483
  });
4323
- const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
5484
+ const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
4324
5485
  return resolve4(dirname3(mkdocsPath), docsDir);
4325
5486
  }
4326
5487
  async function detectTableOfContents(rootDir, projectDir = rootDir) {
@@ -4335,6 +5496,10 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
4335
5496
  ])
4336
5497
  );
4337
5498
  if (mkdocs) return { format: "mkdocs", path: mkdocs };
5499
+ const docfx = firstExisting(
5500
+ [DOCFX_JSON, "toc.yml", "TOC.yml", "toc.yaml"].map((name) => join6(rootDir, name))
5501
+ );
5502
+ if (docfx) return { format: "docfx" };
4338
5503
  const files = await walkMarkdownFiles(rootDir);
4339
5504
  if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
4340
5505
  return { format: "hugo" };
@@ -4354,6 +5519,7 @@ async function loadOutline(rootDir, toc, files) {
4354
5519
  if (explicit && !existsSync(explicit)) {
4355
5520
  throw new Error(`${toc.format} table of contents not found: ${explicit}`);
4356
5521
  }
5522
+ if (toc.format === "docfx") return loadDocfxOutline(rootDir, explicit, files);
4357
5523
  if (toc.format === "gitbook") {
4358
5524
  const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
4359
5525
  const rel = insideRoot(rootDir, path2);
@@ -4396,16 +5562,68 @@ async function loadOutline(rootDir, toc, files) {
4396
5562
  });
4397
5563
  return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
4398
5564
  }
5565
+ async function loadDocfxOutline(rootDir, explicit, files) {
5566
+ const configPath = join6(rootDir, DOCFX_JSON);
5567
+ let project;
5568
+ if (existsSync(configPath)) {
5569
+ let config;
5570
+ try {
5571
+ config = JSON.parse(await readOutlineText(configPath));
5572
+ } catch (error) {
5573
+ throw new Error(`${DOCFX_JSON}: ${error instanceof Error ? error.message : String(error)}`);
5574
+ }
5575
+ project = parseDocfxJson(config, DOCFX_JSON);
5576
+ }
5577
+ const entry = explicit === void 0 ? void 0 : insideRoot(rootDir, explicit);
5578
+ if (entry === null) {
5579
+ throw new Error(`${basename(explicit)} must live inside the content root ${rootDir}`);
5580
+ }
5581
+ const readToc = async (abs, rel) => parseYaml(await readOutlineText(abs), {
5582
+ what: rel,
5583
+ maxBytes: OUTLINE_MAX_BYTES,
5584
+ tolerateUnknownTags: true,
5585
+ maxAliasCount: 100
5586
+ });
5587
+ const tocs = /* @__PURE__ */ new Map();
5588
+ const warnings = [];
5589
+ for (const abs of await walkFiles(rootDir, (name) => /^toc\.ya?ml$/i.test(name))) {
5590
+ const rel = insideRoot(rootDir, abs);
5591
+ try {
5592
+ tocs.set(rel, await readToc(abs, rel));
5593
+ } catch (error) {
5594
+ if (rel === entry) throw error;
5595
+ warnings.push(`${rel}: ${error instanceof Error ? error.message : String(error)}; skipped`);
5596
+ }
5597
+ }
5598
+ if (entry !== void 0 && explicit !== void 0 && !tocs.has(entry)) {
5599
+ tocs.set(entry, await readToc(explicit, entry));
5600
+ }
5601
+ const publishes = project ? (file) => docfxPublishes(project, file) : void 0;
5602
+ const outline = parseDocfxToc({
5603
+ tocs,
5604
+ files: publishes ? files.filter(publishes) : files,
5605
+ ...project ? { project } : {},
5606
+ ...entry !== void 0 ? { entry } : {}
5607
+ });
5608
+ outline.warnings.unshift(...warnings);
5609
+ const path = outline.entry ? join6(rootDir, ...outline.entry.split("/")) : void 0;
5610
+ return {
5611
+ outline,
5612
+ ...path ? { path } : project ? { path: configPath } : {},
5613
+ ...publishes ? { publishes } : {}
5614
+ };
5615
+ }
4399
5616
  var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
4400
- var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
5617
+ var HAS_SCHEME3 = /^[a-z][a-z0-9+.-]*:/i;
4401
5618
  function resolveInTree(ctx, target) {
4402
- const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
4403
- if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
5619
+ const fromRoot = ctx.tildeIsRoot && target.startsWith("~/");
5620
+ const resolved = posix3.normalize(fromRoot ? target.slice(2) : posix3.join(ctx.docDir, target));
5621
+ if (resolved === ".." || resolved.startsWith("../") || posix3.isAbsolute(resolved)) return null;
4404
5622
  return resolved === "." ? "" : resolved;
4405
5623
  }
4406
5624
  function rewriteTarget(ctx, isImage, rawTarget) {
4407
5625
  const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
4408
- if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
5626
+ if (target === "" || HAS_SCHEME3.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
4409
5627
  return null;
4410
5628
  }
4411
5629
  const hashAt = target.search(/[#?]/);
@@ -4468,6 +5686,7 @@ function rewriteLinks(ctx, markdown) {
4468
5686
  return line.replace(
4469
5687
  INLINE_LINK,
4470
5688
  (whole, bang, text, target, title) => {
5689
+ if (bang === "!" && ctx.skipImages) return text;
4471
5690
  const rewritten = rewriteTarget(ctx, bang === "!", target);
4472
5691
  return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
4473
5692
  }
@@ -4479,11 +5698,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
4479
5698
  });
4480
5699
  const toc = opts.toc ?? { format: "folders" };
4481
5700
  const hugo = toc.format === "hugo";
5701
+ const docfx = toc.format === "docfx";
5702
+ const conventions = { hugo, description: hugo || docfx };
4482
5703
  const ignore = new Set(opts.ignore ?? []);
4483
5704
  const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
4484
5705
  let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
4485
- const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
5706
+ const {
5707
+ outline,
5708
+ path: outlinePath,
5709
+ publishes
5710
+ } = await loadOutline(rootDir, toc, files.map(relOf));
4486
5711
  for (const message of outline?.warnings ?? []) warn(message);
5712
+ if (publishes) files = files.filter((abs) => publishes(relOf(abs)));
4487
5713
  if (outline) {
4488
5714
  const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
4489
5715
  files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
@@ -4534,26 +5760,24 @@ async function loadMarkdownCatalog(rootDir, opts) {
4534
5760
  const walk = (topic, chain) => {
4535
5761
  for (const entry of topic.entries) {
4536
5762
  const key = entry.file.normalize("NFC");
4537
- if (placements.has(key)) {
4538
- warn(
4539
- `${entry.file}: listed more than once in the table of contents; the first place wins`
4540
- );
4541
- } else {
4542
- placements.set(key, { chain, entry });
4543
- }
5763
+ const existing = placements.get(key) ?? [];
5764
+ existing.push({ chain, entry });
5765
+ placements.set(key, existing);
4544
5766
  }
4545
5767
  for (const child of topic.children) walk(child, [...chain, child]);
4546
5768
  };
4547
5769
  walk(outline.root, []);
4548
5770
  }
4549
5771
  const outlineTopicIds = /* @__PURE__ */ new Map();
5772
+ let outlineTooDeep = false;
4550
5773
  const ensureOutlineChain = (chain) => {
4551
5774
  if (chain.length === 0) {
4552
5775
  rootTopicUsed = true;
4553
5776
  return [rootTopicId];
4554
5777
  }
5778
+ if (chain.length > MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1) outlineTooDeep = true;
4555
5779
  const ids = [];
4556
- for (const node of chain) {
5780
+ for (const node of chain.slice(0, MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1)) {
4557
5781
  let id = outlineTopicIds.get(node);
4558
5782
  if (!id) {
4559
5783
  id = topicIdFor(node.name, takenTopicIds);
@@ -4594,12 +5818,13 @@ async function loadMarkdownCatalog(rootDir, opts) {
4594
5818
  const fileById = /* @__PURE__ */ new Map();
4595
5819
  const idByRel = /* @__PURE__ */ new Map();
4596
5820
  const seenRel = /* @__PURE__ */ new Set();
5821
+ const unlisted = [];
4597
5822
  for (const abs of files) {
4598
5823
  const rel = relOf(abs);
4599
5824
  const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
4600
5825
  const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
4601
5826
  const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
4602
- const fm = readFrontMatter(raw, rel, hugo);
5827
+ const fm = readFrontMatter(raw, rel, conventions);
4603
5828
  if (fm.unpublished) {
4604
5829
  warn(`${rel}: skipped, ${fm.unpublished} page`);
4605
5830
  continue;
@@ -4613,17 +5838,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
4613
5838
  fileById.set(id, rel);
4614
5839
  idByRel.set(relNoExt, id);
4615
5840
  const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
4616
- let title = fm.title ?? firstHeading(fm.body) ?? stem2;
5841
+ let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
4617
5842
  let ordinal = fm.order;
4618
5843
  let topicPath;
4619
- const placement = placements.get(rel.normalize("NFC"));
5844
+ const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
5845
+ const placement = documentPlacements[0];
4620
5846
  if (placement) {
4621
5847
  topicPath = ensureOutlineChain(placement.chain);
4622
5848
  if (placement.entry.title) title = placement.entry.title;
4623
5849
  if (placement.entry.order !== void 0) ordinal = placement.entry.order;
4624
5850
  } else {
4625
- if (outline)
4626
- warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
5851
+ if (outline) unlisted.push(rel);
4627
5852
  topicPath = await ensureTopicChain(relDir);
4628
5853
  if (hugo && ordinal === void 0) ordinal = fm.weight;
4629
5854
  }
@@ -4647,7 +5872,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
4647
5872
  const parentId = topicPath[topicPath.length - 1];
4648
5873
  topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
4649
5874
  }
4650
- const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
5875
+ const tocReferences = [];
5876
+ const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
5877
+ for (const extra of documentPlacements.slice(1)) {
5878
+ let path = ensureOutlineChain(extra.chain);
5879
+ if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
5880
+ const leaf = path.at(-1);
5881
+ if (placedLeaves.has(leaf)) continue;
5882
+ placedLeaves.add(leaf);
5883
+ const order = extra.entry.order ?? fm.order;
5884
+ tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
5885
+ }
5886
+ const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
4651
5887
  loaded.push({
4652
5888
  rel,
4653
5889
  file: rel,
@@ -4658,6 +5894,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
4658
5894
  ...summary ? { summary } : {},
4659
5895
  language: opts.language,
4660
5896
  topicPath,
5897
+ ...tocReferences.length ? { tocReferences } : {},
5898
+ ...fm.locations ? { locations: fm.locations } : {},
4661
5899
  markdown: fm.body,
4662
5900
  ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
4663
5901
  ...ordinal !== void 0 ? { ordinal } : {},
@@ -4679,12 +5917,22 @@ async function loadMarkdownCatalog(rootDir, opts) {
4679
5917
  }
4680
5918
  };
4681
5919
  nameFromPages(outline.root);
4682
- for (const [file, placement] of placements) {
4683
- if (!seenRel.has(file)) {
4684
- warn(
4685
- `${placement.entry.file}: named by the table of contents but not found among the Markdown files`
4686
- );
4687
- }
5920
+ warnCapped(
5921
+ warn,
5922
+ unlisted,
5923
+ (file) => `${file}: not in the ${outline.format} table of contents; filed under its folder`,
5924
+ (more) => `${more} more files are not in the ${outline.format} table of contents; filed under their folders`
5925
+ );
5926
+ warnCapped(
5927
+ warn,
5928
+ [...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
5929
+ (file) => `${file}: named by the table of contents but not found among the Markdown files`,
5930
+ (more) => `${more} more files named by the table of contents are not among the Markdown files`
5931
+ );
5932
+ if (outlineTooDeep) {
5933
+ warn(
5934
+ `the table of contents nests deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1} levels; deeper sections are folded into their ancestors`
5935
+ );
4688
5936
  }
4689
5937
  }
4690
5938
  if (rootTopicUsed) {
@@ -4702,6 +5950,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
4702
5950
  }
4703
5951
  const assets = /* @__PURE__ */ new Map();
4704
5952
  const documents = [];
5953
+ const linkWarnings = [];
4705
5954
  for (const entry of loaded) {
4706
5955
  const ctx = {
4707
5956
  rootDir,
@@ -4710,8 +5959,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
4710
5959
  idByRel,
4711
5960
  uri: opts.uri,
4712
5961
  assets,
4713
- missingAssets: opts.missingAssets ?? "error",
4714
- warn
5962
+ // A docfx docset draws on dependent repositories (`~/reusable-content`)
5963
+ // that a checkout never holds, so a missing image is expected there.
5964
+ missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
5965
+ skipImages: opts.skipImages ?? false,
5966
+ tildeIsRoot: docfx,
5967
+ warn: (message) => linkWarnings.push(message)
4715
5968
  };
4716
5969
  documents.push(
4717
5970
  CatalogDocumentSchema.parse({
@@ -4720,6 +5973,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
4720
5973
  })
4721
5974
  );
4722
5975
  }
5976
+ warnCapped(
5977
+ warn,
5978
+ linkWarnings,
5979
+ (message) => message,
5980
+ (more) => `${more} more link and image warnings`
5981
+ );
4723
5982
  return {
4724
5983
  topics,
4725
5984
  documents,
@@ -4840,7 +6099,7 @@ function newerRegistryEntries(registry, installed) {
4840
6099
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
4841
6100
 
4842
6101
  // src/signatures/jcs.ts
4843
- import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
6102
+ import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
4844
6103
  export {
4845
6104
  BGE_SMALL_EN_V15_1,
4846
6105
  BODY_CODEC_MIN_BYTES,
@@ -4880,6 +6139,7 @@ export {
4880
6139
  MULTILINGUAL_E5_SMALL_2,
4881
6140
  PARQUET_EXPORT_VERSION,
4882
6141
  PARQUET_REPORT_PATH,
6142
+ QUERY_STOP_WORDS,
4883
6143
  README_PATH,
4884
6144
  RERANK_CANDIDATES_MAX,
4885
6145
  RERANK_CANDIDATES_MIN,
@@ -4892,13 +6152,14 @@ export {
4892
6152
  SHARD_MAX_CHUNKS,
4893
6153
  SHARD_TARGET_CHUNKS,
4894
6154
  SOURCE_NOTICES_PATH,
6155
+ TITLE_ROUTE_SHARDS,
4895
6156
  TRANSFORMERS_PEER_RANGE,
4896
6157
  ZIP_FIXED_MTIME,
4897
6158
  assertDuckdbCli,
4898
6159
  assetReferences,
4899
6160
  assignOutlineOrders,
4900
6161
  asymmetricTopK,
4901
- canonicalizeJson2 as canonicalizeJson,
6162
+ canonicalizeJson3 as canonicalizeJson,
4902
6163
  centerVector,
4903
6164
  chunkContentHash,
4904
6165
  chunkMarkdownProfile,