@bendyline/gezel-knowledge 1.1.0 → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -2,17 +2,24 @@
2
2
  import {
3
3
  BODY_CODEC_MIN_BYTES,
4
4
  GEZK_APPLICATION_ID,
5
+ GEZK_FORMAT_GENERATIONS,
5
6
  GEZK_FORMAT_VERSION,
6
7
  GEZK_INDEX_SCHEMA_VERSION,
7
8
  GEZK_MIME_TYPE,
9
+ GEZK_SUPPORTED_FORMAT_VERSIONS,
10
+ GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
8
11
  LICENSE_NOTICE_PATH,
9
12
  MANIFEST_PATH,
10
13
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
14
+ MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
15
+ MAX_KNOWLEDGE_TOPIC_DEPTH,
11
16
  MIMETYPE_PATH,
12
17
  README_PATH,
13
18
  ROUTER_DB_PATH,
14
19
  SOURCE_NOTICES_PATH,
15
- ZIP_FIXED_MTIME
20
+ ZIP_FIXED_MTIME,
21
+ isSupportedFormatVersion,
22
+ isSupportedIndexSchemaVersion
16
23
  } from "@bendyline/gezk";
17
24
  var SHARD_TARGET_CHUNKS = 2e5;
18
25
  var SHARD_MAX_CHUNKS = 25e4;
@@ -23,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
23
30
  var ROUTE_SHARDS_PROACTIVE = 3;
24
31
  var ROUTE_SHARDS_EXPLICIT = 6;
25
32
  var RERANK_FINAL_K = 24;
33
+ var RERANK_CANDIDATES_MIN = 1024;
34
+ var RERANK_CANDIDATES_MAX = 4096;
26
35
  function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
27
- return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
36
+ return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
28
37
  }
29
38
 
30
39
  // src/format/ddl.ts
@@ -35,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
35
44
  import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
36
45
 
37
46
  // src/format/quantize.ts
38
- import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
47
+ import {
48
+ centerVector,
49
+ l2Normalize,
50
+ quantizeBinary,
51
+ quantizeBinaryForProfile,
52
+ quantizeInt8,
53
+ rerankScore
54
+ } from "@bendyline/gezk";
39
55
 
40
56
  // src/chunking/markdown-chunker.ts
41
57
  var HEADING = /^(#{1,6})\s+(.*)$/;
@@ -165,11 +181,23 @@ function splitOversizeLine(line, opts) {
165
181
 
166
182
  // src/compiler/compile.ts
167
183
  import { createHash } from "crypto";
168
- import { mkdirSync, rmSync, statSync } from "fs";
184
+ import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
169
185
  import { stat } from "fs/promises";
170
186
  import { join } from "path";
171
187
  import { brotliCompressSync, constants as zlibConstants } from "zlib";
172
- import { GEZK_MANIFEST_KIND, KnowledgeCatalogManifestSchema } from "@bendyline/gezk";
188
+ import {
189
+ GEZK_MANIFEST_KIND,
190
+ KnowledgeAssetPathSchema,
191
+ KnowledgeCatalogManifestSchema,
192
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
193
+ MAX_KNOWLEDGE_ASSET_BYTES,
194
+ MAX_KNOWLEDGE_ASSET_COUNT,
195
+ assetExtension,
196
+ assetKindForExtension,
197
+ canonicalizeJson,
198
+ sniffAssetType,
199
+ svgInertnessProblem
200
+ } from "@bendyline/gezk";
173
201
 
174
202
  // src/archive/write.ts
175
203
  import { createWriteStream } from "fs";
@@ -195,10 +223,10 @@ async function writeGezkArchive(outputPath, entries) {
195
223
  else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
196
224
  else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
197
225
  }
198
- await new Promise((resolve4, reject) => {
226
+ await new Promise((resolve5, reject) => {
199
227
  const out = createWriteStream(outputPath);
200
228
  out.on("error", reject);
201
- out.on("close", () => resolve4());
229
+ out.on("close", () => resolve5());
202
230
  zip.outputStream.on("error", reject);
203
231
  zip.outputStream.pipe(out);
204
232
  zip.end();
@@ -250,7 +278,8 @@ async function compileKnowledgeCatalog(opts) {
250
278
  "a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
251
279
  );
252
280
  }
253
- const topicIds = new Set(opts.topics.map((t) => t.id));
281
+ const topicById = assertTopicForest(opts.topics);
282
+ const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
254
283
  const profile = opts.embeddingProfile;
255
284
  const chunkerOpts = {
256
285
  unit: opts.chunkingProfile.unit,
@@ -263,13 +292,11 @@ async function compileKnowledgeCatalog(opts) {
263
292
  for await (const doc of opts.documents) {
264
293
  if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
265
294
  seenIds.add(doc.id);
266
- const rootTopic = doc.topicPath[0];
267
- if (!rootTopic || !topicIds.has(rootTopic)) {
268
- throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
269
- }
270
295
  prepared.push({
271
296
  doc,
272
297
  topicPathKey: doc.topicPath.join("/"),
298
+ leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
299
+ metaJson: encodeDocumentMeta(doc),
273
300
  chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
274
301
  });
275
302
  }
@@ -336,8 +363,7 @@ async function compileKnowledgeCatalog(opts) {
336
363
  });
337
364
  const topicDocCount = /* @__PURE__ */ new Map();
338
365
  for (const p of prepared) {
339
- const t = p.doc.topicPath[0];
340
- topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
366
+ topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
341
367
  }
342
368
  {
343
369
  const stmt = router.prepare(
@@ -356,9 +382,10 @@ async function compileKnowledgeCatalog(opts) {
356
382
  }
357
383
  {
358
384
  const insertDoc = router.prepare(
359
- `INSERT INTO documents (id, title, slug, summary, language, topic_id, shard_id, chunk_count,
360
- source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob)
361
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
385
+ `INSERT INTO documents (id, title, slug, summary, language, topic_id, ordinal, shard_id,
386
+ chunk_count, source_url, source_revision, source_updated_at, attribution_json, meta_json,
387
+ body_codec, body_blob)
388
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
362
389
  );
363
390
  const insertAlias = router.prepare(
364
391
  "INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
@@ -386,13 +413,15 @@ async function compileKnowledgeCatalog(opts) {
386
413
  p.doc.slug || documentSlug(p.doc.title),
387
414
  p.doc.summary ?? null,
388
415
  p.doc.language,
389
- p.doc.topicPath[0],
416
+ p.leafTopicId,
417
+ p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
390
418
  BigInt(shardOf.get(p.doc.id)),
391
419
  BigInt(p.chunks.length),
392
420
  p.doc.sourceUrl ?? null,
393
421
  p.doc.sourceRevision ?? null,
394
422
  p.doc.sourceUpdatedAt ?? null,
395
423
  p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
424
+ p.metaJson,
396
425
  useBrotli ? "br" : "none",
397
426
  blob
398
427
  );
@@ -449,7 +478,7 @@ async function compileKnowledgeCatalog(opts) {
449
478
  }
450
479
  const row = pendingRows[i];
451
480
  const stmts = vecStmtsFor(shardDb(row.shardId));
452
- stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinary(unit)));
481
+ stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
453
482
  stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
454
483
  const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
455
484
  void shardTotal;
@@ -468,7 +497,7 @@ async function compileKnowledgeCatalog(opts) {
468
497
  const db = shardDb(shardId);
469
498
  shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
470
499
  const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
471
- topics.add(p.doc.topicPath[0]);
500
+ topics.add(p.leafTopicId);
472
501
  shardTopicIds.set(shardId, topics);
473
502
  const insertChunk = db.prepare(
474
503
  `INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
@@ -631,6 +660,10 @@ async function compileKnowledgeCatalog(opts) {
631
660
  sha256: createHash("sha256").update(bytes).digest("hex")
632
661
  });
633
662
  }
663
+ for (const asset of preparedAssets) {
664
+ files.push({ path: asset.path, sizeBytes: asset.sizeBytes, sha256: asset.sha256 });
665
+ }
666
+ assertAssetReferencesDeclared(prepared, preparedAssets);
634
667
  const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
635
668
  kind: GEZK_MANIFEST_KIND,
636
669
  formatVersion: GEZK_FORMAT_VERSION,
@@ -649,13 +682,24 @@ async function compileKnowledgeCatalog(opts) {
649
682
  },
650
683
  embedding: profile,
651
684
  chunking: opts.chunkingProfile,
652
- topics: opts.topics.map((t) => ({ id: t.id, name: t.name })),
685
+ topics: opts.topics.map((t) => ({
686
+ id: t.id,
687
+ name: t.name,
688
+ ...t.parentId ? { parentId: t.parentId } : {},
689
+ sortKey: t.sortKey ?? t.name.toLowerCase(),
690
+ ...t.description ? { description: t.description } : {}
691
+ })),
653
692
  router: {
654
693
  shardTargetChunks: shardTarget,
655
694
  shards: routerShardStats.sort((a, b) => a.id - b.id),
656
695
  totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
657
696
  },
658
- counts: { documents: prepared.length, chunks: totalChunks, shards: shardCount },
697
+ counts: {
698
+ documents: prepared.length,
699
+ chunks: totalChunks,
700
+ shards: shardCount,
701
+ assets: preparedAssets.length
702
+ },
659
703
  files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
660
704
  requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
661
705
  ...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
@@ -672,7 +716,8 @@ async function compileKnowledgeCatalog(opts) {
672
716
  ...Object.entries(extraFiles).map(([path, content]) => ({
673
717
  path,
674
718
  content: Buffer.from(content, "utf8")
675
- }))
719
+ })),
720
+ ...preparedAssets.map((asset) => ({ path: asset.path, content: asset.bytes }))
676
721
  ];
677
722
  await writeGezkArchive(opts.outputPath, archiveEntries);
678
723
  const archiveBytes = (await stat(opts.outputPath)).size;
@@ -688,6 +733,134 @@ async function compileKnowledgeCatalog(opts) {
688
733
  closeAll();
689
734
  }
690
735
  }
736
+ function assertTopicForest(topics) {
737
+ const byId = /* @__PURE__ */ new Map();
738
+ for (const topic of topics) {
739
+ if (byId.has(topic.id)) throw new Error(`duplicate topic id: ${topic.id}`);
740
+ byId.set(topic.id, topic);
741
+ }
742
+ for (const topic of topics) {
743
+ const seen = /* @__PURE__ */ new Set();
744
+ let depth = 0;
745
+ let current = topic;
746
+ while (current) {
747
+ if (seen.has(current.id)) throw new Error(`topic '${topic.id}' sits in a parent cycle`);
748
+ seen.add(current.id);
749
+ depth += 1;
750
+ if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
751
+ throw new Error(`topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`);
752
+ }
753
+ if (current.parentId === void 0) break;
754
+ const parent = byId.get(current.parentId);
755
+ if (!parent) {
756
+ throw new Error(`topic '${current.id}' names an undeclared parent '${current.parentId}'`);
757
+ }
758
+ current = parent;
759
+ }
760
+ }
761
+ return byId;
762
+ }
763
+ function assertTopicPath(topicById, docId, path) {
764
+ for (let i = 0; i < path.length; i++) {
765
+ const id = path[i];
766
+ const topic = topicById.get(id);
767
+ if (!topic) {
768
+ throw new Error(
769
+ `document ${docId} names unknown topic '${id}' (segment ${i} of ${path.join("/")})`
770
+ );
771
+ }
772
+ const expectedParent = i === 0 ? void 0 : path[i - 1];
773
+ if (topic.parentId !== expectedParent) {
774
+ throw new Error(
775
+ `document ${docId}: topic path ${path.join("/")} does not follow the declared tree at '${id}' (its parent is '${topic.parentId ?? "(root)"}')`
776
+ );
777
+ }
778
+ }
779
+ return path[path.length - 1];
780
+ }
781
+ function encodeDocumentMeta(doc) {
782
+ if (!doc.meta || Object.keys(doc.meta).length === 0) return null;
783
+ let json;
784
+ try {
785
+ json = canonicalizeJson(doc.meta);
786
+ } catch (error) {
787
+ throw new Error(
788
+ `document ${doc.id} has metadata that cannot be serialized: ${error instanceof Error ? error.message : String(error)}`
789
+ );
790
+ }
791
+ const bytes = Buffer.byteLength(json, "utf8");
792
+ if (bytes > MAX_KNOWLEDGE_DOCUMENT_META_BYTES) {
793
+ throw new Error(
794
+ `document ${doc.id} has ${bytes} bytes of metadata; the limit is ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES}`
795
+ );
796
+ }
797
+ return json;
798
+ }
799
+ function prepareAssets(assets, extraFiles) {
800
+ for (const path of Object.keys(extraFiles)) {
801
+ if (path.startsWith("assets/")) {
802
+ throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
803
+ }
804
+ }
805
+ if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
806
+ throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
807
+ }
808
+ const seen = /* @__PURE__ */ new Set();
809
+ const prepared = [];
810
+ let total = 0;
811
+ for (const asset of assets) {
812
+ const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
813
+ if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
814
+ const key = asset.path.toLowerCase();
815
+ if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
816
+ seen.add(key);
817
+ if (asset.content === void 0 === (asset.absPath === void 0)) {
818
+ throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
819
+ }
820
+ const bytes = asset.content ?? readFileSync(asset.absPath);
821
+ if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
822
+ throw new Error(
823
+ `asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
824
+ );
825
+ }
826
+ total += bytes.byteLength;
827
+ if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
828
+ throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
829
+ }
830
+ const ext = assetExtension(asset.path);
831
+ const expected = ext ? assetKindForExtension(ext) : null;
832
+ const actual = sniffAssetType(bytes);
833
+ if (expected === null || actual !== expected) {
834
+ throw new Error(
835
+ `asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
836
+ );
837
+ }
838
+ if (ext === "svg") {
839
+ const problem = svgInertnessProblem(bytes);
840
+ if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
841
+ }
842
+ prepared.push({
843
+ path: asset.path,
844
+ bytes,
845
+ sizeBytes: bytes.byteLength,
846
+ sha256: createHash("sha256").update(bytes).digest("hex")
847
+ });
848
+ }
849
+ return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
850
+ }
851
+ function assertAssetReferencesDeclared(prepared, assets) {
852
+ const declared = new Set(assets.map((a) => a.path));
853
+ for (const p of prepared) {
854
+ for (const match of p.doc.markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
855
+ const target = match[1];
856
+ if (!declared.has(target)) {
857
+ throw new Error(
858
+ `document ${p.doc.id} references '${target}', which is not among the catalog's assets`
859
+ );
860
+ }
861
+ }
862
+ }
863
+ }
691
864
  function createCatalogDb(absPath) {
692
865
  const db = new DatabaseSync(absPath);
693
866
  db.exec("PRAGMA page_size=8192");
@@ -960,9 +1133,9 @@ function assertMimetypeEntry(entry, position) {
960
1133
  function assertSupportedFormat(raw) {
961
1134
  const kind = raw?.kind;
962
1135
  const version = raw?.formatVersion;
963
- if (kind !== GEZK_MANIFEST_KIND2 || version !== GEZK_FORMAT_VERSION) {
1136
+ if (kind !== GEZK_MANIFEST_KIND2 || !isSupportedFormatVersion(version)) {
964
1137
  throw new GezkArchiveError(
965
- `unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_FORMAT_VERSION} \u2014 catalogs built for an earlier version must be rebuilt`,
1138
+ `unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_SUPPORTED_FORMAT_VERSIONS.join(", ")} \u2014 catalogs built for another version need a matching reader`,
966
1139
  "format-version"
967
1140
  );
968
1141
  }
@@ -1202,9 +1375,9 @@ function openCatalogDatabase(absPath) {
1202
1375
  );
1203
1376
  }
1204
1377
  const schemaVersion = readPragmaNumber(db, "user_version");
1205
- if (schemaVersion !== GEZK_INDEX_SCHEMA_VERSION) {
1378
+ if (!isSupportedIndexSchemaVersion(schemaVersion)) {
1206
1379
  throw new CatalogOpenError(
1207
- `unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_INDEX_SCHEMA_VERSION}; catalogs built for gezk 0.4 and earlier must be rebuilt)`,
1380
+ `unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.join(", ")}; catalogs built for another generation need a matching reader)`,
1208
1381
  "schema-version"
1209
1382
  );
1210
1383
  }
@@ -1212,6 +1385,7 @@ function openCatalogDatabase(absPath) {
1212
1385
  db.exec("PRAGMA cache_size=-8192");
1213
1386
  return {
1214
1387
  db,
1388
+ schemaVersion,
1215
1389
  close: () => {
1216
1390
  try {
1217
1391
  db.close();
@@ -1234,8 +1408,16 @@ function readPragmaNumber(db, pragma) {
1234
1408
  }
1235
1409
 
1236
1410
  // src/reader/catalog-handle.ts
1411
+ import { readFileSync as readFileSync2 } from "fs";
1237
1412
  import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
1238
1413
  import { brotliDecompressSync } from "zlib";
1414
+ import {
1415
+ KnowledgeEmbeddingProfileSchema,
1416
+ MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
1417
+ assetContentType,
1418
+ embeddingProfileCenter,
1419
+ isKnowledgeAssetPath
1420
+ } from "@bendyline/gezk";
1239
1421
 
1240
1422
  // src/reader/bit-scan.ts
1241
1423
  var POPCOUNT8 = new Uint8Array(256);
@@ -1307,13 +1489,95 @@ function hammingTopK(index, query, k) {
1307
1489
  hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
1308
1490
  return hits;
1309
1491
  }
1492
+ function asymmetricTopK(index, query, k) {
1493
+ const { bits, bytesPerRow, rows } = index;
1494
+ if (Math.ceil(query.length / 8) !== bytesPerRow) {
1495
+ throw new Error(
1496
+ `query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
1497
+ );
1498
+ }
1499
+ const limit = Math.min(k, rows);
1500
+ if (limit <= 0) return [];
1501
+ const lut = new Float32Array(bytesPerRow * 256);
1502
+ for (let p = 0; p < bytesPerRow; p++) {
1503
+ for (let b = 0; b < 256; b++) {
1504
+ let s = 0;
1505
+ for (let i = 0; i < 8; i++) {
1506
+ const d = p * 8 + i;
1507
+ if (d >= query.length) break;
1508
+ const v = query[d];
1509
+ s += b >> i & 1 ? v : -v;
1510
+ }
1511
+ lut[p * 256 + b] = s;
1512
+ }
1513
+ }
1514
+ const heapScore = new Float64Array(limit);
1515
+ const heapId = new Int32Array(limit);
1516
+ let size = 0;
1517
+ const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
1518
+ const swap = (i, j) => {
1519
+ const s = heapScore[i];
1520
+ const id = heapId[i];
1521
+ heapScore[i] = heapScore[j];
1522
+ heapId[i] = heapId[j];
1523
+ heapScore[j] = s;
1524
+ heapId[j] = id;
1525
+ };
1526
+ const siftUp = (start) => {
1527
+ let i = start;
1528
+ while (i > 0) {
1529
+ const parent = i - 1 >> 1;
1530
+ if (!worse(i, parent)) break;
1531
+ swap(i, parent);
1532
+ i = parent;
1533
+ }
1534
+ };
1535
+ const siftDown = (start) => {
1536
+ let i = start;
1537
+ for (; ; ) {
1538
+ const left = 2 * i + 1;
1539
+ if (left >= size) break;
1540
+ const right = left + 1;
1541
+ const child = right < size && worse(right, left) ? right : left;
1542
+ if (!worse(child, i)) break;
1543
+ swap(child, i);
1544
+ i = child;
1545
+ }
1546
+ };
1547
+ let offset = 0;
1548
+ for (let r = 0; r < rows; r++) {
1549
+ let score = 0;
1550
+ for (let p = 0; p < bytesPerRow; p++) {
1551
+ score += lut[p * 256 + bits[offset + p]];
1552
+ }
1553
+ offset += bytesPerRow;
1554
+ if (size < limit) {
1555
+ heapScore[size] = score;
1556
+ heapId[size] = r + 1;
1557
+ size++;
1558
+ siftUp(size - 1);
1559
+ } else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
1560
+ heapScore[0] = score;
1561
+ heapId[0] = r + 1;
1562
+ siftDown(0);
1563
+ }
1564
+ }
1565
+ const hits = [];
1566
+ for (let i = 0; i < size; i++) {
1567
+ hits.push({ chunkId: heapId[i], score: heapScore[i] });
1568
+ }
1569
+ hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
1570
+ return hits;
1571
+ }
1310
1572
 
1311
1573
  // src/reader/catalog-handle.ts
1574
+ var TOPIC_WALK_MAX_DEPTH = 32;
1312
1575
  var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
1313
1576
  var CatalogHandle = class _CatalogHandle {
1314
1577
  constructor(rootDir, router) {
1315
1578
  this.rootDir = rootDir;
1316
1579
  this.router = router;
1580
+ this.schemaVersion = router.schemaVersion;
1317
1581
  this.meta = {};
1318
1582
  for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
1319
1583
  this.meta[row.key] = row.value;
@@ -1325,8 +1589,13 @@ var CatalogHandle = class _CatalogHandle {
1325
1589
  router;
1326
1590
  connections = /* @__PURE__ */ new Map();
1327
1591
  bitIndexes = /* @__PURE__ */ new Map();
1592
+ assetsByPath = null;
1593
+ /** Lazily parsed from the profile echo; `undefined` until first needed. */
1594
+ binaryCenterCache;
1328
1595
  meta;
1329
1596
  shards;
1597
+ /** The router's `PRAGMA user_version` generation; decides which columns exist. */
1598
+ schemaVersion;
1330
1599
  /** Open a catalog's extracted version directory (the dir holding manifest.json). */
1331
1600
  static open(rootDir) {
1332
1601
  const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
@@ -1405,6 +1674,28 @@ var CatalogHandle = class _CatalogHandle {
1405
1674
  this.bitIndexes.set(shard.path, index);
1406
1675
  return index;
1407
1676
  }
1677
+ /**
1678
+ * The stage-1 center the catalog's bits were derived with (a
1679
+ * `centered-sign` profile), or null for plain sign bits. Read from the
1680
+ * router's profile echo, never from this reader's registry: the echo is
1681
+ * the statement of how THIS catalog was built.
1682
+ */
1683
+ binaryCenter() {
1684
+ if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
1685
+ const raw = this.meta.embedding_profile_json;
1686
+ let parsed = null;
1687
+ try {
1688
+ parsed = raw ? JSON.parse(raw) : null;
1689
+ } catch {
1690
+ parsed = null;
1691
+ }
1692
+ const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
1693
+ if (!profile.success) {
1694
+ throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
1695
+ }
1696
+ this.binaryCenterCache = embeddingProfileCenter(profile.data);
1697
+ return this.binaryCenterCache;
1698
+ }
1408
1699
  /** The embedding dimension from the router's profile echo. */
1409
1700
  dimensions() {
1410
1701
  const raw = this.meta.embedding_profile_json;
@@ -1428,40 +1719,124 @@ var CatalogHandle = class _CatalogHandle {
1428
1719
  return target;
1429
1720
  }
1430
1721
  // ── browsing (the shipped table of contents) ──────────────────────────────
1722
+ /** The format version the router was written with (`meta.format_version`). */
1723
+ get formatVersion() {
1724
+ return this.meta.format_version ?? "";
1725
+ }
1726
+ documentColumns() {
1727
+ const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
1728
+ return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
1729
+ }
1730
+ documentOrder() {
1731
+ return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
1732
+ }
1733
+ /**
1734
+ * Every topic with its direct count and a rollup over its subtree. On a
1735
+ * 0.5 catalog every document sits at a root, so both counts agree there.
1736
+ */
1431
1737
  topics() {
1432
1738
  return this.router.db.prepare(
1433
- "SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key"
1739
+ `WITH RECURSIVE sub(root, id, depth) AS (
1740
+ SELECT id, id, 0 FROM topics
1741
+ UNION ALL
1742
+ SELECT sub.root, t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
1743
+ WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1744
+ ),
1745
+ rollup AS (
1746
+ SELECT sub.root AS id, SUM(t.document_count) AS total
1747
+ FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
1748
+ )
1749
+ SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
1750
+ rollup.total AS total_document_count
1751
+ FROM topics t JOIN rollup ON rollup.id = t.id
1752
+ ORDER BY t.sort_key, t.id`
1434
1753
  ).all().map((r) => ({
1435
1754
  id: r.id,
1436
1755
  parentId: r.parent_id,
1437
1756
  name: r.name,
1438
1757
  description: r.description,
1439
1758
  sortKey: r.sort_key,
1440
- documentCount: Number(r.document_count)
1759
+ documentCount: Number(r.document_count),
1760
+ totalDocumentCount: Number(r.total_document_count)
1441
1761
  }));
1442
1762
  }
1763
+ /**
1764
+ * A page of documents, by default including those filed under the topic's
1765
+ * descendants (`descendants: false` lists only the topic's own).
1766
+ */
1443
1767
  documentsPage(opts = {}) {
1444
1768
  const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
1445
1769
  const offset = Math.max(0, opts.offset ?? 0);
1446
- const where = opts.topicId ? "WHERE topic_id = ?" : "";
1447
- const params = opts.topicId ? [opts.topicId] : [];
1770
+ const descendants = opts.descendants ?? true;
1771
+ let scope = "";
1772
+ let where = "";
1773
+ const params = [];
1774
+ if (opts.topicId && descendants) {
1775
+ scope = `WITH RECURSIVE sub(id, depth) AS (
1776
+ SELECT ?, 0
1777
+ UNION ALL
1778
+ SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
1779
+ WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
1780
+ )`;
1781
+ where = "WHERE topic_id IN (SELECT id FROM sub)";
1782
+ params.push(opts.topicId);
1783
+ } else if (opts.topicId) {
1784
+ where = "WHERE topic_id = ?";
1785
+ params.push(opts.topicId);
1786
+ }
1448
1787
  const total = Number(
1449
- this.router.db.prepare(`SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
1788
+ this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
1450
1789
  );
1451
1790
  const documents = this.router.db.prepare(
1452
- `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1453
- source_updated_at, attribution_json
1454
- FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
1791
+ `${scope} SELECT ${this.documentColumns()}
1792
+ FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
1455
1793
  ).all(...params, limit, offset).map(rowToDocumentMeta);
1456
1794
  return { documents, total };
1457
1795
  }
1458
1796
  getDocument(id) {
1459
1797
  const row = this.router.db.prepare(
1460
- `SELECT id, title, slug, summary, language, topic_id, source_url, source_revision,
1461
- source_updated_at, attribution_json, body_codec, body_blob
1798
+ `SELECT ${this.documentColumns()}, body_codec, body_blob
1462
1799
  FROM documents WHERE id = ?`
1463
1800
  ).get(id);
1464
1801
  if (!row) return null;
1802
+ return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
1803
+ }
1804
+ /** Documents whose `topic_id` names no declared topic (a validator check). */
1805
+ documentsWithUndeclaredTopic() {
1806
+ return Number(
1807
+ this.router.db.prepare(
1808
+ "SELECT COUNT(*) AS n FROM documents d LEFT JOIN topics t ON t.id = d.topic_id WHERE t.id IS NULL"
1809
+ ).get().n
1810
+ );
1811
+ }
1812
+ /**
1813
+ * Validator support: every `meta_json` must be a JSON object within the
1814
+ * byte limit. Returns the offending ids (capped) rather than throwing.
1815
+ */
1816
+ checkDocumentMeta(maxBytes, cap = 5) {
1817
+ const invalid = [];
1818
+ const oversize = [];
1819
+ if (this.schemaVersion < 3) return { invalid, oversize };
1820
+ const rows = this.router.db.prepare("SELECT id, meta_json FROM documents WHERE meta_json IS NOT NULL").iterate();
1821
+ for (const row of rows) {
1822
+ if (invalid.length >= cap && oversize.length >= cap) break;
1823
+ if (Buffer.byteLength(row.meta_json, "utf8") > maxBytes) {
1824
+ if (oversize.length < cap) oversize.push(row.id);
1825
+ continue;
1826
+ }
1827
+ if (parseJsonObject(row.meta_json) === null && invalid.length < cap) invalid.push(row.id);
1828
+ }
1829
+ return { invalid, oversize };
1830
+ }
1831
+ /** Validator support: decoded bodies, streamed one at a time. */
1832
+ *documentBodies() {
1833
+ const rows = this.router.db.prepare("SELECT id, body_codec, body_blob FROM documents ORDER BY id").iterate();
1834
+ for (const row of rows) {
1835
+ const id = row.id;
1836
+ yield { id, markdown: this.decodeBody(id, row) };
1837
+ }
1838
+ }
1839
+ decodeBody(id, row) {
1465
1840
  const blob = row.body_blob;
1466
1841
  if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
1467
1842
  throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
@@ -1484,8 +1859,69 @@ var CatalogHandle = class _CatalogHandle {
1484
1859
  } else {
1485
1860
  throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
1486
1861
  }
1487
- const markdown = body.toString("utf8");
1488
- return { ...rowToDocumentMeta(row), markdown };
1862
+ return body.toString("utf8");
1863
+ }
1864
+ // ── assets ────────────────────────────────────────────────────────────────
1865
+ /**
1866
+ * The manifest's `files` entries under `assets/`, read lazily: the
1867
+ * declaration in the manifest is what authorizes serving a file, since
1868
+ * extraction reconciled and hashed every declared entry.
1869
+ */
1870
+ assetIndex() {
1871
+ if (this.assetsByPath) return this.assetsByPath;
1872
+ const index = /* @__PURE__ */ new Map();
1873
+ let raw;
1874
+ try {
1875
+ raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
1876
+ } catch (error) {
1877
+ if (error.code === "ENOENT") {
1878
+ this.assetsByPath = index;
1879
+ return index;
1880
+ }
1881
+ throw new CatalogOpenError(
1882
+ `cannot read ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`,
1883
+ "corrupt"
1884
+ );
1885
+ }
1886
+ const files = raw?.files;
1887
+ if (Array.isArray(files)) {
1888
+ for (const file of files) {
1889
+ const path = file?.path;
1890
+ if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
1891
+ const contentType = assetContentType(path);
1892
+ if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
1893
+ continue;
1894
+ }
1895
+ index.set(path, { path, contentType, sizeBytes: file.sizeBytes, sha256: file.sha256 });
1896
+ }
1897
+ }
1898
+ this.assetsByPath = index;
1899
+ return index;
1900
+ }
1901
+ /** Declared assets, sorted by path. */
1902
+ assets() {
1903
+ return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
1904
+ }
1905
+ /** One declared asset's bytes, or null when the catalog declares no such asset. */
1906
+ readAsset(path) {
1907
+ const info = this.assetIndex().get(path);
1908
+ if (!info) return null;
1909
+ if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
1910
+ throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
1911
+ }
1912
+ let bytes;
1913
+ try {
1914
+ bytes = readFileSync2(this.resolveCatalogPath(path));
1915
+ } catch (error) {
1916
+ throw new CatalogOpenError(
1917
+ `cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
1918
+ "corrupt"
1919
+ );
1920
+ }
1921
+ if (bytes.byteLength !== info.sizeBytes) {
1922
+ throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
1923
+ }
1924
+ return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
1489
1925
  }
1490
1926
  // ── search ────────────────────────────────────────────────────────────────
1491
1927
  /** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
@@ -1542,7 +1978,7 @@ var CatalogHandle = class _CatalogHandle {
1542
1978
  return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
1543
1979
  }
1544
1980
  /**
1545
- * Two-stage semantic search over the routed shards: bit-hamming KNN
1981
+ * Two-stage semantic search over the routed shards: sign-bit scan
1546
1982
  * (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
1547
1983
  */
1548
1984
  searchSemantic(queryVector, opts = {}) {
@@ -1554,14 +1990,15 @@ var CatalogHandle = class _CatalogHandle {
1554
1990
  * after global cross-catalog routing has already spent the S budget.
1555
1991
  */
1556
1992
  searchShards(queryVector, shardIds, finalK) {
1557
- const queryBits = quantizeBinary(queryVector);
1993
+ const center = this.binaryCenter();
1994
+ const scanQuery = center ? centerVector(queryVector, center) : queryVector;
1558
1995
  const hits = [];
1559
1996
  for (const shardId of shardIds) {
1560
1997
  const shard = this.shards.find((s) => s.id === shardId);
1561
1998
  if (!shard) continue;
1562
1999
  const db = this.shardDb(shard);
1563
2000
  const k = rerankK(finalK, shard.chunkCount);
1564
- const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
2001
+ const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
1565
2002
  chunk_id: hit.chunkId
1566
2003
  }));
1567
2004
  if (candidates.length === 0) continue;
@@ -1648,15 +2085,17 @@ var CatalogHandle = class _CatalogHandle {
1648
2085
  }
1649
2086
  }
1650
2087
  };
1651
- function rowToDocumentMeta(row) {
1652
- let attribution = null;
1653
- if (typeof row.attribution_json === "string") {
1654
- try {
1655
- attribution = JSON.parse(row.attribution_json);
1656
- } catch {
1657
- attribution = null;
1658
- }
2088
+ function parseJsonObject(raw) {
2089
+ if (typeof raw !== "string") return null;
2090
+ try {
2091
+ const parsed = JSON.parse(raw);
2092
+ return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
2093
+ } catch {
2094
+ return null;
1659
2095
  }
2096
+ }
2097
+ function rowToDocumentMeta(row) {
2098
+ const ordinal = row.ordinal;
1660
2099
  return {
1661
2100
  id: row.id,
1662
2101
  title: row.title,
@@ -1664,10 +2103,12 @@ function rowToDocumentMeta(row) {
1664
2103
  summary: row.summary ?? null,
1665
2104
  language: row.language,
1666
2105
  topicId: row.topic_id,
2106
+ ordinal: typeof ordinal === "number" || typeof ordinal === "bigint" ? Number(ordinal) : null,
1667
2107
  sourceUrl: row.source_url ?? null,
1668
2108
  sourceRevision: row.source_revision ?? null,
1669
2109
  sourceUpdatedAt: row.source_updated_at ?? null,
1670
- attribution
2110
+ attribution: parseJsonObject(row.attribution_json),
2111
+ meta: parseJsonObject(row.meta_json)
1671
2112
  };
1672
2113
  }
1673
2114
  function parseHeadingPath(raw) {
@@ -1683,7 +2124,17 @@ var ftsQuery = sanitizeFtsQuery;
1683
2124
  // src/reader/validate.ts
1684
2125
  import { readFile } from "fs/promises";
1685
2126
  import { join as join3 } from "path";
1686
- import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3 } from "@bendyline/gezk";
2127
+ import {
2128
+ KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
2129
+ MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2130
+ MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
2131
+ MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
2132
+ assetExtension as assetExtension2,
2133
+ assetKindForExtension as assetKindForExtension2,
2134
+ isKnowledgeAssetPath as isKnowledgeAssetPath2,
2135
+ sniffAssetType as sniffAssetType2,
2136
+ svgInertnessProblem as svgInertnessProblem2
2137
+ } from "@bendyline/gezk";
1687
2138
  async function validateExtractedCatalog(rootDir, opts = {}) {
1688
2139
  const checks = [];
1689
2140
  const check = (name, ok, detail) => {
@@ -1750,8 +2201,43 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1750
2201
  handle.meta.embedding_profile_id === manifest.embedding.id,
1751
2202
  `router meta profile ${handle.meta.embedding_profile_id}`
1752
2203
  );
2204
+ check(
2205
+ "meta-format",
2206
+ handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
2207
+ `router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
2208
+ );
1753
2209
  const topics = handle.topics();
1754
2210
  check("toc-present", topics.length >= 1, `${topics.length} topics`);
2211
+ check("topics-tree", ...topicTreeProblem(topics));
2212
+ const undeclared = handle.documentsWithUndeclaredTopic();
2213
+ check(
2214
+ "documents-topic-declared",
2215
+ undeclared === 0,
2216
+ `${undeclared} documents are filed under a topic the router does not declare`
2217
+ );
2218
+ const assetFiles = manifest.files.filter((f) => f.path.startsWith("assets/"));
2219
+ if (manifest.formatVersion === "0.5") {
2220
+ check(
2221
+ "assets-not-in-0.5",
2222
+ assetFiles.length === 0,
2223
+ `${assetFiles.length} assets/ entries in a 0.5 catalog`
2224
+ );
2225
+ } else {
2226
+ const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
2227
+ check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
2228
+ const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
2229
+ const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
2230
+ check(
2231
+ "assets-limits",
2232
+ assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
2233
+ `${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
2234
+ );
2235
+ check(
2236
+ "counts-assets",
2237
+ (manifest.counts.assets ?? 0) === assetFiles.length,
2238
+ `manifest counts ${manifest.counts.assets ?? 0} assets, files declare ${assetFiles.length}`
2239
+ );
2240
+ }
1755
2241
  check(
1756
2242
  "license-notice",
1757
2243
  manifest.files.some((f) => f.path === manifest.license.noticePath),
@@ -1840,6 +2326,45 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1840
2326
  }
1841
2327
  check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
1842
2328
  }
2329
+ if (handle.schemaVersion >= 3) {
2330
+ const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
2331
+ check(
2332
+ "document-meta-json",
2333
+ meta.invalid.length === 0 && meta.oversize.length === 0,
2334
+ `${meta.invalid.length > 0 ? `not a JSON object: ${meta.invalid.join(", ")}` : ""}${meta.oversize.length > 0 ? ` over ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES} bytes: ${meta.oversize.join(", ")}` : ""}`.trim()
2335
+ );
2336
+ const declared = new Set(assetFiles.map((f) => f.path));
2337
+ for (const file of assetFiles) {
2338
+ const ext = assetExtension2(file.path);
2339
+ if (!ext) continue;
2340
+ const bytes = await readFile(join3(rootDir, file.path));
2341
+ const kind = sniffAssetType2(bytes);
2342
+ check(
2343
+ `asset-type:${file.path}`,
2344
+ kind === assetKindForExtension2(ext),
2345
+ `leading bytes say ${kind ?? "unknown"}, the extension says ${assetKindForExtension2(ext)}`
2346
+ );
2347
+ if (ext === "svg") {
2348
+ const problem = svgInertnessProblem2(bytes);
2349
+ check(`asset-svg-inert:${file.path}`, problem === null, problem ?? void 0);
2350
+ }
2351
+ }
2352
+ if (declared.size > 0) {
2353
+ const missing = [];
2354
+ for (const doc of handle.documentBodies()) {
2355
+ for (const target of assetReferences(doc.markdown)) {
2356
+ if (!declared.has(target)) missing.push(`${doc.id} \u2192 ${target}`);
2357
+ if (missing.length >= 5) break;
2358
+ }
2359
+ if (missing.length >= 5) break;
2360
+ }
2361
+ check(
2362
+ "document-asset-refs",
2363
+ missing.length === 0,
2364
+ `undeclared asset references: ${missing.join("; ")}`
2365
+ );
2366
+ }
2367
+ }
1843
2368
  for (const smoke of manifest.smokeQueries ?? []) {
1844
2369
  const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
1845
2370
  const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
@@ -1855,6 +2380,39 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
1855
2380
  }
1856
2381
  return report(manifest);
1857
2382
  }
2383
+ function assetReferences(markdown) {
2384
+ const found = /* @__PURE__ */ new Set();
2385
+ for (const match of markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
2386
+ const target = match[1];
2387
+ if (target) found.add(target);
2388
+ }
2389
+ return [...found];
2390
+ }
2391
+ function topicTreeProblem(topics) {
2392
+ const byId = new Map(topics.map((t) => [t.id, t]));
2393
+ for (const topic of topics) {
2394
+ if (topic.parentId !== null && !byId.has(topic.parentId)) {
2395
+ return [false, `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`];
2396
+ }
2397
+ }
2398
+ const depthOf = /* @__PURE__ */ new Map();
2399
+ for (const topic of topics) {
2400
+ const seen = /* @__PURE__ */ new Set();
2401
+ let depth = 0;
2402
+ let current = topic;
2403
+ while (current) {
2404
+ if (seen.has(current.id)) return [false, `topic '${topic.id}' sits in a parent cycle`];
2405
+ seen.add(current.id);
2406
+ depth += 1;
2407
+ if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
2408
+ return [false, `topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`];
2409
+ }
2410
+ current = current.parentId === null ? void 0 : byId.get(current.parentId);
2411
+ }
2412
+ depthOf.set(topic.id, depth);
2413
+ }
2414
+ return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
2415
+ }
1858
2416
 
1859
2417
  // src/export/duckdb.ts
1860
2418
  import { spawn } from "child_process";
@@ -1892,7 +2450,7 @@ async function runDuckdbScript(cli, sql) {
1892
2450
  return stdout;
1893
2451
  }
1894
2452
  function runDuckdbProcess(binaryPath, args, stdin) {
1895
- return new Promise((resolve4, reject) => {
2453
+ return new Promise((resolve5, reject) => {
1896
2454
  const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
1897
2455
  let stdout = "";
1898
2456
  let stderr = "";
@@ -1909,7 +2467,7 @@ function runDuckdbProcess(binaryPath, args, stdin) {
1909
2467
  (err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
1910
2468
  );
1911
2469
  child.on("close", (code) => {
1912
- if (code === 0) resolve4({ stdout });
2470
+ if (code === 0) resolve5({ stdout });
1913
2471
  else
1914
2472
  reject(
1915
2473
  new DuckdbError(
@@ -1931,7 +2489,7 @@ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/pr
1931
2489
  import { join as join4, resolve as resolve3 } from "path";
1932
2490
  import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
1933
2491
  import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
1934
- var PARQUET_EXPORT_VERSION = 1;
2492
+ var PARQUET_EXPORT_VERSION = 2;
1935
2493
  var DEFAULT_ROW_GROUP_SIZE = 32768;
1936
2494
  var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
1937
2495
  var PARQUET_REPORT_PATH = "parquet-manifest.json";
@@ -2126,9 +2684,10 @@ async function stageDocuments(router, shardId, topicPaths, path) {
2126
2684
  const aliasesOf = router.db.prepare(
2127
2685
  "SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
2128
2686
  );
2687
+ const extra = router.schemaVersion >= 3 ? ", ordinal, meta_json" : ", NULL AS ordinal, NULL AS meta_json";
2129
2688
  const statement = router.db.prepare(
2130
2689
  `SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
2131
- source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
2690
+ source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob${extra}
2132
2691
  FROM documents WHERE shard_id = ? ORDER BY id`
2133
2692
  );
2134
2693
  for (const row of statement.iterate(shardId)) {
@@ -2150,6 +2709,8 @@ async function stageDocuments(router, shardId, topicPaths, path) {
2150
2709
  source_revision: row.source_revision ?? null,
2151
2710
  source_updated_at: row.source_updated_at ?? null,
2152
2711
  attribution: row.attribution_json ?? null,
2712
+ ordinal: row.ordinal == null ? null : Number(row.ordinal),
2713
+ meta: row.meta_json ?? null,
2153
2714
  aliases
2154
2715
  });
2155
2716
  rows++;
@@ -2273,9 +2834,11 @@ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
2273
2834
  source_revision: "VARCHAR",
2274
2835
  source_updated_at: "VARCHAR",
2275
2836
  attribution: "VARCHAR",
2837
+ ordinal: "INTEGER",
2838
+ meta: "VARCHAR",
2276
2839
  aliases: "VARCHAR[]"
2277
2840
  });
2278
- return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2841
+ return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, ordinal, meta, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
2279
2842
  `;
2280
2843
  }
2281
2844
  function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
@@ -2340,6 +2903,394 @@ async function findDuckdbBinary(env = process.env) {
2340
2903
  return null;
2341
2904
  }
2342
2905
 
2906
+ // src/profiles/e5-small-center.ts
2907
+ var MULTILINGUAL_E5_SMALL_CENTER = [
2908
+ 0.046697,
2909
+ -0.013211,
2910
+ -0.042276,
2911
+ -0.049082,
2912
+ 0.060627,
2913
+ -0.032643,
2914
+ 0.029874,
2915
+ 0.033307,
2916
+ 0.058447,
2917
+ -0.012016,
2918
+ 0.046472,
2919
+ 5991e-6,
2920
+ 0.04884,
2921
+ -0.028836,
2922
+ -0.03244,
2923
+ 0.045633,
2924
+ 0.056798,
2925
+ -0.043557,
2926
+ -0.034472,
2927
+ -0.05355,
2928
+ 0.026363,
2929
+ -0.010318,
2930
+ -0.030487,
2931
+ 0.047083,
2932
+ 0.057205,
2933
+ 0.035654,
2934
+ -0.017461,
2935
+ 0.010881,
2936
+ 0.047985,
2937
+ -0.048514,
2938
+ -0.048956,
2939
+ -0.027643,
2940
+ 0.053259,
2941
+ -0.051057,
2942
+ 0.045055,
2943
+ 0.017888,
2944
+ -0.055755,
2945
+ -0.046578,
2946
+ 0.05214,
2947
+ -0.037085,
2948
+ -0.016268,
2949
+ 0.027508,
2950
+ 0.035793,
2951
+ 0.052082,
2952
+ 0.034885,
2953
+ 0.047174,
2954
+ -0.036778,
2955
+ 0.031394,
2956
+ -9156e-6,
2957
+ -0.02486,
2958
+ -0.048897,
2959
+ 0.052576,
2960
+ 0.017289,
2961
+ 0.047794,
2962
+ 0.051939,
2963
+ -0.051261,
2964
+ -0.061345,
2965
+ -0.056267,
2966
+ -0.049305,
2967
+ 0.047241,
2968
+ 0.043763,
2969
+ -0.04506,
2970
+ 0.015752,
2971
+ 0.017543,
2972
+ 0.067856,
2973
+ 0.065763,
2974
+ 0.037218,
2975
+ 0.044877,
2976
+ -0.057149,
2977
+ -0.028072,
2978
+ -0.060566,
2979
+ 0.064004,
2980
+ 6095e-6,
2981
+ -0.039693,
2982
+ 0.021968,
2983
+ 0.02654,
2984
+ 0.025546,
2985
+ -0.048954,
2986
+ 0.026967,
2987
+ -0.045002,
2988
+ -0.055556,
2989
+ -0.034963,
2990
+ -0.03466,
2991
+ 0.032486,
2992
+ -0.049597,
2993
+ 0.058718,
2994
+ 0.048021,
2995
+ -0.066783,
2996
+ 0.059049,
2997
+ -0.04665,
2998
+ 0.052902,
2999
+ 0.031438,
3000
+ -0.061515,
3001
+ -0.05342,
3002
+ -0.068253,
3003
+ -0.06666,
3004
+ -0.04553,
3005
+ 0.059252,
3006
+ 0.050728,
3007
+ -0.030429,
3008
+ 0.067182,
3009
+ -0.038819,
3010
+ 0.049358,
3011
+ -0.055247,
3012
+ -0.032933,
3013
+ 0.041239,
3014
+ 0.035164,
3015
+ -0.066574,
3016
+ 0.05348,
3017
+ -0.052658,
3018
+ -0.034758,
3019
+ 0.037219,
3020
+ 0.064111,
3021
+ 0.047861,
3022
+ -0.04608,
3023
+ -0.041338,
3024
+ -0.016072,
3025
+ -0.046026,
3026
+ 0.0255,
3027
+ -0.041581,
3028
+ 0.059684,
3029
+ -0.019611,
3030
+ -0.050363,
3031
+ -0.061268,
3032
+ -0.057556,
3033
+ -0.03184,
3034
+ 0.018522,
3035
+ 0.046887,
3036
+ 0.054774,
3037
+ 0.012536,
3038
+ 0.033019,
3039
+ 0.033211,
3040
+ 0.029609,
3041
+ 0.050215,
3042
+ 0.035425,
3043
+ 0.076419,
3044
+ -0.055196,
3045
+ -0.010016,
3046
+ -936e-6,
3047
+ -0.023764,
3048
+ -0.059648,
3049
+ 0.054857,
3050
+ -0.030456,
3051
+ 0.041281,
3052
+ 0.063894,
3053
+ 0.035466,
3054
+ 0.069754,
3055
+ -0.026593,
3056
+ 0.066305,
3057
+ -0.028609,
3058
+ 0.05324,
3059
+ -0.022879,
3060
+ 0.04882,
3061
+ 0.032385,
3062
+ 0.042921,
3063
+ -0.043319,
3064
+ -0.065022,
3065
+ -0.044018,
3066
+ 0.04237,
3067
+ 0.051018,
3068
+ -0.048907,
3069
+ -0.053045,
3070
+ -0.057301,
3071
+ -0.019731,
3072
+ -0.060812,
3073
+ -0.043114,
3074
+ 0.021207,
3075
+ 0.052498,
3076
+ -0.038307,
3077
+ -0.037082,
3078
+ -0.043732,
3079
+ 0.045959,
3080
+ -0.044296,
3081
+ 0.057199,
3082
+ -0.011835,
3083
+ 0.054221,
3084
+ -0.046354,
3085
+ 0.043442,
3086
+ 0.063337,
3087
+ 0.041964,
3088
+ -0.032836,
3089
+ -0.020089,
3090
+ -0.056327,
3091
+ -0.05089,
3092
+ -0.053933,
3093
+ -0.035486,
3094
+ -0.046215,
3095
+ 0.025754,
3096
+ 0.032986,
3097
+ -0.025048,
3098
+ -9903e-6,
3099
+ 0.031162,
3100
+ -0.050969,
3101
+ -0.049401,
3102
+ -0.046206,
3103
+ 0.023197,
3104
+ -0.033441,
3105
+ 0.045075,
3106
+ 0.043318,
3107
+ 0.043766,
3108
+ -6601e-6,
3109
+ -0.023077,
3110
+ 0.033917,
3111
+ 0.032279,
3112
+ 0.034874,
3113
+ 5185e-6,
3114
+ -0.057329,
3115
+ 0.048437,
3116
+ -0.033964,
3117
+ 0.044195,
3118
+ 0.051107,
3119
+ -0.056642,
3120
+ -0.070856,
3121
+ 0.045431,
3122
+ -0.047205,
3123
+ -0.021457,
3124
+ 3103e-6,
3125
+ 0.062536,
3126
+ -0.037804,
3127
+ 0.031278,
3128
+ 0.041619,
3129
+ -0.015547,
3130
+ 0.052298,
3131
+ -0.078327,
3132
+ -0.052303,
3133
+ 0.043099,
3134
+ 0.05375,
3135
+ -0.041854,
3136
+ -0.038314,
3137
+ 0.039399,
3138
+ -0.047769,
3139
+ -0.038469,
3140
+ -0.056793,
3141
+ -0.064312,
3142
+ -0.047884,
3143
+ -0.060067,
3144
+ -0.030687,
3145
+ 0.030053,
3146
+ 0.033569,
3147
+ -0.038014,
3148
+ -0.032396,
3149
+ -0.041395,
3150
+ 0.038163,
3151
+ -0.056206,
3152
+ 0.047115,
3153
+ -0.038099,
3154
+ -0.031927,
3155
+ 0.033857,
3156
+ -0.011227,
3157
+ 0.032883,
3158
+ 0.032516,
3159
+ -0.045622,
3160
+ -0.044387,
3161
+ -0.034374,
3162
+ -0.030433,
3163
+ 0.055097,
3164
+ 0.055568,
3165
+ 0.058726,
3166
+ -0.043569,
3167
+ 0.031854,
3168
+ 775e-5,
3169
+ -0.022533,
3170
+ 0.096133,
3171
+ 0.052876,
3172
+ 0.045582,
3173
+ 0.041868,
3174
+ -0.039268,
3175
+ -0.037865,
3176
+ -0.054658,
3177
+ -0.031297,
3178
+ -0.058104,
3179
+ 0.018705,
3180
+ 0.031934,
3181
+ -0.052746,
3182
+ -0.044438,
3183
+ -0.029991,
3184
+ 0.011917,
3185
+ 0.070103,
3186
+ -0.046682,
3187
+ -0.048965,
3188
+ 0.046255,
3189
+ 0.027877,
3190
+ 0.0475,
3191
+ 0.044287,
3192
+ 0.04251,
3193
+ -0.017647,
3194
+ -0.017812,
3195
+ 0.055672,
3196
+ -0.042207,
3197
+ -0.035516,
3198
+ -0.025558,
3199
+ -0.062443,
3200
+ 0.037519,
3201
+ -0.031697,
3202
+ 0.064998,
3203
+ 0.045949,
3204
+ 302e-6,
3205
+ 0.065205,
3206
+ -0.052016,
3207
+ 0.036721,
3208
+ -0.034769,
3209
+ -0.049074,
3210
+ 0.040413,
3211
+ 0.045425,
3212
+ -0.068063,
3213
+ 0.021485,
3214
+ 8549e-6,
3215
+ 0.028179,
3216
+ 0.026914,
3217
+ 0.020493,
3218
+ 0.056155,
3219
+ 0.047202,
3220
+ -0.049085,
3221
+ -0.022624,
3222
+ 0.063734,
3223
+ 0.05471,
3224
+ -0.032258,
3225
+ 0.047761,
3226
+ -0.065033,
3227
+ -0.047818,
3228
+ -0.034254,
3229
+ -0.062863,
3230
+ -0.01362,
3231
+ -0.053904,
3232
+ 0.024415,
3233
+ 0.044367,
3234
+ -0.064056,
3235
+ -0.038419,
3236
+ 0.038997,
3237
+ 1015e-6,
3238
+ 0.038928,
3239
+ -0.045794,
3240
+ -0.038313,
3241
+ 0.043922,
3242
+ -0.05873,
3243
+ -0.020314,
3244
+ -0.020457,
3245
+ 0.045522,
3246
+ -0.057731,
3247
+ -0.062966,
3248
+ 0.023159,
3249
+ 0.020283,
3250
+ -0.057352,
3251
+ 0.044701,
3252
+ -0.03585,
3253
+ -0.052798,
3254
+ 0.056931,
3255
+ -0.035609,
3256
+ -0.035113,
3257
+ 0.026919,
3258
+ 0.059404,
3259
+ -0.084434,
3260
+ 0.014558,
3261
+ 0.06226,
3262
+ -0.044173,
3263
+ 0.044172,
3264
+ -0.046482,
3265
+ -0.034933,
3266
+ 0.052513,
3267
+ 0.044018,
3268
+ -0.057677,
3269
+ -0.033961,
3270
+ 0.046334,
3271
+ 0.065767,
3272
+ 0.044779,
3273
+ 0.063658,
3274
+ -0.022574,
3275
+ -4821e-6,
3276
+ 0.037388,
3277
+ -0.043197,
3278
+ 0.063,
3279
+ 0.048943,
3280
+ -0.049495,
3281
+ 0.014798,
3282
+ -0.018176,
3283
+ -0.03824,
3284
+ -0.045486,
3285
+ 0.053828,
3286
+ -0.056118,
3287
+ -0.067025,
3288
+ 0.027818,
3289
+ 0.050305,
3290
+ 0.037134,
3291
+ 0.049673
3292
+ ];
3293
+
2343
3294
  // src/profiles/registry.ts
2344
3295
  var SHARED_ENCODING = {
2345
3296
  pooling: "mean",
@@ -2370,6 +3321,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
2370
3321
  passageInstruction: "passage: ",
2371
3322
  ...SHARED_ENCODING
2372
3323
  };
3324
+ var MULTILINGUAL_E5_SMALL_2 = {
3325
+ ...MULTILINGUAL_E5_SMALL_1,
3326
+ id: "multilingual-e5-small@2",
3327
+ quantization: {
3328
+ int8: { method: "symmetric-linear", scale: 127 },
3329
+ binary: {
3330
+ method: "centered-sign",
3331
+ threshold: 0,
3332
+ packing: "lsb-first",
3333
+ center: [...MULTILINGUAL_E5_SMALL_CENTER]
3334
+ }
3335
+ }
3336
+ };
2373
3337
  var BGE_SMALL_EN_V15_1 = {
2374
3338
  id: "bge-small-en-v1.5@1",
2375
3339
  model: {
@@ -2388,6 +3352,7 @@ var BGE_SMALL_EN_V15_1 = {
2388
3352
  ...SHARED_ENCODING
2389
3353
  };
2390
3354
  var KNOWLEDGE_EMBEDDING_PROFILES = [
3355
+ MULTILINGUAL_E5_SMALL_2,
2391
3356
  MULTILINGUAL_E5_SMALL_1,
2392
3357
  BGE_SMALL_EN_V15_1
2393
3358
  ];
@@ -2453,9 +3418,9 @@ async function fileDigest(path) {
2453
3418
  const key = `${path} ${size} ${mtimeMs}`;
2454
3419
  const memo = digestMemo.get(key);
2455
3420
  if (memo) return memo;
2456
- const digest = await new Promise((resolve4, reject) => {
3421
+ const digest = await new Promise((resolve5, reject) => {
2457
3422
  const hash = createHash4("sha256");
2458
- createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve4(`sha256:${hash.digest("hex")}`));
3423
+ createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
2459
3424
  });
2460
3425
  if (digestMemo.size >= DIGEST_MEMO_MAX) {
2461
3426
  const oldest = digestMemo.keys().next().value;
@@ -2613,29 +3578,427 @@ async function createProfileEmbedder(profile, opts = {}) {
2613
3578
  }
2614
3579
 
2615
3580
  // src/markdown-adapter/load.ts
3581
+ import { existsSync } from "fs";
2616
3582
  import { readFile as readFile3, readdir } from "fs/promises";
2617
- import { join as join6, relative as relative3, sep } from "path";
2618
- import { CatalogDocumentSchema } from "@bendyline/gezk";
3583
+ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
3584
+ import {
3585
+ CatalogDocumentSchema,
3586
+ KnowledgeDocumentIdSchema,
3587
+ KnowledgeOrdinalSchema,
3588
+ assetExtension as assetExtension3,
3589
+ formatKnowledgeUri,
3590
+ topicSortKeyForOrder
3591
+ } from "@bendyline/gezk";
3592
+
3593
+ // src/markdown-adapter/frontmatter.ts
3594
+ import { parseDocument } from "yaml";
3595
+ var FRONTMATTER_MAX_BYTES = 64 * 1024;
3596
+ var OUTLINE_MAX_BYTES = 1024 * 1024;
3597
+ function parseYaml(text, opts) {
3598
+ const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
3599
+ if (Buffer.byteLength(text, "utf8") > maxBytes) {
3600
+ throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
3601
+ }
3602
+ const doc = parseDocument(text, {
3603
+ version: "1.2",
3604
+ schema: "core",
3605
+ uniqueKeys: true,
3606
+ logLevel: "silent"
3607
+ });
3608
+ const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
3609
+ if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
3610
+ return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
3611
+ }
3612
+ var OPEN = /^---[ \t]*\r?\n/;
3613
+ var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
3614
+ function parseMarkdownFrontMatter(source) {
3615
+ const open = OPEN.exec(source);
3616
+ if (!open) return { data: {}, body: source };
3617
+ const lines = source.slice(open[0].length).split(/\r?\n/);
3618
+ const closeAt = lines.findIndex((line) => CLOSE.test(line));
3619
+ if (closeAt === -1) throw new Error("front matter is not closed with ---");
3620
+ const yamlText = lines.slice(0, closeAt).join("\n");
3621
+ const body = lines.slice(closeAt + 1).join("\n");
3622
+ const parsed = parseYaml(yamlText, { what: "front matter" });
3623
+ if (parsed === null || parsed === void 0) return { data: {}, body };
3624
+ if (typeof parsed !== "object" || Array.isArray(parsed)) {
3625
+ throw new Error("front matter must be a mapping");
3626
+ }
3627
+ return { data: parsed, body };
3628
+ }
3629
+
3630
+ // src/markdown-adapter/outline.ts
3631
+ import { posix } from "path";
3632
+ var HAS_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
3633
+ var MARKDOWN_EXT = /\.(md|markdown)$/i;
3634
+ function newOutlineTopic(name) {
3635
+ return { name, children: [], entries: [] };
3636
+ }
3637
+ function isRecord(value) {
3638
+ return typeof value === "object" && value !== null && !Array.isArray(value);
3639
+ }
3640
+ function isExternal(target) {
3641
+ return HAS_SCHEME.test(target) || target.startsWith("//");
3642
+ }
3643
+ function safeDecode(value) {
3644
+ try {
3645
+ return decodeURIComponent(value);
3646
+ } catch {
3647
+ return value;
3648
+ }
3649
+ }
3650
+ function stem(file) {
3651
+ return posix.basename(file).replace(MARKDOWN_EXT, "");
3652
+ }
3653
+ function resolveOutlinePath(baseDir, target) {
3654
+ const cleaned = safeDecode(
3655
+ target.replace(/\\/g, "/").replace(/[#?].*$/, "").trim()
3656
+ );
3657
+ if (cleaned === "" || posix.isAbsolute(cleaned)) return null;
3658
+ const joined = posix.normalize(posix.join(baseDir, cleaned));
3659
+ if (joined === ".." || joined.startsWith("../")) return null;
3660
+ return joined === "." ? "" : joined;
3661
+ }
3662
+ function assignOutlineOrders(topic) {
3663
+ topic.entries.forEach((entry, i) => {
3664
+ if (entry.order === void 0) entry.order = i + 1;
3665
+ });
3666
+ topic.children.forEach((child, i) => {
3667
+ if (child.order === void 0) child.order = i + 1;
3668
+ assignOutlineOrders(child);
3669
+ });
3670
+ }
3671
+ function titleFromFolderName(name) {
3672
+ const words = name.replace(/[_-]+/g, " ").trim().split(/\s+/).filter(Boolean);
3673
+ if (words.length === 0) return name;
3674
+ return words.map((word) => /[A-Z]/.test(word) ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
3675
+ }
3676
+ var HEADING2 = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
3677
+ var LIST_ITEM = /^([ \t]*)(?:[-*+]|\d+[.)])\s+(.+?)\s*$/;
3678
+ var ITEM_LINK = /^\[([^\]]*)\]\(\s*<?([^)>\s]*)>?(?:\s+"[^"]*")?\s*\)$/;
3679
+ function materializeSummaryTopic(slot) {
3680
+ if (slot.topic) return slot.topic;
3681
+ const topic = newOutlineTopic(slot.title || "Untitled");
3682
+ if (slot.entry) {
3683
+ const at = slot.container.entries.indexOf(slot.entry);
3684
+ if (at >= 0) slot.container.entries.splice(at, 1);
3685
+ topic.entries.push(slot.entry);
3686
+ }
3687
+ slot.container.children.push(topic);
3688
+ slot.topic = topic;
3689
+ return topic;
3690
+ }
3691
+ function parseGitbookSummary(text, summaryFile) {
3692
+ const dir = posix.dirname(summaryFile);
3693
+ const baseDir = dir === "." ? "" : dir;
3694
+ const outline = {
3695
+ format: "gitbook",
3696
+ root: newOutlineTopic(""),
3697
+ consumed: [summaryFile],
3698
+ warnings: []
3699
+ };
3700
+ let container = outline.root;
3701
+ const stack = [];
3702
+ let fence = null;
3703
+ for (const rawLine of text.split(/\r?\n/)) {
3704
+ const line = rawLine.replace(/\t/g, " ");
3705
+ const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
3706
+ if (fenceMatch) {
3707
+ const marker = fenceMatch[1][0];
3708
+ if (fence === null) fence = marker;
3709
+ else if (marker === fence) fence = null;
3710
+ continue;
3711
+ }
3712
+ if (fence !== null) continue;
3713
+ const heading = HEADING2.exec(line);
3714
+ if (heading) {
3715
+ if (heading[1].length === 1) continue;
3716
+ const part = newOutlineTopic(heading[2].replace(/[*_`]/g, "").trim() || "Part");
3717
+ outline.root.children.push(part);
3718
+ container = part;
3719
+ stack.length = 0;
3720
+ continue;
3721
+ }
3722
+ const item = LIST_ITEM.exec(line);
3723
+ if (!item) continue;
3724
+ const indent = item[1].length;
3725
+ while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
3726
+ stack.pop();
3727
+ }
3728
+ const parent = stack[stack.length - 1];
3729
+ const parentTopic = parent ? materializeSummaryTopic(parent) : container;
3730
+ const content = item[2].trim();
3731
+ const link = ITEM_LINK.exec(content);
3732
+ const title = (link ? link[1] : content).replace(/[*_`]/g, "").trim();
3733
+ const slot = { indent, title, container: parentTopic };
3734
+ const target = link?.[2] ?? "";
3735
+ if (link && target !== "" && !isExternal(target)) {
3736
+ const file = resolveOutlinePath(baseDir, target);
3737
+ if (file === null) {
3738
+ outline.warnings.push(`${summaryFile}: '${target}' leaves the catalog tree; skipped`);
3739
+ } else if (!MARKDOWN_EXT.test(file)) {
3740
+ outline.warnings.push(`${summaryFile}: '${target}' is not a Markdown page; skipped`);
3741
+ } else {
3742
+ slot.entry = { file, ...title ? { title } : {} };
3743
+ parentTopic.entries.push(slot.entry);
3744
+ }
3745
+ }
3746
+ stack.push(slot);
3747
+ }
3748
+ assignOutlineOrders(outline.root);
3749
+ return outline;
3750
+ }
3751
+ function parseMkdocsNav(config, docsDir, configFile = "mkdocs.yml") {
3752
+ if (!isRecord(config)) throw new Error(`${configFile}: not a mapping`);
3753
+ const nav = config.nav;
3754
+ if (nav === void 0 || nav === null) return null;
3755
+ if (!Array.isArray(nav)) throw new Error(`${configFile}: 'nav' must be a list`);
3756
+ const outline = {
3757
+ format: "mkdocs",
3758
+ root: newOutlineTopic(""),
3759
+ consumed: [],
3760
+ warnings: []
3761
+ };
3762
+ const add = (container, target, title) => {
3763
+ if (isExternal(target)) return;
3764
+ const file = resolveOutlinePath(docsDir, target);
3765
+ if (file === null) {
3766
+ outline.warnings.push(
3767
+ `${configFile}: nav entry '${target}' leaves the docs directory; skipped`
3768
+ );
3769
+ return;
3770
+ }
3771
+ if (!MARKDOWN_EXT.test(file)) {
3772
+ outline.warnings.push(`${configFile}: nav entry '${target}' is not a Markdown page; skipped`);
3773
+ return;
3774
+ }
3775
+ container.entries.push({ file, ...title ? { title } : {} });
3776
+ };
3777
+ const walk = (items, container) => {
3778
+ for (const item of items) {
3779
+ if (typeof item === "string") {
3780
+ add(container, item);
3781
+ continue;
3782
+ }
3783
+ if (!isRecord(item)) {
3784
+ outline.warnings.push(`${configFile}: unrecognized nav entry ${JSON.stringify(item)}`);
3785
+ continue;
3786
+ }
3787
+ for (const [title, value] of Object.entries(item)) {
3788
+ if (typeof value === "string") {
3789
+ add(container, value, title.trim());
3790
+ } else if (Array.isArray(value)) {
3791
+ const section = newOutlineTopic(title.trim() || "Section");
3792
+ walk(value, section);
3793
+ container.children.push(section);
3794
+ } else {
3795
+ outline.warnings.push(
3796
+ `${configFile}: nav entry '${title}' is neither a page nor a section`
3797
+ );
3798
+ }
3799
+ }
3800
+ }
3801
+ };
3802
+ walk(nav, outline.root);
3803
+ assignOutlineOrders(outline.root);
3804
+ return outline;
3805
+ }
3806
+ function globToRegExp(pattern) {
3807
+ let source = "";
3808
+ for (let i = 0; i < pattern.length; i++) {
3809
+ const ch = pattern[i];
3810
+ if (ch === "*") {
3811
+ if (pattern[i + 1] === "*") {
3812
+ source += ".*";
3813
+ i++;
3814
+ } else source += "[^/]*";
3815
+ } else if (ch === "?") source += "[^/]";
3816
+ else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
3817
+ }
3818
+ return new RegExp(`^${source}$`);
3819
+ }
3820
+ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
3821
+ const outline = {
3822
+ format: "jupyter-book",
3823
+ root: newOutlineTopic(""),
3824
+ consumed: [],
3825
+ warnings: []
3826
+ };
3827
+ const known = new Set(files.map((f) => f.normalize("NFC")));
3828
+ const resolveFile = (spec) => {
3829
+ const candidates = MARKDOWN_EXT.test(spec) ? [spec] : [`${spec}.md`, `${spec}.markdown`];
3830
+ for (const candidate of candidates) {
3831
+ const path = resolveOutlinePath(tocDir, candidate);
3832
+ if (path !== null && known.has(path.normalize("NFC"))) return path;
3833
+ }
3834
+ return null;
3835
+ };
3836
+ const expandGlob = (pattern) => {
3837
+ const regex = globToRegExp(pattern.replace(/\\/g, "/"));
3838
+ const prefix = tocDir ? `${tocDir}/` : "";
3839
+ return files.filter((file) => {
3840
+ if (!file.startsWith(prefix)) return false;
3841
+ const rel = file.slice(prefix.length);
3842
+ return regex.test(rel) || regex.test(rel.replace(MARKDOWN_EXT, ""));
3843
+ }).sort();
3844
+ };
3845
+ const walkItems = (items, container, where) => {
3846
+ if (items === void 0 || items === null) return;
3847
+ if (!Array.isArray(items)) {
3848
+ outline.warnings.push(`${tocFile}: ${where} must be a list`);
3849
+ return;
3850
+ }
3851
+ for (const item of items) {
3852
+ if (!isRecord(item)) {
3853
+ outline.warnings.push(`${tocFile}: ${where} holds an unrecognized entry`);
3854
+ continue;
3855
+ }
3856
+ const title = typeof item.title === "string" ? item.title.trim() : void 0;
3857
+ if (typeof item.url === "string") continue;
3858
+ if (typeof item.glob === "string") {
3859
+ for (const file2 of expandGlob(item.glob)) container.entries.push({ file: file2 });
3860
+ continue;
3861
+ }
3862
+ if (typeof item.file !== "string") {
3863
+ outline.warnings.push(`${tocFile}: ${where} holds an entry without 'file'`);
3864
+ continue;
3865
+ }
3866
+ const file = resolveFile(item.file);
3867
+ if (file === null) {
3868
+ outline.warnings.push(
3869
+ `${tocFile}: '${item.file}' is not a Markdown file in the tree (notebooks are not supported); skipped`
3870
+ );
3871
+ continue;
3872
+ }
3873
+ const entry = { file, ...title ? { title } : {} };
3874
+ const children = item.sections ?? item.chapters ?? item.entries;
3875
+ if (Array.isArray(children) && children.length > 0) {
3876
+ const topic = newOutlineTopic(title ?? titleFromFolderName(stem(file)));
3877
+ if (!title) topic.titleFromFile = file;
3878
+ topic.entries.push(entry);
3879
+ walkItems(children, topic, `'${item.file}'`);
3880
+ container.children.push(topic);
3881
+ } else {
3882
+ container.entries.push(entry);
3883
+ }
3884
+ }
3885
+ };
3886
+ if (Array.isArray(doc)) {
3887
+ for (const item of doc) {
3888
+ if (isRecord(item) && typeof item.part === "string") {
3889
+ const part = newOutlineTopic(item.part.trim() || "Part");
3890
+ walkItems(item.chapters ?? item.sections, part, `part '${item.part}'`);
3891
+ outline.root.children.push(part);
3892
+ } else {
3893
+ walkItems([item], outline.root, "the list");
3894
+ }
3895
+ }
3896
+ } else if (isRecord(doc)) {
3897
+ if (typeof doc.root === "string") {
3898
+ const file = resolveFile(doc.root);
3899
+ if (file === null) {
3900
+ outline.warnings.push(
3901
+ `${tocFile}: root '${doc.root}' is not a Markdown file in the tree; skipped`
3902
+ );
3903
+ } else {
3904
+ outline.root.entries.push({ file });
3905
+ }
3906
+ }
3907
+ if (Array.isArray(doc.parts)) {
3908
+ for (const part of doc.parts) {
3909
+ if (!isRecord(part)) continue;
3910
+ const caption = typeof part.caption === "string" ? part.caption.trim() : "";
3911
+ const topic = newOutlineTopic(caption || "Part");
3912
+ walkItems(part.chapters ?? part.sections, topic, `part '${topic.name}'`);
3913
+ outline.root.children.push(topic);
3914
+ }
3915
+ }
3916
+ walkItems(doc.chapters, outline.root, "chapters");
3917
+ walkItems(doc.sections, outline.root, "sections");
3918
+ } else {
3919
+ throw new Error(`${tocFile}: not a mapping or a list`);
3920
+ }
3921
+ assignOutlineOrders(outline.root);
3922
+ return outline;
3923
+ }
3924
+
3925
+ // src/markdown-adapter/load.ts
2619
3926
  var ROOT_TOPIC_ID = "general";
2620
3927
  var SUMMARY_MAX_CHARS = 280;
2621
- function parseFrontMatter(markdown) {
2622
- if (!markdown.startsWith("---\n") && !markdown.startsWith("---\r\n")) return { body: markdown };
2623
- const end = markdown.indexOf("\n---", 3);
2624
- if (end === -1) return { body: markdown };
2625
- const head = markdown.slice(markdown.indexOf("\n") + 1, end);
2626
- const body = markdown.slice(markdown.indexOf("\n", end + 1) + 1);
2627
- const out = { body };
2628
- for (const line of head.split("\n")) {
2629
- const at = line.indexOf(":");
2630
- if (at === -1) continue;
2631
- const key = line.slice(0, at).trim();
2632
- const value = line.slice(at + 1).trim();
2633
- if (!value) continue;
2634
- if (key === "title") out.title = value;
2635
- else if (key === "summary") out.summary = value;
2636
- else if (key === "aliases") {
2637
- out.aliases = value.split(",").map((a) => a.trim()).filter(Boolean);
3928
+ var TOPIC_SIDECAR = "_topic.yaml";
3929
+ var HUGO_SECTION_PAGE = "_index";
3930
+ var SECTION_PAGE_ORDINAL = -2147483648;
3931
+ var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
3932
+ function readFrontMatter(raw, file, hugo) {
3933
+ let data;
3934
+ let body;
3935
+ try {
3936
+ ({ data, body } = parseMarkdownFrontMatter(raw));
3937
+ } catch (error) {
3938
+ throw new Error(`${file}: ${error instanceof Error ? error.message : String(error)}`);
3939
+ }
3940
+ const out = { meta: {}, body };
3941
+ const str = (key) => {
3942
+ const value = data[key];
3943
+ if (value === void 0 || value === null) return void 0;
3944
+ if (typeof value === "number") return String(value);
3945
+ if (typeof value !== "string")
3946
+ throw new Error(`${file}: front matter '${key}' must be a string`);
3947
+ return value.trim() || void 0;
3948
+ };
3949
+ out.title = str("title");
3950
+ out.summary = str("summary");
3951
+ out.id = str("id");
3952
+ const aliases = data.aliases;
3953
+ if (typeof aliases === "string") {
3954
+ out.aliases = aliases.split(",").map((a) => a.trim()).filter(Boolean);
3955
+ } else if (Array.isArray(aliases)) {
3956
+ out.aliases = aliases.map((a) => String(a).trim()).filter(Boolean);
3957
+ } else if (aliases !== void 0 && aliases !== null) {
3958
+ throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
3959
+ }
3960
+ if (data.order !== void 0 && data.order !== null) {
3961
+ const order = KnowledgeOrdinalSchema.safeParse(data.order);
3962
+ if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
3963
+ out.order = order.data;
3964
+ }
3965
+ const sub = data.subcategory;
3966
+ if (sub !== void 0 && sub !== null) {
3967
+ if (typeof sub !== "object" || Array.isArray(sub)) {
3968
+ throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
3969
+ }
3970
+ const record = sub;
3971
+ if (typeof record.id !== "string" || !record.id.trim()) {
3972
+ throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
2638
3973
  }
3974
+ const subcategory = { id: record.id.trim() };
3975
+ if (record.title !== void 0 && record.title !== null) {
3976
+ if (typeof record.title !== "string") {
3977
+ throw new Error(`${file}: front matter 'subcategory.title' must be a string`);
3978
+ }
3979
+ subcategory.title = record.title.trim();
3980
+ }
3981
+ if (record.order !== void 0 && record.order !== null) {
3982
+ const order = KnowledgeOrdinalSchema.safeParse(record.order);
3983
+ if (!order.success)
3984
+ throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
3985
+ subcategory.order = order.data;
3986
+ }
3987
+ out.subcategory = subcategory;
3988
+ }
3989
+ if (hugo) {
3990
+ if (data.draft === true) out.unpublished = "draft";
3991
+ else if (data.headless === true) out.unpublished = "headless";
3992
+ if (data.weight !== void 0 && data.weight !== null) {
3993
+ const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
3994
+ if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
3995
+ if (weight.data > 0) out.weight = weight.data;
3996
+ }
3997
+ out.description = str("description");
3998
+ }
3999
+ for (const [key, value] of Object.entries(data)) {
4000
+ if (RESERVED_KEYS.has(key) || value === void 0) continue;
4001
+ out.meta[key] = value;
2639
4002
  }
2640
4003
  return out;
2641
4004
  }
@@ -2677,17 +4040,233 @@ async function walkMarkdownFiles(root) {
2677
4040
  await visit(root);
2678
4041
  return out;
2679
4042
  }
2680
- async function loadMarkdownCatalog(rootDir, opts) {
4043
+ function isRecord2(value) {
4044
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4045
+ }
4046
+ async function readTopicSidecar(dir) {
4047
+ const path = join6(dir, TOPIC_SIDECAR);
4048
+ if (!existsSync(path)) return void 0;
4049
+ const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
4050
+ const data = parseYaml(raw, { what: path });
4051
+ if (data === null || data === void 0) return {};
4052
+ if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
4053
+ const out = {};
4054
+ if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
4055
+ if (typeof data.description === "string" && data.description.trim()) {
4056
+ out.description = data.description.trim();
4057
+ }
4058
+ if (data.order !== void 0 && data.order !== null) {
4059
+ const order = KnowledgeOrdinalSchema.safeParse(data.order);
4060
+ if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
4061
+ out.order = order.data;
4062
+ }
4063
+ return out;
4064
+ }
4065
+ function applyOverride(topic, override) {
4066
+ if (!topic || !override) return;
4067
+ if (override.name) topic.name = override.name;
4068
+ if (override.description) topic.description = override.description;
4069
+ if (override.order !== void 0) topic.sortKey = topicSortKeyForOrder(override.order);
4070
+ }
4071
+ function insideRoot(rootDir, abs) {
4072
+ const rel = relative3(rootDir, abs).split(sep).join("/");
4073
+ if (rel === ".." || rel.startsWith("../") || isAbsolute3(rel)) return null;
4074
+ return rel;
4075
+ }
4076
+ function firstExisting(candidates) {
4077
+ return candidates.find((candidate) => existsSync(candidate));
4078
+ }
4079
+ async function readMkdocsDocsDir(mkdocsPath) {
4080
+ const config = parseYaml((await readFile3(mkdocsPath, "utf8")).replace(/^\uFEFF/, ""), {
4081
+ what: basename(mkdocsPath),
4082
+ maxBytes: OUTLINE_MAX_BYTES,
4083
+ tolerateUnknownTags: true,
4084
+ maxAliasCount: 100
4085
+ });
4086
+ const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
4087
+ return resolve4(dirname3(mkdocsPath), docsDir);
4088
+ }
4089
+ async function detectTableOfContents(rootDir, projectDir = rootDir) {
4090
+ const summary = join6(rootDir, "SUMMARY.md");
4091
+ if (existsSync(summary)) return { format: "gitbook", path: summary };
4092
+ const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
4093
+ if (jupyter) return { format: "jupyter-book", path: jupyter };
4094
+ const mkdocs = firstExisting(
4095
+ [rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
4096
+ join6(dir, "mkdocs.yml"),
4097
+ join6(dir, "mkdocs.yaml")
4098
+ ])
4099
+ );
4100
+ if (mkdocs) return { format: "mkdocs", path: mkdocs };
2681
4101
  const files = await walkMarkdownFiles(rootDir);
4102
+ if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
4103
+ return { format: "hugo" };
4104
+ }
4105
+ return { format: "folders" };
4106
+ }
4107
+ async function readOutlineText(path) {
4108
+ const text = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
4109
+ if (Buffer.byteLength(text, "utf8") > OUTLINE_MAX_BYTES) {
4110
+ throw new Error(`${basename(path)} exceeds ${OUTLINE_MAX_BYTES} bytes`);
4111
+ }
4112
+ return text;
4113
+ }
4114
+ async function loadOutline(rootDir, toc, files) {
4115
+ if (toc.format === "folders" || toc.format === "hugo") return { outline: null };
4116
+ const explicit = toc.path ? isAbsolute3(toc.path) ? toc.path : join6(rootDir, toc.path) : void 0;
4117
+ if (explicit && !existsSync(explicit)) {
4118
+ throw new Error(`${toc.format} table of contents not found: ${explicit}`);
4119
+ }
4120
+ if (toc.format === "gitbook") {
4121
+ const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
4122
+ const rel = insideRoot(rootDir, path2);
4123
+ if (!existsSync(path2)) throw new Error(`gitbook table of contents not found: ${path2}`);
4124
+ if (rel === null)
4125
+ throw new Error(`${basename(path2)} must live inside the content root ${rootDir}`);
4126
+ return { outline: parseGitbookSummary(await readOutlineText(path2), rel), path: path2 };
4127
+ }
4128
+ if (toc.format === "mkdocs") {
4129
+ const path2 = explicit ?? firstExisting(
4130
+ [rootDir, dirname3(rootDir)].flatMap((dir) => [
4131
+ join6(dir, "mkdocs.yml"),
4132
+ join6(dir, "mkdocs.yaml")
4133
+ ])
4134
+ );
4135
+ if (!path2) throw new Error(`mkdocs.yml not found in ${rootDir} or its parent`);
4136
+ const config = parseYaml(await readOutlineText(path2), {
4137
+ what: basename(path2),
4138
+ maxBytes: OUTLINE_MAX_BYTES,
4139
+ tolerateUnknownTags: true,
4140
+ maxAliasCount: 100
4141
+ });
4142
+ const docsDir = await readMkdocsDocsDir(path2);
4143
+ const docsRel = insideRoot(rootDir, docsDir);
4144
+ if (docsRel === null) {
4145
+ throw new Error(
4146
+ `${basename(path2)}: docs_dir ${docsDir} is outside the content root ${rootDir}`
4147
+ );
4148
+ }
4149
+ return { outline: parseMkdocsNav(config, docsRel, basename(path2)), path: path2 };
4150
+ }
4151
+ const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
4152
+ if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
4153
+ const tocDir = insideRoot(rootDir, dirname3(path));
4154
+ if (tocDir === null)
4155
+ throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
4156
+ const doc = parseYaml(await readOutlineText(path), {
4157
+ what: basename(path),
4158
+ maxBytes: OUTLINE_MAX_BYTES
4159
+ });
4160
+ return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
4161
+ }
4162
+ var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
4163
+ var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
4164
+ function resolveInTree(ctx, target) {
4165
+ const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
4166
+ if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
4167
+ return resolved === "." ? "" : resolved;
4168
+ }
4169
+ function rewriteTarget(ctx, isImage, rawTarget) {
4170
+ const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
4171
+ if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
4172
+ return null;
4173
+ }
4174
+ const hashAt = target.search(/[#?]/);
4175
+ const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
4176
+ const decoded = safeDecode2(pathPart);
4177
+ const resolved = resolveInTree(ctx, decoded);
4178
+ if (resolved === null) {
4179
+ ctx.warn(`${ctx.file}: link '${target}' leaves the catalog tree; left as is`);
4180
+ return null;
4181
+ }
4182
+ if (isImage) {
4183
+ if (!assetExtension3(resolved)) {
4184
+ ctx.warn(`${ctx.file}: image '${target}' is not a supported asset type; left as is`);
4185
+ return null;
4186
+ }
4187
+ const abs = join6(ctx.rootDir, ...resolved.split("/"));
4188
+ if (!existsSync(abs)) {
4189
+ const message = `${ctx.file}: image '${target}' does not exist in the catalog tree`;
4190
+ if (ctx.missingAssets === "error") throw new Error(message);
4191
+ ctx.warn(`${message}; left as is`);
4192
+ return null;
4193
+ }
4194
+ const archivePath = resolved.startsWith("assets/") ? resolved : `assets/${resolved}`;
4195
+ if (!ctx.assets.has(archivePath))
4196
+ ctx.assets.set(archivePath, { path: archivePath, absPath: abs });
4197
+ return archivePath;
4198
+ }
4199
+ if (!/\.(md|markdown)$/i.test(resolved)) return null;
4200
+ const documentId = ctx.idByRel.get(resolved.replace(/\.(md|markdown)$/i, "").normalize("NFC"));
4201
+ if (!documentId) {
4202
+ ctx.warn(`${ctx.file}: link '${target}' does not name a document in the catalog; left as is`);
4203
+ return null;
4204
+ }
4205
+ if (!ctx.uri) return null;
4206
+ return formatKnowledgeUri({
4207
+ publisherId: ctx.uri.publisherId,
4208
+ catalogId: ctx.uri.catalogId,
4209
+ documentId
4210
+ });
4211
+ }
4212
+ function safeDecode2(value) {
4213
+ try {
4214
+ return decodeURIComponent(value);
4215
+ } catch {
4216
+ return value;
4217
+ }
4218
+ }
4219
+ function rewriteLinks(ctx, markdown) {
4220
+ const lines = markdown.split("\n");
4221
+ let fence = null;
4222
+ return lines.map((line) => {
4223
+ const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
4224
+ if (fenceMatch) {
4225
+ const marker = fenceMatch[1];
4226
+ if (fence === null) fence = marker[0];
4227
+ else if (marker[0] === fence) fence = null;
4228
+ return line;
4229
+ }
4230
+ if (fence !== null) return line;
4231
+ return line.replace(
4232
+ INLINE_LINK,
4233
+ (whole, bang, text, target, title) => {
4234
+ const rewritten = rewriteTarget(ctx, bang === "!", target);
4235
+ return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
4236
+ }
4237
+ );
4238
+ }).join("\n");
4239
+ }
4240
+ async function loadMarkdownCatalog(rootDir, opts) {
4241
+ const warn = opts.onWarning ?? (() => {
4242
+ });
4243
+ const toc = opts.toc ?? { format: "folders" };
4244
+ const hugo = toc.format === "hugo";
4245
+ const ignore = new Set(opts.ignore ?? []);
4246
+ const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
4247
+ let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
4248
+ const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
4249
+ for (const message of outline?.warnings ?? []) warn(message);
4250
+ if (outline) {
4251
+ const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
4252
+ files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
4253
+ }
2682
4254
  if (files.length === 0) {
2683
4255
  throw new Error(`no Markdown files found under ${rootDir}`);
2684
4256
  }
2685
4257
  const takenTopicIds = /* @__PURE__ */ new Set();
2686
4258
  const topicIdByDir = /* @__PURE__ */ new Map();
2687
4259
  const topics = [];
4260
+ const topicById = /* @__PURE__ */ new Map();
2688
4261
  const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
2689
4262
  let rootTopicUsed = false;
2690
- const ensureTopicChain = (relDirPosix) => {
4263
+ let rootOverride;
4264
+ const addTopic = (topic) => {
4265
+ topics.push(topic);
4266
+ topicById.set(topic.id, topic);
4267
+ return topic;
4268
+ };
4269
+ const ensureTopicChain = async (relDirPosix) => {
2691
4270
  if (!relDirPosix) {
2692
4271
  rootTopicUsed = true;
2693
4272
  return [rootTopicId];
@@ -2701,32 +4280,141 @@ async function loadMarkdownCatalog(rootDir, opts) {
2701
4280
  if (!id) {
2702
4281
  id = topicIdFor(segment, takenTopicIds);
2703
4282
  topicIdByDir.set(prefix, id);
2704
- topics.push({
4283
+ const topic = addTopic({
2705
4284
  id,
2706
- name: segment,
4285
+ name: titleFromFolderName(segment),
2707
4286
  ...path.length > 0 ? { parentId: path[path.length - 1] } : {}
2708
4287
  });
4288
+ applyOverride(topic, await readTopicSidecar(join6(rootDir, ...prefix.split("/"))));
4289
+ applyOverride(topic, opts.topics?.[prefix]);
2709
4290
  }
2710
4291
  path.push(id);
2711
4292
  }
2712
4293
  return path;
2713
4294
  };
2714
- const documents = [];
2715
- const seenIds = /* @__PURE__ */ new Set();
4295
+ const placements = /* @__PURE__ */ new Map();
4296
+ if (outline) {
4297
+ const walk = (topic, chain) => {
4298
+ for (const entry of topic.entries) {
4299
+ const key = entry.file.normalize("NFC");
4300
+ if (placements.has(key)) {
4301
+ warn(
4302
+ `${entry.file}: listed more than once in the table of contents; the first place wins`
4303
+ );
4304
+ } else {
4305
+ placements.set(key, { chain, entry });
4306
+ }
4307
+ }
4308
+ for (const child of topic.children) walk(child, [...chain, child]);
4309
+ };
4310
+ walk(outline.root, []);
4311
+ }
4312
+ const outlineTopicIds = /* @__PURE__ */ new Map();
4313
+ const ensureOutlineChain = (chain) => {
4314
+ if (chain.length === 0) {
4315
+ rootTopicUsed = true;
4316
+ return [rootTopicId];
4317
+ }
4318
+ const ids = [];
4319
+ for (const node of chain) {
4320
+ let id = outlineTopicIds.get(node);
4321
+ if (!id) {
4322
+ id = topicIdFor(node.name, takenTopicIds);
4323
+ outlineTopicIds.set(node, id);
4324
+ const topic = {
4325
+ id,
4326
+ name: node.name,
4327
+ ...ids.length > 0 ? { parentId: ids[ids.length - 1] } : {}
4328
+ };
4329
+ if (node.description) topic.description = node.description;
4330
+ if (node.order !== void 0) topic.sortKey = topicSortKeyForOrder(node.order);
4331
+ addTopic(topic);
4332
+ }
4333
+ ids.push(id);
4334
+ }
4335
+ return ids;
4336
+ };
4337
+ const shelves = /* @__PURE__ */ new Map();
4338
+ const ensureShelf = (parentId, sub, file) => {
4339
+ const key = `${parentId}/${sub.id}`;
4340
+ const existing = shelves.get(key);
4341
+ if (existing) {
4342
+ if (existing.title !== sub.title || existing.order !== sub.order) {
4343
+ throw new Error(
4344
+ `${file}: subcategory '${sub.id}' is declared with a different title or order in ${existing.file}`
4345
+ );
4346
+ }
4347
+ return existing.topicId;
4348
+ }
4349
+ const topicId = topicIdFor(`${parentId}-${sub.id}`, takenTopicIds);
4350
+ const topic = { id: topicId, name: sub.title ?? sub.id, parentId };
4351
+ if (sub.order !== void 0) topic.sortKey = topicSortKeyForOrder(sub.order);
4352
+ addTopic(topic);
4353
+ shelves.set(key, { topicId, title: sub.title, order: sub.order, file });
4354
+ return topicId;
4355
+ };
4356
+ const loaded = [];
4357
+ const fileById = /* @__PURE__ */ new Map();
4358
+ const idByRel = /* @__PURE__ */ new Map();
4359
+ const seenRel = /* @__PURE__ */ new Set();
2716
4360
  for (const abs of files) {
2717
- const rel = relative3(rootDir, abs).split(sep).join("/");
4361
+ const rel = relOf(abs);
2718
4362
  const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
2719
- const topicPath = ensureTopicChain(relDir);
2720
- const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
2721
- if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
2722
- seenIds.add(id);
4363
+ const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
2723
4364
  const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
2724
- const fm = parseFrontMatter(raw);
2725
- const stem = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
2726
- const title = fm.title ?? firstHeading(fm.body) ?? stem;
2727
- const summary = fm.summary ?? firstParagraph(fm.body);
2728
- documents.push(
2729
- CatalogDocumentSchema.parse({
4365
+ const fm = readFrontMatter(raw, rel, hugo);
4366
+ if (fm.unpublished) {
4367
+ warn(`${rel}: skipped, ${fm.unpublished} page`);
4368
+ continue;
4369
+ }
4370
+ seenRel.add(rel.normalize("NFC"));
4371
+ const id = fm.id ?? relNoExt;
4372
+ const idCheck = KnowledgeDocumentIdSchema.safeParse(id);
4373
+ if (!idCheck.success) throw new Error(`${rel}: front matter 'id' is not a valid document id`);
4374
+ const previous = fileById.get(id);
4375
+ if (previous) throw new Error(`${rel}: document id '${id}' is already used by ${previous}`);
4376
+ fileById.set(id, rel);
4377
+ idByRel.set(relNoExt, id);
4378
+ const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
4379
+ let title = fm.title ?? firstHeading(fm.body) ?? stem2;
4380
+ let ordinal = fm.order;
4381
+ let topicPath;
4382
+ const placement = placements.get(rel.normalize("NFC"));
4383
+ if (placement) {
4384
+ topicPath = ensureOutlineChain(placement.chain);
4385
+ if (placement.entry.title) title = placement.entry.title;
4386
+ if (placement.entry.order !== void 0) ordinal = placement.entry.order;
4387
+ } else {
4388
+ if (outline)
4389
+ warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
4390
+ topicPath = await ensureTopicChain(relDir);
4391
+ if (hugo && ordinal === void 0) ordinal = fm.weight;
4392
+ }
4393
+ if (hugo && stem2 === HUGO_SECTION_PAGE) {
4394
+ const override = {
4395
+ ...fm.title ? { name: fm.title } : {},
4396
+ ...fm.description ?? fm.summary ? { description: fm.description ?? fm.summary } : {},
4397
+ ...fm.weight !== void 0 ? { order: fm.weight } : {}
4398
+ };
4399
+ if (relDir === "") {
4400
+ rootOverride = override;
4401
+ } else {
4402
+ const leaf = topicById.get(topicPath[topicPath.length - 1]);
4403
+ applyOverride(leaf, override);
4404
+ applyOverride(leaf, opts.topics?.[relDir]);
4405
+ }
4406
+ if (!fm.body.trim()) continue;
4407
+ ordinal = SECTION_PAGE_ORDINAL;
4408
+ }
4409
+ if (fm.subcategory) {
4410
+ const parentId = topicPath[topicPath.length - 1];
4411
+ topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
4412
+ }
4413
+ const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
4414
+ loaded.push({
4415
+ rel,
4416
+ file: rel,
4417
+ doc: {
2730
4418
  id,
2731
4419
  title,
2732
4420
  slug: documentSlug(title),
@@ -2734,9 +4422,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
2734
4422
  language: opts.language,
2735
4423
  topicPath,
2736
4424
  markdown: fm.body,
2737
- ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
2738
- })
4425
+ ...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
4426
+ ...ordinal !== void 0 ? { ordinal } : {},
4427
+ ...Object.keys(fm.meta).length > 0 ? { meta: fm.meta } : {}
4428
+ }
4429
+ });
4430
+ }
4431
+ if (outline) {
4432
+ const titleByRel = new Map(
4433
+ loaded.map((entry) => [entry.rel.normalize("NFC"), entry.doc.title])
2739
4434
  );
4435
+ const nameFromPages = (topic) => {
4436
+ for (const child of topic.children) {
4437
+ const id = outlineTopicIds.get(child);
4438
+ const title = child.titleFromFile ? titleByRel.get(child.titleFromFile.normalize("NFC")) : void 0;
4439
+ const compiled = id ? topicById.get(id) : void 0;
4440
+ if (compiled && title) compiled.name = title;
4441
+ nameFromPages(child);
4442
+ }
4443
+ };
4444
+ nameFromPages(outline.root);
4445
+ for (const [file, placement] of placements) {
4446
+ if (!seenRel.has(file)) {
4447
+ warn(
4448
+ `${placement.entry.file}: named by the table of contents but not found among the Markdown files`
4449
+ );
4450
+ }
4451
+ }
2740
4452
  }
2741
4453
  if (rootTopicUsed) {
2742
4454
  if (takenTopicIds.has(rootTopicId)) {
@@ -2744,9 +4456,39 @@ async function loadMarkdownCatalog(rootDir, opts) {
2744
4456
  `root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
2745
4457
  );
2746
4458
  }
2747
- topics.unshift({ id: rootTopicId, name: opts.rootTopicName ?? "General" });
4459
+ const root = { id: rootTopicId, name: opts.rootTopicName ?? "General" };
4460
+ applyOverride(root, await readTopicSidecar(rootDir));
4461
+ applyOverride(root, rootOverride);
4462
+ applyOverride(root, opts.topics?.[""]);
4463
+ topics.unshift(root);
4464
+ topicById.set(root.id, root);
2748
4465
  }
2749
- return { topics, documents };
4466
+ const assets = /* @__PURE__ */ new Map();
4467
+ const documents = [];
4468
+ for (const entry of loaded) {
4469
+ const ctx = {
4470
+ rootDir,
4471
+ docDir: entry.rel.includes("/") ? entry.rel.slice(0, entry.rel.lastIndexOf("/")) : "",
4472
+ file: entry.file,
4473
+ idByRel,
4474
+ uri: opts.uri,
4475
+ assets,
4476
+ missingAssets: opts.missingAssets ?? "error",
4477
+ warn
4478
+ };
4479
+ documents.push(
4480
+ CatalogDocumentSchema.parse({
4481
+ ...entry.doc,
4482
+ markdown: rewriteLinks(ctx, entry.doc.markdown)
4483
+ })
4484
+ );
4485
+ }
4486
+ return {
4487
+ topics,
4488
+ documents,
4489
+ assets: [...assets.values()].sort((a, b) => a.path < b.path ? -1 : 1),
4490
+ toc: { format: toc.format, ...outlinePath ? { path: outlinePath } : {} }
4491
+ };
2750
4492
  }
2751
4493
 
2752
4494
  // src/registry-client/fetch.ts
@@ -2861,7 +4603,7 @@ function newerRegistryEntries(registry, installed) {
2861
4603
  var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
2862
4604
 
2863
4605
  // src/signatures/jcs.ts
2864
- import { canonicalizeJson } from "@bendyline/gezk";
4606
+ import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
2865
4607
  export {
2866
4608
  BGE_SMALL_EN_V15_1,
2867
4609
  BODY_CODEC_MIN_BYTES,
@@ -2880,9 +4622,12 @@ export {
2880
4622
  GEZEL_MULTILINGUAL_E5_SMALL_1,
2881
4623
  GEZK_APPLICATION_ID,
2882
4624
  GEZK_ARCHIVE_LIMITS,
4625
+ GEZK_FORMAT_GENERATIONS,
2883
4626
  GEZK_FORMAT_VERSION,
2884
4627
  GEZK_INDEX_SCHEMA_VERSION,
2885
4628
  GEZK_MIME_TYPE,
4629
+ GEZK_SUPPORTED_FORMAT_VERSIONS,
4630
+ GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
2886
4631
  GezkArchiveError,
2887
4632
  KNOWLEDGE_EMBEDDING_PROFILES,
2888
4633
  KNOWLEDGE_TOOLCHAIN,
@@ -2891,11 +4636,16 @@ export {
2891
4636
  MANIFEST_PATH,
2892
4637
  MARKDOWN_CHUNKS_2,
2893
4638
  MAX_KNOWLEDGE_DOCUMENT_BYTES,
4639
+ MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
4640
+ MAX_KNOWLEDGE_TOPIC_DEPTH,
2894
4641
  MIMETYPE_PATH,
2895
4642
  MULTILINGUAL_E5_SMALL_1,
4643
+ MULTILINGUAL_E5_SMALL_2,
2896
4644
  PARQUET_EXPORT_VERSION,
2897
4645
  PARQUET_REPORT_PATH,
2898
4646
  README_PATH,
4647
+ RERANK_CANDIDATES_MAX,
4648
+ RERANK_CANDIDATES_MIN,
2899
4649
  RERANK_FINAL_K,
2900
4650
  ROUTER_DB_PATH,
2901
4651
  ROUTER_DDL,
@@ -2907,13 +4657,18 @@ export {
2907
4657
  SOURCE_NOTICES_PATH,
2908
4658
  ZIP_FIXED_MTIME,
2909
4659
  assertDuckdbCli,
2910
- canonicalizeJson,
4660
+ assetReferences,
4661
+ assignOutlineOrders,
4662
+ asymmetricTopK,
4663
+ canonicalizeJson2 as canonicalizeJson,
4664
+ centerVector,
2911
4665
  chunkContentHash,
2912
4666
  chunkMarkdownProfile,
2913
4667
  chunkUid,
2914
4668
  compareCatalogVersions,
2915
4669
  compileKnowledgeCatalog,
2916
4670
  createProfileEmbedder,
4671
+ detectTableOfContents,
2917
4672
  documentSlug,
2918
4673
  duckdbVersion,
2919
4674
  exportCatalogParquet,
@@ -2924,22 +4679,32 @@ export {
2924
4679
  generateKnowledgeSigningKeyPair,
2925
4680
  hammingTopK,
2926
4681
  inspectGezkArchive,
4682
+ isSupportedFormatVersion,
4683
+ isSupportedIndexSchemaVersion,
2927
4684
  knowledgeEmbeddingProfile,
2928
4685
  knowledgeKeyId,
2929
4686
  l2Normalize,
2930
4687
  loadMarkdownCatalog,
2931
4688
  manifestSigningPayload,
4689
+ newOutlineTopic,
2932
4690
  newerRegistryEntries,
2933
4691
  openCatalogDatabase,
4692
+ parseGitbookSummary,
4693
+ parseJupyterBookToc,
4694
+ parseMkdocsNav,
2934
4695
  quantizeBinary,
4696
+ quantizeBinaryForProfile,
2935
4697
  quantizeInt8,
2936
4698
  readGezkManifest,
4699
+ readMkdocsDocsDir,
2937
4700
  rerankK,
2938
4701
  rerankScore,
4702
+ resolveOutlinePath,
2939
4703
  resolveTransformersModelOptions,
2940
4704
  runDuckdbScript,
2941
4705
  signManifest,
2942
4706
  signRegistryIndex,
4707
+ titleFromFolderName,
2943
4708
  transformersCachePath,
2944
4709
  validateExtractedCatalog,
2945
4710
  verifyManifestSignature,