@bendyline/gezel-knowledge 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.ts +238 -12
- package/dist/index.js +1336 -95
- package/package.json +3 -2
package/dist/index.js
CHANGED
|
@@ -2,17 +2,24 @@
|
|
|
2
2
|
import {
|
|
3
3
|
BODY_CODEC_MIN_BYTES,
|
|
4
4
|
GEZK_APPLICATION_ID,
|
|
5
|
+
GEZK_FORMAT_GENERATIONS,
|
|
5
6
|
GEZK_FORMAT_VERSION,
|
|
6
7
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
7
8
|
GEZK_MIME_TYPE,
|
|
9
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
10
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
8
11
|
LICENSE_NOTICE_PATH,
|
|
9
12
|
MANIFEST_PATH,
|
|
10
13
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
14
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
15
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
11
16
|
MIMETYPE_PATH,
|
|
12
17
|
README_PATH,
|
|
13
18
|
ROUTER_DB_PATH,
|
|
14
19
|
SOURCE_NOTICES_PATH,
|
|
15
|
-
ZIP_FIXED_MTIME
|
|
20
|
+
ZIP_FIXED_MTIME,
|
|
21
|
+
isSupportedFormatVersion,
|
|
22
|
+
isSupportedIndexSchemaVersion
|
|
16
23
|
} from "@bendyline/gezk";
|
|
17
24
|
var SHARD_TARGET_CHUNKS = 2e5;
|
|
18
25
|
var SHARD_MAX_CHUNKS = 25e4;
|
|
@@ -165,11 +172,23 @@ function splitOversizeLine(line, opts) {
|
|
|
165
172
|
|
|
166
173
|
// src/compiler/compile.ts
|
|
167
174
|
import { createHash } from "crypto";
|
|
168
|
-
import { mkdirSync, rmSync, statSync } from "fs";
|
|
175
|
+
import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
|
|
169
176
|
import { stat } from "fs/promises";
|
|
170
177
|
import { join } from "path";
|
|
171
178
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
172
|
-
import {
|
|
179
|
+
import {
|
|
180
|
+
GEZK_MANIFEST_KIND,
|
|
181
|
+
KnowledgeAssetPathSchema,
|
|
182
|
+
KnowledgeCatalogManifestSchema,
|
|
183
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
184
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
185
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
186
|
+
assetExtension,
|
|
187
|
+
assetKindForExtension,
|
|
188
|
+
canonicalizeJson,
|
|
189
|
+
sniffAssetType,
|
|
190
|
+
svgInertnessProblem
|
|
191
|
+
} from "@bendyline/gezk";
|
|
173
192
|
|
|
174
193
|
// src/archive/write.ts
|
|
175
194
|
import { createWriteStream } from "fs";
|
|
@@ -195,10 +214,10 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
195
214
|
else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
|
|
196
215
|
else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
|
|
197
216
|
}
|
|
198
|
-
await new Promise((
|
|
217
|
+
await new Promise((resolve5, reject) => {
|
|
199
218
|
const out = createWriteStream(outputPath);
|
|
200
219
|
out.on("error", reject);
|
|
201
|
-
out.on("close", () =>
|
|
220
|
+
out.on("close", () => resolve5());
|
|
202
221
|
zip.outputStream.on("error", reject);
|
|
203
222
|
zip.outputStream.pipe(out);
|
|
204
223
|
zip.end();
|
|
@@ -250,7 +269,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
250
269
|
"a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
|
|
251
270
|
);
|
|
252
271
|
}
|
|
253
|
-
const
|
|
272
|
+
const topicById = assertTopicForest(opts.topics);
|
|
273
|
+
const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
|
|
254
274
|
const profile = opts.embeddingProfile;
|
|
255
275
|
const chunkerOpts = {
|
|
256
276
|
unit: opts.chunkingProfile.unit,
|
|
@@ -263,13 +283,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
263
283
|
for await (const doc of opts.documents) {
|
|
264
284
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
265
285
|
seenIds.add(doc.id);
|
|
266
|
-
const rootTopic = doc.topicPath[0];
|
|
267
|
-
if (!rootTopic || !topicIds.has(rootTopic)) {
|
|
268
|
-
throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
|
|
269
|
-
}
|
|
270
286
|
prepared.push({
|
|
271
287
|
doc,
|
|
272
288
|
topicPathKey: doc.topicPath.join("/"),
|
|
289
|
+
leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
|
|
290
|
+
metaJson: encodeDocumentMeta(doc),
|
|
273
291
|
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
274
292
|
});
|
|
275
293
|
}
|
|
@@ -336,8 +354,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
336
354
|
});
|
|
337
355
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
338
356
|
for (const p of prepared) {
|
|
339
|
-
|
|
340
|
-
topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
|
|
357
|
+
topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
|
|
341
358
|
}
|
|
342
359
|
{
|
|
343
360
|
const stmt = router.prepare(
|
|
@@ -356,9 +373,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
356
373
|
}
|
|
357
374
|
{
|
|
358
375
|
const insertDoc = router.prepare(
|
|
359
|
-
`INSERT INTO documents (id, title, slug, summary, language, topic_id,
|
|
360
|
-
source_url, source_revision, source_updated_at, attribution_json,
|
|
361
|
-
|
|
376
|
+
`INSERT INTO documents (id, title, slug, summary, language, topic_id, ordinal, shard_id,
|
|
377
|
+
chunk_count, source_url, source_revision, source_updated_at, attribution_json, meta_json,
|
|
378
|
+
body_codec, body_blob)
|
|
379
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
362
380
|
);
|
|
363
381
|
const insertAlias = router.prepare(
|
|
364
382
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
@@ -386,13 +404,15 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
386
404
|
p.doc.slug || documentSlug(p.doc.title),
|
|
387
405
|
p.doc.summary ?? null,
|
|
388
406
|
p.doc.language,
|
|
389
|
-
p.
|
|
407
|
+
p.leafTopicId,
|
|
408
|
+
p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
|
|
390
409
|
BigInt(shardOf.get(p.doc.id)),
|
|
391
410
|
BigInt(p.chunks.length),
|
|
392
411
|
p.doc.sourceUrl ?? null,
|
|
393
412
|
p.doc.sourceRevision ?? null,
|
|
394
413
|
p.doc.sourceUpdatedAt ?? null,
|
|
395
414
|
p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
|
|
415
|
+
p.metaJson,
|
|
396
416
|
useBrotli ? "br" : "none",
|
|
397
417
|
blob
|
|
398
418
|
);
|
|
@@ -468,7 +488,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
468
488
|
const db = shardDb(shardId);
|
|
469
489
|
shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
|
|
470
490
|
const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
|
|
471
|
-
topics.add(p.
|
|
491
|
+
topics.add(p.leafTopicId);
|
|
472
492
|
shardTopicIds.set(shardId, topics);
|
|
473
493
|
const insertChunk = db.prepare(
|
|
474
494
|
`INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
|
|
@@ -631,6 +651,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
631
651
|
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
632
652
|
});
|
|
633
653
|
}
|
|
654
|
+
for (const asset of preparedAssets) {
|
|
655
|
+
files.push({ path: asset.path, sizeBytes: asset.sizeBytes, sha256: asset.sha256 });
|
|
656
|
+
}
|
|
657
|
+
assertAssetReferencesDeclared(prepared, preparedAssets);
|
|
634
658
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
635
659
|
kind: GEZK_MANIFEST_KIND,
|
|
636
660
|
formatVersion: GEZK_FORMAT_VERSION,
|
|
@@ -649,13 +673,24 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
649
673
|
},
|
|
650
674
|
embedding: profile,
|
|
651
675
|
chunking: opts.chunkingProfile,
|
|
652
|
-
topics: opts.topics.map((t) => ({
|
|
676
|
+
topics: opts.topics.map((t) => ({
|
|
677
|
+
id: t.id,
|
|
678
|
+
name: t.name,
|
|
679
|
+
...t.parentId ? { parentId: t.parentId } : {},
|
|
680
|
+
sortKey: t.sortKey ?? t.name.toLowerCase(),
|
|
681
|
+
...t.description ? { description: t.description } : {}
|
|
682
|
+
})),
|
|
653
683
|
router: {
|
|
654
684
|
shardTargetChunks: shardTarget,
|
|
655
685
|
shards: routerShardStats.sort((a, b) => a.id - b.id),
|
|
656
686
|
totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
|
|
657
687
|
},
|
|
658
|
-
counts: {
|
|
688
|
+
counts: {
|
|
689
|
+
documents: prepared.length,
|
|
690
|
+
chunks: totalChunks,
|
|
691
|
+
shards: shardCount,
|
|
692
|
+
assets: preparedAssets.length
|
|
693
|
+
},
|
|
659
694
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
660
695
|
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
661
696
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
@@ -672,7 +707,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
672
707
|
...Object.entries(extraFiles).map(([path, content]) => ({
|
|
673
708
|
path,
|
|
674
709
|
content: Buffer.from(content, "utf8")
|
|
675
|
-
}))
|
|
710
|
+
})),
|
|
711
|
+
...preparedAssets.map((asset) => ({ path: asset.path, content: asset.bytes }))
|
|
676
712
|
];
|
|
677
713
|
await writeGezkArchive(opts.outputPath, archiveEntries);
|
|
678
714
|
const archiveBytes = (await stat(opts.outputPath)).size;
|
|
@@ -688,6 +724,134 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
688
724
|
closeAll();
|
|
689
725
|
}
|
|
690
726
|
}
|
|
727
|
+
function assertTopicForest(topics) {
|
|
728
|
+
const byId = /* @__PURE__ */ new Map();
|
|
729
|
+
for (const topic of topics) {
|
|
730
|
+
if (byId.has(topic.id)) throw new Error(`duplicate topic id: ${topic.id}`);
|
|
731
|
+
byId.set(topic.id, topic);
|
|
732
|
+
}
|
|
733
|
+
for (const topic of topics) {
|
|
734
|
+
const seen = /* @__PURE__ */ new Set();
|
|
735
|
+
let depth = 0;
|
|
736
|
+
let current = topic;
|
|
737
|
+
while (current) {
|
|
738
|
+
if (seen.has(current.id)) throw new Error(`topic '${topic.id}' sits in a parent cycle`);
|
|
739
|
+
seen.add(current.id);
|
|
740
|
+
depth += 1;
|
|
741
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
742
|
+
throw new Error(`topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`);
|
|
743
|
+
}
|
|
744
|
+
if (current.parentId === void 0) break;
|
|
745
|
+
const parent = byId.get(current.parentId);
|
|
746
|
+
if (!parent) {
|
|
747
|
+
throw new Error(`topic '${current.id}' names an undeclared parent '${current.parentId}'`);
|
|
748
|
+
}
|
|
749
|
+
current = parent;
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
return byId;
|
|
753
|
+
}
|
|
754
|
+
function assertTopicPath(topicById, docId, path) {
|
|
755
|
+
for (let i = 0; i < path.length; i++) {
|
|
756
|
+
const id = path[i];
|
|
757
|
+
const topic = topicById.get(id);
|
|
758
|
+
if (!topic) {
|
|
759
|
+
throw new Error(
|
|
760
|
+
`document ${docId} names unknown topic '${id}' (segment ${i} of ${path.join("/")})`
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
const expectedParent = i === 0 ? void 0 : path[i - 1];
|
|
764
|
+
if (topic.parentId !== expectedParent) {
|
|
765
|
+
throw new Error(
|
|
766
|
+
`document ${docId}: topic path ${path.join("/")} does not follow the declared tree at '${id}' (its parent is '${topic.parentId ?? "(root)"}')`
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
return path[path.length - 1];
|
|
771
|
+
}
|
|
772
|
+
function encodeDocumentMeta(doc) {
|
|
773
|
+
if (!doc.meta || Object.keys(doc.meta).length === 0) return null;
|
|
774
|
+
let json;
|
|
775
|
+
try {
|
|
776
|
+
json = canonicalizeJson(doc.meta);
|
|
777
|
+
} catch (error) {
|
|
778
|
+
throw new Error(
|
|
779
|
+
`document ${doc.id} has metadata that cannot be serialized: ${error instanceof Error ? error.message : String(error)}`
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
const bytes = Buffer.byteLength(json, "utf8");
|
|
783
|
+
if (bytes > MAX_KNOWLEDGE_DOCUMENT_META_BYTES) {
|
|
784
|
+
throw new Error(
|
|
785
|
+
`document ${doc.id} has ${bytes} bytes of metadata; the limit is ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES}`
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
return json;
|
|
789
|
+
}
|
|
790
|
+
function prepareAssets(assets, extraFiles) {
|
|
791
|
+
for (const path of Object.keys(extraFiles)) {
|
|
792
|
+
if (path.startsWith("assets/")) {
|
|
793
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
797
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
798
|
+
}
|
|
799
|
+
const seen = /* @__PURE__ */ new Set();
|
|
800
|
+
const prepared = [];
|
|
801
|
+
let total = 0;
|
|
802
|
+
for (const asset of assets) {
|
|
803
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
804
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
805
|
+
const key = asset.path.toLowerCase();
|
|
806
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
807
|
+
seen.add(key);
|
|
808
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
809
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
810
|
+
}
|
|
811
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
812
|
+
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
813
|
+
throw new Error(
|
|
814
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
total += bytes.byteLength;
|
|
818
|
+
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
819
|
+
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
820
|
+
}
|
|
821
|
+
const ext = assetExtension(asset.path);
|
|
822
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
823
|
+
const actual = sniffAssetType(bytes);
|
|
824
|
+
if (expected === null || actual !== expected) {
|
|
825
|
+
throw new Error(
|
|
826
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
if (ext === "svg") {
|
|
830
|
+
const problem = svgInertnessProblem(bytes);
|
|
831
|
+
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
832
|
+
}
|
|
833
|
+
prepared.push({
|
|
834
|
+
path: asset.path,
|
|
835
|
+
bytes,
|
|
836
|
+
sizeBytes: bytes.byteLength,
|
|
837
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
841
|
+
}
|
|
842
|
+
function assertAssetReferencesDeclared(prepared, assets) {
|
|
843
|
+
const declared = new Set(assets.map((a) => a.path));
|
|
844
|
+
for (const p of prepared) {
|
|
845
|
+
for (const match of p.doc.markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
846
|
+
const target = match[1];
|
|
847
|
+
if (!declared.has(target)) {
|
|
848
|
+
throw new Error(
|
|
849
|
+
`document ${p.doc.id} references '${target}', which is not among the catalog's assets`
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
691
855
|
function createCatalogDb(absPath) {
|
|
692
856
|
const db = new DatabaseSync(absPath);
|
|
693
857
|
db.exec("PRAGMA page_size=8192");
|
|
@@ -960,9 +1124,9 @@ function assertMimetypeEntry(entry, position) {
|
|
|
960
1124
|
function assertSupportedFormat(raw) {
|
|
961
1125
|
const kind = raw?.kind;
|
|
962
1126
|
const version = raw?.formatVersion;
|
|
963
|
-
if (kind !== GEZK_MANIFEST_KIND2 || version
|
|
1127
|
+
if (kind !== GEZK_MANIFEST_KIND2 || !isSupportedFormatVersion(version)) {
|
|
964
1128
|
throw new GezkArchiveError(
|
|
965
|
-
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${
|
|
1129
|
+
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_SUPPORTED_FORMAT_VERSIONS.join(", ")} \u2014 catalogs built for another version need a matching reader`,
|
|
966
1130
|
"format-version"
|
|
967
1131
|
);
|
|
968
1132
|
}
|
|
@@ -1202,9 +1366,9 @@ function openCatalogDatabase(absPath) {
|
|
|
1202
1366
|
);
|
|
1203
1367
|
}
|
|
1204
1368
|
const schemaVersion = readPragmaNumber(db, "user_version");
|
|
1205
|
-
if (schemaVersion
|
|
1369
|
+
if (!isSupportedIndexSchemaVersion(schemaVersion)) {
|
|
1206
1370
|
throw new CatalogOpenError(
|
|
1207
|
-
`unsupported index schema version ${schemaVersion} (this reader supports ${
|
|
1371
|
+
`unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.join(", ")}; catalogs built for another generation need a matching reader)`,
|
|
1208
1372
|
"schema-version"
|
|
1209
1373
|
);
|
|
1210
1374
|
}
|
|
@@ -1212,6 +1376,7 @@ function openCatalogDatabase(absPath) {
|
|
|
1212
1376
|
db.exec("PRAGMA cache_size=-8192");
|
|
1213
1377
|
return {
|
|
1214
1378
|
db,
|
|
1379
|
+
schemaVersion,
|
|
1215
1380
|
close: () => {
|
|
1216
1381
|
try {
|
|
1217
1382
|
db.close();
|
|
@@ -1234,8 +1399,14 @@ function readPragmaNumber(db, pragma) {
|
|
|
1234
1399
|
}
|
|
1235
1400
|
|
|
1236
1401
|
// src/reader/catalog-handle.ts
|
|
1402
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
1237
1403
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1238
1404
|
import { brotliDecompressSync } from "zlib";
|
|
1405
|
+
import {
|
|
1406
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
|
|
1407
|
+
assetContentType,
|
|
1408
|
+
isKnowledgeAssetPath
|
|
1409
|
+
} from "@bendyline/gezk";
|
|
1239
1410
|
|
|
1240
1411
|
// src/reader/bit-scan.ts
|
|
1241
1412
|
var POPCOUNT8 = new Uint8Array(256);
|
|
@@ -1309,11 +1480,13 @@ function hammingTopK(index, query, k) {
|
|
|
1309
1480
|
}
|
|
1310
1481
|
|
|
1311
1482
|
// src/reader/catalog-handle.ts
|
|
1483
|
+
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
1312
1484
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
1313
1485
|
var CatalogHandle = class _CatalogHandle {
|
|
1314
1486
|
constructor(rootDir, router) {
|
|
1315
1487
|
this.rootDir = rootDir;
|
|
1316
1488
|
this.router = router;
|
|
1489
|
+
this.schemaVersion = router.schemaVersion;
|
|
1317
1490
|
this.meta = {};
|
|
1318
1491
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1319
1492
|
this.meta[row.key] = row.value;
|
|
@@ -1325,8 +1498,11 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1325
1498
|
router;
|
|
1326
1499
|
connections = /* @__PURE__ */ new Map();
|
|
1327
1500
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
1501
|
+
assetsByPath = null;
|
|
1328
1502
|
meta;
|
|
1329
1503
|
shards;
|
|
1504
|
+
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
1505
|
+
schemaVersion;
|
|
1330
1506
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1331
1507
|
static open(rootDir) {
|
|
1332
1508
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
@@ -1428,40 +1604,124 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1428
1604
|
return target;
|
|
1429
1605
|
}
|
|
1430
1606
|
// ── browsing (the shipped table of contents) ──────────────────────────────
|
|
1607
|
+
/** The format version the router was written with (`meta.format_version`). */
|
|
1608
|
+
get formatVersion() {
|
|
1609
|
+
return this.meta.format_version ?? "";
|
|
1610
|
+
}
|
|
1611
|
+
documentColumns() {
|
|
1612
|
+
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1613
|
+
return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
1614
|
+
}
|
|
1615
|
+
documentOrder() {
|
|
1616
|
+
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
1617
|
+
}
|
|
1618
|
+
/**
|
|
1619
|
+
* Every topic with its direct count and a rollup over its subtree. On a
|
|
1620
|
+
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1621
|
+
*/
|
|
1431
1622
|
topics() {
|
|
1432
1623
|
return this.router.db.prepare(
|
|
1433
|
-
|
|
1624
|
+
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1625
|
+
SELECT id, id, 0 FROM topics
|
|
1626
|
+
UNION ALL
|
|
1627
|
+
SELECT sub.root, t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1628
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1629
|
+
),
|
|
1630
|
+
rollup AS (
|
|
1631
|
+
SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
1632
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
1633
|
+
)
|
|
1634
|
+
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1635
|
+
rollup.total AS total_document_count
|
|
1636
|
+
FROM topics t JOIN rollup ON rollup.id = t.id
|
|
1637
|
+
ORDER BY t.sort_key, t.id`
|
|
1434
1638
|
).all().map((r) => ({
|
|
1435
1639
|
id: r.id,
|
|
1436
1640
|
parentId: r.parent_id,
|
|
1437
1641
|
name: r.name,
|
|
1438
1642
|
description: r.description,
|
|
1439
1643
|
sortKey: r.sort_key,
|
|
1440
|
-
documentCount: Number(r.document_count)
|
|
1644
|
+
documentCount: Number(r.document_count),
|
|
1645
|
+
totalDocumentCount: Number(r.total_document_count)
|
|
1441
1646
|
}));
|
|
1442
1647
|
}
|
|
1648
|
+
/**
|
|
1649
|
+
* A page of documents, by default including those filed under the topic's
|
|
1650
|
+
* descendants (`descendants: false` lists only the topic's own).
|
|
1651
|
+
*/
|
|
1443
1652
|
documentsPage(opts = {}) {
|
|
1444
1653
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1445
1654
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1446
|
-
const
|
|
1447
|
-
|
|
1655
|
+
const descendants = opts.descendants ?? true;
|
|
1656
|
+
let scope = "";
|
|
1657
|
+
let where = "";
|
|
1658
|
+
const params = [];
|
|
1659
|
+
if (opts.topicId && descendants) {
|
|
1660
|
+
scope = `WITH RECURSIVE sub(id, depth) AS (
|
|
1661
|
+
SELECT ?, 0
|
|
1662
|
+
UNION ALL
|
|
1663
|
+
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1664
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1665
|
+
)`;
|
|
1666
|
+
where = "WHERE topic_id IN (SELECT id FROM sub)";
|
|
1667
|
+
params.push(opts.topicId);
|
|
1668
|
+
} else if (opts.topicId) {
|
|
1669
|
+
where = "WHERE topic_id = ?";
|
|
1670
|
+
params.push(opts.topicId);
|
|
1671
|
+
}
|
|
1448
1672
|
const total = Number(
|
|
1449
|
-
this.router.db.prepare(
|
|
1673
|
+
this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
|
|
1450
1674
|
);
|
|
1451
1675
|
const documents = this.router.db.prepare(
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
|
|
1676
|
+
`${scope} SELECT ${this.documentColumns()}
|
|
1677
|
+
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1455
1678
|
).all(...params, limit, offset).map(rowToDocumentMeta);
|
|
1456
1679
|
return { documents, total };
|
|
1457
1680
|
}
|
|
1458
1681
|
getDocument(id) {
|
|
1459
1682
|
const row = this.router.db.prepare(
|
|
1460
|
-
`SELECT
|
|
1461
|
-
source_updated_at, attribution_json, body_codec, body_blob
|
|
1683
|
+
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1462
1684
|
FROM documents WHERE id = ?`
|
|
1463
1685
|
).get(id);
|
|
1464
1686
|
if (!row) return null;
|
|
1687
|
+
return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
|
|
1688
|
+
}
|
|
1689
|
+
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
1690
|
+
documentsWithUndeclaredTopic() {
|
|
1691
|
+
return Number(
|
|
1692
|
+
this.router.db.prepare(
|
|
1693
|
+
"SELECT COUNT(*) AS n FROM documents d LEFT JOIN topics t ON t.id = d.topic_id WHERE t.id IS NULL"
|
|
1694
|
+
).get().n
|
|
1695
|
+
);
|
|
1696
|
+
}
|
|
1697
|
+
/**
|
|
1698
|
+
* Validator support: every `meta_json` must be a JSON object within the
|
|
1699
|
+
* byte limit. Returns the offending ids (capped) rather than throwing.
|
|
1700
|
+
*/
|
|
1701
|
+
checkDocumentMeta(maxBytes, cap = 5) {
|
|
1702
|
+
const invalid = [];
|
|
1703
|
+
const oversize = [];
|
|
1704
|
+
if (this.schemaVersion < 3) return { invalid, oversize };
|
|
1705
|
+
const rows = this.router.db.prepare("SELECT id, meta_json FROM documents WHERE meta_json IS NOT NULL").iterate();
|
|
1706
|
+
for (const row of rows) {
|
|
1707
|
+
if (invalid.length >= cap && oversize.length >= cap) break;
|
|
1708
|
+
if (Buffer.byteLength(row.meta_json, "utf8") > maxBytes) {
|
|
1709
|
+
if (oversize.length < cap) oversize.push(row.id);
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1712
|
+
if (parseJsonObject(row.meta_json) === null && invalid.length < cap) invalid.push(row.id);
|
|
1713
|
+
}
|
|
1714
|
+
return { invalid, oversize };
|
|
1715
|
+
}
|
|
1716
|
+
/** Validator support: decoded bodies, streamed one at a time. */
|
|
1717
|
+
*documentBodies() {
|
|
1718
|
+
const rows = this.router.db.prepare("SELECT id, body_codec, body_blob FROM documents ORDER BY id").iterate();
|
|
1719
|
+
for (const row of rows) {
|
|
1720
|
+
const id = row.id;
|
|
1721
|
+
yield { id, markdown: this.decodeBody(id, row) };
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
decodeBody(id, row) {
|
|
1465
1725
|
const blob = row.body_blob;
|
|
1466
1726
|
if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
|
|
1467
1727
|
throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
|
|
@@ -1484,8 +1744,69 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1484
1744
|
} else {
|
|
1485
1745
|
throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
|
|
1486
1746
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1747
|
+
return body.toString("utf8");
|
|
1748
|
+
}
|
|
1749
|
+
// ── assets ────────────────────────────────────────────────────────────────
|
|
1750
|
+
/**
|
|
1751
|
+
* The manifest's `files` entries under `assets/`, read lazily: the
|
|
1752
|
+
* declaration in the manifest is what authorizes serving a file, since
|
|
1753
|
+
* extraction reconciled and hashed every declared entry.
|
|
1754
|
+
*/
|
|
1755
|
+
assetIndex() {
|
|
1756
|
+
if (this.assetsByPath) return this.assetsByPath;
|
|
1757
|
+
const index = /* @__PURE__ */ new Map();
|
|
1758
|
+
let raw;
|
|
1759
|
+
try {
|
|
1760
|
+
raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
1761
|
+
} catch (error) {
|
|
1762
|
+
if (error.code === "ENOENT") {
|
|
1763
|
+
this.assetsByPath = index;
|
|
1764
|
+
return index;
|
|
1765
|
+
}
|
|
1766
|
+
throw new CatalogOpenError(
|
|
1767
|
+
`cannot read ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1768
|
+
"corrupt"
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
const files = raw?.files;
|
|
1772
|
+
if (Array.isArray(files)) {
|
|
1773
|
+
for (const file of files) {
|
|
1774
|
+
const path = file?.path;
|
|
1775
|
+
if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
|
|
1776
|
+
const contentType = assetContentType(path);
|
|
1777
|
+
if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
|
|
1778
|
+
continue;
|
|
1779
|
+
}
|
|
1780
|
+
index.set(path, { path, contentType, sizeBytes: file.sizeBytes, sha256: file.sha256 });
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
this.assetsByPath = index;
|
|
1784
|
+
return index;
|
|
1785
|
+
}
|
|
1786
|
+
/** Declared assets, sorted by path. */
|
|
1787
|
+
assets() {
|
|
1788
|
+
return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
1789
|
+
}
|
|
1790
|
+
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
1791
|
+
readAsset(path) {
|
|
1792
|
+
const info = this.assetIndex().get(path);
|
|
1793
|
+
if (!info) return null;
|
|
1794
|
+
if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
|
|
1795
|
+
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
1796
|
+
}
|
|
1797
|
+
let bytes;
|
|
1798
|
+
try {
|
|
1799
|
+
bytes = readFileSync2(this.resolveCatalogPath(path));
|
|
1800
|
+
} catch (error) {
|
|
1801
|
+
throw new CatalogOpenError(
|
|
1802
|
+
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1803
|
+
"corrupt"
|
|
1804
|
+
);
|
|
1805
|
+
}
|
|
1806
|
+
if (bytes.byteLength !== info.sizeBytes) {
|
|
1807
|
+
throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
|
|
1808
|
+
}
|
|
1809
|
+
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1489
1810
|
}
|
|
1490
1811
|
// ── search ────────────────────────────────────────────────────────────────
|
|
1491
1812
|
/** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
|
|
@@ -1648,15 +1969,17 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1648
1969
|
}
|
|
1649
1970
|
}
|
|
1650
1971
|
};
|
|
1651
|
-
function
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
}
|
|
1972
|
+
function parseJsonObject(raw) {
|
|
1973
|
+
if (typeof raw !== "string") return null;
|
|
1974
|
+
try {
|
|
1975
|
+
const parsed = JSON.parse(raw);
|
|
1976
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
1977
|
+
} catch {
|
|
1978
|
+
return null;
|
|
1659
1979
|
}
|
|
1980
|
+
}
|
|
1981
|
+
function rowToDocumentMeta(row) {
|
|
1982
|
+
const ordinal = row.ordinal;
|
|
1660
1983
|
return {
|
|
1661
1984
|
id: row.id,
|
|
1662
1985
|
title: row.title,
|
|
@@ -1664,10 +1987,12 @@ function rowToDocumentMeta(row) {
|
|
|
1664
1987
|
summary: row.summary ?? null,
|
|
1665
1988
|
language: row.language,
|
|
1666
1989
|
topicId: row.topic_id,
|
|
1990
|
+
ordinal: typeof ordinal === "number" || typeof ordinal === "bigint" ? Number(ordinal) : null,
|
|
1667
1991
|
sourceUrl: row.source_url ?? null,
|
|
1668
1992
|
sourceRevision: row.source_revision ?? null,
|
|
1669
1993
|
sourceUpdatedAt: row.source_updated_at ?? null,
|
|
1670
|
-
attribution
|
|
1994
|
+
attribution: parseJsonObject(row.attribution_json),
|
|
1995
|
+
meta: parseJsonObject(row.meta_json)
|
|
1671
1996
|
};
|
|
1672
1997
|
}
|
|
1673
1998
|
function parseHeadingPath(raw) {
|
|
@@ -1683,7 +2008,17 @@ var ftsQuery = sanitizeFtsQuery;
|
|
|
1683
2008
|
// src/reader/validate.ts
|
|
1684
2009
|
import { readFile } from "fs/promises";
|
|
1685
2010
|
import { join as join3 } from "path";
|
|
1686
|
-
import {
|
|
2011
|
+
import {
|
|
2012
|
+
KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
|
|
2013
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2014
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
|
|
2015
|
+
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2016
|
+
assetExtension as assetExtension2,
|
|
2017
|
+
assetKindForExtension as assetKindForExtension2,
|
|
2018
|
+
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
2019
|
+
sniffAssetType as sniffAssetType2,
|
|
2020
|
+
svgInertnessProblem as svgInertnessProblem2
|
|
2021
|
+
} from "@bendyline/gezk";
|
|
1687
2022
|
async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
1688
2023
|
const checks = [];
|
|
1689
2024
|
const check = (name, ok, detail) => {
|
|
@@ -1750,8 +2085,43 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1750
2085
|
handle.meta.embedding_profile_id === manifest.embedding.id,
|
|
1751
2086
|
`router meta profile ${handle.meta.embedding_profile_id}`
|
|
1752
2087
|
);
|
|
2088
|
+
check(
|
|
2089
|
+
"meta-format",
|
|
2090
|
+
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2091
|
+
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2092
|
+
);
|
|
1753
2093
|
const topics = handle.topics();
|
|
1754
2094
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2095
|
+
check("topics-tree", ...topicTreeProblem(topics));
|
|
2096
|
+
const undeclared = handle.documentsWithUndeclaredTopic();
|
|
2097
|
+
check(
|
|
2098
|
+
"documents-topic-declared",
|
|
2099
|
+
undeclared === 0,
|
|
2100
|
+
`${undeclared} documents are filed under a topic the router does not declare`
|
|
2101
|
+
);
|
|
2102
|
+
const assetFiles = manifest.files.filter((f) => f.path.startsWith("assets/"));
|
|
2103
|
+
if (manifest.formatVersion === "0.5") {
|
|
2104
|
+
check(
|
|
2105
|
+
"assets-not-in-0.5",
|
|
2106
|
+
assetFiles.length === 0,
|
|
2107
|
+
`${assetFiles.length} assets/ entries in a 0.5 catalog`
|
|
2108
|
+
);
|
|
2109
|
+
} else {
|
|
2110
|
+
const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
|
|
2111
|
+
check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
|
|
2112
|
+
const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
|
|
2113
|
+
const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
|
|
2114
|
+
check(
|
|
2115
|
+
"assets-limits",
|
|
2116
|
+
assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2117
|
+
`${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
|
|
2118
|
+
);
|
|
2119
|
+
check(
|
|
2120
|
+
"counts-assets",
|
|
2121
|
+
(manifest.counts.assets ?? 0) === assetFiles.length,
|
|
2122
|
+
`manifest counts ${manifest.counts.assets ?? 0} assets, files declare ${assetFiles.length}`
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
1755
2125
|
check(
|
|
1756
2126
|
"license-notice",
|
|
1757
2127
|
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
@@ -1840,6 +2210,45 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1840
2210
|
}
|
|
1841
2211
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
1842
2212
|
}
|
|
2213
|
+
if (handle.schemaVersion >= 3) {
|
|
2214
|
+
const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
|
|
2215
|
+
check(
|
|
2216
|
+
"document-meta-json",
|
|
2217
|
+
meta.invalid.length === 0 && meta.oversize.length === 0,
|
|
2218
|
+
`${meta.invalid.length > 0 ? `not a JSON object: ${meta.invalid.join(", ")}` : ""}${meta.oversize.length > 0 ? ` over ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES} bytes: ${meta.oversize.join(", ")}` : ""}`.trim()
|
|
2219
|
+
);
|
|
2220
|
+
const declared = new Set(assetFiles.map((f) => f.path));
|
|
2221
|
+
for (const file of assetFiles) {
|
|
2222
|
+
const ext = assetExtension2(file.path);
|
|
2223
|
+
if (!ext) continue;
|
|
2224
|
+
const bytes = await readFile(join3(rootDir, file.path));
|
|
2225
|
+
const kind = sniffAssetType2(bytes);
|
|
2226
|
+
check(
|
|
2227
|
+
`asset-type:${file.path}`,
|
|
2228
|
+
kind === assetKindForExtension2(ext),
|
|
2229
|
+
`leading bytes say ${kind ?? "unknown"}, the extension says ${assetKindForExtension2(ext)}`
|
|
2230
|
+
);
|
|
2231
|
+
if (ext === "svg") {
|
|
2232
|
+
const problem = svgInertnessProblem2(bytes);
|
|
2233
|
+
check(`asset-svg-inert:${file.path}`, problem === null, problem ?? void 0);
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
if (declared.size > 0) {
|
|
2237
|
+
const missing = [];
|
|
2238
|
+
for (const doc of handle.documentBodies()) {
|
|
2239
|
+
for (const target of assetReferences(doc.markdown)) {
|
|
2240
|
+
if (!declared.has(target)) missing.push(`${doc.id} \u2192 ${target}`);
|
|
2241
|
+
if (missing.length >= 5) break;
|
|
2242
|
+
}
|
|
2243
|
+
if (missing.length >= 5) break;
|
|
2244
|
+
}
|
|
2245
|
+
check(
|
|
2246
|
+
"document-asset-refs",
|
|
2247
|
+
missing.length === 0,
|
|
2248
|
+
`undeclared asset references: ${missing.join("; ")}`
|
|
2249
|
+
);
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
1843
2252
|
for (const smoke of manifest.smokeQueries ?? []) {
|
|
1844
2253
|
const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
|
|
1845
2254
|
const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
|
|
@@ -1855,6 +2264,39 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1855
2264
|
}
|
|
1856
2265
|
return report(manifest);
|
|
1857
2266
|
}
|
|
2267
|
+
function assetReferences(markdown) {
|
|
2268
|
+
const found = /* @__PURE__ */ new Set();
|
|
2269
|
+
for (const match of markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
2270
|
+
const target = match[1];
|
|
2271
|
+
if (target) found.add(target);
|
|
2272
|
+
}
|
|
2273
|
+
return [...found];
|
|
2274
|
+
}
|
|
2275
|
+
function topicTreeProblem(topics) {
|
|
2276
|
+
const byId = new Map(topics.map((t) => [t.id, t]));
|
|
2277
|
+
for (const topic of topics) {
|
|
2278
|
+
if (topic.parentId !== null && !byId.has(topic.parentId)) {
|
|
2279
|
+
return [false, `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`];
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
const depthOf = /* @__PURE__ */ new Map();
|
|
2283
|
+
for (const topic of topics) {
|
|
2284
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2285
|
+
let depth = 0;
|
|
2286
|
+
let current = topic;
|
|
2287
|
+
while (current) {
|
|
2288
|
+
if (seen.has(current.id)) return [false, `topic '${topic.id}' sits in a parent cycle`];
|
|
2289
|
+
seen.add(current.id);
|
|
2290
|
+
depth += 1;
|
|
2291
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
2292
|
+
return [false, `topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`];
|
|
2293
|
+
}
|
|
2294
|
+
current = current.parentId === null ? void 0 : byId.get(current.parentId);
|
|
2295
|
+
}
|
|
2296
|
+
depthOf.set(topic.id, depth);
|
|
2297
|
+
}
|
|
2298
|
+
return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
|
|
2299
|
+
}
|
|
1858
2300
|
|
|
1859
2301
|
// src/export/duckdb.ts
|
|
1860
2302
|
import { spawn } from "child_process";
|
|
@@ -1892,7 +2334,7 @@ async function runDuckdbScript(cli, sql) {
|
|
|
1892
2334
|
return stdout;
|
|
1893
2335
|
}
|
|
1894
2336
|
function runDuckdbProcess(binaryPath, args, stdin) {
|
|
1895
|
-
return new Promise((
|
|
2337
|
+
return new Promise((resolve5, reject) => {
|
|
1896
2338
|
const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
1897
2339
|
let stdout = "";
|
|
1898
2340
|
let stderr = "";
|
|
@@ -1909,7 +2351,7 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
1909
2351
|
(err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
|
|
1910
2352
|
);
|
|
1911
2353
|
child.on("close", (code) => {
|
|
1912
|
-
if (code === 0)
|
|
2354
|
+
if (code === 0) resolve5({ stdout });
|
|
1913
2355
|
else
|
|
1914
2356
|
reject(
|
|
1915
2357
|
new DuckdbError(
|
|
@@ -1931,7 +2373,7 @@ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/pr
|
|
|
1931
2373
|
import { join as join4, resolve as resolve3 } from "path";
|
|
1932
2374
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
1933
2375
|
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
1934
|
-
var PARQUET_EXPORT_VERSION =
|
|
2376
|
+
var PARQUET_EXPORT_VERSION = 2;
|
|
1935
2377
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
1936
2378
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
1937
2379
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2126,9 +2568,10 @@ async function stageDocuments(router, shardId, topicPaths, path) {
|
|
|
2126
2568
|
const aliasesOf = router.db.prepare(
|
|
2127
2569
|
"SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
|
|
2128
2570
|
);
|
|
2571
|
+
const extra = router.schemaVersion >= 3 ? ", ordinal, meta_json" : ", NULL AS ordinal, NULL AS meta_json";
|
|
2129
2572
|
const statement = router.db.prepare(
|
|
2130
2573
|
`SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
|
|
2131
|
-
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
|
|
2574
|
+
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob${extra}
|
|
2132
2575
|
FROM documents WHERE shard_id = ? ORDER BY id`
|
|
2133
2576
|
);
|
|
2134
2577
|
for (const row of statement.iterate(shardId)) {
|
|
@@ -2150,6 +2593,8 @@ async function stageDocuments(router, shardId, topicPaths, path) {
|
|
|
2150
2593
|
source_revision: row.source_revision ?? null,
|
|
2151
2594
|
source_updated_at: row.source_updated_at ?? null,
|
|
2152
2595
|
attribution: row.attribution_json ?? null,
|
|
2596
|
+
ordinal: row.ordinal == null ? null : Number(row.ordinal),
|
|
2597
|
+
meta: row.meta_json ?? null,
|
|
2153
2598
|
aliases
|
|
2154
2599
|
});
|
|
2155
2600
|
rows++;
|
|
@@ -2273,9 +2718,11 @@ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
|
2273
2718
|
source_revision: "VARCHAR",
|
|
2274
2719
|
source_updated_at: "VARCHAR",
|
|
2275
2720
|
attribution: "VARCHAR",
|
|
2721
|
+
ordinal: "INTEGER",
|
|
2722
|
+
meta: "VARCHAR",
|
|
2276
2723
|
aliases: "VARCHAR[]"
|
|
2277
2724
|
});
|
|
2278
|
-
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2725
|
+
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, ordinal, meta, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2279
2726
|
`;
|
|
2280
2727
|
}
|
|
2281
2728
|
function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
|
|
@@ -2453,9 +2900,9 @@ async function fileDigest(path) {
|
|
|
2453
2900
|
const key = `${path} ${size} ${mtimeMs}`;
|
|
2454
2901
|
const memo = digestMemo.get(key);
|
|
2455
2902
|
if (memo) return memo;
|
|
2456
|
-
const digest = await new Promise((
|
|
2903
|
+
const digest = await new Promise((resolve5, reject) => {
|
|
2457
2904
|
const hash = createHash4("sha256");
|
|
2458
|
-
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () =>
|
|
2905
|
+
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
2459
2906
|
});
|
|
2460
2907
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
2461
2908
|
const oldest = digestMemo.keys().next().value;
|
|
@@ -2613,29 +3060,427 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
2613
3060
|
}
|
|
2614
3061
|
|
|
2615
3062
|
// src/markdown-adapter/load.ts
|
|
3063
|
+
import { existsSync } from "fs";
|
|
2616
3064
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2617
|
-
import { join as join6, relative as relative3, sep } from "path";
|
|
2618
|
-
import {
|
|
3065
|
+
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3066
|
+
import {
|
|
3067
|
+
CatalogDocumentSchema,
|
|
3068
|
+
KnowledgeDocumentIdSchema,
|
|
3069
|
+
KnowledgeOrdinalSchema,
|
|
3070
|
+
assetExtension as assetExtension3,
|
|
3071
|
+
formatKnowledgeUri,
|
|
3072
|
+
topicSortKeyForOrder
|
|
3073
|
+
} from "@bendyline/gezk";
|
|
3074
|
+
|
|
3075
|
+
// src/markdown-adapter/frontmatter.ts
|
|
3076
|
+
import { parseDocument } from "yaml";
|
|
3077
|
+
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
3078
|
+
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
3079
|
+
function parseYaml(text, opts) {
|
|
3080
|
+
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
3081
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
3082
|
+
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
3083
|
+
}
|
|
3084
|
+
const doc = parseDocument(text, {
|
|
3085
|
+
version: "1.2",
|
|
3086
|
+
schema: "core",
|
|
3087
|
+
uniqueKeys: true,
|
|
3088
|
+
logLevel: "silent"
|
|
3089
|
+
});
|
|
3090
|
+
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
3091
|
+
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
3092
|
+
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
3093
|
+
}
|
|
3094
|
+
var OPEN = /^---[ \t]*\r?\n/;
|
|
3095
|
+
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
3096
|
+
function parseMarkdownFrontMatter(source) {
|
|
3097
|
+
const open = OPEN.exec(source);
|
|
3098
|
+
if (!open) return { data: {}, body: source };
|
|
3099
|
+
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
3100
|
+
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
3101
|
+
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
3102
|
+
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
3103
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
3104
|
+
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
3105
|
+
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
3106
|
+
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3107
|
+
throw new Error("front matter must be a mapping");
|
|
3108
|
+
}
|
|
3109
|
+
return { data: parsed, body };
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
// src/markdown-adapter/outline.ts
|
|
3113
|
+
import { posix } from "path";
|
|
3114
|
+
var HAS_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
|
|
3115
|
+
var MARKDOWN_EXT = /\.(md|markdown)$/i;
|
|
3116
|
+
function newOutlineTopic(name) {
|
|
3117
|
+
return { name, children: [], entries: [] };
|
|
3118
|
+
}
|
|
3119
|
+
function isRecord(value) {
|
|
3120
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3121
|
+
}
|
|
3122
|
+
function isExternal(target) {
|
|
3123
|
+
return HAS_SCHEME.test(target) || target.startsWith("//");
|
|
3124
|
+
}
|
|
3125
|
+
function safeDecode(value) {
|
|
3126
|
+
try {
|
|
3127
|
+
return decodeURIComponent(value);
|
|
3128
|
+
} catch {
|
|
3129
|
+
return value;
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
function stem(file) {
|
|
3133
|
+
return posix.basename(file).replace(MARKDOWN_EXT, "");
|
|
3134
|
+
}
|
|
3135
|
+
function resolveOutlinePath(baseDir, target) {
|
|
3136
|
+
const cleaned = safeDecode(
|
|
3137
|
+
target.replace(/\\/g, "/").replace(/[#?].*$/, "").trim()
|
|
3138
|
+
);
|
|
3139
|
+
if (cleaned === "" || posix.isAbsolute(cleaned)) return null;
|
|
3140
|
+
const joined = posix.normalize(posix.join(baseDir, cleaned));
|
|
3141
|
+
if (joined === ".." || joined.startsWith("../")) return null;
|
|
3142
|
+
return joined === "." ? "" : joined;
|
|
3143
|
+
}
|
|
3144
|
+
function assignOutlineOrders(topic) {
|
|
3145
|
+
topic.entries.forEach((entry, i) => {
|
|
3146
|
+
if (entry.order === void 0) entry.order = i + 1;
|
|
3147
|
+
});
|
|
3148
|
+
topic.children.forEach((child, i) => {
|
|
3149
|
+
if (child.order === void 0) child.order = i + 1;
|
|
3150
|
+
assignOutlineOrders(child);
|
|
3151
|
+
});
|
|
3152
|
+
}
|
|
3153
|
+
function titleFromFolderName(name) {
|
|
3154
|
+
const words = name.replace(/[_-]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
3155
|
+
if (words.length === 0) return name;
|
|
3156
|
+
return words.map((word) => /[A-Z]/.test(word) ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
3157
|
+
}
|
|
3158
|
+
var HEADING2 = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
|
|
3159
|
+
var LIST_ITEM = /^([ \t]*)(?:[-*+]|\d+[.)])\s+(.+?)\s*$/;
|
|
3160
|
+
var ITEM_LINK = /^\[([^\]]*)\]\(\s*<?([^)>\s]*)>?(?:\s+"[^"]*")?\s*\)$/;
|
|
3161
|
+
function materializeSummaryTopic(slot) {
|
|
3162
|
+
if (slot.topic) return slot.topic;
|
|
3163
|
+
const topic = newOutlineTopic(slot.title || "Untitled");
|
|
3164
|
+
if (slot.entry) {
|
|
3165
|
+
const at = slot.container.entries.indexOf(slot.entry);
|
|
3166
|
+
if (at >= 0) slot.container.entries.splice(at, 1);
|
|
3167
|
+
topic.entries.push(slot.entry);
|
|
3168
|
+
}
|
|
3169
|
+
slot.container.children.push(topic);
|
|
3170
|
+
slot.topic = topic;
|
|
3171
|
+
return topic;
|
|
3172
|
+
}
|
|
3173
|
+
function parseGitbookSummary(text, summaryFile) {
|
|
3174
|
+
const dir = posix.dirname(summaryFile);
|
|
3175
|
+
const baseDir = dir === "." ? "" : dir;
|
|
3176
|
+
const outline = {
|
|
3177
|
+
format: "gitbook",
|
|
3178
|
+
root: newOutlineTopic(""),
|
|
3179
|
+
consumed: [summaryFile],
|
|
3180
|
+
warnings: []
|
|
3181
|
+
};
|
|
3182
|
+
let container = outline.root;
|
|
3183
|
+
const stack = [];
|
|
3184
|
+
let fence = null;
|
|
3185
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
3186
|
+
const line = rawLine.replace(/\t/g, " ");
|
|
3187
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
3188
|
+
if (fenceMatch) {
|
|
3189
|
+
const marker = fenceMatch[1][0];
|
|
3190
|
+
if (fence === null) fence = marker;
|
|
3191
|
+
else if (marker === fence) fence = null;
|
|
3192
|
+
continue;
|
|
3193
|
+
}
|
|
3194
|
+
if (fence !== null) continue;
|
|
3195
|
+
const heading = HEADING2.exec(line);
|
|
3196
|
+
if (heading) {
|
|
3197
|
+
if (heading[1].length === 1) continue;
|
|
3198
|
+
const part = newOutlineTopic(heading[2].replace(/[*_`]/g, "").trim() || "Part");
|
|
3199
|
+
outline.root.children.push(part);
|
|
3200
|
+
container = part;
|
|
3201
|
+
stack.length = 0;
|
|
3202
|
+
continue;
|
|
3203
|
+
}
|
|
3204
|
+
const item = LIST_ITEM.exec(line);
|
|
3205
|
+
if (!item) continue;
|
|
3206
|
+
const indent = item[1].length;
|
|
3207
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
|
|
3208
|
+
stack.pop();
|
|
3209
|
+
}
|
|
3210
|
+
const parent = stack[stack.length - 1];
|
|
3211
|
+
const parentTopic = parent ? materializeSummaryTopic(parent) : container;
|
|
3212
|
+
const content = item[2].trim();
|
|
3213
|
+
const link = ITEM_LINK.exec(content);
|
|
3214
|
+
const title = (link ? link[1] : content).replace(/[*_`]/g, "").trim();
|
|
3215
|
+
const slot = { indent, title, container: parentTopic };
|
|
3216
|
+
const target = link?.[2] ?? "";
|
|
3217
|
+
if (link && target !== "" && !isExternal(target)) {
|
|
3218
|
+
const file = resolveOutlinePath(baseDir, target);
|
|
3219
|
+
if (file === null) {
|
|
3220
|
+
outline.warnings.push(`${summaryFile}: '${target}' leaves the catalog tree; skipped`);
|
|
3221
|
+
} else if (!MARKDOWN_EXT.test(file)) {
|
|
3222
|
+
outline.warnings.push(`${summaryFile}: '${target}' is not a Markdown page; skipped`);
|
|
3223
|
+
} else {
|
|
3224
|
+
slot.entry = { file, ...title ? { title } : {} };
|
|
3225
|
+
parentTopic.entries.push(slot.entry);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
stack.push(slot);
|
|
3229
|
+
}
|
|
3230
|
+
assignOutlineOrders(outline.root);
|
|
3231
|
+
return outline;
|
|
3232
|
+
}
|
|
3233
|
+
function parseMkdocsNav(config, docsDir, configFile = "mkdocs.yml") {
|
|
3234
|
+
if (!isRecord(config)) throw new Error(`${configFile}: not a mapping`);
|
|
3235
|
+
const nav = config.nav;
|
|
3236
|
+
if (nav === void 0 || nav === null) return null;
|
|
3237
|
+
if (!Array.isArray(nav)) throw new Error(`${configFile}: 'nav' must be a list`);
|
|
3238
|
+
const outline = {
|
|
3239
|
+
format: "mkdocs",
|
|
3240
|
+
root: newOutlineTopic(""),
|
|
3241
|
+
consumed: [],
|
|
3242
|
+
warnings: []
|
|
3243
|
+
};
|
|
3244
|
+
const add = (container, target, title) => {
|
|
3245
|
+
if (isExternal(target)) return;
|
|
3246
|
+
const file = resolveOutlinePath(docsDir, target);
|
|
3247
|
+
if (file === null) {
|
|
3248
|
+
outline.warnings.push(
|
|
3249
|
+
`${configFile}: nav entry '${target}' leaves the docs directory; skipped`
|
|
3250
|
+
);
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
3253
|
+
if (!MARKDOWN_EXT.test(file)) {
|
|
3254
|
+
outline.warnings.push(`${configFile}: nav entry '${target}' is not a Markdown page; skipped`);
|
|
3255
|
+
return;
|
|
3256
|
+
}
|
|
3257
|
+
container.entries.push({ file, ...title ? { title } : {} });
|
|
3258
|
+
};
|
|
3259
|
+
const walk = (items, container) => {
|
|
3260
|
+
for (const item of items) {
|
|
3261
|
+
if (typeof item === "string") {
|
|
3262
|
+
add(container, item);
|
|
3263
|
+
continue;
|
|
3264
|
+
}
|
|
3265
|
+
if (!isRecord(item)) {
|
|
3266
|
+
outline.warnings.push(`${configFile}: unrecognized nav entry ${JSON.stringify(item)}`);
|
|
3267
|
+
continue;
|
|
3268
|
+
}
|
|
3269
|
+
for (const [title, value] of Object.entries(item)) {
|
|
3270
|
+
if (typeof value === "string") {
|
|
3271
|
+
add(container, value, title.trim());
|
|
3272
|
+
} else if (Array.isArray(value)) {
|
|
3273
|
+
const section = newOutlineTopic(title.trim() || "Section");
|
|
3274
|
+
walk(value, section);
|
|
3275
|
+
container.children.push(section);
|
|
3276
|
+
} else {
|
|
3277
|
+
outline.warnings.push(
|
|
3278
|
+
`${configFile}: nav entry '${title}' is neither a page nor a section`
|
|
3279
|
+
);
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
};
|
|
3284
|
+
walk(nav, outline.root);
|
|
3285
|
+
assignOutlineOrders(outline.root);
|
|
3286
|
+
return outline;
|
|
3287
|
+
}
|
|
3288
|
+
function globToRegExp(pattern) {
|
|
3289
|
+
let source = "";
|
|
3290
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
3291
|
+
const ch = pattern[i];
|
|
3292
|
+
if (ch === "*") {
|
|
3293
|
+
if (pattern[i + 1] === "*") {
|
|
3294
|
+
source += ".*";
|
|
3295
|
+
i++;
|
|
3296
|
+
} else source += "[^/]*";
|
|
3297
|
+
} else if (ch === "?") source += "[^/]";
|
|
3298
|
+
else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
3299
|
+
}
|
|
3300
|
+
return new RegExp(`^${source}$`);
|
|
3301
|
+
}
|
|
3302
|
+
function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
|
|
3303
|
+
const outline = {
|
|
3304
|
+
format: "jupyter-book",
|
|
3305
|
+
root: newOutlineTopic(""),
|
|
3306
|
+
consumed: [],
|
|
3307
|
+
warnings: []
|
|
3308
|
+
};
|
|
3309
|
+
const known = new Set(files.map((f) => f.normalize("NFC")));
|
|
3310
|
+
const resolveFile = (spec) => {
|
|
3311
|
+
const candidates = MARKDOWN_EXT.test(spec) ? [spec] : [`${spec}.md`, `${spec}.markdown`];
|
|
3312
|
+
for (const candidate of candidates) {
|
|
3313
|
+
const path = resolveOutlinePath(tocDir, candidate);
|
|
3314
|
+
if (path !== null && known.has(path.normalize("NFC"))) return path;
|
|
3315
|
+
}
|
|
3316
|
+
return null;
|
|
3317
|
+
};
|
|
3318
|
+
const expandGlob = (pattern) => {
|
|
3319
|
+
const regex = globToRegExp(pattern.replace(/\\/g, "/"));
|
|
3320
|
+
const prefix = tocDir ? `${tocDir}/` : "";
|
|
3321
|
+
return files.filter((file) => {
|
|
3322
|
+
if (!file.startsWith(prefix)) return false;
|
|
3323
|
+
const rel = file.slice(prefix.length);
|
|
3324
|
+
return regex.test(rel) || regex.test(rel.replace(MARKDOWN_EXT, ""));
|
|
3325
|
+
}).sort();
|
|
3326
|
+
};
|
|
3327
|
+
const walkItems = (items, container, where) => {
|
|
3328
|
+
if (items === void 0 || items === null) return;
|
|
3329
|
+
if (!Array.isArray(items)) {
|
|
3330
|
+
outline.warnings.push(`${tocFile}: ${where} must be a list`);
|
|
3331
|
+
return;
|
|
3332
|
+
}
|
|
3333
|
+
for (const item of items) {
|
|
3334
|
+
if (!isRecord(item)) {
|
|
3335
|
+
outline.warnings.push(`${tocFile}: ${where} holds an unrecognized entry`);
|
|
3336
|
+
continue;
|
|
3337
|
+
}
|
|
3338
|
+
const title = typeof item.title === "string" ? item.title.trim() : void 0;
|
|
3339
|
+
if (typeof item.url === "string") continue;
|
|
3340
|
+
if (typeof item.glob === "string") {
|
|
3341
|
+
for (const file2 of expandGlob(item.glob)) container.entries.push({ file: file2 });
|
|
3342
|
+
continue;
|
|
3343
|
+
}
|
|
3344
|
+
if (typeof item.file !== "string") {
|
|
3345
|
+
outline.warnings.push(`${tocFile}: ${where} holds an entry without 'file'`);
|
|
3346
|
+
continue;
|
|
3347
|
+
}
|
|
3348
|
+
const file = resolveFile(item.file);
|
|
3349
|
+
if (file === null) {
|
|
3350
|
+
outline.warnings.push(
|
|
3351
|
+
`${tocFile}: '${item.file}' is not a Markdown file in the tree (notebooks are not supported); skipped`
|
|
3352
|
+
);
|
|
3353
|
+
continue;
|
|
3354
|
+
}
|
|
3355
|
+
const entry = { file, ...title ? { title } : {} };
|
|
3356
|
+
const children = item.sections ?? item.chapters ?? item.entries;
|
|
3357
|
+
if (Array.isArray(children) && children.length > 0) {
|
|
3358
|
+
const topic = newOutlineTopic(title ?? titleFromFolderName(stem(file)));
|
|
3359
|
+
if (!title) topic.titleFromFile = file;
|
|
3360
|
+
topic.entries.push(entry);
|
|
3361
|
+
walkItems(children, topic, `'${item.file}'`);
|
|
3362
|
+
container.children.push(topic);
|
|
3363
|
+
} else {
|
|
3364
|
+
container.entries.push(entry);
|
|
3365
|
+
}
|
|
3366
|
+
}
|
|
3367
|
+
};
|
|
3368
|
+
if (Array.isArray(doc)) {
|
|
3369
|
+
for (const item of doc) {
|
|
3370
|
+
if (isRecord(item) && typeof item.part === "string") {
|
|
3371
|
+
const part = newOutlineTopic(item.part.trim() || "Part");
|
|
3372
|
+
walkItems(item.chapters ?? item.sections, part, `part '${item.part}'`);
|
|
3373
|
+
outline.root.children.push(part);
|
|
3374
|
+
} else {
|
|
3375
|
+
walkItems([item], outline.root, "the list");
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
} else if (isRecord(doc)) {
|
|
3379
|
+
if (typeof doc.root === "string") {
|
|
3380
|
+
const file = resolveFile(doc.root);
|
|
3381
|
+
if (file === null) {
|
|
3382
|
+
outline.warnings.push(
|
|
3383
|
+
`${tocFile}: root '${doc.root}' is not a Markdown file in the tree; skipped`
|
|
3384
|
+
);
|
|
3385
|
+
} else {
|
|
3386
|
+
outline.root.entries.push({ file });
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3389
|
+
if (Array.isArray(doc.parts)) {
|
|
3390
|
+
for (const part of doc.parts) {
|
|
3391
|
+
if (!isRecord(part)) continue;
|
|
3392
|
+
const caption = typeof part.caption === "string" ? part.caption.trim() : "";
|
|
3393
|
+
const topic = newOutlineTopic(caption || "Part");
|
|
3394
|
+
walkItems(part.chapters ?? part.sections, topic, `part '${topic.name}'`);
|
|
3395
|
+
outline.root.children.push(topic);
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
walkItems(doc.chapters, outline.root, "chapters");
|
|
3399
|
+
walkItems(doc.sections, outline.root, "sections");
|
|
3400
|
+
} else {
|
|
3401
|
+
throw new Error(`${tocFile}: not a mapping or a list`);
|
|
3402
|
+
}
|
|
3403
|
+
assignOutlineOrders(outline.root);
|
|
3404
|
+
return outline;
|
|
3405
|
+
}
|
|
3406
|
+
|
|
3407
|
+
// src/markdown-adapter/load.ts
|
|
2619
3408
|
var ROOT_TOPIC_ID = "general";
|
|
2620
3409
|
var SUMMARY_MAX_CHARS = 280;
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
3410
|
+
var TOPIC_SIDECAR = "_topic.yaml";
|
|
3411
|
+
var HUGO_SECTION_PAGE = "_index";
|
|
3412
|
+
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
3413
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
|
|
3414
|
+
function readFrontMatter(raw, file, hugo) {
|
|
3415
|
+
let data;
|
|
3416
|
+
let body;
|
|
3417
|
+
try {
|
|
3418
|
+
({ data, body } = parseMarkdownFrontMatter(raw));
|
|
3419
|
+
} catch (error) {
|
|
3420
|
+
throw new Error(`${file}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3421
|
+
}
|
|
3422
|
+
const out = { meta: {}, body };
|
|
3423
|
+
const str = (key) => {
|
|
3424
|
+
const value = data[key];
|
|
3425
|
+
if (value === void 0 || value === null) return void 0;
|
|
3426
|
+
if (typeof value === "number") return String(value);
|
|
3427
|
+
if (typeof value !== "string")
|
|
3428
|
+
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
3429
|
+
return value.trim() || void 0;
|
|
3430
|
+
};
|
|
3431
|
+
out.title = str("title");
|
|
3432
|
+
out.summary = str("summary");
|
|
3433
|
+
out.id = str("id");
|
|
3434
|
+
const aliases = data.aliases;
|
|
3435
|
+
if (typeof aliases === "string") {
|
|
3436
|
+
out.aliases = aliases.split(",").map((a) => a.trim()).filter(Boolean);
|
|
3437
|
+
} else if (Array.isArray(aliases)) {
|
|
3438
|
+
out.aliases = aliases.map((a) => String(a).trim()).filter(Boolean);
|
|
3439
|
+
} else if (aliases !== void 0 && aliases !== null) {
|
|
3440
|
+
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
3441
|
+
}
|
|
3442
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
3443
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
3444
|
+
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
3445
|
+
out.order = order.data;
|
|
3446
|
+
}
|
|
3447
|
+
const sub = data.subcategory;
|
|
3448
|
+
if (sub !== void 0 && sub !== null) {
|
|
3449
|
+
if (typeof sub !== "object" || Array.isArray(sub)) {
|
|
3450
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
3451
|
+
}
|
|
3452
|
+
const record = sub;
|
|
3453
|
+
if (typeof record.id !== "string" || !record.id.trim()) {
|
|
3454
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
3455
|
+
}
|
|
3456
|
+
const subcategory = { id: record.id.trim() };
|
|
3457
|
+
if (record.title !== void 0 && record.title !== null) {
|
|
3458
|
+
if (typeof record.title !== "string") {
|
|
3459
|
+
throw new Error(`${file}: front matter 'subcategory.title' must be a string`);
|
|
3460
|
+
}
|
|
3461
|
+
subcategory.title = record.title.trim();
|
|
2638
3462
|
}
|
|
3463
|
+
if (record.order !== void 0 && record.order !== null) {
|
|
3464
|
+
const order = KnowledgeOrdinalSchema.safeParse(record.order);
|
|
3465
|
+
if (!order.success)
|
|
3466
|
+
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
3467
|
+
subcategory.order = order.data;
|
|
3468
|
+
}
|
|
3469
|
+
out.subcategory = subcategory;
|
|
3470
|
+
}
|
|
3471
|
+
if (hugo) {
|
|
3472
|
+
if (data.draft === true) out.unpublished = "draft";
|
|
3473
|
+
else if (data.headless === true) out.unpublished = "headless";
|
|
3474
|
+
if (data.weight !== void 0 && data.weight !== null) {
|
|
3475
|
+
const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
|
|
3476
|
+
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
3477
|
+
if (weight.data > 0) out.weight = weight.data;
|
|
3478
|
+
}
|
|
3479
|
+
out.description = str("description");
|
|
3480
|
+
}
|
|
3481
|
+
for (const [key, value] of Object.entries(data)) {
|
|
3482
|
+
if (RESERVED_KEYS.has(key) || value === void 0) continue;
|
|
3483
|
+
out.meta[key] = value;
|
|
2639
3484
|
}
|
|
2640
3485
|
return out;
|
|
2641
3486
|
}
|
|
@@ -2677,17 +3522,233 @@ async function walkMarkdownFiles(root) {
|
|
|
2677
3522
|
await visit(root);
|
|
2678
3523
|
return out;
|
|
2679
3524
|
}
|
|
2680
|
-
|
|
3525
|
+
function isRecord2(value) {
|
|
3526
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3527
|
+
}
|
|
3528
|
+
async function readTopicSidecar(dir) {
|
|
3529
|
+
const path = join6(dir, TOPIC_SIDECAR);
|
|
3530
|
+
if (!existsSync(path)) return void 0;
|
|
3531
|
+
const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
3532
|
+
const data = parseYaml(raw, { what: path });
|
|
3533
|
+
if (data === null || data === void 0) return {};
|
|
3534
|
+
if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
|
|
3535
|
+
const out = {};
|
|
3536
|
+
if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
|
|
3537
|
+
if (typeof data.description === "string" && data.description.trim()) {
|
|
3538
|
+
out.description = data.description.trim();
|
|
3539
|
+
}
|
|
3540
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
3541
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
3542
|
+
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
3543
|
+
out.order = order.data;
|
|
3544
|
+
}
|
|
3545
|
+
return out;
|
|
3546
|
+
}
|
|
3547
|
+
function applyOverride(topic, override) {
|
|
3548
|
+
if (!topic || !override) return;
|
|
3549
|
+
if (override.name) topic.name = override.name;
|
|
3550
|
+
if (override.description) topic.description = override.description;
|
|
3551
|
+
if (override.order !== void 0) topic.sortKey = topicSortKeyForOrder(override.order);
|
|
3552
|
+
}
|
|
3553
|
+
function insideRoot(rootDir, abs) {
|
|
3554
|
+
const rel = relative3(rootDir, abs).split(sep).join("/");
|
|
3555
|
+
if (rel === ".." || rel.startsWith("../") || isAbsolute3(rel)) return null;
|
|
3556
|
+
return rel;
|
|
3557
|
+
}
|
|
3558
|
+
function firstExisting(candidates) {
|
|
3559
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
3560
|
+
}
|
|
3561
|
+
async function readMkdocsDocsDir(mkdocsPath) {
|
|
3562
|
+
const config = parseYaml((await readFile3(mkdocsPath, "utf8")).replace(/^\uFEFF/, ""), {
|
|
3563
|
+
what: basename(mkdocsPath),
|
|
3564
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
3565
|
+
tolerateUnknownTags: true,
|
|
3566
|
+
maxAliasCount: 100
|
|
3567
|
+
});
|
|
3568
|
+
const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
3569
|
+
return resolve4(dirname3(mkdocsPath), docsDir);
|
|
3570
|
+
}
|
|
3571
|
+
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
3572
|
+
const summary = join6(rootDir, "SUMMARY.md");
|
|
3573
|
+
if (existsSync(summary)) return { format: "gitbook", path: summary };
|
|
3574
|
+
const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
3575
|
+
if (jupyter) return { format: "jupyter-book", path: jupyter };
|
|
3576
|
+
const mkdocs = firstExisting(
|
|
3577
|
+
[rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
|
|
3578
|
+
join6(dir, "mkdocs.yml"),
|
|
3579
|
+
join6(dir, "mkdocs.yaml")
|
|
3580
|
+
])
|
|
3581
|
+
);
|
|
3582
|
+
if (mkdocs) return { format: "mkdocs", path: mkdocs };
|
|
2681
3583
|
const files = await walkMarkdownFiles(rootDir);
|
|
3584
|
+
if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
|
|
3585
|
+
return { format: "hugo" };
|
|
3586
|
+
}
|
|
3587
|
+
return { format: "folders" };
|
|
3588
|
+
}
|
|
3589
|
+
async function readOutlineText(path) {
|
|
3590
|
+
const text = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
3591
|
+
if (Buffer.byteLength(text, "utf8") > OUTLINE_MAX_BYTES) {
|
|
3592
|
+
throw new Error(`${basename(path)} exceeds ${OUTLINE_MAX_BYTES} bytes`);
|
|
3593
|
+
}
|
|
3594
|
+
return text;
|
|
3595
|
+
}
|
|
3596
|
+
async function loadOutline(rootDir, toc, files) {
|
|
3597
|
+
if (toc.format === "folders" || toc.format === "hugo") return { outline: null };
|
|
3598
|
+
const explicit = toc.path ? isAbsolute3(toc.path) ? toc.path : join6(rootDir, toc.path) : void 0;
|
|
3599
|
+
if (explicit && !existsSync(explicit)) {
|
|
3600
|
+
throw new Error(`${toc.format} table of contents not found: ${explicit}`);
|
|
3601
|
+
}
|
|
3602
|
+
if (toc.format === "gitbook") {
|
|
3603
|
+
const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
|
|
3604
|
+
const rel = insideRoot(rootDir, path2);
|
|
3605
|
+
if (!existsSync(path2)) throw new Error(`gitbook table of contents not found: ${path2}`);
|
|
3606
|
+
if (rel === null)
|
|
3607
|
+
throw new Error(`${basename(path2)} must live inside the content root ${rootDir}`);
|
|
3608
|
+
return { outline: parseGitbookSummary(await readOutlineText(path2), rel), path: path2 };
|
|
3609
|
+
}
|
|
3610
|
+
if (toc.format === "mkdocs") {
|
|
3611
|
+
const path2 = explicit ?? firstExisting(
|
|
3612
|
+
[rootDir, dirname3(rootDir)].flatMap((dir) => [
|
|
3613
|
+
join6(dir, "mkdocs.yml"),
|
|
3614
|
+
join6(dir, "mkdocs.yaml")
|
|
3615
|
+
])
|
|
3616
|
+
);
|
|
3617
|
+
if (!path2) throw new Error(`mkdocs.yml not found in ${rootDir} or its parent`);
|
|
3618
|
+
const config = parseYaml(await readOutlineText(path2), {
|
|
3619
|
+
what: basename(path2),
|
|
3620
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
3621
|
+
tolerateUnknownTags: true,
|
|
3622
|
+
maxAliasCount: 100
|
|
3623
|
+
});
|
|
3624
|
+
const docsDir = await readMkdocsDocsDir(path2);
|
|
3625
|
+
const docsRel = insideRoot(rootDir, docsDir);
|
|
3626
|
+
if (docsRel === null) {
|
|
3627
|
+
throw new Error(
|
|
3628
|
+
`${basename(path2)}: docs_dir ${docsDir} is outside the content root ${rootDir}`
|
|
3629
|
+
);
|
|
3630
|
+
}
|
|
3631
|
+
return { outline: parseMkdocsNav(config, docsRel, basename(path2)), path: path2 };
|
|
3632
|
+
}
|
|
3633
|
+
const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
3634
|
+
if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
|
|
3635
|
+
const tocDir = insideRoot(rootDir, dirname3(path));
|
|
3636
|
+
if (tocDir === null)
|
|
3637
|
+
throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
|
|
3638
|
+
const doc = parseYaml(await readOutlineText(path), {
|
|
3639
|
+
what: basename(path),
|
|
3640
|
+
maxBytes: OUTLINE_MAX_BYTES
|
|
3641
|
+
});
|
|
3642
|
+
return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
|
|
3643
|
+
}
|
|
3644
|
+
var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
|
|
3645
|
+
var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
|
|
3646
|
+
function resolveInTree(ctx, target) {
|
|
3647
|
+
const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
|
|
3648
|
+
if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
|
|
3649
|
+
return resolved === "." ? "" : resolved;
|
|
3650
|
+
}
|
|
3651
|
+
function rewriteTarget(ctx, isImage, rawTarget) {
|
|
3652
|
+
const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
|
|
3653
|
+
if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
|
|
3654
|
+
return null;
|
|
3655
|
+
}
|
|
3656
|
+
const hashAt = target.search(/[#?]/);
|
|
3657
|
+
const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
|
|
3658
|
+
const decoded = safeDecode2(pathPart);
|
|
3659
|
+
const resolved = resolveInTree(ctx, decoded);
|
|
3660
|
+
if (resolved === null) {
|
|
3661
|
+
ctx.warn(`${ctx.file}: link '${target}' leaves the catalog tree; left as is`);
|
|
3662
|
+
return null;
|
|
3663
|
+
}
|
|
3664
|
+
if (isImage) {
|
|
3665
|
+
if (!assetExtension3(resolved)) {
|
|
3666
|
+
ctx.warn(`${ctx.file}: image '${target}' is not a supported asset type; left as is`);
|
|
3667
|
+
return null;
|
|
3668
|
+
}
|
|
3669
|
+
const abs = join6(ctx.rootDir, ...resolved.split("/"));
|
|
3670
|
+
if (!existsSync(abs)) {
|
|
3671
|
+
const message = `${ctx.file}: image '${target}' does not exist in the catalog tree`;
|
|
3672
|
+
if (ctx.missingAssets === "error") throw new Error(message);
|
|
3673
|
+
ctx.warn(`${message}; left as is`);
|
|
3674
|
+
return null;
|
|
3675
|
+
}
|
|
3676
|
+
const archivePath = resolved.startsWith("assets/") ? resolved : `assets/${resolved}`;
|
|
3677
|
+
if (!ctx.assets.has(archivePath))
|
|
3678
|
+
ctx.assets.set(archivePath, { path: archivePath, absPath: abs });
|
|
3679
|
+
return archivePath;
|
|
3680
|
+
}
|
|
3681
|
+
if (!/\.(md|markdown)$/i.test(resolved)) return null;
|
|
3682
|
+
const documentId = ctx.idByRel.get(resolved.replace(/\.(md|markdown)$/i, "").normalize("NFC"));
|
|
3683
|
+
if (!documentId) {
|
|
3684
|
+
ctx.warn(`${ctx.file}: link '${target}' does not name a document in the catalog; left as is`);
|
|
3685
|
+
return null;
|
|
3686
|
+
}
|
|
3687
|
+
if (!ctx.uri) return null;
|
|
3688
|
+
return formatKnowledgeUri({
|
|
3689
|
+
publisherId: ctx.uri.publisherId,
|
|
3690
|
+
catalogId: ctx.uri.catalogId,
|
|
3691
|
+
documentId
|
|
3692
|
+
});
|
|
3693
|
+
}
|
|
3694
|
+
function safeDecode2(value) {
|
|
3695
|
+
try {
|
|
3696
|
+
return decodeURIComponent(value);
|
|
3697
|
+
} catch {
|
|
3698
|
+
return value;
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
function rewriteLinks(ctx, markdown) {
|
|
3702
|
+
const lines = markdown.split("\n");
|
|
3703
|
+
let fence = null;
|
|
3704
|
+
return lines.map((line) => {
|
|
3705
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
3706
|
+
if (fenceMatch) {
|
|
3707
|
+
const marker = fenceMatch[1];
|
|
3708
|
+
if (fence === null) fence = marker[0];
|
|
3709
|
+
else if (marker[0] === fence) fence = null;
|
|
3710
|
+
return line;
|
|
3711
|
+
}
|
|
3712
|
+
if (fence !== null) return line;
|
|
3713
|
+
return line.replace(
|
|
3714
|
+
INLINE_LINK,
|
|
3715
|
+
(whole, bang, text, target, title) => {
|
|
3716
|
+
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
3717
|
+
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
3718
|
+
}
|
|
3719
|
+
);
|
|
3720
|
+
}).join("\n");
|
|
3721
|
+
}
|
|
3722
|
+
async function loadMarkdownCatalog(rootDir, opts) {
|
|
3723
|
+
const warn = opts.onWarning ?? (() => {
|
|
3724
|
+
});
|
|
3725
|
+
const toc = opts.toc ?? { format: "folders" };
|
|
3726
|
+
const hugo = toc.format === "hugo";
|
|
3727
|
+
const ignore = new Set(opts.ignore ?? []);
|
|
3728
|
+
const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
|
|
3729
|
+
let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
|
|
3730
|
+
const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
|
|
3731
|
+
for (const message of outline?.warnings ?? []) warn(message);
|
|
3732
|
+
if (outline) {
|
|
3733
|
+
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
3734
|
+
files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
|
|
3735
|
+
}
|
|
2682
3736
|
if (files.length === 0) {
|
|
2683
3737
|
throw new Error(`no Markdown files found under ${rootDir}`);
|
|
2684
3738
|
}
|
|
2685
3739
|
const takenTopicIds = /* @__PURE__ */ new Set();
|
|
2686
3740
|
const topicIdByDir = /* @__PURE__ */ new Map();
|
|
2687
3741
|
const topics = [];
|
|
3742
|
+
const topicById = /* @__PURE__ */ new Map();
|
|
2688
3743
|
const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
|
|
2689
3744
|
let rootTopicUsed = false;
|
|
2690
|
-
|
|
3745
|
+
let rootOverride;
|
|
3746
|
+
const addTopic = (topic) => {
|
|
3747
|
+
topics.push(topic);
|
|
3748
|
+
topicById.set(topic.id, topic);
|
|
3749
|
+
return topic;
|
|
3750
|
+
};
|
|
3751
|
+
const ensureTopicChain = async (relDirPosix) => {
|
|
2691
3752
|
if (!relDirPosix) {
|
|
2692
3753
|
rootTopicUsed = true;
|
|
2693
3754
|
return [rootTopicId];
|
|
@@ -2701,32 +3762,141 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2701
3762
|
if (!id) {
|
|
2702
3763
|
id = topicIdFor(segment, takenTopicIds);
|
|
2703
3764
|
topicIdByDir.set(prefix, id);
|
|
2704
|
-
|
|
3765
|
+
const topic = addTopic({
|
|
2705
3766
|
id,
|
|
2706
|
-
name: segment,
|
|
3767
|
+
name: titleFromFolderName(segment),
|
|
2707
3768
|
...path.length > 0 ? { parentId: path[path.length - 1] } : {}
|
|
2708
3769
|
});
|
|
3770
|
+
applyOverride(topic, await readTopicSidecar(join6(rootDir, ...prefix.split("/"))));
|
|
3771
|
+
applyOverride(topic, opts.topics?.[prefix]);
|
|
2709
3772
|
}
|
|
2710
3773
|
path.push(id);
|
|
2711
3774
|
}
|
|
2712
3775
|
return path;
|
|
2713
3776
|
};
|
|
2714
|
-
const
|
|
2715
|
-
|
|
3777
|
+
const placements = /* @__PURE__ */ new Map();
|
|
3778
|
+
if (outline) {
|
|
3779
|
+
const walk = (topic, chain) => {
|
|
3780
|
+
for (const entry of topic.entries) {
|
|
3781
|
+
const key = entry.file.normalize("NFC");
|
|
3782
|
+
if (placements.has(key)) {
|
|
3783
|
+
warn(
|
|
3784
|
+
`${entry.file}: listed more than once in the table of contents; the first place wins`
|
|
3785
|
+
);
|
|
3786
|
+
} else {
|
|
3787
|
+
placements.set(key, { chain, entry });
|
|
3788
|
+
}
|
|
3789
|
+
}
|
|
3790
|
+
for (const child of topic.children) walk(child, [...chain, child]);
|
|
3791
|
+
};
|
|
3792
|
+
walk(outline.root, []);
|
|
3793
|
+
}
|
|
3794
|
+
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
3795
|
+
const ensureOutlineChain = (chain) => {
|
|
3796
|
+
if (chain.length === 0) {
|
|
3797
|
+
rootTopicUsed = true;
|
|
3798
|
+
return [rootTopicId];
|
|
3799
|
+
}
|
|
3800
|
+
const ids = [];
|
|
3801
|
+
for (const node of chain) {
|
|
3802
|
+
let id = outlineTopicIds.get(node);
|
|
3803
|
+
if (!id) {
|
|
3804
|
+
id = topicIdFor(node.name, takenTopicIds);
|
|
3805
|
+
outlineTopicIds.set(node, id);
|
|
3806
|
+
const topic = {
|
|
3807
|
+
id,
|
|
3808
|
+
name: node.name,
|
|
3809
|
+
...ids.length > 0 ? { parentId: ids[ids.length - 1] } : {}
|
|
3810
|
+
};
|
|
3811
|
+
if (node.description) topic.description = node.description;
|
|
3812
|
+
if (node.order !== void 0) topic.sortKey = topicSortKeyForOrder(node.order);
|
|
3813
|
+
addTopic(topic);
|
|
3814
|
+
}
|
|
3815
|
+
ids.push(id);
|
|
3816
|
+
}
|
|
3817
|
+
return ids;
|
|
3818
|
+
};
|
|
3819
|
+
const shelves = /* @__PURE__ */ new Map();
|
|
3820
|
+
const ensureShelf = (parentId, sub, file) => {
|
|
3821
|
+
const key = `${parentId}/${sub.id}`;
|
|
3822
|
+
const existing = shelves.get(key);
|
|
3823
|
+
if (existing) {
|
|
3824
|
+
if (existing.title !== sub.title || existing.order !== sub.order) {
|
|
3825
|
+
throw new Error(
|
|
3826
|
+
`${file}: subcategory '${sub.id}' is declared with a different title or order in ${existing.file}`
|
|
3827
|
+
);
|
|
3828
|
+
}
|
|
3829
|
+
return existing.topicId;
|
|
3830
|
+
}
|
|
3831
|
+
const topicId = topicIdFor(`${parentId}-${sub.id}`, takenTopicIds);
|
|
3832
|
+
const topic = { id: topicId, name: sub.title ?? sub.id, parentId };
|
|
3833
|
+
if (sub.order !== void 0) topic.sortKey = topicSortKeyForOrder(sub.order);
|
|
3834
|
+
addTopic(topic);
|
|
3835
|
+
shelves.set(key, { topicId, title: sub.title, order: sub.order, file });
|
|
3836
|
+
return topicId;
|
|
3837
|
+
};
|
|
3838
|
+
const loaded = [];
|
|
3839
|
+
const fileById = /* @__PURE__ */ new Map();
|
|
3840
|
+
const idByRel = /* @__PURE__ */ new Map();
|
|
3841
|
+
const seenRel = /* @__PURE__ */ new Set();
|
|
2716
3842
|
for (const abs of files) {
|
|
2717
|
-
const rel =
|
|
3843
|
+
const rel = relOf(abs);
|
|
2718
3844
|
const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
2719
|
-
const
|
|
2720
|
-
const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
2721
|
-
if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
|
|
2722
|
-
seenIds.add(id);
|
|
3845
|
+
const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
2723
3846
|
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
2724
|
-
const fm =
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
3847
|
+
const fm = readFrontMatter(raw, rel, hugo);
|
|
3848
|
+
if (fm.unpublished) {
|
|
3849
|
+
warn(`${rel}: skipped, ${fm.unpublished} page`);
|
|
3850
|
+
continue;
|
|
3851
|
+
}
|
|
3852
|
+
seenRel.add(rel.normalize("NFC"));
|
|
3853
|
+
const id = fm.id ?? relNoExt;
|
|
3854
|
+
const idCheck = KnowledgeDocumentIdSchema.safeParse(id);
|
|
3855
|
+
if (!idCheck.success) throw new Error(`${rel}: front matter 'id' is not a valid document id`);
|
|
3856
|
+
const previous = fileById.get(id);
|
|
3857
|
+
if (previous) throw new Error(`${rel}: document id '${id}' is already used by ${previous}`);
|
|
3858
|
+
fileById.set(id, rel);
|
|
3859
|
+
idByRel.set(relNoExt, id);
|
|
3860
|
+
const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
3861
|
+
let title = fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
3862
|
+
let ordinal = fm.order;
|
|
3863
|
+
let topicPath;
|
|
3864
|
+
const placement = placements.get(rel.normalize("NFC"));
|
|
3865
|
+
if (placement) {
|
|
3866
|
+
topicPath = ensureOutlineChain(placement.chain);
|
|
3867
|
+
if (placement.entry.title) title = placement.entry.title;
|
|
3868
|
+
if (placement.entry.order !== void 0) ordinal = placement.entry.order;
|
|
3869
|
+
} else {
|
|
3870
|
+
if (outline)
|
|
3871
|
+
warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
|
|
3872
|
+
topicPath = await ensureTopicChain(relDir);
|
|
3873
|
+
if (hugo && ordinal === void 0) ordinal = fm.weight;
|
|
3874
|
+
}
|
|
3875
|
+
if (hugo && stem2 === HUGO_SECTION_PAGE) {
|
|
3876
|
+
const override = {
|
|
3877
|
+
...fm.title ? { name: fm.title } : {},
|
|
3878
|
+
...fm.description ?? fm.summary ? { description: fm.description ?? fm.summary } : {},
|
|
3879
|
+
...fm.weight !== void 0 ? { order: fm.weight } : {}
|
|
3880
|
+
};
|
|
3881
|
+
if (relDir === "") {
|
|
3882
|
+
rootOverride = override;
|
|
3883
|
+
} else {
|
|
3884
|
+
const leaf = topicById.get(topicPath[topicPath.length - 1]);
|
|
3885
|
+
applyOverride(leaf, override);
|
|
3886
|
+
applyOverride(leaf, opts.topics?.[relDir]);
|
|
3887
|
+
}
|
|
3888
|
+
if (!fm.body.trim()) continue;
|
|
3889
|
+
ordinal = SECTION_PAGE_ORDINAL;
|
|
3890
|
+
}
|
|
3891
|
+
if (fm.subcategory) {
|
|
3892
|
+
const parentId = topicPath[topicPath.length - 1];
|
|
3893
|
+
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
3894
|
+
}
|
|
3895
|
+
const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
|
|
3896
|
+
loaded.push({
|
|
3897
|
+
rel,
|
|
3898
|
+
file: rel,
|
|
3899
|
+
doc: {
|
|
2730
3900
|
id,
|
|
2731
3901
|
title,
|
|
2732
3902
|
slug: documentSlug(title),
|
|
@@ -2734,9 +3904,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2734
3904
|
language: opts.language,
|
|
2735
3905
|
topicPath,
|
|
2736
3906
|
markdown: fm.body,
|
|
2737
|
-
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
|
|
2738
|
-
|
|
3907
|
+
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
3908
|
+
...ordinal !== void 0 ? { ordinal } : {},
|
|
3909
|
+
...Object.keys(fm.meta).length > 0 ? { meta: fm.meta } : {}
|
|
3910
|
+
}
|
|
3911
|
+
});
|
|
3912
|
+
}
|
|
3913
|
+
if (outline) {
|
|
3914
|
+
const titleByRel = new Map(
|
|
3915
|
+
loaded.map((entry) => [entry.rel.normalize("NFC"), entry.doc.title])
|
|
2739
3916
|
);
|
|
3917
|
+
const nameFromPages = (topic) => {
|
|
3918
|
+
for (const child of topic.children) {
|
|
3919
|
+
const id = outlineTopicIds.get(child);
|
|
3920
|
+
const title = child.titleFromFile ? titleByRel.get(child.titleFromFile.normalize("NFC")) : void 0;
|
|
3921
|
+
const compiled = id ? topicById.get(id) : void 0;
|
|
3922
|
+
if (compiled && title) compiled.name = title;
|
|
3923
|
+
nameFromPages(child);
|
|
3924
|
+
}
|
|
3925
|
+
};
|
|
3926
|
+
nameFromPages(outline.root);
|
|
3927
|
+
for (const [file, placement] of placements) {
|
|
3928
|
+
if (!seenRel.has(file)) {
|
|
3929
|
+
warn(
|
|
3930
|
+
`${placement.entry.file}: named by the table of contents but not found among the Markdown files`
|
|
3931
|
+
);
|
|
3932
|
+
}
|
|
3933
|
+
}
|
|
2740
3934
|
}
|
|
2741
3935
|
if (rootTopicUsed) {
|
|
2742
3936
|
if (takenTopicIds.has(rootTopicId)) {
|
|
@@ -2744,9 +3938,39 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2744
3938
|
`root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
|
|
2745
3939
|
);
|
|
2746
3940
|
}
|
|
2747
|
-
|
|
3941
|
+
const root = { id: rootTopicId, name: opts.rootTopicName ?? "General" };
|
|
3942
|
+
applyOverride(root, await readTopicSidecar(rootDir));
|
|
3943
|
+
applyOverride(root, rootOverride);
|
|
3944
|
+
applyOverride(root, opts.topics?.[""]);
|
|
3945
|
+
topics.unshift(root);
|
|
3946
|
+
topicById.set(root.id, root);
|
|
2748
3947
|
}
|
|
2749
|
-
|
|
3948
|
+
const assets = /* @__PURE__ */ new Map();
|
|
3949
|
+
const documents = [];
|
|
3950
|
+
for (const entry of loaded) {
|
|
3951
|
+
const ctx = {
|
|
3952
|
+
rootDir,
|
|
3953
|
+
docDir: entry.rel.includes("/") ? entry.rel.slice(0, entry.rel.lastIndexOf("/")) : "",
|
|
3954
|
+
file: entry.file,
|
|
3955
|
+
idByRel,
|
|
3956
|
+
uri: opts.uri,
|
|
3957
|
+
assets,
|
|
3958
|
+
missingAssets: opts.missingAssets ?? "error",
|
|
3959
|
+
warn
|
|
3960
|
+
};
|
|
3961
|
+
documents.push(
|
|
3962
|
+
CatalogDocumentSchema.parse({
|
|
3963
|
+
...entry.doc,
|
|
3964
|
+
markdown: rewriteLinks(ctx, entry.doc.markdown)
|
|
3965
|
+
})
|
|
3966
|
+
);
|
|
3967
|
+
}
|
|
3968
|
+
return {
|
|
3969
|
+
topics,
|
|
3970
|
+
documents,
|
|
3971
|
+
assets: [...assets.values()].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
3972
|
+
toc: { format: toc.format, ...outlinePath ? { path: outlinePath } : {} }
|
|
3973
|
+
};
|
|
2750
3974
|
}
|
|
2751
3975
|
|
|
2752
3976
|
// src/registry-client/fetch.ts
|
|
@@ -2861,7 +4085,7 @@ function newerRegistryEntries(registry, installed) {
|
|
|
2861
4085
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
2862
4086
|
|
|
2863
4087
|
// src/signatures/jcs.ts
|
|
2864
|
-
import { canonicalizeJson } from "@bendyline/gezk";
|
|
4088
|
+
import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
|
|
2865
4089
|
export {
|
|
2866
4090
|
BGE_SMALL_EN_V15_1,
|
|
2867
4091
|
BODY_CODEC_MIN_BYTES,
|
|
@@ -2880,9 +4104,12 @@ export {
|
|
|
2880
4104
|
GEZEL_MULTILINGUAL_E5_SMALL_1,
|
|
2881
4105
|
GEZK_APPLICATION_ID,
|
|
2882
4106
|
GEZK_ARCHIVE_LIMITS,
|
|
4107
|
+
GEZK_FORMAT_GENERATIONS,
|
|
2883
4108
|
GEZK_FORMAT_VERSION,
|
|
2884
4109
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
2885
4110
|
GEZK_MIME_TYPE,
|
|
4111
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
4112
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
2886
4113
|
GezkArchiveError,
|
|
2887
4114
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
2888
4115
|
KNOWLEDGE_TOOLCHAIN,
|
|
@@ -2891,6 +4118,8 @@ export {
|
|
|
2891
4118
|
MANIFEST_PATH,
|
|
2892
4119
|
MARKDOWN_CHUNKS_2,
|
|
2893
4120
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
4121
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
4122
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
2894
4123
|
MIMETYPE_PATH,
|
|
2895
4124
|
MULTILINGUAL_E5_SMALL_1,
|
|
2896
4125
|
PARQUET_EXPORT_VERSION,
|
|
@@ -2907,13 +4136,16 @@ export {
|
|
|
2907
4136
|
SOURCE_NOTICES_PATH,
|
|
2908
4137
|
ZIP_FIXED_MTIME,
|
|
2909
4138
|
assertDuckdbCli,
|
|
2910
|
-
|
|
4139
|
+
assetReferences,
|
|
4140
|
+
assignOutlineOrders,
|
|
4141
|
+
canonicalizeJson2 as canonicalizeJson,
|
|
2911
4142
|
chunkContentHash,
|
|
2912
4143
|
chunkMarkdownProfile,
|
|
2913
4144
|
chunkUid,
|
|
2914
4145
|
compareCatalogVersions,
|
|
2915
4146
|
compileKnowledgeCatalog,
|
|
2916
4147
|
createProfileEmbedder,
|
|
4148
|
+
detectTableOfContents,
|
|
2917
4149
|
documentSlug,
|
|
2918
4150
|
duckdbVersion,
|
|
2919
4151
|
exportCatalogParquet,
|
|
@@ -2924,22 +4156,31 @@ export {
|
|
|
2924
4156
|
generateKnowledgeSigningKeyPair,
|
|
2925
4157
|
hammingTopK,
|
|
2926
4158
|
inspectGezkArchive,
|
|
4159
|
+
isSupportedFormatVersion,
|
|
4160
|
+
isSupportedIndexSchemaVersion,
|
|
2927
4161
|
knowledgeEmbeddingProfile,
|
|
2928
4162
|
knowledgeKeyId,
|
|
2929
4163
|
l2Normalize,
|
|
2930
4164
|
loadMarkdownCatalog,
|
|
2931
4165
|
manifestSigningPayload,
|
|
4166
|
+
newOutlineTopic,
|
|
2932
4167
|
newerRegistryEntries,
|
|
2933
4168
|
openCatalogDatabase,
|
|
4169
|
+
parseGitbookSummary,
|
|
4170
|
+
parseJupyterBookToc,
|
|
4171
|
+
parseMkdocsNav,
|
|
2934
4172
|
quantizeBinary,
|
|
2935
4173
|
quantizeInt8,
|
|
2936
4174
|
readGezkManifest,
|
|
4175
|
+
readMkdocsDocsDir,
|
|
2937
4176
|
rerankK,
|
|
2938
4177
|
rerankScore,
|
|
4178
|
+
resolveOutlinePath,
|
|
2939
4179
|
resolveTransformersModelOptions,
|
|
2940
4180
|
runDuckdbScript,
|
|
2941
4181
|
signManifest,
|
|
2942
4182
|
signRegistryIndex,
|
|
4183
|
+
titleFromFolderName,
|
|
2943
4184
|
transformersCachePath,
|
|
2944
4185
|
validateExtractedCatalog,
|
|
2945
4186
|
verifyManifestSignature,
|