@bendyline/gezel-knowledge 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/index.d.ts +322 -24
- package/dist/index.js +1866 -101
- package/package.json +3 -2
package/dist/index.js
CHANGED
|
@@ -2,17 +2,24 @@
|
|
|
2
2
|
import {
|
|
3
3
|
BODY_CODEC_MIN_BYTES,
|
|
4
4
|
GEZK_APPLICATION_ID,
|
|
5
|
+
GEZK_FORMAT_GENERATIONS,
|
|
5
6
|
GEZK_FORMAT_VERSION,
|
|
6
7
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
7
8
|
GEZK_MIME_TYPE,
|
|
9
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
10
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
8
11
|
LICENSE_NOTICE_PATH,
|
|
9
12
|
MANIFEST_PATH,
|
|
10
13
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
14
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
15
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
11
16
|
MIMETYPE_PATH,
|
|
12
17
|
README_PATH,
|
|
13
18
|
ROUTER_DB_PATH,
|
|
14
19
|
SOURCE_NOTICES_PATH,
|
|
15
|
-
ZIP_FIXED_MTIME
|
|
20
|
+
ZIP_FIXED_MTIME,
|
|
21
|
+
isSupportedFormatVersion,
|
|
22
|
+
isSupportedIndexSchemaVersion
|
|
16
23
|
} from "@bendyline/gezk";
|
|
17
24
|
var SHARD_TARGET_CHUNKS = 2e5;
|
|
18
25
|
var SHARD_MAX_CHUNKS = 25e4;
|
|
@@ -23,8 +30,10 @@ var CENTROID_SAMPLE_MAX = 65536;
|
|
|
23
30
|
var ROUTE_SHARDS_PROACTIVE = 3;
|
|
24
31
|
var ROUTE_SHARDS_EXPLICIT = 6;
|
|
25
32
|
var RERANK_FINAL_K = 24;
|
|
33
|
+
var RERANK_CANDIDATES_MIN = 1024;
|
|
34
|
+
var RERANK_CANDIDATES_MAX = 4096;
|
|
26
35
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
27
|
-
return Math.min(
|
|
36
|
+
return Math.min(RERANK_CANDIDATES_MAX, Math.max(RERANK_CANDIDATES_MIN, 32 * finalK), chunkCount);
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
// src/format/ddl.ts
|
|
@@ -35,7 +44,14 @@ import { documentSlug } from "@bendyline/gezk";
|
|
|
35
44
|
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
36
45
|
|
|
37
46
|
// src/format/quantize.ts
|
|
38
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
centerVector,
|
|
49
|
+
l2Normalize,
|
|
50
|
+
quantizeBinary,
|
|
51
|
+
quantizeBinaryForProfile,
|
|
52
|
+
quantizeInt8,
|
|
53
|
+
rerankScore
|
|
54
|
+
} from "@bendyline/gezk";
|
|
39
55
|
|
|
40
56
|
// src/chunking/markdown-chunker.ts
|
|
41
57
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -165,11 +181,23 @@ function splitOversizeLine(line, opts) {
|
|
|
165
181
|
|
|
166
182
|
// src/compiler/compile.ts
|
|
167
183
|
import { createHash } from "crypto";
|
|
168
|
-
import { mkdirSync, rmSync, statSync } from "fs";
|
|
184
|
+
import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
|
|
169
185
|
import { stat } from "fs/promises";
|
|
170
186
|
import { join } from "path";
|
|
171
187
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
172
|
-
import {
|
|
188
|
+
import {
|
|
189
|
+
GEZK_MANIFEST_KIND,
|
|
190
|
+
KnowledgeAssetPathSchema,
|
|
191
|
+
KnowledgeCatalogManifestSchema,
|
|
192
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
193
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
194
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
195
|
+
assetExtension,
|
|
196
|
+
assetKindForExtension,
|
|
197
|
+
canonicalizeJson,
|
|
198
|
+
sniffAssetType,
|
|
199
|
+
svgInertnessProblem
|
|
200
|
+
} from "@bendyline/gezk";
|
|
173
201
|
|
|
174
202
|
// src/archive/write.ts
|
|
175
203
|
import { createWriteStream } from "fs";
|
|
@@ -195,10 +223,10 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
195
223
|
else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
|
|
196
224
|
else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
|
|
197
225
|
}
|
|
198
|
-
await new Promise((
|
|
226
|
+
await new Promise((resolve5, reject) => {
|
|
199
227
|
const out = createWriteStream(outputPath);
|
|
200
228
|
out.on("error", reject);
|
|
201
|
-
out.on("close", () =>
|
|
229
|
+
out.on("close", () => resolve5());
|
|
202
230
|
zip.outputStream.on("error", reject);
|
|
203
231
|
zip.outputStream.pipe(out);
|
|
204
232
|
zip.end();
|
|
@@ -250,7 +278,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
250
278
|
"a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
|
|
251
279
|
);
|
|
252
280
|
}
|
|
253
|
-
const
|
|
281
|
+
const topicById = assertTopicForest(opts.topics);
|
|
282
|
+
const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
|
|
254
283
|
const profile = opts.embeddingProfile;
|
|
255
284
|
const chunkerOpts = {
|
|
256
285
|
unit: opts.chunkingProfile.unit,
|
|
@@ -263,13 +292,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
263
292
|
for await (const doc of opts.documents) {
|
|
264
293
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
265
294
|
seenIds.add(doc.id);
|
|
266
|
-
const rootTopic = doc.topicPath[0];
|
|
267
|
-
if (!rootTopic || !topicIds.has(rootTopic)) {
|
|
268
|
-
throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
|
|
269
|
-
}
|
|
270
295
|
prepared.push({
|
|
271
296
|
doc,
|
|
272
297
|
topicPathKey: doc.topicPath.join("/"),
|
|
298
|
+
leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
|
|
299
|
+
metaJson: encodeDocumentMeta(doc),
|
|
273
300
|
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
274
301
|
});
|
|
275
302
|
}
|
|
@@ -336,8 +363,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
336
363
|
});
|
|
337
364
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
338
365
|
for (const p of prepared) {
|
|
339
|
-
|
|
340
|
-
topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
|
|
366
|
+
topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
|
|
341
367
|
}
|
|
342
368
|
{
|
|
343
369
|
const stmt = router.prepare(
|
|
@@ -356,9 +382,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
356
382
|
}
|
|
357
383
|
{
|
|
358
384
|
const insertDoc = router.prepare(
|
|
359
|
-
`INSERT INTO documents (id, title, slug, summary, language, topic_id,
|
|
360
|
-
source_url, source_revision, source_updated_at, attribution_json,
|
|
361
|
-
|
|
385
|
+
`INSERT INTO documents (id, title, slug, summary, language, topic_id, ordinal, shard_id,
|
|
386
|
+
chunk_count, source_url, source_revision, source_updated_at, attribution_json, meta_json,
|
|
387
|
+
body_codec, body_blob)
|
|
388
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
362
389
|
);
|
|
363
390
|
const insertAlias = router.prepare(
|
|
364
391
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
@@ -386,13 +413,15 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
386
413
|
p.doc.slug || documentSlug(p.doc.title),
|
|
387
414
|
p.doc.summary ?? null,
|
|
388
415
|
p.doc.language,
|
|
389
|
-
p.
|
|
416
|
+
p.leafTopicId,
|
|
417
|
+
p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
|
|
390
418
|
BigInt(shardOf.get(p.doc.id)),
|
|
391
419
|
BigInt(p.chunks.length),
|
|
392
420
|
p.doc.sourceUrl ?? null,
|
|
393
421
|
p.doc.sourceRevision ?? null,
|
|
394
422
|
p.doc.sourceUpdatedAt ?? null,
|
|
395
423
|
p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
|
|
424
|
+
p.metaJson,
|
|
396
425
|
useBrotli ? "br" : "none",
|
|
397
426
|
blob
|
|
398
427
|
);
|
|
@@ -449,7 +478,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
449
478
|
}
|
|
450
479
|
const row = pendingRows[i];
|
|
451
480
|
const stmts = vecStmtsFor(shardDb(row.shardId));
|
|
452
|
-
stmts.vec.run(BigInt(row.chunkId), Buffer.from(
|
|
481
|
+
stmts.vec.run(BigInt(row.chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
|
|
453
482
|
stmts.int8.run(BigInt(row.chunkId), Buffer.from(quantizeInt8(unit).buffer));
|
|
454
483
|
const shardTotal = shardChunkCounts.get(row.shardId) ?? 0;
|
|
455
484
|
void shardTotal;
|
|
@@ -468,7 +497,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
468
497
|
const db = shardDb(shardId);
|
|
469
498
|
shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
|
|
470
499
|
const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
|
|
471
|
-
topics.add(p.
|
|
500
|
+
topics.add(p.leafTopicId);
|
|
472
501
|
shardTopicIds.set(shardId, topics);
|
|
473
502
|
const insertChunk = db.prepare(
|
|
474
503
|
`INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
|
|
@@ -631,6 +660,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
631
660
|
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
632
661
|
});
|
|
633
662
|
}
|
|
663
|
+
for (const asset of preparedAssets) {
|
|
664
|
+
files.push({ path: asset.path, sizeBytes: asset.sizeBytes, sha256: asset.sha256 });
|
|
665
|
+
}
|
|
666
|
+
assertAssetReferencesDeclared(prepared, preparedAssets);
|
|
634
667
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
635
668
|
kind: GEZK_MANIFEST_KIND,
|
|
636
669
|
formatVersion: GEZK_FORMAT_VERSION,
|
|
@@ -649,13 +682,24 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
649
682
|
},
|
|
650
683
|
embedding: profile,
|
|
651
684
|
chunking: opts.chunkingProfile,
|
|
652
|
-
topics: opts.topics.map((t) => ({
|
|
685
|
+
topics: opts.topics.map((t) => ({
|
|
686
|
+
id: t.id,
|
|
687
|
+
name: t.name,
|
|
688
|
+
...t.parentId ? { parentId: t.parentId } : {},
|
|
689
|
+
sortKey: t.sortKey ?? t.name.toLowerCase(),
|
|
690
|
+
...t.description ? { description: t.description } : {}
|
|
691
|
+
})),
|
|
653
692
|
router: {
|
|
654
693
|
shardTargetChunks: shardTarget,
|
|
655
694
|
shards: routerShardStats.sort((a, b) => a.id - b.id),
|
|
656
695
|
totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
|
|
657
696
|
},
|
|
658
|
-
counts: {
|
|
697
|
+
counts: {
|
|
698
|
+
documents: prepared.length,
|
|
699
|
+
chunks: totalChunks,
|
|
700
|
+
shards: shardCount,
|
|
701
|
+
assets: preparedAssets.length
|
|
702
|
+
},
|
|
659
703
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
660
704
|
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
661
705
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
@@ -672,7 +716,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
672
716
|
...Object.entries(extraFiles).map(([path, content]) => ({
|
|
673
717
|
path,
|
|
674
718
|
content: Buffer.from(content, "utf8")
|
|
675
|
-
}))
|
|
719
|
+
})),
|
|
720
|
+
...preparedAssets.map((asset) => ({ path: asset.path, content: asset.bytes }))
|
|
676
721
|
];
|
|
677
722
|
await writeGezkArchive(opts.outputPath, archiveEntries);
|
|
678
723
|
const archiveBytes = (await stat(opts.outputPath)).size;
|
|
@@ -688,6 +733,134 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
688
733
|
closeAll();
|
|
689
734
|
}
|
|
690
735
|
}
|
|
736
|
+
function assertTopicForest(topics) {
|
|
737
|
+
const byId = /* @__PURE__ */ new Map();
|
|
738
|
+
for (const topic of topics) {
|
|
739
|
+
if (byId.has(topic.id)) throw new Error(`duplicate topic id: ${topic.id}`);
|
|
740
|
+
byId.set(topic.id, topic);
|
|
741
|
+
}
|
|
742
|
+
for (const topic of topics) {
|
|
743
|
+
const seen = /* @__PURE__ */ new Set();
|
|
744
|
+
let depth = 0;
|
|
745
|
+
let current = topic;
|
|
746
|
+
while (current) {
|
|
747
|
+
if (seen.has(current.id)) throw new Error(`topic '${topic.id}' sits in a parent cycle`);
|
|
748
|
+
seen.add(current.id);
|
|
749
|
+
depth += 1;
|
|
750
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
751
|
+
throw new Error(`topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`);
|
|
752
|
+
}
|
|
753
|
+
if (current.parentId === void 0) break;
|
|
754
|
+
const parent = byId.get(current.parentId);
|
|
755
|
+
if (!parent) {
|
|
756
|
+
throw new Error(`topic '${current.id}' names an undeclared parent '${current.parentId}'`);
|
|
757
|
+
}
|
|
758
|
+
current = parent;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
return byId;
|
|
762
|
+
}
|
|
763
|
+
function assertTopicPath(topicById, docId, path) {
|
|
764
|
+
for (let i = 0; i < path.length; i++) {
|
|
765
|
+
const id = path[i];
|
|
766
|
+
const topic = topicById.get(id);
|
|
767
|
+
if (!topic) {
|
|
768
|
+
throw new Error(
|
|
769
|
+
`document ${docId} names unknown topic '${id}' (segment ${i} of ${path.join("/")})`
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
const expectedParent = i === 0 ? void 0 : path[i - 1];
|
|
773
|
+
if (topic.parentId !== expectedParent) {
|
|
774
|
+
throw new Error(
|
|
775
|
+
`document ${docId}: topic path ${path.join("/")} does not follow the declared tree at '${id}' (its parent is '${topic.parentId ?? "(root)"}')`
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
return path[path.length - 1];
|
|
780
|
+
}
|
|
781
|
+
function encodeDocumentMeta(doc) {
|
|
782
|
+
if (!doc.meta || Object.keys(doc.meta).length === 0) return null;
|
|
783
|
+
let json;
|
|
784
|
+
try {
|
|
785
|
+
json = canonicalizeJson(doc.meta);
|
|
786
|
+
} catch (error) {
|
|
787
|
+
throw new Error(
|
|
788
|
+
`document ${doc.id} has metadata that cannot be serialized: ${error instanceof Error ? error.message : String(error)}`
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
const bytes = Buffer.byteLength(json, "utf8");
|
|
792
|
+
if (bytes > MAX_KNOWLEDGE_DOCUMENT_META_BYTES) {
|
|
793
|
+
throw new Error(
|
|
794
|
+
`document ${doc.id} has ${bytes} bytes of metadata; the limit is ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES}`
|
|
795
|
+
);
|
|
796
|
+
}
|
|
797
|
+
return json;
|
|
798
|
+
}
|
|
799
|
+
function prepareAssets(assets, extraFiles) {
|
|
800
|
+
for (const path of Object.keys(extraFiles)) {
|
|
801
|
+
if (path.startsWith("assets/")) {
|
|
802
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
806
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
807
|
+
}
|
|
808
|
+
const seen = /* @__PURE__ */ new Set();
|
|
809
|
+
const prepared = [];
|
|
810
|
+
let total = 0;
|
|
811
|
+
for (const asset of assets) {
|
|
812
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
813
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
814
|
+
const key = asset.path.toLowerCase();
|
|
815
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
816
|
+
seen.add(key);
|
|
817
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
818
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
819
|
+
}
|
|
820
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
821
|
+
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
822
|
+
throw new Error(
|
|
823
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
824
|
+
);
|
|
825
|
+
}
|
|
826
|
+
total += bytes.byteLength;
|
|
827
|
+
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
828
|
+
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
829
|
+
}
|
|
830
|
+
const ext = assetExtension(asset.path);
|
|
831
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
832
|
+
const actual = sniffAssetType(bytes);
|
|
833
|
+
if (expected === null || actual !== expected) {
|
|
834
|
+
throw new Error(
|
|
835
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
836
|
+
);
|
|
837
|
+
}
|
|
838
|
+
if (ext === "svg") {
|
|
839
|
+
const problem = svgInertnessProblem(bytes);
|
|
840
|
+
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
841
|
+
}
|
|
842
|
+
prepared.push({
|
|
843
|
+
path: asset.path,
|
|
844
|
+
bytes,
|
|
845
|
+
sizeBytes: bytes.byteLength,
|
|
846
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
847
|
+
});
|
|
848
|
+
}
|
|
849
|
+
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
850
|
+
}
|
|
851
|
+
function assertAssetReferencesDeclared(prepared, assets) {
|
|
852
|
+
const declared = new Set(assets.map((a) => a.path));
|
|
853
|
+
for (const p of prepared) {
|
|
854
|
+
for (const match of p.doc.markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
855
|
+
const target = match[1];
|
|
856
|
+
if (!declared.has(target)) {
|
|
857
|
+
throw new Error(
|
|
858
|
+
`document ${p.doc.id} references '${target}', which is not among the catalog's assets`
|
|
859
|
+
);
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
691
864
|
function createCatalogDb(absPath) {
|
|
692
865
|
const db = new DatabaseSync(absPath);
|
|
693
866
|
db.exec("PRAGMA page_size=8192");
|
|
@@ -960,9 +1133,9 @@ function assertMimetypeEntry(entry, position) {
|
|
|
960
1133
|
function assertSupportedFormat(raw) {
|
|
961
1134
|
const kind = raw?.kind;
|
|
962
1135
|
const version = raw?.formatVersion;
|
|
963
|
-
if (kind !== GEZK_MANIFEST_KIND2 || version
|
|
1136
|
+
if (kind !== GEZK_MANIFEST_KIND2 || !isSupportedFormatVersion(version)) {
|
|
964
1137
|
throw new GezkArchiveError(
|
|
965
|
-
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${
|
|
1138
|
+
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_SUPPORTED_FORMAT_VERSIONS.join(", ")} \u2014 catalogs built for another version need a matching reader`,
|
|
966
1139
|
"format-version"
|
|
967
1140
|
);
|
|
968
1141
|
}
|
|
@@ -1202,9 +1375,9 @@ function openCatalogDatabase(absPath) {
|
|
|
1202
1375
|
);
|
|
1203
1376
|
}
|
|
1204
1377
|
const schemaVersion = readPragmaNumber(db, "user_version");
|
|
1205
|
-
if (schemaVersion
|
|
1378
|
+
if (!isSupportedIndexSchemaVersion(schemaVersion)) {
|
|
1206
1379
|
throw new CatalogOpenError(
|
|
1207
|
-
`unsupported index schema version ${schemaVersion} (this reader supports ${
|
|
1380
|
+
`unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.join(", ")}; catalogs built for another generation need a matching reader)`,
|
|
1208
1381
|
"schema-version"
|
|
1209
1382
|
);
|
|
1210
1383
|
}
|
|
@@ -1212,6 +1385,7 @@ function openCatalogDatabase(absPath) {
|
|
|
1212
1385
|
db.exec("PRAGMA cache_size=-8192");
|
|
1213
1386
|
return {
|
|
1214
1387
|
db,
|
|
1388
|
+
schemaVersion,
|
|
1215
1389
|
close: () => {
|
|
1216
1390
|
try {
|
|
1217
1391
|
db.close();
|
|
@@ -1234,8 +1408,16 @@ function readPragmaNumber(db, pragma) {
|
|
|
1234
1408
|
}
|
|
1235
1409
|
|
|
1236
1410
|
// src/reader/catalog-handle.ts
|
|
1411
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
1237
1412
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1238
1413
|
import { brotliDecompressSync } from "zlib";
|
|
1414
|
+
import {
|
|
1415
|
+
KnowledgeEmbeddingProfileSchema,
|
|
1416
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
|
|
1417
|
+
assetContentType,
|
|
1418
|
+
embeddingProfileCenter,
|
|
1419
|
+
isKnowledgeAssetPath
|
|
1420
|
+
} from "@bendyline/gezk";
|
|
1239
1421
|
|
|
1240
1422
|
// src/reader/bit-scan.ts
|
|
1241
1423
|
var POPCOUNT8 = new Uint8Array(256);
|
|
@@ -1307,13 +1489,95 @@ function hammingTopK(index, query, k) {
|
|
|
1307
1489
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1308
1490
|
return hits;
|
|
1309
1491
|
}
|
|
1492
|
+
function asymmetricTopK(index, query, k) {
|
|
1493
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1494
|
+
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1495
|
+
throw new Error(
|
|
1496
|
+
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1497
|
+
);
|
|
1498
|
+
}
|
|
1499
|
+
const limit = Math.min(k, rows);
|
|
1500
|
+
if (limit <= 0) return [];
|
|
1501
|
+
const lut = new Float32Array(bytesPerRow * 256);
|
|
1502
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1503
|
+
for (let b = 0; b < 256; b++) {
|
|
1504
|
+
let s = 0;
|
|
1505
|
+
for (let i = 0; i < 8; i++) {
|
|
1506
|
+
const d = p * 8 + i;
|
|
1507
|
+
if (d >= query.length) break;
|
|
1508
|
+
const v = query[d];
|
|
1509
|
+
s += b >> i & 1 ? v : -v;
|
|
1510
|
+
}
|
|
1511
|
+
lut[p * 256 + b] = s;
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
const heapScore = new Float64Array(limit);
|
|
1515
|
+
const heapId = new Int32Array(limit);
|
|
1516
|
+
let size = 0;
|
|
1517
|
+
const worse = (i, j) => heapScore[i] < heapScore[j] || heapScore[i] === heapScore[j] && heapId[i] > heapId[j];
|
|
1518
|
+
const swap = (i, j) => {
|
|
1519
|
+
const s = heapScore[i];
|
|
1520
|
+
const id = heapId[i];
|
|
1521
|
+
heapScore[i] = heapScore[j];
|
|
1522
|
+
heapId[i] = heapId[j];
|
|
1523
|
+
heapScore[j] = s;
|
|
1524
|
+
heapId[j] = id;
|
|
1525
|
+
};
|
|
1526
|
+
const siftUp = (start) => {
|
|
1527
|
+
let i = start;
|
|
1528
|
+
while (i > 0) {
|
|
1529
|
+
const parent = i - 1 >> 1;
|
|
1530
|
+
if (!worse(i, parent)) break;
|
|
1531
|
+
swap(i, parent);
|
|
1532
|
+
i = parent;
|
|
1533
|
+
}
|
|
1534
|
+
};
|
|
1535
|
+
const siftDown = (start) => {
|
|
1536
|
+
let i = start;
|
|
1537
|
+
for (; ; ) {
|
|
1538
|
+
const left = 2 * i + 1;
|
|
1539
|
+
if (left >= size) break;
|
|
1540
|
+
const right = left + 1;
|
|
1541
|
+
const child = right < size && worse(right, left) ? right : left;
|
|
1542
|
+
if (!worse(child, i)) break;
|
|
1543
|
+
swap(child, i);
|
|
1544
|
+
i = child;
|
|
1545
|
+
}
|
|
1546
|
+
};
|
|
1547
|
+
let offset = 0;
|
|
1548
|
+
for (let r = 0; r < rows; r++) {
|
|
1549
|
+
let score = 0;
|
|
1550
|
+
for (let p = 0; p < bytesPerRow; p++) {
|
|
1551
|
+
score += lut[p * 256 + bits[offset + p]];
|
|
1552
|
+
}
|
|
1553
|
+
offset += bytesPerRow;
|
|
1554
|
+
if (size < limit) {
|
|
1555
|
+
heapScore[size] = score;
|
|
1556
|
+
heapId[size] = r + 1;
|
|
1557
|
+
size++;
|
|
1558
|
+
siftUp(size - 1);
|
|
1559
|
+
} else if (score > heapScore[0] || score === heapScore[0] && r + 1 < heapId[0]) {
|
|
1560
|
+
heapScore[0] = score;
|
|
1561
|
+
heapId[0] = r + 1;
|
|
1562
|
+
siftDown(0);
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
const hits = [];
|
|
1566
|
+
for (let i = 0; i < size; i++) {
|
|
1567
|
+
hits.push({ chunkId: heapId[i], score: heapScore[i] });
|
|
1568
|
+
}
|
|
1569
|
+
hits.sort((a, b) => b.score - a.score || a.chunkId - b.chunkId);
|
|
1570
|
+
return hits;
|
|
1571
|
+
}
|
|
1310
1572
|
|
|
1311
1573
|
// src/reader/catalog-handle.ts
|
|
1574
|
+
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
1312
1575
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
1313
1576
|
var CatalogHandle = class _CatalogHandle {
|
|
1314
1577
|
constructor(rootDir, router) {
|
|
1315
1578
|
this.rootDir = rootDir;
|
|
1316
1579
|
this.router = router;
|
|
1580
|
+
this.schemaVersion = router.schemaVersion;
|
|
1317
1581
|
this.meta = {};
|
|
1318
1582
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1319
1583
|
this.meta[row.key] = row.value;
|
|
@@ -1325,8 +1589,13 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1325
1589
|
router;
|
|
1326
1590
|
connections = /* @__PURE__ */ new Map();
|
|
1327
1591
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
1592
|
+
assetsByPath = null;
|
|
1593
|
+
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1594
|
+
binaryCenterCache;
|
|
1328
1595
|
meta;
|
|
1329
1596
|
shards;
|
|
1597
|
+
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
1598
|
+
schemaVersion;
|
|
1330
1599
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1331
1600
|
static open(rootDir) {
|
|
1332
1601
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
@@ -1405,6 +1674,28 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1405
1674
|
this.bitIndexes.set(shard.path, index);
|
|
1406
1675
|
return index;
|
|
1407
1676
|
}
|
|
1677
|
+
/**
|
|
1678
|
+
* The stage-1 center the catalog's bits were derived with (a
|
|
1679
|
+
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
1680
|
+
* router's profile echo, never from this reader's registry: the echo is
|
|
1681
|
+
* the statement of how THIS catalog was built.
|
|
1682
|
+
*/
|
|
1683
|
+
binaryCenter() {
|
|
1684
|
+
if (this.binaryCenterCache !== void 0) return this.binaryCenterCache;
|
|
1685
|
+
const raw = this.meta.embedding_profile_json;
|
|
1686
|
+
let parsed = null;
|
|
1687
|
+
try {
|
|
1688
|
+
parsed = raw ? JSON.parse(raw) : null;
|
|
1689
|
+
} catch {
|
|
1690
|
+
parsed = null;
|
|
1691
|
+
}
|
|
1692
|
+
const profile = KnowledgeEmbeddingProfileSchema.safeParse(parsed);
|
|
1693
|
+
if (!profile.success) {
|
|
1694
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1695
|
+
}
|
|
1696
|
+
this.binaryCenterCache = embeddingProfileCenter(profile.data);
|
|
1697
|
+
return this.binaryCenterCache;
|
|
1698
|
+
}
|
|
1408
1699
|
/** The embedding dimension from the router's profile echo. */
|
|
1409
1700
|
dimensions() {
|
|
1410
1701
|
const raw = this.meta.embedding_profile_json;
|
|
@@ -1428,40 +1719,124 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1428
1719
|
return target;
|
|
1429
1720
|
}
|
|
1430
1721
|
// ── browsing (the shipped table of contents) ──────────────────────────────
|
|
1722
|
+
/** The format version the router was written with (`meta.format_version`). */
|
|
1723
|
+
get formatVersion() {
|
|
1724
|
+
return this.meta.format_version ?? "";
|
|
1725
|
+
}
|
|
1726
|
+
documentColumns() {
|
|
1727
|
+
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1728
|
+
return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
1729
|
+
}
|
|
1730
|
+
documentOrder() {
|
|
1731
|
+
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
1732
|
+
}
|
|
1733
|
+
/**
|
|
1734
|
+
* Every topic with its direct count and a rollup over its subtree. On a
|
|
1735
|
+
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1736
|
+
*/
|
|
1431
1737
|
topics() {
|
|
1432
1738
|
return this.router.db.prepare(
|
|
1433
|
-
|
|
1739
|
+
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1740
|
+
SELECT id, id, 0 FROM topics
|
|
1741
|
+
UNION ALL
|
|
1742
|
+
SELECT sub.root, t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1743
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1744
|
+
),
|
|
1745
|
+
rollup AS (
|
|
1746
|
+
SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
1747
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
1748
|
+
)
|
|
1749
|
+
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1750
|
+
rollup.total AS total_document_count
|
|
1751
|
+
FROM topics t JOIN rollup ON rollup.id = t.id
|
|
1752
|
+
ORDER BY t.sort_key, t.id`
|
|
1434
1753
|
).all().map((r) => ({
|
|
1435
1754
|
id: r.id,
|
|
1436
1755
|
parentId: r.parent_id,
|
|
1437
1756
|
name: r.name,
|
|
1438
1757
|
description: r.description,
|
|
1439
1758
|
sortKey: r.sort_key,
|
|
1440
|
-
documentCount: Number(r.document_count)
|
|
1759
|
+
documentCount: Number(r.document_count),
|
|
1760
|
+
totalDocumentCount: Number(r.total_document_count)
|
|
1441
1761
|
}));
|
|
1442
1762
|
}
|
|
1763
|
+
/**
|
|
1764
|
+
* A page of documents, by default including those filed under the topic's
|
|
1765
|
+
* descendants (`descendants: false` lists only the topic's own).
|
|
1766
|
+
*/
|
|
1443
1767
|
documentsPage(opts = {}) {
|
|
1444
1768
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1445
1769
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1446
|
-
const
|
|
1447
|
-
|
|
1770
|
+
const descendants = opts.descendants ?? true;
|
|
1771
|
+
let scope = "";
|
|
1772
|
+
let where = "";
|
|
1773
|
+
const params = [];
|
|
1774
|
+
if (opts.topicId && descendants) {
|
|
1775
|
+
scope = `WITH RECURSIVE sub(id, depth) AS (
|
|
1776
|
+
SELECT ?, 0
|
|
1777
|
+
UNION ALL
|
|
1778
|
+
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1779
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1780
|
+
)`;
|
|
1781
|
+
where = "WHERE topic_id IN (SELECT id FROM sub)";
|
|
1782
|
+
params.push(opts.topicId);
|
|
1783
|
+
} else if (opts.topicId) {
|
|
1784
|
+
where = "WHERE topic_id = ?";
|
|
1785
|
+
params.push(opts.topicId);
|
|
1786
|
+
}
|
|
1448
1787
|
const total = Number(
|
|
1449
|
-
this.router.db.prepare(
|
|
1788
|
+
this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
|
|
1450
1789
|
);
|
|
1451
1790
|
const documents = this.router.db.prepare(
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
|
|
1791
|
+
`${scope} SELECT ${this.documentColumns()}
|
|
1792
|
+
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1455
1793
|
).all(...params, limit, offset).map(rowToDocumentMeta);
|
|
1456
1794
|
return { documents, total };
|
|
1457
1795
|
}
|
|
1458
1796
|
getDocument(id) {
|
|
1459
1797
|
const row = this.router.db.prepare(
|
|
1460
|
-
`SELECT
|
|
1461
|
-
source_updated_at, attribution_json, body_codec, body_blob
|
|
1798
|
+
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1462
1799
|
FROM documents WHERE id = ?`
|
|
1463
1800
|
).get(id);
|
|
1464
1801
|
if (!row) return null;
|
|
1802
|
+
return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
|
|
1803
|
+
}
|
|
1804
|
+
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
1805
|
+
documentsWithUndeclaredTopic() {
|
|
1806
|
+
return Number(
|
|
1807
|
+
this.router.db.prepare(
|
|
1808
|
+
"SELECT COUNT(*) AS n FROM documents d LEFT JOIN topics t ON t.id = d.topic_id WHERE t.id IS NULL"
|
|
1809
|
+
).get().n
|
|
1810
|
+
);
|
|
1811
|
+
}
|
|
1812
|
+
/**
|
|
1813
|
+
* Validator support: every `meta_json` must be a JSON object within the
|
|
1814
|
+
* byte limit. Returns the offending ids (capped) rather than throwing.
|
|
1815
|
+
*/
|
|
1816
|
+
checkDocumentMeta(maxBytes, cap = 5) {
|
|
1817
|
+
const invalid = [];
|
|
1818
|
+
const oversize = [];
|
|
1819
|
+
if (this.schemaVersion < 3) return { invalid, oversize };
|
|
1820
|
+
const rows = this.router.db.prepare("SELECT id, meta_json FROM documents WHERE meta_json IS NOT NULL").iterate();
|
|
1821
|
+
for (const row of rows) {
|
|
1822
|
+
if (invalid.length >= cap && oversize.length >= cap) break;
|
|
1823
|
+
if (Buffer.byteLength(row.meta_json, "utf8") > maxBytes) {
|
|
1824
|
+
if (oversize.length < cap) oversize.push(row.id);
|
|
1825
|
+
continue;
|
|
1826
|
+
}
|
|
1827
|
+
if (parseJsonObject(row.meta_json) === null && invalid.length < cap) invalid.push(row.id);
|
|
1828
|
+
}
|
|
1829
|
+
return { invalid, oversize };
|
|
1830
|
+
}
|
|
1831
|
+
/** Validator support: decoded bodies, streamed one at a time. */
|
|
1832
|
+
*documentBodies() {
|
|
1833
|
+
const rows = this.router.db.prepare("SELECT id, body_codec, body_blob FROM documents ORDER BY id").iterate();
|
|
1834
|
+
for (const row of rows) {
|
|
1835
|
+
const id = row.id;
|
|
1836
|
+
yield { id, markdown: this.decodeBody(id, row) };
|
|
1837
|
+
}
|
|
1838
|
+
}
|
|
1839
|
+
decodeBody(id, row) {
|
|
1465
1840
|
const blob = row.body_blob;
|
|
1466
1841
|
if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
|
|
1467
1842
|
throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
|
|
@@ -1484,8 +1859,69 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1484
1859
|
} else {
|
|
1485
1860
|
throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
|
|
1486
1861
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1862
|
+
return body.toString("utf8");
|
|
1863
|
+
}
|
|
1864
|
+
// ── assets ────────────────────────────────────────────────────────────────
|
|
1865
|
+
/**
|
|
1866
|
+
* The manifest's `files` entries under `assets/`, read lazily: the
|
|
1867
|
+
* declaration in the manifest is what authorizes serving a file, since
|
|
1868
|
+
* extraction reconciled and hashed every declared entry.
|
|
1869
|
+
*/
|
|
1870
|
+
assetIndex() {
|
|
1871
|
+
if (this.assetsByPath) return this.assetsByPath;
|
|
1872
|
+
const index = /* @__PURE__ */ new Map();
|
|
1873
|
+
let raw;
|
|
1874
|
+
try {
|
|
1875
|
+
raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
1876
|
+
} catch (error) {
|
|
1877
|
+
if (error.code === "ENOENT") {
|
|
1878
|
+
this.assetsByPath = index;
|
|
1879
|
+
return index;
|
|
1880
|
+
}
|
|
1881
|
+
throw new CatalogOpenError(
|
|
1882
|
+
`cannot read ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1883
|
+
"corrupt"
|
|
1884
|
+
);
|
|
1885
|
+
}
|
|
1886
|
+
const files = raw?.files;
|
|
1887
|
+
if (Array.isArray(files)) {
|
|
1888
|
+
for (const file of files) {
|
|
1889
|
+
const path = file?.path;
|
|
1890
|
+
if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
|
|
1891
|
+
const contentType = assetContentType(path);
|
|
1892
|
+
if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
|
|
1893
|
+
continue;
|
|
1894
|
+
}
|
|
1895
|
+
index.set(path, { path, contentType, sizeBytes: file.sizeBytes, sha256: file.sha256 });
|
|
1896
|
+
}
|
|
1897
|
+
}
|
|
1898
|
+
this.assetsByPath = index;
|
|
1899
|
+
return index;
|
|
1900
|
+
}
|
|
1901
|
+
/** Declared assets, sorted by path. */
|
|
1902
|
+
assets() {
|
|
1903
|
+
return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
1904
|
+
}
|
|
1905
|
+
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
1906
|
+
readAsset(path) {
|
|
1907
|
+
const info = this.assetIndex().get(path);
|
|
1908
|
+
if (!info) return null;
|
|
1909
|
+
if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
|
|
1910
|
+
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
1911
|
+
}
|
|
1912
|
+
let bytes;
|
|
1913
|
+
try {
|
|
1914
|
+
bytes = readFileSync2(this.resolveCatalogPath(path));
|
|
1915
|
+
} catch (error) {
|
|
1916
|
+
throw new CatalogOpenError(
|
|
1917
|
+
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1918
|
+
"corrupt"
|
|
1919
|
+
);
|
|
1920
|
+
}
|
|
1921
|
+
if (bytes.byteLength !== info.sizeBytes) {
|
|
1922
|
+
throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
|
|
1923
|
+
}
|
|
1924
|
+
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1489
1925
|
}
|
|
1490
1926
|
// ── search ────────────────────────────────────────────────────────────────
|
|
1491
1927
|
/** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
|
|
@@ -1542,7 +1978,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1542
1978
|
return this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
1543
1979
|
}
|
|
1544
1980
|
/**
|
|
1545
|
-
* Two-stage semantic search over the routed shards: bit
|
|
1981
|
+
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
1546
1982
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
1547
1983
|
*/
|
|
1548
1984
|
searchSemantic(queryVector, opts = {}) {
|
|
@@ -1554,14 +1990,15 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1554
1990
|
* after global cross-catalog routing has already spent the S budget.
|
|
1555
1991
|
*/
|
|
1556
1992
|
searchShards(queryVector, shardIds, finalK) {
|
|
1557
|
-
const
|
|
1993
|
+
const center = this.binaryCenter();
|
|
1994
|
+
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
1558
1995
|
const hits = [];
|
|
1559
1996
|
for (const shardId of shardIds) {
|
|
1560
1997
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1561
1998
|
if (!shard) continue;
|
|
1562
1999
|
const db = this.shardDb(shard);
|
|
1563
2000
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1564
|
-
const candidates =
|
|
2001
|
+
const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k).map((hit) => ({
|
|
1565
2002
|
chunk_id: hit.chunkId
|
|
1566
2003
|
}));
|
|
1567
2004
|
if (candidates.length === 0) continue;
|
|
@@ -1648,15 +2085,17 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1648
2085
|
}
|
|
1649
2086
|
}
|
|
1650
2087
|
};
|
|
1651
|
-
function
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
}
|
|
2088
|
+
function parseJsonObject(raw) {
|
|
2089
|
+
if (typeof raw !== "string") return null;
|
|
2090
|
+
try {
|
|
2091
|
+
const parsed = JSON.parse(raw);
|
|
2092
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
2093
|
+
} catch {
|
|
2094
|
+
return null;
|
|
1659
2095
|
}
|
|
2096
|
+
}
|
|
2097
|
+
function rowToDocumentMeta(row) {
|
|
2098
|
+
const ordinal = row.ordinal;
|
|
1660
2099
|
return {
|
|
1661
2100
|
id: row.id,
|
|
1662
2101
|
title: row.title,
|
|
@@ -1664,10 +2103,12 @@ function rowToDocumentMeta(row) {
|
|
|
1664
2103
|
summary: row.summary ?? null,
|
|
1665
2104
|
language: row.language,
|
|
1666
2105
|
topicId: row.topic_id,
|
|
2106
|
+
ordinal: typeof ordinal === "number" || typeof ordinal === "bigint" ? Number(ordinal) : null,
|
|
1667
2107
|
sourceUrl: row.source_url ?? null,
|
|
1668
2108
|
sourceRevision: row.source_revision ?? null,
|
|
1669
2109
|
sourceUpdatedAt: row.source_updated_at ?? null,
|
|
1670
|
-
attribution
|
|
2110
|
+
attribution: parseJsonObject(row.attribution_json),
|
|
2111
|
+
meta: parseJsonObject(row.meta_json)
|
|
1671
2112
|
};
|
|
1672
2113
|
}
|
|
1673
2114
|
function parseHeadingPath(raw) {
|
|
@@ -1683,7 +2124,17 @@ var ftsQuery = sanitizeFtsQuery;
|
|
|
1683
2124
|
// src/reader/validate.ts
|
|
1684
2125
|
import { readFile } from "fs/promises";
|
|
1685
2126
|
import { join as join3 } from "path";
|
|
1686
|
-
import {
|
|
2127
|
+
import {
|
|
2128
|
+
KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
|
|
2129
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2130
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
|
|
2131
|
+
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2132
|
+
assetExtension as assetExtension2,
|
|
2133
|
+
assetKindForExtension as assetKindForExtension2,
|
|
2134
|
+
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
2135
|
+
sniffAssetType as sniffAssetType2,
|
|
2136
|
+
svgInertnessProblem as svgInertnessProblem2
|
|
2137
|
+
} from "@bendyline/gezk";
|
|
1687
2138
|
async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
1688
2139
|
const checks = [];
|
|
1689
2140
|
const check = (name, ok, detail) => {
|
|
@@ -1750,8 +2201,43 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1750
2201
|
handle.meta.embedding_profile_id === manifest.embedding.id,
|
|
1751
2202
|
`router meta profile ${handle.meta.embedding_profile_id}`
|
|
1752
2203
|
);
|
|
2204
|
+
check(
|
|
2205
|
+
"meta-format",
|
|
2206
|
+
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2207
|
+
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2208
|
+
);
|
|
1753
2209
|
const topics = handle.topics();
|
|
1754
2210
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2211
|
+
check("topics-tree", ...topicTreeProblem(topics));
|
|
2212
|
+
const undeclared = handle.documentsWithUndeclaredTopic();
|
|
2213
|
+
check(
|
|
2214
|
+
"documents-topic-declared",
|
|
2215
|
+
undeclared === 0,
|
|
2216
|
+
`${undeclared} documents are filed under a topic the router does not declare`
|
|
2217
|
+
);
|
|
2218
|
+
const assetFiles = manifest.files.filter((f) => f.path.startsWith("assets/"));
|
|
2219
|
+
if (manifest.formatVersion === "0.5") {
|
|
2220
|
+
check(
|
|
2221
|
+
"assets-not-in-0.5",
|
|
2222
|
+
assetFiles.length === 0,
|
|
2223
|
+
`${assetFiles.length} assets/ entries in a 0.5 catalog`
|
|
2224
|
+
);
|
|
2225
|
+
} else {
|
|
2226
|
+
const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
|
|
2227
|
+
check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
|
|
2228
|
+
const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
|
|
2229
|
+
const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
|
|
2230
|
+
check(
|
|
2231
|
+
"assets-limits",
|
|
2232
|
+
assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2233
|
+
`${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
|
|
2234
|
+
);
|
|
2235
|
+
check(
|
|
2236
|
+
"counts-assets",
|
|
2237
|
+
(manifest.counts.assets ?? 0) === assetFiles.length,
|
|
2238
|
+
`manifest counts ${manifest.counts.assets ?? 0} assets, files declare ${assetFiles.length}`
|
|
2239
|
+
);
|
|
2240
|
+
}
|
|
1755
2241
|
check(
|
|
1756
2242
|
"license-notice",
|
|
1757
2243
|
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
@@ -1840,6 +2326,45 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1840
2326
|
}
|
|
1841
2327
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
1842
2328
|
}
|
|
2329
|
+
if (handle.schemaVersion >= 3) {
|
|
2330
|
+
const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
|
|
2331
|
+
check(
|
|
2332
|
+
"document-meta-json",
|
|
2333
|
+
meta.invalid.length === 0 && meta.oversize.length === 0,
|
|
2334
|
+
`${meta.invalid.length > 0 ? `not a JSON object: ${meta.invalid.join(", ")}` : ""}${meta.oversize.length > 0 ? ` over ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES} bytes: ${meta.oversize.join(", ")}` : ""}`.trim()
|
|
2335
|
+
);
|
|
2336
|
+
const declared = new Set(assetFiles.map((f) => f.path));
|
|
2337
|
+
for (const file of assetFiles) {
|
|
2338
|
+
const ext = assetExtension2(file.path);
|
|
2339
|
+
if (!ext) continue;
|
|
2340
|
+
const bytes = await readFile(join3(rootDir, file.path));
|
|
2341
|
+
const kind = sniffAssetType2(bytes);
|
|
2342
|
+
check(
|
|
2343
|
+
`asset-type:${file.path}`,
|
|
2344
|
+
kind === assetKindForExtension2(ext),
|
|
2345
|
+
`leading bytes say ${kind ?? "unknown"}, the extension says ${assetKindForExtension2(ext)}`
|
|
2346
|
+
);
|
|
2347
|
+
if (ext === "svg") {
|
|
2348
|
+
const problem = svgInertnessProblem2(bytes);
|
|
2349
|
+
check(`asset-svg-inert:${file.path}`, problem === null, problem ?? void 0);
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
if (declared.size > 0) {
|
|
2353
|
+
const missing = [];
|
|
2354
|
+
for (const doc of handle.documentBodies()) {
|
|
2355
|
+
for (const target of assetReferences(doc.markdown)) {
|
|
2356
|
+
if (!declared.has(target)) missing.push(`${doc.id} \u2192 ${target}`);
|
|
2357
|
+
if (missing.length >= 5) break;
|
|
2358
|
+
}
|
|
2359
|
+
if (missing.length >= 5) break;
|
|
2360
|
+
}
|
|
2361
|
+
check(
|
|
2362
|
+
"document-asset-refs",
|
|
2363
|
+
missing.length === 0,
|
|
2364
|
+
`undeclared asset references: ${missing.join("; ")}`
|
|
2365
|
+
);
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
1843
2368
|
for (const smoke of manifest.smokeQueries ?? []) {
|
|
1844
2369
|
const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
|
|
1845
2370
|
const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
|
|
@@ -1855,6 +2380,39 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1855
2380
|
}
|
|
1856
2381
|
return report(manifest);
|
|
1857
2382
|
}
|
|
2383
|
+
function assetReferences(markdown) {
|
|
2384
|
+
const found = /* @__PURE__ */ new Set();
|
|
2385
|
+
for (const match of markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
2386
|
+
const target = match[1];
|
|
2387
|
+
if (target) found.add(target);
|
|
2388
|
+
}
|
|
2389
|
+
return [...found];
|
|
2390
|
+
}
|
|
2391
|
+
function topicTreeProblem(topics) {
|
|
2392
|
+
const byId = new Map(topics.map((t) => [t.id, t]));
|
|
2393
|
+
for (const topic of topics) {
|
|
2394
|
+
if (topic.parentId !== null && !byId.has(topic.parentId)) {
|
|
2395
|
+
return [false, `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`];
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
const depthOf = /* @__PURE__ */ new Map();
|
|
2399
|
+
for (const topic of topics) {
|
|
2400
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2401
|
+
let depth = 0;
|
|
2402
|
+
let current = topic;
|
|
2403
|
+
while (current) {
|
|
2404
|
+
if (seen.has(current.id)) return [false, `topic '${topic.id}' sits in a parent cycle`];
|
|
2405
|
+
seen.add(current.id);
|
|
2406
|
+
depth += 1;
|
|
2407
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
2408
|
+
return [false, `topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`];
|
|
2409
|
+
}
|
|
2410
|
+
current = current.parentId === null ? void 0 : byId.get(current.parentId);
|
|
2411
|
+
}
|
|
2412
|
+
depthOf.set(topic.id, depth);
|
|
2413
|
+
}
|
|
2414
|
+
return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
|
|
2415
|
+
}
|
|
1858
2416
|
|
|
1859
2417
|
// src/export/duckdb.ts
|
|
1860
2418
|
import { spawn } from "child_process";
|
|
@@ -1892,7 +2450,7 @@ async function runDuckdbScript(cli, sql) {
|
|
|
1892
2450
|
return stdout;
|
|
1893
2451
|
}
|
|
1894
2452
|
function runDuckdbProcess(binaryPath, args, stdin) {
|
|
1895
|
-
return new Promise((
|
|
2453
|
+
return new Promise((resolve5, reject) => {
|
|
1896
2454
|
const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
1897
2455
|
let stdout = "";
|
|
1898
2456
|
let stderr = "";
|
|
@@ -1909,7 +2467,7 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
1909
2467
|
(err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
|
|
1910
2468
|
);
|
|
1911
2469
|
child.on("close", (code) => {
|
|
1912
|
-
if (code === 0)
|
|
2470
|
+
if (code === 0) resolve5({ stdout });
|
|
1913
2471
|
else
|
|
1914
2472
|
reject(
|
|
1915
2473
|
new DuckdbError(
|
|
@@ -1931,7 +2489,7 @@ import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/pr
|
|
|
1931
2489
|
import { join as join4, resolve as resolve3 } from "path";
|
|
1932
2490
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
1933
2491
|
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
1934
|
-
var PARQUET_EXPORT_VERSION =
|
|
2492
|
+
var PARQUET_EXPORT_VERSION = 2;
|
|
1935
2493
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
1936
2494
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
1937
2495
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2126,9 +2684,10 @@ async function stageDocuments(router, shardId, topicPaths, path) {
|
|
|
2126
2684
|
const aliasesOf = router.db.prepare(
|
|
2127
2685
|
"SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
|
|
2128
2686
|
);
|
|
2687
|
+
const extra = router.schemaVersion >= 3 ? ", ordinal, meta_json" : ", NULL AS ordinal, NULL AS meta_json";
|
|
2129
2688
|
const statement = router.db.prepare(
|
|
2130
2689
|
`SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
|
|
2131
|
-
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob
|
|
2690
|
+
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob${extra}
|
|
2132
2691
|
FROM documents WHERE shard_id = ? ORDER BY id`
|
|
2133
2692
|
);
|
|
2134
2693
|
for (const row of statement.iterate(shardId)) {
|
|
@@ -2150,6 +2709,8 @@ async function stageDocuments(router, shardId, topicPaths, path) {
|
|
|
2150
2709
|
source_revision: row.source_revision ?? null,
|
|
2151
2710
|
source_updated_at: row.source_updated_at ?? null,
|
|
2152
2711
|
attribution: row.attribution_json ?? null,
|
|
2712
|
+
ordinal: row.ordinal == null ? null : Number(row.ordinal),
|
|
2713
|
+
meta: row.meta_json ?? null,
|
|
2153
2714
|
aliases
|
|
2154
2715
|
});
|
|
2155
2716
|
rows++;
|
|
@@ -2273,9 +2834,11 @@ function documentsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
|
2273
2834
|
source_revision: "VARCHAR",
|
|
2274
2835
|
source_updated_at: "VARCHAR",
|
|
2275
2836
|
attribution: "VARCHAR",
|
|
2837
|
+
ordinal: "INTEGER",
|
|
2838
|
+
meta: "VARCHAR",
|
|
2276
2839
|
aliases: "VARCHAR[]"
|
|
2277
2840
|
});
|
|
2278
|
-
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2841
|
+
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, ordinal, meta, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2279
2842
|
`;
|
|
2280
2843
|
}
|
|
2281
2844
|
function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
|
|
@@ -2340,6 +2903,394 @@ async function findDuckdbBinary(env = process.env) {
|
|
|
2340
2903
|
return null;
|
|
2341
2904
|
}
|
|
2342
2905
|
|
|
2906
|
+
// src/profiles/e5-small-center.ts
|
|
2907
|
+
var MULTILINGUAL_E5_SMALL_CENTER = [
|
|
2908
|
+
0.046697,
|
|
2909
|
+
-0.013211,
|
|
2910
|
+
-0.042276,
|
|
2911
|
+
-0.049082,
|
|
2912
|
+
0.060627,
|
|
2913
|
+
-0.032643,
|
|
2914
|
+
0.029874,
|
|
2915
|
+
0.033307,
|
|
2916
|
+
0.058447,
|
|
2917
|
+
-0.012016,
|
|
2918
|
+
0.046472,
|
|
2919
|
+
5991e-6,
|
|
2920
|
+
0.04884,
|
|
2921
|
+
-0.028836,
|
|
2922
|
+
-0.03244,
|
|
2923
|
+
0.045633,
|
|
2924
|
+
0.056798,
|
|
2925
|
+
-0.043557,
|
|
2926
|
+
-0.034472,
|
|
2927
|
+
-0.05355,
|
|
2928
|
+
0.026363,
|
|
2929
|
+
-0.010318,
|
|
2930
|
+
-0.030487,
|
|
2931
|
+
0.047083,
|
|
2932
|
+
0.057205,
|
|
2933
|
+
0.035654,
|
|
2934
|
+
-0.017461,
|
|
2935
|
+
0.010881,
|
|
2936
|
+
0.047985,
|
|
2937
|
+
-0.048514,
|
|
2938
|
+
-0.048956,
|
|
2939
|
+
-0.027643,
|
|
2940
|
+
0.053259,
|
|
2941
|
+
-0.051057,
|
|
2942
|
+
0.045055,
|
|
2943
|
+
0.017888,
|
|
2944
|
+
-0.055755,
|
|
2945
|
+
-0.046578,
|
|
2946
|
+
0.05214,
|
|
2947
|
+
-0.037085,
|
|
2948
|
+
-0.016268,
|
|
2949
|
+
0.027508,
|
|
2950
|
+
0.035793,
|
|
2951
|
+
0.052082,
|
|
2952
|
+
0.034885,
|
|
2953
|
+
0.047174,
|
|
2954
|
+
-0.036778,
|
|
2955
|
+
0.031394,
|
|
2956
|
+
-9156e-6,
|
|
2957
|
+
-0.02486,
|
|
2958
|
+
-0.048897,
|
|
2959
|
+
0.052576,
|
|
2960
|
+
0.017289,
|
|
2961
|
+
0.047794,
|
|
2962
|
+
0.051939,
|
|
2963
|
+
-0.051261,
|
|
2964
|
+
-0.061345,
|
|
2965
|
+
-0.056267,
|
|
2966
|
+
-0.049305,
|
|
2967
|
+
0.047241,
|
|
2968
|
+
0.043763,
|
|
2969
|
+
-0.04506,
|
|
2970
|
+
0.015752,
|
|
2971
|
+
0.017543,
|
|
2972
|
+
0.067856,
|
|
2973
|
+
0.065763,
|
|
2974
|
+
0.037218,
|
|
2975
|
+
0.044877,
|
|
2976
|
+
-0.057149,
|
|
2977
|
+
-0.028072,
|
|
2978
|
+
-0.060566,
|
|
2979
|
+
0.064004,
|
|
2980
|
+
6095e-6,
|
|
2981
|
+
-0.039693,
|
|
2982
|
+
0.021968,
|
|
2983
|
+
0.02654,
|
|
2984
|
+
0.025546,
|
|
2985
|
+
-0.048954,
|
|
2986
|
+
0.026967,
|
|
2987
|
+
-0.045002,
|
|
2988
|
+
-0.055556,
|
|
2989
|
+
-0.034963,
|
|
2990
|
+
-0.03466,
|
|
2991
|
+
0.032486,
|
|
2992
|
+
-0.049597,
|
|
2993
|
+
0.058718,
|
|
2994
|
+
0.048021,
|
|
2995
|
+
-0.066783,
|
|
2996
|
+
0.059049,
|
|
2997
|
+
-0.04665,
|
|
2998
|
+
0.052902,
|
|
2999
|
+
0.031438,
|
|
3000
|
+
-0.061515,
|
|
3001
|
+
-0.05342,
|
|
3002
|
+
-0.068253,
|
|
3003
|
+
-0.06666,
|
|
3004
|
+
-0.04553,
|
|
3005
|
+
0.059252,
|
|
3006
|
+
0.050728,
|
|
3007
|
+
-0.030429,
|
|
3008
|
+
0.067182,
|
|
3009
|
+
-0.038819,
|
|
3010
|
+
0.049358,
|
|
3011
|
+
-0.055247,
|
|
3012
|
+
-0.032933,
|
|
3013
|
+
0.041239,
|
|
3014
|
+
0.035164,
|
|
3015
|
+
-0.066574,
|
|
3016
|
+
0.05348,
|
|
3017
|
+
-0.052658,
|
|
3018
|
+
-0.034758,
|
|
3019
|
+
0.037219,
|
|
3020
|
+
0.064111,
|
|
3021
|
+
0.047861,
|
|
3022
|
+
-0.04608,
|
|
3023
|
+
-0.041338,
|
|
3024
|
+
-0.016072,
|
|
3025
|
+
-0.046026,
|
|
3026
|
+
0.0255,
|
|
3027
|
+
-0.041581,
|
|
3028
|
+
0.059684,
|
|
3029
|
+
-0.019611,
|
|
3030
|
+
-0.050363,
|
|
3031
|
+
-0.061268,
|
|
3032
|
+
-0.057556,
|
|
3033
|
+
-0.03184,
|
|
3034
|
+
0.018522,
|
|
3035
|
+
0.046887,
|
|
3036
|
+
0.054774,
|
|
3037
|
+
0.012536,
|
|
3038
|
+
0.033019,
|
|
3039
|
+
0.033211,
|
|
3040
|
+
0.029609,
|
|
3041
|
+
0.050215,
|
|
3042
|
+
0.035425,
|
|
3043
|
+
0.076419,
|
|
3044
|
+
-0.055196,
|
|
3045
|
+
-0.010016,
|
|
3046
|
+
-936e-6,
|
|
3047
|
+
-0.023764,
|
|
3048
|
+
-0.059648,
|
|
3049
|
+
0.054857,
|
|
3050
|
+
-0.030456,
|
|
3051
|
+
0.041281,
|
|
3052
|
+
0.063894,
|
|
3053
|
+
0.035466,
|
|
3054
|
+
0.069754,
|
|
3055
|
+
-0.026593,
|
|
3056
|
+
0.066305,
|
|
3057
|
+
-0.028609,
|
|
3058
|
+
0.05324,
|
|
3059
|
+
-0.022879,
|
|
3060
|
+
0.04882,
|
|
3061
|
+
0.032385,
|
|
3062
|
+
0.042921,
|
|
3063
|
+
-0.043319,
|
|
3064
|
+
-0.065022,
|
|
3065
|
+
-0.044018,
|
|
3066
|
+
0.04237,
|
|
3067
|
+
0.051018,
|
|
3068
|
+
-0.048907,
|
|
3069
|
+
-0.053045,
|
|
3070
|
+
-0.057301,
|
|
3071
|
+
-0.019731,
|
|
3072
|
+
-0.060812,
|
|
3073
|
+
-0.043114,
|
|
3074
|
+
0.021207,
|
|
3075
|
+
0.052498,
|
|
3076
|
+
-0.038307,
|
|
3077
|
+
-0.037082,
|
|
3078
|
+
-0.043732,
|
|
3079
|
+
0.045959,
|
|
3080
|
+
-0.044296,
|
|
3081
|
+
0.057199,
|
|
3082
|
+
-0.011835,
|
|
3083
|
+
0.054221,
|
|
3084
|
+
-0.046354,
|
|
3085
|
+
0.043442,
|
|
3086
|
+
0.063337,
|
|
3087
|
+
0.041964,
|
|
3088
|
+
-0.032836,
|
|
3089
|
+
-0.020089,
|
|
3090
|
+
-0.056327,
|
|
3091
|
+
-0.05089,
|
|
3092
|
+
-0.053933,
|
|
3093
|
+
-0.035486,
|
|
3094
|
+
-0.046215,
|
|
3095
|
+
0.025754,
|
|
3096
|
+
0.032986,
|
|
3097
|
+
-0.025048,
|
|
3098
|
+
-9903e-6,
|
|
3099
|
+
0.031162,
|
|
3100
|
+
-0.050969,
|
|
3101
|
+
-0.049401,
|
|
3102
|
+
-0.046206,
|
|
3103
|
+
0.023197,
|
|
3104
|
+
-0.033441,
|
|
3105
|
+
0.045075,
|
|
3106
|
+
0.043318,
|
|
3107
|
+
0.043766,
|
|
3108
|
+
-6601e-6,
|
|
3109
|
+
-0.023077,
|
|
3110
|
+
0.033917,
|
|
3111
|
+
0.032279,
|
|
3112
|
+
0.034874,
|
|
3113
|
+
5185e-6,
|
|
3114
|
+
-0.057329,
|
|
3115
|
+
0.048437,
|
|
3116
|
+
-0.033964,
|
|
3117
|
+
0.044195,
|
|
3118
|
+
0.051107,
|
|
3119
|
+
-0.056642,
|
|
3120
|
+
-0.070856,
|
|
3121
|
+
0.045431,
|
|
3122
|
+
-0.047205,
|
|
3123
|
+
-0.021457,
|
|
3124
|
+
3103e-6,
|
|
3125
|
+
0.062536,
|
|
3126
|
+
-0.037804,
|
|
3127
|
+
0.031278,
|
|
3128
|
+
0.041619,
|
|
3129
|
+
-0.015547,
|
|
3130
|
+
0.052298,
|
|
3131
|
+
-0.078327,
|
|
3132
|
+
-0.052303,
|
|
3133
|
+
0.043099,
|
|
3134
|
+
0.05375,
|
|
3135
|
+
-0.041854,
|
|
3136
|
+
-0.038314,
|
|
3137
|
+
0.039399,
|
|
3138
|
+
-0.047769,
|
|
3139
|
+
-0.038469,
|
|
3140
|
+
-0.056793,
|
|
3141
|
+
-0.064312,
|
|
3142
|
+
-0.047884,
|
|
3143
|
+
-0.060067,
|
|
3144
|
+
-0.030687,
|
|
3145
|
+
0.030053,
|
|
3146
|
+
0.033569,
|
|
3147
|
+
-0.038014,
|
|
3148
|
+
-0.032396,
|
|
3149
|
+
-0.041395,
|
|
3150
|
+
0.038163,
|
|
3151
|
+
-0.056206,
|
|
3152
|
+
0.047115,
|
|
3153
|
+
-0.038099,
|
|
3154
|
+
-0.031927,
|
|
3155
|
+
0.033857,
|
|
3156
|
+
-0.011227,
|
|
3157
|
+
0.032883,
|
|
3158
|
+
0.032516,
|
|
3159
|
+
-0.045622,
|
|
3160
|
+
-0.044387,
|
|
3161
|
+
-0.034374,
|
|
3162
|
+
-0.030433,
|
|
3163
|
+
0.055097,
|
|
3164
|
+
0.055568,
|
|
3165
|
+
0.058726,
|
|
3166
|
+
-0.043569,
|
|
3167
|
+
0.031854,
|
|
3168
|
+
775e-5,
|
|
3169
|
+
-0.022533,
|
|
3170
|
+
0.096133,
|
|
3171
|
+
0.052876,
|
|
3172
|
+
0.045582,
|
|
3173
|
+
0.041868,
|
|
3174
|
+
-0.039268,
|
|
3175
|
+
-0.037865,
|
|
3176
|
+
-0.054658,
|
|
3177
|
+
-0.031297,
|
|
3178
|
+
-0.058104,
|
|
3179
|
+
0.018705,
|
|
3180
|
+
0.031934,
|
|
3181
|
+
-0.052746,
|
|
3182
|
+
-0.044438,
|
|
3183
|
+
-0.029991,
|
|
3184
|
+
0.011917,
|
|
3185
|
+
0.070103,
|
|
3186
|
+
-0.046682,
|
|
3187
|
+
-0.048965,
|
|
3188
|
+
0.046255,
|
|
3189
|
+
0.027877,
|
|
3190
|
+
0.0475,
|
|
3191
|
+
0.044287,
|
|
3192
|
+
0.04251,
|
|
3193
|
+
-0.017647,
|
|
3194
|
+
-0.017812,
|
|
3195
|
+
0.055672,
|
|
3196
|
+
-0.042207,
|
|
3197
|
+
-0.035516,
|
|
3198
|
+
-0.025558,
|
|
3199
|
+
-0.062443,
|
|
3200
|
+
0.037519,
|
|
3201
|
+
-0.031697,
|
|
3202
|
+
0.064998,
|
|
3203
|
+
0.045949,
|
|
3204
|
+
302e-6,
|
|
3205
|
+
0.065205,
|
|
3206
|
+
-0.052016,
|
|
3207
|
+
0.036721,
|
|
3208
|
+
-0.034769,
|
|
3209
|
+
-0.049074,
|
|
3210
|
+
0.040413,
|
|
3211
|
+
0.045425,
|
|
3212
|
+
-0.068063,
|
|
3213
|
+
0.021485,
|
|
3214
|
+
8549e-6,
|
|
3215
|
+
0.028179,
|
|
3216
|
+
0.026914,
|
|
3217
|
+
0.020493,
|
|
3218
|
+
0.056155,
|
|
3219
|
+
0.047202,
|
|
3220
|
+
-0.049085,
|
|
3221
|
+
-0.022624,
|
|
3222
|
+
0.063734,
|
|
3223
|
+
0.05471,
|
|
3224
|
+
-0.032258,
|
|
3225
|
+
0.047761,
|
|
3226
|
+
-0.065033,
|
|
3227
|
+
-0.047818,
|
|
3228
|
+
-0.034254,
|
|
3229
|
+
-0.062863,
|
|
3230
|
+
-0.01362,
|
|
3231
|
+
-0.053904,
|
|
3232
|
+
0.024415,
|
|
3233
|
+
0.044367,
|
|
3234
|
+
-0.064056,
|
|
3235
|
+
-0.038419,
|
|
3236
|
+
0.038997,
|
|
3237
|
+
1015e-6,
|
|
3238
|
+
0.038928,
|
|
3239
|
+
-0.045794,
|
|
3240
|
+
-0.038313,
|
|
3241
|
+
0.043922,
|
|
3242
|
+
-0.05873,
|
|
3243
|
+
-0.020314,
|
|
3244
|
+
-0.020457,
|
|
3245
|
+
0.045522,
|
|
3246
|
+
-0.057731,
|
|
3247
|
+
-0.062966,
|
|
3248
|
+
0.023159,
|
|
3249
|
+
0.020283,
|
|
3250
|
+
-0.057352,
|
|
3251
|
+
0.044701,
|
|
3252
|
+
-0.03585,
|
|
3253
|
+
-0.052798,
|
|
3254
|
+
0.056931,
|
|
3255
|
+
-0.035609,
|
|
3256
|
+
-0.035113,
|
|
3257
|
+
0.026919,
|
|
3258
|
+
0.059404,
|
|
3259
|
+
-0.084434,
|
|
3260
|
+
0.014558,
|
|
3261
|
+
0.06226,
|
|
3262
|
+
-0.044173,
|
|
3263
|
+
0.044172,
|
|
3264
|
+
-0.046482,
|
|
3265
|
+
-0.034933,
|
|
3266
|
+
0.052513,
|
|
3267
|
+
0.044018,
|
|
3268
|
+
-0.057677,
|
|
3269
|
+
-0.033961,
|
|
3270
|
+
0.046334,
|
|
3271
|
+
0.065767,
|
|
3272
|
+
0.044779,
|
|
3273
|
+
0.063658,
|
|
3274
|
+
-0.022574,
|
|
3275
|
+
-4821e-6,
|
|
3276
|
+
0.037388,
|
|
3277
|
+
-0.043197,
|
|
3278
|
+
0.063,
|
|
3279
|
+
0.048943,
|
|
3280
|
+
-0.049495,
|
|
3281
|
+
0.014798,
|
|
3282
|
+
-0.018176,
|
|
3283
|
+
-0.03824,
|
|
3284
|
+
-0.045486,
|
|
3285
|
+
0.053828,
|
|
3286
|
+
-0.056118,
|
|
3287
|
+
-0.067025,
|
|
3288
|
+
0.027818,
|
|
3289
|
+
0.050305,
|
|
3290
|
+
0.037134,
|
|
3291
|
+
0.049673
|
|
3292
|
+
];
|
|
3293
|
+
|
|
2343
3294
|
// src/profiles/registry.ts
|
|
2344
3295
|
var SHARED_ENCODING = {
|
|
2345
3296
|
pooling: "mean",
|
|
@@ -2370,6 +3321,19 @@ var MULTILINGUAL_E5_SMALL_1 = {
|
|
|
2370
3321
|
passageInstruction: "passage: ",
|
|
2371
3322
|
...SHARED_ENCODING
|
|
2372
3323
|
};
|
|
3324
|
+
var MULTILINGUAL_E5_SMALL_2 = {
|
|
3325
|
+
...MULTILINGUAL_E5_SMALL_1,
|
|
3326
|
+
id: "multilingual-e5-small@2",
|
|
3327
|
+
quantization: {
|
|
3328
|
+
int8: { method: "symmetric-linear", scale: 127 },
|
|
3329
|
+
binary: {
|
|
3330
|
+
method: "centered-sign",
|
|
3331
|
+
threshold: 0,
|
|
3332
|
+
packing: "lsb-first",
|
|
3333
|
+
center: [...MULTILINGUAL_E5_SMALL_CENTER]
|
|
3334
|
+
}
|
|
3335
|
+
}
|
|
3336
|
+
};
|
|
2373
3337
|
var BGE_SMALL_EN_V15_1 = {
|
|
2374
3338
|
id: "bge-small-en-v1.5@1",
|
|
2375
3339
|
model: {
|
|
@@ -2388,6 +3352,7 @@ var BGE_SMALL_EN_V15_1 = {
|
|
|
2388
3352
|
...SHARED_ENCODING
|
|
2389
3353
|
};
|
|
2390
3354
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
3355
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
2391
3356
|
MULTILINGUAL_E5_SMALL_1,
|
|
2392
3357
|
BGE_SMALL_EN_V15_1
|
|
2393
3358
|
];
|
|
@@ -2453,9 +3418,9 @@ async function fileDigest(path) {
|
|
|
2453
3418
|
const key = `${path} ${size} ${mtimeMs}`;
|
|
2454
3419
|
const memo = digestMemo.get(key);
|
|
2455
3420
|
if (memo) return memo;
|
|
2456
|
-
const digest = await new Promise((
|
|
3421
|
+
const digest = await new Promise((resolve5, reject) => {
|
|
2457
3422
|
const hash = createHash4("sha256");
|
|
2458
|
-
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () =>
|
|
3423
|
+
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
2459
3424
|
});
|
|
2460
3425
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
2461
3426
|
const oldest = digestMemo.keys().next().value;
|
|
@@ -2613,29 +3578,427 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
2613
3578
|
}
|
|
2614
3579
|
|
|
2615
3580
|
// src/markdown-adapter/load.ts
|
|
3581
|
+
import { existsSync } from "fs";
|
|
2616
3582
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
2617
|
-
import { join as join6, relative as relative3, sep } from "path";
|
|
2618
|
-
import {
|
|
3583
|
+
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3584
|
+
import {
|
|
3585
|
+
CatalogDocumentSchema,
|
|
3586
|
+
KnowledgeDocumentIdSchema,
|
|
3587
|
+
KnowledgeOrdinalSchema,
|
|
3588
|
+
assetExtension as assetExtension3,
|
|
3589
|
+
formatKnowledgeUri,
|
|
3590
|
+
topicSortKeyForOrder
|
|
3591
|
+
} from "@bendyline/gezk";
|
|
3592
|
+
|
|
3593
|
+
// src/markdown-adapter/frontmatter.ts
|
|
3594
|
+
import { parseDocument } from "yaml";
|
|
3595
|
+
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
3596
|
+
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
3597
|
+
function parseYaml(text, opts) {
|
|
3598
|
+
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
3599
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
3600
|
+
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
3601
|
+
}
|
|
3602
|
+
const doc = parseDocument(text, {
|
|
3603
|
+
version: "1.2",
|
|
3604
|
+
schema: "core",
|
|
3605
|
+
uniqueKeys: true,
|
|
3606
|
+
logLevel: "silent"
|
|
3607
|
+
});
|
|
3608
|
+
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
3609
|
+
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
3610
|
+
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
3611
|
+
}
|
|
3612
|
+
var OPEN = /^---[ \t]*\r?\n/;
|
|
3613
|
+
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
3614
|
+
function parseMarkdownFrontMatter(source) {
|
|
3615
|
+
const open = OPEN.exec(source);
|
|
3616
|
+
if (!open) return { data: {}, body: source };
|
|
3617
|
+
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
3618
|
+
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
3619
|
+
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
3620
|
+
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
3621
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
3622
|
+
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
3623
|
+
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
3624
|
+
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3625
|
+
throw new Error("front matter must be a mapping");
|
|
3626
|
+
}
|
|
3627
|
+
return { data: parsed, body };
|
|
3628
|
+
}
|
|
3629
|
+
|
|
3630
|
+
// src/markdown-adapter/outline.ts
|
|
3631
|
+
import { posix } from "path";
|
|
3632
|
+
var HAS_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
|
|
3633
|
+
var MARKDOWN_EXT = /\.(md|markdown)$/i;
|
|
3634
|
+
function newOutlineTopic(name) {
|
|
3635
|
+
return { name, children: [], entries: [] };
|
|
3636
|
+
}
|
|
3637
|
+
function isRecord(value) {
|
|
3638
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3639
|
+
}
|
|
3640
|
+
function isExternal(target) {
|
|
3641
|
+
return HAS_SCHEME.test(target) || target.startsWith("//");
|
|
3642
|
+
}
|
|
3643
|
+
function safeDecode(value) {
|
|
3644
|
+
try {
|
|
3645
|
+
return decodeURIComponent(value);
|
|
3646
|
+
} catch {
|
|
3647
|
+
return value;
|
|
3648
|
+
}
|
|
3649
|
+
}
|
|
3650
|
+
function stem(file) {
|
|
3651
|
+
return posix.basename(file).replace(MARKDOWN_EXT, "");
|
|
3652
|
+
}
|
|
3653
|
+
function resolveOutlinePath(baseDir, target) {
|
|
3654
|
+
const cleaned = safeDecode(
|
|
3655
|
+
target.replace(/\\/g, "/").replace(/[#?].*$/, "").trim()
|
|
3656
|
+
);
|
|
3657
|
+
if (cleaned === "" || posix.isAbsolute(cleaned)) return null;
|
|
3658
|
+
const joined = posix.normalize(posix.join(baseDir, cleaned));
|
|
3659
|
+
if (joined === ".." || joined.startsWith("../")) return null;
|
|
3660
|
+
return joined === "." ? "" : joined;
|
|
3661
|
+
}
|
|
3662
|
+
function assignOutlineOrders(topic) {
|
|
3663
|
+
topic.entries.forEach((entry, i) => {
|
|
3664
|
+
if (entry.order === void 0) entry.order = i + 1;
|
|
3665
|
+
});
|
|
3666
|
+
topic.children.forEach((child, i) => {
|
|
3667
|
+
if (child.order === void 0) child.order = i + 1;
|
|
3668
|
+
assignOutlineOrders(child);
|
|
3669
|
+
});
|
|
3670
|
+
}
|
|
3671
|
+
function titleFromFolderName(name) {
|
|
3672
|
+
const words = name.replace(/[_-]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
3673
|
+
if (words.length === 0) return name;
|
|
3674
|
+
return words.map((word) => /[A-Z]/.test(word) ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
3675
|
+
}
|
|
3676
|
+
var HEADING2 = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
|
|
3677
|
+
var LIST_ITEM = /^([ \t]*)(?:[-*+]|\d+[.)])\s+(.+?)\s*$/;
|
|
3678
|
+
var ITEM_LINK = /^\[([^\]]*)\]\(\s*<?([^)>\s]*)>?(?:\s+"[^"]*")?\s*\)$/;
|
|
3679
|
+
function materializeSummaryTopic(slot) {
|
|
3680
|
+
if (slot.topic) return slot.topic;
|
|
3681
|
+
const topic = newOutlineTopic(slot.title || "Untitled");
|
|
3682
|
+
if (slot.entry) {
|
|
3683
|
+
const at = slot.container.entries.indexOf(slot.entry);
|
|
3684
|
+
if (at >= 0) slot.container.entries.splice(at, 1);
|
|
3685
|
+
topic.entries.push(slot.entry);
|
|
3686
|
+
}
|
|
3687
|
+
slot.container.children.push(topic);
|
|
3688
|
+
slot.topic = topic;
|
|
3689
|
+
return topic;
|
|
3690
|
+
}
|
|
3691
|
+
function parseGitbookSummary(text, summaryFile) {
|
|
3692
|
+
const dir = posix.dirname(summaryFile);
|
|
3693
|
+
const baseDir = dir === "." ? "" : dir;
|
|
3694
|
+
const outline = {
|
|
3695
|
+
format: "gitbook",
|
|
3696
|
+
root: newOutlineTopic(""),
|
|
3697
|
+
consumed: [summaryFile],
|
|
3698
|
+
warnings: []
|
|
3699
|
+
};
|
|
3700
|
+
let container = outline.root;
|
|
3701
|
+
const stack = [];
|
|
3702
|
+
let fence = null;
|
|
3703
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
3704
|
+
const line = rawLine.replace(/\t/g, " ");
|
|
3705
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
3706
|
+
if (fenceMatch) {
|
|
3707
|
+
const marker = fenceMatch[1][0];
|
|
3708
|
+
if (fence === null) fence = marker;
|
|
3709
|
+
else if (marker === fence) fence = null;
|
|
3710
|
+
continue;
|
|
3711
|
+
}
|
|
3712
|
+
if (fence !== null) continue;
|
|
3713
|
+
const heading = HEADING2.exec(line);
|
|
3714
|
+
if (heading) {
|
|
3715
|
+
if (heading[1].length === 1) continue;
|
|
3716
|
+
const part = newOutlineTopic(heading[2].replace(/[*_`]/g, "").trim() || "Part");
|
|
3717
|
+
outline.root.children.push(part);
|
|
3718
|
+
container = part;
|
|
3719
|
+
stack.length = 0;
|
|
3720
|
+
continue;
|
|
3721
|
+
}
|
|
3722
|
+
const item = LIST_ITEM.exec(line);
|
|
3723
|
+
if (!item) continue;
|
|
3724
|
+
const indent = item[1].length;
|
|
3725
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
|
|
3726
|
+
stack.pop();
|
|
3727
|
+
}
|
|
3728
|
+
const parent = stack[stack.length - 1];
|
|
3729
|
+
const parentTopic = parent ? materializeSummaryTopic(parent) : container;
|
|
3730
|
+
const content = item[2].trim();
|
|
3731
|
+
const link = ITEM_LINK.exec(content);
|
|
3732
|
+
const title = (link ? link[1] : content).replace(/[*_`]/g, "").trim();
|
|
3733
|
+
const slot = { indent, title, container: parentTopic };
|
|
3734
|
+
const target = link?.[2] ?? "";
|
|
3735
|
+
if (link && target !== "" && !isExternal(target)) {
|
|
3736
|
+
const file = resolveOutlinePath(baseDir, target);
|
|
3737
|
+
if (file === null) {
|
|
3738
|
+
outline.warnings.push(`${summaryFile}: '${target}' leaves the catalog tree; skipped`);
|
|
3739
|
+
} else if (!MARKDOWN_EXT.test(file)) {
|
|
3740
|
+
outline.warnings.push(`${summaryFile}: '${target}' is not a Markdown page; skipped`);
|
|
3741
|
+
} else {
|
|
3742
|
+
slot.entry = { file, ...title ? { title } : {} };
|
|
3743
|
+
parentTopic.entries.push(slot.entry);
|
|
3744
|
+
}
|
|
3745
|
+
}
|
|
3746
|
+
stack.push(slot);
|
|
3747
|
+
}
|
|
3748
|
+
assignOutlineOrders(outline.root);
|
|
3749
|
+
return outline;
|
|
3750
|
+
}
|
|
3751
|
+
function parseMkdocsNav(config, docsDir, configFile = "mkdocs.yml") {
|
|
3752
|
+
if (!isRecord(config)) throw new Error(`${configFile}: not a mapping`);
|
|
3753
|
+
const nav = config.nav;
|
|
3754
|
+
if (nav === void 0 || nav === null) return null;
|
|
3755
|
+
if (!Array.isArray(nav)) throw new Error(`${configFile}: 'nav' must be a list`);
|
|
3756
|
+
const outline = {
|
|
3757
|
+
format: "mkdocs",
|
|
3758
|
+
root: newOutlineTopic(""),
|
|
3759
|
+
consumed: [],
|
|
3760
|
+
warnings: []
|
|
3761
|
+
};
|
|
3762
|
+
const add = (container, target, title) => {
|
|
3763
|
+
if (isExternal(target)) return;
|
|
3764
|
+
const file = resolveOutlinePath(docsDir, target);
|
|
3765
|
+
if (file === null) {
|
|
3766
|
+
outline.warnings.push(
|
|
3767
|
+
`${configFile}: nav entry '${target}' leaves the docs directory; skipped`
|
|
3768
|
+
);
|
|
3769
|
+
return;
|
|
3770
|
+
}
|
|
3771
|
+
if (!MARKDOWN_EXT.test(file)) {
|
|
3772
|
+
outline.warnings.push(`${configFile}: nav entry '${target}' is not a Markdown page; skipped`);
|
|
3773
|
+
return;
|
|
3774
|
+
}
|
|
3775
|
+
container.entries.push({ file, ...title ? { title } : {} });
|
|
3776
|
+
};
|
|
3777
|
+
const walk = (items, container) => {
|
|
3778
|
+
for (const item of items) {
|
|
3779
|
+
if (typeof item === "string") {
|
|
3780
|
+
add(container, item);
|
|
3781
|
+
continue;
|
|
3782
|
+
}
|
|
3783
|
+
if (!isRecord(item)) {
|
|
3784
|
+
outline.warnings.push(`${configFile}: unrecognized nav entry ${JSON.stringify(item)}`);
|
|
3785
|
+
continue;
|
|
3786
|
+
}
|
|
3787
|
+
for (const [title, value] of Object.entries(item)) {
|
|
3788
|
+
if (typeof value === "string") {
|
|
3789
|
+
add(container, value, title.trim());
|
|
3790
|
+
} else if (Array.isArray(value)) {
|
|
3791
|
+
const section = newOutlineTopic(title.trim() || "Section");
|
|
3792
|
+
walk(value, section);
|
|
3793
|
+
container.children.push(section);
|
|
3794
|
+
} else {
|
|
3795
|
+
outline.warnings.push(
|
|
3796
|
+
`${configFile}: nav entry '${title}' is neither a page nor a section`
|
|
3797
|
+
);
|
|
3798
|
+
}
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
};
|
|
3802
|
+
walk(nav, outline.root);
|
|
3803
|
+
assignOutlineOrders(outline.root);
|
|
3804
|
+
return outline;
|
|
3805
|
+
}
|
|
3806
|
+
function globToRegExp(pattern) {
|
|
3807
|
+
let source = "";
|
|
3808
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
3809
|
+
const ch = pattern[i];
|
|
3810
|
+
if (ch === "*") {
|
|
3811
|
+
if (pattern[i + 1] === "*") {
|
|
3812
|
+
source += ".*";
|
|
3813
|
+
i++;
|
|
3814
|
+
} else source += "[^/]*";
|
|
3815
|
+
} else if (ch === "?") source += "[^/]";
|
|
3816
|
+
else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
3817
|
+
}
|
|
3818
|
+
return new RegExp(`^${source}$`);
|
|
3819
|
+
}
|
|
3820
|
+
function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
|
|
3821
|
+
const outline = {
|
|
3822
|
+
format: "jupyter-book",
|
|
3823
|
+
root: newOutlineTopic(""),
|
|
3824
|
+
consumed: [],
|
|
3825
|
+
warnings: []
|
|
3826
|
+
};
|
|
3827
|
+
const known = new Set(files.map((f) => f.normalize("NFC")));
|
|
3828
|
+
const resolveFile = (spec) => {
|
|
3829
|
+
const candidates = MARKDOWN_EXT.test(spec) ? [spec] : [`${spec}.md`, `${spec}.markdown`];
|
|
3830
|
+
for (const candidate of candidates) {
|
|
3831
|
+
const path = resolveOutlinePath(tocDir, candidate);
|
|
3832
|
+
if (path !== null && known.has(path.normalize("NFC"))) return path;
|
|
3833
|
+
}
|
|
3834
|
+
return null;
|
|
3835
|
+
};
|
|
3836
|
+
const expandGlob = (pattern) => {
|
|
3837
|
+
const regex = globToRegExp(pattern.replace(/\\/g, "/"));
|
|
3838
|
+
const prefix = tocDir ? `${tocDir}/` : "";
|
|
3839
|
+
return files.filter((file) => {
|
|
3840
|
+
if (!file.startsWith(prefix)) return false;
|
|
3841
|
+
const rel = file.slice(prefix.length);
|
|
3842
|
+
return regex.test(rel) || regex.test(rel.replace(MARKDOWN_EXT, ""));
|
|
3843
|
+
}).sort();
|
|
3844
|
+
};
|
|
3845
|
+
const walkItems = (items, container, where) => {
|
|
3846
|
+
if (items === void 0 || items === null) return;
|
|
3847
|
+
if (!Array.isArray(items)) {
|
|
3848
|
+
outline.warnings.push(`${tocFile}: ${where} must be a list`);
|
|
3849
|
+
return;
|
|
3850
|
+
}
|
|
3851
|
+
for (const item of items) {
|
|
3852
|
+
if (!isRecord(item)) {
|
|
3853
|
+
outline.warnings.push(`${tocFile}: ${where} holds an unrecognized entry`);
|
|
3854
|
+
continue;
|
|
3855
|
+
}
|
|
3856
|
+
const title = typeof item.title === "string" ? item.title.trim() : void 0;
|
|
3857
|
+
if (typeof item.url === "string") continue;
|
|
3858
|
+
if (typeof item.glob === "string") {
|
|
3859
|
+
for (const file2 of expandGlob(item.glob)) container.entries.push({ file: file2 });
|
|
3860
|
+
continue;
|
|
3861
|
+
}
|
|
3862
|
+
if (typeof item.file !== "string") {
|
|
3863
|
+
outline.warnings.push(`${tocFile}: ${where} holds an entry without 'file'`);
|
|
3864
|
+
continue;
|
|
3865
|
+
}
|
|
3866
|
+
const file = resolveFile(item.file);
|
|
3867
|
+
if (file === null) {
|
|
3868
|
+
outline.warnings.push(
|
|
3869
|
+
`${tocFile}: '${item.file}' is not a Markdown file in the tree (notebooks are not supported); skipped`
|
|
3870
|
+
);
|
|
3871
|
+
continue;
|
|
3872
|
+
}
|
|
3873
|
+
const entry = { file, ...title ? { title } : {} };
|
|
3874
|
+
const children = item.sections ?? item.chapters ?? item.entries;
|
|
3875
|
+
if (Array.isArray(children) && children.length > 0) {
|
|
3876
|
+
const topic = newOutlineTopic(title ?? titleFromFolderName(stem(file)));
|
|
3877
|
+
if (!title) topic.titleFromFile = file;
|
|
3878
|
+
topic.entries.push(entry);
|
|
3879
|
+
walkItems(children, topic, `'${item.file}'`);
|
|
3880
|
+
container.children.push(topic);
|
|
3881
|
+
} else {
|
|
3882
|
+
container.entries.push(entry);
|
|
3883
|
+
}
|
|
3884
|
+
}
|
|
3885
|
+
};
|
|
3886
|
+
if (Array.isArray(doc)) {
|
|
3887
|
+
for (const item of doc) {
|
|
3888
|
+
if (isRecord(item) && typeof item.part === "string") {
|
|
3889
|
+
const part = newOutlineTopic(item.part.trim() || "Part");
|
|
3890
|
+
walkItems(item.chapters ?? item.sections, part, `part '${item.part}'`);
|
|
3891
|
+
outline.root.children.push(part);
|
|
3892
|
+
} else {
|
|
3893
|
+
walkItems([item], outline.root, "the list");
|
|
3894
|
+
}
|
|
3895
|
+
}
|
|
3896
|
+
} else if (isRecord(doc)) {
|
|
3897
|
+
if (typeof doc.root === "string") {
|
|
3898
|
+
const file = resolveFile(doc.root);
|
|
3899
|
+
if (file === null) {
|
|
3900
|
+
outline.warnings.push(
|
|
3901
|
+
`${tocFile}: root '${doc.root}' is not a Markdown file in the tree; skipped`
|
|
3902
|
+
);
|
|
3903
|
+
} else {
|
|
3904
|
+
outline.root.entries.push({ file });
|
|
3905
|
+
}
|
|
3906
|
+
}
|
|
3907
|
+
if (Array.isArray(doc.parts)) {
|
|
3908
|
+
for (const part of doc.parts) {
|
|
3909
|
+
if (!isRecord(part)) continue;
|
|
3910
|
+
const caption = typeof part.caption === "string" ? part.caption.trim() : "";
|
|
3911
|
+
const topic = newOutlineTopic(caption || "Part");
|
|
3912
|
+
walkItems(part.chapters ?? part.sections, topic, `part '${topic.name}'`);
|
|
3913
|
+
outline.root.children.push(topic);
|
|
3914
|
+
}
|
|
3915
|
+
}
|
|
3916
|
+
walkItems(doc.chapters, outline.root, "chapters");
|
|
3917
|
+
walkItems(doc.sections, outline.root, "sections");
|
|
3918
|
+
} else {
|
|
3919
|
+
throw new Error(`${tocFile}: not a mapping or a list`);
|
|
3920
|
+
}
|
|
3921
|
+
assignOutlineOrders(outline.root);
|
|
3922
|
+
return outline;
|
|
3923
|
+
}
|
|
3924
|
+
|
|
3925
|
+
// src/markdown-adapter/load.ts
|
|
2619
3926
|
var ROOT_TOPIC_ID = "general";
|
|
2620
3927
|
var SUMMARY_MAX_CHARS = 280;
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
3928
|
+
var TOPIC_SIDECAR = "_topic.yaml";
|
|
3929
|
+
var HUGO_SECTION_PAGE = "_index";
|
|
3930
|
+
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
3931
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
|
|
3932
|
+
function readFrontMatter(raw, file, hugo) {
|
|
3933
|
+
let data;
|
|
3934
|
+
let body;
|
|
3935
|
+
try {
|
|
3936
|
+
({ data, body } = parseMarkdownFrontMatter(raw));
|
|
3937
|
+
} catch (error) {
|
|
3938
|
+
throw new Error(`${file}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3939
|
+
}
|
|
3940
|
+
const out = { meta: {}, body };
|
|
3941
|
+
const str = (key) => {
|
|
3942
|
+
const value = data[key];
|
|
3943
|
+
if (value === void 0 || value === null) return void 0;
|
|
3944
|
+
if (typeof value === "number") return String(value);
|
|
3945
|
+
if (typeof value !== "string")
|
|
3946
|
+
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
3947
|
+
return value.trim() || void 0;
|
|
3948
|
+
};
|
|
3949
|
+
out.title = str("title");
|
|
3950
|
+
out.summary = str("summary");
|
|
3951
|
+
out.id = str("id");
|
|
3952
|
+
const aliases = data.aliases;
|
|
3953
|
+
if (typeof aliases === "string") {
|
|
3954
|
+
out.aliases = aliases.split(",").map((a) => a.trim()).filter(Boolean);
|
|
3955
|
+
} else if (Array.isArray(aliases)) {
|
|
3956
|
+
out.aliases = aliases.map((a) => String(a).trim()).filter(Boolean);
|
|
3957
|
+
} else if (aliases !== void 0 && aliases !== null) {
|
|
3958
|
+
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
3959
|
+
}
|
|
3960
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
3961
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
3962
|
+
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
3963
|
+
out.order = order.data;
|
|
3964
|
+
}
|
|
3965
|
+
const sub = data.subcategory;
|
|
3966
|
+
if (sub !== void 0 && sub !== null) {
|
|
3967
|
+
if (typeof sub !== "object" || Array.isArray(sub)) {
|
|
3968
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
3969
|
+
}
|
|
3970
|
+
const record = sub;
|
|
3971
|
+
if (typeof record.id !== "string" || !record.id.trim()) {
|
|
3972
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
2638
3973
|
}
|
|
3974
|
+
const subcategory = { id: record.id.trim() };
|
|
3975
|
+
if (record.title !== void 0 && record.title !== null) {
|
|
3976
|
+
if (typeof record.title !== "string") {
|
|
3977
|
+
throw new Error(`${file}: front matter 'subcategory.title' must be a string`);
|
|
3978
|
+
}
|
|
3979
|
+
subcategory.title = record.title.trim();
|
|
3980
|
+
}
|
|
3981
|
+
if (record.order !== void 0 && record.order !== null) {
|
|
3982
|
+
const order = KnowledgeOrdinalSchema.safeParse(record.order);
|
|
3983
|
+
if (!order.success)
|
|
3984
|
+
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
3985
|
+
subcategory.order = order.data;
|
|
3986
|
+
}
|
|
3987
|
+
out.subcategory = subcategory;
|
|
3988
|
+
}
|
|
3989
|
+
if (hugo) {
|
|
3990
|
+
if (data.draft === true) out.unpublished = "draft";
|
|
3991
|
+
else if (data.headless === true) out.unpublished = "headless";
|
|
3992
|
+
if (data.weight !== void 0 && data.weight !== null) {
|
|
3993
|
+
const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
|
|
3994
|
+
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
3995
|
+
if (weight.data > 0) out.weight = weight.data;
|
|
3996
|
+
}
|
|
3997
|
+
out.description = str("description");
|
|
3998
|
+
}
|
|
3999
|
+
for (const [key, value] of Object.entries(data)) {
|
|
4000
|
+
if (RESERVED_KEYS.has(key) || value === void 0) continue;
|
|
4001
|
+
out.meta[key] = value;
|
|
2639
4002
|
}
|
|
2640
4003
|
return out;
|
|
2641
4004
|
}
|
|
@@ -2677,17 +4040,233 @@ async function walkMarkdownFiles(root) {
|
|
|
2677
4040
|
await visit(root);
|
|
2678
4041
|
return out;
|
|
2679
4042
|
}
|
|
2680
|
-
|
|
4043
|
+
function isRecord2(value) {
|
|
4044
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4045
|
+
}
|
|
4046
|
+
async function readTopicSidecar(dir) {
|
|
4047
|
+
const path = join6(dir, TOPIC_SIDECAR);
|
|
4048
|
+
if (!existsSync(path)) return void 0;
|
|
4049
|
+
const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
4050
|
+
const data = parseYaml(raw, { what: path });
|
|
4051
|
+
if (data === null || data === void 0) return {};
|
|
4052
|
+
if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
|
|
4053
|
+
const out = {};
|
|
4054
|
+
if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
|
|
4055
|
+
if (typeof data.description === "string" && data.description.trim()) {
|
|
4056
|
+
out.description = data.description.trim();
|
|
4057
|
+
}
|
|
4058
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
4059
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
4060
|
+
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
4061
|
+
out.order = order.data;
|
|
4062
|
+
}
|
|
4063
|
+
return out;
|
|
4064
|
+
}
|
|
4065
|
+
function applyOverride(topic, override) {
|
|
4066
|
+
if (!topic || !override) return;
|
|
4067
|
+
if (override.name) topic.name = override.name;
|
|
4068
|
+
if (override.description) topic.description = override.description;
|
|
4069
|
+
if (override.order !== void 0) topic.sortKey = topicSortKeyForOrder(override.order);
|
|
4070
|
+
}
|
|
4071
|
+
function insideRoot(rootDir, abs) {
|
|
4072
|
+
const rel = relative3(rootDir, abs).split(sep).join("/");
|
|
4073
|
+
if (rel === ".." || rel.startsWith("../") || isAbsolute3(rel)) return null;
|
|
4074
|
+
return rel;
|
|
4075
|
+
}
|
|
4076
|
+
function firstExisting(candidates) {
|
|
4077
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
4078
|
+
}
|
|
4079
|
+
async function readMkdocsDocsDir(mkdocsPath) {
|
|
4080
|
+
const config = parseYaml((await readFile3(mkdocsPath, "utf8")).replace(/^\uFEFF/, ""), {
|
|
4081
|
+
what: basename(mkdocsPath),
|
|
4082
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
4083
|
+
tolerateUnknownTags: true,
|
|
4084
|
+
maxAliasCount: 100
|
|
4085
|
+
});
|
|
4086
|
+
const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
4087
|
+
return resolve4(dirname3(mkdocsPath), docsDir);
|
|
4088
|
+
}
|
|
4089
|
+
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
4090
|
+
const summary = join6(rootDir, "SUMMARY.md");
|
|
4091
|
+
if (existsSync(summary)) return { format: "gitbook", path: summary };
|
|
4092
|
+
const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
4093
|
+
if (jupyter) return { format: "jupyter-book", path: jupyter };
|
|
4094
|
+
const mkdocs = firstExisting(
|
|
4095
|
+
[rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
|
|
4096
|
+
join6(dir, "mkdocs.yml"),
|
|
4097
|
+
join6(dir, "mkdocs.yaml")
|
|
4098
|
+
])
|
|
4099
|
+
);
|
|
4100
|
+
if (mkdocs) return { format: "mkdocs", path: mkdocs };
|
|
2681
4101
|
const files = await walkMarkdownFiles(rootDir);
|
|
4102
|
+
if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
|
|
4103
|
+
return { format: "hugo" };
|
|
4104
|
+
}
|
|
4105
|
+
return { format: "folders" };
|
|
4106
|
+
}
|
|
4107
|
+
async function readOutlineText(path) {
|
|
4108
|
+
const text = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
4109
|
+
if (Buffer.byteLength(text, "utf8") > OUTLINE_MAX_BYTES) {
|
|
4110
|
+
throw new Error(`${basename(path)} exceeds ${OUTLINE_MAX_BYTES} bytes`);
|
|
4111
|
+
}
|
|
4112
|
+
return text;
|
|
4113
|
+
}
|
|
4114
|
+
async function loadOutline(rootDir, toc, files) {
|
|
4115
|
+
if (toc.format === "folders" || toc.format === "hugo") return { outline: null };
|
|
4116
|
+
const explicit = toc.path ? isAbsolute3(toc.path) ? toc.path : join6(rootDir, toc.path) : void 0;
|
|
4117
|
+
if (explicit && !existsSync(explicit)) {
|
|
4118
|
+
throw new Error(`${toc.format} table of contents not found: ${explicit}`);
|
|
4119
|
+
}
|
|
4120
|
+
if (toc.format === "gitbook") {
|
|
4121
|
+
const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
|
|
4122
|
+
const rel = insideRoot(rootDir, path2);
|
|
4123
|
+
if (!existsSync(path2)) throw new Error(`gitbook table of contents not found: ${path2}`);
|
|
4124
|
+
if (rel === null)
|
|
4125
|
+
throw new Error(`${basename(path2)} must live inside the content root ${rootDir}`);
|
|
4126
|
+
return { outline: parseGitbookSummary(await readOutlineText(path2), rel), path: path2 };
|
|
4127
|
+
}
|
|
4128
|
+
if (toc.format === "mkdocs") {
|
|
4129
|
+
const path2 = explicit ?? firstExisting(
|
|
4130
|
+
[rootDir, dirname3(rootDir)].flatMap((dir) => [
|
|
4131
|
+
join6(dir, "mkdocs.yml"),
|
|
4132
|
+
join6(dir, "mkdocs.yaml")
|
|
4133
|
+
])
|
|
4134
|
+
);
|
|
4135
|
+
if (!path2) throw new Error(`mkdocs.yml not found in ${rootDir} or its parent`);
|
|
4136
|
+
const config = parseYaml(await readOutlineText(path2), {
|
|
4137
|
+
what: basename(path2),
|
|
4138
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
4139
|
+
tolerateUnknownTags: true,
|
|
4140
|
+
maxAliasCount: 100
|
|
4141
|
+
});
|
|
4142
|
+
const docsDir = await readMkdocsDocsDir(path2);
|
|
4143
|
+
const docsRel = insideRoot(rootDir, docsDir);
|
|
4144
|
+
if (docsRel === null) {
|
|
4145
|
+
throw new Error(
|
|
4146
|
+
`${basename(path2)}: docs_dir ${docsDir} is outside the content root ${rootDir}`
|
|
4147
|
+
);
|
|
4148
|
+
}
|
|
4149
|
+
return { outline: parseMkdocsNav(config, docsRel, basename(path2)), path: path2 };
|
|
4150
|
+
}
|
|
4151
|
+
const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
4152
|
+
if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
|
|
4153
|
+
const tocDir = insideRoot(rootDir, dirname3(path));
|
|
4154
|
+
if (tocDir === null)
|
|
4155
|
+
throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
|
|
4156
|
+
const doc = parseYaml(await readOutlineText(path), {
|
|
4157
|
+
what: basename(path),
|
|
4158
|
+
maxBytes: OUTLINE_MAX_BYTES
|
|
4159
|
+
});
|
|
4160
|
+
return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
|
|
4161
|
+
}
|
|
4162
|
+
var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
|
|
4163
|
+
var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
|
|
4164
|
+
function resolveInTree(ctx, target) {
|
|
4165
|
+
const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
|
|
4166
|
+
if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
|
|
4167
|
+
return resolved === "." ? "" : resolved;
|
|
4168
|
+
}
|
|
4169
|
+
function rewriteTarget(ctx, isImage, rawTarget) {
|
|
4170
|
+
const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
|
|
4171
|
+
if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
|
|
4172
|
+
return null;
|
|
4173
|
+
}
|
|
4174
|
+
const hashAt = target.search(/[#?]/);
|
|
4175
|
+
const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
|
|
4176
|
+
const decoded = safeDecode2(pathPart);
|
|
4177
|
+
const resolved = resolveInTree(ctx, decoded);
|
|
4178
|
+
if (resolved === null) {
|
|
4179
|
+
ctx.warn(`${ctx.file}: link '${target}' leaves the catalog tree; left as is`);
|
|
4180
|
+
return null;
|
|
4181
|
+
}
|
|
4182
|
+
if (isImage) {
|
|
4183
|
+
if (!assetExtension3(resolved)) {
|
|
4184
|
+
ctx.warn(`${ctx.file}: image '${target}' is not a supported asset type; left as is`);
|
|
4185
|
+
return null;
|
|
4186
|
+
}
|
|
4187
|
+
const abs = join6(ctx.rootDir, ...resolved.split("/"));
|
|
4188
|
+
if (!existsSync(abs)) {
|
|
4189
|
+
const message = `${ctx.file}: image '${target}' does not exist in the catalog tree`;
|
|
4190
|
+
if (ctx.missingAssets === "error") throw new Error(message);
|
|
4191
|
+
ctx.warn(`${message}; left as is`);
|
|
4192
|
+
return null;
|
|
4193
|
+
}
|
|
4194
|
+
const archivePath = resolved.startsWith("assets/") ? resolved : `assets/${resolved}`;
|
|
4195
|
+
if (!ctx.assets.has(archivePath))
|
|
4196
|
+
ctx.assets.set(archivePath, { path: archivePath, absPath: abs });
|
|
4197
|
+
return archivePath;
|
|
4198
|
+
}
|
|
4199
|
+
if (!/\.(md|markdown)$/i.test(resolved)) return null;
|
|
4200
|
+
const documentId = ctx.idByRel.get(resolved.replace(/\.(md|markdown)$/i, "").normalize("NFC"));
|
|
4201
|
+
if (!documentId) {
|
|
4202
|
+
ctx.warn(`${ctx.file}: link '${target}' does not name a document in the catalog; left as is`);
|
|
4203
|
+
return null;
|
|
4204
|
+
}
|
|
4205
|
+
if (!ctx.uri) return null;
|
|
4206
|
+
return formatKnowledgeUri({
|
|
4207
|
+
publisherId: ctx.uri.publisherId,
|
|
4208
|
+
catalogId: ctx.uri.catalogId,
|
|
4209
|
+
documentId
|
|
4210
|
+
});
|
|
4211
|
+
}
|
|
4212
|
+
function safeDecode2(value) {
|
|
4213
|
+
try {
|
|
4214
|
+
return decodeURIComponent(value);
|
|
4215
|
+
} catch {
|
|
4216
|
+
return value;
|
|
4217
|
+
}
|
|
4218
|
+
}
|
|
4219
|
+
function rewriteLinks(ctx, markdown) {
|
|
4220
|
+
const lines = markdown.split("\n");
|
|
4221
|
+
let fence = null;
|
|
4222
|
+
return lines.map((line) => {
|
|
4223
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
4224
|
+
if (fenceMatch) {
|
|
4225
|
+
const marker = fenceMatch[1];
|
|
4226
|
+
if (fence === null) fence = marker[0];
|
|
4227
|
+
else if (marker[0] === fence) fence = null;
|
|
4228
|
+
return line;
|
|
4229
|
+
}
|
|
4230
|
+
if (fence !== null) return line;
|
|
4231
|
+
return line.replace(
|
|
4232
|
+
INLINE_LINK,
|
|
4233
|
+
(whole, bang, text, target, title) => {
|
|
4234
|
+
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
4235
|
+
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
4236
|
+
}
|
|
4237
|
+
);
|
|
4238
|
+
}).join("\n");
|
|
4239
|
+
}
|
|
4240
|
+
async function loadMarkdownCatalog(rootDir, opts) {
|
|
4241
|
+
const warn = opts.onWarning ?? (() => {
|
|
4242
|
+
});
|
|
4243
|
+
const toc = opts.toc ?? { format: "folders" };
|
|
4244
|
+
const hugo = toc.format === "hugo";
|
|
4245
|
+
const ignore = new Set(opts.ignore ?? []);
|
|
4246
|
+
const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
|
|
4247
|
+
let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
|
|
4248
|
+
const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
|
|
4249
|
+
for (const message of outline?.warnings ?? []) warn(message);
|
|
4250
|
+
if (outline) {
|
|
4251
|
+
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
4252
|
+
files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
|
|
4253
|
+
}
|
|
2682
4254
|
if (files.length === 0) {
|
|
2683
4255
|
throw new Error(`no Markdown files found under ${rootDir}`);
|
|
2684
4256
|
}
|
|
2685
4257
|
const takenTopicIds = /* @__PURE__ */ new Set();
|
|
2686
4258
|
const topicIdByDir = /* @__PURE__ */ new Map();
|
|
2687
4259
|
const topics = [];
|
|
4260
|
+
const topicById = /* @__PURE__ */ new Map();
|
|
2688
4261
|
const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
|
|
2689
4262
|
let rootTopicUsed = false;
|
|
2690
|
-
|
|
4263
|
+
let rootOverride;
|
|
4264
|
+
const addTopic = (topic) => {
|
|
4265
|
+
topics.push(topic);
|
|
4266
|
+
topicById.set(topic.id, topic);
|
|
4267
|
+
return topic;
|
|
4268
|
+
};
|
|
4269
|
+
const ensureTopicChain = async (relDirPosix) => {
|
|
2691
4270
|
if (!relDirPosix) {
|
|
2692
4271
|
rootTopicUsed = true;
|
|
2693
4272
|
return [rootTopicId];
|
|
@@ -2701,32 +4280,141 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2701
4280
|
if (!id) {
|
|
2702
4281
|
id = topicIdFor(segment, takenTopicIds);
|
|
2703
4282
|
topicIdByDir.set(prefix, id);
|
|
2704
|
-
|
|
4283
|
+
const topic = addTopic({
|
|
2705
4284
|
id,
|
|
2706
|
-
name: segment,
|
|
4285
|
+
name: titleFromFolderName(segment),
|
|
2707
4286
|
...path.length > 0 ? { parentId: path[path.length - 1] } : {}
|
|
2708
4287
|
});
|
|
4288
|
+
applyOverride(topic, await readTopicSidecar(join6(rootDir, ...prefix.split("/"))));
|
|
4289
|
+
applyOverride(topic, opts.topics?.[prefix]);
|
|
2709
4290
|
}
|
|
2710
4291
|
path.push(id);
|
|
2711
4292
|
}
|
|
2712
4293
|
return path;
|
|
2713
4294
|
};
|
|
2714
|
-
const
|
|
2715
|
-
|
|
4295
|
+
const placements = /* @__PURE__ */ new Map();
|
|
4296
|
+
if (outline) {
|
|
4297
|
+
const walk = (topic, chain) => {
|
|
4298
|
+
for (const entry of topic.entries) {
|
|
4299
|
+
const key = entry.file.normalize("NFC");
|
|
4300
|
+
if (placements.has(key)) {
|
|
4301
|
+
warn(
|
|
4302
|
+
`${entry.file}: listed more than once in the table of contents; the first place wins`
|
|
4303
|
+
);
|
|
4304
|
+
} else {
|
|
4305
|
+
placements.set(key, { chain, entry });
|
|
4306
|
+
}
|
|
4307
|
+
}
|
|
4308
|
+
for (const child of topic.children) walk(child, [...chain, child]);
|
|
4309
|
+
};
|
|
4310
|
+
walk(outline.root, []);
|
|
4311
|
+
}
|
|
4312
|
+
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
4313
|
+
const ensureOutlineChain = (chain) => {
|
|
4314
|
+
if (chain.length === 0) {
|
|
4315
|
+
rootTopicUsed = true;
|
|
4316
|
+
return [rootTopicId];
|
|
4317
|
+
}
|
|
4318
|
+
const ids = [];
|
|
4319
|
+
for (const node of chain) {
|
|
4320
|
+
let id = outlineTopicIds.get(node);
|
|
4321
|
+
if (!id) {
|
|
4322
|
+
id = topicIdFor(node.name, takenTopicIds);
|
|
4323
|
+
outlineTopicIds.set(node, id);
|
|
4324
|
+
const topic = {
|
|
4325
|
+
id,
|
|
4326
|
+
name: node.name,
|
|
4327
|
+
...ids.length > 0 ? { parentId: ids[ids.length - 1] } : {}
|
|
4328
|
+
};
|
|
4329
|
+
if (node.description) topic.description = node.description;
|
|
4330
|
+
if (node.order !== void 0) topic.sortKey = topicSortKeyForOrder(node.order);
|
|
4331
|
+
addTopic(topic);
|
|
4332
|
+
}
|
|
4333
|
+
ids.push(id);
|
|
4334
|
+
}
|
|
4335
|
+
return ids;
|
|
4336
|
+
};
|
|
4337
|
+
const shelves = /* @__PURE__ */ new Map();
|
|
4338
|
+
const ensureShelf = (parentId, sub, file) => {
|
|
4339
|
+
const key = `${parentId}/${sub.id}`;
|
|
4340
|
+
const existing = shelves.get(key);
|
|
4341
|
+
if (existing) {
|
|
4342
|
+
if (existing.title !== sub.title || existing.order !== sub.order) {
|
|
4343
|
+
throw new Error(
|
|
4344
|
+
`${file}: subcategory '${sub.id}' is declared with a different title or order in ${existing.file}`
|
|
4345
|
+
);
|
|
4346
|
+
}
|
|
4347
|
+
return existing.topicId;
|
|
4348
|
+
}
|
|
4349
|
+
const topicId = topicIdFor(`${parentId}-${sub.id}`, takenTopicIds);
|
|
4350
|
+
const topic = { id: topicId, name: sub.title ?? sub.id, parentId };
|
|
4351
|
+
if (sub.order !== void 0) topic.sortKey = topicSortKeyForOrder(sub.order);
|
|
4352
|
+
addTopic(topic);
|
|
4353
|
+
shelves.set(key, { topicId, title: sub.title, order: sub.order, file });
|
|
4354
|
+
return topicId;
|
|
4355
|
+
};
|
|
4356
|
+
const loaded = [];
|
|
4357
|
+
const fileById = /* @__PURE__ */ new Map();
|
|
4358
|
+
const idByRel = /* @__PURE__ */ new Map();
|
|
4359
|
+
const seenRel = /* @__PURE__ */ new Set();
|
|
2716
4360
|
for (const abs of files) {
|
|
2717
|
-
const rel =
|
|
4361
|
+
const rel = relOf(abs);
|
|
2718
4362
|
const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
2719
|
-
const
|
|
2720
|
-
const id = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
2721
|
-
if (seenIds.has(id)) throw new Error(`duplicate document id from path: ${rel}`);
|
|
2722
|
-
seenIds.add(id);
|
|
4363
|
+
const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
2723
4364
|
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
2724
|
-
const fm =
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
4365
|
+
const fm = readFrontMatter(raw, rel, hugo);
|
|
4366
|
+
if (fm.unpublished) {
|
|
4367
|
+
warn(`${rel}: skipped, ${fm.unpublished} page`);
|
|
4368
|
+
continue;
|
|
4369
|
+
}
|
|
4370
|
+
seenRel.add(rel.normalize("NFC"));
|
|
4371
|
+
const id = fm.id ?? relNoExt;
|
|
4372
|
+
const idCheck = KnowledgeDocumentIdSchema.safeParse(id);
|
|
4373
|
+
if (!idCheck.success) throw new Error(`${rel}: front matter 'id' is not a valid document id`);
|
|
4374
|
+
const previous = fileById.get(id);
|
|
4375
|
+
if (previous) throw new Error(`${rel}: document id '${id}' is already used by ${previous}`);
|
|
4376
|
+
fileById.set(id, rel);
|
|
4377
|
+
idByRel.set(relNoExt, id);
|
|
4378
|
+
const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
4379
|
+
let title = fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
4380
|
+
let ordinal = fm.order;
|
|
4381
|
+
let topicPath;
|
|
4382
|
+
const placement = placements.get(rel.normalize("NFC"));
|
|
4383
|
+
if (placement) {
|
|
4384
|
+
topicPath = ensureOutlineChain(placement.chain);
|
|
4385
|
+
if (placement.entry.title) title = placement.entry.title;
|
|
4386
|
+
if (placement.entry.order !== void 0) ordinal = placement.entry.order;
|
|
4387
|
+
} else {
|
|
4388
|
+
if (outline)
|
|
4389
|
+
warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
|
|
4390
|
+
topicPath = await ensureTopicChain(relDir);
|
|
4391
|
+
if (hugo && ordinal === void 0) ordinal = fm.weight;
|
|
4392
|
+
}
|
|
4393
|
+
if (hugo && stem2 === HUGO_SECTION_PAGE) {
|
|
4394
|
+
const override = {
|
|
4395
|
+
...fm.title ? { name: fm.title } : {},
|
|
4396
|
+
...fm.description ?? fm.summary ? { description: fm.description ?? fm.summary } : {},
|
|
4397
|
+
...fm.weight !== void 0 ? { order: fm.weight } : {}
|
|
4398
|
+
};
|
|
4399
|
+
if (relDir === "") {
|
|
4400
|
+
rootOverride = override;
|
|
4401
|
+
} else {
|
|
4402
|
+
const leaf = topicById.get(topicPath[topicPath.length - 1]);
|
|
4403
|
+
applyOverride(leaf, override);
|
|
4404
|
+
applyOverride(leaf, opts.topics?.[relDir]);
|
|
4405
|
+
}
|
|
4406
|
+
if (!fm.body.trim()) continue;
|
|
4407
|
+
ordinal = SECTION_PAGE_ORDINAL;
|
|
4408
|
+
}
|
|
4409
|
+
if (fm.subcategory) {
|
|
4410
|
+
const parentId = topicPath[topicPath.length - 1];
|
|
4411
|
+
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
4412
|
+
}
|
|
4413
|
+
const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
|
|
4414
|
+
loaded.push({
|
|
4415
|
+
rel,
|
|
4416
|
+
file: rel,
|
|
4417
|
+
doc: {
|
|
2730
4418
|
id,
|
|
2731
4419
|
title,
|
|
2732
4420
|
slug: documentSlug(title),
|
|
@@ -2734,9 +4422,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2734
4422
|
language: opts.language,
|
|
2735
4423
|
topicPath,
|
|
2736
4424
|
markdown: fm.body,
|
|
2737
|
-
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
|
|
2738
|
-
|
|
4425
|
+
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
4426
|
+
...ordinal !== void 0 ? { ordinal } : {},
|
|
4427
|
+
...Object.keys(fm.meta).length > 0 ? { meta: fm.meta } : {}
|
|
4428
|
+
}
|
|
4429
|
+
});
|
|
4430
|
+
}
|
|
4431
|
+
if (outline) {
|
|
4432
|
+
const titleByRel = new Map(
|
|
4433
|
+
loaded.map((entry) => [entry.rel.normalize("NFC"), entry.doc.title])
|
|
2739
4434
|
);
|
|
4435
|
+
const nameFromPages = (topic) => {
|
|
4436
|
+
for (const child of topic.children) {
|
|
4437
|
+
const id = outlineTopicIds.get(child);
|
|
4438
|
+
const title = child.titleFromFile ? titleByRel.get(child.titleFromFile.normalize("NFC")) : void 0;
|
|
4439
|
+
const compiled = id ? topicById.get(id) : void 0;
|
|
4440
|
+
if (compiled && title) compiled.name = title;
|
|
4441
|
+
nameFromPages(child);
|
|
4442
|
+
}
|
|
4443
|
+
};
|
|
4444
|
+
nameFromPages(outline.root);
|
|
4445
|
+
for (const [file, placement] of placements) {
|
|
4446
|
+
if (!seenRel.has(file)) {
|
|
4447
|
+
warn(
|
|
4448
|
+
`${placement.entry.file}: named by the table of contents but not found among the Markdown files`
|
|
4449
|
+
);
|
|
4450
|
+
}
|
|
4451
|
+
}
|
|
2740
4452
|
}
|
|
2741
4453
|
if (rootTopicUsed) {
|
|
2742
4454
|
if (takenTopicIds.has(rootTopicId)) {
|
|
@@ -2744,9 +4456,39 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2744
4456
|
`root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
|
|
2745
4457
|
);
|
|
2746
4458
|
}
|
|
2747
|
-
|
|
4459
|
+
const root = { id: rootTopicId, name: opts.rootTopicName ?? "General" };
|
|
4460
|
+
applyOverride(root, await readTopicSidecar(rootDir));
|
|
4461
|
+
applyOverride(root, rootOverride);
|
|
4462
|
+
applyOverride(root, opts.topics?.[""]);
|
|
4463
|
+
topics.unshift(root);
|
|
4464
|
+
topicById.set(root.id, root);
|
|
2748
4465
|
}
|
|
2749
|
-
|
|
4466
|
+
const assets = /* @__PURE__ */ new Map();
|
|
4467
|
+
const documents = [];
|
|
4468
|
+
for (const entry of loaded) {
|
|
4469
|
+
const ctx = {
|
|
4470
|
+
rootDir,
|
|
4471
|
+
docDir: entry.rel.includes("/") ? entry.rel.slice(0, entry.rel.lastIndexOf("/")) : "",
|
|
4472
|
+
file: entry.file,
|
|
4473
|
+
idByRel,
|
|
4474
|
+
uri: opts.uri,
|
|
4475
|
+
assets,
|
|
4476
|
+
missingAssets: opts.missingAssets ?? "error",
|
|
4477
|
+
warn
|
|
4478
|
+
};
|
|
4479
|
+
documents.push(
|
|
4480
|
+
CatalogDocumentSchema.parse({
|
|
4481
|
+
...entry.doc,
|
|
4482
|
+
markdown: rewriteLinks(ctx, entry.doc.markdown)
|
|
4483
|
+
})
|
|
4484
|
+
);
|
|
4485
|
+
}
|
|
4486
|
+
return {
|
|
4487
|
+
topics,
|
|
4488
|
+
documents,
|
|
4489
|
+
assets: [...assets.values()].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
4490
|
+
toc: { format: toc.format, ...outlinePath ? { path: outlinePath } : {} }
|
|
4491
|
+
};
|
|
2750
4492
|
}
|
|
2751
4493
|
|
|
2752
4494
|
// src/registry-client/fetch.ts
|
|
@@ -2861,7 +4603,7 @@ function newerRegistryEntries(registry, installed) {
|
|
|
2861
4603
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
2862
4604
|
|
|
2863
4605
|
// src/signatures/jcs.ts
|
|
2864
|
-
import { canonicalizeJson } from "@bendyline/gezk";
|
|
4606
|
+
import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
|
|
2865
4607
|
export {
|
|
2866
4608
|
BGE_SMALL_EN_V15_1,
|
|
2867
4609
|
BODY_CODEC_MIN_BYTES,
|
|
@@ -2880,9 +4622,12 @@ export {
|
|
|
2880
4622
|
GEZEL_MULTILINGUAL_E5_SMALL_1,
|
|
2881
4623
|
GEZK_APPLICATION_ID,
|
|
2882
4624
|
GEZK_ARCHIVE_LIMITS,
|
|
4625
|
+
GEZK_FORMAT_GENERATIONS,
|
|
2883
4626
|
GEZK_FORMAT_VERSION,
|
|
2884
4627
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
2885
4628
|
GEZK_MIME_TYPE,
|
|
4629
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
4630
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
2886
4631
|
GezkArchiveError,
|
|
2887
4632
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
2888
4633
|
KNOWLEDGE_TOOLCHAIN,
|
|
@@ -2891,11 +4636,16 @@ export {
|
|
|
2891
4636
|
MANIFEST_PATH,
|
|
2892
4637
|
MARKDOWN_CHUNKS_2,
|
|
2893
4638
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
4639
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
4640
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
2894
4641
|
MIMETYPE_PATH,
|
|
2895
4642
|
MULTILINGUAL_E5_SMALL_1,
|
|
4643
|
+
MULTILINGUAL_E5_SMALL_2,
|
|
2896
4644
|
PARQUET_EXPORT_VERSION,
|
|
2897
4645
|
PARQUET_REPORT_PATH,
|
|
2898
4646
|
README_PATH,
|
|
4647
|
+
RERANK_CANDIDATES_MAX,
|
|
4648
|
+
RERANK_CANDIDATES_MIN,
|
|
2899
4649
|
RERANK_FINAL_K,
|
|
2900
4650
|
ROUTER_DB_PATH,
|
|
2901
4651
|
ROUTER_DDL,
|
|
@@ -2907,13 +4657,18 @@ export {
|
|
|
2907
4657
|
SOURCE_NOTICES_PATH,
|
|
2908
4658
|
ZIP_FIXED_MTIME,
|
|
2909
4659
|
assertDuckdbCli,
|
|
2910
|
-
|
|
4660
|
+
assetReferences,
|
|
4661
|
+
assignOutlineOrders,
|
|
4662
|
+
asymmetricTopK,
|
|
4663
|
+
canonicalizeJson2 as canonicalizeJson,
|
|
4664
|
+
centerVector,
|
|
2911
4665
|
chunkContentHash,
|
|
2912
4666
|
chunkMarkdownProfile,
|
|
2913
4667
|
chunkUid,
|
|
2914
4668
|
compareCatalogVersions,
|
|
2915
4669
|
compileKnowledgeCatalog,
|
|
2916
4670
|
createProfileEmbedder,
|
|
4671
|
+
detectTableOfContents,
|
|
2917
4672
|
documentSlug,
|
|
2918
4673
|
duckdbVersion,
|
|
2919
4674
|
exportCatalogParquet,
|
|
@@ -2924,22 +4679,32 @@ export {
|
|
|
2924
4679
|
generateKnowledgeSigningKeyPair,
|
|
2925
4680
|
hammingTopK,
|
|
2926
4681
|
inspectGezkArchive,
|
|
4682
|
+
isSupportedFormatVersion,
|
|
4683
|
+
isSupportedIndexSchemaVersion,
|
|
2927
4684
|
knowledgeEmbeddingProfile,
|
|
2928
4685
|
knowledgeKeyId,
|
|
2929
4686
|
l2Normalize,
|
|
2930
4687
|
loadMarkdownCatalog,
|
|
2931
4688
|
manifestSigningPayload,
|
|
4689
|
+
newOutlineTopic,
|
|
2932
4690
|
newerRegistryEntries,
|
|
2933
4691
|
openCatalogDatabase,
|
|
4692
|
+
parseGitbookSummary,
|
|
4693
|
+
parseJupyterBookToc,
|
|
4694
|
+
parseMkdocsNav,
|
|
2934
4695
|
quantizeBinary,
|
|
4696
|
+
quantizeBinaryForProfile,
|
|
2935
4697
|
quantizeInt8,
|
|
2936
4698
|
readGezkManifest,
|
|
4699
|
+
readMkdocsDocsDir,
|
|
2937
4700
|
rerankK,
|
|
2938
4701
|
rerankScore,
|
|
4702
|
+
resolveOutlinePath,
|
|
2939
4703
|
resolveTransformersModelOptions,
|
|
2940
4704
|
runDuckdbScript,
|
|
2941
4705
|
signManifest,
|
|
2942
4706
|
signRegistryIndex,
|
|
4707
|
+
titleFromFolderName,
|
|
2943
4708
|
transformersCachePath,
|
|
2944
4709
|
validateExtractedCatalog,
|
|
2945
4710
|
verifyManifestSignature,
|