@bendyline/gezel-knowledge 1.2.1 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +175 -49
- package/dist/index.js +1462 -201
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -180,12 +180,11 @@ function splitOversizeLine(line, opts) {
|
|
|
180
180
|
}
|
|
181
181
|
|
|
182
182
|
// src/compiler/compile.ts
|
|
183
|
-
import { createHash } from "crypto";
|
|
183
|
+
import { createHash as createHash2 } from "crypto";
|
|
184
184
|
import {
|
|
185
185
|
closeSync,
|
|
186
186
|
mkdirSync,
|
|
187
187
|
openSync,
|
|
188
|
-
readFileSync,
|
|
189
188
|
readSync,
|
|
190
189
|
rmSync,
|
|
191
190
|
statSync,
|
|
@@ -196,16 +195,13 @@ import { join } from "path";
|
|
|
196
195
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
197
196
|
import {
|
|
198
197
|
GEZK_MANIFEST_KIND,
|
|
199
|
-
KnowledgeAssetPathSchema,
|
|
200
198
|
KnowledgeCatalogManifestSchema,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
assetExtension,
|
|
205
|
-
assetKindForExtension,
|
|
199
|
+
KnowledgeLocationsSchema,
|
|
200
|
+
KnowledgeOrdinalSchema,
|
|
201
|
+
KnowledgeTocReferenceSchema,
|
|
206
202
|
canonicalizeJson,
|
|
207
|
-
|
|
208
|
-
|
|
203
|
+
normalizeLongitude,
|
|
204
|
+
spatialManifest
|
|
209
205
|
} from "@bendyline/gezk";
|
|
210
206
|
|
|
211
207
|
// src/archive/write.ts
|
|
@@ -251,28 +247,306 @@ var nodeRequire2 = createRequire2(import.meta.url);
|
|
|
251
247
|
var sqlite = nodeRequire2("node:sqlite");
|
|
252
248
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
253
249
|
|
|
250
|
+
// src/query-stopwords.ts
|
|
251
|
+
var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
252
|
+
// articles + determiners
|
|
253
|
+
"a",
|
|
254
|
+
"an",
|
|
255
|
+
"the",
|
|
256
|
+
"this",
|
|
257
|
+
"that",
|
|
258
|
+
"these",
|
|
259
|
+
"those",
|
|
260
|
+
"some",
|
|
261
|
+
"any",
|
|
262
|
+
"all",
|
|
263
|
+
"each",
|
|
264
|
+
"both",
|
|
265
|
+
"every",
|
|
266
|
+
"other",
|
|
267
|
+
"another",
|
|
268
|
+
"same",
|
|
269
|
+
"such",
|
|
270
|
+
// pronouns
|
|
271
|
+
"i",
|
|
272
|
+
"me",
|
|
273
|
+
"my",
|
|
274
|
+
"we",
|
|
275
|
+
"us",
|
|
276
|
+
"our",
|
|
277
|
+
"you",
|
|
278
|
+
"your",
|
|
279
|
+
"it",
|
|
280
|
+
"its",
|
|
281
|
+
"he",
|
|
282
|
+
"him",
|
|
283
|
+
"his",
|
|
284
|
+
"she",
|
|
285
|
+
"her",
|
|
286
|
+
"they",
|
|
287
|
+
"them",
|
|
288
|
+
"their",
|
|
289
|
+
// be / have / do
|
|
290
|
+
"am",
|
|
291
|
+
"is",
|
|
292
|
+
"are",
|
|
293
|
+
"was",
|
|
294
|
+
"were",
|
|
295
|
+
"be",
|
|
296
|
+
"been",
|
|
297
|
+
"being",
|
|
298
|
+
"have",
|
|
299
|
+
"has",
|
|
300
|
+
"had",
|
|
301
|
+
"do",
|
|
302
|
+
"does",
|
|
303
|
+
"did",
|
|
304
|
+
"done",
|
|
305
|
+
// modals
|
|
306
|
+
"can",
|
|
307
|
+
"could",
|
|
308
|
+
"will",
|
|
309
|
+
"would",
|
|
310
|
+
"shall",
|
|
311
|
+
"should",
|
|
312
|
+
"may",
|
|
313
|
+
"might",
|
|
314
|
+
"must",
|
|
315
|
+
// question words
|
|
316
|
+
"how",
|
|
317
|
+
"what",
|
|
318
|
+
"when",
|
|
319
|
+
"where",
|
|
320
|
+
"which",
|
|
321
|
+
"who",
|
|
322
|
+
"whom",
|
|
323
|
+
"why",
|
|
324
|
+
// prepositions + conjunctions
|
|
325
|
+
"about",
|
|
326
|
+
"and",
|
|
327
|
+
"as",
|
|
328
|
+
"at",
|
|
329
|
+
"but",
|
|
330
|
+
"by",
|
|
331
|
+
"for",
|
|
332
|
+
"from",
|
|
333
|
+
"if",
|
|
334
|
+
"in",
|
|
335
|
+
"into",
|
|
336
|
+
"of",
|
|
337
|
+
"on",
|
|
338
|
+
"or",
|
|
339
|
+
"so",
|
|
340
|
+
"than",
|
|
341
|
+
"then",
|
|
342
|
+
"there",
|
|
343
|
+
"to",
|
|
344
|
+
"with",
|
|
345
|
+
// request scaffolding + degree words
|
|
346
|
+
"please",
|
|
347
|
+
"help",
|
|
348
|
+
"let",
|
|
349
|
+
"tell",
|
|
350
|
+
"want",
|
|
351
|
+
"need",
|
|
352
|
+
"create",
|
|
353
|
+
"make",
|
|
354
|
+
"give",
|
|
355
|
+
"get",
|
|
356
|
+
"got",
|
|
357
|
+
"use",
|
|
358
|
+
"using",
|
|
359
|
+
"just",
|
|
360
|
+
"like",
|
|
361
|
+
"also",
|
|
362
|
+
"very",
|
|
363
|
+
"only",
|
|
364
|
+
"new",
|
|
365
|
+
"more",
|
|
366
|
+
"most",
|
|
367
|
+
"much",
|
|
368
|
+
"many",
|
|
369
|
+
"few",
|
|
370
|
+
"own",
|
|
371
|
+
// conversational scaffolding
|
|
372
|
+
"hey",
|
|
373
|
+
"hi",
|
|
374
|
+
"hiya",
|
|
375
|
+
"hello",
|
|
376
|
+
"thanks",
|
|
377
|
+
"thank",
|
|
378
|
+
"okay",
|
|
379
|
+
"ok",
|
|
380
|
+
"yep",
|
|
381
|
+
"yeah",
|
|
382
|
+
"yes",
|
|
383
|
+
"nope",
|
|
384
|
+
"good",
|
|
385
|
+
"great",
|
|
386
|
+
"fine",
|
|
387
|
+
"nice",
|
|
388
|
+
"cool",
|
|
389
|
+
"ready",
|
|
390
|
+
"going",
|
|
391
|
+
"doing",
|
|
392
|
+
"sounds",
|
|
393
|
+
"morning",
|
|
394
|
+
"afternoon",
|
|
395
|
+
"evening",
|
|
396
|
+
"night",
|
|
397
|
+
// Apostrophes are token boundaries, so contractions otherwise leave these
|
|
398
|
+
// fragments looking like rare, highly distinctive query terms.
|
|
399
|
+
"s",
|
|
400
|
+
"t",
|
|
401
|
+
"re",
|
|
402
|
+
"ve",
|
|
403
|
+
"ll",
|
|
404
|
+
"d",
|
|
405
|
+
"m",
|
|
406
|
+
"isn",
|
|
407
|
+
"aren",
|
|
408
|
+
"wasn",
|
|
409
|
+
"weren",
|
|
410
|
+
"don",
|
|
411
|
+
"doesn",
|
|
412
|
+
"didn",
|
|
413
|
+
"hasn",
|
|
414
|
+
"haven",
|
|
415
|
+
"hadn",
|
|
416
|
+
"won",
|
|
417
|
+
"wouldn",
|
|
418
|
+
"shouldn",
|
|
419
|
+
"couldn"
|
|
420
|
+
]);
|
|
421
|
+
|
|
254
422
|
// src/reader/fts-query.ts
|
|
255
|
-
function
|
|
423
|
+
function contentTokens(query) {
|
|
256
424
|
const tokens = [
|
|
257
425
|
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
258
426
|
];
|
|
427
|
+
const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
|
|
428
|
+
return content.length > 0 ? content : tokens;
|
|
429
|
+
}
|
|
430
|
+
function contentFtsQuery(query) {
|
|
431
|
+
const tokens = contentTokens(query);
|
|
432
|
+
return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
|
|
433
|
+
}
|
|
434
|
+
function selectiveFtsQuery(query, rows, rowsMatching) {
|
|
435
|
+
const tokens = contentTokens(query);
|
|
259
436
|
if (tokens.length === 0) return null;
|
|
260
|
-
|
|
437
|
+
const counted = tokens.map((token) => {
|
|
438
|
+
const phrase = quoteToken(token);
|
|
439
|
+
return { phrase, hits: rowsMatching(phrase) };
|
|
440
|
+
});
|
|
441
|
+
const kept = counted.filter((c) => c.hits * 2 <= rows);
|
|
442
|
+
const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
|
|
443
|
+
return chosen.map((c) => c.phrase).join(" OR ");
|
|
261
444
|
}
|
|
262
445
|
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
263
446
|
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
264
|
-
function documentFtsTopIds(db,
|
|
265
|
-
const
|
|
266
|
-
return
|
|
447
|
+
function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
|
|
448
|
+
const match = contentFtsQuery(query);
|
|
449
|
+
if (!match || allowedDocumentIds?.size === 0) return [];
|
|
450
|
+
const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
|
|
451
|
+
const title = query.normalize("NFKC").trim();
|
|
452
|
+
const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
|
|
453
|
+
const rest = db.prepare(
|
|
267
454
|
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
268
|
-
|
|
269
|
-
|
|
455
|
+
${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
|
|
456
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
457
|
+
).all(
|
|
458
|
+
...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
|
|
459
|
+
);
|
|
460
|
+
for (const row of rest) {
|
|
461
|
+
if (ids.length >= limit) break;
|
|
462
|
+
if (!ids.includes(row.document_id)) ids.push(row.document_id);
|
|
463
|
+
}
|
|
464
|
+
return ids;
|
|
465
|
+
}
|
|
466
|
+
var NAME_STOPWORDS = new Set(
|
|
467
|
+
"a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
|
|
468
|
+
);
|
|
469
|
+
var NAMED_TITLE_CANDIDATES = 256;
|
|
470
|
+
var TITLE_DF_CACHE_MAX = 5e4;
|
|
471
|
+
var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
|
|
472
|
+
var SAME_WORDS_SCORE = Number.MAX_VALUE;
|
|
473
|
+
function nameWords(text) {
|
|
474
|
+
return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
475
|
+
}
|
|
476
|
+
function quoteToken(token) {
|
|
477
|
+
return `"${token.replace(/"/g, '""')}"`;
|
|
478
|
+
}
|
|
479
|
+
var titleStatsByDb = /* @__PURE__ */ new WeakMap();
|
|
480
|
+
function titleStats(db) {
|
|
481
|
+
let stats = titleStatsByDb.get(db);
|
|
482
|
+
if (!stats) {
|
|
483
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
|
|
484
|
+
stats = {
|
|
485
|
+
documents: Math.max(1, Number(row.c)),
|
|
486
|
+
df: /* @__PURE__ */ new Map(),
|
|
487
|
+
dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
|
|
488
|
+
candidates: db.prepare(
|
|
489
|
+
`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
490
|
+
ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
491
|
+
)
|
|
492
|
+
};
|
|
493
|
+
titleStatsByDb.set(db, stats);
|
|
494
|
+
}
|
|
495
|
+
return stats;
|
|
496
|
+
}
|
|
497
|
+
function titleDf(stats, word) {
|
|
498
|
+
let df = stats.df.get(word);
|
|
499
|
+
if (df === void 0) {
|
|
500
|
+
if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
|
|
501
|
+
const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
|
|
502
|
+
df = Number(row.c);
|
|
503
|
+
stats.df.set(word, df);
|
|
504
|
+
}
|
|
505
|
+
return df;
|
|
506
|
+
}
|
|
507
|
+
function namedTitleMatches(db, query, limit, allowedDocumentIds) {
|
|
508
|
+
if (allowedDocumentIds?.size === 0) return [];
|
|
509
|
+
const raw = query.normalize("NFKC").trim();
|
|
510
|
+
const words = [...new Set(nameWords(raw))].slice(0, 16);
|
|
511
|
+
const content = words.filter((w) => !NAME_STOPWORDS.has(w));
|
|
512
|
+
if (content.length === 0 || limit <= 0) return [];
|
|
513
|
+
const asked = new Set(words);
|
|
514
|
+
const capitalized = new Set(
|
|
515
|
+
(raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
|
|
516
|
+
);
|
|
517
|
+
const queryWords = nameWords(raw).join(" ");
|
|
518
|
+
const stats = titleStats(db);
|
|
519
|
+
const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
|
|
520
|
+
const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
521
|
+
AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
|
|
522
|
+
const named = [];
|
|
523
|
+
for (const row of rows) {
|
|
524
|
+
const titleWords = nameWords(row.title);
|
|
525
|
+
if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
|
|
526
|
+
const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
|
|
527
|
+
if (distinct.length === 0) continue;
|
|
528
|
+
let score;
|
|
529
|
+
if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
|
|
530
|
+
score = EXACT_TITLE_SCORE;
|
|
531
|
+
} else if (titleWords.join(" ") === queryWords) {
|
|
532
|
+
score = SAME_WORDS_SCORE;
|
|
533
|
+
} else {
|
|
534
|
+
score = 0;
|
|
535
|
+
for (const w of distinct) {
|
|
536
|
+
score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
named.push({ documentId: row.document_id, score, order: named.length });
|
|
540
|
+
}
|
|
541
|
+
named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
|
|
542
|
+
return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
|
|
543
|
+
}
|
|
544
|
+
function compareScores(a, b) {
|
|
545
|
+
return a === b ? 0 : a > b ? 1 : -1;
|
|
270
546
|
}
|
|
271
547
|
var SMOKE_QUERY_TOP_N = 10;
|
|
272
548
|
function documentSmokeQueryMisses(db, smoke) {
|
|
273
|
-
const
|
|
274
|
-
if (!match) return [...smoke.expectedDocumentIds];
|
|
275
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
549
|
+
const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
|
|
276
550
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
277
551
|
}
|
|
278
552
|
|
|
@@ -284,6 +558,92 @@ var KNOWLEDGE_TOOLCHAIN = {
|
|
|
284
558
|
version: nodeRequire3("../package.json").version
|
|
285
559
|
};
|
|
286
560
|
|
|
561
|
+
// src/compiler/assets.ts
|
|
562
|
+
import { createHash } from "crypto";
|
|
563
|
+
import { readFileSync } from "fs";
|
|
564
|
+
import {
|
|
565
|
+
KnowledgeAssetPathSchema,
|
|
566
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
567
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
568
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
569
|
+
assetExtension,
|
|
570
|
+
assetKindForExtension,
|
|
571
|
+
sniffAssetType,
|
|
572
|
+
svgInertnessProblem
|
|
573
|
+
} from "@bendyline/gezk";
|
|
574
|
+
function prepareAssets(assets, extraFiles, opts) {
|
|
575
|
+
for (const path of Object.keys(extraFiles)) {
|
|
576
|
+
if (path.startsWith("assets/")) {
|
|
577
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
581
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
582
|
+
}
|
|
583
|
+
const seen = /* @__PURE__ */ new Set();
|
|
584
|
+
const prepared = [];
|
|
585
|
+
const skippedPaths = /* @__PURE__ */ new Set();
|
|
586
|
+
const rejectContent = (path, message) => {
|
|
587
|
+
if (opts.invalidAssets !== "warn") throw new Error(message);
|
|
588
|
+
skippedPaths.add(path);
|
|
589
|
+
opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
|
|
590
|
+
};
|
|
591
|
+
let total = 0;
|
|
592
|
+
for (const asset of assets) {
|
|
593
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
594
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
595
|
+
const key = asset.path.toLowerCase();
|
|
596
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
597
|
+
seen.add(key);
|
|
598
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
599
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
600
|
+
}
|
|
601
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
602
|
+
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
603
|
+
rejectContent(
|
|
604
|
+
asset.path,
|
|
605
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
606
|
+
);
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
const ext = assetExtension(asset.path);
|
|
610
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
611
|
+
const actual = sniffAssetType(bytes);
|
|
612
|
+
if (expected === null || actual !== expected) {
|
|
613
|
+
rejectContent(
|
|
614
|
+
asset.path,
|
|
615
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
616
|
+
);
|
|
617
|
+
continue;
|
|
618
|
+
}
|
|
619
|
+
if (ext === "svg") {
|
|
620
|
+
const problem = svgInertnessProblem(bytes);
|
|
621
|
+
if (problem) {
|
|
622
|
+
rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
total += bytes.byteLength;
|
|
627
|
+
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
628
|
+
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
629
|
+
}
|
|
630
|
+
prepared.push({
|
|
631
|
+
path: asset.path,
|
|
632
|
+
bytes,
|
|
633
|
+
sizeBytes: bytes.byteLength,
|
|
634
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
|
|
638
|
+
}
|
|
639
|
+
function omitSkippedAssetReferences(markdown, skippedPaths) {
|
|
640
|
+
if (skippedPaths.size === 0) return markdown;
|
|
641
|
+
return markdown.replace(
|
|
642
|
+
/!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
|
|
643
|
+
(whole, text, target) => skippedPaths.has(target) ? text : whole
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
|
|
287
647
|
// src/compiler/compile.ts
|
|
288
648
|
async function compileKnowledgeCatalog(opts) {
|
|
289
649
|
if (opts.topics.length === 0) {
|
|
@@ -292,7 +652,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
292
652
|
);
|
|
293
653
|
}
|
|
294
654
|
const topicById = assertTopicForest(opts.topics);
|
|
295
|
-
const
|
|
655
|
+
const { assets: preparedAssets, skippedPaths } = prepareAssets(
|
|
656
|
+
opts.assets ?? [],
|
|
657
|
+
opts.extraFiles ?? {},
|
|
658
|
+
opts
|
|
659
|
+
);
|
|
296
660
|
const profile = opts.embeddingProfile;
|
|
297
661
|
const chunkerOpts = {
|
|
298
662
|
unit: opts.chunkingProfile.unit,
|
|
@@ -302,13 +666,40 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
302
666
|
};
|
|
303
667
|
const prepared = [];
|
|
304
668
|
const seenIds = /* @__PURE__ */ new Set();
|
|
305
|
-
for await (const
|
|
669
|
+
for await (const input of opts.documents) {
|
|
670
|
+
const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
|
|
671
|
+
...location,
|
|
672
|
+
longitude: normalizeLongitude(location.longitude)
|
|
673
|
+
})).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
674
|
+
const doc = {
|
|
675
|
+
...input,
|
|
676
|
+
locations,
|
|
677
|
+
markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
|
|
678
|
+
};
|
|
306
679
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
307
680
|
seenIds.add(doc.id);
|
|
681
|
+
const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
|
|
682
|
+
const placements = [
|
|
683
|
+
{
|
|
684
|
+
topicId: leafTopicId,
|
|
685
|
+
ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
|
|
686
|
+
}
|
|
687
|
+
];
|
|
688
|
+
const leaves = /* @__PURE__ */ new Set([leafTopicId]);
|
|
689
|
+
for (const raw of doc.tocReferences ?? []) {
|
|
690
|
+
const reference = KnowledgeTocReferenceSchema.parse(raw);
|
|
691
|
+
const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
|
|
692
|
+
if (leaves.has(topicId))
|
|
693
|
+
throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
|
|
694
|
+
leaves.add(topicId);
|
|
695
|
+
placements.push({ topicId, ordinal: reference.ordinal ?? null });
|
|
696
|
+
}
|
|
697
|
+
placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
|
|
308
698
|
prepared.push({
|
|
309
699
|
doc,
|
|
310
700
|
topicPathKey: doc.topicPath.join("/"),
|
|
311
|
-
leafTopicId
|
|
701
|
+
leafTopicId,
|
|
702
|
+
placements,
|
|
312
703
|
metaJson: encodeDocumentMeta(doc),
|
|
313
704
|
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
314
705
|
});
|
|
@@ -411,7 +802,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
411
802
|
});
|
|
412
803
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
413
804
|
for (const p of prepared) {
|
|
414
|
-
|
|
805
|
+
for (const placement of p.placements) {
|
|
806
|
+
topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
|
|
807
|
+
}
|
|
415
808
|
}
|
|
416
809
|
{
|
|
417
810
|
const stmt = router.prepare(
|
|
@@ -435,6 +828,13 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
435
828
|
body_codec, body_blob)
|
|
436
829
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
437
830
|
);
|
|
831
|
+
const insertLocation = router.prepare(
|
|
832
|
+
`INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
|
|
833
|
+
VALUES (?, ?, ?, ?, ?, ?)`
|
|
834
|
+
);
|
|
835
|
+
const insertPlacement = router.prepare(
|
|
836
|
+
"INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
|
|
837
|
+
);
|
|
438
838
|
const insertAlias = router.prepare(
|
|
439
839
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
440
840
|
);
|
|
@@ -473,6 +873,29 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
473
873
|
useBrotli ? "br" : "none",
|
|
474
874
|
blob
|
|
475
875
|
);
|
|
876
|
+
for (const location of p.doc.locations ?? []) {
|
|
877
|
+
const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
|
|
878
|
+
if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
|
|
879
|
+
throw new Error(
|
|
880
|
+
`document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
insertLocation.run(
|
|
884
|
+
p.doc.id,
|
|
885
|
+
location.id,
|
|
886
|
+
location.latitude,
|
|
887
|
+
location.longitude,
|
|
888
|
+
location.role,
|
|
889
|
+
provenance
|
|
890
|
+
);
|
|
891
|
+
}
|
|
892
|
+
for (const placement of p.placements) {
|
|
893
|
+
insertPlacement.run(
|
|
894
|
+
placement.topicId,
|
|
895
|
+
p.doc.id,
|
|
896
|
+
placement.ordinal === null ? null : BigInt(placement.ordinal)
|
|
897
|
+
);
|
|
898
|
+
}
|
|
476
899
|
const aliases = [...new Set(p.doc.aliases ?? [])].sort();
|
|
477
900
|
for (const alias of aliases) insertAlias.run(alias, p.doc.id);
|
|
478
901
|
insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
|
|
@@ -705,7 +1128,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
705
1128
|
files.push({
|
|
706
1129
|
path,
|
|
707
1130
|
sizeBytes: bytes.length,
|
|
708
|
-
sha256:
|
|
1131
|
+
sha256: createHash2("sha256").update(bytes).digest("hex")
|
|
709
1132
|
});
|
|
710
1133
|
}
|
|
711
1134
|
for (const asset of preparedAssets) {
|
|
@@ -749,6 +1172,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
749
1172
|
assets: preparedAssets.length
|
|
750
1173
|
},
|
|
751
1174
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
1175
|
+
spatial: spatialManifest(
|
|
1176
|
+
prepared.flatMap(
|
|
1177
|
+
(p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
|
|
1178
|
+
)
|
|
1179
|
+
),
|
|
752
1180
|
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
753
1181
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
754
1182
|
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
@@ -844,58 +1272,6 @@ function encodeDocumentMeta(doc) {
|
|
|
844
1272
|
}
|
|
845
1273
|
return json;
|
|
846
1274
|
}
|
|
847
|
-
function prepareAssets(assets, extraFiles) {
|
|
848
|
-
for (const path of Object.keys(extraFiles)) {
|
|
849
|
-
if (path.startsWith("assets/")) {
|
|
850
|
-
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
854
|
-
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
855
|
-
}
|
|
856
|
-
const seen = /* @__PURE__ */ new Set();
|
|
857
|
-
const prepared = [];
|
|
858
|
-
let total = 0;
|
|
859
|
-
for (const asset of assets) {
|
|
860
|
-
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
861
|
-
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
862
|
-
const key = asset.path.toLowerCase();
|
|
863
|
-
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
864
|
-
seen.add(key);
|
|
865
|
-
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
866
|
-
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
867
|
-
}
|
|
868
|
-
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
869
|
-
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
870
|
-
throw new Error(
|
|
871
|
-
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
872
|
-
);
|
|
873
|
-
}
|
|
874
|
-
total += bytes.byteLength;
|
|
875
|
-
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
876
|
-
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
877
|
-
}
|
|
878
|
-
const ext = assetExtension(asset.path);
|
|
879
|
-
const expected = ext ? assetKindForExtension(ext) : null;
|
|
880
|
-
const actual = sniffAssetType(bytes);
|
|
881
|
-
if (expected === null || actual !== expected) {
|
|
882
|
-
throw new Error(
|
|
883
|
-
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
884
|
-
);
|
|
885
|
-
}
|
|
886
|
-
if (ext === "svg") {
|
|
887
|
-
const problem = svgInertnessProblem(bytes);
|
|
888
|
-
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
889
|
-
}
|
|
890
|
-
prepared.push({
|
|
891
|
-
path: asset.path,
|
|
892
|
-
bytes,
|
|
893
|
-
sizeBytes: bytes.byteLength,
|
|
894
|
-
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
895
|
-
});
|
|
896
|
-
}
|
|
897
|
-
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
898
|
-
}
|
|
899
1275
|
function assertAssetReferencesDeclared(prepared, assets) {
|
|
900
1276
|
const declared = new Set(assets.map((a) => a.path));
|
|
901
1277
|
for (const p of prepared) {
|
|
@@ -1110,7 +1486,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1110
1486
|
return assignment.map((s) => renumber.get(s));
|
|
1111
1487
|
}
|
|
1112
1488
|
function seedFor(catalogId, shardId) {
|
|
1113
|
-
const digest =
|
|
1489
|
+
const digest = createHash2("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
1114
1490
|
return digest.readBigUInt64LE(24);
|
|
1115
1491
|
}
|
|
1116
1492
|
function makePrng(seed) {
|
|
@@ -1195,7 +1571,7 @@ function kMeans(samples, k, seed) {
|
|
|
1195
1571
|
}
|
|
1196
1572
|
|
|
1197
1573
|
// src/archive/read.ts
|
|
1198
|
-
import { createHash as
|
|
1574
|
+
import { createHash as createHash3 } from "crypto";
|
|
1199
1575
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
1200
1576
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1201
1577
|
import { createRequire as createRequire4 } from "module";
|
|
@@ -1485,7 +1861,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1485
1861
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1486
1862
|
return;
|
|
1487
1863
|
}
|
|
1488
|
-
const hasher =
|
|
1864
|
+
const hasher = createHash3("sha256");
|
|
1489
1865
|
let sizeBytes = 0;
|
|
1490
1866
|
const verify = new Transform({
|
|
1491
1867
|
transform(chunk, _encoding, callback) {
|
|
@@ -1608,7 +1984,7 @@ function readPragmaNumber(db, pragma) {
|
|
|
1608
1984
|
}
|
|
1609
1985
|
|
|
1610
1986
|
// src/reader/catalog-handle.ts
|
|
1611
|
-
import { readFileSync as
|
|
1987
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
1612
1988
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1613
1989
|
import { brotliDecompressSync } from "zlib";
|
|
1614
1990
|
import {
|
|
@@ -1689,14 +2065,14 @@ function hammingTopK(index, query, k) {
|
|
|
1689
2065
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1690
2066
|
return hits;
|
|
1691
2067
|
}
|
|
1692
|
-
function asymmetricTopK(index, query, k) {
|
|
2068
|
+
function asymmetricTopK(index, query, k, eligibleChunkIds) {
|
|
1693
2069
|
const { bits, bytesPerRow, rows } = index;
|
|
1694
2070
|
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1695
2071
|
throw new Error(
|
|
1696
2072
|
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1697
2073
|
);
|
|
1698
2074
|
}
|
|
1699
|
-
const limit = Math.min(k, rows);
|
|
2075
|
+
const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
|
|
1700
2076
|
if (limit <= 0) return [];
|
|
1701
2077
|
const lut = new Float32Array(bytesPerRow * 256);
|
|
1702
2078
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
@@ -1746,6 +2122,10 @@ function asymmetricTopK(index, query, k) {
|
|
|
1746
2122
|
};
|
|
1747
2123
|
let offset = 0;
|
|
1748
2124
|
for (let r = 0; r < rows; r++) {
|
|
2125
|
+
if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
|
|
2126
|
+
offset += bytesPerRow;
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
1749
2129
|
let score = 0;
|
|
1750
2130
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
1751
2131
|
score += lut[p * 256 + bits[offset + p]];
|
|
@@ -1770,14 +2150,126 @@ function asymmetricTopK(index, query, k) {
|
|
|
1770
2150
|
return hits;
|
|
1771
2151
|
}
|
|
1772
2152
|
|
|
2153
|
+
// src/reader/spatial-index.ts
|
|
2154
|
+
import {
|
|
2155
|
+
KnowledgeLocationSchema,
|
|
2156
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
|
|
2157
|
+
KnowledgeRadiusSchema,
|
|
2158
|
+
locationDistanceMeters,
|
|
2159
|
+
locationInBounds,
|
|
2160
|
+
normalizeLongitude as normalizeLongitude2,
|
|
2161
|
+
radiusBounds,
|
|
2162
|
+
spatialManifest as spatialManifest2
|
|
2163
|
+
} from "@bendyline/gezk";
|
|
2164
|
+
var DocumentSpatialIndex = class {
|
|
2165
|
+
constructor(db, allowLegacy) {
|
|
2166
|
+
this.db = db;
|
|
2167
|
+
this.allowLegacy = allowLegacy;
|
|
2168
|
+
this.hasTable = Boolean(
|
|
2169
|
+
db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
db;
|
|
2173
|
+
allowLegacy;
|
|
2174
|
+
hasTable;
|
|
2175
|
+
legacy;
|
|
2176
|
+
decode(row) {
|
|
2177
|
+
const provenance = row.provenance_json;
|
|
2178
|
+
if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
|
|
2179
|
+
throw new Error("location provenance exceeds 16384 bytes");
|
|
2180
|
+
return {
|
|
2181
|
+
documentId: String(row.document_id),
|
|
2182
|
+
location: KnowledgeLocationSchema.parse({
|
|
2183
|
+
id: row.location_id,
|
|
2184
|
+
latitude: row.latitude,
|
|
2185
|
+
longitude: row.longitude,
|
|
2186
|
+
role: row.role,
|
|
2187
|
+
...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
|
|
2188
|
+
})
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
legacyRows() {
|
|
2192
|
+
if (this.legacy) return this.legacy;
|
|
2193
|
+
this.legacy = [];
|
|
2194
|
+
if (!this.allowLegacy) return this.legacy;
|
|
2195
|
+
for (const row of this.db.prepare(
|
|
2196
|
+
"SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
|
|
2197
|
+
).all()) {
|
|
2198
|
+
try {
|
|
2199
|
+
const meta = JSON.parse(String(row.meta_json));
|
|
2200
|
+
if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
|
|
2201
|
+
continue;
|
|
2202
|
+
const point = KnowledgeLocationSchema.safeParse({
|
|
2203
|
+
id: "qualla-coordinate",
|
|
2204
|
+
role: "subject",
|
|
2205
|
+
latitude: meta.coordinates.lat,
|
|
2206
|
+
longitude: meta.coordinates.lng,
|
|
2207
|
+
provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
|
|
2208
|
+
});
|
|
2209
|
+
if (point.success)
|
|
2210
|
+
this.legacy.push({
|
|
2211
|
+
documentId: String(row.id),
|
|
2212
|
+
location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
|
|
2213
|
+
});
|
|
2214
|
+
} catch {
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
return this.legacy;
|
|
2218
|
+
}
|
|
2219
|
+
rows() {
|
|
2220
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
|
|
2221
|
+
}
|
|
2222
|
+
locations(documentId) {
|
|
2223
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
|
|
2224
|
+
}
|
|
2225
|
+
integrity() {
|
|
2226
|
+
const rows = this.rows();
|
|
2227
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
2228
|
+
for (const row of rows) {
|
|
2229
|
+
if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
|
|
2230
|
+
throw new Error("location longitude must be in [-180, 180)");
|
|
2231
|
+
const points = grouped.get(row.documentId) ?? [];
|
|
2232
|
+
points.push(row.location);
|
|
2233
|
+
grouped.set(row.documentId, points);
|
|
2234
|
+
if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
|
|
2235
|
+
throw new Error("location references missing document");
|
|
2236
|
+
}
|
|
2237
|
+
for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
|
|
2238
|
+
return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
|
|
2239
|
+
}
|
|
2240
|
+
matches(raw) {
|
|
2241
|
+
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
2242
|
+
const boxes = radiusBounds(radius);
|
|
2243
|
+
const candidates = this.hasTable ? boxes.flatMap(
|
|
2244
|
+
(box) => this.db.prepare(`SELECT * FROM document_locations
|
|
2245
|
+
WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
|
|
2246
|
+
) : this.legacyRows().filter(
|
|
2247
|
+
(row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
|
|
2248
|
+
);
|
|
2249
|
+
const matches = /* @__PURE__ */ new Map();
|
|
2250
|
+
for (const { documentId, location } of candidates) {
|
|
2251
|
+
const distanceMeters = locationDistanceMeters(radius, location);
|
|
2252
|
+
if (distanceMeters > radius.radiusMeters) continue;
|
|
2253
|
+
const previous = matches.get(documentId);
|
|
2254
|
+
if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
|
|
2255
|
+
matches.set(documentId, { distanceMeters, matchedLocation: location });
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
return matches;
|
|
2259
|
+
}
|
|
2260
|
+
};
|
|
2261
|
+
|
|
1773
2262
|
// src/reader/catalog-handle.ts
|
|
1774
2263
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
1775
2264
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
2265
|
+
var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
|
|
2266
|
+
var TITLE_ROUTE_SHARDS = 1;
|
|
1776
2267
|
var CatalogHandle = class _CatalogHandle {
|
|
1777
2268
|
constructor(rootDir, router) {
|
|
1778
2269
|
this.rootDir = rootDir;
|
|
1779
2270
|
this.router = router;
|
|
1780
2271
|
this.schemaVersion = router.schemaVersion;
|
|
2272
|
+
this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
1781
2273
|
this.meta = {};
|
|
1782
2274
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1783
2275
|
this.meta[row.key] = row.value;
|
|
@@ -1789,6 +2281,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1789
2281
|
router;
|
|
1790
2282
|
connections = /* @__PURE__ */ new Map();
|
|
1791
2283
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
2284
|
+
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
2285
|
+
chunkTermRows = /* @__PURE__ */ new Map();
|
|
2286
|
+
spatial;
|
|
1792
2287
|
assetsByPath = null;
|
|
1793
2288
|
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1794
2289
|
binaryCenterCache;
|
|
@@ -1799,7 +2294,12 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1799
2294
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1800
2295
|
static open(rootDir) {
|
|
1801
2296
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
1802
|
-
|
|
2297
|
+
try {
|
|
2298
|
+
return new _CatalogHandle(rootDir, router);
|
|
2299
|
+
} catch (error) {
|
|
2300
|
+
router.close();
|
|
2301
|
+
throw error;
|
|
2302
|
+
}
|
|
1803
2303
|
}
|
|
1804
2304
|
close() {
|
|
1805
2305
|
for (const conn of this.connections.values()) conn.close();
|
|
@@ -1874,6 +2374,24 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1874
2374
|
this.bitIndexes.set(shard.path, index);
|
|
1875
2375
|
return index;
|
|
1876
2376
|
}
|
|
2377
|
+
/**
|
|
2378
|
+
* Load the next shard's sign bits that are not resident yet, if they fit
|
|
2379
|
+
* the resident budget. False when nothing is left to load. One shard per
|
|
2380
|
+
* call, so a host can keep answering queries between shards while it warms
|
|
2381
|
+
* a large catalog, and never evicts a shard it loaded a moment ago.
|
|
2382
|
+
*/
|
|
2383
|
+
prewarmNextShard() {
|
|
2384
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
2385
|
+
let resident = 0;
|
|
2386
|
+
for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
|
|
2387
|
+
for (const shard of this.shards) {
|
|
2388
|
+
if (this.bitIndexes.has(shard.path)) continue;
|
|
2389
|
+
if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
|
|
2390
|
+
this.shardBits(shard);
|
|
2391
|
+
return true;
|
|
2392
|
+
}
|
|
2393
|
+
return false;
|
|
2394
|
+
}
|
|
1877
2395
|
/**
|
|
1878
2396
|
* The stage-1 center the catalog's bits were derived with (a
|
|
1879
2397
|
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
@@ -1923,9 +2441,13 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1923
2441
|
get formatVersion() {
|
|
1924
2442
|
return this.meta.format_version ?? "";
|
|
1925
2443
|
}
|
|
1926
|
-
documentColumns() {
|
|
2444
|
+
documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
|
|
1927
2445
|
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1928
|
-
|
|
2446
|
+
const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
2447
|
+
return columns.split(", ").map((column) => {
|
|
2448
|
+
const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
|
|
2449
|
+
return prefix ? `${prefix}.${column} AS ${column}` : column;
|
|
2450
|
+
}).join(", ");
|
|
1929
2451
|
}
|
|
1930
2452
|
documentOrder() {
|
|
1931
2453
|
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
@@ -1935,6 +2457,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1935
2457
|
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1936
2458
|
*/
|
|
1937
2459
|
topics() {
|
|
2460
|
+
const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
|
|
2461
|
+
FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
2462
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
|
|
1938
2463
|
return this.router.db.prepare(
|
|
1939
2464
|
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1940
2465
|
SELECT id, id, 0 FROM topics
|
|
@@ -1943,8 +2468,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1943
2468
|
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1944
2469
|
),
|
|
1945
2470
|
rollup AS (
|
|
1946
|
-
|
|
1947
|
-
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
2471
|
+
${rollup}
|
|
1948
2472
|
)
|
|
1949
2473
|
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1950
2474
|
rollup.total AS total_document_count
|
|
@@ -1968,6 +2492,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1968
2492
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1969
2493
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1970
2494
|
const descendants = opts.descendants ?? true;
|
|
2495
|
+
if (this.schemaVersion >= 4 && opts.topicId) {
|
|
2496
|
+
return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
|
|
2497
|
+
}
|
|
1971
2498
|
let scope = "";
|
|
1972
2499
|
let where = "";
|
|
1973
2500
|
const params = [];
|
|
@@ -1990,16 +2517,101 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1990
2517
|
const documents = this.router.db.prepare(
|
|
1991
2518
|
`${scope} SELECT ${this.documentColumns()}
|
|
1992
2519
|
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1993
|
-
).all(...params, limit, offset).map(
|
|
2520
|
+
).all(...params, limit, offset).map((row) => this.documentMeta(row));
|
|
1994
2521
|
return { documents, total };
|
|
1995
2522
|
}
|
|
2523
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
2524
|
+
sharedDocumentsPage(topicId, descendants, limit, offset) {
|
|
2525
|
+
const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
|
|
2526
|
+
JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
|
|
2527
|
+
const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
|
|
2528
|
+
SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
|
|
2529
|
+
FROM topic_documents td JOIN topics t ON t.id = td.topic_id
|
|
2530
|
+
WHERE td.topic_id IN (SELECT id FROM sub)
|
|
2531
|
+
)`;
|
|
2532
|
+
const total = Number(
|
|
2533
|
+
this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
|
|
2534
|
+
);
|
|
2535
|
+
const documents = this.router.db.prepare(
|
|
2536
|
+
`${scope}, ranked AS (
|
|
2537
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2538
|
+
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2539
|
+
) SELECT ${this.documentColumns("d", "p")}
|
|
2540
|
+
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
|
|
2541
|
+
ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
|
|
2542
|
+
).all(topicId, limit, offset).map((row) => this.documentMeta(row));
|
|
2543
|
+
return { documents, total };
|
|
2544
|
+
}
|
|
2545
|
+
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
2546
|
+
tocIntegrity() {
|
|
2547
|
+
const count = (sql) => Number(this.router.db.prepare(sql).get().n);
|
|
2548
|
+
const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
|
|
2549
|
+
const incorrectCounts = count(
|
|
2550
|
+
`SELECT COUNT(*) AS n FROM topics t LEFT JOIN
|
|
2551
|
+
(SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
|
|
2552
|
+
WHERE t.document_count != COALESCE(r.n, 0)`
|
|
2553
|
+
);
|
|
2554
|
+
if (this.schemaVersion < 4)
|
|
2555
|
+
return {
|
|
2556
|
+
placements: count("SELECT COUNT(*) AS n FROM documents"),
|
|
2557
|
+
invalidReferences: 0,
|
|
2558
|
+
missingPrimary: 0,
|
|
2559
|
+
incorrectCounts,
|
|
2560
|
+
duplicateReferences: 0
|
|
2561
|
+
};
|
|
2562
|
+
return {
|
|
2563
|
+
placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
|
|
2564
|
+
invalidReferences: count(
|
|
2565
|
+
`SELECT COUNT(*) AS n FROM topic_documents r
|
|
2566
|
+
LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
|
|
2567
|
+
WHERE d.id IS NULL OR t.id IS NULL OR
|
|
2568
|
+
(r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
|
|
2569
|
+
),
|
|
2570
|
+
missingPrimary: count(
|
|
2571
|
+
`SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
|
|
2572
|
+
SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
|
|
2573
|
+
),
|
|
2574
|
+
incorrectCounts,
|
|
2575
|
+
duplicateReferences: count(
|
|
2576
|
+
"SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
|
|
2577
|
+
)
|
|
2578
|
+
};
|
|
2579
|
+
}
|
|
1996
2580
|
getDocument(id) {
|
|
1997
2581
|
const row = this.router.db.prepare(
|
|
1998
2582
|
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1999
2583
|
FROM documents WHERE id = ?`
|
|
2000
2584
|
).get(id);
|
|
2001
2585
|
if (!row) return null;
|
|
2002
|
-
return { ...
|
|
2586
|
+
return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
|
|
2587
|
+
}
|
|
2588
|
+
/**
|
|
2589
|
+
* One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
|
|
2590
|
+
* shard the document's chunks live in. Null when the document or the
|
|
2591
|
+
* passage is not in this catalog, which is what a fabricated or stale
|
|
2592
|
+
* citation looks like.
|
|
2593
|
+
*/
|
|
2594
|
+
getChunk(documentId, chunkUid2) {
|
|
2595
|
+
const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
|
|
2596
|
+
if (!doc) return null;
|
|
2597
|
+
const shardId = Number(doc.shard_id);
|
|
2598
|
+
const shard = this.shards.find((s) => s.id === shardId);
|
|
2599
|
+
if (!shard) return null;
|
|
2600
|
+
const row = this.shardDb(shard).prepare(
|
|
2601
|
+
`SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
|
|
2602
|
+
FROM chunks WHERE chunk_uid = ? AND document_id = ?`
|
|
2603
|
+
).get(chunkUid2, documentId);
|
|
2604
|
+
if (!row) return null;
|
|
2605
|
+
return {
|
|
2606
|
+
chunkUid: row.chunk_uid,
|
|
2607
|
+
documentId: row.document_id,
|
|
2608
|
+
title: row.title,
|
|
2609
|
+
headingPath: parseHeadingPath(row.heading_path),
|
|
2610
|
+
lineStart: Number(row.line_start),
|
|
2611
|
+
lineEnd: Number(row.line_end),
|
|
2612
|
+
text: row.text,
|
|
2613
|
+
shardId
|
|
2614
|
+
};
|
|
2003
2615
|
}
|
|
2004
2616
|
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
2005
2617
|
documentsWithUndeclaredTopic() {
|
|
@@ -2072,7 +2684,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2072
2684
|
const index = /* @__PURE__ */ new Map();
|
|
2073
2685
|
let raw;
|
|
2074
2686
|
try {
|
|
2075
|
-
raw = JSON.parse(
|
|
2687
|
+
raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
2076
2688
|
} catch (error) {
|
|
2077
2689
|
if (error.code === "ENOENT") {
|
|
2078
2690
|
this.assetsByPath = index;
|
|
@@ -2111,7 +2723,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2111
2723
|
}
|
|
2112
2724
|
let bytes;
|
|
2113
2725
|
try {
|
|
2114
|
-
bytes =
|
|
2726
|
+
bytes = readFileSync3(this.resolveCatalogPath(path));
|
|
2115
2727
|
} catch (error) {
|
|
2116
2728
|
throw new CatalogOpenError(
|
|
2117
2729
|
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -2123,19 +2735,59 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2123
2735
|
}
|
|
2124
2736
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
2125
2737
|
}
|
|
2738
|
+
documentMeta(row) {
|
|
2739
|
+
const document = rowToDocumentMeta(row);
|
|
2740
|
+
const locations = this.spatial.locations(document.id);
|
|
2741
|
+
return locations.length ? { ...document, locations } : document;
|
|
2742
|
+
}
|
|
2743
|
+
documentLocations(documentId) {
|
|
2744
|
+
return this.spatial.locations(documentId);
|
|
2745
|
+
}
|
|
2746
|
+
documentLocationRows() {
|
|
2747
|
+
return this.spatial.rows();
|
|
2748
|
+
}
|
|
2749
|
+
spatialIntegrity() {
|
|
2750
|
+
return this.spatial.integrity();
|
|
2751
|
+
}
|
|
2752
|
+
spatialMatches(radius) {
|
|
2753
|
+
return this.spatial.matches(radius);
|
|
2754
|
+
}
|
|
2755
|
+
nearbyDocuments(radius, opts = {}) {
|
|
2756
|
+
const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
|
|
2757
|
+
const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
|
|
2758
|
+
const ordered = [...this.spatialMatches(radius)].sort(
|
|
2759
|
+
(a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
|
|
2760
|
+
);
|
|
2761
|
+
const get = this.router.db.prepare(
|
|
2762
|
+
`SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
|
|
2763
|
+
);
|
|
2764
|
+
return {
|
|
2765
|
+
total: ordered.length,
|
|
2766
|
+
documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
|
|
2767
|
+
...this.documentMeta(get.get(id)),
|
|
2768
|
+
...match
|
|
2769
|
+
}))
|
|
2770
|
+
};
|
|
2771
|
+
}
|
|
2772
|
+
shardsForDocuments(ids) {
|
|
2773
|
+
if (!ids.size) return [];
|
|
2774
|
+
return this.router.db.prepare(
|
|
2775
|
+
"SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
|
|
2776
|
+
).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
|
|
2777
|
+
}
|
|
2126
2778
|
// ── search ────────────────────────────────────────────────────────────────
|
|
2127
2779
|
/**
|
|
2128
2780
|
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
2129
2781
|
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
2130
2782
|
*/
|
|
2131
|
-
searchDocumentsFts(query, limit = 10) {
|
|
2132
|
-
const match = ftsQuery(query);
|
|
2133
|
-
if (!match) return [];
|
|
2783
|
+
searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
|
|
2134
2784
|
try {
|
|
2135
|
-
return documentFtsTopIds(this.router.db,
|
|
2136
|
-
documentId,
|
|
2137
|
-
|
|
2138
|
-
|
|
2785
|
+
return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
|
|
2786
|
+
(documentId, i) => ({
|
|
2787
|
+
documentId,
|
|
2788
|
+
rank: i
|
|
2789
|
+
})
|
|
2790
|
+
);
|
|
2139
2791
|
} catch {
|
|
2140
2792
|
return [];
|
|
2141
2793
|
}
|
|
@@ -2175,24 +2827,73 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2175
2827
|
}
|
|
2176
2828
|
return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
|
|
2177
2829
|
}
|
|
2178
|
-
/**
|
|
2179
|
-
|
|
2830
|
+
/**
|
|
2831
|
+
* Top-S shard ids within THIS catalog (single-catalog callers, tests).
|
|
2832
|
+
* With the raw `query`, also the shard of the page it names by title
|
|
2833
|
+
* (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
|
|
2834
|
+
*/
|
|
2835
|
+
routeShards(queryVector, shardBudget, query) {
|
|
2180
2836
|
if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
|
|
2181
|
-
|
|
2837
|
+
const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
2838
|
+
if (query) {
|
|
2839
|
+
for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
|
|
2840
|
+
if (!routed.includes(named.shardId)) routed.push(named.shardId);
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
return routed;
|
|
2844
|
+
}
|
|
2845
|
+
/**
|
|
2846
|
+
* Title-assisted routing: the shards holding the pages `query` names by
|
|
2847
|
+
* title (`namedTitleMatches`), most specific name first, at most `limit`
|
|
2848
|
+
* distinct shards. Centroid routing compares a question with averages of
|
|
2849
|
+
* whole shards, and e5 scores those in a band a few hundredths wide, so
|
|
2850
|
+
* the shard holding the very page a question names can rank 8th of 10:
|
|
2851
|
+
* "Who were the members of ABBA?" never reached the ABBA article in the
|
|
2852
|
+
* Wikipedia music catalog. Its title does — the vector arm scans that
|
|
2853
|
+
* shard too. Measured on the 10-shard music and film-tv catalogs, one such
|
|
2854
|
+
* shard lifted end-to-end recall@8 on "What is <title>?" questions from
|
|
2855
|
+
* 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
|
|
2856
|
+
*
|
|
2857
|
+
* `score` is the named match's specificity (fts-query.ts), comparable
|
|
2858
|
+
* across catalogs closely enough for the daemon to choose which catalogs'
|
|
2859
|
+
* shards to add when several name a page. Empty on any FTS error.
|
|
2860
|
+
*/
|
|
2861
|
+
titleRouteShards(query, limit) {
|
|
2862
|
+
if (limit <= 0) return [];
|
|
2863
|
+
try {
|
|
2864
|
+
const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
|
|
2865
|
+
const out = [];
|
|
2866
|
+
for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
|
|
2867
|
+
const row = shardOf.get(named.documentId);
|
|
2868
|
+
if (!row) continue;
|
|
2869
|
+
const shardId = Number(row.shard_id);
|
|
2870
|
+
if (out.some((o) => o.shardId === shardId)) continue;
|
|
2871
|
+
out.push({ shardId, score: named.score });
|
|
2872
|
+
if (out.length >= limit) break;
|
|
2873
|
+
}
|
|
2874
|
+
return out.sort((a, b) => compareScores(b.score, a.score));
|
|
2875
|
+
} catch {
|
|
2876
|
+
return [];
|
|
2877
|
+
}
|
|
2182
2878
|
}
|
|
2183
2879
|
/**
|
|
2184
2880
|
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
2185
2881
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
2186
2882
|
*/
|
|
2187
2883
|
searchSemantic(queryVector, opts = {}) {
|
|
2188
|
-
const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
|
|
2189
|
-
return this.searchShards(
|
|
2884
|
+
const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
|
|
2885
|
+
return this.searchShards(
|
|
2886
|
+
queryVector,
|
|
2887
|
+
shardIds,
|
|
2888
|
+
opts.finalK ?? RERANK_FINAL_K,
|
|
2889
|
+
opts.allowedDocumentIds
|
|
2890
|
+
);
|
|
2190
2891
|
}
|
|
2191
2892
|
/**
|
|
2192
2893
|
* The scan stage against an EXPLICIT shard list — what the daemon calls
|
|
2193
2894
|
* after global cross-catalog routing has already spent the S budget.
|
|
2194
2895
|
*/
|
|
2195
|
-
searchShards(queryVector, shardIds, finalK) {
|
|
2896
|
+
searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
|
|
2196
2897
|
const center = this.binaryCenter();
|
|
2197
2898
|
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
2198
2899
|
const hits = [];
|
|
@@ -2200,10 +2901,17 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2200
2901
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2201
2902
|
if (!shard) continue;
|
|
2202
2903
|
const db = this.shardDb(shard);
|
|
2203
|
-
const
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2904
|
+
const eligible = allowedDocumentIds ? new Set(
|
|
2905
|
+
db.prepare(
|
|
2906
|
+
"SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
|
|
2907
|
+
).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
|
|
2908
|
+
) : void 0;
|
|
2909
|
+
const k = rerankK(finalK, eligible?.size ?? shard.chunkCount);
|
|
2910
|
+
const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k, eligible).map(
|
|
2911
|
+
(hit) => ({
|
|
2912
|
+
chunk_id: hit.chunkId
|
|
2913
|
+
})
|
|
2914
|
+
);
|
|
2207
2915
|
if (candidates.length === 0) continue;
|
|
2208
2916
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
2209
2917
|
const reranked = candidates.map((c) => {
|
|
@@ -2237,20 +2945,26 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2237
2945
|
return hits;
|
|
2238
2946
|
}
|
|
2239
2947
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
2240
|
-
searchChunksFts(query, shardIds, limitPerShard = 12) {
|
|
2241
|
-
const match = ftsQuery(query);
|
|
2242
|
-
if (!match) return [];
|
|
2948
|
+
searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
|
|
2243
2949
|
const hits = [];
|
|
2244
2950
|
for (const shardId of shardIds) {
|
|
2245
2951
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2246
2952
|
if (!shard) continue;
|
|
2247
2953
|
const db = this.shardDb(shard);
|
|
2248
2954
|
try {
|
|
2955
|
+
const match = selectiveFtsQuery(
|
|
2956
|
+
query,
|
|
2957
|
+
shard.chunkCount,
|
|
2958
|
+
(phrase) => this.chunkFtsRows(shard, db, phrase)
|
|
2959
|
+
);
|
|
2960
|
+
if (!match) return hits;
|
|
2249
2961
|
const rows = db.prepare(
|
|
2250
2962
|
`SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
|
|
2251
2963
|
FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
|
|
2252
|
-
WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
|
|
2253
|
-
).all(
|
|
2964
|
+
WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
|
|
2965
|
+
).all(
|
|
2966
|
+
...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
|
|
2967
|
+
);
|
|
2254
2968
|
for (const row of rows) {
|
|
2255
2969
|
hits.push({
|
|
2256
2970
|
chunkUid: row.chunk_uid,
|
|
@@ -2269,6 +2983,18 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2269
2983
|
}
|
|
2270
2984
|
return hits;
|
|
2271
2985
|
}
|
|
2986
|
+
/** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
|
|
2987
|
+
chunkFtsRows(shard, db, phrase) {
|
|
2988
|
+
const key = `${shard.path}\0${phrase}`;
|
|
2989
|
+
let rows = this.chunkTermRows.get(key);
|
|
2990
|
+
if (rows === void 0) {
|
|
2991
|
+
if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
|
|
2992
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
|
|
2993
|
+
rows = Number(row.c);
|
|
2994
|
+
this.chunkTermRows.set(key, rows);
|
|
2995
|
+
}
|
|
2996
|
+
return rows;
|
|
2997
|
+
}
|
|
2272
2998
|
/**
|
|
2273
2999
|
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
2274
3000
|
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
@@ -2322,7 +3048,6 @@ function parseHeadingPath(raw) {
|
|
|
2322
3048
|
return [];
|
|
2323
3049
|
}
|
|
2324
3050
|
}
|
|
2325
|
-
var ftsQuery = sanitizeFtsQuery;
|
|
2326
3051
|
|
|
2327
3052
|
// src/reader/validate.ts
|
|
2328
3053
|
import { readFile } from "fs/promises";
|
|
@@ -2334,6 +3059,7 @@ import {
|
|
|
2334
3059
|
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2335
3060
|
assetExtension as assetExtension2,
|
|
2336
3061
|
assetKindForExtension as assetKindForExtension2,
|
|
3062
|
+
canonicalizeJson as canonicalizeJson2,
|
|
2337
3063
|
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
2338
3064
|
sniffAssetType as sniffAssetType2,
|
|
2339
3065
|
svgInertnessProblem as svgInertnessProblem2
|
|
@@ -2409,6 +3135,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2409
3135
|
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2410
3136
|
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2411
3137
|
);
|
|
3138
|
+
if (manifest.formatVersion === "0.7") {
|
|
3139
|
+
try {
|
|
3140
|
+
const spatial = handle.spatialIntegrity();
|
|
3141
|
+
check("document-locations-table", spatial.hasTable);
|
|
3142
|
+
check(
|
|
3143
|
+
"document-locations-summary",
|
|
3144
|
+
Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
|
|
3145
|
+
"spatial counts and coverage must match router points"
|
|
3146
|
+
);
|
|
3147
|
+
} catch (error) {
|
|
3148
|
+
check(
|
|
3149
|
+
"document-locations-integrity",
|
|
3150
|
+
false,
|
|
3151
|
+
error instanceof Error ? error.message : String(error)
|
|
3152
|
+
);
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
2412
3155
|
const topics = handle.topics();
|
|
2413
3156
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2414
3157
|
check("topics-tree", ...topicTreeProblem(topics));
|
|
@@ -2446,11 +3189,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2446
3189
|
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
2447
3190
|
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
2448
3191
|
);
|
|
3192
|
+
const toc = handle.tocIntegrity();
|
|
3193
|
+
check(
|
|
3194
|
+
"toc-references",
|
|
3195
|
+
toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
|
|
3196
|
+
`${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
|
|
3197
|
+
);
|
|
3198
|
+
check(
|
|
3199
|
+
"toc-counts",
|
|
3200
|
+
toc.incorrectCounts === 0,
|
|
3201
|
+
`${toc.incorrectCounts} incorrect per-topic counts`
|
|
3202
|
+
);
|
|
2449
3203
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
2450
3204
|
check(
|
|
2451
3205
|
"counts-documents",
|
|
2452
|
-
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum ===
|
|
2453
|
-
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
|
|
3206
|
+
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
|
|
3207
|
+
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
|
|
2454
3208
|
);
|
|
2455
3209
|
check(
|
|
2456
3210
|
"counts-shards",
|
|
@@ -2578,6 +3332,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2578
3332
|
);
|
|
2579
3333
|
}
|
|
2580
3334
|
}
|
|
3335
|
+
} catch (error) {
|
|
3336
|
+
check("catalog-structure", false, error instanceof Error ? error.message : String(error));
|
|
2581
3337
|
} finally {
|
|
2582
3338
|
handle.close();
|
|
2583
3339
|
}
|
|
@@ -2686,13 +3442,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
2686
3442
|
}
|
|
2687
3443
|
|
|
2688
3444
|
// src/export/parquet.ts
|
|
2689
|
-
import { createHash as
|
|
3445
|
+
import { createHash as createHash4 } from "crypto";
|
|
2690
3446
|
import { createWriteStream as createWriteStream3 } from "fs";
|
|
2691
3447
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2692
3448
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2693
3449
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2694
3450
|
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2695
|
-
var PARQUET_EXPORT_VERSION =
|
|
3451
|
+
var PARQUET_EXPORT_VERSION = 4;
|
|
2696
3452
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2697
3453
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2698
3454
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2739,6 +3495,34 @@ async function exportCatalogParquet(opts) {
|
|
|
2739
3495
|
onProgress: opts.onProgress
|
|
2740
3496
|
})
|
|
2741
3497
|
);
|
|
3498
|
+
const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
3499
|
+
const locations = spatial.rows();
|
|
3500
|
+
if (spatial.hasTable || locations.length) {
|
|
3501
|
+
const path = join4(staging, "document-locations.ndjson");
|
|
3502
|
+
const out = ndjsonWriter(path);
|
|
3503
|
+
for (const { documentId, location } of locations)
|
|
3504
|
+
await out.write({
|
|
3505
|
+
document_id: documentId,
|
|
3506
|
+
location_id: location.id,
|
|
3507
|
+
latitude: location.latitude,
|
|
3508
|
+
longitude: location.longitude,
|
|
3509
|
+
role: location.role,
|
|
3510
|
+
provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
|
|
3511
|
+
});
|
|
3512
|
+
await out.close();
|
|
3513
|
+
const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
|
|
3514
|
+
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
|
|
3515
|
+
files.push(
|
|
3516
|
+
await writeParquet(opts.duckdb, {
|
|
3517
|
+
outDir,
|
|
3518
|
+
file: "document-locations.parquet",
|
|
3519
|
+
table: "document_locations",
|
|
3520
|
+
rows: locations.length,
|
|
3521
|
+
sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
|
|
3522
|
+
onProgress: opts.onProgress
|
|
3523
|
+
})
|
|
3524
|
+
);
|
|
3525
|
+
}
|
|
2742
3526
|
for (const shard of manifest.router.shards) {
|
|
2743
3527
|
const tag = String(shard.id).padStart(3, "0");
|
|
2744
3528
|
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
@@ -3081,7 +3865,7 @@ async function writeParquet(cli, spec) {
|
|
|
3081
3865
|
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
3082
3866
|
rows: spec.rows,
|
|
3083
3867
|
sizeBytes: (await stat2(target)).size,
|
|
3084
|
-
sha256:
|
|
3868
|
+
sha256: createHash4("sha256").update(bytes).digest("hex")
|
|
3085
3869
|
};
|
|
3086
3870
|
}
|
|
3087
3871
|
async function findDuckdbBinary(env = process.env) {
|
|
@@ -3582,7 +4366,7 @@ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
|
3582
4366
|
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
3583
4367
|
|
|
3584
4368
|
// src/embedding/artifact-verify.ts
|
|
3585
|
-
import { createHash as
|
|
4369
|
+
import { createHash as createHash5 } from "crypto";
|
|
3586
4370
|
import { createReadStream } from "fs";
|
|
3587
4371
|
import { stat as stat3 } from "fs/promises";
|
|
3588
4372
|
import { join as join5 } from "path";
|
|
@@ -3626,7 +4410,7 @@ async function fileDigest(path) {
|
|
|
3626
4410
|
const memo = digestMemo.get(key);
|
|
3627
4411
|
if (memo) return memo;
|
|
3628
4412
|
const digest = await new Promise((resolve5, reject) => {
|
|
3629
|
-
const hash =
|
|
4413
|
+
const hash = createHash5("sha256");
|
|
3630
4414
|
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
3631
4415
|
});
|
|
3632
4416
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
@@ -3771,11 +4555,13 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3771
4555
|
const [pipe, tokenizer] = await Promise.all([
|
|
3772
4556
|
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3773
4557
|
...modelOptions,
|
|
4558
|
+
...opts.localFilesOnly ? { local_files_only: true } : {},
|
|
3774
4559
|
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
|
|
3775
4560
|
...progress
|
|
3776
4561
|
}),
|
|
3777
4562
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3778
4563
|
revision: modelOptions.revision,
|
|
4564
|
+
...opts.localFilesOnly ? { local_files_only: true } : {},
|
|
3779
4565
|
...progress
|
|
3780
4566
|
})
|
|
3781
4567
|
]);
|
|
@@ -3817,52 +4603,20 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3817
4603
|
// src/markdown-adapter/load.ts
|
|
3818
4604
|
import { existsSync } from "fs";
|
|
3819
4605
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
3820
|
-
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as
|
|
4606
|
+
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3821
4607
|
import {
|
|
3822
4608
|
CatalogDocumentSchema,
|
|
3823
4609
|
KnowledgeDocumentIdSchema,
|
|
3824
|
-
|
|
4610
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
|
|
4611
|
+
KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
|
|
4612
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
|
|
3825
4613
|
assetExtension as assetExtension3,
|
|
3826
4614
|
formatKnowledgeUri,
|
|
3827
4615
|
topicSortKeyForOrder
|
|
3828
4616
|
} from "@bendyline/gezk";
|
|
3829
4617
|
|
|
3830
|
-
// src/markdown-adapter/
|
|
3831
|
-
import {
|
|
3832
|
-
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
3833
|
-
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
3834
|
-
function parseYaml(text, opts) {
|
|
3835
|
-
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
3836
|
-
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
3837
|
-
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
3838
|
-
}
|
|
3839
|
-
const doc = parseDocument(text, {
|
|
3840
|
-
version: "1.2",
|
|
3841
|
-
schema: "core",
|
|
3842
|
-
uniqueKeys: true,
|
|
3843
|
-
logLevel: "silent"
|
|
3844
|
-
});
|
|
3845
|
-
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
3846
|
-
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
3847
|
-
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
3848
|
-
}
|
|
3849
|
-
var OPEN = /^---[ \t]*\r?\n/;
|
|
3850
|
-
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
3851
|
-
function parseMarkdownFrontMatter(source) {
|
|
3852
|
-
const open = OPEN.exec(source);
|
|
3853
|
-
if (!open) return { data: {}, body: source };
|
|
3854
|
-
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
3855
|
-
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
3856
|
-
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
3857
|
-
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
3858
|
-
const body = lines.slice(closeAt + 1).join("\n");
|
|
3859
|
-
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
3860
|
-
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
3861
|
-
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3862
|
-
throw new Error("front matter must be a mapping");
|
|
3863
|
-
}
|
|
3864
|
-
return { data: parsed, body };
|
|
3865
|
-
}
|
|
4618
|
+
// src/markdown-adapter/docfx.ts
|
|
4619
|
+
import { posix as posix2 } from "path";
|
|
3866
4620
|
|
|
3867
4621
|
// src/markdown-adapter/outline.ts
|
|
3868
4622
|
import { posix } from "path";
|
|
@@ -4159,14 +4913,408 @@ function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
|
|
|
4159
4913
|
return outline;
|
|
4160
4914
|
}
|
|
4161
4915
|
|
|
4916
|
+
// src/markdown-adapter/docfx.ts
|
|
4917
|
+
var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
|
|
4918
|
+
var MARKDOWN_EXT2 = /\.(md|markdown)$/i;
|
|
4919
|
+
var TOC_FILE = /^toc\.(ya?ml|json|md)$/i;
|
|
4920
|
+
var LANDING_PAGE = /(^|\/)index\.(ya?ml|md)$/i;
|
|
4921
|
+
var WARNING_EXAMPLES = 5;
|
|
4922
|
+
function isRecord2(value) {
|
|
4923
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4924
|
+
}
|
|
4925
|
+
function normalizeFolder(value) {
|
|
4926
|
+
if (typeof value !== "string") return "";
|
|
4927
|
+
const cleaned = posix2.normalize(value.replace(/\\/g, "/")).replace(/^\.?\/+|\/+$/g, "");
|
|
4928
|
+
return cleaned === "." ? "" : cleaned;
|
|
4929
|
+
}
|
|
4930
|
+
function dirOf(file) {
|
|
4931
|
+
const dir = posix2.dirname(file);
|
|
4932
|
+
return dir === "." ? "" : dir;
|
|
4933
|
+
}
|
|
4934
|
+
function parentOf(dir) {
|
|
4935
|
+
return dir === "" ? null : dirOf(dir);
|
|
4936
|
+
}
|
|
4937
|
+
function isUnder(file, dir) {
|
|
4938
|
+
return dir === "" || file.startsWith(`${dir}/`);
|
|
4939
|
+
}
|
|
4940
|
+
function escapeRegExp(text) {
|
|
4941
|
+
return text.replace(/[.+^$()|[\]\\]/g, "\\$&");
|
|
4942
|
+
}
|
|
4943
|
+
function docfxGlob(pattern) {
|
|
4944
|
+
const p = pattern.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
4945
|
+
let source = "";
|
|
4946
|
+
for (let i = 0; i < p.length; i++) {
|
|
4947
|
+
const ch = p[i];
|
|
4948
|
+
if (ch === "*") {
|
|
4949
|
+
if (p[i + 1] === "*") {
|
|
4950
|
+
if (p[i + 2] === "/") {
|
|
4951
|
+
source += "(?:.*/)?";
|
|
4952
|
+
i += 2;
|
|
4953
|
+
} else {
|
|
4954
|
+
source += ".*";
|
|
4955
|
+
i += 1;
|
|
4956
|
+
}
|
|
4957
|
+
} else source += "[^/]*";
|
|
4958
|
+
} else if (ch === "?") {
|
|
4959
|
+
source += "[^/]";
|
|
4960
|
+
} else if (ch === "{" && p.indexOf("}", i) > i) {
|
|
4961
|
+
const close = p.indexOf("}", i);
|
|
4962
|
+
source += `(?:${p.slice(i + 1, close).split(",").map(escapeRegExp).join("|")})`;
|
|
4963
|
+
i = close;
|
|
4964
|
+
} else {
|
|
4965
|
+
source += escapeRegExp(ch);
|
|
4966
|
+
}
|
|
4967
|
+
}
|
|
4968
|
+
return new RegExp(`^${source}$`, "i");
|
|
4969
|
+
}
|
|
4970
|
+
function globList(value) {
|
|
4971
|
+
const list = typeof value === "string" ? [value] : Array.isArray(value) ? value : [];
|
|
4972
|
+
return list.filter((g) => typeof g === "string" && g.trim() !== "").map(docfxGlob);
|
|
4973
|
+
}
|
|
4974
|
+
function parseDocfxJson(config, configFile = "docfx.json") {
|
|
4975
|
+
if (!isRecord2(config)) throw new Error(`${configFile}: not a JSON object`);
|
|
4976
|
+
const build = isRecord2(config.build) ? config.build : {};
|
|
4977
|
+
const content = [];
|
|
4978
|
+
for (const group of Array.isArray(build.content) ? build.content : []) {
|
|
4979
|
+
if (!isRecord2(group)) continue;
|
|
4980
|
+
const include = globList(group.files);
|
|
4981
|
+
if (include.length === 0) continue;
|
|
4982
|
+
content.push({
|
|
4983
|
+
src: normalizeFolder(group.src),
|
|
4984
|
+
dest: normalizeFolder(group.dest),
|
|
4985
|
+
include,
|
|
4986
|
+
exclude: globList(group.exclude)
|
|
4987
|
+
});
|
|
4988
|
+
}
|
|
4989
|
+
const meta = isRecord2(build.globalMetadata) ? build.globalMetadata : {};
|
|
4990
|
+
const breadcrumb = meta.breadcrumb_path;
|
|
4991
|
+
return {
|
|
4992
|
+
content,
|
|
4993
|
+
...typeof breadcrumb === "string" && breadcrumb.trim() ? { breadcrumbPath: breadcrumb.trim() } : {}
|
|
4994
|
+
};
|
|
4995
|
+
}
|
|
4996
|
+
function docfxPublishes(project, file) {
|
|
4997
|
+
if (project.content.length === 0) return true;
|
|
4998
|
+
return project.content.some((group) => {
|
|
4999
|
+
if (!isUnder(file, group.src)) return false;
|
|
5000
|
+
const rel = group.src === "" ? file : file.slice(group.src.length + 1);
|
|
5001
|
+
return group.include.some((r) => r.test(rel)) && !group.exclude.some((r) => r.test(rel));
|
|
5002
|
+
});
|
|
5003
|
+
}
|
|
5004
|
+
function tocItems(doc) {
|
|
5005
|
+
if (Array.isArray(doc)) return doc;
|
|
5006
|
+
if (isRecord2(doc) && Array.isArray(doc.items)) return doc.items;
|
|
5007
|
+
return [];
|
|
5008
|
+
}
|
|
5009
|
+
function isBreadcrumbToc(doc) {
|
|
5010
|
+
let tocHref = false;
|
|
5011
|
+
const visit = (items) => {
|
|
5012
|
+
for (const item of items) {
|
|
5013
|
+
if (!isRecord2(item)) continue;
|
|
5014
|
+
if (typeof item.href === "string") return false;
|
|
5015
|
+
if (typeof item.tocHref === "string") tocHref = true;
|
|
5016
|
+
if (Array.isArray(item.items) && !visit(item.items)) return false;
|
|
5017
|
+
}
|
|
5018
|
+
return true;
|
|
5019
|
+
};
|
|
5020
|
+
return visit(tocItems(doc)) && tocHref;
|
|
5021
|
+
}
|
|
5022
|
+
function basePrefixes(url) {
|
|
5023
|
+
const segments = url.split("/").filter(Boolean);
|
|
5024
|
+
const out = ["/"];
|
|
5025
|
+
for (let i = 1; i < segments.length; i++) out.push(`/${segments.slice(0, i).join("/")}/`);
|
|
5026
|
+
return out;
|
|
5027
|
+
}
|
|
5028
|
+
function parseDocfxToc(input) {
|
|
5029
|
+
const outline = {
|
|
5030
|
+
format: "docfx",
|
|
5031
|
+
root: newOutlineTopic(""),
|
|
5032
|
+
consumed: input.files.filter((file) => TOC_FILE.test(posix2.basename(file))),
|
|
5033
|
+
warnings: []
|
|
5034
|
+
};
|
|
5035
|
+
const known = new Set(input.files.map((file) => file.normalize("NFC")));
|
|
5036
|
+
const groups = input.project?.content.length ? [...input.project.content].sort((a, b) => b.dest.length - a.dest.length) : [{ src: "", dest: "", include: [], exclude: [] }];
|
|
5037
|
+
const tocByDir = /* @__PURE__ */ new Map();
|
|
5038
|
+
for (const file of [...input.tocs.keys()].sort()) {
|
|
5039
|
+
if (/\.ya?ml$/i.test(file) && !tocByDir.has(dirOf(file))) tocByDir.set(dirOf(file), file);
|
|
5040
|
+
}
|
|
5041
|
+
const sourcesOf = (sitePath) => {
|
|
5042
|
+
const out = [];
|
|
5043
|
+
for (const group of groups) {
|
|
5044
|
+
if (group.dest !== "" && sitePath !== group.dest && !sitePath.startsWith(`${group.dest}/`))
|
|
5045
|
+
continue;
|
|
5046
|
+
const rest = sitePath.slice(group.dest.length).replace(/^\/+/, "");
|
|
5047
|
+
out.push(group.src && rest ? `${group.src}/${rest}` : group.src || rest);
|
|
5048
|
+
}
|
|
5049
|
+
return out;
|
|
5050
|
+
};
|
|
5051
|
+
const tocAt = (path) => {
|
|
5052
|
+
const trimmed = path.replace(/\/+$/, "");
|
|
5053
|
+
return TOC_FILE.test(posix2.basename(trimmed)) ? tocByDir.get(dirOf(trimmed)) : tocByDir.get(trimmed);
|
|
5054
|
+
};
|
|
5055
|
+
const pageAt = (path) => {
|
|
5056
|
+
const candidates = path === "" || path.endsWith("/") ? [`${path}index.md`] : MARKDOWN_EXT2.test(path) ? [path] : [`${path}.md`, `${path}/index.md`];
|
|
5057
|
+
return candidates.find((candidate) => known.has(candidate.normalize("NFC")));
|
|
5058
|
+
};
|
|
5059
|
+
const resolveSite = (url, base2) => {
|
|
5060
|
+
const path = url.replace(/[?#].*$/, "");
|
|
5061
|
+
let below;
|
|
5062
|
+
if (`${path}/` === base2) below = "";
|
|
5063
|
+
else if (path.startsWith(base2)) below = path.slice(base2.length);
|
|
5064
|
+
else return null;
|
|
5065
|
+
for (const source of sourcesOf(below)) {
|
|
5066
|
+
const page = pageAt(source);
|
|
5067
|
+
if (page) return { kind: "page", file: page };
|
|
5068
|
+
const toc = tocAt(source);
|
|
5069
|
+
if (toc) return { kind: "toc", file: toc };
|
|
5070
|
+
}
|
|
5071
|
+
return null;
|
|
5072
|
+
};
|
|
5073
|
+
let entry = input.entry;
|
|
5074
|
+
let base;
|
|
5075
|
+
const breadcrumb = input.project?.breadcrumbPath;
|
|
5076
|
+
if (!entry && breadcrumb) {
|
|
5077
|
+
for (const candidate of basePrefixes(breadcrumb)) {
|
|
5078
|
+
const target = resolveSite(breadcrumb, candidate);
|
|
5079
|
+
if (target?.kind === "toc") {
|
|
5080
|
+
entry = target.file;
|
|
5081
|
+
base = candidate;
|
|
5082
|
+
break;
|
|
5083
|
+
}
|
|
5084
|
+
}
|
|
5085
|
+
if (!entry) {
|
|
5086
|
+
outline.warnings.push(
|
|
5087
|
+
`docfx.json: breadcrumb_path '${breadcrumb}' names no toc.yml in the tree; ignored`
|
|
5088
|
+
);
|
|
5089
|
+
}
|
|
5090
|
+
}
|
|
5091
|
+
entry ??= tocByDir.get("");
|
|
5092
|
+
if (entry && base === void 0) {
|
|
5093
|
+
const urls = [];
|
|
5094
|
+
const collect = (items) => {
|
|
5095
|
+
for (const item of items) {
|
|
5096
|
+
if (!isRecord2(item)) continue;
|
|
5097
|
+
for (const key of ["tocHref", "topicHref", "href"]) {
|
|
5098
|
+
const value = item[key];
|
|
5099
|
+
if (typeof value === "string" && value.startsWith("/") && !value.startsWith("//")) {
|
|
5100
|
+
urls.push(value);
|
|
5101
|
+
}
|
|
5102
|
+
}
|
|
5103
|
+
if (Array.isArray(item.items)) collect(item.items);
|
|
5104
|
+
}
|
|
5105
|
+
};
|
|
5106
|
+
collect(tocItems(input.tocs.get(entry)));
|
|
5107
|
+
let best = { base: "/", hits: 0 };
|
|
5108
|
+
for (const candidate of [...new Set(urls.flatMap(basePrefixes))].sort(
|
|
5109
|
+
(a, b) => a.length - b.length
|
|
5110
|
+
)) {
|
|
5111
|
+
const hits = urls.filter((url) => resolveSite(url, candidate) !== null).length;
|
|
5112
|
+
if (hits > best.hits) best = { base: candidate, hits };
|
|
5113
|
+
}
|
|
5114
|
+
base = best.base;
|
|
5115
|
+
}
|
|
5116
|
+
const siteBase = base ?? "/";
|
|
5117
|
+
const classify = (href, tocDir) => {
|
|
5118
|
+
const value = href.trim();
|
|
5119
|
+
if (value === "" || value.startsWith("#") || HAS_SCHEME2.test(value) || value.startsWith("//"))
|
|
5120
|
+
return null;
|
|
5121
|
+
if (value.startsWith("/")) return resolveSite(value, siteBase);
|
|
5122
|
+
const path = resolveOutlinePath(tocDir, value);
|
|
5123
|
+
if (path === null) return null;
|
|
5124
|
+
if (MARKDOWN_EXT2.test(path)) return { kind: "page", file: path };
|
|
5125
|
+
const toc = tocAt(path);
|
|
5126
|
+
return toc ? { kind: "toc", file: toc } : null;
|
|
5127
|
+
};
|
|
5128
|
+
const crossLinks = /* @__PURE__ */ new WeakSet();
|
|
5129
|
+
const included = /* @__PURE__ */ new Set();
|
|
5130
|
+
const homes = /* @__PURE__ */ new Map();
|
|
5131
|
+
const pageEntry = (file, tocDir) => {
|
|
5132
|
+
const entry2 = { file };
|
|
5133
|
+
if (!isUnder(file, tocDir)) crossLinks.add(entry2);
|
|
5134
|
+
return entry2;
|
|
5135
|
+
};
|
|
5136
|
+
const walkItems = (items, tocFile, container, followTocHref) => {
|
|
5137
|
+
const tocDir = dirOf(tocFile);
|
|
5138
|
+
for (const item of items) {
|
|
5139
|
+
if (!isRecord2(item)) continue;
|
|
5140
|
+
const name = typeof item.name === "string" ? item.name.trim() : "";
|
|
5141
|
+
const href = typeof item.href === "string" ? classify(item.href, tocDir) : null;
|
|
5142
|
+
const topicHref = typeof item.topicHref === "string" ? classify(item.topicHref, tocDir) : null;
|
|
5143
|
+
const tocHref = followTocHref && typeof item.tocHref === "string" ? classify(item.tocHref, tocDir) : null;
|
|
5144
|
+
const page = [href, topicHref].find((target) => target?.kind === "page");
|
|
5145
|
+
const nested = [href, tocHref].find(
|
|
5146
|
+
(target) => target?.kind === "toc" && !included.has(target.file)
|
|
5147
|
+
);
|
|
5148
|
+
const children = Array.isArray(item.items) && item.items.length > 0 ? item.items : null;
|
|
5149
|
+
if (!nested && !children) {
|
|
5150
|
+
if (page) container.entries.push(pageEntry(page.file, tocDir));
|
|
5151
|
+
continue;
|
|
5152
|
+
}
|
|
5153
|
+
const topic = newOutlineTopic(
|
|
5154
|
+
name || (nested ? titleFromFolderName(posix2.basename(dirOf(nested.file))) : "Untitled")
|
|
5155
|
+
);
|
|
5156
|
+
if (!name && page) topic.titleFromFile = page.file;
|
|
5157
|
+
if (page) topic.entries.push(pageEntry(page.file, tocDir));
|
|
5158
|
+
if (nested) includeToc(nested.file, topic, false);
|
|
5159
|
+
if (children) walkItems(children, tocFile, topic, followTocHref);
|
|
5160
|
+
container.children.push(topic);
|
|
5161
|
+
}
|
|
5162
|
+
};
|
|
5163
|
+
const includeToc = (file, container, followTocHref) => {
|
|
5164
|
+
if (included.has(file)) return;
|
|
5165
|
+
included.add(file);
|
|
5166
|
+
const dir = dirOf(file);
|
|
5167
|
+
if (container !== outline.root && !homes.has(dir)) homes.set(dir, container);
|
|
5168
|
+
walkItems(tocItems(input.tocs.get(file)), file, container, followTocHref);
|
|
5169
|
+
};
|
|
5170
|
+
const homeOf = (dir) => {
|
|
5171
|
+
for (let d = dir; d !== null; d = parentOf(d)) {
|
|
5172
|
+
const home = homes.get(d);
|
|
5173
|
+
if (home) return home;
|
|
5174
|
+
}
|
|
5175
|
+
return void 0;
|
|
5176
|
+
};
|
|
5177
|
+
if (entry) {
|
|
5178
|
+
outline.entry = entry;
|
|
5179
|
+
includeToc(entry, outline.root, true);
|
|
5180
|
+
}
|
|
5181
|
+
const lone = outline.root.entries.length === 0 && outline.root.children.length === 1 ? outline.root.children[0] : void 0;
|
|
5182
|
+
const top = lone ?? outline.root;
|
|
5183
|
+
const orphans = [...input.tocs.keys()].filter((file) => !included.has(file) && /\.ya?ml$/i.test(file)).sort((a, b) => a.split("/").length - b.split("/").length || (a < b ? -1 : a > b ? 1 : 0));
|
|
5184
|
+
for (const file of orphans) {
|
|
5185
|
+
if (included.has(file)) continue;
|
|
5186
|
+
const doc = input.tocs.get(file);
|
|
5187
|
+
if (isBreadcrumbToc(doc)) continue;
|
|
5188
|
+
const parent = parentOf(dirOf(file));
|
|
5189
|
+
const topic = newOutlineTopic(orphanTopicName(file, doc));
|
|
5190
|
+
((parent === null ? void 0 : homeOf(parent)) ?? top).children.push(topic);
|
|
5191
|
+
includeToc(file, topic, false);
|
|
5192
|
+
}
|
|
5193
|
+
const listed = /* @__PURE__ */ new Set();
|
|
5194
|
+
const collectListed = (topic) => {
|
|
5195
|
+
for (const entry2 of topic.entries) {
|
|
5196
|
+
if (!crossLinks.has(entry2)) listed.add(entry2.file.normalize("NFC"));
|
|
5197
|
+
}
|
|
5198
|
+
topic.children.forEach(collectListed);
|
|
5199
|
+
};
|
|
5200
|
+
collectListed(outline.root);
|
|
5201
|
+
const placed = /* @__PURE__ */ new Set();
|
|
5202
|
+
const missing = [];
|
|
5203
|
+
const duplicates = [];
|
|
5204
|
+
const settle = (topic) => {
|
|
5205
|
+
topic.entries = topic.entries.filter((entry2) => {
|
|
5206
|
+
const key = entry2.file.normalize("NFC");
|
|
5207
|
+
if (!known.has(key)) {
|
|
5208
|
+
missing.push(entry2.file);
|
|
5209
|
+
return false;
|
|
5210
|
+
}
|
|
5211
|
+
if (crossLinks.has(entry2) && listed.has(key)) return false;
|
|
5212
|
+
if (placed.has(key)) {
|
|
5213
|
+
if (!crossLinks.has(entry2)) duplicates.push(entry2.file);
|
|
5214
|
+
return false;
|
|
5215
|
+
}
|
|
5216
|
+
placed.add(key);
|
|
5217
|
+
return true;
|
|
5218
|
+
});
|
|
5219
|
+
topic.children.forEach(settle);
|
|
5220
|
+
};
|
|
5221
|
+
settle(outline.root);
|
|
5222
|
+
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
5223
|
+
const homed = [];
|
|
5224
|
+
for (const file of input.files) {
|
|
5225
|
+
const key = file.normalize("NFC");
|
|
5226
|
+
if (placed.has(key) || consumed.has(key)) continue;
|
|
5227
|
+
const home = homeOf(dirOf(file));
|
|
5228
|
+
if (!home) continue;
|
|
5229
|
+
home.entries.push({ file });
|
|
5230
|
+
homed.push(file);
|
|
5231
|
+
}
|
|
5232
|
+
if (lone) {
|
|
5233
|
+
outline.root.entries = lone.entries;
|
|
5234
|
+
outline.root.children = lone.children;
|
|
5235
|
+
}
|
|
5236
|
+
assignOutlineOrders(outline.root);
|
|
5237
|
+
const report = (files, what) => {
|
|
5238
|
+
if (files.length === 0) return;
|
|
5239
|
+
const named = files.slice(0, WARNING_EXAMPLES).join(", ");
|
|
5240
|
+
const more = files.length > WARNING_EXAMPLES ? `, and ${files.length - WARNING_EXAMPLES} more` : "";
|
|
5241
|
+
outline.warnings.push(`${files.length} ${what}: ${named}${more}`);
|
|
5242
|
+
};
|
|
5243
|
+
report(
|
|
5244
|
+
[...new Set(missing)],
|
|
5245
|
+
"pages named by a toc.yml are not among the published Markdown files"
|
|
5246
|
+
);
|
|
5247
|
+
report([...new Set(duplicates)], "pages are listed more than once; the first place wins");
|
|
5248
|
+
report(homed, "pages are in no toc.yml; filed with their folder's section");
|
|
5249
|
+
return outline;
|
|
5250
|
+
}
|
|
5251
|
+
function orphanTopicName(file, doc) {
|
|
5252
|
+
for (const item of tocItems(doc)) {
|
|
5253
|
+
if (!isRecord2(item) || typeof item.name !== "string" || typeof item.href !== "string") continue;
|
|
5254
|
+
if (!LANDING_PAGE.test(item.href.replace(/[?#].*$/, ""))) continue;
|
|
5255
|
+
const name = item.name.trim().replace(/\s+(documentation|docs)$/i, "").trim();
|
|
5256
|
+
if (name) return name;
|
|
5257
|
+
}
|
|
5258
|
+
const folder = posix2.basename(dirOf(file));
|
|
5259
|
+
return folder ? titleFromFolderName(folder) : "Documentation";
|
|
5260
|
+
}
|
|
5261
|
+
|
|
5262
|
+
// src/markdown-adapter/frontmatter.ts
|
|
5263
|
+
import { parseDocument } from "yaml";
|
|
5264
|
+
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
5265
|
+
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
5266
|
+
function parseYaml(text, opts) {
|
|
5267
|
+
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
5268
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
5269
|
+
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
5270
|
+
}
|
|
5271
|
+
const doc = parseDocument(text, {
|
|
5272
|
+
version: "1.2",
|
|
5273
|
+
schema: "core",
|
|
5274
|
+
uniqueKeys: true,
|
|
5275
|
+
logLevel: "silent"
|
|
5276
|
+
});
|
|
5277
|
+
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
5278
|
+
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
5279
|
+
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
5280
|
+
}
|
|
5281
|
+
var OPEN = /^---[ \t]*\r?\n/;
|
|
5282
|
+
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
5283
|
+
function parseMarkdownFrontMatter(source) {
|
|
5284
|
+
const open = OPEN.exec(source);
|
|
5285
|
+
if (!open) return { data: {}, body: source };
|
|
5286
|
+
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
5287
|
+
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
5288
|
+
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
5289
|
+
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
5290
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
5291
|
+
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
5292
|
+
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
5293
|
+
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
5294
|
+
throw new Error("front matter must be a mapping");
|
|
5295
|
+
}
|
|
5296
|
+
return { data: parsed, body };
|
|
5297
|
+
}
|
|
5298
|
+
|
|
4162
5299
|
// src/markdown-adapter/load.ts
|
|
4163
5300
|
var ROOT_TOPIC_ID = "general";
|
|
4164
5301
|
var SUMMARY_MAX_CHARS = 280;
|
|
4165
5302
|
var TOPIC_SIDECAR = "_topic.yaml";
|
|
4166
5303
|
var HUGO_SECTION_PAGE = "_index";
|
|
5304
|
+
var DOCFX_JSON = "docfx.json";
|
|
5305
|
+
var WARNING_EXAMPLES2 = 20;
|
|
4167
5306
|
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
4168
|
-
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
4169
|
-
|
|
5307
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
5308
|
+
"title",
|
|
5309
|
+
"summary",
|
|
5310
|
+
"aliases",
|
|
5311
|
+
"id",
|
|
5312
|
+
"order",
|
|
5313
|
+
"subcategory",
|
|
5314
|
+
"locations"
|
|
5315
|
+
]);
|
|
5316
|
+
function readFrontMatter(raw, file, conventions) {
|
|
5317
|
+
const { hugo } = conventions;
|
|
4170
5318
|
let data;
|
|
4171
5319
|
let body;
|
|
4172
5320
|
try {
|
|
@@ -4183,6 +5331,12 @@ function readFrontMatter(raw, file, hugo) {
|
|
|
4183
5331
|
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
4184
5332
|
return value.trim() || void 0;
|
|
4185
5333
|
};
|
|
5334
|
+
if (data.locations !== void 0) {
|
|
5335
|
+
const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
|
|
5336
|
+
if (!locations.success)
|
|
5337
|
+
throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
|
|
5338
|
+
out.locations = locations.data;
|
|
5339
|
+
}
|
|
4186
5340
|
out.title = str("title");
|
|
4187
5341
|
out.summary = str("summary");
|
|
4188
5342
|
out.id = str("id");
|
|
@@ -4195,7 +5349,7 @@ function readFrontMatter(raw, file, hugo) {
|
|
|
4195
5349
|
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
4196
5350
|
}
|
|
4197
5351
|
if (data.order !== void 0 && data.order !== null) {
|
|
4198
|
-
const order =
|
|
5352
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4199
5353
|
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
4200
5354
|
out.order = order.data;
|
|
4201
5355
|
}
|
|
@@ -4216,7 +5370,7 @@ function readFrontMatter(raw, file, hugo) {
|
|
|
4216
5370
|
subcategory.title = record.title.trim();
|
|
4217
5371
|
}
|
|
4218
5372
|
if (record.order !== void 0 && record.order !== null) {
|
|
4219
|
-
const order =
|
|
5373
|
+
const order = KnowledgeOrdinalSchema2.safeParse(record.order);
|
|
4220
5374
|
if (!order.success)
|
|
4221
5375
|
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
4222
5376
|
subcategory.order = order.data;
|
|
@@ -4227,12 +5381,12 @@ function readFrontMatter(raw, file, hugo) {
|
|
|
4227
5381
|
if (data.draft === true) out.unpublished = "draft";
|
|
4228
5382
|
else if (data.headless === true) out.unpublished = "headless";
|
|
4229
5383
|
if (data.weight !== void 0 && data.weight !== null) {
|
|
4230
|
-
const weight =
|
|
5384
|
+
const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
|
|
4231
5385
|
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
4232
5386
|
if (weight.data > 0) out.weight = weight.data;
|
|
4233
5387
|
}
|
|
4234
|
-
out.description = str("description");
|
|
4235
5388
|
}
|
|
5389
|
+
if (conventions.description) out.description = str("description");
|
|
4236
5390
|
for (const [key, value] of Object.entries(data)) {
|
|
4237
5391
|
if (RESERVED_KEYS.has(key) || value === void 0) continue;
|
|
4238
5392
|
out.meta[key] = value;
|
|
@@ -4261,7 +5415,7 @@ function topicIdFor(name, taken) {
|
|
|
4261
5415
|
taken.add(id);
|
|
4262
5416
|
return id;
|
|
4263
5417
|
}
|
|
4264
|
-
async function
|
|
5418
|
+
async function walkFiles(root, matches) {
|
|
4265
5419
|
const out = [];
|
|
4266
5420
|
const visit = async (dir) => {
|
|
4267
5421
|
const entries = (await readdir(dir, { withFileTypes: true })).sort(
|
|
@@ -4271,13 +5425,20 @@ async function walkMarkdownFiles(root) {
|
|
|
4271
5425
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
4272
5426
|
const abs = join6(dir, entry.name);
|
|
4273
5427
|
if (entry.isDirectory()) await visit(abs);
|
|
4274
|
-
else if (entry.isFile() &&
|
|
5428
|
+
else if (entry.isFile() && matches(entry.name)) out.push(abs);
|
|
4275
5429
|
}
|
|
4276
5430
|
};
|
|
4277
5431
|
await visit(root);
|
|
4278
5432
|
return out;
|
|
4279
5433
|
}
|
|
4280
|
-
function
|
|
5434
|
+
function walkMarkdownFiles(root) {
|
|
5435
|
+
return walkFiles(root, (name) => /\.(md|markdown)$/i.test(name));
|
|
5436
|
+
}
|
|
5437
|
+
function warnCapped(warn, items, each, rest) {
|
|
5438
|
+
for (const item of items.slice(0, WARNING_EXAMPLES2)) warn(each(item));
|
|
5439
|
+
if (items.length > WARNING_EXAMPLES2) warn(rest(items.length - WARNING_EXAMPLES2));
|
|
5440
|
+
}
|
|
5441
|
+
function isRecord3(value) {
|
|
4281
5442
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4282
5443
|
}
|
|
4283
5444
|
async function readTopicSidecar(dir) {
|
|
@@ -4286,14 +5447,14 @@ async function readTopicSidecar(dir) {
|
|
|
4286
5447
|
const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
4287
5448
|
const data = parseYaml(raw, { what: path });
|
|
4288
5449
|
if (data === null || data === void 0) return {};
|
|
4289
|
-
if (!
|
|
5450
|
+
if (!isRecord3(data)) throw new Error(`${path}: must be a mapping`);
|
|
4290
5451
|
const out = {};
|
|
4291
5452
|
if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
|
|
4292
5453
|
if (typeof data.description === "string" && data.description.trim()) {
|
|
4293
5454
|
out.description = data.description.trim();
|
|
4294
5455
|
}
|
|
4295
5456
|
if (data.order !== void 0 && data.order !== null) {
|
|
4296
|
-
const order =
|
|
5457
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4297
5458
|
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
4298
5459
|
out.order = order.data;
|
|
4299
5460
|
}
|
|
@@ -4320,7 +5481,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
|
|
|
4320
5481
|
tolerateUnknownTags: true,
|
|
4321
5482
|
maxAliasCount: 100
|
|
4322
5483
|
});
|
|
4323
|
-
const docsDir =
|
|
5484
|
+
const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
4324
5485
|
return resolve4(dirname3(mkdocsPath), docsDir);
|
|
4325
5486
|
}
|
|
4326
5487
|
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
@@ -4335,6 +5496,10 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
|
4335
5496
|
])
|
|
4336
5497
|
);
|
|
4337
5498
|
if (mkdocs) return { format: "mkdocs", path: mkdocs };
|
|
5499
|
+
const docfx = firstExisting(
|
|
5500
|
+
[DOCFX_JSON, "toc.yml", "TOC.yml", "toc.yaml"].map((name) => join6(rootDir, name))
|
|
5501
|
+
);
|
|
5502
|
+
if (docfx) return { format: "docfx" };
|
|
4338
5503
|
const files = await walkMarkdownFiles(rootDir);
|
|
4339
5504
|
if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
|
|
4340
5505
|
return { format: "hugo" };
|
|
@@ -4354,6 +5519,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4354
5519
|
if (explicit && !existsSync(explicit)) {
|
|
4355
5520
|
throw new Error(`${toc.format} table of contents not found: ${explicit}`);
|
|
4356
5521
|
}
|
|
5522
|
+
if (toc.format === "docfx") return loadDocfxOutline(rootDir, explicit, files);
|
|
4357
5523
|
if (toc.format === "gitbook") {
|
|
4358
5524
|
const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
|
|
4359
5525
|
const rel = insideRoot(rootDir, path2);
|
|
@@ -4396,16 +5562,68 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4396
5562
|
});
|
|
4397
5563
|
return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
|
|
4398
5564
|
}
|
|
5565
|
+
async function loadDocfxOutline(rootDir, explicit, files) {
|
|
5566
|
+
const configPath = join6(rootDir, DOCFX_JSON);
|
|
5567
|
+
let project;
|
|
5568
|
+
if (existsSync(configPath)) {
|
|
5569
|
+
let config;
|
|
5570
|
+
try {
|
|
5571
|
+
config = JSON.parse(await readOutlineText(configPath));
|
|
5572
|
+
} catch (error) {
|
|
5573
|
+
throw new Error(`${DOCFX_JSON}: ${error instanceof Error ? error.message : String(error)}`);
|
|
5574
|
+
}
|
|
5575
|
+
project = parseDocfxJson(config, DOCFX_JSON);
|
|
5576
|
+
}
|
|
5577
|
+
const entry = explicit === void 0 ? void 0 : insideRoot(rootDir, explicit);
|
|
5578
|
+
if (entry === null) {
|
|
5579
|
+
throw new Error(`${basename(explicit)} must live inside the content root ${rootDir}`);
|
|
5580
|
+
}
|
|
5581
|
+
const readToc = async (abs, rel) => parseYaml(await readOutlineText(abs), {
|
|
5582
|
+
what: rel,
|
|
5583
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
5584
|
+
tolerateUnknownTags: true,
|
|
5585
|
+
maxAliasCount: 100
|
|
5586
|
+
});
|
|
5587
|
+
const tocs = /* @__PURE__ */ new Map();
|
|
5588
|
+
const warnings = [];
|
|
5589
|
+
for (const abs of await walkFiles(rootDir, (name) => /^toc\.ya?ml$/i.test(name))) {
|
|
5590
|
+
const rel = insideRoot(rootDir, abs);
|
|
5591
|
+
try {
|
|
5592
|
+
tocs.set(rel, await readToc(abs, rel));
|
|
5593
|
+
} catch (error) {
|
|
5594
|
+
if (rel === entry) throw error;
|
|
5595
|
+
warnings.push(`${rel}: ${error instanceof Error ? error.message : String(error)}; skipped`);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
if (entry !== void 0 && explicit !== void 0 && !tocs.has(entry)) {
|
|
5599
|
+
tocs.set(entry, await readToc(explicit, entry));
|
|
5600
|
+
}
|
|
5601
|
+
const publishes = project ? (file) => docfxPublishes(project, file) : void 0;
|
|
5602
|
+
const outline = parseDocfxToc({
|
|
5603
|
+
tocs,
|
|
5604
|
+
files: publishes ? files.filter(publishes) : files,
|
|
5605
|
+
...project ? { project } : {},
|
|
5606
|
+
...entry !== void 0 ? { entry } : {}
|
|
5607
|
+
});
|
|
5608
|
+
outline.warnings.unshift(...warnings);
|
|
5609
|
+
const path = outline.entry ? join6(rootDir, ...outline.entry.split("/")) : void 0;
|
|
5610
|
+
return {
|
|
5611
|
+
outline,
|
|
5612
|
+
...path ? { path } : project ? { path: configPath } : {},
|
|
5613
|
+
...publishes ? { publishes } : {}
|
|
5614
|
+
};
|
|
5615
|
+
}
|
|
4399
5616
|
var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
|
|
4400
|
-
var
|
|
5617
|
+
var HAS_SCHEME3 = /^[a-z][a-z0-9+.-]*:/i;
|
|
4401
5618
|
function resolveInTree(ctx, target) {
|
|
4402
|
-
const
|
|
4403
|
-
|
|
5619
|
+
const fromRoot = ctx.tildeIsRoot && target.startsWith("~/");
|
|
5620
|
+
const resolved = posix3.normalize(fromRoot ? target.slice(2) : posix3.join(ctx.docDir, target));
|
|
5621
|
+
if (resolved === ".." || resolved.startsWith("../") || posix3.isAbsolute(resolved)) return null;
|
|
4404
5622
|
return resolved === "." ? "" : resolved;
|
|
4405
5623
|
}
|
|
4406
5624
|
function rewriteTarget(ctx, isImage, rawTarget) {
|
|
4407
5625
|
const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
|
|
4408
|
-
if (target === "" ||
|
|
5626
|
+
if (target === "" || HAS_SCHEME3.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
|
|
4409
5627
|
return null;
|
|
4410
5628
|
}
|
|
4411
5629
|
const hashAt = target.search(/[#?]/);
|
|
@@ -4468,6 +5686,7 @@ function rewriteLinks(ctx, markdown) {
|
|
|
4468
5686
|
return line.replace(
|
|
4469
5687
|
INLINE_LINK,
|
|
4470
5688
|
(whole, bang, text, target, title) => {
|
|
5689
|
+
if (bang === "!" && ctx.skipImages) return text;
|
|
4471
5690
|
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
4472
5691
|
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
4473
5692
|
}
|
|
@@ -4479,11 +5698,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4479
5698
|
});
|
|
4480
5699
|
const toc = opts.toc ?? { format: "folders" };
|
|
4481
5700
|
const hugo = toc.format === "hugo";
|
|
5701
|
+
const docfx = toc.format === "docfx";
|
|
5702
|
+
const conventions = { hugo, description: hugo || docfx };
|
|
4482
5703
|
const ignore = new Set(opts.ignore ?? []);
|
|
4483
5704
|
const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
|
|
4484
5705
|
let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
|
|
4485
|
-
const {
|
|
5706
|
+
const {
|
|
5707
|
+
outline,
|
|
5708
|
+
path: outlinePath,
|
|
5709
|
+
publishes
|
|
5710
|
+
} = await loadOutline(rootDir, toc, files.map(relOf));
|
|
4486
5711
|
for (const message of outline?.warnings ?? []) warn(message);
|
|
5712
|
+
if (publishes) files = files.filter((abs) => publishes(relOf(abs)));
|
|
4487
5713
|
if (outline) {
|
|
4488
5714
|
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
4489
5715
|
files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
|
|
@@ -4534,26 +5760,24 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4534
5760
|
const walk = (topic, chain) => {
|
|
4535
5761
|
for (const entry of topic.entries) {
|
|
4536
5762
|
const key = entry.file.normalize("NFC");
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
);
|
|
4541
|
-
} else {
|
|
4542
|
-
placements.set(key, { chain, entry });
|
|
4543
|
-
}
|
|
5763
|
+
const existing = placements.get(key) ?? [];
|
|
5764
|
+
existing.push({ chain, entry });
|
|
5765
|
+
placements.set(key, existing);
|
|
4544
5766
|
}
|
|
4545
5767
|
for (const child of topic.children) walk(child, [...chain, child]);
|
|
4546
5768
|
};
|
|
4547
5769
|
walk(outline.root, []);
|
|
4548
5770
|
}
|
|
4549
5771
|
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
5772
|
+
let outlineTooDeep = false;
|
|
4550
5773
|
const ensureOutlineChain = (chain) => {
|
|
4551
5774
|
if (chain.length === 0) {
|
|
4552
5775
|
rootTopicUsed = true;
|
|
4553
5776
|
return [rootTopicId];
|
|
4554
5777
|
}
|
|
5778
|
+
if (chain.length > MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1) outlineTooDeep = true;
|
|
4555
5779
|
const ids = [];
|
|
4556
|
-
for (const node of chain) {
|
|
5780
|
+
for (const node of chain.slice(0, MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1)) {
|
|
4557
5781
|
let id = outlineTopicIds.get(node);
|
|
4558
5782
|
if (!id) {
|
|
4559
5783
|
id = topicIdFor(node.name, takenTopicIds);
|
|
@@ -4594,12 +5818,13 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4594
5818
|
const fileById = /* @__PURE__ */ new Map();
|
|
4595
5819
|
const idByRel = /* @__PURE__ */ new Map();
|
|
4596
5820
|
const seenRel = /* @__PURE__ */ new Set();
|
|
5821
|
+
const unlisted = [];
|
|
4597
5822
|
for (const abs of files) {
|
|
4598
5823
|
const rel = relOf(abs);
|
|
4599
5824
|
const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
4600
5825
|
const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
4601
5826
|
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
4602
|
-
const fm = readFrontMatter(raw, rel,
|
|
5827
|
+
const fm = readFrontMatter(raw, rel, conventions);
|
|
4603
5828
|
if (fm.unpublished) {
|
|
4604
5829
|
warn(`${rel}: skipped, ${fm.unpublished} page`);
|
|
4605
5830
|
continue;
|
|
@@ -4613,17 +5838,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4613
5838
|
fileById.set(id, rel);
|
|
4614
5839
|
idByRel.set(relNoExt, id);
|
|
4615
5840
|
const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
4616
|
-
let title = fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
5841
|
+
let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
4617
5842
|
let ordinal = fm.order;
|
|
4618
5843
|
let topicPath;
|
|
4619
|
-
const
|
|
5844
|
+
const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
|
|
5845
|
+
const placement = documentPlacements[0];
|
|
4620
5846
|
if (placement) {
|
|
4621
5847
|
topicPath = ensureOutlineChain(placement.chain);
|
|
4622
5848
|
if (placement.entry.title) title = placement.entry.title;
|
|
4623
5849
|
if (placement.entry.order !== void 0) ordinal = placement.entry.order;
|
|
4624
5850
|
} else {
|
|
4625
|
-
if (outline)
|
|
4626
|
-
warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
|
|
5851
|
+
if (outline) unlisted.push(rel);
|
|
4627
5852
|
topicPath = await ensureTopicChain(relDir);
|
|
4628
5853
|
if (hugo && ordinal === void 0) ordinal = fm.weight;
|
|
4629
5854
|
}
|
|
@@ -4647,7 +5872,18 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4647
5872
|
const parentId = topicPath[topicPath.length - 1];
|
|
4648
5873
|
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
4649
5874
|
}
|
|
4650
|
-
const
|
|
5875
|
+
const tocReferences = [];
|
|
5876
|
+
const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
|
|
5877
|
+
for (const extra of documentPlacements.slice(1)) {
|
|
5878
|
+
let path = ensureOutlineChain(extra.chain);
|
|
5879
|
+
if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
|
|
5880
|
+
const leaf = path.at(-1);
|
|
5881
|
+
if (placedLeaves.has(leaf)) continue;
|
|
5882
|
+
placedLeaves.add(leaf);
|
|
5883
|
+
const order = extra.entry.order ?? fm.order;
|
|
5884
|
+
tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
|
|
5885
|
+
}
|
|
5886
|
+
const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
|
|
4651
5887
|
loaded.push({
|
|
4652
5888
|
rel,
|
|
4653
5889
|
file: rel,
|
|
@@ -4658,6 +5894,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4658
5894
|
...summary ? { summary } : {},
|
|
4659
5895
|
language: opts.language,
|
|
4660
5896
|
topicPath,
|
|
5897
|
+
...tocReferences.length ? { tocReferences } : {},
|
|
5898
|
+
...fm.locations ? { locations: fm.locations } : {},
|
|
4661
5899
|
markdown: fm.body,
|
|
4662
5900
|
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
4663
5901
|
...ordinal !== void 0 ? { ordinal } : {},
|
|
@@ -4679,12 +5917,22 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4679
5917
|
}
|
|
4680
5918
|
};
|
|
4681
5919
|
nameFromPages(outline.root);
|
|
4682
|
-
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
4686
|
-
|
|
4687
|
-
|
|
5920
|
+
warnCapped(
|
|
5921
|
+
warn,
|
|
5922
|
+
unlisted,
|
|
5923
|
+
(file) => `${file}: not in the ${outline.format} table of contents; filed under its folder`,
|
|
5924
|
+
(more) => `${more} more files are not in the ${outline.format} table of contents; filed under their folders`
|
|
5925
|
+
);
|
|
5926
|
+
warnCapped(
|
|
5927
|
+
warn,
|
|
5928
|
+
[...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
|
|
5929
|
+
(file) => `${file}: named by the table of contents but not found among the Markdown files`,
|
|
5930
|
+
(more) => `${more} more files named by the table of contents are not among the Markdown files`
|
|
5931
|
+
);
|
|
5932
|
+
if (outlineTooDeep) {
|
|
5933
|
+
warn(
|
|
5934
|
+
`the table of contents nests deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH2 - 1} levels; deeper sections are folded into their ancestors`
|
|
5935
|
+
);
|
|
4688
5936
|
}
|
|
4689
5937
|
}
|
|
4690
5938
|
if (rootTopicUsed) {
|
|
@@ -4702,6 +5950,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4702
5950
|
}
|
|
4703
5951
|
const assets = /* @__PURE__ */ new Map();
|
|
4704
5952
|
const documents = [];
|
|
5953
|
+
const linkWarnings = [];
|
|
4705
5954
|
for (const entry of loaded) {
|
|
4706
5955
|
const ctx = {
|
|
4707
5956
|
rootDir,
|
|
@@ -4710,8 +5959,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4710
5959
|
idByRel,
|
|
4711
5960
|
uri: opts.uri,
|
|
4712
5961
|
assets,
|
|
4713
|
-
|
|
4714
|
-
|
|
5962
|
+
// A docfx docset draws on dependent repositories (`~/reusable-content`)
|
|
5963
|
+
// that a checkout never holds, so a missing image is expected there.
|
|
5964
|
+
missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
|
|
5965
|
+
skipImages: opts.skipImages ?? false,
|
|
5966
|
+
tildeIsRoot: docfx,
|
|
5967
|
+
warn: (message) => linkWarnings.push(message)
|
|
4715
5968
|
};
|
|
4716
5969
|
documents.push(
|
|
4717
5970
|
CatalogDocumentSchema.parse({
|
|
@@ -4720,6 +5973,12 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4720
5973
|
})
|
|
4721
5974
|
);
|
|
4722
5975
|
}
|
|
5976
|
+
warnCapped(
|
|
5977
|
+
warn,
|
|
5978
|
+
linkWarnings,
|
|
5979
|
+
(message) => message,
|
|
5980
|
+
(more) => `${more} more link and image warnings`
|
|
5981
|
+
);
|
|
4723
5982
|
return {
|
|
4724
5983
|
topics,
|
|
4725
5984
|
documents,
|
|
@@ -4840,7 +6099,7 @@ function newerRegistryEntries(registry, installed) {
|
|
|
4840
6099
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
4841
6100
|
|
|
4842
6101
|
// src/signatures/jcs.ts
|
|
4843
|
-
import { canonicalizeJson as
|
|
6102
|
+
import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
|
|
4844
6103
|
export {
|
|
4845
6104
|
BGE_SMALL_EN_V15_1,
|
|
4846
6105
|
BODY_CODEC_MIN_BYTES,
|
|
@@ -4880,6 +6139,7 @@ export {
|
|
|
4880
6139
|
MULTILINGUAL_E5_SMALL_2,
|
|
4881
6140
|
PARQUET_EXPORT_VERSION,
|
|
4882
6141
|
PARQUET_REPORT_PATH,
|
|
6142
|
+
QUERY_STOP_WORDS,
|
|
4883
6143
|
README_PATH,
|
|
4884
6144
|
RERANK_CANDIDATES_MAX,
|
|
4885
6145
|
RERANK_CANDIDATES_MIN,
|
|
@@ -4892,13 +6152,14 @@ export {
|
|
|
4892
6152
|
SHARD_MAX_CHUNKS,
|
|
4893
6153
|
SHARD_TARGET_CHUNKS,
|
|
4894
6154
|
SOURCE_NOTICES_PATH,
|
|
6155
|
+
TITLE_ROUTE_SHARDS,
|
|
4895
6156
|
TRANSFORMERS_PEER_RANGE,
|
|
4896
6157
|
ZIP_FIXED_MTIME,
|
|
4897
6158
|
assertDuckdbCli,
|
|
4898
6159
|
assetReferences,
|
|
4899
6160
|
assignOutlineOrders,
|
|
4900
6161
|
asymmetricTopK,
|
|
4901
|
-
|
|
6162
|
+
canonicalizeJson3 as canonicalizeJson,
|
|
4902
6163
|
centerVector,
|
|
4903
6164
|
chunkContentHash,
|
|
4904
6165
|
chunkMarkdownProfile,
|