@bendyline/gezel-knowledge 1.2.2 → 1.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +161 -40
- package/dist/index.js +954 -141
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -180,12 +180,11 @@ function splitOversizeLine(line, opts) {
|
|
|
180
180
|
}
|
|
181
181
|
|
|
182
182
|
// src/compiler/compile.ts
|
|
183
|
-
import { createHash } from "crypto";
|
|
183
|
+
import { createHash as createHash2 } from "crypto";
|
|
184
184
|
import {
|
|
185
185
|
closeSync,
|
|
186
186
|
mkdirSync,
|
|
187
187
|
openSync,
|
|
188
|
-
readFileSync,
|
|
189
188
|
readSync,
|
|
190
189
|
rmSync,
|
|
191
190
|
statSync,
|
|
@@ -196,16 +195,13 @@ import { join } from "path";
|
|
|
196
195
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
197
196
|
import {
|
|
198
197
|
GEZK_MANIFEST_KIND,
|
|
199
|
-
KnowledgeAssetPathSchema,
|
|
200
198
|
KnowledgeCatalogManifestSchema,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
assetExtension,
|
|
205
|
-
assetKindForExtension,
|
|
199
|
+
KnowledgeLocationsSchema,
|
|
200
|
+
KnowledgeOrdinalSchema,
|
|
201
|
+
KnowledgeTocReferenceSchema,
|
|
206
202
|
canonicalizeJson,
|
|
207
|
-
|
|
208
|
-
|
|
203
|
+
normalizeLongitude,
|
|
204
|
+
spatialManifest
|
|
209
205
|
} from "@bendyline/gezk";
|
|
210
206
|
|
|
211
207
|
// src/archive/write.ts
|
|
@@ -251,28 +247,306 @@ var nodeRequire2 = createRequire2(import.meta.url);
|
|
|
251
247
|
var sqlite = nodeRequire2("node:sqlite");
|
|
252
248
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
253
249
|
|
|
250
|
+
// src/query-stopwords.ts
|
|
251
|
+
var QUERY_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
252
|
+
// articles + determiners
|
|
253
|
+
"a",
|
|
254
|
+
"an",
|
|
255
|
+
"the",
|
|
256
|
+
"this",
|
|
257
|
+
"that",
|
|
258
|
+
"these",
|
|
259
|
+
"those",
|
|
260
|
+
"some",
|
|
261
|
+
"any",
|
|
262
|
+
"all",
|
|
263
|
+
"each",
|
|
264
|
+
"both",
|
|
265
|
+
"every",
|
|
266
|
+
"other",
|
|
267
|
+
"another",
|
|
268
|
+
"same",
|
|
269
|
+
"such",
|
|
270
|
+
// pronouns
|
|
271
|
+
"i",
|
|
272
|
+
"me",
|
|
273
|
+
"my",
|
|
274
|
+
"we",
|
|
275
|
+
"us",
|
|
276
|
+
"our",
|
|
277
|
+
"you",
|
|
278
|
+
"your",
|
|
279
|
+
"it",
|
|
280
|
+
"its",
|
|
281
|
+
"he",
|
|
282
|
+
"him",
|
|
283
|
+
"his",
|
|
284
|
+
"she",
|
|
285
|
+
"her",
|
|
286
|
+
"they",
|
|
287
|
+
"them",
|
|
288
|
+
"their",
|
|
289
|
+
// be / have / do
|
|
290
|
+
"am",
|
|
291
|
+
"is",
|
|
292
|
+
"are",
|
|
293
|
+
"was",
|
|
294
|
+
"were",
|
|
295
|
+
"be",
|
|
296
|
+
"been",
|
|
297
|
+
"being",
|
|
298
|
+
"have",
|
|
299
|
+
"has",
|
|
300
|
+
"had",
|
|
301
|
+
"do",
|
|
302
|
+
"does",
|
|
303
|
+
"did",
|
|
304
|
+
"done",
|
|
305
|
+
// modals
|
|
306
|
+
"can",
|
|
307
|
+
"could",
|
|
308
|
+
"will",
|
|
309
|
+
"would",
|
|
310
|
+
"shall",
|
|
311
|
+
"should",
|
|
312
|
+
"may",
|
|
313
|
+
"might",
|
|
314
|
+
"must",
|
|
315
|
+
// question words
|
|
316
|
+
"how",
|
|
317
|
+
"what",
|
|
318
|
+
"when",
|
|
319
|
+
"where",
|
|
320
|
+
"which",
|
|
321
|
+
"who",
|
|
322
|
+
"whom",
|
|
323
|
+
"why",
|
|
324
|
+
// prepositions + conjunctions
|
|
325
|
+
"about",
|
|
326
|
+
"and",
|
|
327
|
+
"as",
|
|
328
|
+
"at",
|
|
329
|
+
"but",
|
|
330
|
+
"by",
|
|
331
|
+
"for",
|
|
332
|
+
"from",
|
|
333
|
+
"if",
|
|
334
|
+
"in",
|
|
335
|
+
"into",
|
|
336
|
+
"of",
|
|
337
|
+
"on",
|
|
338
|
+
"or",
|
|
339
|
+
"so",
|
|
340
|
+
"than",
|
|
341
|
+
"then",
|
|
342
|
+
"there",
|
|
343
|
+
"to",
|
|
344
|
+
"with",
|
|
345
|
+
// request scaffolding + degree words
|
|
346
|
+
"please",
|
|
347
|
+
"help",
|
|
348
|
+
"let",
|
|
349
|
+
"tell",
|
|
350
|
+
"want",
|
|
351
|
+
"need",
|
|
352
|
+
"create",
|
|
353
|
+
"make",
|
|
354
|
+
"give",
|
|
355
|
+
"get",
|
|
356
|
+
"got",
|
|
357
|
+
"use",
|
|
358
|
+
"using",
|
|
359
|
+
"just",
|
|
360
|
+
"like",
|
|
361
|
+
"also",
|
|
362
|
+
"very",
|
|
363
|
+
"only",
|
|
364
|
+
"new",
|
|
365
|
+
"more",
|
|
366
|
+
"most",
|
|
367
|
+
"much",
|
|
368
|
+
"many",
|
|
369
|
+
"few",
|
|
370
|
+
"own",
|
|
371
|
+
// conversational scaffolding
|
|
372
|
+
"hey",
|
|
373
|
+
"hi",
|
|
374
|
+
"hiya",
|
|
375
|
+
"hello",
|
|
376
|
+
"thanks",
|
|
377
|
+
"thank",
|
|
378
|
+
"okay",
|
|
379
|
+
"ok",
|
|
380
|
+
"yep",
|
|
381
|
+
"yeah",
|
|
382
|
+
"yes",
|
|
383
|
+
"nope",
|
|
384
|
+
"good",
|
|
385
|
+
"great",
|
|
386
|
+
"fine",
|
|
387
|
+
"nice",
|
|
388
|
+
"cool",
|
|
389
|
+
"ready",
|
|
390
|
+
"going",
|
|
391
|
+
"doing",
|
|
392
|
+
"sounds",
|
|
393
|
+
"morning",
|
|
394
|
+
"afternoon",
|
|
395
|
+
"evening",
|
|
396
|
+
"night",
|
|
397
|
+
// Apostrophes are token boundaries, so contractions otherwise leave these
|
|
398
|
+
// fragments looking like rare, highly distinctive query terms.
|
|
399
|
+
"s",
|
|
400
|
+
"t",
|
|
401
|
+
"re",
|
|
402
|
+
"ve",
|
|
403
|
+
"ll",
|
|
404
|
+
"d",
|
|
405
|
+
"m",
|
|
406
|
+
"isn",
|
|
407
|
+
"aren",
|
|
408
|
+
"wasn",
|
|
409
|
+
"weren",
|
|
410
|
+
"don",
|
|
411
|
+
"doesn",
|
|
412
|
+
"didn",
|
|
413
|
+
"hasn",
|
|
414
|
+
"haven",
|
|
415
|
+
"hadn",
|
|
416
|
+
"won",
|
|
417
|
+
"wouldn",
|
|
418
|
+
"shouldn",
|
|
419
|
+
"couldn"
|
|
420
|
+
]);
|
|
421
|
+
|
|
254
422
|
// src/reader/fts-query.ts
|
|
255
|
-
function
|
|
423
|
+
function contentTokens(query) {
|
|
256
424
|
const tokens = [
|
|
257
425
|
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
258
426
|
];
|
|
427
|
+
const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
|
|
428
|
+
return content.length > 0 ? content : tokens;
|
|
429
|
+
}
|
|
430
|
+
function contentFtsQuery(query) {
|
|
431
|
+
const tokens = contentTokens(query);
|
|
432
|
+
return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
|
|
433
|
+
}
|
|
434
|
+
function selectiveFtsQuery(query, rows, rowsMatching) {
|
|
435
|
+
const tokens = contentTokens(query);
|
|
259
436
|
if (tokens.length === 0) return null;
|
|
260
|
-
|
|
437
|
+
const counted = tokens.map((token) => {
|
|
438
|
+
const phrase = quoteToken(token);
|
|
439
|
+
return { phrase, hits: rowsMatching(phrase) };
|
|
440
|
+
});
|
|
441
|
+
const kept = counted.filter((c) => c.hits * 2 <= rows);
|
|
442
|
+
const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
|
|
443
|
+
return chosen.map((c) => c.phrase).join(" OR ");
|
|
261
444
|
}
|
|
262
445
|
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
263
446
|
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
264
|
-
function documentFtsTopIds(db,
|
|
265
|
-
const
|
|
266
|
-
return
|
|
447
|
+
function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
|
|
448
|
+
const match = contentFtsQuery(query);
|
|
449
|
+
if (!match || allowedDocumentIds?.size === 0) return [];
|
|
450
|
+
const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
|
|
451
|
+
const title = query.normalize("NFKC").trim();
|
|
452
|
+
const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
|
|
453
|
+
const rest = db.prepare(
|
|
267
454
|
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
268
|
-
|
|
269
|
-
|
|
455
|
+
${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
|
|
456
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
457
|
+
).all(
|
|
458
|
+
...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
|
|
459
|
+
);
|
|
460
|
+
for (const row of rest) {
|
|
461
|
+
if (ids.length >= limit) break;
|
|
462
|
+
if (!ids.includes(row.document_id)) ids.push(row.document_id);
|
|
463
|
+
}
|
|
464
|
+
return ids;
|
|
465
|
+
}
|
|
466
|
+
var NAME_STOPWORDS = new Set(
|
|
467
|
+
"a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
|
|
468
|
+
);
|
|
469
|
+
var NAMED_TITLE_CANDIDATES = 256;
|
|
470
|
+
var TITLE_DF_CACHE_MAX = 5e4;
|
|
471
|
+
var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
|
|
472
|
+
var SAME_WORDS_SCORE = Number.MAX_VALUE;
|
|
473
|
+
function nameWords(text) {
|
|
474
|
+
return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
475
|
+
}
|
|
476
|
+
function quoteToken(token) {
|
|
477
|
+
return `"${token.replace(/"/g, '""')}"`;
|
|
478
|
+
}
|
|
479
|
+
var titleStatsByDb = /* @__PURE__ */ new WeakMap();
|
|
480
|
+
function titleStats(db) {
|
|
481
|
+
let stats = titleStatsByDb.get(db);
|
|
482
|
+
if (!stats) {
|
|
483
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
|
|
484
|
+
stats = {
|
|
485
|
+
documents: Math.max(1, Number(row.c)),
|
|
486
|
+
df: /* @__PURE__ */ new Map(),
|
|
487
|
+
dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
|
|
488
|
+
candidates: db.prepare(
|
|
489
|
+
`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
490
|
+
ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
491
|
+
)
|
|
492
|
+
};
|
|
493
|
+
titleStatsByDb.set(db, stats);
|
|
494
|
+
}
|
|
495
|
+
return stats;
|
|
496
|
+
}
|
|
497
|
+
function titleDf(stats, word) {
|
|
498
|
+
let df = stats.df.get(word);
|
|
499
|
+
if (df === void 0) {
|
|
500
|
+
if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
|
|
501
|
+
const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
|
|
502
|
+
df = Number(row.c);
|
|
503
|
+
stats.df.set(word, df);
|
|
504
|
+
}
|
|
505
|
+
return df;
|
|
506
|
+
}
|
|
507
|
+
function namedTitleMatches(db, query, limit, allowedDocumentIds) {
|
|
508
|
+
if (allowedDocumentIds?.size === 0) return [];
|
|
509
|
+
const raw = query.normalize("NFKC").trim();
|
|
510
|
+
const words = [...new Set(nameWords(raw))].slice(0, 16);
|
|
511
|
+
const content = words.filter((w) => !NAME_STOPWORDS.has(w));
|
|
512
|
+
if (content.length === 0 || limit <= 0) return [];
|
|
513
|
+
const asked = new Set(words);
|
|
514
|
+
const capitalized = new Set(
|
|
515
|
+
(raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
|
|
516
|
+
);
|
|
517
|
+
const queryWords = nameWords(raw).join(" ");
|
|
518
|
+
const stats = titleStats(db);
|
|
519
|
+
const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
|
|
520
|
+
const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
521
|
+
AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
|
|
522
|
+
const named = [];
|
|
523
|
+
for (const row of rows) {
|
|
524
|
+
const titleWords = nameWords(row.title);
|
|
525
|
+
if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
|
|
526
|
+
const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
|
|
527
|
+
if (distinct.length === 0) continue;
|
|
528
|
+
let score;
|
|
529
|
+
if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
|
|
530
|
+
score = EXACT_TITLE_SCORE;
|
|
531
|
+
} else if (titleWords.join(" ") === queryWords) {
|
|
532
|
+
score = SAME_WORDS_SCORE;
|
|
533
|
+
} else {
|
|
534
|
+
score = 0;
|
|
535
|
+
for (const w of distinct) {
|
|
536
|
+
score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
named.push({ documentId: row.document_id, score, order: named.length });
|
|
540
|
+
}
|
|
541
|
+
named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
|
|
542
|
+
return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
|
|
543
|
+
}
|
|
544
|
+
function compareScores(a, b) {
|
|
545
|
+
return a === b ? 0 : a > b ? 1 : -1;
|
|
270
546
|
}
|
|
271
547
|
var SMOKE_QUERY_TOP_N = 10;
|
|
272
548
|
function documentSmokeQueryMisses(db, smoke) {
|
|
273
|
-
const
|
|
274
|
-
if (!match) return [...smoke.expectedDocumentIds];
|
|
275
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
549
|
+
const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
|
|
276
550
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
277
551
|
}
|
|
278
552
|
|
|
@@ -284,6 +558,92 @@ var KNOWLEDGE_TOOLCHAIN = {
|
|
|
284
558
|
version: nodeRequire3("../package.json").version
|
|
285
559
|
};
|
|
286
560
|
|
|
561
|
+
// src/compiler/assets.ts
|
|
562
|
+
import { createHash } from "crypto";
|
|
563
|
+
import { readFileSync } from "fs";
|
|
564
|
+
import {
|
|
565
|
+
KnowledgeAssetPathSchema,
|
|
566
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
567
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
568
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
569
|
+
assetExtension,
|
|
570
|
+
assetKindForExtension,
|
|
571
|
+
sniffAssetType,
|
|
572
|
+
svgInertnessProblem
|
|
573
|
+
} from "@bendyline/gezk";
|
|
574
|
+
function prepareAssets(assets, extraFiles, opts) {
|
|
575
|
+
for (const path of Object.keys(extraFiles)) {
|
|
576
|
+
if (path.startsWith("assets/")) {
|
|
577
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
581
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
582
|
+
}
|
|
583
|
+
const seen = /* @__PURE__ */ new Set();
|
|
584
|
+
const prepared = [];
|
|
585
|
+
const skippedPaths = /* @__PURE__ */ new Set();
|
|
586
|
+
const rejectContent = (path, message) => {
|
|
587
|
+
if (opts.invalidAssets !== "warn") throw new Error(message);
|
|
588
|
+
skippedPaths.add(path);
|
|
589
|
+
opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
|
|
590
|
+
};
|
|
591
|
+
let total = 0;
|
|
592
|
+
for (const asset of assets) {
|
|
593
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
594
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
595
|
+
const key = asset.path.toLowerCase();
|
|
596
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
597
|
+
seen.add(key);
|
|
598
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
599
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
600
|
+
}
|
|
601
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
602
|
+
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
603
|
+
rejectContent(
|
|
604
|
+
asset.path,
|
|
605
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
606
|
+
);
|
|
607
|
+
continue;
|
|
608
|
+
}
|
|
609
|
+
const ext = assetExtension(asset.path);
|
|
610
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
611
|
+
const actual = sniffAssetType(bytes);
|
|
612
|
+
if (expected === null || actual !== expected) {
|
|
613
|
+
rejectContent(
|
|
614
|
+
asset.path,
|
|
615
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
616
|
+
);
|
|
617
|
+
continue;
|
|
618
|
+
}
|
|
619
|
+
if (ext === "svg") {
|
|
620
|
+
const problem = svgInertnessProblem(bytes);
|
|
621
|
+
if (problem) {
|
|
622
|
+
rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
total += bytes.byteLength;
|
|
627
|
+
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
628
|
+
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
629
|
+
}
|
|
630
|
+
prepared.push({
|
|
631
|
+
path: asset.path,
|
|
632
|
+
bytes,
|
|
633
|
+
sizeBytes: bytes.byteLength,
|
|
634
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
|
|
638
|
+
}
|
|
639
|
+
function omitSkippedAssetReferences(markdown, skippedPaths) {
|
|
640
|
+
if (skippedPaths.size === 0) return markdown;
|
|
641
|
+
return markdown.replace(
|
|
642
|
+
/!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
|
|
643
|
+
(whole, text, target) => skippedPaths.has(target) ? text : whole
|
|
644
|
+
);
|
|
645
|
+
}
|
|
646
|
+
|
|
287
647
|
// src/compiler/compile.ts
|
|
288
648
|
async function compileKnowledgeCatalog(opts) {
|
|
289
649
|
if (opts.topics.length === 0) {
|
|
@@ -292,7 +652,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
292
652
|
);
|
|
293
653
|
}
|
|
294
654
|
const topicById = assertTopicForest(opts.topics);
|
|
295
|
-
const
|
|
655
|
+
const { assets: preparedAssets, skippedPaths } = prepareAssets(
|
|
656
|
+
opts.assets ?? [],
|
|
657
|
+
opts.extraFiles ?? {},
|
|
658
|
+
opts
|
|
659
|
+
);
|
|
296
660
|
const profile = opts.embeddingProfile;
|
|
297
661
|
const chunkerOpts = {
|
|
298
662
|
unit: opts.chunkingProfile.unit,
|
|
@@ -302,13 +666,40 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
302
666
|
};
|
|
303
667
|
const prepared = [];
|
|
304
668
|
const seenIds = /* @__PURE__ */ new Set();
|
|
305
|
-
for await (const
|
|
669
|
+
for await (const input of opts.documents) {
|
|
670
|
+
const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
|
|
671
|
+
...location,
|
|
672
|
+
longitude: normalizeLongitude(location.longitude)
|
|
673
|
+
})).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
674
|
+
const doc = {
|
|
675
|
+
...input,
|
|
676
|
+
locations,
|
|
677
|
+
markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
|
|
678
|
+
};
|
|
306
679
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
307
680
|
seenIds.add(doc.id);
|
|
681
|
+
const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
|
|
682
|
+
const placements = [
|
|
683
|
+
{
|
|
684
|
+
topicId: leafTopicId,
|
|
685
|
+
ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
|
|
686
|
+
}
|
|
687
|
+
];
|
|
688
|
+
const leaves = /* @__PURE__ */ new Set([leafTopicId]);
|
|
689
|
+
for (const raw of doc.tocReferences ?? []) {
|
|
690
|
+
const reference = KnowledgeTocReferenceSchema.parse(raw);
|
|
691
|
+
const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
|
|
692
|
+
if (leaves.has(topicId))
|
|
693
|
+
throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
|
|
694
|
+
leaves.add(topicId);
|
|
695
|
+
placements.push({ topicId, ordinal: reference.ordinal ?? null });
|
|
696
|
+
}
|
|
697
|
+
placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
|
|
308
698
|
prepared.push({
|
|
309
699
|
doc,
|
|
310
700
|
topicPathKey: doc.topicPath.join("/"),
|
|
311
|
-
leafTopicId
|
|
701
|
+
leafTopicId,
|
|
702
|
+
placements,
|
|
312
703
|
metaJson: encodeDocumentMeta(doc),
|
|
313
704
|
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
314
705
|
});
|
|
@@ -411,7 +802,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
411
802
|
});
|
|
412
803
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
413
804
|
for (const p of prepared) {
|
|
414
|
-
|
|
805
|
+
for (const placement of p.placements) {
|
|
806
|
+
topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
|
|
807
|
+
}
|
|
415
808
|
}
|
|
416
809
|
{
|
|
417
810
|
const stmt = router.prepare(
|
|
@@ -435,6 +828,13 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
435
828
|
body_codec, body_blob)
|
|
436
829
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
437
830
|
);
|
|
831
|
+
const insertLocation = router.prepare(
|
|
832
|
+
`INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
|
|
833
|
+
VALUES (?, ?, ?, ?, ?, ?)`
|
|
834
|
+
);
|
|
835
|
+
const insertPlacement = router.prepare(
|
|
836
|
+
"INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
|
|
837
|
+
);
|
|
438
838
|
const insertAlias = router.prepare(
|
|
439
839
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
440
840
|
);
|
|
@@ -473,6 +873,29 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
473
873
|
useBrotli ? "br" : "none",
|
|
474
874
|
blob
|
|
475
875
|
);
|
|
876
|
+
for (const location of p.doc.locations ?? []) {
|
|
877
|
+
const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
|
|
878
|
+
if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
|
|
879
|
+
throw new Error(
|
|
880
|
+
`document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
insertLocation.run(
|
|
884
|
+
p.doc.id,
|
|
885
|
+
location.id,
|
|
886
|
+
location.latitude,
|
|
887
|
+
location.longitude,
|
|
888
|
+
location.role,
|
|
889
|
+
provenance
|
|
890
|
+
);
|
|
891
|
+
}
|
|
892
|
+
for (const placement of p.placements) {
|
|
893
|
+
insertPlacement.run(
|
|
894
|
+
placement.topicId,
|
|
895
|
+
p.doc.id,
|
|
896
|
+
placement.ordinal === null ? null : BigInt(placement.ordinal)
|
|
897
|
+
);
|
|
898
|
+
}
|
|
476
899
|
const aliases = [...new Set(p.doc.aliases ?? [])].sort();
|
|
477
900
|
for (const alias of aliases) insertAlias.run(alias, p.doc.id);
|
|
478
901
|
insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
|
|
@@ -705,7 +1128,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
705
1128
|
files.push({
|
|
706
1129
|
path,
|
|
707
1130
|
sizeBytes: bytes.length,
|
|
708
|
-
sha256:
|
|
1131
|
+
sha256: createHash2("sha256").update(bytes).digest("hex")
|
|
709
1132
|
});
|
|
710
1133
|
}
|
|
711
1134
|
for (const asset of preparedAssets) {
|
|
@@ -749,6 +1172,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
749
1172
|
assets: preparedAssets.length
|
|
750
1173
|
},
|
|
751
1174
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
1175
|
+
spatial: spatialManifest(
|
|
1176
|
+
prepared.flatMap(
|
|
1177
|
+
(p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
|
|
1178
|
+
)
|
|
1179
|
+
),
|
|
752
1180
|
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
753
1181
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
754
1182
|
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
@@ -844,58 +1272,6 @@ function encodeDocumentMeta(doc) {
|
|
|
844
1272
|
}
|
|
845
1273
|
return json;
|
|
846
1274
|
}
|
|
847
|
-
function prepareAssets(assets, extraFiles) {
|
|
848
|
-
for (const path of Object.keys(extraFiles)) {
|
|
849
|
-
if (path.startsWith("assets/")) {
|
|
850
|
-
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
854
|
-
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
855
|
-
}
|
|
856
|
-
const seen = /* @__PURE__ */ new Set();
|
|
857
|
-
const prepared = [];
|
|
858
|
-
let total = 0;
|
|
859
|
-
for (const asset of assets) {
|
|
860
|
-
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
861
|
-
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
862
|
-
const key = asset.path.toLowerCase();
|
|
863
|
-
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
864
|
-
seen.add(key);
|
|
865
|
-
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
866
|
-
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
867
|
-
}
|
|
868
|
-
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
869
|
-
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
870
|
-
throw new Error(
|
|
871
|
-
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
872
|
-
);
|
|
873
|
-
}
|
|
874
|
-
total += bytes.byteLength;
|
|
875
|
-
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
876
|
-
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
877
|
-
}
|
|
878
|
-
const ext = assetExtension(asset.path);
|
|
879
|
-
const expected = ext ? assetKindForExtension(ext) : null;
|
|
880
|
-
const actual = sniffAssetType(bytes);
|
|
881
|
-
if (expected === null || actual !== expected) {
|
|
882
|
-
throw new Error(
|
|
883
|
-
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
884
|
-
);
|
|
885
|
-
}
|
|
886
|
-
if (ext === "svg") {
|
|
887
|
-
const problem = svgInertnessProblem(bytes);
|
|
888
|
-
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
889
|
-
}
|
|
890
|
-
prepared.push({
|
|
891
|
-
path: asset.path,
|
|
892
|
-
bytes,
|
|
893
|
-
sizeBytes: bytes.byteLength,
|
|
894
|
-
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
895
|
-
});
|
|
896
|
-
}
|
|
897
|
-
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
898
|
-
}
|
|
899
1275
|
function assertAssetReferencesDeclared(prepared, assets) {
|
|
900
1276
|
const declared = new Set(assets.map((a) => a.path));
|
|
901
1277
|
for (const p of prepared) {
|
|
@@ -1110,7 +1486,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1110
1486
|
return assignment.map((s) => renumber.get(s));
|
|
1111
1487
|
}
|
|
1112
1488
|
function seedFor(catalogId, shardId) {
|
|
1113
|
-
const digest =
|
|
1489
|
+
const digest = createHash2("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
1114
1490
|
return digest.readBigUInt64LE(24);
|
|
1115
1491
|
}
|
|
1116
1492
|
function makePrng(seed) {
|
|
@@ -1195,7 +1571,7 @@ function kMeans(samples, k, seed) {
|
|
|
1195
1571
|
}
|
|
1196
1572
|
|
|
1197
1573
|
// src/archive/read.ts
|
|
1198
|
-
import { createHash as
|
|
1574
|
+
import { createHash as createHash3 } from "crypto";
|
|
1199
1575
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
1200
1576
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1201
1577
|
import { createRequire as createRequire4 } from "module";
|
|
@@ -1485,7 +1861,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1485
1861
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1486
1862
|
return;
|
|
1487
1863
|
}
|
|
1488
|
-
const hasher =
|
|
1864
|
+
const hasher = createHash3("sha256");
|
|
1489
1865
|
let sizeBytes = 0;
|
|
1490
1866
|
const verify = new Transform({
|
|
1491
1867
|
transform(chunk, _encoding, callback) {
|
|
@@ -1608,7 +1984,7 @@ function readPragmaNumber(db, pragma) {
|
|
|
1608
1984
|
}
|
|
1609
1985
|
|
|
1610
1986
|
// src/reader/catalog-handle.ts
|
|
1611
|
-
import { readFileSync as
|
|
1987
|
+
import { readFileSync as readFileSync3 } from "fs";
|
|
1612
1988
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1613
1989
|
import { brotliDecompressSync } from "zlib";
|
|
1614
1990
|
import {
|
|
@@ -1689,14 +2065,14 @@ function hammingTopK(index, query, k) {
|
|
|
1689
2065
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1690
2066
|
return hits;
|
|
1691
2067
|
}
|
|
1692
|
-
function asymmetricTopK(index, query, k) {
|
|
2068
|
+
function asymmetricTopK(index, query, k, eligibleChunkIds) {
|
|
1693
2069
|
const { bits, bytesPerRow, rows } = index;
|
|
1694
2070
|
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1695
2071
|
throw new Error(
|
|
1696
2072
|
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1697
2073
|
);
|
|
1698
2074
|
}
|
|
1699
|
-
const limit = Math.min(k, rows);
|
|
2075
|
+
const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
|
|
1700
2076
|
if (limit <= 0) return [];
|
|
1701
2077
|
const lut = new Float32Array(bytesPerRow * 256);
|
|
1702
2078
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
@@ -1746,6 +2122,10 @@ function asymmetricTopK(index, query, k) {
|
|
|
1746
2122
|
};
|
|
1747
2123
|
let offset = 0;
|
|
1748
2124
|
for (let r = 0; r < rows; r++) {
|
|
2125
|
+
if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
|
|
2126
|
+
offset += bytesPerRow;
|
|
2127
|
+
continue;
|
|
2128
|
+
}
|
|
1749
2129
|
let score = 0;
|
|
1750
2130
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
1751
2131
|
score += lut[p * 256 + bits[offset + p]];
|
|
@@ -1770,14 +2150,126 @@ function asymmetricTopK(index, query, k) {
|
|
|
1770
2150
|
return hits;
|
|
1771
2151
|
}
|
|
1772
2152
|
|
|
2153
|
+
// src/reader/spatial-index.ts
|
|
2154
|
+
import {
|
|
2155
|
+
KnowledgeLocationSchema,
|
|
2156
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
|
|
2157
|
+
KnowledgeRadiusSchema,
|
|
2158
|
+
locationDistanceMeters,
|
|
2159
|
+
locationInBounds,
|
|
2160
|
+
normalizeLongitude as normalizeLongitude2,
|
|
2161
|
+
radiusBounds,
|
|
2162
|
+
spatialManifest as spatialManifest2
|
|
2163
|
+
} from "@bendyline/gezk";
|
|
2164
|
+
var DocumentSpatialIndex = class {
|
|
2165
|
+
constructor(db, allowLegacy) {
|
|
2166
|
+
this.db = db;
|
|
2167
|
+
this.allowLegacy = allowLegacy;
|
|
2168
|
+
this.hasTable = Boolean(
|
|
2169
|
+
db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
|
|
2170
|
+
);
|
|
2171
|
+
}
|
|
2172
|
+
db;
|
|
2173
|
+
allowLegacy;
|
|
2174
|
+
hasTable;
|
|
2175
|
+
legacy;
|
|
2176
|
+
decode(row) {
|
|
2177
|
+
const provenance = row.provenance_json;
|
|
2178
|
+
if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
|
|
2179
|
+
throw new Error("location provenance exceeds 16384 bytes");
|
|
2180
|
+
return {
|
|
2181
|
+
documentId: String(row.document_id),
|
|
2182
|
+
location: KnowledgeLocationSchema.parse({
|
|
2183
|
+
id: row.location_id,
|
|
2184
|
+
latitude: row.latitude,
|
|
2185
|
+
longitude: row.longitude,
|
|
2186
|
+
role: row.role,
|
|
2187
|
+
...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
|
|
2188
|
+
})
|
|
2189
|
+
};
|
|
2190
|
+
}
|
|
2191
|
+
legacyRows() {
|
|
2192
|
+
if (this.legacy) return this.legacy;
|
|
2193
|
+
this.legacy = [];
|
|
2194
|
+
if (!this.allowLegacy) return this.legacy;
|
|
2195
|
+
for (const row of this.db.prepare(
|
|
2196
|
+
"SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
|
|
2197
|
+
).all()) {
|
|
2198
|
+
try {
|
|
2199
|
+
const meta = JSON.parse(String(row.meta_json));
|
|
2200
|
+
if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
|
|
2201
|
+
continue;
|
|
2202
|
+
const point = KnowledgeLocationSchema.safeParse({
|
|
2203
|
+
id: "qualla-coordinate",
|
|
2204
|
+
role: "subject",
|
|
2205
|
+
latitude: meta.coordinates.lat,
|
|
2206
|
+
longitude: meta.coordinates.lng,
|
|
2207
|
+
provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
|
|
2208
|
+
});
|
|
2209
|
+
if (point.success)
|
|
2210
|
+
this.legacy.push({
|
|
2211
|
+
documentId: String(row.id),
|
|
2212
|
+
location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
|
|
2213
|
+
});
|
|
2214
|
+
} catch {
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
return this.legacy;
|
|
2218
|
+
}
|
|
2219
|
+
rows() {
|
|
2220
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
|
|
2221
|
+
}
|
|
2222
|
+
locations(documentId) {
|
|
2223
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
|
|
2224
|
+
}
|
|
2225
|
+
integrity() {
|
|
2226
|
+
const rows = this.rows();
|
|
2227
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
2228
|
+
for (const row of rows) {
|
|
2229
|
+
if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
|
|
2230
|
+
throw new Error("location longitude must be in [-180, 180)");
|
|
2231
|
+
const points = grouped.get(row.documentId) ?? [];
|
|
2232
|
+
points.push(row.location);
|
|
2233
|
+
grouped.set(row.documentId, points);
|
|
2234
|
+
if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
|
|
2235
|
+
throw new Error("location references missing document");
|
|
2236
|
+
}
|
|
2237
|
+
for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
|
|
2238
|
+
return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
|
|
2239
|
+
}
|
|
2240
|
+
matches(raw) {
|
|
2241
|
+
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
2242
|
+
const boxes = radiusBounds(radius);
|
|
2243
|
+
const candidates = this.hasTable ? boxes.flatMap(
|
|
2244
|
+
(box) => this.db.prepare(`SELECT * FROM document_locations
|
|
2245
|
+
WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
|
|
2246
|
+
) : this.legacyRows().filter(
|
|
2247
|
+
(row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
|
|
2248
|
+
);
|
|
2249
|
+
const matches = /* @__PURE__ */ new Map();
|
|
2250
|
+
for (const { documentId, location } of candidates) {
|
|
2251
|
+
const distanceMeters = locationDistanceMeters(radius, location);
|
|
2252
|
+
if (distanceMeters > radius.radiusMeters) continue;
|
|
2253
|
+
const previous = matches.get(documentId);
|
|
2254
|
+
if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
|
|
2255
|
+
matches.set(documentId, { distanceMeters, matchedLocation: location });
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
return matches;
|
|
2259
|
+
}
|
|
2260
|
+
};
|
|
2261
|
+
|
|
1773
2262
|
// src/reader/catalog-handle.ts
|
|
1774
2263
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
1775
2264
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
2265
|
+
var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
|
|
2266
|
+
var TITLE_ROUTE_SHARDS = 1;
|
|
1776
2267
|
var CatalogHandle = class _CatalogHandle {
|
|
1777
2268
|
constructor(rootDir, router) {
|
|
1778
2269
|
this.rootDir = rootDir;
|
|
1779
2270
|
this.router = router;
|
|
1780
2271
|
this.schemaVersion = router.schemaVersion;
|
|
2272
|
+
this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
1781
2273
|
this.meta = {};
|
|
1782
2274
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1783
2275
|
this.meta[row.key] = row.value;
|
|
@@ -1789,6 +2281,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1789
2281
|
router;
|
|
1790
2282
|
connections = /* @__PURE__ */ new Map();
|
|
1791
2283
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
2284
|
+
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
2285
|
+
chunkTermRows = /* @__PURE__ */ new Map();
|
|
2286
|
+
spatial;
|
|
1792
2287
|
assetsByPath = null;
|
|
1793
2288
|
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1794
2289
|
binaryCenterCache;
|
|
@@ -1799,7 +2294,12 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1799
2294
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1800
2295
|
static open(rootDir) {
|
|
1801
2296
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
1802
|
-
|
|
2297
|
+
try {
|
|
2298
|
+
return new _CatalogHandle(rootDir, router);
|
|
2299
|
+
} catch (error) {
|
|
2300
|
+
router.close();
|
|
2301
|
+
throw error;
|
|
2302
|
+
}
|
|
1803
2303
|
}
|
|
1804
2304
|
close() {
|
|
1805
2305
|
for (const conn of this.connections.values()) conn.close();
|
|
@@ -1874,6 +2374,24 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1874
2374
|
this.bitIndexes.set(shard.path, index);
|
|
1875
2375
|
return index;
|
|
1876
2376
|
}
|
|
2377
|
+
/**
|
|
2378
|
+
* Load the next shard's sign bits that are not resident yet, if they fit
|
|
2379
|
+
* the resident budget. False when nothing is left to load. One shard per
|
|
2380
|
+
* call, so a host can keep answering queries between shards while it warms
|
|
2381
|
+
* a large catalog, and never evicts a shard it loaded a moment ago.
|
|
2382
|
+
*/
|
|
2383
|
+
prewarmNextShard() {
|
|
2384
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
2385
|
+
let resident = 0;
|
|
2386
|
+
for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
|
|
2387
|
+
for (const shard of this.shards) {
|
|
2388
|
+
if (this.bitIndexes.has(shard.path)) continue;
|
|
2389
|
+
if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
|
|
2390
|
+
this.shardBits(shard);
|
|
2391
|
+
return true;
|
|
2392
|
+
}
|
|
2393
|
+
return false;
|
|
2394
|
+
}
|
|
1877
2395
|
/**
|
|
1878
2396
|
* The stage-1 center the catalog's bits were derived with (a
|
|
1879
2397
|
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
@@ -1923,9 +2441,13 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1923
2441
|
get formatVersion() {
|
|
1924
2442
|
return this.meta.format_version ?? "";
|
|
1925
2443
|
}
|
|
1926
|
-
documentColumns() {
|
|
2444
|
+
documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
|
|
1927
2445
|
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1928
|
-
|
|
2446
|
+
const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
2447
|
+
return columns.split(", ").map((column) => {
|
|
2448
|
+
const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
|
|
2449
|
+
return prefix ? `${prefix}.${column} AS ${column}` : column;
|
|
2450
|
+
}).join(", ");
|
|
1929
2451
|
}
|
|
1930
2452
|
documentOrder() {
|
|
1931
2453
|
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
@@ -1935,6 +2457,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1935
2457
|
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1936
2458
|
*/
|
|
1937
2459
|
topics() {
|
|
2460
|
+
const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
|
|
2461
|
+
FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
2462
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
|
|
1938
2463
|
return this.router.db.prepare(
|
|
1939
2464
|
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1940
2465
|
SELECT id, id, 0 FROM topics
|
|
@@ -1943,8 +2468,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1943
2468
|
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1944
2469
|
),
|
|
1945
2470
|
rollup AS (
|
|
1946
|
-
|
|
1947
|
-
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
2471
|
+
${rollup}
|
|
1948
2472
|
)
|
|
1949
2473
|
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1950
2474
|
rollup.total AS total_document_count
|
|
@@ -1968,6 +2492,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1968
2492
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1969
2493
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1970
2494
|
const descendants = opts.descendants ?? true;
|
|
2495
|
+
if (this.schemaVersion >= 4 && opts.topicId) {
|
|
2496
|
+
return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
|
|
2497
|
+
}
|
|
1971
2498
|
let scope = "";
|
|
1972
2499
|
let where = "";
|
|
1973
2500
|
const params = [];
|
|
@@ -1990,16 +2517,101 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1990
2517
|
const documents = this.router.db.prepare(
|
|
1991
2518
|
`${scope} SELECT ${this.documentColumns()}
|
|
1992
2519
|
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1993
|
-
).all(...params, limit, offset).map(
|
|
2520
|
+
).all(...params, limit, offset).map((row) => this.documentMeta(row));
|
|
1994
2521
|
return { documents, total };
|
|
1995
2522
|
}
|
|
2523
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
2524
|
+
sharedDocumentsPage(topicId, descendants, limit, offset) {
|
|
2525
|
+
const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
|
|
2526
|
+
JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
|
|
2527
|
+
const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
|
|
2528
|
+
SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
|
|
2529
|
+
FROM topic_documents td JOIN topics t ON t.id = td.topic_id
|
|
2530
|
+
WHERE td.topic_id IN (SELECT id FROM sub)
|
|
2531
|
+
)`;
|
|
2532
|
+
const total = Number(
|
|
2533
|
+
this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
|
|
2534
|
+
);
|
|
2535
|
+
const documents = this.router.db.prepare(
|
|
2536
|
+
`${scope}, ranked AS (
|
|
2537
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2538
|
+
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2539
|
+
) SELECT ${this.documentColumns("d", "p")}
|
|
2540
|
+
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
|
|
2541
|
+
ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
|
|
2542
|
+
).all(topicId, limit, offset).map((row) => this.documentMeta(row));
|
|
2543
|
+
return { documents, total };
|
|
2544
|
+
}
|
|
2545
|
+
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
2546
|
+
tocIntegrity() {
|
|
2547
|
+
const count = (sql) => Number(this.router.db.prepare(sql).get().n);
|
|
2548
|
+
const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
|
|
2549
|
+
const incorrectCounts = count(
|
|
2550
|
+
`SELECT COUNT(*) AS n FROM topics t LEFT JOIN
|
|
2551
|
+
(SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
|
|
2552
|
+
WHERE t.document_count != COALESCE(r.n, 0)`
|
|
2553
|
+
);
|
|
2554
|
+
if (this.schemaVersion < 4)
|
|
2555
|
+
return {
|
|
2556
|
+
placements: count("SELECT COUNT(*) AS n FROM documents"),
|
|
2557
|
+
invalidReferences: 0,
|
|
2558
|
+
missingPrimary: 0,
|
|
2559
|
+
incorrectCounts,
|
|
2560
|
+
duplicateReferences: 0
|
|
2561
|
+
};
|
|
2562
|
+
return {
|
|
2563
|
+
placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
|
|
2564
|
+
invalidReferences: count(
|
|
2565
|
+
`SELECT COUNT(*) AS n FROM topic_documents r
|
|
2566
|
+
LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
|
|
2567
|
+
WHERE d.id IS NULL OR t.id IS NULL OR
|
|
2568
|
+
(r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
|
|
2569
|
+
),
|
|
2570
|
+
missingPrimary: count(
|
|
2571
|
+
`SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
|
|
2572
|
+
SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
|
|
2573
|
+
),
|
|
2574
|
+
incorrectCounts,
|
|
2575
|
+
duplicateReferences: count(
|
|
2576
|
+
"SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
|
|
2577
|
+
)
|
|
2578
|
+
};
|
|
2579
|
+
}
|
|
1996
2580
|
getDocument(id) {
|
|
1997
2581
|
const row = this.router.db.prepare(
|
|
1998
2582
|
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1999
2583
|
FROM documents WHERE id = ?`
|
|
2000
2584
|
).get(id);
|
|
2001
2585
|
if (!row) return null;
|
|
2002
|
-
return { ...
|
|
2586
|
+
return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
|
|
2587
|
+
}
|
|
2588
|
+
/**
|
|
2589
|
+
* One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
|
|
2590
|
+
* shard the document's chunks live in. Null when the document or the
|
|
2591
|
+
* passage is not in this catalog, which is what a fabricated or stale
|
|
2592
|
+
* citation looks like.
|
|
2593
|
+
*/
|
|
2594
|
+
getChunk(documentId, chunkUid2) {
|
|
2595
|
+
const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
|
|
2596
|
+
if (!doc) return null;
|
|
2597
|
+
const shardId = Number(doc.shard_id);
|
|
2598
|
+
const shard = this.shards.find((s) => s.id === shardId);
|
|
2599
|
+
if (!shard) return null;
|
|
2600
|
+
const row = this.shardDb(shard).prepare(
|
|
2601
|
+
`SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
|
|
2602
|
+
FROM chunks WHERE chunk_uid = ? AND document_id = ?`
|
|
2603
|
+
).get(chunkUid2, documentId);
|
|
2604
|
+
if (!row) return null;
|
|
2605
|
+
return {
|
|
2606
|
+
chunkUid: row.chunk_uid,
|
|
2607
|
+
documentId: row.document_id,
|
|
2608
|
+
title: row.title,
|
|
2609
|
+
headingPath: parseHeadingPath(row.heading_path),
|
|
2610
|
+
lineStart: Number(row.line_start),
|
|
2611
|
+
lineEnd: Number(row.line_end),
|
|
2612
|
+
text: row.text,
|
|
2613
|
+
shardId
|
|
2614
|
+
};
|
|
2003
2615
|
}
|
|
2004
2616
|
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
2005
2617
|
documentsWithUndeclaredTopic() {
|
|
@@ -2072,7 +2684,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2072
2684
|
const index = /* @__PURE__ */ new Map();
|
|
2073
2685
|
let raw;
|
|
2074
2686
|
try {
|
|
2075
|
-
raw = JSON.parse(
|
|
2687
|
+
raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
2076
2688
|
} catch (error) {
|
|
2077
2689
|
if (error.code === "ENOENT") {
|
|
2078
2690
|
this.assetsByPath = index;
|
|
@@ -2111,7 +2723,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2111
2723
|
}
|
|
2112
2724
|
let bytes;
|
|
2113
2725
|
try {
|
|
2114
|
-
bytes =
|
|
2726
|
+
bytes = readFileSync3(this.resolveCatalogPath(path));
|
|
2115
2727
|
} catch (error) {
|
|
2116
2728
|
throw new CatalogOpenError(
|
|
2117
2729
|
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -2123,19 +2735,59 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2123
2735
|
}
|
|
2124
2736
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
2125
2737
|
}
|
|
2738
|
+
documentMeta(row) {
|
|
2739
|
+
const document = rowToDocumentMeta(row);
|
|
2740
|
+
const locations = this.spatial.locations(document.id);
|
|
2741
|
+
return locations.length ? { ...document, locations } : document;
|
|
2742
|
+
}
|
|
2743
|
+
documentLocations(documentId) {
|
|
2744
|
+
return this.spatial.locations(documentId);
|
|
2745
|
+
}
|
|
2746
|
+
documentLocationRows() {
|
|
2747
|
+
return this.spatial.rows();
|
|
2748
|
+
}
|
|
2749
|
+
spatialIntegrity() {
|
|
2750
|
+
return this.spatial.integrity();
|
|
2751
|
+
}
|
|
2752
|
+
spatialMatches(radius) {
|
|
2753
|
+
return this.spatial.matches(radius);
|
|
2754
|
+
}
|
|
2755
|
+
nearbyDocuments(radius, opts = {}) {
|
|
2756
|
+
const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
|
|
2757
|
+
const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
|
|
2758
|
+
const ordered = [...this.spatialMatches(radius)].sort(
|
|
2759
|
+
(a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
|
|
2760
|
+
);
|
|
2761
|
+
const get = this.router.db.prepare(
|
|
2762
|
+
`SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
|
|
2763
|
+
);
|
|
2764
|
+
return {
|
|
2765
|
+
total: ordered.length,
|
|
2766
|
+
documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
|
|
2767
|
+
...this.documentMeta(get.get(id)),
|
|
2768
|
+
...match
|
|
2769
|
+
}))
|
|
2770
|
+
};
|
|
2771
|
+
}
|
|
2772
|
+
shardsForDocuments(ids) {
|
|
2773
|
+
if (!ids.size) return [];
|
|
2774
|
+
return this.router.db.prepare(
|
|
2775
|
+
"SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
|
|
2776
|
+
).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
|
|
2777
|
+
}
|
|
2126
2778
|
// ── search ────────────────────────────────────────────────────────────────
|
|
2127
2779
|
/**
|
|
2128
2780
|
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
2129
2781
|
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
2130
2782
|
*/
|
|
2131
|
-
searchDocumentsFts(query, limit = 10) {
|
|
2132
|
-
const match = ftsQuery(query);
|
|
2133
|
-
if (!match) return [];
|
|
2783
|
+
searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
|
|
2134
2784
|
try {
|
|
2135
|
-
return documentFtsTopIds(this.router.db,
|
|
2136
|
-
documentId,
|
|
2137
|
-
|
|
2138
|
-
|
|
2785
|
+
return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
|
|
2786
|
+
(documentId, i) => ({
|
|
2787
|
+
documentId,
|
|
2788
|
+
rank: i
|
|
2789
|
+
})
|
|
2790
|
+
);
|
|
2139
2791
|
} catch {
|
|
2140
2792
|
return [];
|
|
2141
2793
|
}
|
|
@@ -2175,24 +2827,73 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2175
2827
|
}
|
|
2176
2828
|
return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
|
|
2177
2829
|
}
|
|
2178
|
-
/**
|
|
2179
|
-
|
|
2830
|
+
/**
|
|
2831
|
+
* Top-S shard ids within THIS catalog (single-catalog callers, tests).
|
|
2832
|
+
* With the raw `query`, also the shard of the page it names by title
|
|
2833
|
+
* (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
|
|
2834
|
+
*/
|
|
2835
|
+
routeShards(queryVector, shardBudget, query) {
|
|
2180
2836
|
if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
|
|
2181
|
-
|
|
2837
|
+
const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
2838
|
+
if (query) {
|
|
2839
|
+
for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
|
|
2840
|
+
if (!routed.includes(named.shardId)) routed.push(named.shardId);
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
return routed;
|
|
2844
|
+
}
|
|
2845
|
+
/**
|
|
2846
|
+
* Title-assisted routing: the shards holding the pages `query` names by
|
|
2847
|
+
* title (`namedTitleMatches`), most specific name first, at most `limit`
|
|
2848
|
+
* distinct shards. Centroid routing compares a question with averages of
|
|
2849
|
+
* whole shards, and e5 scores those in a band a few hundredths wide, so
|
|
2850
|
+
* the shard holding the very page a question names can rank 8th of 10:
|
|
2851
|
+
* "Who were the members of ABBA?" never reached the ABBA article in the
|
|
2852
|
+
* Wikipedia music catalog. Its title does — the vector arm scans that
|
|
2853
|
+
* shard too. Measured on the 10-shard music and film-tv catalogs, one such
|
|
2854
|
+
* shard lifted end-to-end recall@8 on "What is <title>?" questions from
|
|
2855
|
+
* 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
|
|
2856
|
+
*
|
|
2857
|
+
* `score` is the named match's specificity (fts-query.ts), comparable
|
|
2858
|
+
* across catalogs closely enough for the daemon to choose which catalogs'
|
|
2859
|
+
* shards to add when several name a page. Empty on any FTS error.
|
|
2860
|
+
*/
|
|
2861
|
+
titleRouteShards(query, limit) {
|
|
2862
|
+
if (limit <= 0) return [];
|
|
2863
|
+
try {
|
|
2864
|
+
const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
|
|
2865
|
+
const out = [];
|
|
2866
|
+
for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
|
|
2867
|
+
const row = shardOf.get(named.documentId);
|
|
2868
|
+
if (!row) continue;
|
|
2869
|
+
const shardId = Number(row.shard_id);
|
|
2870
|
+
if (out.some((o) => o.shardId === shardId)) continue;
|
|
2871
|
+
out.push({ shardId, score: named.score });
|
|
2872
|
+
if (out.length >= limit) break;
|
|
2873
|
+
}
|
|
2874
|
+
return out.sort((a, b) => compareScores(b.score, a.score));
|
|
2875
|
+
} catch {
|
|
2876
|
+
return [];
|
|
2877
|
+
}
|
|
2182
2878
|
}
|
|
2183
2879
|
/**
|
|
2184
2880
|
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
2185
2881
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
2186
2882
|
*/
|
|
2187
2883
|
searchSemantic(queryVector, opts = {}) {
|
|
2188
|
-
const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
|
|
2189
|
-
return this.searchShards(
|
|
2884
|
+
const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
|
|
2885
|
+
return this.searchShards(
|
|
2886
|
+
queryVector,
|
|
2887
|
+
shardIds,
|
|
2888
|
+
opts.finalK ?? RERANK_FINAL_K,
|
|
2889
|
+
opts.allowedDocumentIds
|
|
2890
|
+
);
|
|
2190
2891
|
}
|
|
2191
2892
|
/**
|
|
2192
2893
|
* The scan stage against an EXPLICIT shard list — what the daemon calls
|
|
2193
2894
|
* after global cross-catalog routing has already spent the S budget.
|
|
2194
2895
|
*/
|
|
2195
|
-
searchShards(queryVector, shardIds, finalK) {
|
|
2896
|
+
searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
|
|
2196
2897
|
const center = this.binaryCenter();
|
|
2197
2898
|
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
2198
2899
|
const hits = [];
|
|
@@ -2200,10 +2901,17 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2200
2901
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2201
2902
|
if (!shard) continue;
|
|
2202
2903
|
const db = this.shardDb(shard);
|
|
2203
|
-
const
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2904
|
+
const eligible = allowedDocumentIds ? new Set(
|
|
2905
|
+
db.prepare(
|
|
2906
|
+
"SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
|
|
2907
|
+
).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
|
|
2908
|
+
) : void 0;
|
|
2909
|
+
const k = rerankK(finalK, eligible?.size ?? shard.chunkCount);
|
|
2910
|
+
const candidates = asymmetricTopK(this.shardBits(shard), scanQuery, k, eligible).map(
|
|
2911
|
+
(hit) => ({
|
|
2912
|
+
chunk_id: hit.chunkId
|
|
2913
|
+
})
|
|
2914
|
+
);
|
|
2207
2915
|
if (candidates.length === 0) continue;
|
|
2208
2916
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
2209
2917
|
const reranked = candidates.map((c) => {
|
|
@@ -2237,20 +2945,26 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2237
2945
|
return hits;
|
|
2238
2946
|
}
|
|
2239
2947
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
2240
|
-
searchChunksFts(query, shardIds, limitPerShard = 12) {
|
|
2241
|
-
const match = ftsQuery(query);
|
|
2242
|
-
if (!match) return [];
|
|
2948
|
+
searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
|
|
2243
2949
|
const hits = [];
|
|
2244
2950
|
for (const shardId of shardIds) {
|
|
2245
2951
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2246
2952
|
if (!shard) continue;
|
|
2247
2953
|
const db = this.shardDb(shard);
|
|
2248
2954
|
try {
|
|
2955
|
+
const match = selectiveFtsQuery(
|
|
2956
|
+
query,
|
|
2957
|
+
shard.chunkCount,
|
|
2958
|
+
(phrase) => this.chunkFtsRows(shard, db, phrase)
|
|
2959
|
+
);
|
|
2960
|
+
if (!match) return hits;
|
|
2249
2961
|
const rows = db.prepare(
|
|
2250
2962
|
`SELECT c.chunk_uid, c.document_id, c.title, c.heading_path, c.line_start, c.line_end, c.text
|
|
2251
2963
|
FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
|
|
2252
|
-
WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
|
|
2253
|
-
).all(
|
|
2964
|
+
WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
|
|
2965
|
+
).all(
|
|
2966
|
+
...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
|
|
2967
|
+
);
|
|
2254
2968
|
for (const row of rows) {
|
|
2255
2969
|
hits.push({
|
|
2256
2970
|
chunkUid: row.chunk_uid,
|
|
@@ -2269,6 +2983,18 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2269
2983
|
}
|
|
2270
2984
|
return hits;
|
|
2271
2985
|
}
|
|
2986
|
+
/** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
|
|
2987
|
+
chunkFtsRows(shard, db, phrase) {
|
|
2988
|
+
const key = `${shard.path}\0${phrase}`;
|
|
2989
|
+
let rows = this.chunkTermRows.get(key);
|
|
2990
|
+
if (rows === void 0) {
|
|
2991
|
+
if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
|
|
2992
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
|
|
2993
|
+
rows = Number(row.c);
|
|
2994
|
+
this.chunkTermRows.set(key, rows);
|
|
2995
|
+
}
|
|
2996
|
+
return rows;
|
|
2997
|
+
}
|
|
2272
2998
|
/**
|
|
2273
2999
|
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
2274
3000
|
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
@@ -2322,7 +3048,6 @@ function parseHeadingPath(raw) {
|
|
|
2322
3048
|
return [];
|
|
2323
3049
|
}
|
|
2324
3050
|
}
|
|
2325
|
-
var ftsQuery = sanitizeFtsQuery;
|
|
2326
3051
|
|
|
2327
3052
|
// src/reader/validate.ts
|
|
2328
3053
|
import { readFile } from "fs/promises";
|
|
@@ -2334,6 +3059,7 @@ import {
|
|
|
2334
3059
|
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2335
3060
|
assetExtension as assetExtension2,
|
|
2336
3061
|
assetKindForExtension as assetKindForExtension2,
|
|
3062
|
+
canonicalizeJson as canonicalizeJson2,
|
|
2337
3063
|
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
2338
3064
|
sniffAssetType as sniffAssetType2,
|
|
2339
3065
|
svgInertnessProblem as svgInertnessProblem2
|
|
@@ -2409,6 +3135,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2409
3135
|
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2410
3136
|
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2411
3137
|
);
|
|
3138
|
+
if (manifest.formatVersion === "0.7") {
|
|
3139
|
+
try {
|
|
3140
|
+
const spatial = handle.spatialIntegrity();
|
|
3141
|
+
check("document-locations-table", spatial.hasTable);
|
|
3142
|
+
check(
|
|
3143
|
+
"document-locations-summary",
|
|
3144
|
+
Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
|
|
3145
|
+
"spatial counts and coverage must match router points"
|
|
3146
|
+
);
|
|
3147
|
+
} catch (error) {
|
|
3148
|
+
check(
|
|
3149
|
+
"document-locations-integrity",
|
|
3150
|
+
false,
|
|
3151
|
+
error instanceof Error ? error.message : String(error)
|
|
3152
|
+
);
|
|
3153
|
+
}
|
|
3154
|
+
}
|
|
2412
3155
|
const topics = handle.topics();
|
|
2413
3156
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2414
3157
|
check("topics-tree", ...topicTreeProblem(topics));
|
|
@@ -2446,11 +3189,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2446
3189
|
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
2447
3190
|
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
2448
3191
|
);
|
|
3192
|
+
const toc = handle.tocIntegrity();
|
|
3193
|
+
check(
|
|
3194
|
+
"toc-references",
|
|
3195
|
+
toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
|
|
3196
|
+
`${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
|
|
3197
|
+
);
|
|
3198
|
+
check(
|
|
3199
|
+
"toc-counts",
|
|
3200
|
+
toc.incorrectCounts === 0,
|
|
3201
|
+
`${toc.incorrectCounts} incorrect per-topic counts`
|
|
3202
|
+
);
|
|
2449
3203
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
2450
3204
|
check(
|
|
2451
3205
|
"counts-documents",
|
|
2452
|
-
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum ===
|
|
2453
|
-
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
|
|
3206
|
+
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
|
|
3207
|
+
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
|
|
2454
3208
|
);
|
|
2455
3209
|
check(
|
|
2456
3210
|
"counts-shards",
|
|
@@ -2578,6 +3332,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2578
3332
|
);
|
|
2579
3333
|
}
|
|
2580
3334
|
}
|
|
3335
|
+
} catch (error) {
|
|
3336
|
+
check("catalog-structure", false, error instanceof Error ? error.message : String(error));
|
|
2581
3337
|
} finally {
|
|
2582
3338
|
handle.close();
|
|
2583
3339
|
}
|
|
@@ -2686,13 +3442,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
2686
3442
|
}
|
|
2687
3443
|
|
|
2688
3444
|
// src/export/parquet.ts
|
|
2689
|
-
import { createHash as
|
|
3445
|
+
import { createHash as createHash4 } from "crypto";
|
|
2690
3446
|
import { createWriteStream as createWriteStream3 } from "fs";
|
|
2691
3447
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2692
3448
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2693
3449
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2694
3450
|
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2695
|
-
var PARQUET_EXPORT_VERSION =
|
|
3451
|
+
var PARQUET_EXPORT_VERSION = 4;
|
|
2696
3452
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2697
3453
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2698
3454
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2739,6 +3495,34 @@ async function exportCatalogParquet(opts) {
|
|
|
2739
3495
|
onProgress: opts.onProgress
|
|
2740
3496
|
})
|
|
2741
3497
|
);
|
|
3498
|
+
const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
3499
|
+
const locations = spatial.rows();
|
|
3500
|
+
if (spatial.hasTable || locations.length) {
|
|
3501
|
+
const path = join4(staging, "document-locations.ndjson");
|
|
3502
|
+
const out = ndjsonWriter(path);
|
|
3503
|
+
for (const { documentId, location } of locations)
|
|
3504
|
+
await out.write({
|
|
3505
|
+
document_id: documentId,
|
|
3506
|
+
location_id: location.id,
|
|
3507
|
+
latitude: location.latitude,
|
|
3508
|
+
longitude: location.longitude,
|
|
3509
|
+
role: location.role,
|
|
3510
|
+
provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
|
|
3511
|
+
});
|
|
3512
|
+
await out.close();
|
|
3513
|
+
const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
|
|
3514
|
+
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
|
|
3515
|
+
files.push(
|
|
3516
|
+
await writeParquet(opts.duckdb, {
|
|
3517
|
+
outDir,
|
|
3518
|
+
file: "document-locations.parquet",
|
|
3519
|
+
table: "document_locations",
|
|
3520
|
+
rows: locations.length,
|
|
3521
|
+
sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
|
|
3522
|
+
onProgress: opts.onProgress
|
|
3523
|
+
})
|
|
3524
|
+
);
|
|
3525
|
+
}
|
|
2742
3526
|
for (const shard of manifest.router.shards) {
|
|
2743
3527
|
const tag = String(shard.id).padStart(3, "0");
|
|
2744
3528
|
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
@@ -3081,7 +3865,7 @@ async function writeParquet(cli, spec) {
|
|
|
3081
3865
|
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
3082
3866
|
rows: spec.rows,
|
|
3083
3867
|
sizeBytes: (await stat2(target)).size,
|
|
3084
|
-
sha256:
|
|
3868
|
+
sha256: createHash4("sha256").update(bytes).digest("hex")
|
|
3085
3869
|
};
|
|
3086
3870
|
}
|
|
3087
3871
|
async function findDuckdbBinary(env = process.env) {
|
|
@@ -3582,7 +4366,7 @@ var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
|
3582
4366
|
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
3583
4367
|
|
|
3584
4368
|
// src/embedding/artifact-verify.ts
|
|
3585
|
-
import { createHash as
|
|
4369
|
+
import { createHash as createHash5 } from "crypto";
|
|
3586
4370
|
import { createReadStream } from "fs";
|
|
3587
4371
|
import { stat as stat3 } from "fs/promises";
|
|
3588
4372
|
import { join as join5 } from "path";
|
|
@@ -3626,7 +4410,7 @@ async function fileDigest(path) {
|
|
|
3626
4410
|
const memo = digestMemo.get(key);
|
|
3627
4411
|
if (memo) return memo;
|
|
3628
4412
|
const digest = await new Promise((resolve5, reject) => {
|
|
3629
|
-
const hash =
|
|
4413
|
+
const hash = createHash5("sha256");
|
|
3630
4414
|
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
3631
4415
|
});
|
|
3632
4416
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
@@ -3771,11 +4555,13 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3771
4555
|
const [pipe, tokenizer] = await Promise.all([
|
|
3772
4556
|
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3773
4557
|
...modelOptions,
|
|
4558
|
+
...opts.localFilesOnly ? { local_files_only: true } : {},
|
|
3774
4559
|
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {},
|
|
3775
4560
|
...progress
|
|
3776
4561
|
}),
|
|
3777
4562
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3778
4563
|
revision: modelOptions.revision,
|
|
4564
|
+
...opts.localFilesOnly ? { local_files_only: true } : {},
|
|
3779
4565
|
...progress
|
|
3780
4566
|
})
|
|
3781
4567
|
]);
|
|
@@ -3821,7 +4607,8 @@ import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6
|
|
|
3821
4607
|
import {
|
|
3822
4608
|
CatalogDocumentSchema,
|
|
3823
4609
|
KnowledgeDocumentIdSchema,
|
|
3824
|
-
|
|
4610
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
|
|
4611
|
+
KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
|
|
3825
4612
|
MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
|
|
3826
4613
|
assetExtension as assetExtension3,
|
|
3827
4614
|
formatKnowledgeUri,
|
|
@@ -4517,7 +5304,15 @@ var HUGO_SECTION_PAGE = "_index";
|
|
|
4517
5304
|
var DOCFX_JSON = "docfx.json";
|
|
4518
5305
|
var WARNING_EXAMPLES2 = 20;
|
|
4519
5306
|
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
4520
|
-
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
5307
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
5308
|
+
"title",
|
|
5309
|
+
"summary",
|
|
5310
|
+
"aliases",
|
|
5311
|
+
"id",
|
|
5312
|
+
"order",
|
|
5313
|
+
"subcategory",
|
|
5314
|
+
"locations"
|
|
5315
|
+
]);
|
|
4521
5316
|
function readFrontMatter(raw, file, conventions) {
|
|
4522
5317
|
const { hugo } = conventions;
|
|
4523
5318
|
let data;
|
|
@@ -4536,6 +5331,12 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4536
5331
|
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
4537
5332
|
return value.trim() || void 0;
|
|
4538
5333
|
};
|
|
5334
|
+
if (data.locations !== void 0) {
|
|
5335
|
+
const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
|
|
5336
|
+
if (!locations.success)
|
|
5337
|
+
throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
|
|
5338
|
+
out.locations = locations.data;
|
|
5339
|
+
}
|
|
4539
5340
|
out.title = str("title");
|
|
4540
5341
|
out.summary = str("summary");
|
|
4541
5342
|
out.id = str("id");
|
|
@@ -4548,7 +5349,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4548
5349
|
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
4549
5350
|
}
|
|
4550
5351
|
if (data.order !== void 0 && data.order !== null) {
|
|
4551
|
-
const order =
|
|
5352
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4552
5353
|
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
4553
5354
|
out.order = order.data;
|
|
4554
5355
|
}
|
|
@@ -4569,7 +5370,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4569
5370
|
subcategory.title = record.title.trim();
|
|
4570
5371
|
}
|
|
4571
5372
|
if (record.order !== void 0 && record.order !== null) {
|
|
4572
|
-
const order =
|
|
5373
|
+
const order = KnowledgeOrdinalSchema2.safeParse(record.order);
|
|
4573
5374
|
if (!order.success)
|
|
4574
5375
|
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
4575
5376
|
subcategory.order = order.data;
|
|
@@ -4580,7 +5381,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4580
5381
|
if (data.draft === true) out.unpublished = "draft";
|
|
4581
5382
|
else if (data.headless === true) out.unpublished = "headless";
|
|
4582
5383
|
if (data.weight !== void 0 && data.weight !== null) {
|
|
4583
|
-
const weight =
|
|
5384
|
+
const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
|
|
4584
5385
|
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
4585
5386
|
if (weight.data > 0) out.weight = weight.data;
|
|
4586
5387
|
}
|
|
@@ -4653,7 +5454,7 @@ async function readTopicSidecar(dir) {
|
|
|
4653
5454
|
out.description = data.description.trim();
|
|
4654
5455
|
}
|
|
4655
5456
|
if (data.order !== void 0 && data.order !== null) {
|
|
4656
|
-
const order =
|
|
5457
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4657
5458
|
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
4658
5459
|
out.order = order.data;
|
|
4659
5460
|
}
|
|
@@ -4885,6 +5686,7 @@ function rewriteLinks(ctx, markdown) {
|
|
|
4885
5686
|
return line.replace(
|
|
4886
5687
|
INLINE_LINK,
|
|
4887
5688
|
(whole, bang, text, target, title) => {
|
|
5689
|
+
if (bang === "!" && ctx.skipImages) return text;
|
|
4888
5690
|
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
4889
5691
|
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
4890
5692
|
}
|
|
@@ -4955,22 +5757,16 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4955
5757
|
};
|
|
4956
5758
|
const placements = /* @__PURE__ */ new Map();
|
|
4957
5759
|
if (outline) {
|
|
4958
|
-
const duplicates = [];
|
|
4959
5760
|
const walk = (topic, chain) => {
|
|
4960
5761
|
for (const entry of topic.entries) {
|
|
4961
5762
|
const key = entry.file.normalize("NFC");
|
|
4962
|
-
|
|
4963
|
-
|
|
5763
|
+
const existing = placements.get(key) ?? [];
|
|
5764
|
+
existing.push({ chain, entry });
|
|
5765
|
+
placements.set(key, existing);
|
|
4964
5766
|
}
|
|
4965
5767
|
for (const child of topic.children) walk(child, [...chain, child]);
|
|
4966
5768
|
};
|
|
4967
5769
|
walk(outline.root, []);
|
|
4968
|
-
warnCapped(
|
|
4969
|
-
warn,
|
|
4970
|
-
duplicates,
|
|
4971
|
-
(file) => `${file}: listed more than once in the table of contents; the first place wins`,
|
|
4972
|
-
(more) => `${more} more files are listed more than once in the table of contents`
|
|
4973
|
-
);
|
|
4974
5770
|
}
|
|
4975
5771
|
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
4976
5772
|
let outlineTooDeep = false;
|
|
@@ -5045,7 +5841,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5045
5841
|
let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
5046
5842
|
let ordinal = fm.order;
|
|
5047
5843
|
let topicPath;
|
|
5048
|
-
const
|
|
5844
|
+
const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
|
|
5845
|
+
const placement = documentPlacements[0];
|
|
5049
5846
|
if (placement) {
|
|
5050
5847
|
topicPath = ensureOutlineChain(placement.chain);
|
|
5051
5848
|
if (placement.entry.title) title = placement.entry.title;
|
|
@@ -5075,6 +5872,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5075
5872
|
const parentId = topicPath[topicPath.length - 1];
|
|
5076
5873
|
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
5077
5874
|
}
|
|
5875
|
+
const tocReferences = [];
|
|
5876
|
+
const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
|
|
5877
|
+
for (const extra of documentPlacements.slice(1)) {
|
|
5878
|
+
let path = ensureOutlineChain(extra.chain);
|
|
5879
|
+
if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
|
|
5880
|
+
const leaf = path.at(-1);
|
|
5881
|
+
if (placedLeaves.has(leaf)) continue;
|
|
5882
|
+
placedLeaves.add(leaf);
|
|
5883
|
+
const order = extra.entry.order ?? fm.order;
|
|
5884
|
+
tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
|
|
5885
|
+
}
|
|
5078
5886
|
const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
|
|
5079
5887
|
loaded.push({
|
|
5080
5888
|
rel,
|
|
@@ -5086,6 +5894,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5086
5894
|
...summary ? { summary } : {},
|
|
5087
5895
|
language: opts.language,
|
|
5088
5896
|
topicPath,
|
|
5897
|
+
...tocReferences.length ? { tocReferences } : {},
|
|
5898
|
+
...fm.locations ? { locations: fm.locations } : {},
|
|
5089
5899
|
markdown: fm.body,
|
|
5090
5900
|
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
5091
5901
|
...ordinal !== void 0 ? { ordinal } : {},
|
|
@@ -5115,7 +5925,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5115
5925
|
);
|
|
5116
5926
|
warnCapped(
|
|
5117
5927
|
warn,
|
|
5118
|
-
[...placements].filter(([file]) => !seenRel.has(file)).map(([
|
|
5928
|
+
[...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
|
|
5119
5929
|
(file) => `${file}: named by the table of contents but not found among the Markdown files`,
|
|
5120
5930
|
(more) => `${more} more files named by the table of contents are not among the Markdown files`
|
|
5121
5931
|
);
|
|
@@ -5152,6 +5962,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5152
5962
|
// A docfx docset draws on dependent repositories (`~/reusable-content`)
|
|
5153
5963
|
// that a checkout never holds, so a missing image is expected there.
|
|
5154
5964
|
missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
|
|
5965
|
+
skipImages: opts.skipImages ?? false,
|
|
5155
5966
|
tildeIsRoot: docfx,
|
|
5156
5967
|
warn: (message) => linkWarnings.push(message)
|
|
5157
5968
|
};
|
|
@@ -5288,7 +6099,7 @@ function newerRegistryEntries(registry, installed) {
|
|
|
5288
6099
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
5289
6100
|
|
|
5290
6101
|
// src/signatures/jcs.ts
|
|
5291
|
-
import { canonicalizeJson as
|
|
6102
|
+
import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
|
|
5292
6103
|
export {
|
|
5293
6104
|
BGE_SMALL_EN_V15_1,
|
|
5294
6105
|
BODY_CODEC_MIN_BYTES,
|
|
@@ -5328,6 +6139,7 @@ export {
|
|
|
5328
6139
|
MULTILINGUAL_E5_SMALL_2,
|
|
5329
6140
|
PARQUET_EXPORT_VERSION,
|
|
5330
6141
|
PARQUET_REPORT_PATH,
|
|
6142
|
+
QUERY_STOP_WORDS,
|
|
5331
6143
|
README_PATH,
|
|
5332
6144
|
RERANK_CANDIDATES_MAX,
|
|
5333
6145
|
RERANK_CANDIDATES_MIN,
|
|
@@ -5340,13 +6152,14 @@ export {
|
|
|
5340
6152
|
SHARD_MAX_CHUNKS,
|
|
5341
6153
|
SHARD_TARGET_CHUNKS,
|
|
5342
6154
|
SOURCE_NOTICES_PATH,
|
|
6155
|
+
TITLE_ROUTE_SHARDS,
|
|
5343
6156
|
TRANSFORMERS_PEER_RANGE,
|
|
5344
6157
|
ZIP_FIXED_MTIME,
|
|
5345
6158
|
assertDuckdbCli,
|
|
5346
6159
|
assetReferences,
|
|
5347
6160
|
assignOutlineOrders,
|
|
5348
6161
|
asymmetricTopK,
|
|
5349
|
-
|
|
6162
|
+
canonicalizeJson3 as canonicalizeJson,
|
|
5350
6163
|
centerVector,
|
|
5351
6164
|
chunkContentHash,
|
|
5352
6165
|
chunkMarkdownProfile,
|