@bendyline/gezel-knowledge 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -6
- package/dist/index.d.ts +510 -44
- package/dist/index.js +2118 -239
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
ROUTER_DB_PATH,
|
|
19
19
|
SOURCE_NOTICES_PATH,
|
|
20
20
|
ZIP_FIXED_MTIME,
|
|
21
|
+
formatAtLeast,
|
|
21
22
|
isSupportedFormatVersion,
|
|
22
23
|
isSupportedIndexSchemaVersion
|
|
23
24
|
} from "@bendyline/gezk";
|
|
@@ -37,7 +38,7 @@ function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY)
|
|
|
37
38
|
}
|
|
38
39
|
|
|
39
40
|
// src/format/ddl.ts
|
|
40
|
-
import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
|
|
41
|
+
import { ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, shardDdlFor } from "@bendyline/gezk";
|
|
41
42
|
|
|
42
43
|
// src/format/ids.ts
|
|
43
44
|
import { documentSlug } from "@bendyline/gezk";
|
|
@@ -47,6 +48,7 @@ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
|
47
48
|
import {
|
|
48
49
|
centerVector,
|
|
49
50
|
l2Normalize,
|
|
51
|
+
profileUnitVector,
|
|
50
52
|
quantizeBinary,
|
|
51
53
|
quantizeBinaryForProfile,
|
|
52
54
|
quantizeInt8,
|
|
@@ -180,12 +182,11 @@ function splitOversizeLine(line, opts) {
|
|
|
180
182
|
}
|
|
181
183
|
|
|
182
184
|
// src/compiler/compile.ts
|
|
183
|
-
import { createHash } from "crypto";
|
|
185
|
+
import { createHash as createHash3 } from "crypto";
|
|
184
186
|
import {
|
|
185
187
|
closeSync,
|
|
186
188
|
mkdirSync,
|
|
187
189
|
openSync,
|
|
188
|
-
readFileSync,
|
|
189
190
|
readSync,
|
|
190
191
|
rmSync,
|
|
191
192
|
statSync,
|
|
@@ -196,16 +197,15 @@ import { join } from "path";
|
|
|
196
197
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
197
198
|
import {
|
|
198
199
|
GEZK_MANIFEST_KIND,
|
|
199
|
-
KnowledgeAssetPathSchema,
|
|
200
200
|
KnowledgeCatalogManifestSchema,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
assetExtension,
|
|
205
|
-
assetKindForExtension,
|
|
201
|
+
KnowledgeLocationsSchema,
|
|
202
|
+
KnowledgeOrdinalSchema,
|
|
203
|
+
KnowledgeTocReferenceSchema,
|
|
206
204
|
canonicalizeJson,
|
|
207
|
-
|
|
208
|
-
|
|
205
|
+
embeddingProfileMinimumFormat,
|
|
206
|
+
isKnowledgeMediaAssetPath,
|
|
207
|
+
normalizeLongitude,
|
|
208
|
+
spatialManifest
|
|
209
209
|
} from "@bendyline/gezk";
|
|
210
210
|
|
|
211
211
|
// src/archive/write.ts
|
|
@@ -251,28 +251,137 @@ var nodeRequire2 = createRequire2(import.meta.url);
|
|
|
251
251
|
var sqlite = nodeRequire2("node:sqlite");
|
|
252
252
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
253
253
|
|
|
254
|
+
// src/query-stopwords.ts
|
|
255
|
+
import { QUERY_STOP_WORDS } from "@bendyline/gezk";
|
|
256
|
+
|
|
254
257
|
// src/reader/fts-query.ts
|
|
255
|
-
function
|
|
258
|
+
function contentTokens(query) {
|
|
256
259
|
const tokens = [
|
|
257
260
|
...new Set((query.normalize("NFKC").match(/[\p{L}\p{N}_]+/gu) ?? []).slice(0, 16))
|
|
258
261
|
];
|
|
262
|
+
const content = tokens.filter((t) => !QUERY_STOP_WORDS.has(t.toLowerCase()));
|
|
263
|
+
return content.length > 0 ? content : tokens;
|
|
264
|
+
}
|
|
265
|
+
function contentFtsQuery(query) {
|
|
266
|
+
const tokens = contentTokens(query);
|
|
267
|
+
return tokens.length > 0 ? tokens.map(quoteToken).join(" OR ") : null;
|
|
268
|
+
}
|
|
269
|
+
function selectiveFtsQuery(query, rows, rowsMatching) {
|
|
270
|
+
const tokens = contentTokens(query);
|
|
259
271
|
if (tokens.length === 0) return null;
|
|
260
|
-
|
|
272
|
+
const counted = tokens.map((token) => {
|
|
273
|
+
const phrase = quoteToken(token);
|
|
274
|
+
return { phrase, hits: rowsMatching(phrase) };
|
|
275
|
+
});
|
|
276
|
+
const kept = counted.filter((c) => c.hits * 2 <= rows);
|
|
277
|
+
const chosen = kept.length > 0 ? kept : [counted.reduce((rarest, c) => c.hits < rarest.hits ? c : rarest)];
|
|
278
|
+
return chosen.map((c) => c.phrase).join(" OR ");
|
|
261
279
|
}
|
|
262
280
|
var DOCUMENT_FTS_WEIGHTS = { title: 10, summary: 1, aliases: 5 };
|
|
263
281
|
var DOCUMENT_FTS_ORDER = `bm25(fts_documents, ${DOCUMENT_FTS_WEIGHTS.title}.0, ${DOCUMENT_FTS_WEIGHTS.summary}.0, ${DOCUMENT_FTS_WEIGHTS.aliases}.0)`;
|
|
264
|
-
function documentFtsTopIds(db,
|
|
265
|
-
const
|
|
266
|
-
return
|
|
282
|
+
function documentFtsTopIds(db, query, limit, allowedDocumentIds) {
|
|
283
|
+
const match = contentFtsQuery(query);
|
|
284
|
+
if (!match || allowedDocumentIds?.size === 0) return [];
|
|
285
|
+
const scope = allowedDocumentIds ? JSON.stringify([...allowedDocumentIds]) : null;
|
|
286
|
+
const title = query.normalize("NFKC").trim();
|
|
287
|
+
const ids = namedTitleMatches(db, query, limit, allowedDocumentIds).map((m) => m.documentId);
|
|
288
|
+
const rest = db.prepare(
|
|
267
289
|
`SELECT document_id FROM fts_documents WHERE fts_documents MATCH ?
|
|
268
|
-
|
|
269
|
-
|
|
290
|
+
${scope === null ? "" : "AND document_id IN (SELECT value FROM json_each(?))"}
|
|
291
|
+
ORDER BY (title = ? COLLATE NOCASE) DESC, ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
292
|
+
).all(
|
|
293
|
+
...scope === null ? [match, title, limit + ids.length] : [match, scope, title, limit + ids.length]
|
|
294
|
+
);
|
|
295
|
+
for (const row of rest) {
|
|
296
|
+
if (ids.length >= limit) break;
|
|
297
|
+
if (!ids.includes(row.document_id)) ids.push(row.document_id);
|
|
298
|
+
}
|
|
299
|
+
return ids;
|
|
300
|
+
}
|
|
301
|
+
var NAME_STOPWORDS = new Set(
|
|
302
|
+
"a an the of in on at to for by with from and or nor but as into onto about than then is are was were be been being am do does did done has have had having who whom whose what which when where why how whats wheres it its this that these those there their them they he him his she her hers i me my we us our you your not no yes if so can could would should will shall may might must".split(" ")
|
|
303
|
+
);
|
|
304
|
+
var NAMED_TITLE_CANDIDATES = 256;
|
|
305
|
+
var TITLE_DF_CACHE_MAX = 5e4;
|
|
306
|
+
var EXACT_TITLE_SCORE = Number.POSITIVE_INFINITY;
|
|
307
|
+
var SAME_WORDS_SCORE = Number.MAX_VALUE;
|
|
308
|
+
function nameWords(text) {
|
|
309
|
+
return text.normalize("NFKC").toLowerCase().normalize("NFD").replace(/\p{M}+/gu, "").match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
310
|
+
}
|
|
311
|
+
function quoteToken(token) {
|
|
312
|
+
return `"${token.replace(/"/g, '""')}"`;
|
|
313
|
+
}
|
|
314
|
+
var titleStatsByDb = /* @__PURE__ */ new WeakMap();
|
|
315
|
+
function titleStats(db) {
|
|
316
|
+
let stats = titleStatsByDb.get(db);
|
|
317
|
+
if (!stats) {
|
|
318
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM documents").get();
|
|
319
|
+
stats = {
|
|
320
|
+
documents: Math.max(1, Number(row.c)),
|
|
321
|
+
df: /* @__PURE__ */ new Map(),
|
|
322
|
+
dfQuery: db.prepare("SELECT COUNT(*) AS c FROM fts_documents WHERE fts_documents MATCH ?"),
|
|
323
|
+
candidates: db.prepare(
|
|
324
|
+
`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
325
|
+
ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`
|
|
326
|
+
)
|
|
327
|
+
};
|
|
328
|
+
titleStatsByDb.set(db, stats);
|
|
329
|
+
}
|
|
330
|
+
return stats;
|
|
331
|
+
}
|
|
332
|
+
function titleDf(stats, word) {
|
|
333
|
+
let df = stats.df.get(word);
|
|
334
|
+
if (df === void 0) {
|
|
335
|
+
if (stats.df.size >= TITLE_DF_CACHE_MAX) stats.df.clear();
|
|
336
|
+
const row = stats.dfQuery.get(`title : ${quoteToken(word)}`);
|
|
337
|
+
df = Number(row.c);
|
|
338
|
+
stats.df.set(word, df);
|
|
339
|
+
}
|
|
340
|
+
return df;
|
|
341
|
+
}
|
|
342
|
+
function namedTitleMatches(db, query, limit, allowedDocumentIds) {
|
|
343
|
+
if (allowedDocumentIds?.size === 0) return [];
|
|
344
|
+
const raw = query.normalize("NFKC").trim();
|
|
345
|
+
const words = [...new Set(nameWords(raw))].slice(0, 16);
|
|
346
|
+
const content = words.filter((w) => !NAME_STOPWORDS.has(w));
|
|
347
|
+
if (content.length === 0 || limit <= 0) return [];
|
|
348
|
+
const asked = new Set(words);
|
|
349
|
+
const capitalized = new Set(
|
|
350
|
+
(raw.match(/[\p{L}\p{N}]+/gu) ?? []).filter((w, i) => i > 0 && /^\p{Lu}/u.test(w)).flatMap(nameWords)
|
|
351
|
+
);
|
|
352
|
+
const queryWords = nameWords(raw).join(" ");
|
|
353
|
+
const stats = titleStats(db);
|
|
354
|
+
const expression = `title : (${content.map(quoteToken).join(" OR ")})`;
|
|
355
|
+
const rows = allowedDocumentIds ? db.prepare(`SELECT document_id, title FROM fts_documents WHERE fts_documents MATCH ?
|
|
356
|
+
AND document_id IN (SELECT value FROM json_each(?)) ORDER BY ${DOCUMENT_FTS_ORDER} LIMIT ?`).all(expression, JSON.stringify([...allowedDocumentIds]), NAMED_TITLE_CANDIDATES) : stats.candidates.all(expression, NAMED_TITLE_CANDIDATES);
|
|
357
|
+
const named = [];
|
|
358
|
+
for (const row of rows) {
|
|
359
|
+
const titleWords = nameWords(row.title);
|
|
360
|
+
if (titleWords.length === 0 || !titleWords.every((w) => asked.has(w))) continue;
|
|
361
|
+
const distinct = [...new Set(titleWords)].filter((w) => !NAME_STOPWORDS.has(w));
|
|
362
|
+
if (distinct.length === 0) continue;
|
|
363
|
+
let score;
|
|
364
|
+
if (row.title.normalize("NFKC").trim().toLowerCase() === raw.toLowerCase()) {
|
|
365
|
+
score = EXACT_TITLE_SCORE;
|
|
366
|
+
} else if (titleWords.join(" ") === queryWords) {
|
|
367
|
+
score = SAME_WORDS_SCORE;
|
|
368
|
+
} else {
|
|
369
|
+
score = 0;
|
|
370
|
+
for (const w of distinct) {
|
|
371
|
+
score += (capitalized.has(w) ? 2 : 1) * Math.log(stats.documents / Math.max(1, titleDf(stats, w)));
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
named.push({ documentId: row.document_id, score, order: named.length });
|
|
375
|
+
}
|
|
376
|
+
named.sort((a, b) => compareScores(b.score, a.score) || a.order - b.order);
|
|
377
|
+
return named.slice(0, limit).map(({ documentId, score }) => ({ documentId, score }));
|
|
378
|
+
}
|
|
379
|
+
function compareScores(a, b) {
|
|
380
|
+
return a === b ? 0 : a > b ? 1 : -1;
|
|
270
381
|
}
|
|
271
382
|
var SMOKE_QUERY_TOP_N = 10;
|
|
272
383
|
function documentSmokeQueryMisses(db, smoke) {
|
|
273
|
-
const
|
|
274
|
-
if (!match) return [...smoke.expectedDocumentIds];
|
|
275
|
-
const top = documentFtsTopIds(db, match, SMOKE_QUERY_TOP_N, smoke.query);
|
|
384
|
+
const top = documentFtsTopIds(db, smoke.query, SMOKE_QUERY_TOP_N);
|
|
276
385
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
277
386
|
}
|
|
278
387
|
|
|
@@ -284,6 +393,216 @@ var KNOWLEDGE_TOOLCHAIN = {
|
|
|
284
393
|
version: nodeRequire3("../package.json").version
|
|
285
394
|
};
|
|
286
395
|
|
|
396
|
+
// src/compiler/assets.ts
|
|
397
|
+
import { createHash } from "crypto";
|
|
398
|
+
import { readFileSync } from "fs";
|
|
399
|
+
import {
|
|
400
|
+
KnowledgeAssetPathSchema,
|
|
401
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
402
|
+
assetExtension,
|
|
403
|
+
assetKindForExtension,
|
|
404
|
+
knowledgeAssetLimitsProblem,
|
|
405
|
+
maxKnowledgeAssetBytes,
|
|
406
|
+
sniffAssetType,
|
|
407
|
+
svgInertnessProblem
|
|
408
|
+
} from "@bendyline/gezk";
|
|
409
|
+
function prepareAssets(assets, extraFiles, opts) {
|
|
410
|
+
for (const path of Object.keys(extraFiles)) {
|
|
411
|
+
if (path.startsWith("assets/")) {
|
|
412
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
416
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
417
|
+
}
|
|
418
|
+
const seen = /* @__PURE__ */ new Set();
|
|
419
|
+
const prepared = [];
|
|
420
|
+
const skippedPaths = /* @__PURE__ */ new Set();
|
|
421
|
+
const rejectContent = (path, message) => {
|
|
422
|
+
if (opts.invalidAssets !== "warn") throw new Error(message);
|
|
423
|
+
skippedPaths.add(path);
|
|
424
|
+
opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
|
|
425
|
+
};
|
|
426
|
+
for (const asset of assets) {
|
|
427
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
428
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
429
|
+
const key = asset.path.toLowerCase();
|
|
430
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
431
|
+
seen.add(key);
|
|
432
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
433
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
434
|
+
}
|
|
435
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
436
|
+
const limit = maxKnowledgeAssetBytes(asset.path);
|
|
437
|
+
if (bytes.byteLength > limit) {
|
|
438
|
+
rejectContent(
|
|
439
|
+
asset.path,
|
|
440
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${limit}`
|
|
441
|
+
);
|
|
442
|
+
continue;
|
|
443
|
+
}
|
|
444
|
+
const ext = assetExtension(asset.path);
|
|
445
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
446
|
+
const actual = sniffAssetType(bytes);
|
|
447
|
+
if (expected === null || actual !== expected) {
|
|
448
|
+
rejectContent(
|
|
449
|
+
asset.path,
|
|
450
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
451
|
+
);
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
if (ext === "svg") {
|
|
455
|
+
const problem = svgInertnessProblem(bytes);
|
|
456
|
+
if (problem) {
|
|
457
|
+
rejectContent(asset.path, `asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
prepared.push({
|
|
462
|
+
path: asset.path,
|
|
463
|
+
bytes,
|
|
464
|
+
sizeBytes: bytes.byteLength,
|
|
465
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
466
|
+
...asset.attribution ? { attribution: asset.attribution } : {}
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
const totals = knowledgeAssetLimitsProblem(prepared);
|
|
470
|
+
if (totals) throw new Error(totals);
|
|
471
|
+
return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
|
|
472
|
+
}
|
|
473
|
+
function omitSkippedAssetReferences(markdown, skippedPaths) {
|
|
474
|
+
if (skippedPaths.size === 0) return markdown;
|
|
475
|
+
return markdown.replace(
|
|
476
|
+
/!?\[((?:\\.|[^\[\]\\\n]|\[[^\]\n]*\])*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+(?:"[^"]*"|'[^']*'|\([^)]*\)))?\s*\)/g,
|
|
477
|
+
(whole, text, target) => skippedPaths.has(target) ? text : whole
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
// src/compiler/media-rows.ts
|
|
482
|
+
import { createHash as createHash2 } from "crypto";
|
|
483
|
+
import {
|
|
484
|
+
assetContentType,
|
|
485
|
+
assetModality
|
|
486
|
+
} from "@bendyline/gezk";
|
|
487
|
+
var MEDIA_ROW_TEXT_MAX_TOKENS = 128;
|
|
488
|
+
var MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
|
|
489
|
+
var MEDIA_REFERENCE = /!\[([^\]\n]*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+"[^"\n]*")?\s*\)/g;
|
|
490
|
+
function mediaReferences(markdown) {
|
|
491
|
+
const out = [];
|
|
492
|
+
markdown.split("\n").forEach((text, index) => {
|
|
493
|
+
for (const match of text.matchAll(MEDIA_REFERENCE)) {
|
|
494
|
+
out.push({
|
|
495
|
+
alt: (match[1] ?? "").trim(),
|
|
496
|
+
path: match[2],
|
|
497
|
+
line: index + 1,
|
|
498
|
+
caption: text.replace(MEDIA_REFERENCE, " ").replace(/\s+/g, " ").trim()
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
return out;
|
|
503
|
+
}
|
|
504
|
+
function humanizeAssetName(path) {
|
|
505
|
+
const base = path.slice(path.lastIndexOf("/") + 1);
|
|
506
|
+
const stem2 = base.includes(".") ? base.slice(0, base.lastIndexOf(".")) : base;
|
|
507
|
+
return stem2.replace(/[-_.]+/g, " ").replace(/\s+/g, " ").trim();
|
|
508
|
+
}
|
|
509
|
+
function boundedText(parts, countTokens) {
|
|
510
|
+
const unique = [];
|
|
511
|
+
for (const part of parts.map((p) => p.trim()).filter(Boolean)) {
|
|
512
|
+
if (!unique.some((u) => u.toLowerCase() === part.toLowerCase())) unique.push(part);
|
|
513
|
+
}
|
|
514
|
+
let text = unique.join("\n").normalize("NFC");
|
|
515
|
+
if (countTokens(text) <= MEDIA_ROW_TEXT_MAX_TOKENS) return text;
|
|
516
|
+
const words = text.split(/(\s+)/);
|
|
517
|
+
let lo = 0;
|
|
518
|
+
let hi = words.length;
|
|
519
|
+
while (lo < hi) {
|
|
520
|
+
const mid = Math.ceil((lo + hi) / 2);
|
|
521
|
+
if (countTokens(words.slice(0, mid).join("")) <= MEDIA_ROW_TEXT_MAX_TOKENS) lo = mid;
|
|
522
|
+
else hi = mid - 1;
|
|
523
|
+
}
|
|
524
|
+
text = words.slice(0, lo).join("").trim();
|
|
525
|
+
return text;
|
|
526
|
+
}
|
|
527
|
+
function headingAt(chunks, line) {
|
|
528
|
+
const inside = chunks.find((c) => c.lineStart <= line && line <= c.lineEnd);
|
|
529
|
+
if (inside) return inside.headingPath;
|
|
530
|
+
let before;
|
|
531
|
+
for (const c of chunks) if (c.lineEnd < line) before = c;
|
|
532
|
+
return before?.headingPath ?? [];
|
|
533
|
+
}
|
|
534
|
+
function windowHash(assetSha256, startMs, endMs) {
|
|
535
|
+
if (startMs === null || endMs === null) return assetSha256;
|
|
536
|
+
return createHash2("sha256").update(`${assetSha256}\0${startMs}\0${endMs}`).digest("hex");
|
|
537
|
+
}
|
|
538
|
+
async function collectMediaRows(args) {
|
|
539
|
+
const byPath = new Map(args.assets.map((a) => [a.path, a]));
|
|
540
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
541
|
+
const warned = /* @__PURE__ */ new Set();
|
|
542
|
+
const out = /* @__PURE__ */ new Map();
|
|
543
|
+
for (const doc of args.documents) {
|
|
544
|
+
const rows = [];
|
|
545
|
+
const seen = /* @__PURE__ */ new Set();
|
|
546
|
+
for (const ref of mediaReferences(doc.markdown)) {
|
|
547
|
+
if (seen.has(ref.path)) continue;
|
|
548
|
+
seen.add(ref.path);
|
|
549
|
+
const asset = byPath.get(ref.path);
|
|
550
|
+
const modality = assetModality(ref.path);
|
|
551
|
+
const mimeType = assetContentType(ref.path);
|
|
552
|
+
if (!asset || !modality || !mimeType || mimeType === "image/svg+xml") continue;
|
|
553
|
+
const block = modality === "audio" ? args.profile.media?.audio : modality === "video" ? args.profile.media?.video : args.profile.media?.image;
|
|
554
|
+
if (!block) {
|
|
555
|
+
if (!warned.has(modality)) {
|
|
556
|
+
warned.add(modality);
|
|
557
|
+
args.onWarning?.(
|
|
558
|
+
`profile ${args.profile.id} describes no ${modality} encoder; ${modality} references get no media rows`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
let pending = embedded.get(ref.path);
|
|
564
|
+
if (!pending) {
|
|
565
|
+
pending = args.embedMedia({ modality, path: ref.path, bytes: asset.bytes });
|
|
566
|
+
embedded.set(ref.path, pending);
|
|
567
|
+
}
|
|
568
|
+
const results = [...await pending].sort((a, b) => (a.startMs ?? 0) - (b.startMs ?? 0));
|
|
569
|
+
const headingPath = headingAt(doc.chunks, ref.line);
|
|
570
|
+
for (const result of results) {
|
|
571
|
+
const startMs = modality === "image" ? null : result.startMs ?? 0;
|
|
572
|
+
const endMs = modality === "image" ? null : result.endMs ?? null;
|
|
573
|
+
if (startMs !== null && (endMs === null || endMs <= startMs)) {
|
|
574
|
+
throw new Error(`${ref.path}: a ${modality} window needs startMs < endMs`);
|
|
575
|
+
}
|
|
576
|
+
rows.push({
|
|
577
|
+
modality,
|
|
578
|
+
assetPath: ref.path,
|
|
579
|
+
mimeType,
|
|
580
|
+
width: result.width ?? null,
|
|
581
|
+
height: result.height ?? null,
|
|
582
|
+
startMs,
|
|
583
|
+
endMs,
|
|
584
|
+
attributionJson: asset.attribution ? JSON.stringify(asset.attribution) : null,
|
|
585
|
+
headingPath,
|
|
586
|
+
line: ref.line,
|
|
587
|
+
text: boundedText(
|
|
588
|
+
[ref.alt, ref.caption, humanizeAssetName(ref.path), result.transcript ?? ""],
|
|
589
|
+
args.countTokens
|
|
590
|
+
),
|
|
591
|
+
contentHash: windowHash(asset.sha256, startMs, endMs),
|
|
592
|
+
vector: result.vector
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (rows.length > MAX_MEDIA_ROWS_PER_DOCUMENT) {
|
|
597
|
+
throw new Error(
|
|
598
|
+
`document ${doc.id} has ${rows.length} media rows; the limit is ${MAX_MEDIA_ROWS_PER_DOCUMENT}`
|
|
599
|
+
);
|
|
600
|
+
}
|
|
601
|
+
if (rows.length > 0) out.set(doc.id, rows);
|
|
602
|
+
}
|
|
603
|
+
return out;
|
|
604
|
+
}
|
|
605
|
+
|
|
287
606
|
// src/compiler/compile.ts
|
|
288
607
|
async function compileKnowledgeCatalog(opts) {
|
|
289
608
|
if (opts.topics.length === 0) {
|
|
@@ -292,7 +611,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
292
611
|
);
|
|
293
612
|
}
|
|
294
613
|
const topicById = assertTopicForest(opts.topics);
|
|
295
|
-
const
|
|
614
|
+
const { assets: preparedAssets, skippedPaths } = prepareAssets(
|
|
615
|
+
opts.assets ?? [],
|
|
616
|
+
opts.extraFiles ?? {},
|
|
617
|
+
opts
|
|
618
|
+
);
|
|
296
619
|
const profile = opts.embeddingProfile;
|
|
297
620
|
const chunkerOpts = {
|
|
298
621
|
unit: opts.chunkingProfile.unit,
|
|
@@ -302,23 +625,75 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
302
625
|
};
|
|
303
626
|
const prepared = [];
|
|
304
627
|
const seenIds = /* @__PURE__ */ new Set();
|
|
305
|
-
for await (const
|
|
628
|
+
for await (const input of opts.documents) {
|
|
629
|
+
const locations = KnowledgeLocationsSchema.parse(input.locations ?? []).map((location) => ({
|
|
630
|
+
...location,
|
|
631
|
+
longitude: normalizeLongitude(location.longitude)
|
|
632
|
+
})).sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
|
|
633
|
+
const doc = {
|
|
634
|
+
...input,
|
|
635
|
+
locations,
|
|
636
|
+
markdown: omitSkippedAssetReferences(input.markdown, skippedPaths)
|
|
637
|
+
};
|
|
306
638
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
307
639
|
seenIds.add(doc.id);
|
|
640
|
+
const leafTopicId = assertTopicPath(topicById, doc.id, doc.topicPath);
|
|
641
|
+
const placements = [
|
|
642
|
+
{
|
|
643
|
+
topicId: leafTopicId,
|
|
644
|
+
ordinal: doc.ordinal === void 0 ? null : KnowledgeOrdinalSchema.parse(doc.ordinal)
|
|
645
|
+
}
|
|
646
|
+
];
|
|
647
|
+
const leaves = /* @__PURE__ */ new Set([leafTopicId]);
|
|
648
|
+
for (const raw of doc.tocReferences ?? []) {
|
|
649
|
+
const reference = KnowledgeTocReferenceSchema.parse(raw);
|
|
650
|
+
const topicId = assertTopicPath(topicById, doc.id, reference.topicPath);
|
|
651
|
+
if (leaves.has(topicId))
|
|
652
|
+
throw new Error(`document ${doc.id} has a repeated TOC placement at '${topicId}'`);
|
|
653
|
+
leaves.add(topicId);
|
|
654
|
+
placements.push({ topicId, ordinal: reference.ordinal ?? null });
|
|
655
|
+
}
|
|
656
|
+
placements.sort((a, b) => a.topicId < b.topicId ? -1 : a.topicId > b.topicId ? 1 : 0);
|
|
308
657
|
prepared.push({
|
|
309
658
|
doc,
|
|
310
659
|
topicPathKey: doc.topicPath.join("/"),
|
|
311
|
-
leafTopicId
|
|
660
|
+
leafTopicId,
|
|
661
|
+
placements,
|
|
312
662
|
metaJson: encodeDocumentMeta(doc),
|
|
313
|
-
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
663
|
+
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts),
|
|
664
|
+
media: []
|
|
314
665
|
});
|
|
315
666
|
}
|
|
316
667
|
prepared.sort(
|
|
317
668
|
(a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
|
|
318
669
|
);
|
|
319
670
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
671
|
+
if (opts.embedMedia) {
|
|
672
|
+
const rows = await collectMediaRows({
|
|
673
|
+
documents: prepared.map((p) => ({
|
|
674
|
+
id: p.doc.id,
|
|
675
|
+
title: p.doc.title,
|
|
676
|
+
markdown: normalizeMarkdown(p.doc.markdown),
|
|
677
|
+
chunks: p.chunks
|
|
678
|
+
})),
|
|
679
|
+
assets: preparedAssets,
|
|
680
|
+
profile,
|
|
681
|
+
embedMedia: opts.embedMedia,
|
|
682
|
+
countTokens: opts.countTokens,
|
|
683
|
+
onWarning: opts.onWarning
|
|
684
|
+
});
|
|
685
|
+
for (const p of prepared) p.media = rows.get(p.doc.id) ?? [];
|
|
686
|
+
}
|
|
687
|
+
const mediaCounts = { image: 0, video: 0, audio: 0 };
|
|
688
|
+
for (const p of prepared) for (const row of p.media) mediaCounts[row.modality] += 1;
|
|
689
|
+
const totalMedia = mediaCounts.image + mediaCounts.video + mediaCounts.audio;
|
|
690
|
+
const totalRows = totalChunks + totalMedia;
|
|
691
|
+
const rowsOf = (p) => p.chunks.length + p.media.length;
|
|
692
|
+
const formatVersion = catalogFormatVersion(profile, preparedAssets, totalMedia);
|
|
693
|
+
const indexSchemaVersion = GEZK_FORMAT_GENERATIONS[formatVersion];
|
|
694
|
+
const shardDdl = shardDdlFor(indexSchemaVersion);
|
|
320
695
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
321
|
-
const embedded =
|
|
696
|
+
const embedded = totalRows <= shardTarget;
|
|
322
697
|
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
323
698
|
const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
|
|
324
699
|
p.doc.title,
|
|
@@ -335,7 +710,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
335
710
|
docVectors = await stageVectors({
|
|
336
711
|
prepared,
|
|
337
712
|
stagedPath,
|
|
338
|
-
|
|
713
|
+
profile,
|
|
339
714
|
batchSize: opts.embedBatchSize ?? 32,
|
|
340
715
|
embed: opts.embed,
|
|
341
716
|
embedInput,
|
|
@@ -348,11 +723,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
348
723
|
if (embedded) {
|
|
349
724
|
for (const p of prepared) shardOf.set(p.doc.id, 0);
|
|
350
725
|
} else if (semanticFill) {
|
|
351
|
-
shardCount = Math.ceil(
|
|
352
|
-
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(
|
|
726
|
+
shardCount = Math.ceil(totalRows / shardTarget);
|
|
727
|
+
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalRows / shardCount * 1.02));
|
|
353
728
|
const assignment = semanticShards(
|
|
354
729
|
docVectors,
|
|
355
|
-
prepared.map(
|
|
730
|
+
prepared.map(rowsOf),
|
|
356
731
|
shardCount,
|
|
357
732
|
capacity,
|
|
358
733
|
seedFor(`${opts.catalog.id}#shard-fill`, 0)
|
|
@@ -364,19 +739,19 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
364
739
|
let current = 0;
|
|
365
740
|
let filled = 0;
|
|
366
741
|
for (const p of prepared) {
|
|
367
|
-
if (filled > 0 && filled + p
|
|
742
|
+
if (filled > 0 && filled + rowsOf(p) > shardTarget) {
|
|
368
743
|
current++;
|
|
369
744
|
filled = 0;
|
|
370
745
|
}
|
|
371
746
|
shardOf.set(p.doc.id, current);
|
|
372
|
-
filled += p
|
|
747
|
+
filled += rowsOf(p);
|
|
373
748
|
}
|
|
374
749
|
shardCount = current + 1;
|
|
375
750
|
}
|
|
376
751
|
const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
|
|
377
752
|
const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
|
|
378
753
|
const shardDbs = /* @__PURE__ */ new Map();
|
|
379
|
-
const router = createCatalogDb(routerStaged);
|
|
754
|
+
const router = createCatalogDb(routerStaged, indexSchemaVersion);
|
|
380
755
|
const closeAll = () => {
|
|
381
756
|
for (const db of [router, ...shardDbs.values()]) {
|
|
382
757
|
try {
|
|
@@ -389,11 +764,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
389
764
|
router.exec(ROUTER_DDL);
|
|
390
765
|
router.exec("BEGIN");
|
|
391
766
|
if (embedded) {
|
|
392
|
-
router.exec(
|
|
767
|
+
router.exec(shardDdl.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
|
|
393
768
|
}
|
|
394
769
|
const commonMeta = {
|
|
395
|
-
format_version: String(
|
|
396
|
-
index_schema_version: String(
|
|
770
|
+
format_version: String(formatVersion),
|
|
771
|
+
index_schema_version: String(indexSchemaVersion),
|
|
397
772
|
catalog_id: opts.catalog.id,
|
|
398
773
|
catalog_version: opts.catalog.version,
|
|
399
774
|
embedding_profile_id: profile.id,
|
|
@@ -411,7 +786,9 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
411
786
|
});
|
|
412
787
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
413
788
|
for (const p of prepared) {
|
|
414
|
-
|
|
789
|
+
for (const placement of p.placements) {
|
|
790
|
+
topicDocCount.set(placement.topicId, (topicDocCount.get(placement.topicId) ?? 0) + 1);
|
|
791
|
+
}
|
|
415
792
|
}
|
|
416
793
|
{
|
|
417
794
|
const stmt = router.prepare(
|
|
@@ -435,6 +812,13 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
435
812
|
body_codec, body_blob)
|
|
436
813
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
437
814
|
);
|
|
815
|
+
const insertLocation = router.prepare(
|
|
816
|
+
`INSERT INTO document_locations (document_id, location_id, latitude, longitude, role, provenance_json)
|
|
817
|
+
VALUES (?, ?, ?, ?, ?, ?)`
|
|
818
|
+
);
|
|
819
|
+
const insertPlacement = router.prepare(
|
|
820
|
+
"INSERT INTO topic_documents (topic_id, document_id, ordinal) VALUES (?, ?, ?)"
|
|
821
|
+
);
|
|
438
822
|
const insertAlias = router.prepare(
|
|
439
823
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
440
824
|
);
|
|
@@ -464,7 +848,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
464
848
|
p.leafTopicId,
|
|
465
849
|
p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
|
|
466
850
|
BigInt(shardOf.get(p.doc.id)),
|
|
467
|
-
BigInt(p
|
|
851
|
+
BigInt(rowsOf(p)),
|
|
468
852
|
p.doc.sourceUrl ?? null,
|
|
469
853
|
p.doc.sourceRevision ?? null,
|
|
470
854
|
p.doc.sourceUpdatedAt ?? null,
|
|
@@ -473,6 +857,29 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
473
857
|
useBrotli ? "br" : "none",
|
|
474
858
|
blob
|
|
475
859
|
);
|
|
860
|
+
for (const location of p.doc.locations ?? []) {
|
|
861
|
+
const provenance = location.provenance ? canonicalizeJson(location.provenance) : null;
|
|
862
|
+
if (provenance !== null && Buffer.byteLength(provenance) > 16384) {
|
|
863
|
+
throw new Error(
|
|
864
|
+
`document ${p.doc.id} location ${location.id} provenance exceeds 16384 bytes`
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
insertLocation.run(
|
|
868
|
+
p.doc.id,
|
|
869
|
+
location.id,
|
|
870
|
+
location.latitude,
|
|
871
|
+
location.longitude,
|
|
872
|
+
location.role,
|
|
873
|
+
provenance
|
|
874
|
+
);
|
|
875
|
+
}
|
|
876
|
+
for (const placement of p.placements) {
|
|
877
|
+
insertPlacement.run(
|
|
878
|
+
placement.topicId,
|
|
879
|
+
p.doc.id,
|
|
880
|
+
placement.ordinal === null ? null : BigInt(placement.ordinal)
|
|
881
|
+
);
|
|
882
|
+
}
|
|
476
883
|
const aliases = [...new Set(p.doc.aliases ?? [])].sort();
|
|
477
884
|
for (const alias of aliases) insertAlias.run(alias, p.doc.id);
|
|
478
885
|
insertFts.run(p.doc.title, p.doc.summary ?? "", aliases.join(" "), p.doc.id);
|
|
@@ -486,8 +893,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
486
893
|
if (embedded) return router;
|
|
487
894
|
let db = shardDbs.get(id);
|
|
488
895
|
if (!db) {
|
|
489
|
-
db = createCatalogDb(join(opts.workDir, shardPath(id)));
|
|
490
|
-
db.exec(
|
|
896
|
+
db = createCatalogDb(join(opts.workDir, shardPath(id)), indexSchemaVersion);
|
|
897
|
+
db.exec(shardDdl);
|
|
491
898
|
db.exec("BEGIN");
|
|
492
899
|
shardDbs.set(id, db);
|
|
493
900
|
}
|
|
@@ -518,7 +925,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
518
925
|
);
|
|
519
926
|
}
|
|
520
927
|
for (let i = 0; i < vectors.length; i++) {
|
|
521
|
-
const
|
|
928
|
+
const raw = vectors[i];
|
|
929
|
+
const unit = staged ? l2Normalize(raw) : profileUnitVector(profile, raw);
|
|
522
930
|
if (unit.length !== profile.dimensions) {
|
|
523
931
|
throw new Error(
|
|
524
932
|
`embedder returned dim ${unit.length}, profile expects ${profile.dimensions}`
|
|
@@ -582,6 +990,46 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
582
990
|
pendingRows.push({ shardId, chunkId });
|
|
583
991
|
if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
|
|
584
992
|
}
|
|
993
|
+
if (p.media.length > 0) {
|
|
994
|
+
const insertMedia = db.prepare(
|
|
995
|
+
`INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
|
|
996
|
+
line_start, line_end, token_count, content_hash, text, modality, asset_path, mime_type,
|
|
997
|
+
width, height, start_ms, end_ms, attribution_json)
|
|
998
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
|
|
999
|
+
);
|
|
1000
|
+
const stmts = vecStmtsFor(db);
|
|
1001
|
+
p.media.forEach((media, index) => {
|
|
1002
|
+
const ordinal = p.chunks.length + index;
|
|
1003
|
+
const headingText = media.headingPath.join(" > ");
|
|
1004
|
+
const row = insertMedia.get(
|
|
1005
|
+
chunkUid(p.doc.id, ordinal, `media:${media.modality}:${media.contentHash}`),
|
|
1006
|
+
p.doc.id,
|
|
1007
|
+
BigInt(ordinal),
|
|
1008
|
+
p.doc.title,
|
|
1009
|
+
JSON.stringify(media.headingPath),
|
|
1010
|
+
headingText,
|
|
1011
|
+
BigInt(media.line),
|
|
1012
|
+
BigInt(media.line),
|
|
1013
|
+
BigInt(opts.countTokens(media.text)),
|
|
1014
|
+
media.contentHash,
|
|
1015
|
+
media.text,
|
|
1016
|
+
media.modality,
|
|
1017
|
+
media.assetPath,
|
|
1018
|
+
media.mimeType,
|
|
1019
|
+
media.width === null ? null : BigInt(media.width),
|
|
1020
|
+
media.height === null ? null : BigInt(media.height),
|
|
1021
|
+
media.startMs === null ? null : BigInt(media.startMs),
|
|
1022
|
+
media.endMs === null ? null : BigInt(media.endMs),
|
|
1023
|
+
media.attributionJson
|
|
1024
|
+
);
|
|
1025
|
+
const chunkId = Number(row.id);
|
|
1026
|
+
insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, media.text);
|
|
1027
|
+
shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
|
|
1028
|
+
const unit = profileUnitVector(profile, media.vector);
|
|
1029
|
+
stmts.vec.run(BigInt(chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
|
|
1030
|
+
stmts.int8.run(BigInt(chunkId), Buffer.from(quantizeInt8(unit).buffer));
|
|
1031
|
+
});
|
|
1032
|
+
}
|
|
585
1033
|
}
|
|
586
1034
|
await flushEmbeds();
|
|
587
1035
|
staged?.close();
|
|
@@ -593,7 +1041,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
593
1041
|
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
594
1042
|
});
|
|
595
1043
|
}
|
|
596
|
-
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(
|
|
1044
|
+
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalRows) });
|
|
597
1045
|
const finalDir = join(opts.workDir, "final");
|
|
598
1046
|
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
599
1047
|
const vacuumed = [];
|
|
@@ -705,7 +1153,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
705
1153
|
files.push({
|
|
706
1154
|
path,
|
|
707
1155
|
sizeBytes: bytes.length,
|
|
708
|
-
sha256:
|
|
1156
|
+
sha256: createHash3("sha256").update(bytes).digest("hex")
|
|
709
1157
|
});
|
|
710
1158
|
}
|
|
711
1159
|
for (const asset of preparedAssets) {
|
|
@@ -714,8 +1162,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
714
1162
|
assertAssetReferencesDeclared(prepared, preparedAssets);
|
|
715
1163
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
716
1164
|
kind: GEZK_MANIFEST_KIND,
|
|
717
|
-
formatVersion
|
|
718
|
-
indexSchemaVersion
|
|
1165
|
+
formatVersion,
|
|
1166
|
+
indexSchemaVersion,
|
|
719
1167
|
id: opts.catalog.id,
|
|
720
1168
|
version: opts.catalog.version,
|
|
721
1169
|
name: opts.catalog.name,
|
|
@@ -744,12 +1192,18 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
744
1192
|
},
|
|
745
1193
|
counts: {
|
|
746
1194
|
documents: prepared.length,
|
|
747
|
-
chunks:
|
|
1195
|
+
chunks: totalRows,
|
|
748
1196
|
shards: shardCount,
|
|
749
|
-
assets: preparedAssets.length
|
|
1197
|
+
assets: preparedAssets.length,
|
|
1198
|
+
...formatVersion === "0.8" ? { media: mediaCounts } : {}
|
|
750
1199
|
},
|
|
751
1200
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
752
|
-
|
|
1201
|
+
spatial: spatialManifest(
|
|
1202
|
+
prepared.flatMap(
|
|
1203
|
+
(p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
|
|
1204
|
+
)
|
|
1205
|
+
),
|
|
1206
|
+
requires: { formatVersion, features: [] },
|
|
753
1207
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
754
1208
|
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
755
1209
|
});
|
|
@@ -773,7 +1227,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
773
1227
|
return {
|
|
774
1228
|
manifest,
|
|
775
1229
|
documents: prepared.length,
|
|
776
|
-
chunks:
|
|
1230
|
+
chunks: totalRows,
|
|
1231
|
+
media: mediaCounts,
|
|
777
1232
|
shards: shardCount,
|
|
778
1233
|
archiveBytes
|
|
779
1234
|
};
|
|
@@ -844,58 +1299,6 @@ function encodeDocumentMeta(doc) {
|
|
|
844
1299
|
}
|
|
845
1300
|
return json;
|
|
846
1301
|
}
|
|
847
|
-
function prepareAssets(assets, extraFiles) {
|
|
848
|
-
for (const path of Object.keys(extraFiles)) {
|
|
849
|
-
if (path.startsWith("assets/")) {
|
|
850
|
-
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
851
|
-
}
|
|
852
|
-
}
|
|
853
|
-
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
854
|
-
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
855
|
-
}
|
|
856
|
-
const seen = /* @__PURE__ */ new Set();
|
|
857
|
-
const prepared = [];
|
|
858
|
-
let total = 0;
|
|
859
|
-
for (const asset of assets) {
|
|
860
|
-
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
861
|
-
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
862
|
-
const key = asset.path.toLowerCase();
|
|
863
|
-
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
864
|
-
seen.add(key);
|
|
865
|
-
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
866
|
-
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
867
|
-
}
|
|
868
|
-
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
869
|
-
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
870
|
-
throw new Error(
|
|
871
|
-
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
872
|
-
);
|
|
873
|
-
}
|
|
874
|
-
total += bytes.byteLength;
|
|
875
|
-
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
876
|
-
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
877
|
-
}
|
|
878
|
-
const ext = assetExtension(asset.path);
|
|
879
|
-
const expected = ext ? assetKindForExtension(ext) : null;
|
|
880
|
-
const actual = sniffAssetType(bytes);
|
|
881
|
-
if (expected === null || actual !== expected) {
|
|
882
|
-
throw new Error(
|
|
883
|
-
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
884
|
-
);
|
|
885
|
-
}
|
|
886
|
-
if (ext === "svg") {
|
|
887
|
-
const problem = svgInertnessProblem(bytes);
|
|
888
|
-
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
889
|
-
}
|
|
890
|
-
prepared.push({
|
|
891
|
-
path: asset.path,
|
|
892
|
-
bytes,
|
|
893
|
-
sizeBytes: bytes.byteLength,
|
|
894
|
-
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
895
|
-
});
|
|
896
|
-
}
|
|
897
|
-
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
898
|
-
}
|
|
899
1302
|
function assertAssetReferencesDeclared(prepared, assets) {
|
|
900
1303
|
const declared = new Set(assets.map((a) => a.path));
|
|
901
1304
|
for (const p of prepared) {
|
|
@@ -909,11 +1312,15 @@ function assertAssetReferencesDeclared(prepared, assets) {
|
|
|
909
1312
|
}
|
|
910
1313
|
}
|
|
911
1314
|
}
|
|
912
|
-
function
|
|
1315
|
+
function catalogFormatVersion(profile, assets, mediaRows = 0) {
|
|
1316
|
+
const needsMedia = mediaRows > 0 || assets.some((asset) => isKnowledgeMediaAssetPath(asset.path));
|
|
1317
|
+
return needsMedia || embeddingProfileMinimumFormat(profile) === "0.8" ? "0.8" : "0.7";
|
|
1318
|
+
}
|
|
1319
|
+
function createCatalogDb(absPath, indexSchemaVersion) {
|
|
913
1320
|
const db = new DatabaseSync(absPath);
|
|
914
1321
|
db.exec("PRAGMA page_size=8192");
|
|
915
1322
|
db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
|
|
916
|
-
db.exec(`PRAGMA user_version=${
|
|
1323
|
+
db.exec(`PRAGMA user_version=${indexSchemaVersion}`);
|
|
917
1324
|
db.exec("PRAGMA journal_mode=DELETE");
|
|
918
1325
|
return db;
|
|
919
1326
|
}
|
|
@@ -958,7 +1365,8 @@ ${path.join(" > ")}
|
|
|
958
1365
|
return header;
|
|
959
1366
|
}
|
|
960
1367
|
async function stageVectors(args) {
|
|
961
|
-
const { prepared,
|
|
1368
|
+
const { prepared, profile } = args;
|
|
1369
|
+
const { dimensions } = profile;
|
|
962
1370
|
const docVectors = prepared.map(() => new Float32Array(dimensions));
|
|
963
1371
|
const fd = openSync(args.stagedPath, "w");
|
|
964
1372
|
let texts = [];
|
|
@@ -972,10 +1380,7 @@ async function stageVectors(args) {
|
|
|
972
1380
|
}
|
|
973
1381
|
const block = new Float32Array(texts.length * dimensions);
|
|
974
1382
|
for (let i = 0; i < vectors.length; i++) {
|
|
975
|
-
const unit =
|
|
976
|
-
if (unit.length !== dimensions) {
|
|
977
|
-
throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
|
|
978
|
-
}
|
|
1383
|
+
const unit = profileUnitVector(profile, vectors[i]);
|
|
979
1384
|
block.set(unit, i * dimensions);
|
|
980
1385
|
const sum = docVectors[owners[i]];
|
|
981
1386
|
for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
|
|
@@ -999,7 +1404,9 @@ async function stageVectors(args) {
|
|
|
999
1404
|
} finally {
|
|
1000
1405
|
closeSync(fd);
|
|
1001
1406
|
}
|
|
1002
|
-
return docVectors.map(
|
|
1407
|
+
return docVectors.map(
|
|
1408
|
+
(v) => v.some((x) => x !== 0) ? Float32Array.from(l2Normalize(Array.from(v))) : v
|
|
1409
|
+
);
|
|
1003
1410
|
}
|
|
1004
1411
|
var StagedVectorReader = class {
|
|
1005
1412
|
constructor(path, dimensions) {
|
|
@@ -1041,7 +1448,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1041
1448
|
return sum;
|
|
1042
1449
|
};
|
|
1043
1450
|
const stride = Math.max(1, Math.ceil(n / 2e4));
|
|
1044
|
-
const sample = docVectors.filter((
|
|
1451
|
+
const sample = docVectors.filter((v, i) => i % stride === 0 && v.some((x) => x !== 0));
|
|
1045
1452
|
const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
|
|
1046
1453
|
for (let i = 0; centers.length < shardCount; i++) {
|
|
1047
1454
|
centers.push(
|
|
@@ -1110,7 +1517,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1110
1517
|
return assignment.map((s) => renumber.get(s));
|
|
1111
1518
|
}
|
|
1112
1519
|
function seedFor(catalogId, shardId) {
|
|
1113
|
-
const digest =
|
|
1520
|
+
const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
1114
1521
|
return digest.readBigUInt64LE(24);
|
|
1115
1522
|
}
|
|
1116
1523
|
function makePrng(seed) {
|
|
@@ -1195,7 +1602,7 @@ function kMeans(samples, k, seed) {
|
|
|
1195
1602
|
}
|
|
1196
1603
|
|
|
1197
1604
|
// src/archive/read.ts
|
|
1198
|
-
import { createHash as
|
|
1605
|
+
import { createHash as createHash4 } from "crypto";
|
|
1199
1606
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
1200
1607
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1201
1608
|
import { createRequire as createRequire4 } from "module";
|
|
@@ -1485,7 +1892,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1485
1892
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1486
1893
|
return;
|
|
1487
1894
|
}
|
|
1488
|
-
const hasher =
|
|
1895
|
+
const hasher = createHash4("sha256");
|
|
1489
1896
|
let sizeBytes = 0;
|
|
1490
1897
|
const verify = new Transform({
|
|
1491
1898
|
transform(chunk, _encoding, callback) {
|
|
@@ -1555,6 +1962,14 @@ var CatalogOpenError = class extends Error {
|
|
|
1555
1962
|
}
|
|
1556
1963
|
reason;
|
|
1557
1964
|
};
|
|
1965
|
+
var CatalogQueryError = class extends Error {
|
|
1966
|
+
constructor(message, reason) {
|
|
1967
|
+
super(message);
|
|
1968
|
+
this.reason = reason;
|
|
1969
|
+
this.name = "CatalogQueryError";
|
|
1970
|
+
}
|
|
1971
|
+
reason;
|
|
1972
|
+
};
|
|
1558
1973
|
function openCatalogDatabase(absPath) {
|
|
1559
1974
|
const uri = `${pathToFileURL(absPath).href}?immutable=1`;
|
|
1560
1975
|
let db;
|
|
@@ -1608,15 +2023,15 @@ function readPragmaNumber(db, pragma) {
|
|
|
1608
2023
|
}
|
|
1609
2024
|
|
|
1610
2025
|
// src/reader/catalog-handle.ts
|
|
1611
|
-
import { readFileSync as
|
|
2026
|
+
import { readFileSync as readFileSync3, statSync as statSync2 } from "fs";
|
|
1612
2027
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1613
2028
|
import { brotliDecompressSync } from "zlib";
|
|
1614
2029
|
import {
|
|
1615
2030
|
KnowledgeEmbeddingProfileSchema,
|
|
1616
|
-
|
|
1617
|
-
assetContentType,
|
|
2031
|
+
assetContentType as assetContentType2,
|
|
1618
2032
|
embeddingProfileCenter,
|
|
1619
|
-
isKnowledgeAssetPath
|
|
2033
|
+
isKnowledgeAssetPath,
|
|
2034
|
+
maxKnowledgeAssetBytes as maxKnowledgeAssetBytes2
|
|
1620
2035
|
} from "@bendyline/gezk";
|
|
1621
2036
|
|
|
1622
2037
|
// src/reader/bit-scan.ts
|
|
@@ -1689,14 +2104,14 @@ function hammingTopK(index, query, k) {
|
|
|
1689
2104
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1690
2105
|
return hits;
|
|
1691
2106
|
}
|
|
1692
|
-
function asymmetricTopK(index, query, k) {
|
|
2107
|
+
function asymmetricTopK(index, query, k, eligibleChunkIds, excludedChunkIds) {
|
|
1693
2108
|
const { bits, bytesPerRow, rows } = index;
|
|
1694
2109
|
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
1695
2110
|
throw new Error(
|
|
1696
2111
|
`query has ${query.length} dimensions, shard rows have ${bytesPerRow} bytes of bits`
|
|
1697
2112
|
);
|
|
1698
2113
|
}
|
|
1699
|
-
const limit = Math.min(k, rows);
|
|
2114
|
+
const limit = Math.min(k, eligibleChunkIds?.size ?? rows);
|
|
1700
2115
|
if (limit <= 0) return [];
|
|
1701
2116
|
const lut = new Float32Array(bytesPerRow * 256);
|
|
1702
2117
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
@@ -1746,6 +2161,10 @@ function asymmetricTopK(index, query, k) {
|
|
|
1746
2161
|
};
|
|
1747
2162
|
let offset = 0;
|
|
1748
2163
|
for (let r = 0; r < rows; r++) {
|
|
2164
|
+
if (eligibleChunkIds && !eligibleChunkIds.has(r + 1) || excludedChunkIds?.has(r + 1)) {
|
|
2165
|
+
offset += bytesPerRow;
|
|
2166
|
+
continue;
|
|
2167
|
+
}
|
|
1749
2168
|
let score = 0;
|
|
1750
2169
|
for (let p = 0; p < bytesPerRow; p++) {
|
|
1751
2170
|
score += lut[p * 256 + bits[offset + p]];
|
|
@@ -1770,14 +2189,160 @@ function asymmetricTopK(index, query, k) {
|
|
|
1770
2189
|
return hits;
|
|
1771
2190
|
}
|
|
1772
2191
|
|
|
2192
|
+
// src/reader/spatial-index.ts
|
|
2193
|
+
import {
|
|
2194
|
+
KnowledgeLocationSchema,
|
|
2195
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema2,
|
|
2196
|
+
KnowledgeRadiusSchema,
|
|
2197
|
+
locationDistanceMeters,
|
|
2198
|
+
locationInBounds,
|
|
2199
|
+
normalizeLongitude as normalizeLongitude2,
|
|
2200
|
+
radiusBounds,
|
|
2201
|
+
spatialManifest as spatialManifest2
|
|
2202
|
+
} from "@bendyline/gezk";
|
|
2203
|
+
var DocumentSpatialIndex = class {
|
|
2204
|
+
constructor(db, allowLegacy) {
|
|
2205
|
+
this.db = db;
|
|
2206
|
+
this.allowLegacy = allowLegacy;
|
|
2207
|
+
this.hasTable = Boolean(
|
|
2208
|
+
db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='document_locations'").get()
|
|
2209
|
+
);
|
|
2210
|
+
}
|
|
2211
|
+
db;
|
|
2212
|
+
allowLegacy;
|
|
2213
|
+
hasTable;
|
|
2214
|
+
legacy;
|
|
2215
|
+
decode(row) {
|
|
2216
|
+
const provenance = row.provenance_json;
|
|
2217
|
+
if (typeof provenance === "string" && Buffer.byteLength(provenance) > 16384)
|
|
2218
|
+
throw new Error("location provenance exceeds 16384 bytes");
|
|
2219
|
+
return {
|
|
2220
|
+
documentId: String(row.document_id),
|
|
2221
|
+
location: KnowledgeLocationSchema.parse({
|
|
2222
|
+
id: row.location_id,
|
|
2223
|
+
latitude: row.latitude,
|
|
2224
|
+
longitude: row.longitude,
|
|
2225
|
+
role: row.role,
|
|
2226
|
+
...provenance === null ? {} : { provenance: JSON.parse(String(provenance)) }
|
|
2227
|
+
})
|
|
2228
|
+
};
|
|
2229
|
+
}
|
|
2230
|
+
legacyRows() {
|
|
2231
|
+
if (this.legacy) return this.legacy;
|
|
2232
|
+
this.legacy = [];
|
|
2233
|
+
if (!this.allowLegacy) return this.legacy;
|
|
2234
|
+
for (const row of this.db.prepare(
|
|
2235
|
+
"SELECT id, meta_json FROM documents WHERE id LIKE 'qualla-%' AND meta_json IS NOT NULL"
|
|
2236
|
+
).all()) {
|
|
2237
|
+
try {
|
|
2238
|
+
const meta = JSON.parse(String(row.meta_json));
|
|
2239
|
+
if (typeof meta.region !== "string" || !Array.isArray(meta.quallaArticleIds) || !meta.coordinates)
|
|
2240
|
+
continue;
|
|
2241
|
+
const point = KnowledgeLocationSchema.safeParse({
|
|
2242
|
+
id: "qualla-coordinate",
|
|
2243
|
+
role: "subject",
|
|
2244
|
+
latitude: meta.coordinates.lat,
|
|
2245
|
+
longitude: meta.coordinates.lng,
|
|
2246
|
+
provenance: { adapter: "qualla-regional-meta@1", articleIds: meta.quallaArticleIds }
|
|
2247
|
+
});
|
|
2248
|
+
if (point.success)
|
|
2249
|
+
this.legacy.push({
|
|
2250
|
+
documentId: String(row.id),
|
|
2251
|
+
location: { ...point.data, longitude: normalizeLongitude2(point.data.longitude) }
|
|
2252
|
+
});
|
|
2253
|
+
} catch {
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
return this.legacy;
|
|
2257
|
+
}
|
|
2258
|
+
rows() {
|
|
2259
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations ORDER BY document_id, location_id").all().map((row) => this.decode(row)) : this.legacyRows();
|
|
2260
|
+
}
|
|
2261
|
+
locations(documentId) {
|
|
2262
|
+
return this.hasTable ? this.db.prepare("SELECT * FROM document_locations WHERE document_id = ? ORDER BY location_id").all(documentId).map((row) => this.decode(row).location) : this.legacyRows().filter((row) => row.documentId === documentId).map((row) => row.location);
|
|
2263
|
+
}
|
|
2264
|
+
integrity() {
|
|
2265
|
+
const rows = this.rows();
|
|
2266
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
2267
|
+
for (const row of rows) {
|
|
2268
|
+
if (normalizeLongitude2(row.location.longitude) !== row.location.longitude)
|
|
2269
|
+
throw new Error("location longitude must be in [-180, 180)");
|
|
2270
|
+
const points = grouped.get(row.documentId) ?? [];
|
|
2271
|
+
points.push(row.location);
|
|
2272
|
+
grouped.set(row.documentId, points);
|
|
2273
|
+
if (!this.db.prepare("SELECT 1 FROM documents WHERE id = ?").get(row.documentId))
|
|
2274
|
+
throw new Error("location references missing document");
|
|
2275
|
+
}
|
|
2276
|
+
for (const points of grouped.values()) KnowledgeLocationsSchema2.parse(points);
|
|
2277
|
+
return { hasTable: this.hasTable, manifest: spatialManifest2(rows) };
|
|
2278
|
+
}
|
|
2279
|
+
matches(raw) {
|
|
2280
|
+
const radius = KnowledgeRadiusSchema.parse(raw);
|
|
2281
|
+
const boxes = radiusBounds(radius);
|
|
2282
|
+
const candidates = this.hasTable ? boxes.flatMap(
|
|
2283
|
+
(box) => this.db.prepare(`SELECT * FROM document_locations
|
|
2284
|
+
WHERE role = 'subject' AND latitude BETWEEN ? AND ? AND longitude BETWEEN ? AND ?`).all(box.south, box.north, box.west, box.east).map((row) => this.decode(row))
|
|
2285
|
+
) : this.legacyRows().filter(
|
|
2286
|
+
(row) => row.location.role === "subject" && boxes.some((box) => locationInBounds(row.location, box))
|
|
2287
|
+
);
|
|
2288
|
+
const matches = /* @__PURE__ */ new Map();
|
|
2289
|
+
for (const { documentId, location } of candidates) {
|
|
2290
|
+
const distanceMeters = locationDistanceMeters(radius, location);
|
|
2291
|
+
if (distanceMeters > radius.radiusMeters) continue;
|
|
2292
|
+
const previous = matches.get(documentId);
|
|
2293
|
+
if (!previous || distanceMeters < previous.distanceMeters || distanceMeters === previous.distanceMeters && location.id < previous.matchedLocation.id) {
|
|
2294
|
+
matches.set(documentId, { distanceMeters, matchedLocation: location });
|
|
2295
|
+
}
|
|
2296
|
+
}
|
|
2297
|
+
return matches;
|
|
2298
|
+
}
|
|
2299
|
+
};
|
|
2300
|
+
|
|
1773
2301
|
// src/reader/catalog-handle.ts
|
|
1774
2302
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
2303
|
+
var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
|
|
2304
|
+
var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
|
|
2305
|
+
function mediaOf(row) {
|
|
2306
|
+
const modality = row.modality;
|
|
2307
|
+
if (!modality || modality === "text") return void 0;
|
|
2308
|
+
const num = (v) => v === null || v === void 0 ? void 0 : Number(v);
|
|
2309
|
+
let attribution;
|
|
2310
|
+
if (typeof row.attribution_json === "string") {
|
|
2311
|
+
try {
|
|
2312
|
+
const parsed = JSON.parse(row.attribution_json);
|
|
2313
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
2314
|
+
attribution = parsed;
|
|
2315
|
+
}
|
|
2316
|
+
} catch {
|
|
2317
|
+
attribution = void 0;
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
const media = {
|
|
2321
|
+
modality,
|
|
2322
|
+
assetPath: row.asset_path,
|
|
2323
|
+
mimeType: row.mime_type
|
|
2324
|
+
};
|
|
2325
|
+
const width = num(row.width);
|
|
2326
|
+
const height = num(row.height);
|
|
2327
|
+
const startMs = num(row.start_ms);
|
|
2328
|
+
const endMs = num(row.end_ms);
|
|
2329
|
+
if (width !== void 0) media.width = width;
|
|
2330
|
+
if (height !== void 0) media.height = height;
|
|
2331
|
+
if (startMs !== void 0) media.startMs = startMs;
|
|
2332
|
+
if (endMs !== void 0) media.endMs = endMs;
|
|
2333
|
+
if (typeof row.thumbnail_path === "string") media.thumbnailAssetPath = row.thumbnail_path;
|
|
2334
|
+
if (attribution) media.attribution = attribution;
|
|
2335
|
+
return media;
|
|
2336
|
+
}
|
|
1775
2337
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
2338
|
+
var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
|
|
2339
|
+
var TITLE_ROUTE_SHARDS = 1;
|
|
1776
2340
|
var CatalogHandle = class _CatalogHandle {
|
|
1777
2341
|
constructor(rootDir, router) {
|
|
1778
2342
|
this.rootDir = rootDir;
|
|
1779
2343
|
this.router = router;
|
|
1780
2344
|
this.schemaVersion = router.schemaVersion;
|
|
2345
|
+
this.spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
1781
2346
|
this.meta = {};
|
|
1782
2347
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1783
2348
|
this.meta[row.key] = row.value;
|
|
@@ -1789,6 +2354,10 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1789
2354
|
router;
|
|
1790
2355
|
connections = /* @__PURE__ */ new Map();
|
|
1791
2356
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
2357
|
+
mediaIndexes = /* @__PURE__ */ new Map();
|
|
2358
|
+
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
2359
|
+
chunkTermRows = /* @__PURE__ */ new Map();
|
|
2360
|
+
spatial;
|
|
1792
2361
|
assetsByPath = null;
|
|
1793
2362
|
/** Lazily parsed from the profile echo; `undefined` until first needed. */
|
|
1794
2363
|
binaryCenterCache;
|
|
@@ -1799,7 +2368,12 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1799
2368
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1800
2369
|
static open(rootDir) {
|
|
1801
2370
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
1802
|
-
|
|
2371
|
+
try {
|
|
2372
|
+
return new _CatalogHandle(rootDir, router);
|
|
2373
|
+
} catch (error) {
|
|
2374
|
+
router.close();
|
|
2375
|
+
throw error;
|
|
2376
|
+
}
|
|
1803
2377
|
}
|
|
1804
2378
|
close() {
|
|
1805
2379
|
for (const conn of this.connections.values()) conn.close();
|
|
@@ -1874,6 +2448,24 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1874
2448
|
this.bitIndexes.set(shard.path, index);
|
|
1875
2449
|
return index;
|
|
1876
2450
|
}
|
|
2451
|
+
/**
|
|
2452
|
+
* Load the next shard's sign bits that are not resident yet, if they fit
|
|
2453
|
+
* the resident budget. False when nothing is left to load. One shard per
|
|
2454
|
+
* call, so a host can keep answering queries between shards while it warms
|
|
2455
|
+
* a large catalog, and never evicts a shard it loaded a moment ago.
|
|
2456
|
+
*/
|
|
2457
|
+
prewarmNextShard() {
|
|
2458
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
2459
|
+
let resident = 0;
|
|
2460
|
+
for (const index of this.bitIndexes.values()) resident += index.bits.byteLength;
|
|
2461
|
+
for (const shard of this.shards) {
|
|
2462
|
+
if (this.bitIndexes.has(shard.path)) continue;
|
|
2463
|
+
if (resident + shard.chunkCount * bytesPerRow > BIT_INDEX_BUDGET_BYTES) return false;
|
|
2464
|
+
this.shardBits(shard);
|
|
2465
|
+
return true;
|
|
2466
|
+
}
|
|
2467
|
+
return false;
|
|
2468
|
+
}
|
|
1877
2469
|
/**
|
|
1878
2470
|
* The stage-1 center the catalog's bits were derived with (a
|
|
1879
2471
|
* `centered-sign` profile), or null for plain sign bits. Read from the
|
|
@@ -1896,6 +2488,127 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1896
2488
|
this.binaryCenterCache = embeddingProfileCenter(profile.data);
|
|
1897
2489
|
return this.binaryCenterCache;
|
|
1898
2490
|
}
|
|
2491
|
+
/** The chunk columns this catalog's schema has. */
|
|
2492
|
+
chunkColumns() {
|
|
2493
|
+
return this.schemaVersion >= 5 ? `${CHUNK_COLUMNS}, ${MEDIA_COLUMNS}` : CHUNK_COLUMNS;
|
|
2494
|
+
}
|
|
2495
|
+
/** A shard's media rows with their int8 vectors (empty before index schema 5). */
|
|
2496
|
+
shardMedia(shard) {
|
|
2497
|
+
const cached = this.mediaIndexes.get(shard.path);
|
|
2498
|
+
if (cached) return cached;
|
|
2499
|
+
const empty = {
|
|
2500
|
+
ids: [],
|
|
2501
|
+
modalities: [],
|
|
2502
|
+
int8: new Int8Array(0),
|
|
2503
|
+
idSet: /* @__PURE__ */ new Set()
|
|
2504
|
+
};
|
|
2505
|
+
if (this.schemaVersion < 5) {
|
|
2506
|
+
this.mediaIndexes.set(shard.path, empty);
|
|
2507
|
+
return empty;
|
|
2508
|
+
}
|
|
2509
|
+
const dims = this.dimensions();
|
|
2510
|
+
const rows = this.shardDb(shard).prepare(
|
|
2511
|
+
`SELECT c.id, c.modality, v.v FROM chunks c JOIN chunk_vectors_int8 v ON v.chunk_id = c.id
|
|
2512
|
+
WHERE c.modality <> 'text' ORDER BY c.id`
|
|
2513
|
+
).all();
|
|
2514
|
+
const int8 = new Int8Array(rows.length * dims);
|
|
2515
|
+
rows.forEach((row, i) => {
|
|
2516
|
+
if (row.v.byteLength !== dims) {
|
|
2517
|
+
throw new CatalogOpenError(`int8 vector width ${row.v.byteLength} != ${dims}`, "corrupt");
|
|
2518
|
+
}
|
|
2519
|
+
int8.set(new Int8Array(row.v.buffer, row.v.byteOffset, dims), i * dims);
|
|
2520
|
+
});
|
|
2521
|
+
const ids = rows.map((r) => Number(r.id));
|
|
2522
|
+
const index = {
|
|
2523
|
+
ids,
|
|
2524
|
+
modalities: rows.map((r) => r.modality),
|
|
2525
|
+
int8,
|
|
2526
|
+
idSet: new Set(ids)
|
|
2527
|
+
};
|
|
2528
|
+
this.mediaIndexes.set(shard.path, index);
|
|
2529
|
+
return index;
|
|
2530
|
+
}
|
|
2531
|
+
/** Media rows this catalog holds, by modality (all zero before index schema 5). */
|
|
2532
|
+
mediaCounts() {
|
|
2533
|
+
const counts = { image: 0, video: 0, audio: 0 };
|
|
2534
|
+
for (const shard of this.shards) {
|
|
2535
|
+
for (const modality of this.shardMedia(shard).modalities) counts[modality] += 1;
|
|
2536
|
+
}
|
|
2537
|
+
return counts;
|
|
2538
|
+
}
|
|
2539
|
+
/**
|
|
2540
|
+
* The media lane: every media row of every shard scored exactly against
|
|
2541
|
+
* the query (int8 rerank), keeping the best `perModality` of each kind.
|
|
2542
|
+
* Images sit at a lower cosine than text in a shared space, so in the text
|
|
2543
|
+
* lane's sign-bit pre-filter they would rarely survive the cut; media rows
|
|
2544
|
+
* are few enough (capped per catalog) to score them all.
|
|
2545
|
+
*/
|
|
2546
|
+
searchMedia(queryVector, opts = {}) {
|
|
2547
|
+
this.assertQueryWidth(queryVector);
|
|
2548
|
+
const perModality = opts.perModality ?? 4;
|
|
2549
|
+
const dims = this.dimensions();
|
|
2550
|
+
const best = /* @__PURE__ */ new Map();
|
|
2551
|
+
for (const shard of this.shards) {
|
|
2552
|
+
const index = this.shardMedia(shard);
|
|
2553
|
+
for (let i = 0; i < index.ids.length; i++) {
|
|
2554
|
+
let dot = 0;
|
|
2555
|
+
const base = i * dims;
|
|
2556
|
+
for (let d = 0; d < dims; d++)
|
|
2557
|
+
dot += queryVector[d] * (index.int8[base + d] / 127);
|
|
2558
|
+
const modality = index.modalities[i];
|
|
2559
|
+
const list = best.get(modality) ?? [];
|
|
2560
|
+
list.push({ shard, id: index.ids[i], cosine: dot });
|
|
2561
|
+
best.set(modality, list);
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
const hits = [];
|
|
2565
|
+
for (const list of best.values()) {
|
|
2566
|
+
list.sort((a, b) => b.cosine - a.cosine || a.id - b.id);
|
|
2567
|
+
let kept = 0;
|
|
2568
|
+
for (const candidate of list) {
|
|
2569
|
+
if (kept >= perModality) break;
|
|
2570
|
+
const row = this.shardDb(candidate.shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`).get(BigInt(candidate.id));
|
|
2571
|
+
if (!row) continue;
|
|
2572
|
+
if (opts.allowedDocumentIds && !opts.allowedDocumentIds.has(row.document_id))
|
|
2573
|
+
continue;
|
|
2574
|
+
kept++;
|
|
2575
|
+
hits.push({
|
|
2576
|
+
...this.hitFromRow(row, candidate.shard.id),
|
|
2577
|
+
cosine: candidate.cosine,
|
|
2578
|
+
source: "vector"
|
|
2579
|
+
});
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
return hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
|
|
2583
|
+
}
|
|
2584
|
+
hitFromRow(row, shardId) {
|
|
2585
|
+
const media = mediaOf(row);
|
|
2586
|
+
return {
|
|
2587
|
+
chunkUid: row.chunk_uid,
|
|
2588
|
+
documentId: row.document_id,
|
|
2589
|
+
title: row.title,
|
|
2590
|
+
headingPath: parseHeadingPath(row.heading_path),
|
|
2591
|
+
lineStart: Number(row.line_start),
|
|
2592
|
+
lineEnd: Number(row.line_end),
|
|
2593
|
+
text: row.text,
|
|
2594
|
+
shardId,
|
|
2595
|
+
...media ? { media } : {}
|
|
2596
|
+
};
|
|
2597
|
+
}
|
|
2598
|
+
/**
|
|
2599
|
+
* Refuse a query vector whose width is not the catalog's stored width,
|
|
2600
|
+
* before any scan: routing and rerank would otherwise score a silent
|
|
2601
|
+
* prefix or fail deep inside a shard with an untyped error.
|
|
2602
|
+
*/
|
|
2603
|
+
assertQueryWidth(queryVector) {
|
|
2604
|
+
const dims = this.dimensions();
|
|
2605
|
+
if (queryVector.length !== dims) {
|
|
2606
|
+
throw new CatalogQueryError(
|
|
2607
|
+
`query vector has ${queryVector.length} dimensions, the catalog stores ${dims}`,
|
|
2608
|
+
"dimension"
|
|
2609
|
+
);
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
1899
2612
|
/** The embedding dimension from the router's profile echo. */
|
|
1900
2613
|
dimensions() {
|
|
1901
2614
|
const raw = this.meta.embedding_profile_json;
|
|
@@ -1923,9 +2636,13 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1923
2636
|
get formatVersion() {
|
|
1924
2637
|
return this.meta.format_version ?? "";
|
|
1925
2638
|
}
|
|
1926
|
-
documentColumns() {
|
|
2639
|
+
documentColumns(documentPrefix = "", placementPrefix = documentPrefix) {
|
|
1927
2640
|
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1928
|
-
|
|
2641
|
+
const columns = this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
2642
|
+
return columns.split(", ").map((column) => {
|
|
2643
|
+
const prefix = column === "topic_id" || column === "ordinal" ? placementPrefix : documentPrefix;
|
|
2644
|
+
return prefix ? `${prefix}.${column} AS ${column}` : column;
|
|
2645
|
+
}).join(", ");
|
|
1929
2646
|
}
|
|
1930
2647
|
documentOrder() {
|
|
1931
2648
|
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
@@ -1935,6 +2652,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1935
2652
|
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1936
2653
|
*/
|
|
1937
2654
|
topics() {
|
|
2655
|
+
const rollup = this.schemaVersion >= 4 ? `SELECT sub.root AS id, COUNT(DISTINCT td.document_id) AS total
|
|
2656
|
+
FROM sub LEFT JOIN topic_documents td ON td.topic_id = sub.id GROUP BY sub.root` : `SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
2657
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root`;
|
|
1938
2658
|
return this.router.db.prepare(
|
|
1939
2659
|
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1940
2660
|
SELECT id, id, 0 FROM topics
|
|
@@ -1943,8 +2663,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1943
2663
|
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1944
2664
|
),
|
|
1945
2665
|
rollup AS (
|
|
1946
|
-
|
|
1947
|
-
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
2666
|
+
${rollup}
|
|
1948
2667
|
)
|
|
1949
2668
|
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1950
2669
|
rollup.total AS total_document_count
|
|
@@ -1968,6 +2687,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1968
2687
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1969
2688
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1970
2689
|
const descendants = opts.descendants ?? true;
|
|
2690
|
+
if (this.schemaVersion >= 4 && opts.topicId) {
|
|
2691
|
+
return this.sharedDocumentsPage(opts.topicId, descendants, limit, offset);
|
|
2692
|
+
}
|
|
1971
2693
|
let scope = "";
|
|
1972
2694
|
let where = "";
|
|
1973
2695
|
const params = [];
|
|
@@ -1990,16 +2712,89 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1990
2712
|
const documents = this.router.db.prepare(
|
|
1991
2713
|
`${scope} SELECT ${this.documentColumns()}
|
|
1992
2714
|
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1993
|
-
).all(...params, limit, offset).map(
|
|
2715
|
+
).all(...params, limit, offset).map((row) => this.documentMeta(row));
|
|
2716
|
+
return { documents, total };
|
|
2717
|
+
}
|
|
2718
|
+
/** A reference is a placement, not a duplicate result or a redirect page. */
|
|
2719
|
+
sharedDocumentsPage(topicId, descendants, limit, offset) {
|
|
2720
|
+
const sub = descendants ? `SELECT ?, 0 UNION ALL SELECT t.id, sub.depth + 1 FROM topics t
|
|
2721
|
+
JOIN sub ON t.parent_id = sub.id WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}` : "SELECT ?, 0";
|
|
2722
|
+
const scope = `WITH RECURSIVE sub(id, depth) AS (${sub}), scoped AS (
|
|
2723
|
+
SELECT td.document_id, td.topic_id, td.ordinal, t.sort_key
|
|
2724
|
+
FROM topic_documents td JOIN topics t ON t.id = td.topic_id
|
|
2725
|
+
WHERE td.topic_id IN (SELECT id FROM sub)
|
|
2726
|
+
)`;
|
|
2727
|
+
const total = Number(
|
|
2728
|
+
this.router.db.prepare(`${scope} SELECT COUNT(DISTINCT document_id) AS n FROM scoped`).get(topicId).n
|
|
2729
|
+
);
|
|
2730
|
+
const documents = this.router.db.prepare(
|
|
2731
|
+
`${scope}, ranked AS (
|
|
2732
|
+
SELECT *, ROW_NUMBER() OVER (PARTITION BY document_id
|
|
2733
|
+
ORDER BY (ordinal IS NULL), ordinal, sort_key, topic_id) AS placement_rank FROM scoped
|
|
2734
|
+
) SELECT ${this.documentColumns("d", "p")}
|
|
2735
|
+
FROM documents d JOIN ranked p ON p.document_id = d.id AND p.placement_rank = 1
|
|
2736
|
+
ORDER BY (p.ordinal IS NULL), p.ordinal, d.slug, d.id LIMIT ? OFFSET ?`
|
|
2737
|
+
).all(topicId, limit, offset).map((row) => this.documentMeta(row));
|
|
1994
2738
|
return { documents, total };
|
|
1995
2739
|
}
|
|
2740
|
+
/** Integrity facts kept separate from browse counts, which include shared references. */
|
|
2741
|
+
tocIntegrity() {
|
|
2742
|
+
const count = (sql) => Number(this.router.db.prepare(sql).get().n);
|
|
2743
|
+
const relation = this.schemaVersion >= 4 ? "topic_documents" : "documents";
|
|
2744
|
+
const incorrectCounts = count(
|
|
2745
|
+
`SELECT COUNT(*) AS n FROM topics t LEFT JOIN
|
|
2746
|
+
(SELECT topic_id, COUNT(*) AS n FROM ${relation} GROUP BY topic_id) r ON r.topic_id = t.id
|
|
2747
|
+
WHERE t.document_count != COALESCE(r.n, 0)`
|
|
2748
|
+
);
|
|
2749
|
+
if (this.schemaVersion < 4)
|
|
2750
|
+
return {
|
|
2751
|
+
placements: count("SELECT COUNT(*) AS n FROM documents"),
|
|
2752
|
+
invalidReferences: 0,
|
|
2753
|
+
missingPrimary: 0,
|
|
2754
|
+
incorrectCounts,
|
|
2755
|
+
duplicateReferences: 0
|
|
2756
|
+
};
|
|
2757
|
+
return {
|
|
2758
|
+
placements: count("SELECT COUNT(*) AS n FROM topic_documents"),
|
|
2759
|
+
invalidReferences: count(
|
|
2760
|
+
`SELECT COUNT(*) AS n FROM topic_documents r
|
|
2761
|
+
LEFT JOIN documents d ON d.id = r.document_id LEFT JOIN topics t ON t.id = r.topic_id
|
|
2762
|
+
WHERE d.id IS NULL OR t.id IS NULL OR
|
|
2763
|
+
(r.ordinal IS NOT NULL AND (typeof(r.ordinal) != 'integer' OR r.ordinal < -2147483648 OR r.ordinal > 2147483647))`
|
|
2764
|
+
),
|
|
2765
|
+
missingPrimary: count(
|
|
2766
|
+
`SELECT COUNT(*) AS n FROM documents d WHERE NOT EXISTS (
|
|
2767
|
+
SELECT 1 FROM topic_documents r WHERE r.document_id = d.id AND r.topic_id = d.topic_id AND r.ordinal IS d.ordinal)`
|
|
2768
|
+
),
|
|
2769
|
+
incorrectCounts,
|
|
2770
|
+
duplicateReferences: count(
|
|
2771
|
+
"SELECT COUNT(*) AS n FROM (SELECT topic_id, document_id FROM topic_documents GROUP BY topic_id, document_id HAVING COUNT(*) > 1)"
|
|
2772
|
+
)
|
|
2773
|
+
};
|
|
2774
|
+
}
|
|
1996
2775
|
getDocument(id) {
|
|
1997
2776
|
const row = this.router.db.prepare(
|
|
1998
2777
|
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1999
2778
|
FROM documents WHERE id = ?`
|
|
2000
2779
|
).get(id);
|
|
2001
2780
|
if (!row) return null;
|
|
2002
|
-
return { ...
|
|
2781
|
+
return { ...this.documentMeta(row), markdown: this.decodeBody(id, row) };
|
|
2782
|
+
}
|
|
2783
|
+
/**
|
|
2784
|
+
* One passage by its citation id (`knowledge://…#chunk=<uid>`), from the
|
|
2785
|
+
* shard the document's chunks live in. Null when the document or the
|
|
2786
|
+
* passage is not in this catalog, which is what a fabricated or stale
|
|
2787
|
+
* citation looks like.
|
|
2788
|
+
*/
|
|
2789
|
+
getChunk(documentId, chunkUid2) {
|
|
2790
|
+
const doc = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?").get(documentId);
|
|
2791
|
+
if (!doc) return null;
|
|
2792
|
+
const shardId = Number(doc.shard_id);
|
|
2793
|
+
const shard = this.shards.find((s) => s.id === shardId);
|
|
2794
|
+
if (!shard) return null;
|
|
2795
|
+
const row = this.shardDb(shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE chunk_uid = ? AND document_id = ?`).get(chunkUid2, documentId);
|
|
2796
|
+
if (!row) return null;
|
|
2797
|
+
return this.hitFromRow(row, shardId);
|
|
2003
2798
|
}
|
|
2004
2799
|
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
2005
2800
|
documentsWithUndeclaredTopic() {
|
|
@@ -2072,7 +2867,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2072
2867
|
const index = /* @__PURE__ */ new Map();
|
|
2073
2868
|
let raw;
|
|
2074
2869
|
try {
|
|
2075
|
-
raw = JSON.parse(
|
|
2870
|
+
raw = JSON.parse(readFileSync3(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
2076
2871
|
} catch (error) {
|
|
2077
2872
|
if (error.code === "ENOENT") {
|
|
2078
2873
|
this.assetsByPath = index;
|
|
@@ -2088,7 +2883,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2088
2883
|
for (const file of files) {
|
|
2089
2884
|
const path = file?.path;
|
|
2090
2885
|
if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
|
|
2091
|
-
const contentType =
|
|
2886
|
+
const contentType = assetContentType2(path);
|
|
2092
2887
|
if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
|
|
2093
2888
|
continue;
|
|
2094
2889
|
}
|
|
@@ -2102,16 +2897,42 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2102
2897
|
assets() {
|
|
2103
2898
|
return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
2104
2899
|
}
|
|
2900
|
+
/**
|
|
2901
|
+
* Where a declared asset lives on disk, so a large one (video, audio) can be
|
|
2902
|
+
* streamed with byte ranges instead of read whole. Checks containment and
|
|
2903
|
+
* the manifest size; the content hash was verified once, at extraction.
|
|
2904
|
+
*/
|
|
2905
|
+
assetFile(path) {
|
|
2906
|
+
const info = this.assetIndex().get(path);
|
|
2907
|
+
if (!info) return null;
|
|
2908
|
+
if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
|
|
2909
|
+
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
2910
|
+
}
|
|
2911
|
+
const absPath = this.resolveCatalogPath(path);
|
|
2912
|
+
let size;
|
|
2913
|
+
try {
|
|
2914
|
+
size = statSync2(absPath).size;
|
|
2915
|
+
} catch (error) {
|
|
2916
|
+
throw new CatalogOpenError(
|
|
2917
|
+
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2918
|
+
"corrupt"
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
if (size !== info.sizeBytes) {
|
|
2922
|
+
throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
|
|
2923
|
+
}
|
|
2924
|
+
return { ...info, absPath };
|
|
2925
|
+
}
|
|
2105
2926
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
2106
2927
|
readAsset(path) {
|
|
2107
2928
|
const info = this.assetIndex().get(path);
|
|
2108
2929
|
if (!info) return null;
|
|
2109
|
-
if (info.sizeBytes >
|
|
2930
|
+
if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
|
|
2110
2931
|
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
2111
2932
|
}
|
|
2112
2933
|
let bytes;
|
|
2113
2934
|
try {
|
|
2114
|
-
bytes =
|
|
2935
|
+
bytes = readFileSync3(this.resolveCatalogPath(path));
|
|
2115
2936
|
} catch (error) {
|
|
2116
2937
|
throw new CatalogOpenError(
|
|
2117
2938
|
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
@@ -2123,19 +2944,59 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2123
2944
|
}
|
|
2124
2945
|
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
2125
2946
|
}
|
|
2947
|
+
documentMeta(row) {
|
|
2948
|
+
const document = rowToDocumentMeta(row);
|
|
2949
|
+
const locations = this.spatial.locations(document.id);
|
|
2950
|
+
return locations.length ? { ...document, locations } : document;
|
|
2951
|
+
}
|
|
2952
|
+
documentLocations(documentId) {
|
|
2953
|
+
return this.spatial.locations(documentId);
|
|
2954
|
+
}
|
|
2955
|
+
documentLocationRows() {
|
|
2956
|
+
return this.spatial.rows();
|
|
2957
|
+
}
|
|
2958
|
+
spatialIntegrity() {
|
|
2959
|
+
return this.spatial.integrity();
|
|
2960
|
+
}
|
|
2961
|
+
spatialMatches(radius) {
|
|
2962
|
+
return this.spatial.matches(radius);
|
|
2963
|
+
}
|
|
2964
|
+
nearbyDocuments(radius, opts = {}) {
|
|
2965
|
+
const offset = Math.max(0, Math.trunc(opts.offset ?? 0));
|
|
2966
|
+
const limit = Math.max(1, Math.min(500, Math.trunc(opts.limit ?? 50)));
|
|
2967
|
+
const ordered = [...this.spatialMatches(radius)].sort(
|
|
2968
|
+
(a, b) => a[1].distanceMeters - b[1].distanceMeters || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0)
|
|
2969
|
+
);
|
|
2970
|
+
const get = this.router.db.prepare(
|
|
2971
|
+
`SELECT ${this.documentColumns()} FROM documents d WHERE d.id = ?`
|
|
2972
|
+
);
|
|
2973
|
+
return {
|
|
2974
|
+
total: ordered.length,
|
|
2975
|
+
documents: ordered.slice(offset, offset + limit).map(([id, match]) => ({
|
|
2976
|
+
...this.documentMeta(get.get(id)),
|
|
2977
|
+
...match
|
|
2978
|
+
}))
|
|
2979
|
+
};
|
|
2980
|
+
}
|
|
2981
|
+
shardsForDocuments(ids) {
|
|
2982
|
+
if (!ids.size) return [];
|
|
2983
|
+
return this.router.db.prepare(
|
|
2984
|
+
"SELECT DISTINCT shard_id FROM documents WHERE id IN (SELECT value FROM json_each(?)) ORDER BY shard_id"
|
|
2985
|
+
).all(JSON.stringify([...ids])).map((row) => Number(row.shard_id));
|
|
2986
|
+
}
|
|
2126
2987
|
// ── search ────────────────────────────────────────────────────────────────
|
|
2127
2988
|
/**
|
|
2128
2989
|
* Catalog-wide title/summary/alias FTS — always runs regardless of routing.
|
|
2129
2990
|
* Name-first: an exact title, then title-weighted BM25 (see fts-query.ts).
|
|
2130
2991
|
*/
|
|
2131
|
-
searchDocumentsFts(query, limit = 10) {
|
|
2132
|
-
const match = ftsQuery(query);
|
|
2133
|
-
if (!match) return [];
|
|
2992
|
+
searchDocumentsFts(query, limit = 10, allowedDocumentIds) {
|
|
2134
2993
|
try {
|
|
2135
|
-
return documentFtsTopIds(this.router.db,
|
|
2136
|
-
documentId,
|
|
2137
|
-
|
|
2138
|
-
|
|
2994
|
+
return documentFtsTopIds(this.router.db, query, limit, allowedDocumentIds).map(
|
|
2995
|
+
(documentId, i) => ({
|
|
2996
|
+
documentId,
|
|
2997
|
+
rank: i
|
|
2998
|
+
})
|
|
2999
|
+
);
|
|
2139
3000
|
} catch {
|
|
2140
3001
|
return [];
|
|
2141
3002
|
}
|
|
@@ -2159,6 +3020,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2159
3020
|
* takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
|
|
2160
3021
|
*/
|
|
2161
3022
|
scoreShards(queryVector) {
|
|
3023
|
+
this.assertQueryWidth(queryVector);
|
|
2162
3024
|
const rows = this.router.db.prepare("SELECT shard_id, embedding FROM route_centroids").all();
|
|
2163
3025
|
const best = /* @__PURE__ */ new Map();
|
|
2164
3026
|
for (const row of rows) {
|
|
@@ -2168,31 +3030,80 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2168
3030
|
Math.floor(row.embedding.byteLength / 4)
|
|
2169
3031
|
);
|
|
2170
3032
|
let dot = 0;
|
|
2171
|
-
|
|
2172
|
-
for (let i = 0; i < n; i++) dot += centroid[i] * queryVector[i];
|
|
3033
|
+
for (let i = 0; i < queryVector.length; i++) dot += centroid[i] * queryVector[i];
|
|
2173
3034
|
const id = Number(row.shard_id);
|
|
2174
3035
|
if (dot > (best.get(id) ?? Number.NEGATIVE_INFINITY)) best.set(id, dot);
|
|
2175
3036
|
}
|
|
2176
3037
|
return [...best.entries()].map(([shardId, score]) => ({ shardId, score }));
|
|
2177
3038
|
}
|
|
2178
|
-
/**
|
|
2179
|
-
|
|
3039
|
+
/**
|
|
3040
|
+
* Top-S shard ids within THIS catalog (single-catalog callers, tests).
|
|
3041
|
+
* With the raw `query`, also the shard of the page it names by title
|
|
3042
|
+
* (`titleRouteShards`), so the result can exceed S by `TITLE_ROUTE_SHARDS`.
|
|
3043
|
+
*/
|
|
3044
|
+
routeShards(queryVector, shardBudget, query) {
|
|
2180
3045
|
if (this.shards.length <= shardBudget) return this.shards.map((s) => s.id);
|
|
2181
|
-
|
|
3046
|
+
const routed = this.scoreShards(queryVector).sort((a, b) => b.score - a.score).slice(0, shardBudget).map((s) => s.shardId);
|
|
3047
|
+
if (query) {
|
|
3048
|
+
for (const named of this.titleRouteShards(query, TITLE_ROUTE_SHARDS)) {
|
|
3049
|
+
if (!routed.includes(named.shardId)) routed.push(named.shardId);
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
return routed;
|
|
3053
|
+
}
|
|
3054
|
+
/**
|
|
3055
|
+
* Title-assisted routing: the shards holding the pages `query` names by
|
|
3056
|
+
* title (`namedTitleMatches`), most specific name first, at most `limit`
|
|
3057
|
+
* distinct shards. Centroid routing compares a question with averages of
|
|
3058
|
+
* whole shards, and e5 scores those in a band a few hundredths wide, so
|
|
3059
|
+
* the shard holding the very page a question names can rank 8th of 10:
|
|
3060
|
+
* "Who were the members of ABBA?" never reached the ABBA article in the
|
|
3061
|
+
* Wikipedia music catalog. Its title does — the vector arm scans that
|
|
3062
|
+
* shard too. Measured on the 10-shard music and film-tv catalogs, one such
|
|
3063
|
+
* shard lifted end-to-end recall@8 on "What is <title>?" questions from
|
|
3064
|
+
* 86% to 92% and 84% to 91%, at about 0.15 extra shard scans per query.
|
|
3065
|
+
*
|
|
3066
|
+
* `score` is the named match's specificity (fts-query.ts), comparable
|
|
3067
|
+
* across catalogs closely enough for the daemon to choose which catalogs'
|
|
3068
|
+
* shards to add when several name a page. Empty on any FTS error.
|
|
3069
|
+
*/
|
|
3070
|
+
titleRouteShards(query, limit) {
|
|
3071
|
+
if (limit <= 0) return [];
|
|
3072
|
+
try {
|
|
3073
|
+
const shardOf = this.router.db.prepare("SELECT shard_id FROM documents WHERE id = ?");
|
|
3074
|
+
const out = [];
|
|
3075
|
+
for (const named of namedTitleMatches(this.router.db, query, limit * 4)) {
|
|
3076
|
+
const row = shardOf.get(named.documentId);
|
|
3077
|
+
if (!row) continue;
|
|
3078
|
+
const shardId = Number(row.shard_id);
|
|
3079
|
+
if (out.some((o) => o.shardId === shardId)) continue;
|
|
3080
|
+
out.push({ shardId, score: named.score });
|
|
3081
|
+
if (out.length >= limit) break;
|
|
3082
|
+
}
|
|
3083
|
+
return out.sort((a, b) => compareScores(b.score, a.score));
|
|
3084
|
+
} catch {
|
|
3085
|
+
return [];
|
|
3086
|
+
}
|
|
2182
3087
|
}
|
|
2183
3088
|
/**
|
|
2184
3089
|
* Two-stage semantic search over the routed shards: sign-bit scan
|
|
2185
3090
|
* (stage 1) → int8 rerank (stage 2). Sequential by design (§9).
|
|
2186
3091
|
*/
|
|
2187
3092
|
searchSemantic(queryVector, opts = {}) {
|
|
2188
|
-
const shardIds = this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT);
|
|
2189
|
-
return this.searchShards(
|
|
3093
|
+
const shardIds = opts.allowedDocumentIds ? this.shardsForDocuments(opts.allowedDocumentIds) : this.routeShards(queryVector, opts.shardBudget ?? ROUTE_SHARDS_EXPLICIT, opts.query);
|
|
3094
|
+
return this.searchShards(
|
|
3095
|
+
queryVector,
|
|
3096
|
+
shardIds,
|
|
3097
|
+
opts.finalK ?? RERANK_FINAL_K,
|
|
3098
|
+
opts.allowedDocumentIds
|
|
3099
|
+
);
|
|
2190
3100
|
}
|
|
2191
3101
|
/**
|
|
2192
3102
|
* The scan stage against an EXPLICIT shard list — what the daemon calls
|
|
2193
3103
|
* after global cross-catalog routing has already spent the S budget.
|
|
2194
3104
|
*/
|
|
2195
|
-
searchShards(queryVector, shardIds, finalK) {
|
|
3105
|
+
searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
|
|
3106
|
+
this.assertQueryWidth(queryVector);
|
|
2196
3107
|
const center = this.binaryCenter();
|
|
2197
3108
|
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
2198
3109
|
const hits = [];
|
|
@@ -2200,8 +3111,21 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2200
3111
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2201
3112
|
if (!shard) continue;
|
|
2202
3113
|
const db = this.shardDb(shard);
|
|
2203
|
-
const
|
|
2204
|
-
|
|
3114
|
+
const eligible = allowedDocumentIds ? new Set(
|
|
3115
|
+
db.prepare(
|
|
3116
|
+
"SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
|
|
3117
|
+
).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
|
|
3118
|
+
) : void 0;
|
|
3119
|
+
const media = this.shardMedia(shard);
|
|
3120
|
+
const k = rerankK(finalK, eligible?.size ?? shard.chunkCount - media.ids.length);
|
|
3121
|
+
const excluded = media.ids.length > 0 ? media.idSet : void 0;
|
|
3122
|
+
const candidates = asymmetricTopK(
|
|
3123
|
+
this.shardBits(shard),
|
|
3124
|
+
scanQuery,
|
|
3125
|
+
k,
|
|
3126
|
+
eligible,
|
|
3127
|
+
excluded
|
|
3128
|
+
).map((hit) => ({
|
|
2205
3129
|
chunk_id: hit.chunkId
|
|
2206
3130
|
}));
|
|
2207
3131
|
if (candidates.length === 0) continue;
|
|
@@ -2212,63 +3136,57 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2212
3136
|
const int8 = new Int8Array(row.v.buffer, row.v.byteOffset, row.v.byteLength);
|
|
2213
3137
|
return { chunkId: Number(c.chunk_id), cosine: rerankScore(queryVector, int8) };
|
|
2214
3138
|
}).filter((r) => r !== null).sort((a, b) => b.cosine - a.cosine).slice(0, finalK);
|
|
2215
|
-
const getChunk = db.prepare(
|
|
2216
|
-
`SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
|
|
2217
|
-
FROM chunks WHERE id = ?`
|
|
2218
|
-
);
|
|
3139
|
+
const getChunk = db.prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`);
|
|
2219
3140
|
for (const r of reranked) {
|
|
2220
3141
|
const row = getChunk.get(BigInt(r.chunkId));
|
|
2221
3142
|
if (!row) continue;
|
|
2222
|
-
hits.push({
|
|
2223
|
-
chunkUid: row.chunk_uid,
|
|
2224
|
-
documentId: row.document_id,
|
|
2225
|
-
title: row.title,
|
|
2226
|
-
headingPath: parseHeadingPath(row.heading_path),
|
|
2227
|
-
lineStart: Number(row.line_start),
|
|
2228
|
-
lineEnd: Number(row.line_end),
|
|
2229
|
-
text: row.text,
|
|
2230
|
-
cosine: r.cosine,
|
|
2231
|
-
source: "vector",
|
|
2232
|
-
shardId
|
|
2233
|
-
});
|
|
3143
|
+
hits.push({ ...this.hitFromRow(row, shardId), cosine: r.cosine, source: "vector" });
|
|
2234
3144
|
}
|
|
2235
3145
|
}
|
|
2236
3146
|
hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
|
|
2237
3147
|
return hits;
|
|
2238
3148
|
}
|
|
2239
3149
|
/** Chunk-body FTS over the routed shards (explicit search only). */
|
|
2240
|
-
searchChunksFts(query, shardIds, limitPerShard = 12) {
|
|
2241
|
-
const match = ftsQuery(query);
|
|
2242
|
-
if (!match) return [];
|
|
3150
|
+
searchChunksFts(query, shardIds, limitPerShard = 12, allowedDocumentIds) {
|
|
2243
3151
|
const hits = [];
|
|
2244
3152
|
for (const shardId of shardIds) {
|
|
2245
3153
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2246
3154
|
if (!shard) continue;
|
|
2247
3155
|
const db = this.shardDb(shard);
|
|
2248
3156
|
try {
|
|
3157
|
+
const match = selectiveFtsQuery(
|
|
3158
|
+
query,
|
|
3159
|
+
shard.chunkCount,
|
|
3160
|
+
(phrase) => this.chunkFtsRows(shard, db, phrase)
|
|
3161
|
+
);
|
|
3162
|
+
if (!match) return hits;
|
|
2249
3163
|
const rows = db.prepare(
|
|
2250
|
-
`SELECT
|
|
3164
|
+
`SELECT ${this.chunkColumns().split(", ").map((column) => `c.${column}`).join(", ")}
|
|
2251
3165
|
FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
|
|
2252
|
-
WHERE fts_chunks MATCH ? ORDER BY f.rank LIMIT ?`
|
|
2253
|
-
).all(
|
|
3166
|
+
WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
|
|
3167
|
+
).all(
|
|
3168
|
+
...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
|
|
3169
|
+
);
|
|
2254
3170
|
for (const row of rows) {
|
|
2255
|
-
hits.push({
|
|
2256
|
-
chunkUid: row.chunk_uid,
|
|
2257
|
-
documentId: row.document_id,
|
|
2258
|
-
title: row.title,
|
|
2259
|
-
headingPath: parseHeadingPath(row.heading_path),
|
|
2260
|
-
lineStart: Number(row.line_start),
|
|
2261
|
-
lineEnd: Number(row.line_end),
|
|
2262
|
-
text: row.text,
|
|
2263
|
-
source: "fts",
|
|
2264
|
-
shardId
|
|
2265
|
-
});
|
|
3171
|
+
hits.push({ ...this.hitFromRow(row, shardId), source: "fts" });
|
|
2266
3172
|
}
|
|
2267
3173
|
} catch {
|
|
2268
3174
|
}
|
|
2269
3175
|
}
|
|
2270
3176
|
return hits;
|
|
2271
3177
|
}
|
|
3178
|
+
/** Rows of one shard's `fts_chunks` an FTS5 phrase matches — what `selectiveFtsQuery` prunes on. */
|
|
3179
|
+
chunkFtsRows(shard, db, phrase) {
|
|
3180
|
+
const key = `${shard.path}\0${phrase}`;
|
|
3181
|
+
let rows = this.chunkTermRows.get(key);
|
|
3182
|
+
if (rows === void 0) {
|
|
3183
|
+
if (this.chunkTermRows.size >= CHUNK_TERM_ROWS_CACHE_MAX) this.chunkTermRows.clear();
|
|
3184
|
+
const row = db.prepare("SELECT COUNT(*) AS c FROM fts_chunks WHERE fts_chunks MATCH ?").get(phrase);
|
|
3185
|
+
rows = Number(row.c);
|
|
3186
|
+
this.chunkTermRows.set(key, rows);
|
|
3187
|
+
}
|
|
3188
|
+
return rows;
|
|
3189
|
+
}
|
|
2272
3190
|
/**
|
|
2273
3191
|
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
2274
3192
|
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
@@ -2322,19 +3240,20 @@ function parseHeadingPath(raw) {
|
|
|
2322
3240
|
return [];
|
|
2323
3241
|
}
|
|
2324
3242
|
}
|
|
2325
|
-
var ftsQuery = sanitizeFtsQuery;
|
|
2326
3243
|
|
|
2327
3244
|
// src/reader/validate.ts
|
|
2328
|
-
import { readFile } from "fs/promises";
|
|
3245
|
+
import { open, readFile } from "fs/promises";
|
|
2329
3246
|
import { join as join3 } from "path";
|
|
2330
3247
|
import {
|
|
2331
3248
|
KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
|
|
2332
|
-
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2333
|
-
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
|
|
2334
|
-
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2335
3249
|
assetExtension as assetExtension2,
|
|
2336
3250
|
assetKindForExtension as assetKindForExtension2,
|
|
3251
|
+
assetModality as assetModality2,
|
|
3252
|
+
canonicalizeJson as canonicalizeJson2,
|
|
3253
|
+
formatAtLeast as formatAtLeast2,
|
|
2337
3254
|
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
3255
|
+
isKnowledgeMediaAssetPath as isKnowledgeMediaAssetPath2,
|
|
3256
|
+
knowledgeAssetLimitsProblem as knowledgeAssetLimitsProblem2,
|
|
2338
3257
|
sniffAssetType as sniffAssetType2,
|
|
2339
3258
|
svgInertnessProblem as svgInertnessProblem2
|
|
2340
3259
|
} from "@bendyline/gezk";
|
|
@@ -2409,6 +3328,23 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2409
3328
|
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2410
3329
|
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2411
3330
|
);
|
|
3331
|
+
if (formatAtLeast2(manifest.formatVersion, "0.7")) {
|
|
3332
|
+
try {
|
|
3333
|
+
const spatial = handle.spatialIntegrity();
|
|
3334
|
+
check("document-locations-table", spatial.hasTable);
|
|
3335
|
+
check(
|
|
3336
|
+
"document-locations-summary",
|
|
3337
|
+
Boolean(manifest.spatial) && canonicalizeJson2(spatial.manifest) === canonicalizeJson2(manifest.spatial),
|
|
3338
|
+
"spatial counts and coverage must match router points"
|
|
3339
|
+
);
|
|
3340
|
+
} catch (error) {
|
|
3341
|
+
check(
|
|
3342
|
+
"document-locations-integrity",
|
|
3343
|
+
false,
|
|
3344
|
+
error instanceof Error ? error.message : String(error)
|
|
3345
|
+
);
|
|
3346
|
+
}
|
|
3347
|
+
}
|
|
2412
3348
|
const topics = handle.topics();
|
|
2413
3349
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2414
3350
|
check("topics-tree", ...topicTreeProblem(topics));
|
|
@@ -2428,13 +3364,16 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2428
3364
|
} else {
|
|
2429
3365
|
const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
|
|
2430
3366
|
check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
3367
|
+
if (!formatAtLeast2(manifest.formatVersion, "0.8")) {
|
|
3368
|
+
const media = assetFiles.filter((f) => isKnowledgeMediaAssetPath2(f.path));
|
|
3369
|
+
check(
|
|
3370
|
+
"assets-media-in-0.8",
|
|
3371
|
+
media.length === 0,
|
|
3372
|
+
`audio/video assets need format 0.8: ${media.map((f) => f.path).join(", ")}`
|
|
3373
|
+
);
|
|
3374
|
+
}
|
|
3375
|
+
const limits = knowledgeAssetLimitsProblem2(assetFiles);
|
|
3376
|
+
check("assets-limits", limits === null, limits ?? "");
|
|
2438
3377
|
check(
|
|
2439
3378
|
"counts-assets",
|
|
2440
3379
|
(manifest.counts.assets ?? 0) === assetFiles.length,
|
|
@@ -2446,11 +3385,22 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2446
3385
|
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
2447
3386
|
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
2448
3387
|
);
|
|
3388
|
+
const toc = handle.tocIntegrity();
|
|
3389
|
+
check(
|
|
3390
|
+
"toc-references",
|
|
3391
|
+
toc.invalidReferences === 0 && toc.missingPrimary === 0 && toc.duplicateReferences === 0,
|
|
3392
|
+
`${toc.invalidReferences} invalid references, ${toc.missingPrimary} missing primary placements, ${toc.duplicateReferences} duplicate placements`
|
|
3393
|
+
);
|
|
3394
|
+
check(
|
|
3395
|
+
"toc-counts",
|
|
3396
|
+
toc.incorrectCounts === 0,
|
|
3397
|
+
`${toc.incorrectCounts} incorrect per-topic counts`
|
|
3398
|
+
);
|
|
2449
3399
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
2450
3400
|
check(
|
|
2451
3401
|
"counts-documents",
|
|
2452
|
-
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum ===
|
|
2453
|
-
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, manifest ${manifest.counts.documents}`
|
|
3402
|
+
handle.documentsPage({ limit: 1 }).total === manifest.counts.documents && topicDocSum === toc.placements,
|
|
3403
|
+
`documents table ${handle.documentsPage({ limit: 1 }).total}, topic sum ${topicDocSum}, placements ${toc.placements}, manifest documents ${manifest.counts.documents}`
|
|
2454
3404
|
);
|
|
2455
3405
|
check(
|
|
2456
3406
|
"counts-shards",
|
|
@@ -2482,6 +3432,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2482
3432
|
bodies.maxCompressedBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024,
|
|
2483
3433
|
`largest compressed body is ${bodies.maxCompressedBytes} bytes`
|
|
2484
3434
|
);
|
|
3435
|
+
const mediaRowCounts = { image: 0, video: 0, audio: 0 };
|
|
2485
3436
|
for (const shard of handle.shards) {
|
|
2486
3437
|
const conn = openCatalogDatabase(handle.resolveCatalogPath(shard.path));
|
|
2487
3438
|
const db = conn.db;
|
|
@@ -2518,6 +3469,21 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2518
3469
|
badBit === 0 && badInt8 === 0,
|
|
2519
3470
|
`${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
|
|
2520
3471
|
);
|
|
3472
|
+
if (handle.schemaVersion >= 5) {
|
|
3473
|
+
const rows = db.prepare(
|
|
3474
|
+
`SELECT modality, asset_path, thumbnail_path FROM chunks WHERE modality <> 'text'`
|
|
3475
|
+
).all();
|
|
3476
|
+
const declaredAssets = new Set(assetFiles.map((f) => f.path));
|
|
3477
|
+
const wrong = rows.filter(
|
|
3478
|
+
(r) => !declaredAssets.has(r.asset_path) || assetModality2(r.asset_path) !== r.modality || r.thumbnail_path !== null && (!declaredAssets.has(r.thumbnail_path) || assetModality2(r.thumbnail_path) !== "image")
|
|
3479
|
+
).slice(0, 5).map((r) => `${r.modality} ${r.asset_path}`);
|
|
3480
|
+
check(
|
|
3481
|
+
`media-rows:${shard.path}`,
|
|
3482
|
+
wrong.length === 0,
|
|
3483
|
+
`media rows naming an undeclared or mismatched asset: ${wrong.join(", ")}`
|
|
3484
|
+
);
|
|
3485
|
+
for (const r of rows) mediaRowCounts[r.modality] += 1;
|
|
3486
|
+
}
|
|
2521
3487
|
const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
|
|
2522
3488
|
check(
|
|
2523
3489
|
`chunk-ids-dense:${shard.path}`,
|
|
@@ -2529,6 +3495,14 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2529
3495
|
}
|
|
2530
3496
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
2531
3497
|
}
|
|
3498
|
+
if (handle.schemaVersion >= 5) {
|
|
3499
|
+
const counted = manifest.counts.media ?? { image: 0, video: 0, audio: 0 };
|
|
3500
|
+
check(
|
|
3501
|
+
"counts-media",
|
|
3502
|
+
counted.image === mediaRowCounts.image && counted.video === mediaRowCounts.video && counted.audio === mediaRowCounts.audio,
|
|
3503
|
+
`manifest counts ${JSON.stringify(counted)}, shards hold ${JSON.stringify(mediaRowCounts)}`
|
|
3504
|
+
);
|
|
3505
|
+
}
|
|
2532
3506
|
if (handle.schemaVersion >= 3) {
|
|
2533
3507
|
const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
|
|
2534
3508
|
check(
|
|
@@ -2540,7 +3514,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2540
3514
|
for (const file of assetFiles) {
|
|
2541
3515
|
const ext = assetExtension2(file.path);
|
|
2542
3516
|
if (!ext) continue;
|
|
2543
|
-
const bytes = await readFile(join3(rootDir, file.path));
|
|
3517
|
+
const bytes = isKnowledgeMediaAssetPath2(file.path) ? await readLeadingBytes(join3(rootDir, file.path), 64) : await readFile(join3(rootDir, file.path));
|
|
2544
3518
|
const kind = sniffAssetType2(bytes);
|
|
2545
3519
|
check(
|
|
2546
3520
|
`asset-type:${file.path}`,
|
|
@@ -2578,6 +3552,8 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
2578
3552
|
);
|
|
2579
3553
|
}
|
|
2580
3554
|
}
|
|
3555
|
+
} catch (error) {
|
|
3556
|
+
check("catalog-structure", false, error instanceof Error ? error.message : String(error));
|
|
2581
3557
|
} finally {
|
|
2582
3558
|
handle.close();
|
|
2583
3559
|
}
|
|
@@ -2616,6 +3592,16 @@ function topicTreeProblem(topics) {
|
|
|
2616
3592
|
}
|
|
2617
3593
|
return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
|
|
2618
3594
|
}
|
|
3595
|
+
async function readLeadingBytes(path, length) {
|
|
3596
|
+
const handle = await open(path, "r");
|
|
3597
|
+
try {
|
|
3598
|
+
const buffer = new Uint8Array(length);
|
|
3599
|
+
const { bytesRead } = await handle.read(buffer, 0, length, 0);
|
|
3600
|
+
return buffer.subarray(0, bytesRead);
|
|
3601
|
+
} finally {
|
|
3602
|
+
await handle.close();
|
|
3603
|
+
}
|
|
3604
|
+
}
|
|
2619
3605
|
|
|
2620
3606
|
// src/export/duckdb.ts
|
|
2621
3607
|
import { spawn } from "child_process";
|
|
@@ -2686,13 +3672,13 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
2686
3672
|
}
|
|
2687
3673
|
|
|
2688
3674
|
// src/export/parquet.ts
|
|
2689
|
-
import { createHash as
|
|
3675
|
+
import { createHash as createHash5 } from "crypto";
|
|
2690
3676
|
import { createWriteStream as createWriteStream3 } from "fs";
|
|
2691
3677
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2692
3678
|
import { join as join4, resolve as resolve3 } from "path";
|
|
2693
3679
|
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2694
3680
|
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4, embeddingProfileCenter as embeddingProfileCenter2 } from "@bendyline/gezk";
|
|
2695
|
-
var PARQUET_EXPORT_VERSION =
|
|
3681
|
+
var PARQUET_EXPORT_VERSION = 4;
|
|
2696
3682
|
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2697
3683
|
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2698
3684
|
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
@@ -2739,6 +3725,34 @@ async function exportCatalogParquet(opts) {
|
|
|
2739
3725
|
onProgress: opts.onProgress
|
|
2740
3726
|
})
|
|
2741
3727
|
);
|
|
3728
|
+
const spatial = new DocumentSpatialIndex(router.db, router.schemaVersion === 3);
|
|
3729
|
+
const locations = spatial.rows();
|
|
3730
|
+
if (spatial.hasTable || locations.length) {
|
|
3731
|
+
const path = join4(staging, "document-locations.ndjson");
|
|
3732
|
+
const out = ndjsonWriter(path);
|
|
3733
|
+
for (const { documentId, location } of locations)
|
|
3734
|
+
await out.write({
|
|
3735
|
+
document_id: documentId,
|
|
3736
|
+
location_id: location.id,
|
|
3737
|
+
latitude: location.latitude,
|
|
3738
|
+
longitude: location.longitude,
|
|
3739
|
+
role: location.role,
|
|
3740
|
+
provenance_json: location.provenance ? JSON.stringify(location.provenance) : null
|
|
3741
|
+
});
|
|
3742
|
+
await out.close();
|
|
3743
|
+
const columns = "{document_id:'VARCHAR',location_id:'VARCHAR',latitude:'DOUBLE',longitude:'DOUBLE',role:'VARCHAR',provenance_json:'VARCHAR'}";
|
|
3744
|
+
const source = locations.length ? `read_ndjson(${sqlString(path)}, columns=${columns})` : "(SELECT NULL::VARCHAR document_id, NULL::VARCHAR location_id, NULL::DOUBLE latitude, NULL::DOUBLE longitude, NULL::VARCHAR role, NULL::VARCHAR provenance_json WHERE false)";
|
|
3745
|
+
files.push(
|
|
3746
|
+
await writeParquet(opts.duckdb, {
|
|
3747
|
+
outDir,
|
|
3748
|
+
file: "document-locations.parquet",
|
|
3749
|
+
table: "document_locations",
|
|
3750
|
+
rows: locations.length,
|
|
3751
|
+
sql: `${PRELUDE}COPY (SELECT * FROM ${source}) TO ${sqlString(join4(outDir, "document-locations.parquet"))} ${copyOptions(rowGroupSize, metadata)};`,
|
|
3752
|
+
onProgress: opts.onProgress
|
|
3753
|
+
})
|
|
3754
|
+
);
|
|
3755
|
+
}
|
|
2742
3756
|
for (const shard of manifest.router.shards) {
|
|
2743
3757
|
const tag = String(shard.id).padStart(3, "0");
|
|
2744
3758
|
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
@@ -3081,7 +4095,7 @@ async function writeParquet(cli, spec) {
|
|
|
3081
4095
|
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
3082
4096
|
rows: spec.rows,
|
|
3083
4097
|
sizeBytes: (await stat2(target)).size,
|
|
3084
|
-
sha256:
|
|
4098
|
+
sha256: createHash5("sha256").update(bytes).digest("hex")
|
|
3085
4099
|
};
|
|
3086
4100
|
}
|
|
3087
4101
|
async function findDuckdbBinary(env = process.env) {
|
|
@@ -3498,6 +4512,522 @@ var MULTILINGUAL_E5_SMALL_CENTER = [
|
|
|
3498
4512
|
0.049673
|
|
3499
4513
|
];
|
|
3500
4514
|
|
|
4515
|
+
// src/profiles/embeddinggemma-2-center.ts
|
|
4516
|
+
var EMBEDDINGGEMMA_2_512_CENTER = [
|
|
4517
|
+
-3987e-6,
|
|
4518
|
+
0.017212,
|
|
4519
|
+
0.023034,
|
|
4520
|
+
-0.020831,
|
|
4521
|
+
0.013136,
|
|
4522
|
+
0.018836,
|
|
4523
|
+
7334e-6,
|
|
4524
|
+
-0.022294,
|
|
4525
|
+
5809e-6,
|
|
4526
|
+
-0.147176,
|
|
4527
|
+
-5646e-6,
|
|
4528
|
+
-0.022884,
|
|
4529
|
+
-0.011497,
|
|
4530
|
+
1292e-6,
|
|
4531
|
+
0.014257,
|
|
4532
|
+
-1878e-6,
|
|
4533
|
+
0.051834,
|
|
4534
|
+
-0.012991,
|
|
4535
|
+
933e-6,
|
|
4536
|
+
0.022558,
|
|
4537
|
+
-0.034902,
|
|
4538
|
+
-0.030019,
|
|
4539
|
+
0.017879,
|
|
4540
|
+
0.015619,
|
|
4541
|
+
-0.014837,
|
|
4542
|
+
0.028657,
|
|
4543
|
+
0.017575,
|
|
4544
|
+
-0.045617,
|
|
4545
|
+
-0.03244,
|
|
4546
|
+
0.23277,
|
|
4547
|
+
-0.021971,
|
|
4548
|
+
-0.026583,
|
|
4549
|
+
0.011485,
|
|
4550
|
+
-0.01468,
|
|
4551
|
+
0.044649,
|
|
4552
|
+
0.018019,
|
|
4553
|
+
-0.011519,
|
|
4554
|
+
0.035574,
|
|
4555
|
+
512e-5,
|
|
4556
|
+
-0.030136,
|
|
4557
|
+
4583e-6,
|
|
4558
|
+
0.018314,
|
|
4559
|
+
-0.028859,
|
|
4560
|
+
0.027199,
|
|
4561
|
+
0.020695,
|
|
4562
|
+
-849e-6,
|
|
4563
|
+
0.015867,
|
|
4564
|
+
-0.024724,
|
|
4565
|
+
-8979e-6,
|
|
4566
|
+
0.023625,
|
|
4567
|
+
0.028948,
|
|
4568
|
+
-0.025744,
|
|
4569
|
+
-1547e-6,
|
|
4570
|
+
0.0281,
|
|
4571
|
+
0.024258,
|
|
4572
|
+
0.02262,
|
|
4573
|
+
1386e-6,
|
|
4574
|
+
0.015002,
|
|
4575
|
+
3764e-6,
|
|
4576
|
+
0.016978,
|
|
4577
|
+
0.019178,
|
|
4578
|
+
0.012229,
|
|
4579
|
+
-193e-6,
|
|
4580
|
+
0.032654,
|
|
4581
|
+
-0.030602,
|
|
4582
|
+
-1749e-6,
|
|
4583
|
+
0.027403,
|
|
4584
|
+
-4047e-6,
|
|
4585
|
+
0.031605,
|
|
4586
|
+
0.023884,
|
|
4587
|
+
0.015804,
|
|
4588
|
+
3971e-6,
|
|
4589
|
+
0.021826,
|
|
4590
|
+
0.02457,
|
|
4591
|
+
-0.023111,
|
|
4592
|
+
-0.032593,
|
|
4593
|
+
-0.090226,
|
|
4594
|
+
0.020798,
|
|
4595
|
+
6402e-6,
|
|
4596
|
+
0.015607,
|
|
4597
|
+
-8457e-6,
|
|
4598
|
+
-0.010067,
|
|
4599
|
+
6362e-6,
|
|
4600
|
+
9973e-6,
|
|
4601
|
+
0.0122,
|
|
4602
|
+
-0.020958,
|
|
4603
|
+
-0.031666,
|
|
4604
|
+
0.034654,
|
|
4605
|
+
-0.012847,
|
|
4606
|
+
-1119e-6,
|
|
4607
|
+
0.024259,
|
|
4608
|
+
0.018097,
|
|
4609
|
+
9105e-6,
|
|
4610
|
+
-9523e-6,
|
|
4611
|
+
-0.022682,
|
|
4612
|
+
-5167e-6,
|
|
4613
|
+
-0.022959,
|
|
4614
|
+
0.012175,
|
|
4615
|
+
-0.020933,
|
|
4616
|
+
0.014024,
|
|
4617
|
+
-4898e-6,
|
|
4618
|
+
-2018e-6,
|
|
4619
|
+
-7178e-6,
|
|
4620
|
+
0.038098,
|
|
4621
|
+
0.016307,
|
|
4622
|
+
5668e-6,
|
|
4623
|
+
-0.014475,
|
|
4624
|
+
661e-6,
|
|
4625
|
+
-1287e-6,
|
|
4626
|
+
-0.025024,
|
|
4627
|
+
5704e-6,
|
|
4628
|
+
-0.32188,
|
|
4629
|
+
-0.026611,
|
|
4630
|
+
0.032111,
|
|
4631
|
+
-9511e-6,
|
|
4632
|
+
0.260031,
|
|
4633
|
+
0.027274,
|
|
4634
|
+
-909e-5,
|
|
4635
|
+
0.055461,
|
|
4636
|
+
0.024395,
|
|
4637
|
+
-0.017894,
|
|
4638
|
+
0.061775,
|
|
4639
|
+
-8529e-6,
|
|
4640
|
+
-1795e-6,
|
|
4641
|
+
-0.017545,
|
|
4642
|
+
0.011492,
|
|
4643
|
+
0.044594,
|
|
4644
|
+
38e-4,
|
|
4645
|
+
-0.02231,
|
|
4646
|
+
0.029977,
|
|
4647
|
+
0.010629,
|
|
4648
|
+
-489e-5,
|
|
4649
|
+
2646e-6,
|
|
4650
|
+
6408e-6,
|
|
4651
|
+
-0.014453,
|
|
4652
|
+
-0.020764,
|
|
4653
|
+
-0.018043,
|
|
4654
|
+
-0.015625,
|
|
4655
|
+
0.074213,
|
|
4656
|
+
-0.012319,
|
|
4657
|
+
-8038e-6,
|
|
4658
|
+
-6265e-6,
|
|
4659
|
+
0.012548,
|
|
4660
|
+
-621e-6,
|
|
4661
|
+
-0.021893,
|
|
4662
|
+
0.039889,
|
|
4663
|
+
3623e-6,
|
|
4664
|
+
-0.029173,
|
|
4665
|
+
-716e-6,
|
|
4666
|
+
-0.013168,
|
|
4667
|
+
-6674e-6,
|
|
4668
|
+
-0.010069,
|
|
4669
|
+
-0.02451,
|
|
4670
|
+
0.015114,
|
|
4671
|
+
-0.012945,
|
|
4672
|
+
1561e-6,
|
|
4673
|
+
0.013299,
|
|
4674
|
+
-0.029098,
|
|
4675
|
+
0.01304,
|
|
4676
|
+
0.013519,
|
|
4677
|
+
-0.014282,
|
|
4678
|
+
0.026849,
|
|
4679
|
+
-2462e-6,
|
|
4680
|
+
0.019473,
|
|
4681
|
+
-8227e-6,
|
|
4682
|
+
-822e-6,
|
|
4683
|
+
-7241e-6,
|
|
4684
|
+
4259e-6,
|
|
4685
|
+
-0.016026,
|
|
4686
|
+
7987e-6,
|
|
4687
|
+
-7065e-6,
|
|
4688
|
+
-1229e-6,
|
|
4689
|
+
-0.01583,
|
|
4690
|
+
0.021277,
|
|
4691
|
+
0.013837,
|
|
4692
|
+
0.024282,
|
|
4693
|
+
-7589e-6,
|
|
4694
|
+
-0.016577,
|
|
4695
|
+
7774e-6,
|
|
4696
|
+
0.010479,
|
|
4697
|
+
-0.012539,
|
|
4698
|
+
-0.020799,
|
|
4699
|
+
0.026506,
|
|
4700
|
+
8733e-6,
|
|
4701
|
+
-0.015001,
|
|
4702
|
+
2151e-6,
|
|
4703
|
+
-0.011922,
|
|
4704
|
+
-0.02785,
|
|
4705
|
+
-0.060631,
|
|
4706
|
+
3372e-6,
|
|
4707
|
+
0.022441,
|
|
4708
|
+
-0.012448,
|
|
4709
|
+
6827e-6,
|
|
4710
|
+
-7047e-6,
|
|
4711
|
+
1249e-6,
|
|
4712
|
+
-2934e-6,
|
|
4713
|
+
4559e-6,
|
|
4714
|
+
-0.011409,
|
|
4715
|
+
-0.012564,
|
|
4716
|
+
5788e-6,
|
|
4717
|
+
-0.020554,
|
|
4718
|
+
-4292e-6,
|
|
4719
|
+
0.01927,
|
|
4720
|
+
0.013487,
|
|
4721
|
+
-0.015484,
|
|
4722
|
+
9704e-6,
|
|
4723
|
+
5617e-6,
|
|
4724
|
+
0.02447,
|
|
4725
|
+
2056e-6,
|
|
4726
|
+
0.011381,
|
|
4727
|
+
0.039831,
|
|
4728
|
+
-7878e-6,
|
|
4729
|
+
-0.011366,
|
|
4730
|
+
-814e-5,
|
|
4731
|
+
8099e-6,
|
|
4732
|
+
6637e-6,
|
|
4733
|
+
154e-6,
|
|
4734
|
+
0.072273,
|
|
4735
|
+
7195e-6,
|
|
4736
|
+
-0.011899,
|
|
4737
|
+
0.016669,
|
|
4738
|
+
3331e-6,
|
|
4739
|
+
0.032787,
|
|
4740
|
+
-0.012251,
|
|
4741
|
+
-6049e-6,
|
|
4742
|
+
-0.017814,
|
|
4743
|
+
0.055136,
|
|
4744
|
+
0.028157,
|
|
4745
|
+
-0.028898,
|
|
4746
|
+
0.035646,
|
|
4747
|
+
0.036174,
|
|
4748
|
+
-5621e-6,
|
|
4749
|
+
0.050018,
|
|
4750
|
+
-0.040307,
|
|
4751
|
+
0.060677,
|
|
4752
|
+
58e-6,
|
|
4753
|
+
-0.045134,
|
|
4754
|
+
4752e-6,
|
|
4755
|
+
7161e-6,
|
|
4756
|
+
0.040398,
|
|
4757
|
+
-0.051202,
|
|
4758
|
+
-572e-6,
|
|
4759
|
+
6186e-6,
|
|
4760
|
+
-0.010613,
|
|
4761
|
+
0.039894,
|
|
4762
|
+
-0.014924,
|
|
4763
|
+
49e-4,
|
|
4764
|
+
-0.011771,
|
|
4765
|
+
9397e-6,
|
|
4766
|
+
9457e-6,
|
|
4767
|
+
0.025576,
|
|
4768
|
+
2251e-6,
|
|
4769
|
+
0.020638,
|
|
4770
|
+
-0.029722,
|
|
4771
|
+
7545e-6,
|
|
4772
|
+
-533e-6,
|
|
4773
|
+
0.026445,
|
|
4774
|
+
921e-6,
|
|
4775
|
+
9681e-6,
|
|
4776
|
+
-959e-6,
|
|
4777
|
+
-0.018021,
|
|
4778
|
+
807e-5,
|
|
4779
|
+
8041e-6,
|
|
4780
|
+
-8004e-6,
|
|
4781
|
+
4422e-6,
|
|
4782
|
+
-2478e-6,
|
|
4783
|
+
7376e-6,
|
|
4784
|
+
-828e-5,
|
|
4785
|
+
8006e-6,
|
|
4786
|
+
-3182e-6,
|
|
4787
|
+
5771e-6,
|
|
4788
|
+
-526e-5,
|
|
4789
|
+
0.01422,
|
|
4790
|
+
-3677e-6,
|
|
4791
|
+
-0.010823,
|
|
4792
|
+
0.019873,
|
|
4793
|
+
3328e-6,
|
|
4794
|
+
4427e-6,
|
|
4795
|
+
0.02227,
|
|
4796
|
+
-0.014147,
|
|
4797
|
+
0.014306,
|
|
4798
|
+
-5089e-6,
|
|
4799
|
+
-0.010849,
|
|
4800
|
+
322e-5,
|
|
4801
|
+
-8249e-6,
|
|
4802
|
+
0.019141,
|
|
4803
|
+
-0.012199,
|
|
4804
|
+
0.015451,
|
|
4805
|
+
0.026615,
|
|
4806
|
+
0.02352,
|
|
4807
|
+
0.011117,
|
|
4808
|
+
-399e-6,
|
|
4809
|
+
0.016821,
|
|
4810
|
+
0.023447,
|
|
4811
|
+
6638e-6,
|
|
4812
|
+
8598e-6,
|
|
4813
|
+
-0.02239,
|
|
4814
|
+
0.017325,
|
|
4815
|
+
0.022729,
|
|
4816
|
+
9922e-6,
|
|
4817
|
+
-4099e-6,
|
|
4818
|
+
9124e-6,
|
|
4819
|
+
-8221e-6,
|
|
4820
|
+
-7554e-6,
|
|
4821
|
+
0.046752,
|
|
4822
|
+
0.027253,
|
|
4823
|
+
437e-6,
|
|
4824
|
+
4693e-6,
|
|
4825
|
+
7093e-6,
|
|
4826
|
+
-0.014217,
|
|
4827
|
+
-0.024195,
|
|
4828
|
+
2768e-6,
|
|
4829
|
+
-0.012022,
|
|
4830
|
+
-0.012142,
|
|
4831
|
+
-4326e-6,
|
|
4832
|
+
-0.01054,
|
|
4833
|
+
0.01168,
|
|
4834
|
+
4821e-6,
|
|
4835
|
+
-5609e-6,
|
|
4836
|
+
-0.096205,
|
|
4837
|
+
-0.018675,
|
|
4838
|
+
-0.017776,
|
|
4839
|
+
-0.010107,
|
|
4840
|
+
-0.012825,
|
|
4841
|
+
-644e-6,
|
|
4842
|
+
0.011283,
|
|
4843
|
+
3759e-6,
|
|
4844
|
+
-0.025286,
|
|
4845
|
+
2306e-6,
|
|
4846
|
+
9735e-6,
|
|
4847
|
+
-158e-5,
|
|
4848
|
+
0.012131,
|
|
4849
|
+
-0.018451,
|
|
4850
|
+
0.014135,
|
|
4851
|
+
6421e-6,
|
|
4852
|
+
-7006e-6,
|
|
4853
|
+
-6122e-6,
|
|
4854
|
+
0.013727,
|
|
4855
|
+
8605e-6,
|
|
4856
|
+
-6385e-6,
|
|
4857
|
+
-0.016703,
|
|
4858
|
+
-0.184351,
|
|
4859
|
+
4162e-6,
|
|
4860
|
+
0.022412,
|
|
4861
|
+
-0.021583,
|
|
4862
|
+
-5397e-6,
|
|
4863
|
+
-0.011593,
|
|
4864
|
+
-1118e-6,
|
|
4865
|
+
0.013629,
|
|
4866
|
+
4367e-6,
|
|
4867
|
+
238e-5,
|
|
4868
|
+
0.030207,
|
|
4869
|
+
2765e-6,
|
|
4870
|
+
0.015452,
|
|
4871
|
+
-0.023998,
|
|
4872
|
+
0.016889,
|
|
4873
|
+
5828e-6,
|
|
4874
|
+
0.01799,
|
|
4875
|
+
-0.019521,
|
|
4876
|
+
0.012821,
|
|
4877
|
+
-0.021567,
|
|
4878
|
+
5999e-6,
|
|
4879
|
+
0.017122,
|
|
4880
|
+
-9796e-6,
|
|
4881
|
+
6654e-6,
|
|
4882
|
+
291e-5,
|
|
4883
|
+
-0.017359,
|
|
4884
|
+
9342e-6,
|
|
4885
|
+
-929e-6,
|
|
4886
|
+
-0.012544,
|
|
4887
|
+
-8234e-6,
|
|
4888
|
+
0.016922,
|
|
4889
|
+
0.018215,
|
|
4890
|
+
-758e-6,
|
|
4891
|
+
5614e-6,
|
|
4892
|
+
0.02137,
|
|
4893
|
+
7615e-6,
|
|
4894
|
+
0.010542,
|
|
4895
|
+
635e-6,
|
|
4896
|
+
0.023702,
|
|
4897
|
+
0.020834,
|
|
4898
|
+
0.013032,
|
|
4899
|
+
0.011873,
|
|
4900
|
+
-2792e-6,
|
|
4901
|
+
0.04144,
|
|
4902
|
+
-0.02252,
|
|
4903
|
+
0.027235,
|
|
4904
|
+
-0.012274,
|
|
4905
|
+
-4002e-6,
|
|
4906
|
+
0.022356,
|
|
4907
|
+
-0.020197,
|
|
4908
|
+
0.270597,
|
|
4909
|
+
-3966e-6,
|
|
4910
|
+
-4187e-6,
|
|
4911
|
+
-0.014922,
|
|
4912
|
+
-0.013515,
|
|
4913
|
+
-0.01073,
|
|
4914
|
+
5186e-6,
|
|
4915
|
+
-0.01013,
|
|
4916
|
+
-0.024027,
|
|
4917
|
+
-897e-6,
|
|
4918
|
+
0.023069,
|
|
4919
|
+
0.021741,
|
|
4920
|
+
573e-6,
|
|
4921
|
+
-7467e-6,
|
|
4922
|
+
823e-5,
|
|
4923
|
+
0.037651,
|
|
4924
|
+
-0.017797,
|
|
4925
|
+
-0.014499,
|
|
4926
|
+
2699e-6,
|
|
4927
|
+
-0.010982,
|
|
4928
|
+
-0.027587,
|
|
4929
|
+
0.028585,
|
|
4930
|
+
-6217e-6,
|
|
4931
|
+
-9743e-6,
|
|
4932
|
+
7437e-6,
|
|
4933
|
+
-0.023614,
|
|
4934
|
+
-0.023206,
|
|
4935
|
+
-0.029976,
|
|
4936
|
+
-903e-6,
|
|
4937
|
+
-5667e-6,
|
|
4938
|
+
-0.011557,
|
|
4939
|
+
0.011335,
|
|
4940
|
+
-0.019149,
|
|
4941
|
+
0.027405,
|
|
4942
|
+
-0.032284,
|
|
4943
|
+
-0.010479,
|
|
4944
|
+
-0.013504,
|
|
4945
|
+
0.013693,
|
|
4946
|
+
-6407e-6,
|
|
4947
|
+
8791e-6,
|
|
4948
|
+
-4461e-6,
|
|
4949
|
+
-2256e-6,
|
|
4950
|
+
-0.015897,
|
|
4951
|
+
8837e-6,
|
|
4952
|
+
-1417e-6,
|
|
4953
|
+
0.02557,
|
|
4954
|
+
-0.012118,
|
|
4955
|
+
-0.021653,
|
|
4956
|
+
3051e-6,
|
|
4957
|
+
-4168e-6,
|
|
4958
|
+
-0.01862,
|
|
4959
|
+
-9461e-6,
|
|
4960
|
+
-0.01088,
|
|
4961
|
+
-982e-5,
|
|
4962
|
+
-0.011997,
|
|
4963
|
+
-0.015663,
|
|
4964
|
+
-0.021258,
|
|
4965
|
+
5679e-6,
|
|
4966
|
+
0.012566,
|
|
4967
|
+
1521e-6,
|
|
4968
|
+
-8604e-6,
|
|
4969
|
+
0.012445,
|
|
4970
|
+
8421e-6,
|
|
4971
|
+
-0.033856,
|
|
4972
|
+
-0.019957,
|
|
4973
|
+
146e-6,
|
|
4974
|
+
-185e-6,
|
|
4975
|
+
781e-6,
|
|
4976
|
+
0.018164,
|
|
4977
|
+
4102e-6,
|
|
4978
|
+
-0.022985,
|
|
4979
|
+
-8788e-6,
|
|
4980
|
+
-4097e-6,
|
|
4981
|
+
0.02217,
|
|
4982
|
+
-0.019959,
|
|
4983
|
+
-0.011669,
|
|
4984
|
+
5313e-6,
|
|
4985
|
+
381e-6,
|
|
4986
|
+
3747e-6,
|
|
4987
|
+
-0.036681,
|
|
4988
|
+
-6177e-6,
|
|
4989
|
+
-0.014834,
|
|
4990
|
+
-0.031706,
|
|
4991
|
+
0.010642,
|
|
4992
|
+
1258e-6,
|
|
4993
|
+
0.01296,
|
|
4994
|
+
0.016435,
|
|
4995
|
+
-8722e-6,
|
|
4996
|
+
-0.010652,
|
|
4997
|
+
9787e-6,
|
|
4998
|
+
-474e-5,
|
|
4999
|
+
0.011086,
|
|
5000
|
+
-0.020615,
|
|
5001
|
+
0.021049,
|
|
5002
|
+
-468e-5,
|
|
5003
|
+
8645e-6,
|
|
5004
|
+
-9937e-6,
|
|
5005
|
+
216e-5,
|
|
5006
|
+
0.010686,
|
|
5007
|
+
-6216e-6,
|
|
5008
|
+
-0.011397,
|
|
5009
|
+
-5593e-6,
|
|
5010
|
+
9687e-6,
|
|
5011
|
+
0.017522,
|
|
5012
|
+
-6329e-6,
|
|
5013
|
+
9519e-6,
|
|
5014
|
+
-0.022554,
|
|
5015
|
+
-0.02169,
|
|
5016
|
+
3307e-6,
|
|
5017
|
+
-4782e-6,
|
|
5018
|
+
1506e-6,
|
|
5019
|
+
1607e-6,
|
|
5020
|
+
-3791e-6,
|
|
5021
|
+
-3601e-6,
|
|
5022
|
+
0.030061,
|
|
5023
|
+
-468e-5,
|
|
5024
|
+
-0.186431,
|
|
5025
|
+
-1492e-6,
|
|
5026
|
+
0.02242,
|
|
5027
|
+
-8878e-6,
|
|
5028
|
+
0.01108
|
|
5029
|
+
];
|
|
5030
|
+
|
|
3501
5031
|
// src/profiles/registry.ts
|
|
3502
5032
|
var SHARED_ENCODING = {
|
|
3503
5033
|
pooling: "mean",
|
|
@@ -3558,7 +5088,94 @@ var BGE_SMALL_EN_V15_1 = {
|
|
|
3558
5088
|
passageInstruction: "",
|
|
3559
5089
|
...SHARED_ENCODING
|
|
3560
5090
|
};
|
|
5091
|
+
var GEMMA2_REPO = "onnx-community/embeddinggemma-2-ONNX";
|
|
5092
|
+
var GEMMA2_PROCESSOR_CONFIG = {
|
|
5093
|
+
path: "processor_config.json",
|
|
5094
|
+
digest: "sha256:168f6a08522f3ce5dea596d94d003af2fd691742d4f41fe1f9d8cce76bfbf69c"
|
|
5095
|
+
};
|
|
5096
|
+
var EMBEDDINGGEMMA_2_512_1 = {
|
|
5097
|
+
id: "embeddinggemma-2-512@1",
|
|
5098
|
+
model: {
|
|
5099
|
+
repo: GEMMA2_REPO,
|
|
5100
|
+
revision: "daa72c51243991dfcaf9f9137d2c573d8f7790c0",
|
|
5101
|
+
onnxFile: "onnx/model_quantized.onnx",
|
|
5102
|
+
onnxDigest: "sha256:d06edd601f851c633a2519304cbeb8dc6170d7ceb61b436625c17fb9b6e74953",
|
|
5103
|
+
files: [
|
|
5104
|
+
{
|
|
5105
|
+
path: "onnx/model_quantized.onnx_data",
|
|
5106
|
+
digest: "sha256:278a7ff1248c3618e4bd11a607fc54f7bdc7778854230f3956d3f86bd9db4f3b"
|
|
5107
|
+
},
|
|
5108
|
+
{
|
|
5109
|
+
path: "config.json",
|
|
5110
|
+
digest: "sha256:8d011bfe08b5e345bbe0b81e5c6fd02c381920b345b986047bc2a33ce7b90d1d"
|
|
5111
|
+
},
|
|
5112
|
+
{
|
|
5113
|
+
path: "tokenizer_config.json",
|
|
5114
|
+
digest: "sha256:17bd5d6e9364ca49a534e1502076593317c298d4a663623091ed45388f004874"
|
|
5115
|
+
}
|
|
5116
|
+
]
|
|
5117
|
+
},
|
|
5118
|
+
tokenizer: {
|
|
5119
|
+
kind: "gemma-bpe",
|
|
5120
|
+
file: "tokenizer.json",
|
|
5121
|
+
digest: "sha256:4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4"
|
|
5122
|
+
},
|
|
5123
|
+
pooling: "mean",
|
|
5124
|
+
normalized: true,
|
|
5125
|
+
dimensions: 512,
|
|
5126
|
+
truncation: { method: "prefix", sourceDimensions: 768 },
|
|
5127
|
+
maxTokens: 512,
|
|
5128
|
+
queryInstruction: "task: search result | query: ",
|
|
5129
|
+
passageInstruction: "title: none | text: ",
|
|
5130
|
+
vectorEncoding: "bit+int8",
|
|
5131
|
+
distance: { stage1: "hamming", stage2: "cosine" },
|
|
5132
|
+
quantization: {
|
|
5133
|
+
int8: { method: "symmetric-linear", scale: 127 },
|
|
5134
|
+
binary: {
|
|
5135
|
+
method: "centered-sign",
|
|
5136
|
+
threshold: 0,
|
|
5137
|
+
packing: "lsb-first",
|
|
5138
|
+
center: [...EMBEDDINGGEMMA_2_512_CENTER]
|
|
5139
|
+
}
|
|
5140
|
+
},
|
|
5141
|
+
media: {
|
|
5142
|
+
image: {
|
|
5143
|
+
encoder: {
|
|
5144
|
+
onnxFile: "onnx/vision_encoder_quantized.onnx",
|
|
5145
|
+
onnxDigest: "sha256:bb0de2df53a2448a32dc7908a187c168c8afd514d4d6f674f7f46024875fa4e3",
|
|
5146
|
+
files: [
|
|
5147
|
+
{
|
|
5148
|
+
path: "onnx/vision_encoder_quantized.onnx_data",
|
|
5149
|
+
digest: "sha256:3dabd69c0a36e9a8771ad82030dde74daa5a0e02b7047a5d3f3382b1137bab89"
|
|
5150
|
+
},
|
|
5151
|
+
GEMMA2_PROCESSOR_CONFIG
|
|
5152
|
+
]
|
|
5153
|
+
},
|
|
5154
|
+
tokenBudget: 280,
|
|
5155
|
+
resample: "bicubic",
|
|
5156
|
+
alpha: "composite-white"
|
|
5157
|
+
},
|
|
5158
|
+
video: { framesPerSecond: 1, maxFrames: 32, tokenBudgetPerFrame: 140 },
|
|
5159
|
+
audio: {
|
|
5160
|
+
encoder: {
|
|
5161
|
+
onnxFile: "onnx/audio_encoder_quantized.onnx",
|
|
5162
|
+
onnxDigest: "sha256:04a9a9094ba76fb169e4c69be45a4b621580188654d6be59eab860eadcd42d3c",
|
|
5163
|
+
files: [
|
|
5164
|
+
{
|
|
5165
|
+
path: "onnx/audio_encoder_quantized.onnx_data",
|
|
5166
|
+
digest: "sha256:aa6361d898e1f1d6303f8dd2b5fabf4fd3629e15fd709e9cb06ac5cb9416a030"
|
|
5167
|
+
},
|
|
5168
|
+
GEMMA2_PROCESSOR_CONFIG
|
|
5169
|
+
]
|
|
5170
|
+
},
|
|
5171
|
+
sampleRate: 16e3,
|
|
5172
|
+
channels: 1,
|
|
5173
|
+
maxWindowMs: 3e4
|
|
5174
|
+
}
|
|
5175
|
+
}
|
|
5176
|
+
};
|
|
3561
5177
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
5178
|
+
EMBEDDINGGEMMA_2_512_1,
|
|
3562
5179
|
MULTILINGUAL_E5_SMALL_2,
|
|
3563
5180
|
MULTILINGUAL_E5_SMALL_1,
|
|
3564
5181
|
BGE_SMALL_EN_V15_1
|
|
@@ -3579,10 +5196,13 @@ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
|
3579
5196
|
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
3580
5197
|
|
|
3581
5198
|
// src/embedding/profile-embedder.ts
|
|
3582
|
-
import {
|
|
5199
|
+
import {
|
|
5200
|
+
embeddingProfileArtifacts as embeddingProfileArtifacts3,
|
|
5201
|
+
profileUnitVector as profileUnitVector2
|
|
5202
|
+
} from "@bendyline/gezk";
|
|
3583
5203
|
|
|
3584
5204
|
// src/embedding/artifact-verify.ts
|
|
3585
|
-
import { createHash as
|
|
5205
|
+
import { createHash as createHash6 } from "crypto";
|
|
3586
5206
|
import { createReadStream } from "fs";
|
|
3587
5207
|
import { stat as stat3 } from "fs/promises";
|
|
3588
5208
|
import { join as join5 } from "path";
|
|
@@ -3626,7 +5246,7 @@ async function fileDigest(path) {
|
|
|
3626
5246
|
const memo = digestMemo.get(key);
|
|
3627
5247
|
if (memo) return memo;
|
|
3628
5248
|
const digest = await new Promise((resolve5, reject) => {
|
|
3629
|
-
const hash =
|
|
5249
|
+
const hash = createHash6("sha256");
|
|
3630
5250
|
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
3631
5251
|
});
|
|
3632
5252
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
@@ -3636,6 +5256,45 @@ async function fileDigest(path) {
|
|
|
3636
5256
|
digestMemo.set(key, digest);
|
|
3637
5257
|
return digest;
|
|
3638
5258
|
}
|
|
5259
|
+
function mediaEncoderPins(profile, modalities) {
|
|
5260
|
+
const encoders = /* @__PURE__ */ new Set();
|
|
5261
|
+
const media = profile.media;
|
|
5262
|
+
for (const modality of modalities) {
|
|
5263
|
+
const encoder = modality === "audio" ? media?.audio?.encoder : media?.image?.encoder;
|
|
5264
|
+
if (encoder) encoders.add(encoder);
|
|
5265
|
+
}
|
|
5266
|
+
const pins = [];
|
|
5267
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5268
|
+
for (const encoder of encoders) {
|
|
5269
|
+
const files = [
|
|
5270
|
+
{ path: encoder.onnxFile, digest: encoder.onnxDigest },
|
|
5271
|
+
...encoder.files ?? []
|
|
5272
|
+
];
|
|
5273
|
+
for (const { path, digest } of files) {
|
|
5274
|
+
if (!digest || seen.has(path)) continue;
|
|
5275
|
+
seen.add(path);
|
|
5276
|
+
pins.push({ file: path, expected: digest });
|
|
5277
|
+
}
|
|
5278
|
+
}
|
|
5279
|
+
return pins;
|
|
5280
|
+
}
|
|
5281
|
+
async function verifyMediaArtifacts(profile, modalities, opts) {
|
|
5282
|
+
const pins = mediaEncoderPins(profile, modalities);
|
|
5283
|
+
if (pins.length === 0) return { status: "unpinned", checks: [] };
|
|
5284
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5285
|
+
const checks = [];
|
|
5286
|
+
for (const { file, expected } of pins) {
|
|
5287
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
|
|
5288
|
+
checks.push({ role: "media", file, path, expected, actual: await fileDigest(path) });
|
|
5289
|
+
}
|
|
5290
|
+
if (checks.some((c) => c.actual === null)) {
|
|
5291
|
+
throw new EmbedderArtifactError("missing", profile.id, checks);
|
|
5292
|
+
}
|
|
5293
|
+
if (checks.some((c) => c.actual !== c.expected)) {
|
|
5294
|
+
throw new EmbedderArtifactError("mismatch", profile.id, checks);
|
|
5295
|
+
}
|
|
5296
|
+
return { status: "verified", checks };
|
|
5297
|
+
}
|
|
3639
5298
|
async function verifyProfileArtifacts(profile, opts) {
|
|
3640
5299
|
const artifacts = embeddingProfileArtifacts(profile);
|
|
3641
5300
|
const revision = opts.revision ?? profile.model.revision;
|
|
@@ -3650,6 +5309,9 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
3650
5309
|
expected: artifacts.tokenizerDigest
|
|
3651
5310
|
});
|
|
3652
5311
|
}
|
|
5312
|
+
for (const extra of artifacts.files) {
|
|
5313
|
+
if (extra.digest) pinned.push({ role: "file", file: extra.path, expected: extra.digest });
|
|
5314
|
+
}
|
|
3653
5315
|
if (pinned.length === 0) return { status: "unpinned", checks: [] };
|
|
3654
5316
|
const checks = [];
|
|
3655
5317
|
for (const { role, file, expected } of pinned) {
|
|
@@ -3665,10 +5327,89 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
3665
5327
|
return { status: "verified", checks };
|
|
3666
5328
|
}
|
|
3667
5329
|
|
|
5330
|
+
// src/embedding/pinned-files.ts
|
|
5331
|
+
import { createHash as createHash7 } from "crypto";
|
|
5332
|
+
import { createWriteStream as createWriteStream4 } from "fs";
|
|
5333
|
+
import { mkdir as mkdir4, rename, rm as rm2, stat as stat4 } from "fs/promises";
|
|
5334
|
+
import { dirname as dirname3 } from "path";
|
|
5335
|
+
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
5336
|
+
function isExternalDataFile(path) {
|
|
5337
|
+
return /\.onnx_data(_\d+)?$/.test(path);
|
|
5338
|
+
}
|
|
5339
|
+
function pinnedLoadFiles(profile, modalities = []) {
|
|
5340
|
+
const artifacts = embeddingProfileArtifacts2(profile);
|
|
5341
|
+
const files = [
|
|
5342
|
+
{
|
|
5343
|
+
path: artifacts.onnxFile,
|
|
5344
|
+
...artifacts.onnxDigest ? { digest: artifacts.onnxDigest } : {}
|
|
5345
|
+
},
|
|
5346
|
+
{
|
|
5347
|
+
path: artifacts.tokenizerFile,
|
|
5348
|
+
...artifacts.tokenizerDigest ? { digest: artifacts.tokenizerDigest } : {}
|
|
5349
|
+
},
|
|
5350
|
+
...artifacts.files.map((f) => ({ path: f.path, ...f.digest ? { digest: f.digest } : {} })),
|
|
5351
|
+
...mediaEncoderPins(profile, modalities).map((p) => ({ path: p.file, digest: p.expected }))
|
|
5352
|
+
];
|
|
5353
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5354
|
+
return files.filter((f) => !seen.has(f.path) && seen.add(f.path));
|
|
5355
|
+
}
|
|
5356
|
+
async function isFile(path) {
|
|
5357
|
+
return stat4(path).then((s) => s.isFile()).catch(() => false);
|
|
5358
|
+
}
|
|
5359
|
+
async function missingPinnedFiles(profile, opts) {
|
|
5360
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5361
|
+
const missing = [];
|
|
5362
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5363
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5364
|
+
if (!await isFile(path)) missing.push(file.path);
|
|
5365
|
+
}
|
|
5366
|
+
return missing;
|
|
5367
|
+
}
|
|
5368
|
+
async function prefetchExternalData(profile, opts) {
|
|
5369
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5370
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
5371
|
+
const fetched = [];
|
|
5372
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5373
|
+
if (!isExternalDataFile(file.path)) continue;
|
|
5374
|
+
const dest = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5375
|
+
if (await isFile(dest)) continue;
|
|
5376
|
+
const url = `https://huggingface.co/${profile.model.repo}/resolve/${revision}/${file.path}`;
|
|
5377
|
+
const res = await fetchImpl(url);
|
|
5378
|
+
if (!res.ok || !res.body) throw new Error(`${file.path}: download failed (HTTP ${res.status})`);
|
|
5379
|
+
await mkdir4(dirname3(dest), { recursive: true });
|
|
5380
|
+
const partial = `${dest}.partial`;
|
|
5381
|
+
const hash = createHash7("sha256");
|
|
5382
|
+
try {
|
|
5383
|
+
const out = createWriteStream4(partial);
|
|
5384
|
+
const reader = res.body.getReader();
|
|
5385
|
+
for (; ; ) {
|
|
5386
|
+
const { done, value } = await reader.read();
|
|
5387
|
+
if (done) break;
|
|
5388
|
+
hash.update(value);
|
|
5389
|
+
if (!out.write(value)) await new Promise((r) => out.once("drain", () => r()));
|
|
5390
|
+
}
|
|
5391
|
+
await new Promise((resolve5, reject) => {
|
|
5392
|
+
out.on("error", reject);
|
|
5393
|
+
out.end(() => resolve5());
|
|
5394
|
+
});
|
|
5395
|
+
const digest = `sha256:${hash.digest("hex")}`;
|
|
5396
|
+
if (file.digest && digest !== file.digest) {
|
|
5397
|
+
throw new Error(`${file.path}: downloaded ${digest}, profile pins ${file.digest}`);
|
|
5398
|
+
}
|
|
5399
|
+
await rename(partial, dest);
|
|
5400
|
+
} catch (err) {
|
|
5401
|
+
await rm2(partial, { force: true });
|
|
5402
|
+
throw err;
|
|
5403
|
+
}
|
|
5404
|
+
fetched.push(file.path);
|
|
5405
|
+
}
|
|
5406
|
+
return fetched;
|
|
5407
|
+
}
|
|
5408
|
+
|
|
3668
5409
|
// src/embedding/profile-embedder.ts
|
|
3669
5410
|
var MAX_BATCH = 8;
|
|
3670
5411
|
var MAX_CHARS = 8e3;
|
|
3671
|
-
var TRANSFORMERS_PEER_RANGE = "^3.
|
|
5412
|
+
var TRANSFORMERS_PEER_RANGE = "^4.3.1";
|
|
3672
5413
|
var EmbedderUnavailableError = class extends Error {
|
|
3673
5414
|
isActionable = true;
|
|
3674
5415
|
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
@@ -3712,7 +5453,7 @@ var POOLING_BY_PROFILE = {
|
|
|
3712
5453
|
cls: "cls"
|
|
3713
5454
|
};
|
|
3714
5455
|
function resolveTransformersModelOptions(profile) {
|
|
3715
|
-
const { onnxFile, tokenizerFile } =
|
|
5456
|
+
const { onnxFile, tokenizerFile } = embeddingProfileArtifacts3(profile);
|
|
3716
5457
|
if (tokenizerFile !== "tokenizer.json") {
|
|
3717
5458
|
throw new EmbedderUnavailableError(
|
|
3718
5459
|
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
@@ -3737,6 +5478,41 @@ function transformersGraphOptions(onnxFile) {
|
|
|
3737
5478
|
if (modelFileName === "") return null;
|
|
3738
5479
|
return { dtype, subfolder, model_file_name: modelFileName };
|
|
3739
5480
|
}
|
|
5481
|
+
function isMultimodalEncoderConfig(config) {
|
|
5482
|
+
return Boolean(config.vision_config || config.audio_config);
|
|
5483
|
+
}
|
|
5484
|
+
function pinnedExternalDataChunks(profile) {
|
|
5485
|
+
const { onnxFile, files } = embeddingProfileArtifacts3(profile);
|
|
5486
|
+
const prefix = `${onnxFile}_data`;
|
|
5487
|
+
return files.filter((f) => f.path === prefix || /^_\d+$/.test(f.path.slice(prefix.length))).length;
|
|
5488
|
+
}
|
|
5489
|
+
function encoderRows(outputs, attentionMask) {
|
|
5490
|
+
const pooled = outputs.sentence_embedding;
|
|
5491
|
+
if (pooled) {
|
|
5492
|
+
const [batch2 = 0, width2 = 0] = pooled.dims;
|
|
5493
|
+
return Array.from(
|
|
5494
|
+
{ length: batch2 },
|
|
5495
|
+
(_, b) => Array.from({ length: width2 }, (_2, d) => pooled.data[b * width2 + d])
|
|
5496
|
+
);
|
|
5497
|
+
}
|
|
5498
|
+
const hidden = outputs.last_hidden_state;
|
|
5499
|
+
if (!hidden || !attentionMask) {
|
|
5500
|
+
throw new Error("the encoder returned neither sentence_embedding nor last_hidden_state");
|
|
5501
|
+
}
|
|
5502
|
+
const [batch = 0, tokens = 0, width = 0] = hidden.dims;
|
|
5503
|
+
return Array.from({ length: batch }, (_, b) => {
|
|
5504
|
+
const sum = new Array(width).fill(0);
|
|
5505
|
+
let count = 0;
|
|
5506
|
+
for (let t = 0; t < tokens; t++) {
|
|
5507
|
+
if (!Number(attentionMask.data[b * tokens + t])) continue;
|
|
5508
|
+
count++;
|
|
5509
|
+
const base = (b * tokens + t) * width;
|
|
5510
|
+
for (let d = 0; d < width; d++)
|
|
5511
|
+
sum[d] = sum[d] + hidden.data[base + d];
|
|
5512
|
+
}
|
|
5513
|
+
return sum.map((v) => v / Math.max(1, count));
|
|
5514
|
+
});
|
|
5515
|
+
}
|
|
3740
5516
|
async function loadTransformers() {
|
|
3741
5517
|
try {
|
|
3742
5518
|
const specifier = "@huggingface/transformers";
|
|
@@ -3752,6 +5528,20 @@ async function loadTransformers() {
|
|
|
3752
5528
|
throw err;
|
|
3753
5529
|
}
|
|
3754
5530
|
}
|
|
5531
|
+
async function preparePinnedLoad(profile, opts) {
|
|
5532
|
+
const files = pinnedLoadFiles(profile, opts.modalities);
|
|
5533
|
+
if (!files.some((f) => isExternalDataFile(f.path))) return;
|
|
5534
|
+
if (opts.localFilesOnly) {
|
|
5535
|
+
const missing = await missingPinnedFiles(profile, opts);
|
|
5536
|
+
if (missing.length > 0) {
|
|
5537
|
+
throw new EmbedderUnavailableError(
|
|
5538
|
+
`the model files for profile ${profile.id} are not all installed (missing: ${missing.join(", ")})`
|
|
5539
|
+
);
|
|
5540
|
+
}
|
|
5541
|
+
return;
|
|
5542
|
+
}
|
|
5543
|
+
await prefetchExternalData(profile, opts);
|
|
5544
|
+
}
|
|
3755
5545
|
async function createProfileEmbedder(profile, opts = {}) {
|
|
3756
5546
|
const pooling = POOLING_BY_PROFILE[profile.pooling];
|
|
3757
5547
|
if (!pooling) {
|
|
@@ -3768,14 +5558,47 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3768
5558
|
transformers.env.allowRemoteModels = true;
|
|
3769
5559
|
}
|
|
3770
5560
|
const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
|
|
3771
|
-
const
|
|
3772
|
-
|
|
5561
|
+
const localOnly = opts.localFilesOnly ? { local_files_only: true } : {};
|
|
5562
|
+
if (!opts.transformers) {
|
|
5563
|
+
await preparePinnedLoad(profile, {
|
|
5564
|
+
cacheDir: cacheDir ?? transformers.env.cacheDir ?? "",
|
|
5565
|
+
...opts.localFilesOnly ? { localFilesOnly: true } : {}
|
|
5566
|
+
});
|
|
5567
|
+
}
|
|
5568
|
+
const sessionOptions = opts.sessionOptions ? { session_options: opts.sessionOptions } : {};
|
|
5569
|
+
const config = transformers.AutoConfig ? await transformers.AutoConfig.from_pretrained(profile.model.repo, {
|
|
5570
|
+
revision: modelOptions.revision,
|
|
5571
|
+
...localOnly,
|
|
5572
|
+
...progress
|
|
5573
|
+
}) : null;
|
|
5574
|
+
const multimodal = config !== null && isMultimodalEncoderConfig(config);
|
|
5575
|
+
if (multimodal && !transformers.AutoModel) {
|
|
5576
|
+
throw new EmbedderUnavailableError(
|
|
5577
|
+
`profile ${profile.id} is a multimodal encoder; this runtime cannot load its text session`
|
|
5578
|
+
);
|
|
5579
|
+
}
|
|
5580
|
+
const externalData = pinnedExternalDataChunks(profile);
|
|
5581
|
+
const [runner, tokenizer] = await Promise.all([
|
|
5582
|
+
multimodal ? transformers.AutoModel.from_pretrained(
|
|
5583
|
+
profile.model.repo,
|
|
5584
|
+
{
|
|
5585
|
+
...modelOptions,
|
|
5586
|
+
config: { ...config, vision_config: null, audio_config: null },
|
|
5587
|
+
...externalData > 0 ? { use_external_data_format: externalData } : {},
|
|
5588
|
+
...localOnly,
|
|
5589
|
+
...sessionOptions,
|
|
5590
|
+
...progress
|
|
5591
|
+
}
|
|
5592
|
+
) : transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3773
5593
|
...modelOptions,
|
|
3774
|
-
...
|
|
5594
|
+
...externalData > 0 ? { use_external_data_format: externalData } : {},
|
|
5595
|
+
...localOnly,
|
|
5596
|
+
...sessionOptions,
|
|
3775
5597
|
...progress
|
|
3776
5598
|
}),
|
|
3777
5599
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3778
5600
|
revision: modelOptions.revision,
|
|
5601
|
+
...localOnly,
|
|
3779
5602
|
...progress
|
|
3780
5603
|
})
|
|
3781
5604
|
]);
|
|
@@ -3785,17 +5608,23 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3785
5608
|
cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
|
|
3786
5609
|
});
|
|
3787
5610
|
} catch (err) {
|
|
3788
|
-
await
|
|
5611
|
+
await runner.dispose?.().catch(() => {
|
|
3789
5612
|
});
|
|
3790
5613
|
throw err;
|
|
3791
5614
|
}
|
|
3792
5615
|
const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
|
|
5616
|
+
const runBatch = multimodal ? async (slice) => {
|
|
5617
|
+
const inputs = tokenizer(slice, {
|
|
5618
|
+
padding: true,
|
|
5619
|
+
truncation: true,
|
|
5620
|
+
max_length: profile.maxTokens
|
|
5621
|
+
});
|
|
5622
|
+
return encoderRows(await runner(inputs), inputs.attention_mask);
|
|
5623
|
+
} : async (slice) => (await runner(slice, { pooling, normalize: true })).tolist();
|
|
3793
5624
|
const embed = async (texts) => {
|
|
3794
5625
|
const out = [];
|
|
3795
5626
|
for (let i = 0; i < texts.length; i += MAX_BATCH) {
|
|
3796
|
-
|
|
3797
|
-
const result = await pipe(slice, { pooling, normalize: true });
|
|
3798
|
-
out.push(...result.tolist());
|
|
5627
|
+
out.push(...await runBatch(texts.slice(i, i + MAX_BATCH).map(cap)));
|
|
3799
5628
|
}
|
|
3800
5629
|
return out;
|
|
3801
5630
|
};
|
|
@@ -3805,11 +5634,12 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3805
5634
|
embed,
|
|
3806
5635
|
embedQuery: async (text) => {
|
|
3807
5636
|
const [vector] = await embed([`${profile.queryInstruction}${text}`]);
|
|
3808
|
-
|
|
5637
|
+
if (!vector) throw new Error(`profile ${profile.id} returned no query vector`);
|
|
5638
|
+
return profileUnitVector2(profile, vector);
|
|
3809
5639
|
},
|
|
3810
5640
|
countTokens: (text) => tokenizer.encode(text).length,
|
|
3811
5641
|
dispose: async () => {
|
|
3812
|
-
await
|
|
5642
|
+
await runner.dispose?.();
|
|
3813
5643
|
}
|
|
3814
5644
|
};
|
|
3815
5645
|
}
|
|
@@ -3817,11 +5647,12 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
3817
5647
|
// src/markdown-adapter/load.ts
|
|
3818
5648
|
import { existsSync } from "fs";
|
|
3819
5649
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
3820
|
-
import { basename, dirname as
|
|
5650
|
+
import { basename, dirname as dirname4, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3821
5651
|
import {
|
|
3822
5652
|
CatalogDocumentSchema,
|
|
3823
5653
|
KnowledgeDocumentIdSchema,
|
|
3824
|
-
|
|
5654
|
+
KnowledgeLocationsSchema as KnowledgeLocationsSchema3,
|
|
5655
|
+
KnowledgeOrdinalSchema as KnowledgeOrdinalSchema2,
|
|
3825
5656
|
MAX_KNOWLEDGE_TOPIC_DEPTH as MAX_KNOWLEDGE_TOPIC_DEPTH2,
|
|
3826
5657
|
assetExtension as assetExtension3,
|
|
3827
5658
|
formatKnowledgeUri,
|
|
@@ -4494,9 +6325,9 @@ function parseYaml(text, opts) {
|
|
|
4494
6325
|
var OPEN = /^---[ \t]*\r?\n/;
|
|
4495
6326
|
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
4496
6327
|
function parseMarkdownFrontMatter(source) {
|
|
4497
|
-
const
|
|
4498
|
-
if (!
|
|
4499
|
-
const lines = source.slice(
|
|
6328
|
+
const open2 = OPEN.exec(source);
|
|
6329
|
+
if (!open2) return { data: {}, body: source };
|
|
6330
|
+
const lines = source.slice(open2[0].length).split(/\r?\n/);
|
|
4500
6331
|
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
4501
6332
|
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
4502
6333
|
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
@@ -4517,7 +6348,15 @@ var HUGO_SECTION_PAGE = "_index";
|
|
|
4517
6348
|
var DOCFX_JSON = "docfx.json";
|
|
4518
6349
|
var WARNING_EXAMPLES2 = 20;
|
|
4519
6350
|
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
4520
|
-
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
6351
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set([
|
|
6352
|
+
"title",
|
|
6353
|
+
"summary",
|
|
6354
|
+
"aliases",
|
|
6355
|
+
"id",
|
|
6356
|
+
"order",
|
|
6357
|
+
"subcategory",
|
|
6358
|
+
"locations"
|
|
6359
|
+
]);
|
|
4521
6360
|
function readFrontMatter(raw, file, conventions) {
|
|
4522
6361
|
const { hugo } = conventions;
|
|
4523
6362
|
let data;
|
|
@@ -4536,6 +6375,12 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4536
6375
|
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
4537
6376
|
return value.trim() || void 0;
|
|
4538
6377
|
};
|
|
6378
|
+
if (data.locations !== void 0) {
|
|
6379
|
+
const locations = KnowledgeLocationsSchema3.safeParse(data.locations);
|
|
6380
|
+
if (!locations.success)
|
|
6381
|
+
throw new Error(`${file}: invalid front matter locations: ${locations.error.message}`);
|
|
6382
|
+
out.locations = locations.data;
|
|
6383
|
+
}
|
|
4539
6384
|
out.title = str("title");
|
|
4540
6385
|
out.summary = str("summary");
|
|
4541
6386
|
out.id = str("id");
|
|
@@ -4548,7 +6393,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4548
6393
|
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
4549
6394
|
}
|
|
4550
6395
|
if (data.order !== void 0 && data.order !== null) {
|
|
4551
|
-
const order =
|
|
6396
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4552
6397
|
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
4553
6398
|
out.order = order.data;
|
|
4554
6399
|
}
|
|
@@ -4569,7 +6414,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4569
6414
|
subcategory.title = record.title.trim();
|
|
4570
6415
|
}
|
|
4571
6416
|
if (record.order !== void 0 && record.order !== null) {
|
|
4572
|
-
const order =
|
|
6417
|
+
const order = KnowledgeOrdinalSchema2.safeParse(record.order);
|
|
4573
6418
|
if (!order.success)
|
|
4574
6419
|
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
4575
6420
|
subcategory.order = order.data;
|
|
@@ -4580,7 +6425,7 @@ function readFrontMatter(raw, file, conventions) {
|
|
|
4580
6425
|
if (data.draft === true) out.unpublished = "draft";
|
|
4581
6426
|
else if (data.headless === true) out.unpublished = "headless";
|
|
4582
6427
|
if (data.weight !== void 0 && data.weight !== null) {
|
|
4583
|
-
const weight =
|
|
6428
|
+
const weight = KnowledgeOrdinalSchema2.safeParse(data.weight);
|
|
4584
6429
|
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
4585
6430
|
if (weight.data > 0) out.weight = weight.data;
|
|
4586
6431
|
}
|
|
@@ -4653,7 +6498,7 @@ async function readTopicSidecar(dir) {
|
|
|
4653
6498
|
out.description = data.description.trim();
|
|
4654
6499
|
}
|
|
4655
6500
|
if (data.order !== void 0 && data.order !== null) {
|
|
4656
|
-
const order =
|
|
6501
|
+
const order = KnowledgeOrdinalSchema2.safeParse(data.order);
|
|
4657
6502
|
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
4658
6503
|
out.order = order.data;
|
|
4659
6504
|
}
|
|
@@ -4681,7 +6526,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
|
|
|
4681
6526
|
maxAliasCount: 100
|
|
4682
6527
|
});
|
|
4683
6528
|
const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
4684
|
-
return resolve4(
|
|
6529
|
+
return resolve4(dirname4(mkdocsPath), docsDir);
|
|
4685
6530
|
}
|
|
4686
6531
|
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
4687
6532
|
const summary = join6(rootDir, "SUMMARY.md");
|
|
@@ -4689,7 +6534,7 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
|
4689
6534
|
const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
4690
6535
|
if (jupyter) return { format: "jupyter-book", path: jupyter };
|
|
4691
6536
|
const mkdocs = firstExisting(
|
|
4692
|
-
[rootDir, projectDir,
|
|
6537
|
+
[rootDir, projectDir, dirname4(rootDir)].flatMap((dir) => [
|
|
4693
6538
|
join6(dir, "mkdocs.yml"),
|
|
4694
6539
|
join6(dir, "mkdocs.yaml")
|
|
4695
6540
|
])
|
|
@@ -4729,7 +6574,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4729
6574
|
}
|
|
4730
6575
|
if (toc.format === "mkdocs") {
|
|
4731
6576
|
const path2 = explicit ?? firstExisting(
|
|
4732
|
-
[rootDir,
|
|
6577
|
+
[rootDir, dirname4(rootDir)].flatMap((dir) => [
|
|
4733
6578
|
join6(dir, "mkdocs.yml"),
|
|
4734
6579
|
join6(dir, "mkdocs.yaml")
|
|
4735
6580
|
])
|
|
@@ -4752,7 +6597,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
4752
6597
|
}
|
|
4753
6598
|
const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
4754
6599
|
if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
|
|
4755
|
-
const tocDir = insideRoot(rootDir,
|
|
6600
|
+
const tocDir = insideRoot(rootDir, dirname4(path));
|
|
4756
6601
|
if (tocDir === null)
|
|
4757
6602
|
throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
|
|
4758
6603
|
const doc = parseYaml(await readOutlineText(path), {
|
|
@@ -4885,6 +6730,7 @@ function rewriteLinks(ctx, markdown) {
|
|
|
4885
6730
|
return line.replace(
|
|
4886
6731
|
INLINE_LINK,
|
|
4887
6732
|
(whole, bang, text, target, title) => {
|
|
6733
|
+
if (bang === "!" && ctx.skipImages) return text;
|
|
4888
6734
|
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
4889
6735
|
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
4890
6736
|
}
|
|
@@ -4955,22 +6801,16 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
4955
6801
|
};
|
|
4956
6802
|
const placements = /* @__PURE__ */ new Map();
|
|
4957
6803
|
if (outline) {
|
|
4958
|
-
const duplicates = [];
|
|
4959
6804
|
const walk = (topic, chain) => {
|
|
4960
6805
|
for (const entry of topic.entries) {
|
|
4961
6806
|
const key = entry.file.normalize("NFC");
|
|
4962
|
-
|
|
4963
|
-
|
|
6807
|
+
const existing = placements.get(key) ?? [];
|
|
6808
|
+
existing.push({ chain, entry });
|
|
6809
|
+
placements.set(key, existing);
|
|
4964
6810
|
}
|
|
4965
6811
|
for (const child of topic.children) walk(child, [...chain, child]);
|
|
4966
6812
|
};
|
|
4967
6813
|
walk(outline.root, []);
|
|
4968
|
-
warnCapped(
|
|
4969
|
-
warn,
|
|
4970
|
-
duplicates,
|
|
4971
|
-
(file) => `${file}: listed more than once in the table of contents; the first place wins`,
|
|
4972
|
-
(more) => `${more} more files are listed more than once in the table of contents`
|
|
4973
|
-
);
|
|
4974
6814
|
}
|
|
4975
6815
|
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
4976
6816
|
let outlineTooDeep = false;
|
|
@@ -5045,7 +6885,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5045
6885
|
let title = docfx ? firstHeading(fm.body) ?? fm.title ?? stem2 : fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
5046
6886
|
let ordinal = fm.order;
|
|
5047
6887
|
let topicPath;
|
|
5048
|
-
const
|
|
6888
|
+
const documentPlacements = placements.get(rel.normalize("NFC")) ?? [];
|
|
6889
|
+
const placement = documentPlacements[0];
|
|
5049
6890
|
if (placement) {
|
|
5050
6891
|
topicPath = ensureOutlineChain(placement.chain);
|
|
5051
6892
|
if (placement.entry.title) title = placement.entry.title;
|
|
@@ -5075,6 +6916,17 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5075
6916
|
const parentId = topicPath[topicPath.length - 1];
|
|
5076
6917
|
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
5077
6918
|
}
|
|
6919
|
+
const tocReferences = [];
|
|
6920
|
+
const placedLeaves = /* @__PURE__ */ new Set([topicPath.at(-1)]);
|
|
6921
|
+
for (const extra of documentPlacements.slice(1)) {
|
|
6922
|
+
let path = ensureOutlineChain(extra.chain);
|
|
6923
|
+
if (fm.subcategory) path = [...path, ensureShelf(path.at(-1), fm.subcategory, rel)];
|
|
6924
|
+
const leaf = path.at(-1);
|
|
6925
|
+
if (placedLeaves.has(leaf)) continue;
|
|
6926
|
+
placedLeaves.add(leaf);
|
|
6927
|
+
const order = extra.entry.order ?? fm.order;
|
|
6928
|
+
tocReferences.push({ topicPath: path, ...order !== void 0 ? { ordinal: order } : {} });
|
|
6929
|
+
}
|
|
5078
6930
|
const summary = fm.summary ?? fm.description ?? firstParagraph(fm.body);
|
|
5079
6931
|
loaded.push({
|
|
5080
6932
|
rel,
|
|
@@ -5086,6 +6938,8 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5086
6938
|
...summary ? { summary } : {},
|
|
5087
6939
|
language: opts.language,
|
|
5088
6940
|
topicPath,
|
|
6941
|
+
...tocReferences.length ? { tocReferences } : {},
|
|
6942
|
+
...fm.locations ? { locations: fm.locations } : {},
|
|
5089
6943
|
markdown: fm.body,
|
|
5090
6944
|
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
5091
6945
|
...ordinal !== void 0 ? { ordinal } : {},
|
|
@@ -5115,7 +6969,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5115
6969
|
);
|
|
5116
6970
|
warnCapped(
|
|
5117
6971
|
warn,
|
|
5118
|
-
[...placements].filter(([file]) => !seenRel.has(file)).map(([
|
|
6972
|
+
[...placements].filter(([file]) => !seenRel.has(file)).map(([file]) => file),
|
|
5119
6973
|
(file) => `${file}: named by the table of contents but not found among the Markdown files`,
|
|
5120
6974
|
(more) => `${more} more files named by the table of contents are not among the Markdown files`
|
|
5121
6975
|
);
|
|
@@ -5152,6 +7006,7 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
5152
7006
|
// A docfx docset draws on dependent repositories (`~/reusable-content`)
|
|
5153
7007
|
// that a checkout never holds, so a missing image is expected there.
|
|
5154
7008
|
missingAssets: opts.missingAssets ?? (docfx ? "warn" : "error"),
|
|
7009
|
+
skipImages: opts.skipImages ?? false,
|
|
5155
7010
|
tildeIsRoot: docfx,
|
|
5156
7011
|
warn: (message) => linkWarnings.push(message)
|
|
5157
7012
|
};
|
|
@@ -5288,7 +7143,7 @@ function newerRegistryEntries(registry, installed) {
|
|
|
5288
7143
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
5289
7144
|
|
|
5290
7145
|
// src/signatures/jcs.ts
|
|
5291
|
-
import { canonicalizeJson as
|
|
7146
|
+
import { canonicalizeJson as canonicalizeJson3 } from "@bendyline/gezk";
|
|
5292
7147
|
export {
|
|
5293
7148
|
BGE_SMALL_EN_V15_1,
|
|
5294
7149
|
BODY_CODEC_MIN_BYTES,
|
|
@@ -5299,7 +7154,9 @@ export {
|
|
|
5299
7154
|
CENTROID_SAMPLE_MAX,
|
|
5300
7155
|
CatalogHandle,
|
|
5301
7156
|
CatalogOpenError,
|
|
7157
|
+
CatalogQueryError,
|
|
5302
7158
|
DuckdbError,
|
|
7159
|
+
EMBEDDINGGEMMA_2_512_1,
|
|
5303
7160
|
EmbedderArtifactError,
|
|
5304
7161
|
EmbedderUnavailableError,
|
|
5305
7162
|
GEZEL_BGE_SMALL_EN_V15_1,
|
|
@@ -5323,11 +7180,14 @@ export {
|
|
|
5323
7180
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
5324
7181
|
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
5325
7182
|
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
7183
|
+
MAX_MEDIA_ROWS_PER_DOCUMENT,
|
|
7184
|
+
MEDIA_ROW_TEXT_MAX_TOKENS,
|
|
5326
7185
|
MIMETYPE_PATH,
|
|
5327
7186
|
MULTILINGUAL_E5_SMALL_1,
|
|
5328
7187
|
MULTILINGUAL_E5_SMALL_2,
|
|
5329
7188
|
PARQUET_EXPORT_VERSION,
|
|
5330
7189
|
PARQUET_REPORT_PATH,
|
|
7190
|
+
QUERY_STOP_WORDS,
|
|
5331
7191
|
README_PATH,
|
|
5332
7192
|
RERANK_CANDIDATES_MAX,
|
|
5333
7193
|
RERANK_CANDIDATES_MIN,
|
|
@@ -5337,20 +7197,25 @@ export {
|
|
|
5337
7197
|
ROUTE_SHARDS_EXPLICIT,
|
|
5338
7198
|
ROUTE_SHARDS_PROACTIVE,
|
|
5339
7199
|
SHARD_DDL,
|
|
7200
|
+
SHARD_DDL_V4,
|
|
7201
|
+
SHARD_DDL_V5,
|
|
5340
7202
|
SHARD_MAX_CHUNKS,
|
|
5341
7203
|
SHARD_TARGET_CHUNKS,
|
|
5342
7204
|
SOURCE_NOTICES_PATH,
|
|
7205
|
+
TITLE_ROUTE_SHARDS,
|
|
5343
7206
|
TRANSFORMERS_PEER_RANGE,
|
|
5344
7207
|
ZIP_FIXED_MTIME,
|
|
5345
7208
|
assertDuckdbCli,
|
|
5346
7209
|
assetReferences,
|
|
5347
7210
|
assignOutlineOrders,
|
|
5348
7211
|
asymmetricTopK,
|
|
5349
|
-
|
|
7212
|
+
canonicalizeJson3 as canonicalizeJson,
|
|
7213
|
+
catalogFormatVersion,
|
|
5350
7214
|
centerVector,
|
|
5351
7215
|
chunkContentHash,
|
|
5352
7216
|
chunkMarkdownProfile,
|
|
5353
7217
|
chunkUid,
|
|
7218
|
+
collectMediaRows,
|
|
5354
7219
|
compareCatalogVersions,
|
|
5355
7220
|
compileKnowledgeCatalog,
|
|
5356
7221
|
createProfileEmbedder,
|
|
@@ -5362,9 +7227,13 @@ export {
|
|
|
5362
7227
|
fetchKnowledgeRegistry,
|
|
5363
7228
|
findDuckdbBinary,
|
|
5364
7229
|
findRegistryEntry,
|
|
7230
|
+
formatAtLeast,
|
|
5365
7231
|
generateKnowledgeSigningKeyPair,
|
|
5366
7232
|
hammingTopK,
|
|
7233
|
+
humanizeAssetName,
|
|
5367
7234
|
inspectGezkArchive,
|
|
7235
|
+
isExternalDataFile,
|
|
7236
|
+
isMultimodalEncoderConfig,
|
|
5368
7237
|
isSupportedFormatVersion,
|
|
5369
7238
|
isSupportedIndexSchemaVersion,
|
|
5370
7239
|
knowledgeEmbeddingProfile,
|
|
@@ -5372,6 +7241,9 @@ export {
|
|
|
5372
7241
|
l2Normalize,
|
|
5373
7242
|
loadMarkdownCatalog,
|
|
5374
7243
|
manifestSigningPayload,
|
|
7244
|
+
mediaEncoderPins,
|
|
7245
|
+
mediaReferences,
|
|
7246
|
+
missingPinnedFiles,
|
|
5375
7247
|
newOutlineTopic,
|
|
5376
7248
|
newerRegistryEntries,
|
|
5377
7249
|
openCatalogDatabase,
|
|
@@ -5379,6 +7251,11 @@ export {
|
|
|
5379
7251
|
parseGitbookSummary,
|
|
5380
7252
|
parseJupyterBookToc,
|
|
5381
7253
|
parseMkdocsNav,
|
|
7254
|
+
pinnedExternalDataChunks,
|
|
7255
|
+
pinnedLoadFiles,
|
|
7256
|
+
prefetchExternalData,
|
|
7257
|
+
preparePinnedLoad,
|
|
7258
|
+
profileUnitVector,
|
|
5382
7259
|
quantizeBinary,
|
|
5383
7260
|
quantizeBinaryForProfile,
|
|
5384
7261
|
quantizeInt8,
|
|
@@ -5389,6 +7266,7 @@ export {
|
|
|
5389
7266
|
resolveOutlinePath,
|
|
5390
7267
|
resolveTransformersModelOptions,
|
|
5391
7268
|
runDuckdbScript,
|
|
7269
|
+
shardDdlFor,
|
|
5392
7270
|
signManifest,
|
|
5393
7271
|
signRegistryIndex,
|
|
5394
7272
|
titleFromFolderName,
|
|
@@ -5396,6 +7274,7 @@ export {
|
|
|
5396
7274
|
transformersGraphOptions,
|
|
5397
7275
|
validateExtractedCatalog,
|
|
5398
7276
|
verifyManifestSignature,
|
|
7277
|
+
verifyMediaArtifacts,
|
|
5399
7278
|
verifyProfileArtifacts,
|
|
5400
7279
|
verifyRegistryIndex,
|
|
5401
7280
|
writeGezkArchive
|