@bendyline/gezel-knowledge 1.2.3 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -6
- package/dist/index.d.ts +385 -40
- package/dist/index.js +1381 -315
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -18,6 +18,7 @@ import {
|
|
|
18
18
|
ROUTER_DB_PATH,
|
|
19
19
|
SOURCE_NOTICES_PATH,
|
|
20
20
|
ZIP_FIXED_MTIME,
|
|
21
|
+
formatAtLeast,
|
|
21
22
|
isSupportedFormatVersion,
|
|
22
23
|
isSupportedIndexSchemaVersion
|
|
23
24
|
} from "@bendyline/gezk";
|
|
@@ -37,7 +38,7 @@ function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY)
|
|
|
37
38
|
}
|
|
38
39
|
|
|
39
40
|
// src/format/ddl.ts
|
|
40
|
-
import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
|
|
41
|
+
import { ROUTER_DDL, SHARD_DDL, SHARD_DDL_V4, SHARD_DDL_V5, shardDdlFor } from "@bendyline/gezk";
|
|
41
42
|
|
|
42
43
|
// src/format/ids.ts
|
|
43
44
|
import { documentSlug } from "@bendyline/gezk";
|
|
@@ -47,6 +48,7 @@ import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
|
47
48
|
import {
|
|
48
49
|
centerVector,
|
|
49
50
|
l2Normalize,
|
|
51
|
+
profileUnitVector,
|
|
50
52
|
quantizeBinary,
|
|
51
53
|
quantizeBinaryForProfile,
|
|
52
54
|
quantizeInt8,
|
|
@@ -180,7 +182,7 @@ function splitOversizeLine(line, opts) {
|
|
|
180
182
|
}
|
|
181
183
|
|
|
182
184
|
// src/compiler/compile.ts
|
|
183
|
-
import { createHash as
|
|
185
|
+
import { createHash as createHash3 } from "crypto";
|
|
184
186
|
import {
|
|
185
187
|
closeSync,
|
|
186
188
|
mkdirSync,
|
|
@@ -200,6 +202,8 @@ import {
|
|
|
200
202
|
KnowledgeOrdinalSchema,
|
|
201
203
|
KnowledgeTocReferenceSchema,
|
|
202
204
|
canonicalizeJson,
|
|
205
|
+
embeddingProfileMinimumFormat,
|
|
206
|
+
isKnowledgeMediaAssetPath,
|
|
203
207
|
normalizeLongitude,
|
|
204
208
|
spatialManifest
|
|
205
209
|
} from "@bendyline/gezk";
|
|
@@ -248,176 +252,7 @@ var sqlite = nodeRequire2("node:sqlite");
|
|
|
248
252
|
var DatabaseSync = sqlite.DatabaseSync;
|
|
249
253
|
|
|
250
254
|
// src/query-stopwords.ts
|
|
251
|
-
|
|
252
|
-
// articles + determiners
|
|
253
|
-
"a",
|
|
254
|
-
"an",
|
|
255
|
-
"the",
|
|
256
|
-
"this",
|
|
257
|
-
"that",
|
|
258
|
-
"these",
|
|
259
|
-
"those",
|
|
260
|
-
"some",
|
|
261
|
-
"any",
|
|
262
|
-
"all",
|
|
263
|
-
"each",
|
|
264
|
-
"both",
|
|
265
|
-
"every",
|
|
266
|
-
"other",
|
|
267
|
-
"another",
|
|
268
|
-
"same",
|
|
269
|
-
"such",
|
|
270
|
-
// pronouns
|
|
271
|
-
"i",
|
|
272
|
-
"me",
|
|
273
|
-
"my",
|
|
274
|
-
"we",
|
|
275
|
-
"us",
|
|
276
|
-
"our",
|
|
277
|
-
"you",
|
|
278
|
-
"your",
|
|
279
|
-
"it",
|
|
280
|
-
"its",
|
|
281
|
-
"he",
|
|
282
|
-
"him",
|
|
283
|
-
"his",
|
|
284
|
-
"she",
|
|
285
|
-
"her",
|
|
286
|
-
"they",
|
|
287
|
-
"them",
|
|
288
|
-
"their",
|
|
289
|
-
// be / have / do
|
|
290
|
-
"am",
|
|
291
|
-
"is",
|
|
292
|
-
"are",
|
|
293
|
-
"was",
|
|
294
|
-
"were",
|
|
295
|
-
"be",
|
|
296
|
-
"been",
|
|
297
|
-
"being",
|
|
298
|
-
"have",
|
|
299
|
-
"has",
|
|
300
|
-
"had",
|
|
301
|
-
"do",
|
|
302
|
-
"does",
|
|
303
|
-
"did",
|
|
304
|
-
"done",
|
|
305
|
-
// modals
|
|
306
|
-
"can",
|
|
307
|
-
"could",
|
|
308
|
-
"will",
|
|
309
|
-
"would",
|
|
310
|
-
"shall",
|
|
311
|
-
"should",
|
|
312
|
-
"may",
|
|
313
|
-
"might",
|
|
314
|
-
"must",
|
|
315
|
-
// question words
|
|
316
|
-
"how",
|
|
317
|
-
"what",
|
|
318
|
-
"when",
|
|
319
|
-
"where",
|
|
320
|
-
"which",
|
|
321
|
-
"who",
|
|
322
|
-
"whom",
|
|
323
|
-
"why",
|
|
324
|
-
// prepositions + conjunctions
|
|
325
|
-
"about",
|
|
326
|
-
"and",
|
|
327
|
-
"as",
|
|
328
|
-
"at",
|
|
329
|
-
"but",
|
|
330
|
-
"by",
|
|
331
|
-
"for",
|
|
332
|
-
"from",
|
|
333
|
-
"if",
|
|
334
|
-
"in",
|
|
335
|
-
"into",
|
|
336
|
-
"of",
|
|
337
|
-
"on",
|
|
338
|
-
"or",
|
|
339
|
-
"so",
|
|
340
|
-
"than",
|
|
341
|
-
"then",
|
|
342
|
-
"there",
|
|
343
|
-
"to",
|
|
344
|
-
"with",
|
|
345
|
-
// request scaffolding + degree words
|
|
346
|
-
"please",
|
|
347
|
-
"help",
|
|
348
|
-
"let",
|
|
349
|
-
"tell",
|
|
350
|
-
"want",
|
|
351
|
-
"need",
|
|
352
|
-
"create",
|
|
353
|
-
"make",
|
|
354
|
-
"give",
|
|
355
|
-
"get",
|
|
356
|
-
"got",
|
|
357
|
-
"use",
|
|
358
|
-
"using",
|
|
359
|
-
"just",
|
|
360
|
-
"like",
|
|
361
|
-
"also",
|
|
362
|
-
"very",
|
|
363
|
-
"only",
|
|
364
|
-
"new",
|
|
365
|
-
"more",
|
|
366
|
-
"most",
|
|
367
|
-
"much",
|
|
368
|
-
"many",
|
|
369
|
-
"few",
|
|
370
|
-
"own",
|
|
371
|
-
// conversational scaffolding
|
|
372
|
-
"hey",
|
|
373
|
-
"hi",
|
|
374
|
-
"hiya",
|
|
375
|
-
"hello",
|
|
376
|
-
"thanks",
|
|
377
|
-
"thank",
|
|
378
|
-
"okay",
|
|
379
|
-
"ok",
|
|
380
|
-
"yep",
|
|
381
|
-
"yeah",
|
|
382
|
-
"yes",
|
|
383
|
-
"nope",
|
|
384
|
-
"good",
|
|
385
|
-
"great",
|
|
386
|
-
"fine",
|
|
387
|
-
"nice",
|
|
388
|
-
"cool",
|
|
389
|
-
"ready",
|
|
390
|
-
"going",
|
|
391
|
-
"doing",
|
|
392
|
-
"sounds",
|
|
393
|
-
"morning",
|
|
394
|
-
"afternoon",
|
|
395
|
-
"evening",
|
|
396
|
-
"night",
|
|
397
|
-
// Apostrophes are token boundaries, so contractions otherwise leave these
|
|
398
|
-
// fragments looking like rare, highly distinctive query terms.
|
|
399
|
-
"s",
|
|
400
|
-
"t",
|
|
401
|
-
"re",
|
|
402
|
-
"ve",
|
|
403
|
-
"ll",
|
|
404
|
-
"d",
|
|
405
|
-
"m",
|
|
406
|
-
"isn",
|
|
407
|
-
"aren",
|
|
408
|
-
"wasn",
|
|
409
|
-
"weren",
|
|
410
|
-
"don",
|
|
411
|
-
"doesn",
|
|
412
|
-
"didn",
|
|
413
|
-
"hasn",
|
|
414
|
-
"haven",
|
|
415
|
-
"hadn",
|
|
416
|
-
"won",
|
|
417
|
-
"wouldn",
|
|
418
|
-
"shouldn",
|
|
419
|
-
"couldn"
|
|
420
|
-
]);
|
|
255
|
+
import { QUERY_STOP_WORDS } from "@bendyline/gezk";
|
|
421
256
|
|
|
422
257
|
// src/reader/fts-query.ts
|
|
423
258
|
function contentTokens(query) {
|
|
@@ -563,11 +398,11 @@ import { createHash } from "crypto";
|
|
|
563
398
|
import { readFileSync } from "fs";
|
|
564
399
|
import {
|
|
565
400
|
KnowledgeAssetPathSchema,
|
|
566
|
-
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
567
|
-
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
568
401
|
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
569
402
|
assetExtension,
|
|
570
403
|
assetKindForExtension,
|
|
404
|
+
knowledgeAssetLimitsProblem,
|
|
405
|
+
maxKnowledgeAssetBytes,
|
|
571
406
|
sniffAssetType,
|
|
572
407
|
svgInertnessProblem
|
|
573
408
|
} from "@bendyline/gezk";
|
|
@@ -588,7 +423,6 @@ function prepareAssets(assets, extraFiles, opts) {
|
|
|
588
423
|
skippedPaths.add(path);
|
|
589
424
|
opts.onWarning?.(`${message}; skipped (references replaced with their text)`);
|
|
590
425
|
};
|
|
591
|
-
let total = 0;
|
|
592
426
|
for (const asset of assets) {
|
|
593
427
|
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
594
428
|
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
@@ -599,10 +433,11 @@ function prepareAssets(assets, extraFiles, opts) {
|
|
|
599
433
|
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
600
434
|
}
|
|
601
435
|
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
602
|
-
|
|
436
|
+
const limit = maxKnowledgeAssetBytes(asset.path);
|
|
437
|
+
if (bytes.byteLength > limit) {
|
|
603
438
|
rejectContent(
|
|
604
439
|
asset.path,
|
|
605
|
-
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${
|
|
440
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${limit}`
|
|
606
441
|
);
|
|
607
442
|
continue;
|
|
608
443
|
}
|
|
@@ -623,17 +458,16 @@ function prepareAssets(assets, extraFiles, opts) {
|
|
|
623
458
|
continue;
|
|
624
459
|
}
|
|
625
460
|
}
|
|
626
|
-
total += bytes.byteLength;
|
|
627
|
-
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
628
|
-
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
629
|
-
}
|
|
630
461
|
prepared.push({
|
|
631
462
|
path: asset.path,
|
|
632
463
|
bytes,
|
|
633
464
|
sizeBytes: bytes.byteLength,
|
|
634
|
-
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
465
|
+
sha256: createHash("sha256").update(bytes).digest("hex"),
|
|
466
|
+
...asset.attribution ? { attribution: asset.attribution } : {}
|
|
635
467
|
});
|
|
636
468
|
}
|
|
469
|
+
const totals = knowledgeAssetLimitsProblem(prepared);
|
|
470
|
+
if (totals) throw new Error(totals);
|
|
637
471
|
return { assets: prepared.sort((a, b) => a.path < b.path ? -1 : 1), skippedPaths };
|
|
638
472
|
}
|
|
639
473
|
function omitSkippedAssetReferences(markdown, skippedPaths) {
|
|
@@ -644,6 +478,131 @@ function omitSkippedAssetReferences(markdown, skippedPaths) {
|
|
|
644
478
|
);
|
|
645
479
|
}
|
|
646
480
|
|
|
481
|
+
// src/compiler/media-rows.ts
|
|
482
|
+
import { createHash as createHash2 } from "crypto";
|
|
483
|
+
import {
|
|
484
|
+
assetContentType,
|
|
485
|
+
assetModality
|
|
486
|
+
} from "@bendyline/gezk";
|
|
487
|
+
var MEDIA_ROW_TEXT_MAX_TOKENS = 128;
|
|
488
|
+
var MAX_MEDIA_ROWS_PER_DOCUMENT = 1024;
|
|
489
|
+
var MEDIA_REFERENCE = /!\[([^\]\n]*)\]\(\s*<?(assets\/[^)\s>]+)>?(?:\s+"[^"\n]*")?\s*\)/g;
|
|
490
|
+
function mediaReferences(markdown) {
|
|
491
|
+
const out = [];
|
|
492
|
+
markdown.split("\n").forEach((text, index) => {
|
|
493
|
+
for (const match of text.matchAll(MEDIA_REFERENCE)) {
|
|
494
|
+
out.push({
|
|
495
|
+
alt: (match[1] ?? "").trim(),
|
|
496
|
+
path: match[2],
|
|
497
|
+
line: index + 1,
|
|
498
|
+
caption: text.replace(MEDIA_REFERENCE, " ").replace(/\s+/g, " ").trim()
|
|
499
|
+
});
|
|
500
|
+
}
|
|
501
|
+
});
|
|
502
|
+
return out;
|
|
503
|
+
}
|
|
504
|
+
function humanizeAssetName(path) {
|
|
505
|
+
const base = path.slice(path.lastIndexOf("/") + 1);
|
|
506
|
+
const stem2 = base.includes(".") ? base.slice(0, base.lastIndexOf(".")) : base;
|
|
507
|
+
return stem2.replace(/[-_.]+/g, " ").replace(/\s+/g, " ").trim();
|
|
508
|
+
}
|
|
509
|
+
function boundedText(parts, countTokens) {
|
|
510
|
+
const unique = [];
|
|
511
|
+
for (const part of parts.map((p) => p.trim()).filter(Boolean)) {
|
|
512
|
+
if (!unique.some((u) => u.toLowerCase() === part.toLowerCase())) unique.push(part);
|
|
513
|
+
}
|
|
514
|
+
let text = unique.join("\n").normalize("NFC");
|
|
515
|
+
if (countTokens(text) <= MEDIA_ROW_TEXT_MAX_TOKENS) return text;
|
|
516
|
+
const words = text.split(/(\s+)/);
|
|
517
|
+
let lo = 0;
|
|
518
|
+
let hi = words.length;
|
|
519
|
+
while (lo < hi) {
|
|
520
|
+
const mid = Math.ceil((lo + hi) / 2);
|
|
521
|
+
if (countTokens(words.slice(0, mid).join("")) <= MEDIA_ROW_TEXT_MAX_TOKENS) lo = mid;
|
|
522
|
+
else hi = mid - 1;
|
|
523
|
+
}
|
|
524
|
+
text = words.slice(0, lo).join("").trim();
|
|
525
|
+
return text;
|
|
526
|
+
}
|
|
527
|
+
function headingAt(chunks, line) {
|
|
528
|
+
const inside = chunks.find((c) => c.lineStart <= line && line <= c.lineEnd);
|
|
529
|
+
if (inside) return inside.headingPath;
|
|
530
|
+
let before;
|
|
531
|
+
for (const c of chunks) if (c.lineEnd < line) before = c;
|
|
532
|
+
return before?.headingPath ?? [];
|
|
533
|
+
}
|
|
534
|
+
function windowHash(assetSha256, startMs, endMs) {
|
|
535
|
+
if (startMs === null || endMs === null) return assetSha256;
|
|
536
|
+
return createHash2("sha256").update(`${assetSha256}\0${startMs}\0${endMs}`).digest("hex");
|
|
537
|
+
}
|
|
538
|
+
async function collectMediaRows(args) {
|
|
539
|
+
const byPath = new Map(args.assets.map((a) => [a.path, a]));
|
|
540
|
+
const embedded = /* @__PURE__ */ new Map();
|
|
541
|
+
const warned = /* @__PURE__ */ new Set();
|
|
542
|
+
const out = /* @__PURE__ */ new Map();
|
|
543
|
+
for (const doc of args.documents) {
|
|
544
|
+
const rows = [];
|
|
545
|
+
const seen = /* @__PURE__ */ new Set();
|
|
546
|
+
for (const ref of mediaReferences(doc.markdown)) {
|
|
547
|
+
if (seen.has(ref.path)) continue;
|
|
548
|
+
seen.add(ref.path);
|
|
549
|
+
const asset = byPath.get(ref.path);
|
|
550
|
+
const modality = assetModality(ref.path);
|
|
551
|
+
const mimeType = assetContentType(ref.path);
|
|
552
|
+
if (!asset || !modality || !mimeType || mimeType === "image/svg+xml") continue;
|
|
553
|
+
const block = modality === "audio" ? args.profile.media?.audio : modality === "video" ? args.profile.media?.video : args.profile.media?.image;
|
|
554
|
+
if (!block) {
|
|
555
|
+
if (!warned.has(modality)) {
|
|
556
|
+
warned.add(modality);
|
|
557
|
+
args.onWarning?.(
|
|
558
|
+
`profile ${args.profile.id} describes no ${modality} encoder; ${modality} references get no media rows`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
let pending = embedded.get(ref.path);
|
|
564
|
+
if (!pending) {
|
|
565
|
+
pending = args.embedMedia({ modality, path: ref.path, bytes: asset.bytes });
|
|
566
|
+
embedded.set(ref.path, pending);
|
|
567
|
+
}
|
|
568
|
+
const results = [...await pending].sort((a, b) => (a.startMs ?? 0) - (b.startMs ?? 0));
|
|
569
|
+
const headingPath = headingAt(doc.chunks, ref.line);
|
|
570
|
+
for (const result of results) {
|
|
571
|
+
const startMs = modality === "image" ? null : result.startMs ?? 0;
|
|
572
|
+
const endMs = modality === "image" ? null : result.endMs ?? null;
|
|
573
|
+
if (startMs !== null && (endMs === null || endMs <= startMs)) {
|
|
574
|
+
throw new Error(`${ref.path}: a ${modality} window needs startMs < endMs`);
|
|
575
|
+
}
|
|
576
|
+
rows.push({
|
|
577
|
+
modality,
|
|
578
|
+
assetPath: ref.path,
|
|
579
|
+
mimeType,
|
|
580
|
+
width: result.width ?? null,
|
|
581
|
+
height: result.height ?? null,
|
|
582
|
+
startMs,
|
|
583
|
+
endMs,
|
|
584
|
+
attributionJson: asset.attribution ? JSON.stringify(asset.attribution) : null,
|
|
585
|
+
headingPath,
|
|
586
|
+
line: ref.line,
|
|
587
|
+
text: boundedText(
|
|
588
|
+
[ref.alt, ref.caption, humanizeAssetName(ref.path), result.transcript ?? ""],
|
|
589
|
+
args.countTokens
|
|
590
|
+
),
|
|
591
|
+
contentHash: windowHash(asset.sha256, startMs, endMs),
|
|
592
|
+
vector: result.vector
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
if (rows.length > MAX_MEDIA_ROWS_PER_DOCUMENT) {
|
|
597
|
+
throw new Error(
|
|
598
|
+
`document ${doc.id} has ${rows.length} media rows; the limit is ${MAX_MEDIA_ROWS_PER_DOCUMENT}`
|
|
599
|
+
);
|
|
600
|
+
}
|
|
601
|
+
if (rows.length > 0) out.set(doc.id, rows);
|
|
602
|
+
}
|
|
603
|
+
return out;
|
|
604
|
+
}
|
|
605
|
+
|
|
647
606
|
// src/compiler/compile.ts
|
|
648
607
|
async function compileKnowledgeCatalog(opts) {
|
|
649
608
|
if (opts.topics.length === 0) {
|
|
@@ -701,15 +660,40 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
701
660
|
leafTopicId,
|
|
702
661
|
placements,
|
|
703
662
|
metaJson: encodeDocumentMeta(doc),
|
|
704
|
-
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
663
|
+
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts),
|
|
664
|
+
media: []
|
|
705
665
|
});
|
|
706
666
|
}
|
|
707
667
|
prepared.sort(
|
|
708
668
|
(a, b) => a.topicPathKey === b.topicPathKey ? a.doc.id < b.doc.id ? -1 : a.doc.id > b.doc.id ? 1 : 0 : a.topicPathKey < b.topicPathKey ? -1 : 1
|
|
709
669
|
);
|
|
710
670
|
const totalChunks = prepared.reduce((sum, p) => sum + p.chunks.length, 0);
|
|
671
|
+
if (opts.embedMedia) {
|
|
672
|
+
const rows = await collectMediaRows({
|
|
673
|
+
documents: prepared.map((p) => ({
|
|
674
|
+
id: p.doc.id,
|
|
675
|
+
title: p.doc.title,
|
|
676
|
+
markdown: normalizeMarkdown(p.doc.markdown),
|
|
677
|
+
chunks: p.chunks
|
|
678
|
+
})),
|
|
679
|
+
assets: preparedAssets,
|
|
680
|
+
profile,
|
|
681
|
+
embedMedia: opts.embedMedia,
|
|
682
|
+
countTokens: opts.countTokens,
|
|
683
|
+
onWarning: opts.onWarning
|
|
684
|
+
});
|
|
685
|
+
for (const p of prepared) p.media = rows.get(p.doc.id) ?? [];
|
|
686
|
+
}
|
|
687
|
+
const mediaCounts = { image: 0, video: 0, audio: 0 };
|
|
688
|
+
for (const p of prepared) for (const row of p.media) mediaCounts[row.modality] += 1;
|
|
689
|
+
const totalMedia = mediaCounts.image + mediaCounts.video + mediaCounts.audio;
|
|
690
|
+
const totalRows = totalChunks + totalMedia;
|
|
691
|
+
const rowsOf = (p) => p.chunks.length + p.media.length;
|
|
692
|
+
const formatVersion = catalogFormatVersion(profile, preparedAssets, totalMedia);
|
|
693
|
+
const indexSchemaVersion = GEZK_FORMAT_GENERATIONS[formatVersion];
|
|
694
|
+
const shardDdl = shardDdlFor(indexSchemaVersion);
|
|
711
695
|
const shardTarget = opts.shardTargetChunks ?? SHARD_TARGET_CHUNKS;
|
|
712
|
-
const embedded =
|
|
696
|
+
const embedded = totalRows <= shardTarget;
|
|
713
697
|
const semanticFill = opts.shardFill === "semantic" && !embedded;
|
|
714
698
|
const embedInput = (p, chunk) => `${profile.passageInstruction}${buildContextHeader(
|
|
715
699
|
p.doc.title,
|
|
@@ -726,7 +710,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
726
710
|
docVectors = await stageVectors({
|
|
727
711
|
prepared,
|
|
728
712
|
stagedPath,
|
|
729
|
-
|
|
713
|
+
profile,
|
|
730
714
|
batchSize: opts.embedBatchSize ?? 32,
|
|
731
715
|
embed: opts.embed,
|
|
732
716
|
embedInput,
|
|
@@ -739,11 +723,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
739
723
|
if (embedded) {
|
|
740
724
|
for (const p of prepared) shardOf.set(p.doc.id, 0);
|
|
741
725
|
} else if (semanticFill) {
|
|
742
|
-
shardCount = Math.ceil(
|
|
743
|
-
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(
|
|
726
|
+
shardCount = Math.ceil(totalRows / shardTarget);
|
|
727
|
+
const capacity = Math.min(SHARD_MAX_CHUNKS, Math.ceil(totalRows / shardCount * 1.02));
|
|
744
728
|
const assignment = semanticShards(
|
|
745
729
|
docVectors,
|
|
746
|
-
prepared.map(
|
|
730
|
+
prepared.map(rowsOf),
|
|
747
731
|
shardCount,
|
|
748
732
|
capacity,
|
|
749
733
|
seedFor(`${opts.catalog.id}#shard-fill`, 0)
|
|
@@ -755,19 +739,19 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
755
739
|
let current = 0;
|
|
756
740
|
let filled = 0;
|
|
757
741
|
for (const p of prepared) {
|
|
758
|
-
if (filled > 0 && filled + p
|
|
742
|
+
if (filled > 0 && filled + rowsOf(p) > shardTarget) {
|
|
759
743
|
current++;
|
|
760
744
|
filled = 0;
|
|
761
745
|
}
|
|
762
746
|
shardOf.set(p.doc.id, current);
|
|
763
|
-
filled += p
|
|
747
|
+
filled += rowsOf(p);
|
|
764
748
|
}
|
|
765
749
|
shardCount = current + 1;
|
|
766
750
|
}
|
|
767
751
|
const routerStaged = join(opts.workDir, ROUTER_DB_PATH);
|
|
768
752
|
const shardPath = (id) => embedded ? ROUTER_DB_PATH : `index/shards/${String(id).padStart(3, "0")}.db`;
|
|
769
753
|
const shardDbs = /* @__PURE__ */ new Map();
|
|
770
|
-
const router = createCatalogDb(routerStaged);
|
|
754
|
+
const router = createCatalogDb(routerStaged, indexSchemaVersion);
|
|
771
755
|
const closeAll = () => {
|
|
772
756
|
for (const db of [router, ...shardDbs.values()]) {
|
|
773
757
|
try {
|
|
@@ -780,11 +764,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
780
764
|
router.exec(ROUTER_DDL);
|
|
781
765
|
router.exec("BEGIN");
|
|
782
766
|
if (embedded) {
|
|
783
|
-
router.exec(
|
|
767
|
+
router.exec(shardDdl.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
|
|
784
768
|
}
|
|
785
769
|
const commonMeta = {
|
|
786
|
-
format_version: String(
|
|
787
|
-
index_schema_version: String(
|
|
770
|
+
format_version: String(formatVersion),
|
|
771
|
+
index_schema_version: String(indexSchemaVersion),
|
|
788
772
|
catalog_id: opts.catalog.id,
|
|
789
773
|
catalog_version: opts.catalog.version,
|
|
790
774
|
embedding_profile_id: profile.id,
|
|
@@ -864,7 +848,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
864
848
|
p.leafTopicId,
|
|
865
849
|
p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
|
|
866
850
|
BigInt(shardOf.get(p.doc.id)),
|
|
867
|
-
BigInt(p
|
|
851
|
+
BigInt(rowsOf(p)),
|
|
868
852
|
p.doc.sourceUrl ?? null,
|
|
869
853
|
p.doc.sourceRevision ?? null,
|
|
870
854
|
p.doc.sourceUpdatedAt ?? null,
|
|
@@ -909,8 +893,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
909
893
|
if (embedded) return router;
|
|
910
894
|
let db = shardDbs.get(id);
|
|
911
895
|
if (!db) {
|
|
912
|
-
db = createCatalogDb(join(opts.workDir, shardPath(id)));
|
|
913
|
-
db.exec(
|
|
896
|
+
db = createCatalogDb(join(opts.workDir, shardPath(id)), indexSchemaVersion);
|
|
897
|
+
db.exec(shardDdl);
|
|
914
898
|
db.exec("BEGIN");
|
|
915
899
|
shardDbs.set(id, db);
|
|
916
900
|
}
|
|
@@ -941,7 +925,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
941
925
|
);
|
|
942
926
|
}
|
|
943
927
|
for (let i = 0; i < vectors.length; i++) {
|
|
944
|
-
const
|
|
928
|
+
const raw = vectors[i];
|
|
929
|
+
const unit = staged ? l2Normalize(raw) : profileUnitVector(profile, raw);
|
|
945
930
|
if (unit.length !== profile.dimensions) {
|
|
946
931
|
throw new Error(
|
|
947
932
|
`embedder returned dim ${unit.length}, profile expects ${profile.dimensions}`
|
|
@@ -1005,6 +990,46 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1005
990
|
pendingRows.push({ shardId, chunkId });
|
|
1006
991
|
if (pendingTexts.length >= embedBatchSize) await flushEmbeds();
|
|
1007
992
|
}
|
|
993
|
+
if (p.media.length > 0) {
|
|
994
|
+
const insertMedia = db.prepare(
|
|
995
|
+
`INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
|
|
996
|
+
line_start, line_end, token_count, content_hash, text, modality, asset_path, mime_type,
|
|
997
|
+
width, height, start_ms, end_ms, attribution_json)
|
|
998
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id`
|
|
999
|
+
);
|
|
1000
|
+
const stmts = vecStmtsFor(db);
|
|
1001
|
+
p.media.forEach((media, index) => {
|
|
1002
|
+
const ordinal = p.chunks.length + index;
|
|
1003
|
+
const headingText = media.headingPath.join(" > ");
|
|
1004
|
+
const row = insertMedia.get(
|
|
1005
|
+
chunkUid(p.doc.id, ordinal, `media:${media.modality}:${media.contentHash}`),
|
|
1006
|
+
p.doc.id,
|
|
1007
|
+
BigInt(ordinal),
|
|
1008
|
+
p.doc.title,
|
|
1009
|
+
JSON.stringify(media.headingPath),
|
|
1010
|
+
headingText,
|
|
1011
|
+
BigInt(media.line),
|
|
1012
|
+
BigInt(media.line),
|
|
1013
|
+
BigInt(opts.countTokens(media.text)),
|
|
1014
|
+
media.contentHash,
|
|
1015
|
+
media.text,
|
|
1016
|
+
media.modality,
|
|
1017
|
+
media.assetPath,
|
|
1018
|
+
media.mimeType,
|
|
1019
|
+
media.width === null ? null : BigInt(media.width),
|
|
1020
|
+
media.height === null ? null : BigInt(media.height),
|
|
1021
|
+
media.startMs === null ? null : BigInt(media.startMs),
|
|
1022
|
+
media.endMs === null ? null : BigInt(media.endMs),
|
|
1023
|
+
media.attributionJson
|
|
1024
|
+
);
|
|
1025
|
+
const chunkId = Number(row.id);
|
|
1026
|
+
insertChunkFts.run(BigInt(chunkId), p.doc.title, headingText, media.text);
|
|
1027
|
+
shardChunkCounts.set(shardId, (shardChunkCounts.get(shardId) ?? 0) + 1);
|
|
1028
|
+
const unit = profileUnitVector(profile, media.vector);
|
|
1029
|
+
stmts.vec.run(BigInt(chunkId), Buffer.from(quantizeBinaryForProfile(profile, unit)));
|
|
1030
|
+
stmts.int8.run(BigInt(chunkId), Buffer.from(quantizeInt8(unit).buffer));
|
|
1031
|
+
});
|
|
1032
|
+
}
|
|
1008
1033
|
}
|
|
1009
1034
|
await flushEmbeds();
|
|
1010
1035
|
staged?.close();
|
|
@@ -1016,7 +1041,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1016
1041
|
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
1017
1042
|
});
|
|
1018
1043
|
}
|
|
1019
|
-
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(
|
|
1044
|
+
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalRows) });
|
|
1020
1045
|
const finalDir = join(opts.workDir, "final");
|
|
1021
1046
|
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
1022
1047
|
const vacuumed = [];
|
|
@@ -1128,7 +1153,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1128
1153
|
files.push({
|
|
1129
1154
|
path,
|
|
1130
1155
|
sizeBytes: bytes.length,
|
|
1131
|
-
sha256:
|
|
1156
|
+
sha256: createHash3("sha256").update(bytes).digest("hex")
|
|
1132
1157
|
});
|
|
1133
1158
|
}
|
|
1134
1159
|
for (const asset of preparedAssets) {
|
|
@@ -1137,8 +1162,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1137
1162
|
assertAssetReferencesDeclared(prepared, preparedAssets);
|
|
1138
1163
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
1139
1164
|
kind: GEZK_MANIFEST_KIND,
|
|
1140
|
-
formatVersion
|
|
1141
|
-
indexSchemaVersion
|
|
1165
|
+
formatVersion,
|
|
1166
|
+
indexSchemaVersion,
|
|
1142
1167
|
id: opts.catalog.id,
|
|
1143
1168
|
version: opts.catalog.version,
|
|
1144
1169
|
name: opts.catalog.name,
|
|
@@ -1167,9 +1192,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1167
1192
|
},
|
|
1168
1193
|
counts: {
|
|
1169
1194
|
documents: prepared.length,
|
|
1170
|
-
chunks:
|
|
1195
|
+
chunks: totalRows,
|
|
1171
1196
|
shards: shardCount,
|
|
1172
|
-
assets: preparedAssets.length
|
|
1197
|
+
assets: preparedAssets.length,
|
|
1198
|
+
...formatVersion === "0.8" ? { media: mediaCounts } : {}
|
|
1173
1199
|
},
|
|
1174
1200
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
1175
1201
|
spatial: spatialManifest(
|
|
@@ -1177,7 +1203,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1177
1203
|
(p) => (p.doc.locations ?? []).map((location) => ({ documentId: p.doc.id, location }))
|
|
1178
1204
|
)
|
|
1179
1205
|
),
|
|
1180
|
-
requires: { formatVersion
|
|
1206
|
+
requires: { formatVersion, features: [] },
|
|
1181
1207
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
1182
1208
|
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
1183
1209
|
});
|
|
@@ -1201,7 +1227,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
1201
1227
|
return {
|
|
1202
1228
|
manifest,
|
|
1203
1229
|
documents: prepared.length,
|
|
1204
|
-
chunks:
|
|
1230
|
+
chunks: totalRows,
|
|
1231
|
+
media: mediaCounts,
|
|
1205
1232
|
shards: shardCount,
|
|
1206
1233
|
archiveBytes
|
|
1207
1234
|
};
|
|
@@ -1285,11 +1312,15 @@ function assertAssetReferencesDeclared(prepared, assets) {
|
|
|
1285
1312
|
}
|
|
1286
1313
|
}
|
|
1287
1314
|
}
|
|
1288
|
-
function
|
|
1315
|
+
function catalogFormatVersion(profile, assets, mediaRows = 0) {
|
|
1316
|
+
const needsMedia = mediaRows > 0 || assets.some((asset) => isKnowledgeMediaAssetPath(asset.path));
|
|
1317
|
+
return needsMedia || embeddingProfileMinimumFormat(profile) === "0.8" ? "0.8" : "0.7";
|
|
1318
|
+
}
|
|
1319
|
+
function createCatalogDb(absPath, indexSchemaVersion) {
|
|
1289
1320
|
const db = new DatabaseSync(absPath);
|
|
1290
1321
|
db.exec("PRAGMA page_size=8192");
|
|
1291
1322
|
db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
|
|
1292
|
-
db.exec(`PRAGMA user_version=${
|
|
1323
|
+
db.exec(`PRAGMA user_version=${indexSchemaVersion}`);
|
|
1293
1324
|
db.exec("PRAGMA journal_mode=DELETE");
|
|
1294
1325
|
return db;
|
|
1295
1326
|
}
|
|
@@ -1334,7 +1365,8 @@ ${path.join(" > ")}
|
|
|
1334
1365
|
return header;
|
|
1335
1366
|
}
|
|
1336
1367
|
async function stageVectors(args) {
|
|
1337
|
-
const { prepared,
|
|
1368
|
+
const { prepared, profile } = args;
|
|
1369
|
+
const { dimensions } = profile;
|
|
1338
1370
|
const docVectors = prepared.map(() => new Float32Array(dimensions));
|
|
1339
1371
|
const fd = openSync(args.stagedPath, "w");
|
|
1340
1372
|
let texts = [];
|
|
@@ -1348,10 +1380,7 @@ async function stageVectors(args) {
|
|
|
1348
1380
|
}
|
|
1349
1381
|
const block = new Float32Array(texts.length * dimensions);
|
|
1350
1382
|
for (let i = 0; i < vectors.length; i++) {
|
|
1351
|
-
const unit =
|
|
1352
|
-
if (unit.length !== dimensions) {
|
|
1353
|
-
throw new Error(`embedder returned dim ${unit.length}, profile expects ${dimensions}`);
|
|
1354
|
-
}
|
|
1383
|
+
const unit = profileUnitVector(profile, vectors[i]);
|
|
1355
1384
|
block.set(unit, i * dimensions);
|
|
1356
1385
|
const sum = docVectors[owners[i]];
|
|
1357
1386
|
for (let d = 0; d < dimensions; d++) sum[d] = sum[d] + unit[d];
|
|
@@ -1375,7 +1404,9 @@ async function stageVectors(args) {
|
|
|
1375
1404
|
} finally {
|
|
1376
1405
|
closeSync(fd);
|
|
1377
1406
|
}
|
|
1378
|
-
return docVectors.map(
|
|
1407
|
+
return docVectors.map(
|
|
1408
|
+
(v) => v.some((x) => x !== 0) ? Float32Array.from(l2Normalize(Array.from(v))) : v
|
|
1409
|
+
);
|
|
1379
1410
|
}
|
|
1380
1411
|
var StagedVectorReader = class {
|
|
1381
1412
|
constructor(path, dimensions) {
|
|
@@ -1417,7 +1448,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1417
1448
|
return sum;
|
|
1418
1449
|
};
|
|
1419
1450
|
const stride = Math.max(1, Math.ceil(n / 2e4));
|
|
1420
|
-
const sample = docVectors.filter((
|
|
1451
|
+
const sample = docVectors.filter((v, i) => i % stride === 0 && v.some((x) => x !== 0));
|
|
1421
1452
|
const centers = kMeans(sample, shardCount, seed).map((c) => c.centroid);
|
|
1422
1453
|
for (let i = 0; centers.length < shardCount; i++) {
|
|
1423
1454
|
centers.push(
|
|
@@ -1486,7 +1517,7 @@ function semanticShards(docVectors, docChunks, shardCount, capacity, seed) {
|
|
|
1486
1517
|
return assignment.map((s) => renumber.get(s));
|
|
1487
1518
|
}
|
|
1488
1519
|
function seedFor(catalogId, shardId) {
|
|
1489
|
-
const digest =
|
|
1520
|
+
const digest = createHash3("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
1490
1521
|
return digest.readBigUInt64LE(24);
|
|
1491
1522
|
}
|
|
1492
1523
|
function makePrng(seed) {
|
|
@@ -1571,7 +1602,7 @@ function kMeans(samples, k, seed) {
|
|
|
1571
1602
|
}
|
|
1572
1603
|
|
|
1573
1604
|
// src/archive/read.ts
|
|
1574
|
-
import { createHash as
|
|
1605
|
+
import { createHash as createHash4 } from "crypto";
|
|
1575
1606
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
1576
1607
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
1577
1608
|
import { createRequire as createRequire4 } from "module";
|
|
@@ -1861,7 +1892,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1861
1892
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1862
1893
|
return;
|
|
1863
1894
|
}
|
|
1864
|
-
const hasher =
|
|
1895
|
+
const hasher = createHash4("sha256");
|
|
1865
1896
|
let sizeBytes = 0;
|
|
1866
1897
|
const verify = new Transform({
|
|
1867
1898
|
transform(chunk, _encoding, callback) {
|
|
@@ -1931,6 +1962,14 @@ var CatalogOpenError = class extends Error {
|
|
|
1931
1962
|
}
|
|
1932
1963
|
reason;
|
|
1933
1964
|
};
|
|
1965
|
+
var CatalogQueryError = class extends Error {
|
|
1966
|
+
constructor(message, reason) {
|
|
1967
|
+
super(message);
|
|
1968
|
+
this.reason = reason;
|
|
1969
|
+
this.name = "CatalogQueryError";
|
|
1970
|
+
}
|
|
1971
|
+
reason;
|
|
1972
|
+
};
|
|
1934
1973
|
function openCatalogDatabase(absPath) {
|
|
1935
1974
|
const uri = `${pathToFileURL(absPath).href}?immutable=1`;
|
|
1936
1975
|
let db;
|
|
@@ -1984,15 +2023,15 @@ function readPragmaNumber(db, pragma) {
|
|
|
1984
2023
|
}
|
|
1985
2024
|
|
|
1986
2025
|
// src/reader/catalog-handle.ts
|
|
1987
|
-
import { readFileSync as readFileSync3 } from "fs";
|
|
2026
|
+
import { readFileSync as readFileSync3, statSync as statSync2 } from "fs";
|
|
1988
2027
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1989
2028
|
import { brotliDecompressSync } from "zlib";
|
|
1990
2029
|
import {
|
|
1991
2030
|
KnowledgeEmbeddingProfileSchema,
|
|
1992
|
-
|
|
1993
|
-
assetContentType,
|
|
2031
|
+
assetContentType as assetContentType2,
|
|
1994
2032
|
embeddingProfileCenter,
|
|
1995
|
-
isKnowledgeAssetPath
|
|
2033
|
+
isKnowledgeAssetPath,
|
|
2034
|
+
maxKnowledgeAssetBytes as maxKnowledgeAssetBytes2
|
|
1996
2035
|
} from "@bendyline/gezk";
|
|
1997
2036
|
|
|
1998
2037
|
// src/reader/bit-scan.ts
|
|
@@ -2065,7 +2104,7 @@ function hammingTopK(index, query, k) {
|
|
|
2065
2104
|
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
2066
2105
|
return hits;
|
|
2067
2106
|
}
|
|
2068
|
-
function asymmetricTopK(index, query, k, eligibleChunkIds) {
|
|
2107
|
+
function asymmetricTopK(index, query, k, eligibleChunkIds, excludedChunkIds) {
|
|
2069
2108
|
const { bits, bytesPerRow, rows } = index;
|
|
2070
2109
|
if (Math.ceil(query.length / 8) !== bytesPerRow) {
|
|
2071
2110
|
throw new Error(
|
|
@@ -2122,7 +2161,7 @@ function asymmetricTopK(index, query, k, eligibleChunkIds) {
|
|
|
2122
2161
|
};
|
|
2123
2162
|
let offset = 0;
|
|
2124
2163
|
for (let r = 0; r < rows; r++) {
|
|
2125
|
-
if (eligibleChunkIds && !eligibleChunkIds.has(r + 1)) {
|
|
2164
|
+
if (eligibleChunkIds && !eligibleChunkIds.has(r + 1) || excludedChunkIds?.has(r + 1)) {
|
|
2126
2165
|
offset += bytesPerRow;
|
|
2127
2166
|
continue;
|
|
2128
2167
|
}
|
|
@@ -2261,6 +2300,40 @@ var DocumentSpatialIndex = class {
|
|
|
2261
2300
|
|
|
2262
2301
|
// src/reader/catalog-handle.ts
|
|
2263
2302
|
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
2303
|
+
var CHUNK_COLUMNS = "chunk_uid, document_id, title, heading_path, line_start, line_end, text";
|
|
2304
|
+
var MEDIA_COLUMNS = "modality, asset_path, mime_type, width, height, start_ms, end_ms, thumbnail_path, attribution_json";
|
|
2305
|
+
function mediaOf(row) {
|
|
2306
|
+
const modality = row.modality;
|
|
2307
|
+
if (!modality || modality === "text") return void 0;
|
|
2308
|
+
const num = (v) => v === null || v === void 0 ? void 0 : Number(v);
|
|
2309
|
+
let attribution;
|
|
2310
|
+
if (typeof row.attribution_json === "string") {
|
|
2311
|
+
try {
|
|
2312
|
+
const parsed = JSON.parse(row.attribution_json);
|
|
2313
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
2314
|
+
attribution = parsed;
|
|
2315
|
+
}
|
|
2316
|
+
} catch {
|
|
2317
|
+
attribution = void 0;
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
const media = {
|
|
2321
|
+
modality,
|
|
2322
|
+
assetPath: row.asset_path,
|
|
2323
|
+
mimeType: row.mime_type
|
|
2324
|
+
};
|
|
2325
|
+
const width = num(row.width);
|
|
2326
|
+
const height = num(row.height);
|
|
2327
|
+
const startMs = num(row.start_ms);
|
|
2328
|
+
const endMs = num(row.end_ms);
|
|
2329
|
+
if (width !== void 0) media.width = width;
|
|
2330
|
+
if (height !== void 0) media.height = height;
|
|
2331
|
+
if (startMs !== void 0) media.startMs = startMs;
|
|
2332
|
+
if (endMs !== void 0) media.endMs = endMs;
|
|
2333
|
+
if (typeof row.thumbnail_path === "string") media.thumbnailAssetPath = row.thumbnail_path;
|
|
2334
|
+
if (attribution) media.attribution = attribution;
|
|
2335
|
+
return media;
|
|
2336
|
+
}
|
|
2264
2337
|
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
2265
2338
|
var CHUNK_TERM_ROWS_CACHE_MAX = 5e4;
|
|
2266
2339
|
var TITLE_ROUTE_SHARDS = 1;
|
|
@@ -2281,6 +2354,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2281
2354
|
router;
|
|
2282
2355
|
connections = /* @__PURE__ */ new Map();
|
|
2283
2356
|
bitIndexes = /* @__PURE__ */ new Map();
|
|
2357
|
+
mediaIndexes = /* @__PURE__ */ new Map();
|
|
2284
2358
|
/** `shard path \0 phrase` → rows the phrase matches in that shard's `fts_chunks`. */
|
|
2285
2359
|
chunkTermRows = /* @__PURE__ */ new Map();
|
|
2286
2360
|
spatial;
|
|
@@ -2414,6 +2488,127 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2414
2488
|
this.binaryCenterCache = embeddingProfileCenter(profile.data);
|
|
2415
2489
|
return this.binaryCenterCache;
|
|
2416
2490
|
}
|
|
2491
|
+
/** The chunk columns this catalog's schema has. */
|
|
2492
|
+
chunkColumns() {
|
|
2493
|
+
return this.schemaVersion >= 5 ? `${CHUNK_COLUMNS}, ${MEDIA_COLUMNS}` : CHUNK_COLUMNS;
|
|
2494
|
+
}
|
|
2495
|
+
/** A shard's media rows with their int8 vectors (empty before index schema 5). */
|
|
2496
|
+
shardMedia(shard) {
|
|
2497
|
+
const cached = this.mediaIndexes.get(shard.path);
|
|
2498
|
+
if (cached) return cached;
|
|
2499
|
+
const empty = {
|
|
2500
|
+
ids: [],
|
|
2501
|
+
modalities: [],
|
|
2502
|
+
int8: new Int8Array(0),
|
|
2503
|
+
idSet: /* @__PURE__ */ new Set()
|
|
2504
|
+
};
|
|
2505
|
+
if (this.schemaVersion < 5) {
|
|
2506
|
+
this.mediaIndexes.set(shard.path, empty);
|
|
2507
|
+
return empty;
|
|
2508
|
+
}
|
|
2509
|
+
const dims = this.dimensions();
|
|
2510
|
+
const rows = this.shardDb(shard).prepare(
|
|
2511
|
+
`SELECT c.id, c.modality, v.v FROM chunks c JOIN chunk_vectors_int8 v ON v.chunk_id = c.id
|
|
2512
|
+
WHERE c.modality <> 'text' ORDER BY c.id`
|
|
2513
|
+
).all();
|
|
2514
|
+
const int8 = new Int8Array(rows.length * dims);
|
|
2515
|
+
rows.forEach((row, i) => {
|
|
2516
|
+
if (row.v.byteLength !== dims) {
|
|
2517
|
+
throw new CatalogOpenError(`int8 vector width ${row.v.byteLength} != ${dims}`, "corrupt");
|
|
2518
|
+
}
|
|
2519
|
+
int8.set(new Int8Array(row.v.buffer, row.v.byteOffset, dims), i * dims);
|
|
2520
|
+
});
|
|
2521
|
+
const ids = rows.map((r) => Number(r.id));
|
|
2522
|
+
const index = {
|
|
2523
|
+
ids,
|
|
2524
|
+
modalities: rows.map((r) => r.modality),
|
|
2525
|
+
int8,
|
|
2526
|
+
idSet: new Set(ids)
|
|
2527
|
+
};
|
|
2528
|
+
this.mediaIndexes.set(shard.path, index);
|
|
2529
|
+
return index;
|
|
2530
|
+
}
|
|
2531
|
+
/** Media rows this catalog holds, by modality (all zero before index schema 5). */
|
|
2532
|
+
mediaCounts() {
|
|
2533
|
+
const counts = { image: 0, video: 0, audio: 0 };
|
|
2534
|
+
for (const shard of this.shards) {
|
|
2535
|
+
for (const modality of this.shardMedia(shard).modalities) counts[modality] += 1;
|
|
2536
|
+
}
|
|
2537
|
+
return counts;
|
|
2538
|
+
}
|
|
2539
|
+
/**
|
|
2540
|
+
* The media lane: every media row of every shard scored exactly against
|
|
2541
|
+
* the query (int8 rerank), keeping the best `perModality` of each kind.
|
|
2542
|
+
* Images sit at a lower cosine than text in a shared space, so in the text
|
|
2543
|
+
* lane's sign-bit pre-filter they would rarely survive the cut; media rows
|
|
2544
|
+
* are few enough (capped per catalog) to score them all.
|
|
2545
|
+
*/
|
|
2546
|
+
searchMedia(queryVector, opts = {}) {
|
|
2547
|
+
this.assertQueryWidth(queryVector);
|
|
2548
|
+
const perModality = opts.perModality ?? 4;
|
|
2549
|
+
const dims = this.dimensions();
|
|
2550
|
+
const best = /* @__PURE__ */ new Map();
|
|
2551
|
+
for (const shard of this.shards) {
|
|
2552
|
+
const index = this.shardMedia(shard);
|
|
2553
|
+
for (let i = 0; i < index.ids.length; i++) {
|
|
2554
|
+
let dot = 0;
|
|
2555
|
+
const base = i * dims;
|
|
2556
|
+
for (let d = 0; d < dims; d++)
|
|
2557
|
+
dot += queryVector[d] * (index.int8[base + d] / 127);
|
|
2558
|
+
const modality = index.modalities[i];
|
|
2559
|
+
const list = best.get(modality) ?? [];
|
|
2560
|
+
list.push({ shard, id: index.ids[i], cosine: dot });
|
|
2561
|
+
best.set(modality, list);
|
|
2562
|
+
}
|
|
2563
|
+
}
|
|
2564
|
+
const hits = [];
|
|
2565
|
+
for (const list of best.values()) {
|
|
2566
|
+
list.sort((a, b) => b.cosine - a.cosine || a.id - b.id);
|
|
2567
|
+
let kept = 0;
|
|
2568
|
+
for (const candidate of list) {
|
|
2569
|
+
if (kept >= perModality) break;
|
|
2570
|
+
const row = this.shardDb(candidate.shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`).get(BigInt(candidate.id));
|
|
2571
|
+
if (!row) continue;
|
|
2572
|
+
if (opts.allowedDocumentIds && !opts.allowedDocumentIds.has(row.document_id))
|
|
2573
|
+
continue;
|
|
2574
|
+
kept++;
|
|
2575
|
+
hits.push({
|
|
2576
|
+
...this.hitFromRow(row, candidate.shard.id),
|
|
2577
|
+
cosine: candidate.cosine,
|
|
2578
|
+
source: "vector"
|
|
2579
|
+
});
|
|
2580
|
+
}
|
|
2581
|
+
}
|
|
2582
|
+
return hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
|
|
2583
|
+
}
|
|
2584
|
+
hitFromRow(row, shardId) {
|
|
2585
|
+
const media = mediaOf(row);
|
|
2586
|
+
return {
|
|
2587
|
+
chunkUid: row.chunk_uid,
|
|
2588
|
+
documentId: row.document_id,
|
|
2589
|
+
title: row.title,
|
|
2590
|
+
headingPath: parseHeadingPath(row.heading_path),
|
|
2591
|
+
lineStart: Number(row.line_start),
|
|
2592
|
+
lineEnd: Number(row.line_end),
|
|
2593
|
+
text: row.text,
|
|
2594
|
+
shardId,
|
|
2595
|
+
...media ? { media } : {}
|
|
2596
|
+
};
|
|
2597
|
+
}
|
|
2598
|
+
/**
|
|
2599
|
+
* Refuse a query vector whose width is not the catalog's stored width,
|
|
2600
|
+
* before any scan: routing and rerank would otherwise score a silent
|
|
2601
|
+
* prefix or fail deep inside a shard with an untyped error.
|
|
2602
|
+
*/
|
|
2603
|
+
assertQueryWidth(queryVector) {
|
|
2604
|
+
const dims = this.dimensions();
|
|
2605
|
+
if (queryVector.length !== dims) {
|
|
2606
|
+
throw new CatalogQueryError(
|
|
2607
|
+
`query vector has ${queryVector.length} dimensions, the catalog stores ${dims}`,
|
|
2608
|
+
"dimension"
|
|
2609
|
+
);
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
2417
2612
|
/** The embedding dimension from the router's profile echo. */
|
|
2418
2613
|
dimensions() {
|
|
2419
2614
|
const raw = this.meta.embedding_profile_json;
|
|
@@ -2597,21 +2792,9 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2597
2792
|
const shardId = Number(doc.shard_id);
|
|
2598
2793
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
2599
2794
|
if (!shard) return null;
|
|
2600
|
-
const row = this.shardDb(shard).prepare(
|
|
2601
|
-
`SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
|
|
2602
|
-
FROM chunks WHERE chunk_uid = ? AND document_id = ?`
|
|
2603
|
-
).get(chunkUid2, documentId);
|
|
2795
|
+
const row = this.shardDb(shard).prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE chunk_uid = ? AND document_id = ?`).get(chunkUid2, documentId);
|
|
2604
2796
|
if (!row) return null;
|
|
2605
|
-
return
|
|
2606
|
-
chunkUid: row.chunk_uid,
|
|
2607
|
-
documentId: row.document_id,
|
|
2608
|
-
title: row.title,
|
|
2609
|
-
headingPath: parseHeadingPath(row.heading_path),
|
|
2610
|
-
lineStart: Number(row.line_start),
|
|
2611
|
-
lineEnd: Number(row.line_end),
|
|
2612
|
-
text: row.text,
|
|
2613
|
-
shardId
|
|
2614
|
-
};
|
|
2797
|
+
return this.hitFromRow(row, shardId);
|
|
2615
2798
|
}
|
|
2616
2799
|
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
2617
2800
|
documentsWithUndeclaredTopic() {
|
|
@@ -2700,7 +2883,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2700
2883
|
for (const file of files) {
|
|
2701
2884
|
const path = file?.path;
|
|
2702
2885
|
if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
|
|
2703
|
-
const contentType =
|
|
2886
|
+
const contentType = assetContentType2(path);
|
|
2704
2887
|
if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
|
|
2705
2888
|
continue;
|
|
2706
2889
|
}
|
|
@@ -2714,11 +2897,37 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2714
2897
|
assets() {
|
|
2715
2898
|
return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
2716
2899
|
}
|
|
2900
|
+
/**
|
|
2901
|
+
* Where a declared asset lives on disk, so a large one (video, audio) can be
|
|
2902
|
+
* streamed with byte ranges instead of read whole. Checks containment and
|
|
2903
|
+
* the manifest size; the content hash was verified once, at extraction.
|
|
2904
|
+
*/
|
|
2905
|
+
assetFile(path) {
|
|
2906
|
+
const info = this.assetIndex().get(path);
|
|
2907
|
+
if (!info) return null;
|
|
2908
|
+
if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
|
|
2909
|
+
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
2910
|
+
}
|
|
2911
|
+
const absPath = this.resolveCatalogPath(path);
|
|
2912
|
+
let size;
|
|
2913
|
+
try {
|
|
2914
|
+
size = statSync2(absPath).size;
|
|
2915
|
+
} catch (error) {
|
|
2916
|
+
throw new CatalogOpenError(
|
|
2917
|
+
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2918
|
+
"corrupt"
|
|
2919
|
+
);
|
|
2920
|
+
}
|
|
2921
|
+
if (size !== info.sizeBytes) {
|
|
2922
|
+
throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
|
|
2923
|
+
}
|
|
2924
|
+
return { ...info, absPath };
|
|
2925
|
+
}
|
|
2717
2926
|
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
2718
2927
|
readAsset(path) {
|
|
2719
2928
|
const info = this.assetIndex().get(path);
|
|
2720
2929
|
if (!info) return null;
|
|
2721
|
-
if (info.sizeBytes >
|
|
2930
|
+
if (info.sizeBytes > maxKnowledgeAssetBytes2(path)) {
|
|
2722
2931
|
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
2723
2932
|
}
|
|
2724
2933
|
let bytes;
|
|
@@ -2811,6 +3020,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2811
3020
|
* takes the top-S GLOBALLY, so multiple catalogs share one scan budget.
|
|
2812
3021
|
*/
|
|
2813
3022
|
scoreShards(queryVector) {
|
|
3023
|
+
this.assertQueryWidth(queryVector);
|
|
2814
3024
|
const rows = this.router.db.prepare("SELECT shard_id, embedding FROM route_centroids").all();
|
|
2815
3025
|
const best = /* @__PURE__ */ new Map();
|
|
2816
3026
|
for (const row of rows) {
|
|
@@ -2820,8 +3030,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2820
3030
|
Math.floor(row.embedding.byteLength / 4)
|
|
2821
3031
|
);
|
|
2822
3032
|
let dot = 0;
|
|
2823
|
-
|
|
2824
|
-
for (let i = 0; i < n; i++) dot += centroid[i] * queryVector[i];
|
|
3033
|
+
for (let i = 0; i < queryVector.length; i++) dot += centroid[i] * queryVector[i];
|
|
2825
3034
|
const id = Number(row.shard_id);
|
|
2826
3035
|
if (dot > (best.get(id) ?? Number.NEGATIVE_INFINITY)) best.set(id, dot);
|
|
2827
3036
|
}
|
|
@@ -2894,6 +3103,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2894
3103
|
* after global cross-catalog routing has already spent the S budget.
|
|
2895
3104
|
*/
|
|
2896
3105
|
searchShards(queryVector, shardIds, finalK, allowedDocumentIds) {
|
|
3106
|
+
this.assertQueryWidth(queryVector);
|
|
2897
3107
|
const center = this.binaryCenter();
|
|
2898
3108
|
const scanQuery = center ? centerVector(queryVector, center) : queryVector;
|
|
2899
3109
|
const hits = [];
|
|
@@ -2906,12 +3116,18 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2906
3116
|
"SELECT id FROM chunks WHERE document_id IN (SELECT value FROM json_each(?))"
|
|
2907
3117
|
).all(JSON.stringify([...allowedDocumentIds])).map((row) => Number(row.id))
|
|
2908
3118
|
) : void 0;
|
|
2909
|
-
const
|
|
2910
|
-
const
|
|
2911
|
-
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
3119
|
+
const media = this.shardMedia(shard);
|
|
3120
|
+
const k = rerankK(finalK, eligible?.size ?? shard.chunkCount - media.ids.length);
|
|
3121
|
+
const excluded = media.ids.length > 0 ? media.idSet : void 0;
|
|
3122
|
+
const candidates = asymmetricTopK(
|
|
3123
|
+
this.shardBits(shard),
|
|
3124
|
+
scanQuery,
|
|
3125
|
+
k,
|
|
3126
|
+
eligible,
|
|
3127
|
+
excluded
|
|
3128
|
+
).map((hit) => ({
|
|
3129
|
+
chunk_id: hit.chunkId
|
|
3130
|
+
}));
|
|
2915
3131
|
if (candidates.length === 0) continue;
|
|
2916
3132
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
2917
3133
|
const reranked = candidates.map((c) => {
|
|
@@ -2920,25 +3136,11 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2920
3136
|
const int8 = new Int8Array(row.v.buffer, row.v.byteOffset, row.v.byteLength);
|
|
2921
3137
|
return { chunkId: Number(c.chunk_id), cosine: rerankScore(queryVector, int8) };
|
|
2922
3138
|
}).filter((r) => r !== null).sort((a, b) => b.cosine - a.cosine).slice(0, finalK);
|
|
2923
|
-
const getChunk = db.prepare(
|
|
2924
|
-
`SELECT chunk_uid, document_id, title, heading_path, line_start, line_end, text
|
|
2925
|
-
FROM chunks WHERE id = ?`
|
|
2926
|
-
);
|
|
3139
|
+
const getChunk = db.prepare(`SELECT ${this.chunkColumns()} FROM chunks WHERE id = ?`);
|
|
2927
3140
|
for (const r of reranked) {
|
|
2928
3141
|
const row = getChunk.get(BigInt(r.chunkId));
|
|
2929
3142
|
if (!row) continue;
|
|
2930
|
-
hits.push({
|
|
2931
|
-
chunkUid: row.chunk_uid,
|
|
2932
|
-
documentId: row.document_id,
|
|
2933
|
-
title: row.title,
|
|
2934
|
-
headingPath: parseHeadingPath(row.heading_path),
|
|
2935
|
-
lineStart: Number(row.line_start),
|
|
2936
|
-
lineEnd: Number(row.line_end),
|
|
2937
|
-
text: row.text,
|
|
2938
|
-
cosine: r.cosine,
|
|
2939
|
-
source: "vector",
|
|
2940
|
-
shardId
|
|
2941
|
-
});
|
|
3143
|
+
hits.push({ ...this.hitFromRow(row, shardId), cosine: r.cosine, source: "vector" });
|
|
2942
3144
|
}
|
|
2943
3145
|
}
|
|
2944
3146
|
hits.sort((a, b) => (b.cosine ?? 0) - (a.cosine ?? 0));
|
|
@@ -2959,24 +3161,14 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
2959
3161
|
);
|
|
2960
3162
|
if (!match) return hits;
|
|
2961
3163
|
const rows = db.prepare(
|
|
2962
|
-
`SELECT
|
|
3164
|
+
`SELECT ${this.chunkColumns().split(", ").map((column) => `c.${column}`).join(", ")}
|
|
2963
3165
|
FROM fts_chunks f JOIN chunks c ON c.id = f.rowid
|
|
2964
3166
|
WHERE fts_chunks MATCH ? ${allowedDocumentIds ? "AND c.document_id IN (SELECT value FROM json_each(?))" : ""} ORDER BY f.rank LIMIT ?`
|
|
2965
3167
|
).all(
|
|
2966
3168
|
...allowedDocumentIds ? [match, JSON.stringify([...allowedDocumentIds]), limitPerShard] : [match, limitPerShard]
|
|
2967
3169
|
);
|
|
2968
3170
|
for (const row of rows) {
|
|
2969
|
-
hits.push({
|
|
2970
|
-
chunkUid: row.chunk_uid,
|
|
2971
|
-
documentId: row.document_id,
|
|
2972
|
-
title: row.title,
|
|
2973
|
-
headingPath: parseHeadingPath(row.heading_path),
|
|
2974
|
-
lineStart: Number(row.line_start),
|
|
2975
|
-
lineEnd: Number(row.line_end),
|
|
2976
|
-
text: row.text,
|
|
2977
|
-
source: "fts",
|
|
2978
|
-
shardId
|
|
2979
|
-
});
|
|
3171
|
+
hits.push({ ...this.hitFromRow(row, shardId), source: "fts" });
|
|
2980
3172
|
}
|
|
2981
3173
|
} catch {
|
|
2982
3174
|
}
|
|
@@ -3050,17 +3242,18 @@ function parseHeadingPath(raw) {
|
|
|
3050
3242
|
}
|
|
3051
3243
|
|
|
3052
3244
|
// src/reader/validate.ts
|
|
3053
|
-
import { readFile } from "fs/promises";
|
|
3245
|
+
import { open, readFile } from "fs/promises";
|
|
3054
3246
|
import { join as join3 } from "path";
|
|
3055
3247
|
import {
|
|
3056
3248
|
KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
|
|
3057
|
-
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
3058
|
-
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
|
|
3059
|
-
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
3060
3249
|
assetExtension as assetExtension2,
|
|
3061
3250
|
assetKindForExtension as assetKindForExtension2,
|
|
3251
|
+
assetModality as assetModality2,
|
|
3062
3252
|
canonicalizeJson as canonicalizeJson2,
|
|
3253
|
+
formatAtLeast as formatAtLeast2,
|
|
3063
3254
|
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
3255
|
+
isKnowledgeMediaAssetPath as isKnowledgeMediaAssetPath2,
|
|
3256
|
+
knowledgeAssetLimitsProblem as knowledgeAssetLimitsProblem2,
|
|
3064
3257
|
sniffAssetType as sniffAssetType2,
|
|
3065
3258
|
svgInertnessProblem as svgInertnessProblem2
|
|
3066
3259
|
} from "@bendyline/gezk";
|
|
@@ -3135,7 +3328,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3135
3328
|
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
3136
3329
|
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
3137
3330
|
);
|
|
3138
|
-
if (manifest.formatVersion
|
|
3331
|
+
if (formatAtLeast2(manifest.formatVersion, "0.7")) {
|
|
3139
3332
|
try {
|
|
3140
3333
|
const spatial = handle.spatialIntegrity();
|
|
3141
3334
|
check("document-locations-table", spatial.hasTable);
|
|
@@ -3171,13 +3364,16 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3171
3364
|
} else {
|
|
3172
3365
|
const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
|
|
3173
3366
|
check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
|
|
3174
|
-
|
|
3175
|
-
|
|
3176
|
-
|
|
3177
|
-
|
|
3178
|
-
|
|
3179
|
-
|
|
3180
|
-
|
|
3367
|
+
if (!formatAtLeast2(manifest.formatVersion, "0.8")) {
|
|
3368
|
+
const media = assetFiles.filter((f) => isKnowledgeMediaAssetPath2(f.path));
|
|
3369
|
+
check(
|
|
3370
|
+
"assets-media-in-0.8",
|
|
3371
|
+
media.length === 0,
|
|
3372
|
+
`audio/video assets need format 0.8: ${media.map((f) => f.path).join(", ")}`
|
|
3373
|
+
);
|
|
3374
|
+
}
|
|
3375
|
+
const limits = knowledgeAssetLimitsProblem2(assetFiles);
|
|
3376
|
+
check("assets-limits", limits === null, limits ?? "");
|
|
3181
3377
|
check(
|
|
3182
3378
|
"counts-assets",
|
|
3183
3379
|
(manifest.counts.assets ?? 0) === assetFiles.length,
|
|
@@ -3236,6 +3432,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3236
3432
|
bodies.maxCompressedBytes <= MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024,
|
|
3237
3433
|
`largest compressed body is ${bodies.maxCompressedBytes} bytes`
|
|
3238
3434
|
);
|
|
3435
|
+
const mediaRowCounts = { image: 0, video: 0, audio: 0 };
|
|
3239
3436
|
for (const shard of handle.shards) {
|
|
3240
3437
|
const conn = openCatalogDatabase(handle.resolveCatalogPath(shard.path));
|
|
3241
3438
|
const db = conn.db;
|
|
@@ -3272,6 +3469,21 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3272
3469
|
badBit === 0 && badInt8 === 0,
|
|
3273
3470
|
`${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
|
|
3274
3471
|
);
|
|
3472
|
+
if (handle.schemaVersion >= 5) {
|
|
3473
|
+
const rows = db.prepare(
|
|
3474
|
+
`SELECT modality, asset_path, thumbnail_path FROM chunks WHERE modality <> 'text'`
|
|
3475
|
+
).all();
|
|
3476
|
+
const declaredAssets = new Set(assetFiles.map((f) => f.path));
|
|
3477
|
+
const wrong = rows.filter(
|
|
3478
|
+
(r) => !declaredAssets.has(r.asset_path) || assetModality2(r.asset_path) !== r.modality || r.thumbnail_path !== null && (!declaredAssets.has(r.thumbnail_path) || assetModality2(r.thumbnail_path) !== "image")
|
|
3479
|
+
).slice(0, 5).map((r) => `${r.modality} ${r.asset_path}`);
|
|
3480
|
+
check(
|
|
3481
|
+
`media-rows:${shard.path}`,
|
|
3482
|
+
wrong.length === 0,
|
|
3483
|
+
`media rows naming an undeclared or mismatched asset: ${wrong.join(", ")}`
|
|
3484
|
+
);
|
|
3485
|
+
for (const r of rows) mediaRowCounts[r.modality] += 1;
|
|
3486
|
+
}
|
|
3275
3487
|
const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
|
|
3276
3488
|
check(
|
|
3277
3489
|
`chunk-ids-dense:${shard.path}`,
|
|
@@ -3283,6 +3495,14 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3283
3495
|
}
|
|
3284
3496
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
3285
3497
|
}
|
|
3498
|
+
if (handle.schemaVersion >= 5) {
|
|
3499
|
+
const counted = manifest.counts.media ?? { image: 0, video: 0, audio: 0 };
|
|
3500
|
+
check(
|
|
3501
|
+
"counts-media",
|
|
3502
|
+
counted.image === mediaRowCounts.image && counted.video === mediaRowCounts.video && counted.audio === mediaRowCounts.audio,
|
|
3503
|
+
`manifest counts ${JSON.stringify(counted)}, shards hold ${JSON.stringify(mediaRowCounts)}`
|
|
3504
|
+
);
|
|
3505
|
+
}
|
|
3286
3506
|
if (handle.schemaVersion >= 3) {
|
|
3287
3507
|
const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
|
|
3288
3508
|
check(
|
|
@@ -3294,7 +3514,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
3294
3514
|
for (const file of assetFiles) {
|
|
3295
3515
|
const ext = assetExtension2(file.path);
|
|
3296
3516
|
if (!ext) continue;
|
|
3297
|
-
const bytes = await readFile(join3(rootDir, file.path));
|
|
3517
|
+
const bytes = isKnowledgeMediaAssetPath2(file.path) ? await readLeadingBytes(join3(rootDir, file.path), 64) : await readFile(join3(rootDir, file.path));
|
|
3298
3518
|
const kind = sniffAssetType2(bytes);
|
|
3299
3519
|
check(
|
|
3300
3520
|
`asset-type:${file.path}`,
|
|
@@ -3372,6 +3592,16 @@ function topicTreeProblem(topics) {
|
|
|
3372
3592
|
}
|
|
3373
3593
|
return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
|
|
3374
3594
|
}
|
|
3595
|
+
async function readLeadingBytes(path, length) {
|
|
3596
|
+
const handle = await open(path, "r");
|
|
3597
|
+
try {
|
|
3598
|
+
const buffer = new Uint8Array(length);
|
|
3599
|
+
const { bytesRead } = await handle.read(buffer, 0, length, 0);
|
|
3600
|
+
return buffer.subarray(0, bytesRead);
|
|
3601
|
+
} finally {
|
|
3602
|
+
await handle.close();
|
|
3603
|
+
}
|
|
3604
|
+
}
|
|
3375
3605
|
|
|
3376
3606
|
// src/export/duckdb.ts
|
|
3377
3607
|
import { spawn } from "child_process";
|
|
@@ -3442,7 +3672,7 @@ function runDuckdbProcess(binaryPath, args, stdin) {
|
|
|
3442
3672
|
}
|
|
3443
3673
|
|
|
3444
3674
|
// src/export/parquet.ts
|
|
3445
|
-
import { createHash as
|
|
3675
|
+
import { createHash as createHash5 } from "crypto";
|
|
3446
3676
|
import { createWriteStream as createWriteStream3 } from "fs";
|
|
3447
3677
|
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
3448
3678
|
import { join as join4, resolve as resolve3 } from "path";
|
|
@@ -3865,7 +4095,7 @@ async function writeParquet(cli, spec) {
|
|
|
3865
4095
|
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
3866
4096
|
rows: spec.rows,
|
|
3867
4097
|
sizeBytes: (await stat2(target)).size,
|
|
3868
|
-
sha256:
|
|
4098
|
+
sha256: createHash5("sha256").update(bytes).digest("hex")
|
|
3869
4099
|
};
|
|
3870
4100
|
}
|
|
3871
4101
|
async function findDuckdbBinary(env = process.env) {
|
|
@@ -4282,6 +4512,522 @@ var MULTILINGUAL_E5_SMALL_CENTER = [
|
|
|
4282
4512
|
0.049673
|
|
4283
4513
|
];
|
|
4284
4514
|
|
|
4515
|
+
// src/profiles/embeddinggemma-2-center.ts
|
|
4516
|
+
var EMBEDDINGGEMMA_2_512_CENTER = [
|
|
4517
|
+
-3987e-6,
|
|
4518
|
+
0.017212,
|
|
4519
|
+
0.023034,
|
|
4520
|
+
-0.020831,
|
|
4521
|
+
0.013136,
|
|
4522
|
+
0.018836,
|
|
4523
|
+
7334e-6,
|
|
4524
|
+
-0.022294,
|
|
4525
|
+
5809e-6,
|
|
4526
|
+
-0.147176,
|
|
4527
|
+
-5646e-6,
|
|
4528
|
+
-0.022884,
|
|
4529
|
+
-0.011497,
|
|
4530
|
+
1292e-6,
|
|
4531
|
+
0.014257,
|
|
4532
|
+
-1878e-6,
|
|
4533
|
+
0.051834,
|
|
4534
|
+
-0.012991,
|
|
4535
|
+
933e-6,
|
|
4536
|
+
0.022558,
|
|
4537
|
+
-0.034902,
|
|
4538
|
+
-0.030019,
|
|
4539
|
+
0.017879,
|
|
4540
|
+
0.015619,
|
|
4541
|
+
-0.014837,
|
|
4542
|
+
0.028657,
|
|
4543
|
+
0.017575,
|
|
4544
|
+
-0.045617,
|
|
4545
|
+
-0.03244,
|
|
4546
|
+
0.23277,
|
|
4547
|
+
-0.021971,
|
|
4548
|
+
-0.026583,
|
|
4549
|
+
0.011485,
|
|
4550
|
+
-0.01468,
|
|
4551
|
+
0.044649,
|
|
4552
|
+
0.018019,
|
|
4553
|
+
-0.011519,
|
|
4554
|
+
0.035574,
|
|
4555
|
+
512e-5,
|
|
4556
|
+
-0.030136,
|
|
4557
|
+
4583e-6,
|
|
4558
|
+
0.018314,
|
|
4559
|
+
-0.028859,
|
|
4560
|
+
0.027199,
|
|
4561
|
+
0.020695,
|
|
4562
|
+
-849e-6,
|
|
4563
|
+
0.015867,
|
|
4564
|
+
-0.024724,
|
|
4565
|
+
-8979e-6,
|
|
4566
|
+
0.023625,
|
|
4567
|
+
0.028948,
|
|
4568
|
+
-0.025744,
|
|
4569
|
+
-1547e-6,
|
|
4570
|
+
0.0281,
|
|
4571
|
+
0.024258,
|
|
4572
|
+
0.02262,
|
|
4573
|
+
1386e-6,
|
|
4574
|
+
0.015002,
|
|
4575
|
+
3764e-6,
|
|
4576
|
+
0.016978,
|
|
4577
|
+
0.019178,
|
|
4578
|
+
0.012229,
|
|
4579
|
+
-193e-6,
|
|
4580
|
+
0.032654,
|
|
4581
|
+
-0.030602,
|
|
4582
|
+
-1749e-6,
|
|
4583
|
+
0.027403,
|
|
4584
|
+
-4047e-6,
|
|
4585
|
+
0.031605,
|
|
4586
|
+
0.023884,
|
|
4587
|
+
0.015804,
|
|
4588
|
+
3971e-6,
|
|
4589
|
+
0.021826,
|
|
4590
|
+
0.02457,
|
|
4591
|
+
-0.023111,
|
|
4592
|
+
-0.032593,
|
|
4593
|
+
-0.090226,
|
|
4594
|
+
0.020798,
|
|
4595
|
+
6402e-6,
|
|
4596
|
+
0.015607,
|
|
4597
|
+
-8457e-6,
|
|
4598
|
+
-0.010067,
|
|
4599
|
+
6362e-6,
|
|
4600
|
+
9973e-6,
|
|
4601
|
+
0.0122,
|
|
4602
|
+
-0.020958,
|
|
4603
|
+
-0.031666,
|
|
4604
|
+
0.034654,
|
|
4605
|
+
-0.012847,
|
|
4606
|
+
-1119e-6,
|
|
4607
|
+
0.024259,
|
|
4608
|
+
0.018097,
|
|
4609
|
+
9105e-6,
|
|
4610
|
+
-9523e-6,
|
|
4611
|
+
-0.022682,
|
|
4612
|
+
-5167e-6,
|
|
4613
|
+
-0.022959,
|
|
4614
|
+
0.012175,
|
|
4615
|
+
-0.020933,
|
|
4616
|
+
0.014024,
|
|
4617
|
+
-4898e-6,
|
|
4618
|
+
-2018e-6,
|
|
4619
|
+
-7178e-6,
|
|
4620
|
+
0.038098,
|
|
4621
|
+
0.016307,
|
|
4622
|
+
5668e-6,
|
|
4623
|
+
-0.014475,
|
|
4624
|
+
661e-6,
|
|
4625
|
+
-1287e-6,
|
|
4626
|
+
-0.025024,
|
|
4627
|
+
5704e-6,
|
|
4628
|
+
-0.32188,
|
|
4629
|
+
-0.026611,
|
|
4630
|
+
0.032111,
|
|
4631
|
+
-9511e-6,
|
|
4632
|
+
0.260031,
|
|
4633
|
+
0.027274,
|
|
4634
|
+
-909e-5,
|
|
4635
|
+
0.055461,
|
|
4636
|
+
0.024395,
|
|
4637
|
+
-0.017894,
|
|
4638
|
+
0.061775,
|
|
4639
|
+
-8529e-6,
|
|
4640
|
+
-1795e-6,
|
|
4641
|
+
-0.017545,
|
|
4642
|
+
0.011492,
|
|
4643
|
+
0.044594,
|
|
4644
|
+
38e-4,
|
|
4645
|
+
-0.02231,
|
|
4646
|
+
0.029977,
|
|
4647
|
+
0.010629,
|
|
4648
|
+
-489e-5,
|
|
4649
|
+
2646e-6,
|
|
4650
|
+
6408e-6,
|
|
4651
|
+
-0.014453,
|
|
4652
|
+
-0.020764,
|
|
4653
|
+
-0.018043,
|
|
4654
|
+
-0.015625,
|
|
4655
|
+
0.074213,
|
|
4656
|
+
-0.012319,
|
|
4657
|
+
-8038e-6,
|
|
4658
|
+
-6265e-6,
|
|
4659
|
+
0.012548,
|
|
4660
|
+
-621e-6,
|
|
4661
|
+
-0.021893,
|
|
4662
|
+
0.039889,
|
|
4663
|
+
3623e-6,
|
|
4664
|
+
-0.029173,
|
|
4665
|
+
-716e-6,
|
|
4666
|
+
-0.013168,
|
|
4667
|
+
-6674e-6,
|
|
4668
|
+
-0.010069,
|
|
4669
|
+
-0.02451,
|
|
4670
|
+
0.015114,
|
|
4671
|
+
-0.012945,
|
|
4672
|
+
1561e-6,
|
|
4673
|
+
0.013299,
|
|
4674
|
+
-0.029098,
|
|
4675
|
+
0.01304,
|
|
4676
|
+
0.013519,
|
|
4677
|
+
-0.014282,
|
|
4678
|
+
0.026849,
|
|
4679
|
+
-2462e-6,
|
|
4680
|
+
0.019473,
|
|
4681
|
+
-8227e-6,
|
|
4682
|
+
-822e-6,
|
|
4683
|
+
-7241e-6,
|
|
4684
|
+
4259e-6,
|
|
4685
|
+
-0.016026,
|
|
4686
|
+
7987e-6,
|
|
4687
|
+
-7065e-6,
|
|
4688
|
+
-1229e-6,
|
|
4689
|
+
-0.01583,
|
|
4690
|
+
0.021277,
|
|
4691
|
+
0.013837,
|
|
4692
|
+
0.024282,
|
|
4693
|
+
-7589e-6,
|
|
4694
|
+
-0.016577,
|
|
4695
|
+
7774e-6,
|
|
4696
|
+
0.010479,
|
|
4697
|
+
-0.012539,
|
|
4698
|
+
-0.020799,
|
|
4699
|
+
0.026506,
|
|
4700
|
+
8733e-6,
|
|
4701
|
+
-0.015001,
|
|
4702
|
+
2151e-6,
|
|
4703
|
+
-0.011922,
|
|
4704
|
+
-0.02785,
|
|
4705
|
+
-0.060631,
|
|
4706
|
+
3372e-6,
|
|
4707
|
+
0.022441,
|
|
4708
|
+
-0.012448,
|
|
4709
|
+
6827e-6,
|
|
4710
|
+
-7047e-6,
|
|
4711
|
+
1249e-6,
|
|
4712
|
+
-2934e-6,
|
|
4713
|
+
4559e-6,
|
|
4714
|
+
-0.011409,
|
|
4715
|
+
-0.012564,
|
|
4716
|
+
5788e-6,
|
|
4717
|
+
-0.020554,
|
|
4718
|
+
-4292e-6,
|
|
4719
|
+
0.01927,
|
|
4720
|
+
0.013487,
|
|
4721
|
+
-0.015484,
|
|
4722
|
+
9704e-6,
|
|
4723
|
+
5617e-6,
|
|
4724
|
+
0.02447,
|
|
4725
|
+
2056e-6,
|
|
4726
|
+
0.011381,
|
|
4727
|
+
0.039831,
|
|
4728
|
+
-7878e-6,
|
|
4729
|
+
-0.011366,
|
|
4730
|
+
-814e-5,
|
|
4731
|
+
8099e-6,
|
|
4732
|
+
6637e-6,
|
|
4733
|
+
154e-6,
|
|
4734
|
+
0.072273,
|
|
4735
|
+
7195e-6,
|
|
4736
|
+
-0.011899,
|
|
4737
|
+
0.016669,
|
|
4738
|
+
3331e-6,
|
|
4739
|
+
0.032787,
|
|
4740
|
+
-0.012251,
|
|
4741
|
+
-6049e-6,
|
|
4742
|
+
-0.017814,
|
|
4743
|
+
0.055136,
|
|
4744
|
+
0.028157,
|
|
4745
|
+
-0.028898,
|
|
4746
|
+
0.035646,
|
|
4747
|
+
0.036174,
|
|
4748
|
+
-5621e-6,
|
|
4749
|
+
0.050018,
|
|
4750
|
+
-0.040307,
|
|
4751
|
+
0.060677,
|
|
4752
|
+
58e-6,
|
|
4753
|
+
-0.045134,
|
|
4754
|
+
4752e-6,
|
|
4755
|
+
7161e-6,
|
|
4756
|
+
0.040398,
|
|
4757
|
+
-0.051202,
|
|
4758
|
+
-572e-6,
|
|
4759
|
+
6186e-6,
|
|
4760
|
+
-0.010613,
|
|
4761
|
+
0.039894,
|
|
4762
|
+
-0.014924,
|
|
4763
|
+
49e-4,
|
|
4764
|
+
-0.011771,
|
|
4765
|
+
9397e-6,
|
|
4766
|
+
9457e-6,
|
|
4767
|
+
0.025576,
|
|
4768
|
+
2251e-6,
|
|
4769
|
+
0.020638,
|
|
4770
|
+
-0.029722,
|
|
4771
|
+
7545e-6,
|
|
4772
|
+
-533e-6,
|
|
4773
|
+
0.026445,
|
|
4774
|
+
921e-6,
|
|
4775
|
+
9681e-6,
|
|
4776
|
+
-959e-6,
|
|
4777
|
+
-0.018021,
|
|
4778
|
+
807e-5,
|
|
4779
|
+
8041e-6,
|
|
4780
|
+
-8004e-6,
|
|
4781
|
+
4422e-6,
|
|
4782
|
+
-2478e-6,
|
|
4783
|
+
7376e-6,
|
|
4784
|
+
-828e-5,
|
|
4785
|
+
8006e-6,
|
|
4786
|
+
-3182e-6,
|
|
4787
|
+
5771e-6,
|
|
4788
|
+
-526e-5,
|
|
4789
|
+
0.01422,
|
|
4790
|
+
-3677e-6,
|
|
4791
|
+
-0.010823,
|
|
4792
|
+
0.019873,
|
|
4793
|
+
3328e-6,
|
|
4794
|
+
4427e-6,
|
|
4795
|
+
0.02227,
|
|
4796
|
+
-0.014147,
|
|
4797
|
+
0.014306,
|
|
4798
|
+
-5089e-6,
|
|
4799
|
+
-0.010849,
|
|
4800
|
+
322e-5,
|
|
4801
|
+
-8249e-6,
|
|
4802
|
+
0.019141,
|
|
4803
|
+
-0.012199,
|
|
4804
|
+
0.015451,
|
|
4805
|
+
0.026615,
|
|
4806
|
+
0.02352,
|
|
4807
|
+
0.011117,
|
|
4808
|
+
-399e-6,
|
|
4809
|
+
0.016821,
|
|
4810
|
+
0.023447,
|
|
4811
|
+
6638e-6,
|
|
4812
|
+
8598e-6,
|
|
4813
|
+
-0.02239,
|
|
4814
|
+
0.017325,
|
|
4815
|
+
0.022729,
|
|
4816
|
+
9922e-6,
|
|
4817
|
+
-4099e-6,
|
|
4818
|
+
9124e-6,
|
|
4819
|
+
-8221e-6,
|
|
4820
|
+
-7554e-6,
|
|
4821
|
+
0.046752,
|
|
4822
|
+
0.027253,
|
|
4823
|
+
437e-6,
|
|
4824
|
+
4693e-6,
|
|
4825
|
+
7093e-6,
|
|
4826
|
+
-0.014217,
|
|
4827
|
+
-0.024195,
|
|
4828
|
+
2768e-6,
|
|
4829
|
+
-0.012022,
|
|
4830
|
+
-0.012142,
|
|
4831
|
+
-4326e-6,
|
|
4832
|
+
-0.01054,
|
|
4833
|
+
0.01168,
|
|
4834
|
+
4821e-6,
|
|
4835
|
+
-5609e-6,
|
|
4836
|
+
-0.096205,
|
|
4837
|
+
-0.018675,
|
|
4838
|
+
-0.017776,
|
|
4839
|
+
-0.010107,
|
|
4840
|
+
-0.012825,
|
|
4841
|
+
-644e-6,
|
|
4842
|
+
0.011283,
|
|
4843
|
+
3759e-6,
|
|
4844
|
+
-0.025286,
|
|
4845
|
+
2306e-6,
|
|
4846
|
+
9735e-6,
|
|
4847
|
+
-158e-5,
|
|
4848
|
+
0.012131,
|
|
4849
|
+
-0.018451,
|
|
4850
|
+
0.014135,
|
|
4851
|
+
6421e-6,
|
|
4852
|
+
-7006e-6,
|
|
4853
|
+
-6122e-6,
|
|
4854
|
+
0.013727,
|
|
4855
|
+
8605e-6,
|
|
4856
|
+
-6385e-6,
|
|
4857
|
+
-0.016703,
|
|
4858
|
+
-0.184351,
|
|
4859
|
+
4162e-6,
|
|
4860
|
+
0.022412,
|
|
4861
|
+
-0.021583,
|
|
4862
|
+
-5397e-6,
|
|
4863
|
+
-0.011593,
|
|
4864
|
+
-1118e-6,
|
|
4865
|
+
0.013629,
|
|
4866
|
+
4367e-6,
|
|
4867
|
+
238e-5,
|
|
4868
|
+
0.030207,
|
|
4869
|
+
2765e-6,
|
|
4870
|
+
0.015452,
|
|
4871
|
+
-0.023998,
|
|
4872
|
+
0.016889,
|
|
4873
|
+
5828e-6,
|
|
4874
|
+
0.01799,
|
|
4875
|
+
-0.019521,
|
|
4876
|
+
0.012821,
|
|
4877
|
+
-0.021567,
|
|
4878
|
+
5999e-6,
|
|
4879
|
+
0.017122,
|
|
4880
|
+
-9796e-6,
|
|
4881
|
+
6654e-6,
|
|
4882
|
+
291e-5,
|
|
4883
|
+
-0.017359,
|
|
4884
|
+
9342e-6,
|
|
4885
|
+
-929e-6,
|
|
4886
|
+
-0.012544,
|
|
4887
|
+
-8234e-6,
|
|
4888
|
+
0.016922,
|
|
4889
|
+
0.018215,
|
|
4890
|
+
-758e-6,
|
|
4891
|
+
5614e-6,
|
|
4892
|
+
0.02137,
|
|
4893
|
+
7615e-6,
|
|
4894
|
+
0.010542,
|
|
4895
|
+
635e-6,
|
|
4896
|
+
0.023702,
|
|
4897
|
+
0.020834,
|
|
4898
|
+
0.013032,
|
|
4899
|
+
0.011873,
|
|
4900
|
+
-2792e-6,
|
|
4901
|
+
0.04144,
|
|
4902
|
+
-0.02252,
|
|
4903
|
+
0.027235,
|
|
4904
|
+
-0.012274,
|
|
4905
|
+
-4002e-6,
|
|
4906
|
+
0.022356,
|
|
4907
|
+
-0.020197,
|
|
4908
|
+
0.270597,
|
|
4909
|
+
-3966e-6,
|
|
4910
|
+
-4187e-6,
|
|
4911
|
+
-0.014922,
|
|
4912
|
+
-0.013515,
|
|
4913
|
+
-0.01073,
|
|
4914
|
+
5186e-6,
|
|
4915
|
+
-0.01013,
|
|
4916
|
+
-0.024027,
|
|
4917
|
+
-897e-6,
|
|
4918
|
+
0.023069,
|
|
4919
|
+
0.021741,
|
|
4920
|
+
573e-6,
|
|
4921
|
+
-7467e-6,
|
|
4922
|
+
823e-5,
|
|
4923
|
+
0.037651,
|
|
4924
|
+
-0.017797,
|
|
4925
|
+
-0.014499,
|
|
4926
|
+
2699e-6,
|
|
4927
|
+
-0.010982,
|
|
4928
|
+
-0.027587,
|
|
4929
|
+
0.028585,
|
|
4930
|
+
-6217e-6,
|
|
4931
|
+
-9743e-6,
|
|
4932
|
+
7437e-6,
|
|
4933
|
+
-0.023614,
|
|
4934
|
+
-0.023206,
|
|
4935
|
+
-0.029976,
|
|
4936
|
+
-903e-6,
|
|
4937
|
+
-5667e-6,
|
|
4938
|
+
-0.011557,
|
|
4939
|
+
0.011335,
|
|
4940
|
+
-0.019149,
|
|
4941
|
+
0.027405,
|
|
4942
|
+
-0.032284,
|
|
4943
|
+
-0.010479,
|
|
4944
|
+
-0.013504,
|
|
4945
|
+
0.013693,
|
|
4946
|
+
-6407e-6,
|
|
4947
|
+
8791e-6,
|
|
4948
|
+
-4461e-6,
|
|
4949
|
+
-2256e-6,
|
|
4950
|
+
-0.015897,
|
|
4951
|
+
8837e-6,
|
|
4952
|
+
-1417e-6,
|
|
4953
|
+
0.02557,
|
|
4954
|
+
-0.012118,
|
|
4955
|
+
-0.021653,
|
|
4956
|
+
3051e-6,
|
|
4957
|
+
-4168e-6,
|
|
4958
|
+
-0.01862,
|
|
4959
|
+
-9461e-6,
|
|
4960
|
+
-0.01088,
|
|
4961
|
+
-982e-5,
|
|
4962
|
+
-0.011997,
|
|
4963
|
+
-0.015663,
|
|
4964
|
+
-0.021258,
|
|
4965
|
+
5679e-6,
|
|
4966
|
+
0.012566,
|
|
4967
|
+
1521e-6,
|
|
4968
|
+
-8604e-6,
|
|
4969
|
+
0.012445,
|
|
4970
|
+
8421e-6,
|
|
4971
|
+
-0.033856,
|
|
4972
|
+
-0.019957,
|
|
4973
|
+
146e-6,
|
|
4974
|
+
-185e-6,
|
|
4975
|
+
781e-6,
|
|
4976
|
+
0.018164,
|
|
4977
|
+
4102e-6,
|
|
4978
|
+
-0.022985,
|
|
4979
|
+
-8788e-6,
|
|
4980
|
+
-4097e-6,
|
|
4981
|
+
0.02217,
|
|
4982
|
+
-0.019959,
|
|
4983
|
+
-0.011669,
|
|
4984
|
+
5313e-6,
|
|
4985
|
+
381e-6,
|
|
4986
|
+
3747e-6,
|
|
4987
|
+
-0.036681,
|
|
4988
|
+
-6177e-6,
|
|
4989
|
+
-0.014834,
|
|
4990
|
+
-0.031706,
|
|
4991
|
+
0.010642,
|
|
4992
|
+
1258e-6,
|
|
4993
|
+
0.01296,
|
|
4994
|
+
0.016435,
|
|
4995
|
+
-8722e-6,
|
|
4996
|
+
-0.010652,
|
|
4997
|
+
9787e-6,
|
|
4998
|
+
-474e-5,
|
|
4999
|
+
0.011086,
|
|
5000
|
+
-0.020615,
|
|
5001
|
+
0.021049,
|
|
5002
|
+
-468e-5,
|
|
5003
|
+
8645e-6,
|
|
5004
|
+
-9937e-6,
|
|
5005
|
+
216e-5,
|
|
5006
|
+
0.010686,
|
|
5007
|
+
-6216e-6,
|
|
5008
|
+
-0.011397,
|
|
5009
|
+
-5593e-6,
|
|
5010
|
+
9687e-6,
|
|
5011
|
+
0.017522,
|
|
5012
|
+
-6329e-6,
|
|
5013
|
+
9519e-6,
|
|
5014
|
+
-0.022554,
|
|
5015
|
+
-0.02169,
|
|
5016
|
+
3307e-6,
|
|
5017
|
+
-4782e-6,
|
|
5018
|
+
1506e-6,
|
|
5019
|
+
1607e-6,
|
|
5020
|
+
-3791e-6,
|
|
5021
|
+
-3601e-6,
|
|
5022
|
+
0.030061,
|
|
5023
|
+
-468e-5,
|
|
5024
|
+
-0.186431,
|
|
5025
|
+
-1492e-6,
|
|
5026
|
+
0.02242,
|
|
5027
|
+
-8878e-6,
|
|
5028
|
+
0.01108
|
|
5029
|
+
];
|
|
5030
|
+
|
|
4285
5031
|
// src/profiles/registry.ts
|
|
4286
5032
|
var SHARED_ENCODING = {
|
|
4287
5033
|
pooling: "mean",
|
|
@@ -4342,7 +5088,94 @@ var BGE_SMALL_EN_V15_1 = {
|
|
|
4342
5088
|
passageInstruction: "",
|
|
4343
5089
|
...SHARED_ENCODING
|
|
4344
5090
|
};
|
|
5091
|
+
var GEMMA2_REPO = "onnx-community/embeddinggemma-2-ONNX";
|
|
5092
|
+
var GEMMA2_PROCESSOR_CONFIG = {
|
|
5093
|
+
path: "processor_config.json",
|
|
5094
|
+
digest: "sha256:168f6a08522f3ce5dea596d94d003af2fd691742d4f41fe1f9d8cce76bfbf69c"
|
|
5095
|
+
};
|
|
5096
|
+
var EMBEDDINGGEMMA_2_512_1 = {
|
|
5097
|
+
id: "embeddinggemma-2-512@1",
|
|
5098
|
+
model: {
|
|
5099
|
+
repo: GEMMA2_REPO,
|
|
5100
|
+
revision: "daa72c51243991dfcaf9f9137d2c573d8f7790c0",
|
|
5101
|
+
onnxFile: "onnx/model_quantized.onnx",
|
|
5102
|
+
onnxDigest: "sha256:d06edd601f851c633a2519304cbeb8dc6170d7ceb61b436625c17fb9b6e74953",
|
|
5103
|
+
files: [
|
|
5104
|
+
{
|
|
5105
|
+
path: "onnx/model_quantized.onnx_data",
|
|
5106
|
+
digest: "sha256:278a7ff1248c3618e4bd11a607fc54f7bdc7778854230f3956d3f86bd9db4f3b"
|
|
5107
|
+
},
|
|
5108
|
+
{
|
|
5109
|
+
path: "config.json",
|
|
5110
|
+
digest: "sha256:8d011bfe08b5e345bbe0b81e5c6fd02c381920b345b986047bc2a33ce7b90d1d"
|
|
5111
|
+
},
|
|
5112
|
+
{
|
|
5113
|
+
path: "tokenizer_config.json",
|
|
5114
|
+
digest: "sha256:17bd5d6e9364ca49a534e1502076593317c298d4a663623091ed45388f004874"
|
|
5115
|
+
}
|
|
5116
|
+
]
|
|
5117
|
+
},
|
|
5118
|
+
tokenizer: {
|
|
5119
|
+
kind: "gemma-bpe",
|
|
5120
|
+
file: "tokenizer.json",
|
|
5121
|
+
digest: "sha256:4d777ef5bdc1aa36227abdfb77c3e49e7b9c892d16e1b6bda41c393504828be4"
|
|
5122
|
+
},
|
|
5123
|
+
pooling: "mean",
|
|
5124
|
+
normalized: true,
|
|
5125
|
+
dimensions: 512,
|
|
5126
|
+
truncation: { method: "prefix", sourceDimensions: 768 },
|
|
5127
|
+
maxTokens: 512,
|
|
5128
|
+
queryInstruction: "task: search result | query: ",
|
|
5129
|
+
passageInstruction: "title: none | text: ",
|
|
5130
|
+
vectorEncoding: "bit+int8",
|
|
5131
|
+
distance: { stage1: "hamming", stage2: "cosine" },
|
|
5132
|
+
quantization: {
|
|
5133
|
+
int8: { method: "symmetric-linear", scale: 127 },
|
|
5134
|
+
binary: {
|
|
5135
|
+
method: "centered-sign",
|
|
5136
|
+
threshold: 0,
|
|
5137
|
+
packing: "lsb-first",
|
|
5138
|
+
center: [...EMBEDDINGGEMMA_2_512_CENTER]
|
|
5139
|
+
}
|
|
5140
|
+
},
|
|
5141
|
+
media: {
|
|
5142
|
+
image: {
|
|
5143
|
+
encoder: {
|
|
5144
|
+
onnxFile: "onnx/vision_encoder_quantized.onnx",
|
|
5145
|
+
onnxDigest: "sha256:bb0de2df53a2448a32dc7908a187c168c8afd514d4d6f674f7f46024875fa4e3",
|
|
5146
|
+
files: [
|
|
5147
|
+
{
|
|
5148
|
+
path: "onnx/vision_encoder_quantized.onnx_data",
|
|
5149
|
+
digest: "sha256:3dabd69c0a36e9a8771ad82030dde74daa5a0e02b7047a5d3f3382b1137bab89"
|
|
5150
|
+
},
|
|
5151
|
+
GEMMA2_PROCESSOR_CONFIG
|
|
5152
|
+
]
|
|
5153
|
+
},
|
|
5154
|
+
tokenBudget: 280,
|
|
5155
|
+
resample: "bicubic",
|
|
5156
|
+
alpha: "composite-white"
|
|
5157
|
+
},
|
|
5158
|
+
video: { framesPerSecond: 1, maxFrames: 32, tokenBudgetPerFrame: 140 },
|
|
5159
|
+
audio: {
|
|
5160
|
+
encoder: {
|
|
5161
|
+
onnxFile: "onnx/audio_encoder_quantized.onnx",
|
|
5162
|
+
onnxDigest: "sha256:04a9a9094ba76fb169e4c69be45a4b621580188654d6be59eab860eadcd42d3c",
|
|
5163
|
+
files: [
|
|
5164
|
+
{
|
|
5165
|
+
path: "onnx/audio_encoder_quantized.onnx_data",
|
|
5166
|
+
digest: "sha256:aa6361d898e1f1d6303f8dd2b5fabf4fd3629e15fd709e9cb06ac5cb9416a030"
|
|
5167
|
+
},
|
|
5168
|
+
GEMMA2_PROCESSOR_CONFIG
|
|
5169
|
+
]
|
|
5170
|
+
},
|
|
5171
|
+
sampleRate: 16e3,
|
|
5172
|
+
channels: 1,
|
|
5173
|
+
maxWindowMs: 3e4
|
|
5174
|
+
}
|
|
5175
|
+
}
|
|
5176
|
+
};
|
|
4345
5177
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
5178
|
+
EMBEDDINGGEMMA_2_512_1,
|
|
4346
5179
|
MULTILINGUAL_E5_SMALL_2,
|
|
4347
5180
|
MULTILINGUAL_E5_SMALL_1,
|
|
4348
5181
|
BGE_SMALL_EN_V15_1
|
|
@@ -4363,10 +5196,13 @@ var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
|
4363
5196
|
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
4364
5197
|
|
|
4365
5198
|
// src/embedding/profile-embedder.ts
|
|
4366
|
-
import {
|
|
5199
|
+
import {
|
|
5200
|
+
embeddingProfileArtifacts as embeddingProfileArtifacts3,
|
|
5201
|
+
profileUnitVector as profileUnitVector2
|
|
5202
|
+
} from "@bendyline/gezk";
|
|
4367
5203
|
|
|
4368
5204
|
// src/embedding/artifact-verify.ts
|
|
4369
|
-
import { createHash as
|
|
5205
|
+
import { createHash as createHash6 } from "crypto";
|
|
4370
5206
|
import { createReadStream } from "fs";
|
|
4371
5207
|
import { stat as stat3 } from "fs/promises";
|
|
4372
5208
|
import { join as join5 } from "path";
|
|
@@ -4410,7 +5246,7 @@ async function fileDigest(path) {
|
|
|
4410
5246
|
const memo = digestMemo.get(key);
|
|
4411
5247
|
if (memo) return memo;
|
|
4412
5248
|
const digest = await new Promise((resolve5, reject) => {
|
|
4413
|
-
const hash =
|
|
5249
|
+
const hash = createHash6("sha256");
|
|
4414
5250
|
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
4415
5251
|
});
|
|
4416
5252
|
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
@@ -4420,6 +5256,45 @@ async function fileDigest(path) {
|
|
|
4420
5256
|
digestMemo.set(key, digest);
|
|
4421
5257
|
return digest;
|
|
4422
5258
|
}
|
|
5259
|
+
function mediaEncoderPins(profile, modalities) {
|
|
5260
|
+
const encoders = /* @__PURE__ */ new Set();
|
|
5261
|
+
const media = profile.media;
|
|
5262
|
+
for (const modality of modalities) {
|
|
5263
|
+
const encoder = modality === "audio" ? media?.audio?.encoder : media?.image?.encoder;
|
|
5264
|
+
if (encoder) encoders.add(encoder);
|
|
5265
|
+
}
|
|
5266
|
+
const pins = [];
|
|
5267
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5268
|
+
for (const encoder of encoders) {
|
|
5269
|
+
const files = [
|
|
5270
|
+
{ path: encoder.onnxFile, digest: encoder.onnxDigest },
|
|
5271
|
+
...encoder.files ?? []
|
|
5272
|
+
];
|
|
5273
|
+
for (const { path, digest } of files) {
|
|
5274
|
+
if (!digest || seen.has(path)) continue;
|
|
5275
|
+
seen.add(path);
|
|
5276
|
+
pins.push({ file: path, expected: digest });
|
|
5277
|
+
}
|
|
5278
|
+
}
|
|
5279
|
+
return pins;
|
|
5280
|
+
}
|
|
5281
|
+
async function verifyMediaArtifacts(profile, modalities, opts) {
|
|
5282
|
+
const pins = mediaEncoderPins(profile, modalities);
|
|
5283
|
+
if (pins.length === 0) return { status: "unpinned", checks: [] };
|
|
5284
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5285
|
+
const checks = [];
|
|
5286
|
+
for (const { file, expected } of pins) {
|
|
5287
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
|
|
5288
|
+
checks.push({ role: "media", file, path, expected, actual: await fileDigest(path) });
|
|
5289
|
+
}
|
|
5290
|
+
if (checks.some((c) => c.actual === null)) {
|
|
5291
|
+
throw new EmbedderArtifactError("missing", profile.id, checks);
|
|
5292
|
+
}
|
|
5293
|
+
if (checks.some((c) => c.actual !== c.expected)) {
|
|
5294
|
+
throw new EmbedderArtifactError("mismatch", profile.id, checks);
|
|
5295
|
+
}
|
|
5296
|
+
return { status: "verified", checks };
|
|
5297
|
+
}
|
|
4423
5298
|
async function verifyProfileArtifacts(profile, opts) {
|
|
4424
5299
|
const artifacts = embeddingProfileArtifacts(profile);
|
|
4425
5300
|
const revision = opts.revision ?? profile.model.revision;
|
|
@@ -4434,6 +5309,9 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
4434
5309
|
expected: artifacts.tokenizerDigest
|
|
4435
5310
|
});
|
|
4436
5311
|
}
|
|
5312
|
+
for (const extra of artifacts.files) {
|
|
5313
|
+
if (extra.digest) pinned.push({ role: "file", file: extra.path, expected: extra.digest });
|
|
5314
|
+
}
|
|
4437
5315
|
if (pinned.length === 0) return { status: "unpinned", checks: [] };
|
|
4438
5316
|
const checks = [];
|
|
4439
5317
|
for (const { role, file, expected } of pinned) {
|
|
@@ -4449,10 +5327,89 @@ async function verifyProfileArtifacts(profile, opts) {
|
|
|
4449
5327
|
return { status: "verified", checks };
|
|
4450
5328
|
}
|
|
4451
5329
|
|
|
5330
|
+
// src/embedding/pinned-files.ts
|
|
5331
|
+
import { createHash as createHash7 } from "crypto";
|
|
5332
|
+
import { createWriteStream as createWriteStream4 } from "fs";
|
|
5333
|
+
import { mkdir as mkdir4, rename, rm as rm2, stat as stat4 } from "fs/promises";
|
|
5334
|
+
import { dirname as dirname3 } from "path";
|
|
5335
|
+
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
5336
|
+
function isExternalDataFile(path) {
|
|
5337
|
+
return /\.onnx_data(_\d+)?$/.test(path);
|
|
5338
|
+
}
|
|
5339
|
+
function pinnedLoadFiles(profile, modalities = []) {
|
|
5340
|
+
const artifacts = embeddingProfileArtifacts2(profile);
|
|
5341
|
+
const files = [
|
|
5342
|
+
{
|
|
5343
|
+
path: artifacts.onnxFile,
|
|
5344
|
+
...artifacts.onnxDigest ? { digest: artifacts.onnxDigest } : {}
|
|
5345
|
+
},
|
|
5346
|
+
{
|
|
5347
|
+
path: artifacts.tokenizerFile,
|
|
5348
|
+
...artifacts.tokenizerDigest ? { digest: artifacts.tokenizerDigest } : {}
|
|
5349
|
+
},
|
|
5350
|
+
...artifacts.files.map((f) => ({ path: f.path, ...f.digest ? { digest: f.digest } : {} })),
|
|
5351
|
+
...mediaEncoderPins(profile, modalities).map((p) => ({ path: p.file, digest: p.expected }))
|
|
5352
|
+
];
|
|
5353
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5354
|
+
return files.filter((f) => !seen.has(f.path) && seen.add(f.path));
|
|
5355
|
+
}
|
|
5356
|
+
async function isFile(path) {
|
|
5357
|
+
return stat4(path).then((s) => s.isFile()).catch(() => false);
|
|
5358
|
+
}
|
|
5359
|
+
async function missingPinnedFiles(profile, opts) {
|
|
5360
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5361
|
+
const missing = [];
|
|
5362
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5363
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5364
|
+
if (!await isFile(path)) missing.push(file.path);
|
|
5365
|
+
}
|
|
5366
|
+
return missing;
|
|
5367
|
+
}
|
|
5368
|
+
async function prefetchExternalData(profile, opts) {
|
|
5369
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
5370
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
5371
|
+
const fetched = [];
|
|
5372
|
+
for (const file of pinnedLoadFiles(profile, opts.modalities)) {
|
|
5373
|
+
if (!isExternalDataFile(file.path)) continue;
|
|
5374
|
+
const dest = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file.path);
|
|
5375
|
+
if (await isFile(dest)) continue;
|
|
5376
|
+
const url = `https://huggingface.co/${profile.model.repo}/resolve/${revision}/${file.path}`;
|
|
5377
|
+
const res = await fetchImpl(url);
|
|
5378
|
+
if (!res.ok || !res.body) throw new Error(`${file.path}: download failed (HTTP ${res.status})`);
|
|
5379
|
+
await mkdir4(dirname3(dest), { recursive: true });
|
|
5380
|
+
const partial = `${dest}.partial`;
|
|
5381
|
+
const hash = createHash7("sha256");
|
|
5382
|
+
try {
|
|
5383
|
+
const out = createWriteStream4(partial);
|
|
5384
|
+
const reader = res.body.getReader();
|
|
5385
|
+
for (; ; ) {
|
|
5386
|
+
const { done, value } = await reader.read();
|
|
5387
|
+
if (done) break;
|
|
5388
|
+
hash.update(value);
|
|
5389
|
+
if (!out.write(value)) await new Promise((r) => out.once("drain", () => r()));
|
|
5390
|
+
}
|
|
5391
|
+
await new Promise((resolve5, reject) => {
|
|
5392
|
+
out.on("error", reject);
|
|
5393
|
+
out.end(() => resolve5());
|
|
5394
|
+
});
|
|
5395
|
+
const digest = `sha256:${hash.digest("hex")}`;
|
|
5396
|
+
if (file.digest && digest !== file.digest) {
|
|
5397
|
+
throw new Error(`${file.path}: downloaded ${digest}, profile pins ${file.digest}`);
|
|
5398
|
+
}
|
|
5399
|
+
await rename(partial, dest);
|
|
5400
|
+
} catch (err) {
|
|
5401
|
+
await rm2(partial, { force: true });
|
|
5402
|
+
throw err;
|
|
5403
|
+
}
|
|
5404
|
+
fetched.push(file.path);
|
|
5405
|
+
}
|
|
5406
|
+
return fetched;
|
|
5407
|
+
}
|
|
5408
|
+
|
|
4452
5409
|
// src/embedding/profile-embedder.ts
|
|
4453
5410
|
var MAX_BATCH = 8;
|
|
4454
5411
|
var MAX_CHARS = 8e3;
|
|
4455
|
-
var TRANSFORMERS_PEER_RANGE = "^3.
|
|
5412
|
+
var TRANSFORMERS_PEER_RANGE = "^4.3.1";
|
|
4456
5413
|
var EmbedderUnavailableError = class extends Error {
|
|
4457
5414
|
isActionable = true;
|
|
4458
5415
|
/** True when `@huggingface/transformers` could not be loaded at all, as opposed to a profile it cannot serve. */
|
|
@@ -4496,7 +5453,7 @@ var POOLING_BY_PROFILE = {
|
|
|
4496
5453
|
cls: "cls"
|
|
4497
5454
|
};
|
|
4498
5455
|
function resolveTransformersModelOptions(profile) {
|
|
4499
|
-
const { onnxFile, tokenizerFile } =
|
|
5456
|
+
const { onnxFile, tokenizerFile } = embeddingProfileArtifacts3(profile);
|
|
4500
5457
|
if (tokenizerFile !== "tokenizer.json") {
|
|
4501
5458
|
throw new EmbedderUnavailableError(
|
|
4502
5459
|
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
@@ -4521,6 +5478,41 @@ function transformersGraphOptions(onnxFile) {
|
|
|
4521
5478
|
if (modelFileName === "") return null;
|
|
4522
5479
|
return { dtype, subfolder, model_file_name: modelFileName };
|
|
4523
5480
|
}
|
|
5481
|
+
function isMultimodalEncoderConfig(config) {
|
|
5482
|
+
return Boolean(config.vision_config || config.audio_config);
|
|
5483
|
+
}
|
|
5484
|
+
function pinnedExternalDataChunks(profile) {
|
|
5485
|
+
const { onnxFile, files } = embeddingProfileArtifacts3(profile);
|
|
5486
|
+
const prefix = `${onnxFile}_data`;
|
|
5487
|
+
return files.filter((f) => f.path === prefix || /^_\d+$/.test(f.path.slice(prefix.length))).length;
|
|
5488
|
+
}
|
|
5489
|
+
function encoderRows(outputs, attentionMask) {
|
|
5490
|
+
const pooled = outputs.sentence_embedding;
|
|
5491
|
+
if (pooled) {
|
|
5492
|
+
const [batch2 = 0, width2 = 0] = pooled.dims;
|
|
5493
|
+
return Array.from(
|
|
5494
|
+
{ length: batch2 },
|
|
5495
|
+
(_, b) => Array.from({ length: width2 }, (_2, d) => pooled.data[b * width2 + d])
|
|
5496
|
+
);
|
|
5497
|
+
}
|
|
5498
|
+
const hidden = outputs.last_hidden_state;
|
|
5499
|
+
if (!hidden || !attentionMask) {
|
|
5500
|
+
throw new Error("the encoder returned neither sentence_embedding nor last_hidden_state");
|
|
5501
|
+
}
|
|
5502
|
+
const [batch = 0, tokens = 0, width = 0] = hidden.dims;
|
|
5503
|
+
return Array.from({ length: batch }, (_, b) => {
|
|
5504
|
+
const sum = new Array(width).fill(0);
|
|
5505
|
+
let count = 0;
|
|
5506
|
+
for (let t = 0; t < tokens; t++) {
|
|
5507
|
+
if (!Number(attentionMask.data[b * tokens + t])) continue;
|
|
5508
|
+
count++;
|
|
5509
|
+
const base = (b * tokens + t) * width;
|
|
5510
|
+
for (let d = 0; d < width; d++)
|
|
5511
|
+
sum[d] = sum[d] + hidden.data[base + d];
|
|
5512
|
+
}
|
|
5513
|
+
return sum.map((v) => v / Math.max(1, count));
|
|
5514
|
+
});
|
|
5515
|
+
}
|
|
4524
5516
|
async function loadTransformers() {
|
|
4525
5517
|
try {
|
|
4526
5518
|
const specifier = "@huggingface/transformers";
|
|
@@ -4536,6 +5528,20 @@ async function loadTransformers() {
|
|
|
4536
5528
|
throw err;
|
|
4537
5529
|
}
|
|
4538
5530
|
}
|
|
5531
|
+
async function preparePinnedLoad(profile, opts) {
|
|
5532
|
+
const files = pinnedLoadFiles(profile, opts.modalities);
|
|
5533
|
+
if (!files.some((f) => isExternalDataFile(f.path))) return;
|
|
5534
|
+
if (opts.localFilesOnly) {
|
|
5535
|
+
const missing = await missingPinnedFiles(profile, opts);
|
|
5536
|
+
if (missing.length > 0) {
|
|
5537
|
+
throw new EmbedderUnavailableError(
|
|
5538
|
+
`the model files for profile ${profile.id} are not all installed (missing: ${missing.join(", ")})`
|
|
5539
|
+
);
|
|
5540
|
+
}
|
|
5541
|
+
return;
|
|
5542
|
+
}
|
|
5543
|
+
await prefetchExternalData(profile, opts);
|
|
5544
|
+
}
|
|
4539
5545
|
async function createProfileEmbedder(profile, opts = {}) {
|
|
4540
5546
|
const pooling = POOLING_BY_PROFILE[profile.pooling];
|
|
4541
5547
|
if (!pooling) {
|
|
@@ -4552,16 +5558,47 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
4552
5558
|
transformers.env.allowRemoteModels = true;
|
|
4553
5559
|
}
|
|
4554
5560
|
const progress = opts.onDownloadProgress ? { progress_callback: aggregateDownloadProgress(opts.onDownloadProgress) } : {};
|
|
4555
|
-
const
|
|
4556
|
-
|
|
5561
|
+
const localOnly = opts.localFilesOnly ? { local_files_only: true } : {};
|
|
5562
|
+
if (!opts.transformers) {
|
|
5563
|
+
await preparePinnedLoad(profile, {
|
|
5564
|
+
cacheDir: cacheDir ?? transformers.env.cacheDir ?? "",
|
|
5565
|
+
...opts.localFilesOnly ? { localFilesOnly: true } : {}
|
|
5566
|
+
});
|
|
5567
|
+
}
|
|
5568
|
+
const sessionOptions = opts.sessionOptions ? { session_options: opts.sessionOptions } : {};
|
|
5569
|
+
const config = transformers.AutoConfig ? await transformers.AutoConfig.from_pretrained(profile.model.repo, {
|
|
5570
|
+
revision: modelOptions.revision,
|
|
5571
|
+
...localOnly,
|
|
5572
|
+
...progress
|
|
5573
|
+
}) : null;
|
|
5574
|
+
const multimodal = config !== null && isMultimodalEncoderConfig(config);
|
|
5575
|
+
if (multimodal && !transformers.AutoModel) {
|
|
5576
|
+
throw new EmbedderUnavailableError(
|
|
5577
|
+
`profile ${profile.id} is a multimodal encoder; this runtime cannot load its text session`
|
|
5578
|
+
);
|
|
5579
|
+
}
|
|
5580
|
+
const externalData = pinnedExternalDataChunks(profile);
|
|
5581
|
+
const [runner, tokenizer] = await Promise.all([
|
|
5582
|
+
multimodal ? transformers.AutoModel.from_pretrained(
|
|
5583
|
+
profile.model.repo,
|
|
5584
|
+
{
|
|
5585
|
+
...modelOptions,
|
|
5586
|
+
config: { ...config, vision_config: null, audio_config: null },
|
|
5587
|
+
...externalData > 0 ? { use_external_data_format: externalData } : {},
|
|
5588
|
+
...localOnly,
|
|
5589
|
+
...sessionOptions,
|
|
5590
|
+
...progress
|
|
5591
|
+
}
|
|
5592
|
+
) : transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
4557
5593
|
...modelOptions,
|
|
4558
|
-
...
|
|
4559
|
-
...
|
|
5594
|
+
...externalData > 0 ? { use_external_data_format: externalData } : {},
|
|
5595
|
+
...localOnly,
|
|
5596
|
+
...sessionOptions,
|
|
4560
5597
|
...progress
|
|
4561
5598
|
}),
|
|
4562
5599
|
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
4563
5600
|
revision: modelOptions.revision,
|
|
4564
|
-
...
|
|
5601
|
+
...localOnly,
|
|
4565
5602
|
...progress
|
|
4566
5603
|
})
|
|
4567
5604
|
]);
|
|
@@ -4571,17 +5608,23 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
4571
5608
|
cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
|
|
4572
5609
|
});
|
|
4573
5610
|
} catch (err) {
|
|
4574
|
-
await
|
|
5611
|
+
await runner.dispose?.().catch(() => {
|
|
4575
5612
|
});
|
|
4576
5613
|
throw err;
|
|
4577
5614
|
}
|
|
4578
5615
|
const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
|
|
5616
|
+
const runBatch = multimodal ? async (slice) => {
|
|
5617
|
+
const inputs = tokenizer(slice, {
|
|
5618
|
+
padding: true,
|
|
5619
|
+
truncation: true,
|
|
5620
|
+
max_length: profile.maxTokens
|
|
5621
|
+
});
|
|
5622
|
+
return encoderRows(await runner(inputs), inputs.attention_mask);
|
|
5623
|
+
} : async (slice) => (await runner(slice, { pooling, normalize: true })).tolist();
|
|
4579
5624
|
const embed = async (texts) => {
|
|
4580
5625
|
const out = [];
|
|
4581
5626
|
for (let i = 0; i < texts.length; i += MAX_BATCH) {
|
|
4582
|
-
|
|
4583
|
-
const result = await pipe(slice, { pooling, normalize: true });
|
|
4584
|
-
out.push(...result.tolist());
|
|
5627
|
+
out.push(...await runBatch(texts.slice(i, i + MAX_BATCH).map(cap)));
|
|
4585
5628
|
}
|
|
4586
5629
|
return out;
|
|
4587
5630
|
};
|
|
@@ -4591,11 +5634,12 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
4591
5634
|
embed,
|
|
4592
5635
|
embedQuery: async (text) => {
|
|
4593
5636
|
const [vector] = await embed([`${profile.queryInstruction}${text}`]);
|
|
4594
|
-
|
|
5637
|
+
if (!vector) throw new Error(`profile ${profile.id} returned no query vector`);
|
|
5638
|
+
return profileUnitVector2(profile, vector);
|
|
4595
5639
|
},
|
|
4596
5640
|
countTokens: (text) => tokenizer.encode(text).length,
|
|
4597
5641
|
dispose: async () => {
|
|
4598
|
-
await
|
|
5642
|
+
await runner.dispose?.();
|
|
4599
5643
|
}
|
|
4600
5644
|
};
|
|
4601
5645
|
}
|
|
@@ -4603,7 +5647,7 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
4603
5647
|
// src/markdown-adapter/load.ts
|
|
4604
5648
|
import { existsSync } from "fs";
|
|
4605
5649
|
import { readFile as readFile3, readdir } from "fs/promises";
|
|
4606
|
-
import { basename, dirname as
|
|
5650
|
+
import { basename, dirname as dirname4, isAbsolute as isAbsolute3, join as join6, posix as posix3, relative as relative3, resolve as resolve4, sep } from "path";
|
|
4607
5651
|
import {
|
|
4608
5652
|
CatalogDocumentSchema,
|
|
4609
5653
|
KnowledgeDocumentIdSchema,
|
|
@@ -5281,9 +6325,9 @@ function parseYaml(text, opts) {
|
|
|
5281
6325
|
var OPEN = /^---[ \t]*\r?\n/;
|
|
5282
6326
|
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
5283
6327
|
function parseMarkdownFrontMatter(source) {
|
|
5284
|
-
const
|
|
5285
|
-
if (!
|
|
5286
|
-
const lines = source.slice(
|
|
6328
|
+
const open2 = OPEN.exec(source);
|
|
6329
|
+
if (!open2) return { data: {}, body: source };
|
|
6330
|
+
const lines = source.slice(open2[0].length).split(/\r?\n/);
|
|
5287
6331
|
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
5288
6332
|
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
5289
6333
|
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
@@ -5482,7 +6526,7 @@ async function readMkdocsDocsDir(mkdocsPath) {
|
|
|
5482
6526
|
maxAliasCount: 100
|
|
5483
6527
|
});
|
|
5484
6528
|
const docsDir = isRecord3(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
5485
|
-
return resolve4(
|
|
6529
|
+
return resolve4(dirname4(mkdocsPath), docsDir);
|
|
5486
6530
|
}
|
|
5487
6531
|
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
5488
6532
|
const summary = join6(rootDir, "SUMMARY.md");
|
|
@@ -5490,7 +6534,7 @@ async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
|
5490
6534
|
const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
5491
6535
|
if (jupyter) return { format: "jupyter-book", path: jupyter };
|
|
5492
6536
|
const mkdocs = firstExisting(
|
|
5493
|
-
[rootDir, projectDir,
|
|
6537
|
+
[rootDir, projectDir, dirname4(rootDir)].flatMap((dir) => [
|
|
5494
6538
|
join6(dir, "mkdocs.yml"),
|
|
5495
6539
|
join6(dir, "mkdocs.yaml")
|
|
5496
6540
|
])
|
|
@@ -5530,7 +6574,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
5530
6574
|
}
|
|
5531
6575
|
if (toc.format === "mkdocs") {
|
|
5532
6576
|
const path2 = explicit ?? firstExisting(
|
|
5533
|
-
[rootDir,
|
|
6577
|
+
[rootDir, dirname4(rootDir)].flatMap((dir) => [
|
|
5534
6578
|
join6(dir, "mkdocs.yml"),
|
|
5535
6579
|
join6(dir, "mkdocs.yaml")
|
|
5536
6580
|
])
|
|
@@ -5553,7 +6597,7 @@ async function loadOutline(rootDir, toc, files) {
|
|
|
5553
6597
|
}
|
|
5554
6598
|
const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
5555
6599
|
if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
|
|
5556
|
-
const tocDir = insideRoot(rootDir,
|
|
6600
|
+
const tocDir = insideRoot(rootDir, dirname4(path));
|
|
5557
6601
|
if (tocDir === null)
|
|
5558
6602
|
throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
|
|
5559
6603
|
const doc = parseYaml(await readOutlineText(path), {
|
|
@@ -6110,7 +7154,9 @@ export {
|
|
|
6110
7154
|
CENTROID_SAMPLE_MAX,
|
|
6111
7155
|
CatalogHandle,
|
|
6112
7156
|
CatalogOpenError,
|
|
7157
|
+
CatalogQueryError,
|
|
6113
7158
|
DuckdbError,
|
|
7159
|
+
EMBEDDINGGEMMA_2_512_1,
|
|
6114
7160
|
EmbedderArtifactError,
|
|
6115
7161
|
EmbedderUnavailableError,
|
|
6116
7162
|
GEZEL_BGE_SMALL_EN_V15_1,
|
|
@@ -6134,6 +7180,8 @@ export {
|
|
|
6134
7180
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
6135
7181
|
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
6136
7182
|
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
7183
|
+
MAX_MEDIA_ROWS_PER_DOCUMENT,
|
|
7184
|
+
MEDIA_ROW_TEXT_MAX_TOKENS,
|
|
6137
7185
|
MIMETYPE_PATH,
|
|
6138
7186
|
MULTILINGUAL_E5_SMALL_1,
|
|
6139
7187
|
MULTILINGUAL_E5_SMALL_2,
|
|
@@ -6149,6 +7197,8 @@ export {
|
|
|
6149
7197
|
ROUTE_SHARDS_EXPLICIT,
|
|
6150
7198
|
ROUTE_SHARDS_PROACTIVE,
|
|
6151
7199
|
SHARD_DDL,
|
|
7200
|
+
SHARD_DDL_V4,
|
|
7201
|
+
SHARD_DDL_V5,
|
|
6152
7202
|
SHARD_MAX_CHUNKS,
|
|
6153
7203
|
SHARD_TARGET_CHUNKS,
|
|
6154
7204
|
SOURCE_NOTICES_PATH,
|
|
@@ -6160,10 +7210,12 @@ export {
|
|
|
6160
7210
|
assignOutlineOrders,
|
|
6161
7211
|
asymmetricTopK,
|
|
6162
7212
|
canonicalizeJson3 as canonicalizeJson,
|
|
7213
|
+
catalogFormatVersion,
|
|
6163
7214
|
centerVector,
|
|
6164
7215
|
chunkContentHash,
|
|
6165
7216
|
chunkMarkdownProfile,
|
|
6166
7217
|
chunkUid,
|
|
7218
|
+
collectMediaRows,
|
|
6167
7219
|
compareCatalogVersions,
|
|
6168
7220
|
compileKnowledgeCatalog,
|
|
6169
7221
|
createProfileEmbedder,
|
|
@@ -6175,9 +7227,13 @@ export {
|
|
|
6175
7227
|
fetchKnowledgeRegistry,
|
|
6176
7228
|
findDuckdbBinary,
|
|
6177
7229
|
findRegistryEntry,
|
|
7230
|
+
formatAtLeast,
|
|
6178
7231
|
generateKnowledgeSigningKeyPair,
|
|
6179
7232
|
hammingTopK,
|
|
7233
|
+
humanizeAssetName,
|
|
6180
7234
|
inspectGezkArchive,
|
|
7235
|
+
isExternalDataFile,
|
|
7236
|
+
isMultimodalEncoderConfig,
|
|
6181
7237
|
isSupportedFormatVersion,
|
|
6182
7238
|
isSupportedIndexSchemaVersion,
|
|
6183
7239
|
knowledgeEmbeddingProfile,
|
|
@@ -6185,6 +7241,9 @@ export {
|
|
|
6185
7241
|
l2Normalize,
|
|
6186
7242
|
loadMarkdownCatalog,
|
|
6187
7243
|
manifestSigningPayload,
|
|
7244
|
+
mediaEncoderPins,
|
|
7245
|
+
mediaReferences,
|
|
7246
|
+
missingPinnedFiles,
|
|
6188
7247
|
newOutlineTopic,
|
|
6189
7248
|
newerRegistryEntries,
|
|
6190
7249
|
openCatalogDatabase,
|
|
@@ -6192,6 +7251,11 @@ export {
|
|
|
6192
7251
|
parseGitbookSummary,
|
|
6193
7252
|
parseJupyterBookToc,
|
|
6194
7253
|
parseMkdocsNav,
|
|
7254
|
+
pinnedExternalDataChunks,
|
|
7255
|
+
pinnedLoadFiles,
|
|
7256
|
+
prefetchExternalData,
|
|
7257
|
+
preparePinnedLoad,
|
|
7258
|
+
profileUnitVector,
|
|
6195
7259
|
quantizeBinary,
|
|
6196
7260
|
quantizeBinaryForProfile,
|
|
6197
7261
|
quantizeInt8,
|
|
@@ -6202,6 +7266,7 @@ export {
|
|
|
6202
7266
|
resolveOutlinePath,
|
|
6203
7267
|
resolveTransformersModelOptions,
|
|
6204
7268
|
runDuckdbScript,
|
|
7269
|
+
shardDdlFor,
|
|
6205
7270
|
signManifest,
|
|
6206
7271
|
signRegistryIndex,
|
|
6207
7272
|
titleFromFolderName,
|
|
@@ -6209,6 +7274,7 @@ export {
|
|
|
6209
7274
|
transformersGraphOptions,
|
|
6210
7275
|
validateExtractedCatalog,
|
|
6211
7276
|
verifyManifestSignature,
|
|
7277
|
+
verifyMediaArtifacts,
|
|
6212
7278
|
verifyProfileArtifacts,
|
|
6213
7279
|
verifyRegistryIndex,
|
|
6214
7280
|
writeGezkArchive
|