@bendyline/gezel-knowledge 1.0.7 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -17
- package/dist/index.d.ts +606 -161
- package/dist/index.js +2389 -508
- package/package.json +6 -4
package/dist/index.js
CHANGED
|
@@ -1,13 +1,28 @@
|
|
|
1
1
|
// src/format/constants.ts
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
import {
|
|
3
|
+
BODY_CODEC_MIN_BYTES,
|
|
4
|
+
GEZK_APPLICATION_ID,
|
|
5
|
+
GEZK_FORMAT_GENERATIONS,
|
|
6
|
+
GEZK_FORMAT_VERSION,
|
|
7
|
+
GEZK_INDEX_SCHEMA_VERSION,
|
|
8
|
+
GEZK_MIME_TYPE,
|
|
9
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
10
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
11
|
+
LICENSE_NOTICE_PATH,
|
|
12
|
+
MANIFEST_PATH,
|
|
13
|
+
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
14
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
15
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
16
|
+
MIMETYPE_PATH,
|
|
17
|
+
README_PATH,
|
|
18
|
+
ROUTER_DB_PATH,
|
|
19
|
+
SOURCE_NOTICES_PATH,
|
|
20
|
+
ZIP_FIXED_MTIME,
|
|
21
|
+
isSupportedFormatVersion,
|
|
22
|
+
isSupportedIndexSchemaVersion
|
|
23
|
+
} from "@bendyline/gezk";
|
|
7
24
|
var SHARD_TARGET_CHUNKS = 2e5;
|
|
8
25
|
var SHARD_MAX_CHUNKS = 25e4;
|
|
9
|
-
var BODY_CODEC_MIN_BYTES = 512;
|
|
10
|
-
var MAX_KNOWLEDGE_DOCUMENT_BYTES = 16 * 1024 * 1024;
|
|
11
26
|
var CENTROID_CHUNKS_PER = 12500;
|
|
12
27
|
var CENTROID_MAX_PER_SHARD = 32;
|
|
13
28
|
var CENTROID_KMEANS_MAX_ITER = 25;
|
|
@@ -18,144 +33,16 @@ var RERANK_FINAL_K = 24;
|
|
|
18
33
|
function rerankK(finalK = RERANK_FINAL_K, chunkCount = Number.POSITIVE_INFINITY) {
|
|
19
34
|
return Math.min(512, Math.max(128, 8 * finalK), chunkCount);
|
|
20
35
|
}
|
|
21
|
-
var ZIP_FIXED_MTIME = new Date(Date.UTC(2e3, 0, 1, 0, 0, 0));
|
|
22
36
|
|
|
23
37
|
// src/format/ddl.ts
|
|
24
|
-
|
|
25
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
26
|
-
|
|
27
|
-
CREATE TABLE topics(
|
|
28
|
-
id TEXT PRIMARY KEY, parent_id TEXT, name TEXT NOT NULL,
|
|
29
|
-
description TEXT, sort_key TEXT NOT NULL, document_count INTEGER NOT NULL
|
|
30
|
-
);
|
|
31
|
-
|
|
32
|
-
CREATE TABLE documents(
|
|
33
|
-
id TEXT PRIMARY KEY,
|
|
34
|
-
title TEXT NOT NULL, slug TEXT NOT NULL, summary TEXT,
|
|
35
|
-
language TEXT NOT NULL, topic_id TEXT NOT NULL REFERENCES topics(id),
|
|
36
|
-
shard_id INTEGER NOT NULL,
|
|
37
|
-
chunk_count INTEGER NOT NULL,
|
|
38
|
-
source_url TEXT, source_revision TEXT, source_updated_at TEXT,
|
|
39
|
-
attribution_json TEXT,
|
|
40
|
-
body_codec TEXT NOT NULL CHECK (body_codec IN ('none','br')),
|
|
41
|
-
body_blob BLOB NOT NULL
|
|
42
|
-
);
|
|
43
|
-
CREATE INDEX documents_topic ON documents(topic_id, slug);
|
|
44
|
-
CREATE INDEX documents_shard ON documents(shard_id);
|
|
45
|
-
|
|
46
|
-
CREATE TABLE aliases(alias TEXT NOT NULL, document_id TEXT NOT NULL,
|
|
47
|
-
PRIMARY KEY (alias, document_id)) WITHOUT ROWID;
|
|
48
|
-
|
|
49
|
-
CREATE TABLE shards(
|
|
50
|
-
id INTEGER PRIMARY KEY,
|
|
51
|
-
path TEXT NOT NULL,
|
|
52
|
-
chunk_count INTEGER NOT NULL, document_count INTEGER NOT NULL,
|
|
53
|
-
topic_ids_json TEXT NOT NULL, centroid_count INTEGER NOT NULL,
|
|
54
|
-
bytes INTEGER NOT NULL
|
|
55
|
-
);
|
|
56
|
-
|
|
57
|
-
CREATE TABLE route_centroids(
|
|
58
|
-
id INTEGER PRIMARY KEY,
|
|
59
|
-
shard_id INTEGER NOT NULL REFERENCES shards(id),
|
|
60
|
-
embedding BLOB NOT NULL,
|
|
61
|
-
weight INTEGER NOT NULL
|
|
62
|
-
);
|
|
63
|
-
CREATE INDEX route_centroids_shard ON route_centroids(shard_id);
|
|
64
|
-
|
|
65
|
-
CREATE VIRTUAL TABLE fts_documents USING fts5(
|
|
66
|
-
title, summary, aliases, document_id UNINDEXED,
|
|
67
|
-
tokenize = 'unicode61 remove_diacritics 2', prefix = '2 3'
|
|
68
|
-
);
|
|
69
|
-
`;
|
|
70
|
-
var SHARD_DDL = `
|
|
71
|
-
CREATE TABLE meta(key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
72
|
-
|
|
73
|
-
CREATE TABLE chunks(
|
|
74
|
-
id INTEGER PRIMARY KEY,
|
|
75
|
-
chunk_uid TEXT NOT NULL,
|
|
76
|
-
document_id TEXT NOT NULL,
|
|
77
|
-
ordinal INTEGER NOT NULL,
|
|
78
|
-
title TEXT NOT NULL,
|
|
79
|
-
heading_path TEXT NOT NULL,
|
|
80
|
-
heading_text TEXT NOT NULL,
|
|
81
|
-
line_start INTEGER NOT NULL, line_end INTEGER NOT NULL,
|
|
82
|
-
token_count INTEGER NOT NULL,
|
|
83
|
-
content_hash TEXT NOT NULL,
|
|
84
|
-
text TEXT NOT NULL
|
|
85
|
-
);
|
|
86
|
-
CREATE UNIQUE INDEX chunks_uid ON chunks(chunk_uid);
|
|
87
|
-
CREATE INDEX chunks_document ON chunks(document_id, ordinal);
|
|
88
|
-
|
|
89
|
-
CREATE VIRTUAL TABLE fts_chunks USING fts5(
|
|
90
|
-
title, heading_text, text,
|
|
91
|
-
content = 'chunks', content_rowid = 'id',
|
|
92
|
-
tokenize = 'unicode61 remove_diacritics 2'
|
|
93
|
-
);
|
|
94
|
-
|
|
95
|
-
CREATE TABLE chunk_vectors_int8(
|
|
96
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
97
|
-
v BLOB NOT NULL
|
|
98
|
-
);
|
|
99
|
-
`;
|
|
100
|
-
function vecChunksDdl(dim) {
|
|
101
|
-
return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
|
102
|
-
chunk_id INTEGER PRIMARY KEY,
|
|
103
|
-
embedding bit[${dim}],
|
|
104
|
-
chunk_size=1024
|
|
105
|
-
);`;
|
|
106
|
-
}
|
|
38
|
+
import { ROUTER_DDL, SHARD_DDL } from "@bendyline/gezk";
|
|
107
39
|
|
|
108
40
|
// src/format/ids.ts
|
|
109
|
-
import {
|
|
110
|
-
|
|
111
|
-
const textHash = createHash("sha256").update(chunkText, "utf8").digest();
|
|
112
|
-
return createHash("sha256").update(documentId, "utf8").update(Buffer.from([0])).update(String(ordinal), "utf8").update(Buffer.from([0])).update(textHash).digest("hex").slice(0, 32);
|
|
113
|
-
}
|
|
114
|
-
function chunkContentHash(chunkText) {
|
|
115
|
-
return createHash("sha256").update(chunkText, "utf8").digest("hex");
|
|
116
|
-
}
|
|
117
|
-
function documentSlug(title) {
|
|
118
|
-
const slug = title.toLowerCase().normalize("NFKD").replace(/\p{Mark}/gu, "").replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 80);
|
|
119
|
-
return slug || "untitled";
|
|
120
|
-
}
|
|
41
|
+
import { documentSlug } from "@bendyline/gezk";
|
|
42
|
+
import { chunkContentHash, chunkUid } from "@bendyline/gezk/node";
|
|
121
43
|
|
|
122
44
|
// src/format/quantize.ts
|
|
123
|
-
|
|
124
|
-
const out = new Int8Array(unitVector.length);
|
|
125
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
126
|
-
const q = Math.round(unitVector[i] * 127);
|
|
127
|
-
out[i] = q > 127 ? 127 : q < -127 ? -127 : q;
|
|
128
|
-
}
|
|
129
|
-
return out;
|
|
130
|
-
}
|
|
131
|
-
function quantizeBinary(unitVector) {
|
|
132
|
-
const bytes = Math.ceil(unitVector.length / 8);
|
|
133
|
-
const out = new Uint8Array(bytes);
|
|
134
|
-
for (let i = 0; i < unitVector.length; i++) {
|
|
135
|
-
if (unitVector[i] > 0) {
|
|
136
|
-
out[i >> 3] = out[i >> 3] | 1 << (i & 7);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
return out;
|
|
140
|
-
}
|
|
141
|
-
function rerankScore(query, passageInt8) {
|
|
142
|
-
let dot = 0;
|
|
143
|
-
const n = Math.min(query.length, passageInt8.length);
|
|
144
|
-
for (let i = 0; i < n; i++) dot += query[i] * (passageInt8[i] / 127);
|
|
145
|
-
return dot;
|
|
146
|
-
}
|
|
147
|
-
function l2Normalize(vector) {
|
|
148
|
-
let sum = 0;
|
|
149
|
-
for (let i = 0; i < vector.length; i++) {
|
|
150
|
-
const v = vector[i];
|
|
151
|
-
sum += v * v;
|
|
152
|
-
}
|
|
153
|
-
const norm = Math.sqrt(sum);
|
|
154
|
-
if (!Number.isFinite(norm) || norm === 0) throw new Error("degenerate vector (zero norm)");
|
|
155
|
-
const out = new Float32Array(vector.length);
|
|
156
|
-
for (let i = 0; i < vector.length; i++) out[i] = vector[i] / norm;
|
|
157
|
-
return out;
|
|
158
|
-
}
|
|
45
|
+
import { l2Normalize, quantizeBinary, quantizeInt8, rerankScore } from "@bendyline/gezk";
|
|
159
46
|
|
|
160
47
|
// src/chunking/markdown-chunker.ts
|
|
161
48
|
var HEADING = /^(#{1,6})\s+(.*)$/;
|
|
@@ -284,12 +171,24 @@ function splitOversizeLine(line, opts) {
|
|
|
284
171
|
}
|
|
285
172
|
|
|
286
173
|
// src/compiler/compile.ts
|
|
287
|
-
import { createHash
|
|
288
|
-
import { mkdirSync, rmSync } from "fs";
|
|
174
|
+
import { createHash } from "crypto";
|
|
175
|
+
import { mkdirSync, readFileSync, rmSync, statSync } from "fs";
|
|
289
176
|
import { stat } from "fs/promises";
|
|
290
177
|
import { join } from "path";
|
|
291
178
|
import { brotliCompressSync, constants as zlibConstants } from "zlib";
|
|
292
|
-
import {
|
|
179
|
+
import {
|
|
180
|
+
GEZK_MANIFEST_KIND,
|
|
181
|
+
KnowledgeAssetPathSchema,
|
|
182
|
+
KnowledgeCatalogManifestSchema,
|
|
183
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES,
|
|
184
|
+
MAX_KNOWLEDGE_ASSET_BYTES,
|
|
185
|
+
MAX_KNOWLEDGE_ASSET_COUNT,
|
|
186
|
+
assetExtension,
|
|
187
|
+
assetKindForExtension,
|
|
188
|
+
canonicalizeJson,
|
|
189
|
+
sniffAssetType,
|
|
190
|
+
svgInertnessProblem
|
|
191
|
+
} from "@bendyline/gezk";
|
|
293
192
|
|
|
294
193
|
// src/archive/write.ts
|
|
295
194
|
import { createWriteStream } from "fs";
|
|
@@ -301,9 +200,13 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
301
200
|
const yazl = nodeRequire("yazl");
|
|
302
201
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
303
202
|
const zip = new yazl.ZipFile();
|
|
304
|
-
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress:
|
|
203
|
+
const options = { mtime: ZIP_FIXED_MTIME, mode: 33188, compress: false };
|
|
204
|
+
zip.addBuffer(Buffer.from(GEZK_MIME_TYPE, "utf8"), MIMETYPE_PATH, options);
|
|
305
205
|
const sorted = [...entries].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
|
|
306
206
|
for (const entry of sorted) {
|
|
207
|
+
if (entry.path === MIMETYPE_PATH) {
|
|
208
|
+
throw new Error(`the writer owns the ${MIMETYPE_PATH} entry`);
|
|
209
|
+
}
|
|
307
210
|
if (entry.path.includes("\\") || entry.path.startsWith("/") || entry.path.includes("..")) {
|
|
308
211
|
throw new Error(`unsafe archive path: ${entry.path}`);
|
|
309
212
|
}
|
|
@@ -311,10 +214,10 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
311
214
|
else if (entry.absPath) zip.addFile(entry.absPath, entry.path, options);
|
|
312
215
|
else throw new Error(`archive entry ${entry.path} has neither content nor absPath`);
|
|
313
216
|
}
|
|
314
|
-
await new Promise((
|
|
217
|
+
await new Promise((resolve5, reject) => {
|
|
315
218
|
const out = createWriteStream(outputPath);
|
|
316
219
|
out.on("error", reject);
|
|
317
|
-
out.on("close", () =>
|
|
220
|
+
out.on("close", () => resolve5());
|
|
318
221
|
zip.outputStream.on("error", reject);
|
|
319
222
|
zip.outputStream.pipe(out);
|
|
320
223
|
zip.end();
|
|
@@ -322,27 +225,7 @@ async function writeGezkArchive(outputPath, entries) {
|
|
|
322
225
|
}
|
|
323
226
|
|
|
324
227
|
// src/format/file-hash.ts
|
|
325
|
-
import {
|
|
326
|
-
import { createReadStream } from "fs";
|
|
327
|
-
async function hashFileStreaming(path, maxBytes = Number.MAX_SAFE_INTEGER) {
|
|
328
|
-
const hasher = createHash2("sha256");
|
|
329
|
-
let sizeBytes = 0;
|
|
330
|
-
await new Promise((resolve3, reject) => {
|
|
331
|
-
const stream = createReadStream(path, { highWaterMark: 4 * 1024 * 1024 });
|
|
332
|
-
stream.on("data", (chunk) => {
|
|
333
|
-
const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
|
|
334
|
-
sizeBytes += bytes.byteLength;
|
|
335
|
-
if (sizeBytes > maxBytes) {
|
|
336
|
-
stream.destroy(new Error(`file exceeds ${maxBytes} byte limit`));
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
hasher.update(bytes);
|
|
340
|
-
});
|
|
341
|
-
stream.on("error", reject);
|
|
342
|
-
stream.on("end", resolve3);
|
|
343
|
-
});
|
|
344
|
-
return { sha256: hasher.digest("hex"), sizeBytes };
|
|
345
|
-
}
|
|
228
|
+
import { hashFileStreaming } from "@bendyline/gezk/node";
|
|
346
229
|
|
|
347
230
|
// src/format/node-sqlite.ts
|
|
348
231
|
import { createRequire as createRequire2 } from "module";
|
|
@@ -371,25 +254,13 @@ function documentSmokeQueryMisses(db, smoke) {
|
|
|
371
254
|
return smoke.expectedDocumentIds.filter((id) => !top.includes(id));
|
|
372
255
|
}
|
|
373
256
|
|
|
374
|
-
// src/
|
|
257
|
+
// src/toolchain.ts
|
|
375
258
|
import { createRequire as createRequire3 } from "module";
|
|
376
259
|
var nodeRequire3 = createRequire3(import.meta.url);
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
382
|
-
} catch (err) {
|
|
383
|
-
throw new Error(
|
|
384
|
-
`sqlite-vec extension failed to load \u2014 the compiler cannot build vector tables: ${err instanceof Error ? err.message : String(err)}`
|
|
385
|
-
);
|
|
386
|
-
} finally {
|
|
387
|
-
try {
|
|
388
|
-
db.enableLoadExtension(false);
|
|
389
|
-
} catch {
|
|
390
|
-
}
|
|
391
|
-
}
|
|
392
|
-
}
|
|
260
|
+
var KNOWLEDGE_TOOLCHAIN = {
|
|
261
|
+
name: "@bendyline/gezel-knowledge",
|
|
262
|
+
version: nodeRequire3("../package.json").version
|
|
263
|
+
};
|
|
393
264
|
|
|
394
265
|
// src/compiler/compile.ts
|
|
395
266
|
async function compileKnowledgeCatalog(opts) {
|
|
@@ -398,12 +269,13 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
398
269
|
"a catalog must ship a table of contents: provide at least one topic (a flat corpus gets a single root topic)"
|
|
399
270
|
);
|
|
400
271
|
}
|
|
401
|
-
const
|
|
272
|
+
const topicById = assertTopicForest(opts.topics);
|
|
273
|
+
const preparedAssets = prepareAssets(opts.assets ?? [], opts.extraFiles ?? {});
|
|
402
274
|
const profile = opts.embeddingProfile;
|
|
403
275
|
const chunkerOpts = {
|
|
404
|
-
unit:
|
|
405
|
-
target: opts.chunkingProfile.
|
|
406
|
-
overlap: opts.chunkingProfile.
|
|
276
|
+
unit: opts.chunkingProfile.unit,
|
|
277
|
+
target: opts.chunkingProfile.target,
|
|
278
|
+
overlap: opts.chunkingProfile.overlap,
|
|
407
279
|
countTokens: opts.countTokens
|
|
408
280
|
};
|
|
409
281
|
const prepared = [];
|
|
@@ -411,13 +283,11 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
411
283
|
for await (const doc of opts.documents) {
|
|
412
284
|
if (seenIds.has(doc.id)) throw new Error(`duplicate document id: ${doc.id}`);
|
|
413
285
|
seenIds.add(doc.id);
|
|
414
|
-
const rootTopic = doc.topicPath[0];
|
|
415
|
-
if (!rootTopic || !topicIds.has(rootTopic)) {
|
|
416
|
-
throw new Error(`document ${doc.id} names unknown topic '${rootTopic ?? ""}'`);
|
|
417
|
-
}
|
|
418
286
|
prepared.push({
|
|
419
287
|
doc,
|
|
420
288
|
topicPathKey: doc.topicPath.join("/"),
|
|
289
|
+
leafTopicId: assertTopicPath(topicById, doc.id, doc.topicPath),
|
|
290
|
+
metaJson: encodeDocumentMeta(doc),
|
|
421
291
|
chunks: chunkMarkdownProfile(normalizeMarkdown(doc.markdown), chunkerOpts)
|
|
422
292
|
});
|
|
423
293
|
}
|
|
@@ -463,7 +333,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
463
333
|
router.exec("BEGIN");
|
|
464
334
|
if (embedded) {
|
|
465
335
|
router.exec(SHARD_DDL.replace("CREATE TABLE meta", "CREATE TABLE IF NOT EXISTS meta"));
|
|
466
|
-
router.exec(vecChunksDdl(profile.dimensions));
|
|
467
336
|
}
|
|
468
337
|
const commonMeta = {
|
|
469
338
|
format_version: String(GEZK_FORMAT_VERSION),
|
|
@@ -485,8 +354,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
485
354
|
});
|
|
486
355
|
const topicDocCount = /* @__PURE__ */ new Map();
|
|
487
356
|
for (const p of prepared) {
|
|
488
|
-
|
|
489
|
-
topicDocCount.set(t, (topicDocCount.get(t) ?? 0) + 1);
|
|
357
|
+
topicDocCount.set(p.leafTopicId, (topicDocCount.get(p.leafTopicId) ?? 0) + 1);
|
|
490
358
|
}
|
|
491
359
|
{
|
|
492
360
|
const stmt = router.prepare(
|
|
@@ -505,9 +373,10 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
505
373
|
}
|
|
506
374
|
{
|
|
507
375
|
const insertDoc = router.prepare(
|
|
508
|
-
`INSERT INTO documents (id, title, slug, summary, language, topic_id,
|
|
509
|
-
source_url, source_revision, source_updated_at, attribution_json,
|
|
510
|
-
|
|
376
|
+
`INSERT INTO documents (id, title, slug, summary, language, topic_id, ordinal, shard_id,
|
|
377
|
+
chunk_count, source_url, source_revision, source_updated_at, attribution_json, meta_json,
|
|
378
|
+
body_codec, body_blob)
|
|
379
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
511
380
|
);
|
|
512
381
|
const insertAlias = router.prepare(
|
|
513
382
|
"INSERT OR IGNORE INTO aliases (alias, document_id) VALUES (?, ?)"
|
|
@@ -535,13 +404,15 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
535
404
|
p.doc.slug || documentSlug(p.doc.title),
|
|
536
405
|
p.doc.summary ?? null,
|
|
537
406
|
p.doc.language,
|
|
538
|
-
p.
|
|
407
|
+
p.leafTopicId,
|
|
408
|
+
p.doc.ordinal === void 0 ? null : BigInt(p.doc.ordinal),
|
|
539
409
|
BigInt(shardOf.get(p.doc.id)),
|
|
540
410
|
BigInt(p.chunks.length),
|
|
541
411
|
p.doc.sourceUrl ?? null,
|
|
542
412
|
p.doc.sourceRevision ?? null,
|
|
543
413
|
p.doc.sourceUpdatedAt ?? null,
|
|
544
414
|
p.doc.attribution ? JSON.stringify(p.doc.attribution) : null,
|
|
415
|
+
p.metaJson,
|
|
545
416
|
useBrotli ? "br" : "none",
|
|
546
417
|
blob
|
|
547
418
|
);
|
|
@@ -560,7 +431,6 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
560
431
|
if (!db) {
|
|
561
432
|
db = createCatalogDb(join(opts.workDir, shardPath(id)));
|
|
562
433
|
db.exec(SHARD_DDL);
|
|
563
|
-
db.exec(vecChunksDdl(profile.dimensions));
|
|
564
434
|
db.exec("BEGIN");
|
|
565
435
|
shardDbs.set(id, db);
|
|
566
436
|
}
|
|
@@ -571,7 +441,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
571
441
|
let stmts = vecStmts.get(db);
|
|
572
442
|
if (!stmts) {
|
|
573
443
|
stmts = {
|
|
574
|
-
vec: db.prepare("INSERT INTO
|
|
444
|
+
vec: db.prepare("INSERT INTO chunk_vectors_bit (chunk_id, v) VALUES (?, ?)"),
|
|
575
445
|
int8: db.prepare("INSERT INTO chunk_vectors_int8 (chunk_id, v) VALUES (?, ?)")
|
|
576
446
|
};
|
|
577
447
|
vecStmts.set(db, stmts);
|
|
@@ -618,7 +488,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
618
488
|
const db = shardDb(shardId);
|
|
619
489
|
shardDocCounts.set(shardId, (shardDocCounts.get(shardId) ?? 0) + 1);
|
|
620
490
|
const topics = shardTopicIds.get(shardId) ?? /* @__PURE__ */ new Set();
|
|
621
|
-
topics.add(p.
|
|
491
|
+
topics.add(p.leafTopicId);
|
|
622
492
|
shardTopicIds.set(shardId, topics);
|
|
623
493
|
const insertChunk = db.prepare(
|
|
624
494
|
`INSERT INTO chunks (chunk_uid, document_id, ordinal, title, heading_path, heading_text,
|
|
@@ -650,7 +520,7 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
650
520
|
const header = buildContextHeader(
|
|
651
521
|
p.doc.title,
|
|
652
522
|
chunk.headingPath,
|
|
653
|
-
opts.chunkingProfile.contextHeader.
|
|
523
|
+
opts.chunkingProfile.contextHeader.max,
|
|
654
524
|
opts.countTokens
|
|
655
525
|
);
|
|
656
526
|
pendingTexts.push(`${profile.passageInstruction}${header}${chunk.text}`);
|
|
@@ -659,6 +529,30 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
659
529
|
}
|
|
660
530
|
}
|
|
661
531
|
await flushEmbeds();
|
|
532
|
+
for (let id = 0; id < shardCount; id++) {
|
|
533
|
+
if (!embedded)
|
|
534
|
+
putMeta(shardDb(id), {
|
|
535
|
+
shard_id: String(id),
|
|
536
|
+
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
540
|
+
const finalDir = join(opts.workDir, "final");
|
|
541
|
+
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
542
|
+
const vacuumed = [];
|
|
543
|
+
const vacuumInto = (db, archivePath) => {
|
|
544
|
+
db.exec("COMMIT");
|
|
545
|
+
const dest = join(finalDir, archivePath);
|
|
546
|
+
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
547
|
+
vacuumed.push({ archivePath, absPath: dest });
|
|
548
|
+
return dest;
|
|
549
|
+
};
|
|
550
|
+
const shardBytes = /* @__PURE__ */ new Map();
|
|
551
|
+
if (!embedded) {
|
|
552
|
+
for (let id = 0; id < shardCount; id++) {
|
|
553
|
+
shardBytes.set(id, statSync(vacuumInto(shardDb(id), shardPath(id))).size);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
662
556
|
{
|
|
663
557
|
const insertShard = router.prepare(
|
|
664
558
|
"INSERT INTO shards (id, path, chunk_count, document_count, topic_ids_json, centroid_count, bytes) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
@@ -681,35 +575,14 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
681
575
|
BigInt(shardDocCounts.get(id) ?? 0),
|
|
682
576
|
JSON.stringify([...shardTopicIds.get(id) ?? []].sort()),
|
|
683
577
|
BigInt(centroids.length),
|
|
684
|
-
BigInt(0)
|
|
685
|
-
// authoritative sizes live in the manifest (post-VACUUM)
|
|
578
|
+
BigInt(shardBytes.get(id) ?? 0)
|
|
686
579
|
);
|
|
687
580
|
for (const c of centroids) {
|
|
688
581
|
insertCentroid.run(BigInt(id), Buffer.from(c.centroid.buffer.slice(0)), BigInt(c.weight));
|
|
689
582
|
}
|
|
690
583
|
}
|
|
691
584
|
}
|
|
692
|
-
for (let id = 0; id < shardCount; id++) {
|
|
693
|
-
if (!embedded)
|
|
694
|
-
putMeta(shardDb(id), {
|
|
695
|
-
shard_id: String(id),
|
|
696
|
-
chunk_count: String(shardChunkCounts.get(id) ?? 0)
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
if (embedded) putMeta(router, { shard_id: "0", chunk_count: String(totalChunks) });
|
|
700
|
-
const finalDir = join(opts.workDir, "final");
|
|
701
|
-
mkdirSync(join(finalDir, "index", "shards"), { recursive: true });
|
|
702
|
-
const vacuumed = [];
|
|
703
|
-
const vacuumInto = (db, archivePath) => {
|
|
704
|
-
db.exec("COMMIT");
|
|
705
|
-
const dest = join(finalDir, archivePath);
|
|
706
|
-
db.exec(`VACUUM INTO '${dest.replaceAll("'", "''").replaceAll("\\", "/")}'`);
|
|
707
|
-
vacuumed.push({ archivePath, absPath: dest });
|
|
708
|
-
};
|
|
709
585
|
vacuumInto(router, ROUTER_DB_PATH);
|
|
710
|
-
if (!embedded) {
|
|
711
|
-
for (let id = 0; id < shardCount; id++) vacuumInto(shardDb(id), shardPath(id));
|
|
712
|
-
}
|
|
713
586
|
router.close();
|
|
714
587
|
for (const db of shardDbs.values()) db.close();
|
|
715
588
|
let smokeQueries = opts.smokeQueries;
|
|
@@ -765,17 +638,25 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
765
638
|
});
|
|
766
639
|
}
|
|
767
640
|
}
|
|
768
|
-
const extraFiles =
|
|
641
|
+
const extraFiles = {
|
|
642
|
+
[README_PATH]: defaultReadme(opts),
|
|
643
|
+
[LICENSE_NOTICE_PATH]: defaultLicenseNotice(opts),
|
|
644
|
+
...opts.extraFiles ?? {}
|
|
645
|
+
};
|
|
769
646
|
for (const [path, content] of Object.entries(extraFiles)) {
|
|
770
647
|
const bytes = Buffer.from(content, "utf8");
|
|
771
648
|
files.push({
|
|
772
649
|
path,
|
|
773
650
|
sizeBytes: bytes.length,
|
|
774
|
-
sha256:
|
|
651
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
775
652
|
});
|
|
776
653
|
}
|
|
654
|
+
for (const asset of preparedAssets) {
|
|
655
|
+
files.push({ path: asset.path, sizeBytes: asset.sizeBytes, sha256: asset.sha256 });
|
|
656
|
+
}
|
|
657
|
+
assertAssetReferencesDeclared(prepared, preparedAssets);
|
|
777
658
|
const unsignedManifest = KnowledgeCatalogManifestSchema.parse({
|
|
778
|
-
kind:
|
|
659
|
+
kind: GEZK_MANIFEST_KIND,
|
|
779
660
|
formatVersion: GEZK_FORMAT_VERSION,
|
|
780
661
|
indexSchemaVersion: GEZK_INDEX_SCHEMA_VERSION,
|
|
781
662
|
id: opts.catalog.id,
|
|
@@ -786,20 +667,34 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
786
667
|
publisher: opts.catalog.publisher,
|
|
787
668
|
createdAt: opts.catalog.createdAt,
|
|
788
669
|
...opts.catalog.sourceSnapshot ? { sourceSnapshot: opts.catalog.sourceSnapshot } : {},
|
|
789
|
-
license:
|
|
670
|
+
license: {
|
|
671
|
+
...opts.catalog.license,
|
|
672
|
+
noticePath: opts.catalog.license.noticePath ?? LICENSE_NOTICE_PATH
|
|
673
|
+
},
|
|
790
674
|
embedding: profile,
|
|
791
675
|
chunking: opts.chunkingProfile,
|
|
792
|
-
topics: opts.topics.map((t) => ({
|
|
676
|
+
topics: opts.topics.map((t) => ({
|
|
677
|
+
id: t.id,
|
|
678
|
+
name: t.name,
|
|
679
|
+
...t.parentId ? { parentId: t.parentId } : {},
|
|
680
|
+
sortKey: t.sortKey ?? t.name.toLowerCase(),
|
|
681
|
+
...t.description ? { description: t.description } : {}
|
|
682
|
+
})),
|
|
793
683
|
router: {
|
|
794
684
|
shardTargetChunks: shardTarget,
|
|
795
685
|
shards: routerShardStats.sort((a, b) => a.id - b.id),
|
|
796
686
|
totalCentroids: routerShardStats.reduce((sum, s) => sum + s.centroids, 0)
|
|
797
687
|
},
|
|
798
|
-
counts: {
|
|
688
|
+
counts: {
|
|
689
|
+
documents: prepared.length,
|
|
690
|
+
chunks: totalChunks,
|
|
691
|
+
shards: shardCount,
|
|
692
|
+
assets: preparedAssets.length
|
|
693
|
+
},
|
|
799
694
|
files: [...files].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
800
|
-
|
|
695
|
+
requires: { formatVersion: GEZK_FORMAT_VERSION, features: [] },
|
|
801
696
|
...smokeQueries && smokeQueries.length > 0 ? { smokeQueries } : {},
|
|
802
|
-
|
|
697
|
+
toolchain: opts.toolchain ?? KNOWLEDGE_TOOLCHAIN
|
|
803
698
|
});
|
|
804
699
|
const manifest = opts.finalizeManifest ? KnowledgeCatalogManifestSchema.parse(opts.finalizeManifest(unsignedManifest)) : unsignedManifest;
|
|
805
700
|
const archiveEntries = [
|
|
@@ -812,7 +707,8 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
812
707
|
...Object.entries(extraFiles).map(([path, content]) => ({
|
|
813
708
|
path,
|
|
814
709
|
content: Buffer.from(content, "utf8")
|
|
815
|
-
}))
|
|
710
|
+
})),
|
|
711
|
+
...preparedAssets.map((asset) => ({ path: asset.path, content: asset.bytes }))
|
|
816
712
|
];
|
|
817
713
|
await writeGezkArchive(opts.outputPath, archiveEntries);
|
|
818
714
|
const archiveBytes = (await stat(opts.outputPath)).size;
|
|
@@ -828,15 +724,166 @@ async function compileKnowledgeCatalog(opts) {
|
|
|
828
724
|
closeAll();
|
|
829
725
|
}
|
|
830
726
|
}
|
|
727
|
+
function assertTopicForest(topics) {
|
|
728
|
+
const byId = /* @__PURE__ */ new Map();
|
|
729
|
+
for (const topic of topics) {
|
|
730
|
+
if (byId.has(topic.id)) throw new Error(`duplicate topic id: ${topic.id}`);
|
|
731
|
+
byId.set(topic.id, topic);
|
|
732
|
+
}
|
|
733
|
+
for (const topic of topics) {
|
|
734
|
+
const seen = /* @__PURE__ */ new Set();
|
|
735
|
+
let depth = 0;
|
|
736
|
+
let current = topic;
|
|
737
|
+
while (current) {
|
|
738
|
+
if (seen.has(current.id)) throw new Error(`topic '${topic.id}' sits in a parent cycle`);
|
|
739
|
+
seen.add(current.id);
|
|
740
|
+
depth += 1;
|
|
741
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
742
|
+
throw new Error(`topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`);
|
|
743
|
+
}
|
|
744
|
+
if (current.parentId === void 0) break;
|
|
745
|
+
const parent = byId.get(current.parentId);
|
|
746
|
+
if (!parent) {
|
|
747
|
+
throw new Error(`topic '${current.id}' names an undeclared parent '${current.parentId}'`);
|
|
748
|
+
}
|
|
749
|
+
current = parent;
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
return byId;
|
|
753
|
+
}
|
|
754
|
+
function assertTopicPath(topicById, docId, path) {
|
|
755
|
+
for (let i = 0; i < path.length; i++) {
|
|
756
|
+
const id = path[i];
|
|
757
|
+
const topic = topicById.get(id);
|
|
758
|
+
if (!topic) {
|
|
759
|
+
throw new Error(
|
|
760
|
+
`document ${docId} names unknown topic '${id}' (segment ${i} of ${path.join("/")})`
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
const expectedParent = i === 0 ? void 0 : path[i - 1];
|
|
764
|
+
if (topic.parentId !== expectedParent) {
|
|
765
|
+
throw new Error(
|
|
766
|
+
`document ${docId}: topic path ${path.join("/")} does not follow the declared tree at '${id}' (its parent is '${topic.parentId ?? "(root)"}')`
|
|
767
|
+
);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
return path[path.length - 1];
|
|
771
|
+
}
|
|
772
|
+
function encodeDocumentMeta(doc) {
|
|
773
|
+
if (!doc.meta || Object.keys(doc.meta).length === 0) return null;
|
|
774
|
+
let json;
|
|
775
|
+
try {
|
|
776
|
+
json = canonicalizeJson(doc.meta);
|
|
777
|
+
} catch (error) {
|
|
778
|
+
throw new Error(
|
|
779
|
+
`document ${doc.id} has metadata that cannot be serialized: ${error instanceof Error ? error.message : String(error)}`
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
const bytes = Buffer.byteLength(json, "utf8");
|
|
783
|
+
if (bytes > MAX_KNOWLEDGE_DOCUMENT_META_BYTES) {
|
|
784
|
+
throw new Error(
|
|
785
|
+
`document ${doc.id} has ${bytes} bytes of metadata; the limit is ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES}`
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
return json;
|
|
789
|
+
}
|
|
790
|
+
function prepareAssets(assets, extraFiles) {
|
|
791
|
+
for (const path of Object.keys(extraFiles)) {
|
|
792
|
+
if (path.startsWith("assets/")) {
|
|
793
|
+
throw new Error(`extraFiles cannot carry '${path}': files under assets/ go through 'assets'`);
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
if (assets.length > MAX_KNOWLEDGE_ASSET_COUNT) {
|
|
797
|
+
throw new Error(`${assets.length} assets exceed the limit of ${MAX_KNOWLEDGE_ASSET_COUNT}`);
|
|
798
|
+
}
|
|
799
|
+
const seen = /* @__PURE__ */ new Set();
|
|
800
|
+
const prepared = [];
|
|
801
|
+
let total = 0;
|
|
802
|
+
for (const asset of assets) {
|
|
803
|
+
const parsed = KnowledgeAssetPathSchema.safeParse(asset.path);
|
|
804
|
+
if (!parsed.success) throw new Error(`invalid asset path: ${asset.path}`);
|
|
805
|
+
const key = asset.path.toLowerCase();
|
|
806
|
+
if (seen.has(key)) throw new Error(`duplicate asset path (case-insensitive): ${asset.path}`);
|
|
807
|
+
seen.add(key);
|
|
808
|
+
if (asset.content === void 0 === (asset.absPath === void 0)) {
|
|
809
|
+
throw new Error(`asset ${asset.path} must supply exactly one of content / absPath`);
|
|
810
|
+
}
|
|
811
|
+
const bytes = asset.content ?? readFileSync(asset.absPath);
|
|
812
|
+
if (bytes.byteLength > MAX_KNOWLEDGE_ASSET_BYTES) {
|
|
813
|
+
throw new Error(
|
|
814
|
+
`asset ${asset.path} is ${bytes.byteLength} bytes; the limit is ${MAX_KNOWLEDGE_ASSET_BYTES}`
|
|
815
|
+
);
|
|
816
|
+
}
|
|
817
|
+
total += bytes.byteLength;
|
|
818
|
+
if (total > MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES) {
|
|
819
|
+
throw new Error(`assets exceed ${MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES} bytes in total`);
|
|
820
|
+
}
|
|
821
|
+
const ext = assetExtension(asset.path);
|
|
822
|
+
const expected = ext ? assetKindForExtension(ext) : null;
|
|
823
|
+
const actual = sniffAssetType(bytes);
|
|
824
|
+
if (expected === null || actual !== expected) {
|
|
825
|
+
throw new Error(
|
|
826
|
+
`asset ${asset.path}: the leading bytes say ${actual ?? "unknown"}, the extension says ${expected ?? "unknown"}`
|
|
827
|
+
);
|
|
828
|
+
}
|
|
829
|
+
if (ext === "svg") {
|
|
830
|
+
const problem = svgInertnessProblem(bytes);
|
|
831
|
+
if (problem) throw new Error(`asset ${asset.path} is not an inert SVG: it ${problem}`);
|
|
832
|
+
}
|
|
833
|
+
prepared.push({
|
|
834
|
+
path: asset.path,
|
|
835
|
+
bytes,
|
|
836
|
+
sizeBytes: bytes.byteLength,
|
|
837
|
+
sha256: createHash("sha256").update(bytes).digest("hex")
|
|
838
|
+
});
|
|
839
|
+
}
|
|
840
|
+
return prepared.sort((a, b) => a.path < b.path ? -1 : 1);
|
|
841
|
+
}
|
|
842
|
+
function assertAssetReferencesDeclared(prepared, assets) {
|
|
843
|
+
const declared = new Set(assets.map((a) => a.path));
|
|
844
|
+
for (const p of prepared) {
|
|
845
|
+
for (const match of p.doc.markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
846
|
+
const target = match[1];
|
|
847
|
+
if (!declared.has(target)) {
|
|
848
|
+
throw new Error(
|
|
849
|
+
`document ${p.doc.id} references '${target}', which is not among the catalog's assets`
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
}
|
|
831
855
|
function createCatalogDb(absPath) {
|
|
832
|
-
const db = new DatabaseSync(absPath
|
|
833
|
-
loadVecExtension(db);
|
|
856
|
+
const db = new DatabaseSync(absPath);
|
|
834
857
|
db.exec("PRAGMA page_size=8192");
|
|
835
858
|
db.exec(`PRAGMA application_id=${GEZK_APPLICATION_ID}`);
|
|
836
859
|
db.exec(`PRAGMA user_version=${GEZK_INDEX_SCHEMA_VERSION}`);
|
|
837
860
|
db.exec("PRAGMA journal_mode=DELETE");
|
|
838
861
|
return db;
|
|
839
862
|
}
|
|
863
|
+
function defaultReadme(opts) {
|
|
864
|
+
const c = opts.catalog;
|
|
865
|
+
return [
|
|
866
|
+
`# ${c.name}`,
|
|
867
|
+
"",
|
|
868
|
+
...c.description ? [c.description, ""] : [],
|
|
869
|
+
`A gezk knowledge catalog (${c.id}@${c.version}) published by ${c.publisher.name}.`,
|
|
870
|
+
`Language: ${c.language}. License: ${c.license.name} (see ${c.license.noticePath ?? LICENSE_NOTICE_PATH}).`,
|
|
871
|
+
...c.sourceSnapshot ? [`Source snapshot: ${c.sourceSnapshot.name} (${c.sourceSnapshot.date}).`] : [],
|
|
872
|
+
""
|
|
873
|
+
].join("\n");
|
|
874
|
+
}
|
|
875
|
+
function defaultLicenseNotice(opts) {
|
|
876
|
+
const c = opts.catalog;
|
|
877
|
+
const publisher = c.publisher.url ? `${c.publisher.name} <${c.publisher.url}>` : c.publisher.name;
|
|
878
|
+
const license = c.license.spdx ? `${c.license.name} (SPDX: ${c.license.spdx})` : c.license.name;
|
|
879
|
+
return [
|
|
880
|
+
`${c.name} (${c.id}@${c.version})`,
|
|
881
|
+
`Publisher: ${publisher}`,
|
|
882
|
+
`License: ${license}`,
|
|
883
|
+
...c.license.attributionRequired ? ["Attribution is required when redistributing this content."] : [],
|
|
884
|
+
""
|
|
885
|
+
].join("\n");
|
|
886
|
+
}
|
|
840
887
|
function normalizeMarkdown(md) {
|
|
841
888
|
return md.replace(/\r\n/g, "\n").normalize("NFC");
|
|
842
889
|
}
|
|
@@ -854,7 +901,7 @@ ${path.join(" > ")}
|
|
|
854
901
|
return header;
|
|
855
902
|
}
|
|
856
903
|
function seedFor(catalogId, shardId) {
|
|
857
|
-
const digest =
|
|
904
|
+
const digest = createHash("sha256").update(catalogId, "utf8").update(Buffer.from([0])).update(String(shardId), "utf8").digest();
|
|
858
905
|
return digest.readBigUInt64LE(24);
|
|
859
906
|
}
|
|
860
907
|
function makePrng(seed) {
|
|
@@ -939,14 +986,14 @@ function kMeans(samples, k, seed) {
|
|
|
939
986
|
}
|
|
940
987
|
|
|
941
988
|
// src/archive/read.ts
|
|
942
|
-
import { createHash as
|
|
989
|
+
import { createHash as createHash2 } from "crypto";
|
|
943
990
|
import { createWriteStream as createWriteStream2 } from "fs";
|
|
944
991
|
import { mkdir as mkdir2 } from "fs/promises";
|
|
945
992
|
import { createRequire as createRequire4 } from "module";
|
|
946
993
|
import { dirname as dirname2, isAbsolute, normalize, relative, resolve } from "path";
|
|
947
994
|
import { Transform } from "stream";
|
|
948
995
|
import { pipeline } from "stream/promises";
|
|
949
|
-
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/
|
|
996
|
+
import { GEZK_MANIFEST_KIND as GEZK_MANIFEST_KIND2, KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema2 } from "@bendyline/gezk";
|
|
950
997
|
var nodeRequire4 = createRequire4(import.meta.url);
|
|
951
998
|
var GEZK_ARCHIVE_LIMITS = {
|
|
952
999
|
maxEntries: 16384,
|
|
@@ -1036,7 +1083,7 @@ async function inspectEntries(archivePath) {
|
|
|
1036
1083
|
function reconcileEntries(manifest, entries) {
|
|
1037
1084
|
const declared = new Map(manifest.files.map((file) => [file.path, file]));
|
|
1038
1085
|
for (const path of entries.keys()) {
|
|
1039
|
-
if (path === MANIFEST_PATH) continue;
|
|
1086
|
+
if (path === MANIFEST_PATH || path === MIMETYPE_PATH) continue;
|
|
1040
1087
|
if (!declared.has(path)) {
|
|
1041
1088
|
throw new GezkArchiveError(`archive contains undeclared file: ${path}`, "undeclared-file");
|
|
1042
1089
|
}
|
|
@@ -1066,14 +1113,88 @@ async function inspectGezkArchive(archivePath) {
|
|
|
1066
1113
|
totalUncompressedBytes: inspected.totalUncompressedBytes
|
|
1067
1114
|
};
|
|
1068
1115
|
}
|
|
1116
|
+
function assertMimetypeEntry(entry, position) {
|
|
1117
|
+
if (position !== 0) {
|
|
1118
|
+
throw new GezkArchiveError("the mimetype entry must be the first archive entry", "mimetype");
|
|
1119
|
+
}
|
|
1120
|
+
if (entry.compressionMethod !== 0 || entry.uncompressedSize !== GEZK_MIME_TYPE.length) {
|
|
1121
|
+
throw new GezkArchiveError("the mimetype entry must be stored, not compressed", "mimetype");
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
function assertSupportedFormat(raw) {
|
|
1125
|
+
const kind = raw?.kind;
|
|
1126
|
+
const version = raw?.formatVersion;
|
|
1127
|
+
if (kind !== GEZK_MANIFEST_KIND2 || !isSupportedFormatVersion(version)) {
|
|
1128
|
+
throw new GezkArchiveError(
|
|
1129
|
+
`unsupported gezk format (kind ${String(kind)}, version ${String(version)}); this reader supports ${GEZK_MANIFEST_KIND2} ${GEZK_SUPPORTED_FORMAT_VERSIONS.join(", ")} \u2014 catalogs built for another version need a matching reader`,
|
|
1130
|
+
"format-version"
|
|
1131
|
+
);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
function readEntryText(zip, entry) {
|
|
1135
|
+
return new Promise((resolveText, rejectText) => {
|
|
1136
|
+
zip.openReadStream(entry, (err, stream) => {
|
|
1137
|
+
if (err || !stream) {
|
|
1138
|
+
rejectText(new GezkArchiveError(`${entry.fileName} read failed: ${err?.message}`, "io"));
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
const parts = [];
|
|
1142
|
+
stream.on("data", (d) => parts.push(d));
|
|
1143
|
+
stream.on("end", () => resolveText(Buffer.concat(parts).toString("utf8")));
|
|
1144
|
+
stream.on(
|
|
1145
|
+
"error",
|
|
1146
|
+
(streamErr) => rejectText(
|
|
1147
|
+
new GezkArchiveError(`${entry.fileName} read failed: ${streamErr.message}`, "io")
|
|
1148
|
+
)
|
|
1149
|
+
);
|
|
1150
|
+
});
|
|
1151
|
+
});
|
|
1152
|
+
}
|
|
1069
1153
|
async function readGezkManifest(archivePath) {
|
|
1070
1154
|
const zip = await openZip(archivePath);
|
|
1071
1155
|
return new Promise((resolvePromise, reject) => {
|
|
1156
|
+
let position = 0;
|
|
1157
|
+
let mimetypeSeen = false;
|
|
1158
|
+
const fail = (err) => {
|
|
1159
|
+
zip.close();
|
|
1160
|
+
reject(
|
|
1161
|
+
err instanceof GezkArchiveError ? err : new GezkArchiveError(err instanceof Error ? err.message : String(err), "io")
|
|
1162
|
+
);
|
|
1163
|
+
};
|
|
1072
1164
|
zip.on("entry", (entry) => {
|
|
1165
|
+
const index = position++;
|
|
1166
|
+
if (entry.fileName === MIMETYPE_PATH) {
|
|
1167
|
+
try {
|
|
1168
|
+
assertMimetypeEntry(entry, index);
|
|
1169
|
+
} catch (err) {
|
|
1170
|
+
fail(err);
|
|
1171
|
+
return;
|
|
1172
|
+
}
|
|
1173
|
+
readEntryText(zip, entry).then((text) => {
|
|
1174
|
+
if (text !== GEZK_MIME_TYPE) {
|
|
1175
|
+
fail(
|
|
1176
|
+
new GezkArchiveError(
|
|
1177
|
+
`not a gezk archive (mimetype ${JSON.stringify(text)})`,
|
|
1178
|
+
"mimetype"
|
|
1179
|
+
)
|
|
1180
|
+
);
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
mimetypeSeen = true;
|
|
1184
|
+
zip.readEntry();
|
|
1185
|
+
}, fail);
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1073
1188
|
if (entry.fileName !== MANIFEST_PATH) {
|
|
1074
1189
|
zip.readEntry();
|
|
1075
1190
|
return;
|
|
1076
1191
|
}
|
|
1192
|
+
if (!mimetypeSeen) {
|
|
1193
|
+
fail(
|
|
1194
|
+
new GezkArchiveError("archive does not start with the gezk mimetype entry", "mimetype")
|
|
1195
|
+
);
|
|
1196
|
+
return;
|
|
1197
|
+
}
|
|
1077
1198
|
if (entry.uncompressedSize > GEZK_ARCHIVE_LIMITS.maxManifestBytes) {
|
|
1078
1199
|
zip.close();
|
|
1079
1200
|
reject(new GezkArchiveError("manifest exceeds size limit", "limits"));
|
|
@@ -1090,12 +1211,14 @@ async function readGezkManifest(archivePath) {
|
|
|
1090
1211
|
stream.on("end", () => {
|
|
1091
1212
|
zip.close();
|
|
1092
1213
|
try {
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
)
|
|
1097
|
-
);
|
|
1214
|
+
const raw = JSON.parse(Buffer.concat(parts).toString("utf8"));
|
|
1215
|
+
assertSupportedFormat(raw);
|
|
1216
|
+
resolvePromise(KnowledgeCatalogManifestSchema2.parse(raw));
|
|
1098
1217
|
} catch (parseErr) {
|
|
1218
|
+
if (parseErr instanceof GezkArchiveError) {
|
|
1219
|
+
reject(parseErr);
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1099
1222
|
reject(
|
|
1100
1223
|
new GezkArchiveError(
|
|
1101
1224
|
`invalid manifest: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
|
|
@@ -1153,7 +1276,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1153
1276
|
pipeline(stream, createWriteStream2(target)).then(res, rej);
|
|
1154
1277
|
return;
|
|
1155
1278
|
}
|
|
1156
|
-
const hasher =
|
|
1279
|
+
const hasher = createHash2("sha256");
|
|
1157
1280
|
let sizeBytes = 0;
|
|
1158
1281
|
const verify = new Transform({
|
|
1159
1282
|
transform(chunk, _encoding, callback) {
|
|
@@ -1214,9 +1337,7 @@ async function extractGezkVerified(archivePath, destDir) {
|
|
|
1214
1337
|
}
|
|
1215
1338
|
|
|
1216
1339
|
// src/reader/open.ts
|
|
1217
|
-
import { createRequire as createRequire5 } from "module";
|
|
1218
1340
|
import { pathToFileURL } from "url";
|
|
1219
|
-
var nodeRequire5 = createRequire5(import.meta.url);
|
|
1220
1341
|
var CatalogOpenError = class extends Error {
|
|
1221
1342
|
constructor(message, reason) {
|
|
1222
1343
|
super(message);
|
|
@@ -1229,7 +1350,7 @@ function openCatalogDatabase(absPath) {
|
|
|
1229
1350
|
const uri = `${pathToFileURL(absPath).href}?immutable=1`;
|
|
1230
1351
|
let db;
|
|
1231
1352
|
try {
|
|
1232
|
-
db = new DatabaseSync(uri, { readOnly: true
|
|
1353
|
+
db = new DatabaseSync(uri, { readOnly: true });
|
|
1233
1354
|
} catch (err) {
|
|
1234
1355
|
throw new CatalogOpenError(
|
|
1235
1356
|
`cannot open catalog database ${absPath}: ${err instanceof Error ? err.message : String(err)}`,
|
|
@@ -1237,20 +1358,6 @@ function openCatalogDatabase(absPath) {
|
|
|
1237
1358
|
);
|
|
1238
1359
|
}
|
|
1239
1360
|
try {
|
|
1240
|
-
let vecLoaded = false;
|
|
1241
|
-
try {
|
|
1242
|
-
db.enableLoadExtension(true);
|
|
1243
|
-
const sqliteVec = nodeRequire5("sqlite-vec");
|
|
1244
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
1245
|
-
vecLoaded = true;
|
|
1246
|
-
} catch {
|
|
1247
|
-
vecLoaded = false;
|
|
1248
|
-
} finally {
|
|
1249
|
-
try {
|
|
1250
|
-
db.enableLoadExtension(false);
|
|
1251
|
-
} catch {
|
|
1252
|
-
}
|
|
1253
|
-
}
|
|
1254
1361
|
const appId = readPragmaNumber(db, "application_id");
|
|
1255
1362
|
if (appId !== GEZK_APPLICATION_ID) {
|
|
1256
1363
|
throw new CatalogOpenError(
|
|
@@ -1259,19 +1366,17 @@ function openCatalogDatabase(absPath) {
|
|
|
1259
1366
|
);
|
|
1260
1367
|
}
|
|
1261
1368
|
const schemaVersion = readPragmaNumber(db, "user_version");
|
|
1262
|
-
if (schemaVersion
|
|
1369
|
+
if (!isSupportedIndexSchemaVersion(schemaVersion)) {
|
|
1263
1370
|
throw new CatalogOpenError(
|
|
1264
|
-
`unsupported index schema version ${schemaVersion} (reader supports ${
|
|
1371
|
+
`unsupported index schema version ${schemaVersion} (this reader supports ${GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS.join(", ")}; catalogs built for another generation need a matching reader)`,
|
|
1265
1372
|
"schema-version"
|
|
1266
1373
|
);
|
|
1267
1374
|
}
|
|
1268
|
-
if (!vecLoaded) {
|
|
1269
|
-
throw new CatalogOpenError("sqlite-vec extension failed to load", "vec-unavailable");
|
|
1270
|
-
}
|
|
1271
1375
|
db.exec("PRAGMA mmap_size=536870912");
|
|
1272
1376
|
db.exec("PRAGMA cache_size=-8192");
|
|
1273
1377
|
return {
|
|
1274
1378
|
db,
|
|
1379
|
+
schemaVersion,
|
|
1275
1380
|
close: () => {
|
|
1276
1381
|
try {
|
|
1277
1382
|
db.close();
|
|
@@ -1294,12 +1399,94 @@ function readPragmaNumber(db, pragma) {
|
|
|
1294
1399
|
}
|
|
1295
1400
|
|
|
1296
1401
|
// src/reader/catalog-handle.ts
|
|
1402
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
1297
1403
|
import { isAbsolute as isAbsolute2, join as join2, relative as relative2, resolve as resolve2 } from "path";
|
|
1298
1404
|
import { brotliDecompressSync } from "zlib";
|
|
1405
|
+
import {
|
|
1406
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES2,
|
|
1407
|
+
assetContentType,
|
|
1408
|
+
isKnowledgeAssetPath
|
|
1409
|
+
} from "@bendyline/gezk";
|
|
1410
|
+
|
|
1411
|
+
// src/reader/bit-scan.ts
|
|
1412
|
+
var POPCOUNT8 = new Uint8Array(256);
|
|
1413
|
+
for (let i = 1; i < 256; i++) POPCOUNT8[i] = (i & 1) + POPCOUNT8[i >> 1];
|
|
1414
|
+
function popcount32(x) {
|
|
1415
|
+
let v = x - (x >>> 1 & 1431655765);
|
|
1416
|
+
v = (v & 858993459) + (v >>> 2 & 858993459);
|
|
1417
|
+
return Math.imul(v + (v >>> 4) & 252645135, 16843009) >>> 24;
|
|
1418
|
+
}
|
|
1419
|
+
function hammingAll(index, query, out) {
|
|
1420
|
+
const { bits, bytesPerRow, rows } = index;
|
|
1421
|
+
if (bytesPerRow % 4 === 0 && bits.byteOffset % 4 === 0) {
|
|
1422
|
+
const perRow = bytesPerRow / 4;
|
|
1423
|
+
const words = new Uint32Array(bits.buffer, bits.byteOffset, rows * perRow);
|
|
1424
|
+
const aligned = new Uint8Array(bytesPerRow);
|
|
1425
|
+
aligned.set(query);
|
|
1426
|
+
const qwords = new Uint32Array(aligned.buffer, 0, perRow);
|
|
1427
|
+
let offset2 = 0;
|
|
1428
|
+
for (let r = 0; r < rows; r++) {
|
|
1429
|
+
let d = 0;
|
|
1430
|
+
for (let w = 0; w < perRow; w++) {
|
|
1431
|
+
d += popcount32((words[offset2 + w] ^ qwords[w]) >>> 0);
|
|
1432
|
+
}
|
|
1433
|
+
out[r] = d;
|
|
1434
|
+
offset2 += perRow;
|
|
1435
|
+
}
|
|
1436
|
+
return;
|
|
1437
|
+
}
|
|
1438
|
+
let offset = 0;
|
|
1439
|
+
for (let r = 0; r < rows; r++) {
|
|
1440
|
+
let d = 0;
|
|
1441
|
+
for (let b = 0; b < bytesPerRow; b++) {
|
|
1442
|
+
d += POPCOUNT8[bits[offset + b] ^ query[b]];
|
|
1443
|
+
}
|
|
1444
|
+
out[r] = d;
|
|
1445
|
+
offset += bytesPerRow;
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
function hammingTopK(index, query, k) {
|
|
1449
|
+
if (query.length !== index.bytesPerRow) {
|
|
1450
|
+
throw new Error(`query has ${query.length} bytes, shard rows have ${index.bytesPerRow}`);
|
|
1451
|
+
}
|
|
1452
|
+
const limit = Math.min(k, index.rows);
|
|
1453
|
+
if (limit <= 0) return [];
|
|
1454
|
+
const distances = new Uint16Array(index.rows);
|
|
1455
|
+
hammingAll(index, query, distances);
|
|
1456
|
+
const maxDistance = index.bytesPerRow * 8;
|
|
1457
|
+
const histogram = new Uint32Array(maxDistance + 1);
|
|
1458
|
+
for (let r = 0; r < index.rows; r++) {
|
|
1459
|
+
const d = distances[r];
|
|
1460
|
+
histogram[d] = histogram[d] + 1;
|
|
1461
|
+
}
|
|
1462
|
+
let threshold = 0;
|
|
1463
|
+
let seen = 0;
|
|
1464
|
+
for (; threshold <= maxDistance; threshold++) {
|
|
1465
|
+
seen += histogram[threshold];
|
|
1466
|
+
if (seen >= limit) break;
|
|
1467
|
+
}
|
|
1468
|
+
let atThreshold = limit - (seen - histogram[threshold]);
|
|
1469
|
+
const hits = [];
|
|
1470
|
+
for (let r = 0; r < index.rows && hits.length < limit; r++) {
|
|
1471
|
+
const d = distances[r];
|
|
1472
|
+
if (d < threshold) hits.push({ chunkId: r + 1, distance: d });
|
|
1473
|
+
else if (d === threshold && atThreshold > 0) {
|
|
1474
|
+
hits.push({ chunkId: r + 1, distance: d });
|
|
1475
|
+
atThreshold--;
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
hits.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
|
|
1479
|
+
return hits;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
// src/reader/catalog-handle.ts
|
|
1483
|
+
var TOPIC_WALK_MAX_DEPTH = 32;
|
|
1484
|
+
var BIT_INDEX_BUDGET_BYTES = 256 * 1024 * 1024;
|
|
1299
1485
|
var CatalogHandle = class _CatalogHandle {
|
|
1300
1486
|
constructor(rootDir, router) {
|
|
1301
1487
|
this.rootDir = rootDir;
|
|
1302
1488
|
this.router = router;
|
|
1489
|
+
this.schemaVersion = router.schemaVersion;
|
|
1303
1490
|
this.meta = {};
|
|
1304
1491
|
for (const row of router.db.prepare("SELECT key, value FROM meta").all()) {
|
|
1305
1492
|
this.meta[row.key] = row.value;
|
|
@@ -1310,8 +1497,12 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1310
1497
|
rootDir;
|
|
1311
1498
|
router;
|
|
1312
1499
|
connections = /* @__PURE__ */ new Map();
|
|
1500
|
+
bitIndexes = /* @__PURE__ */ new Map();
|
|
1501
|
+
assetsByPath = null;
|
|
1313
1502
|
meta;
|
|
1314
1503
|
shards;
|
|
1504
|
+
/** The router's `PRAGMA user_version` generation; decides which columns exist. */
|
|
1505
|
+
schemaVersion;
|
|
1315
1506
|
/** Open a catalog's extracted version directory (the dir holding manifest.json). */
|
|
1316
1507
|
static open(rootDir) {
|
|
1317
1508
|
const router = openCatalogDatabase(join2(rootDir, ROUTER_DB_PATH));
|
|
@@ -1320,6 +1511,7 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1320
1511
|
close() {
|
|
1321
1512
|
for (const conn of this.connections.values()) conn.close();
|
|
1322
1513
|
this.connections.clear();
|
|
1514
|
+
this.bitIndexes.clear();
|
|
1323
1515
|
}
|
|
1324
1516
|
/** Integrity check for the router connection, used by deep validation. */
|
|
1325
1517
|
routerQuickCheck() {
|
|
@@ -1350,6 +1542,57 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1350
1542
|
}
|
|
1351
1543
|
return conn.db;
|
|
1352
1544
|
}
|
|
1545
|
+
/**
|
|
1546
|
+
* A shard's sign-bit rows as one contiguous array, loaded on first use and
|
|
1547
|
+
* kept while the resident budget allows. Chunk ids must be dense from 1 —
|
|
1548
|
+
* the row index IS the id — so a gap is a corrupt shard, not a sparse one.
|
|
1549
|
+
*/
|
|
1550
|
+
shardBits(shard) {
|
|
1551
|
+
const cached = this.bitIndexes.get(shard.path);
|
|
1552
|
+
if (cached) return cached;
|
|
1553
|
+
const db = this.shardDb(shard);
|
|
1554
|
+
const span = db.prepare(
|
|
1555
|
+
"SELECT COUNT(*) AS n, MIN(chunk_id) AS lo, MAX(chunk_id) AS hi FROM chunk_vectors_bit"
|
|
1556
|
+
).get();
|
|
1557
|
+
const rows = Number(span.n);
|
|
1558
|
+
if (rows > 0 && (Number(span.lo) !== 1 || Number(span.hi) !== rows)) {
|
|
1559
|
+
throw new CatalogOpenError(`chunk ids are not dense in ${shard.path}`, "corrupt");
|
|
1560
|
+
}
|
|
1561
|
+
const bytesPerRow = Math.ceil(this.dimensions() / 8);
|
|
1562
|
+
const bits = new Uint8Array(rows * bytesPerRow);
|
|
1563
|
+
const rowsIter = db.prepare("SELECT chunk_id, v FROM chunk_vectors_bit ORDER BY chunk_id").iterate();
|
|
1564
|
+
for (const row of rowsIter) {
|
|
1565
|
+
if (row.v.byteLength !== bytesPerRow) {
|
|
1566
|
+
throw new CatalogOpenError(
|
|
1567
|
+
`bit vector width ${row.v.byteLength} != ${bytesPerRow} in ${shard.path}`,
|
|
1568
|
+
"corrupt"
|
|
1569
|
+
);
|
|
1570
|
+
}
|
|
1571
|
+
bits.set(row.v, (Number(row.chunk_id) - 1) * bytesPerRow);
|
|
1572
|
+
}
|
|
1573
|
+
const index = { bits, bytesPerRow, rows };
|
|
1574
|
+
let resident = bits.byteLength;
|
|
1575
|
+
for (const other of this.bitIndexes.values()) resident += other.bits.byteLength;
|
|
1576
|
+
for (const [key, other] of this.bitIndexes) {
|
|
1577
|
+
if (resident <= BIT_INDEX_BUDGET_BYTES) break;
|
|
1578
|
+
this.bitIndexes.delete(key);
|
|
1579
|
+
resident -= other.bits.byteLength;
|
|
1580
|
+
}
|
|
1581
|
+
this.bitIndexes.set(shard.path, index);
|
|
1582
|
+
return index;
|
|
1583
|
+
}
|
|
1584
|
+
/** The embedding dimension from the router's profile echo. */
|
|
1585
|
+
dimensions() {
|
|
1586
|
+
const raw = this.meta.embedding_profile_json;
|
|
1587
|
+
if (raw) {
|
|
1588
|
+
try {
|
|
1589
|
+
const dims = JSON.parse(raw).dimensions;
|
|
1590
|
+
if (typeof dims === "number" && dims > 0) return dims;
|
|
1591
|
+
} catch {
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
throw new CatalogOpenError("router meta lacks a usable embedding profile", "corrupt");
|
|
1595
|
+
}
|
|
1353
1596
|
/** Resolve a manifest/router path without allowing it to escape the catalog. */
|
|
1354
1597
|
resolveCatalogPath(path) {
|
|
1355
1598
|
const root = resolve2(this.rootDir);
|
|
@@ -1361,40 +1604,124 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1361
1604
|
return target;
|
|
1362
1605
|
}
|
|
1363
1606
|
// ── browsing (the shipped table of contents) ──────────────────────────────
|
|
1607
|
+
/** The format version the router was written with (`meta.format_version`). */
|
|
1608
|
+
get formatVersion() {
|
|
1609
|
+
return this.meta.format_version ?? "";
|
|
1610
|
+
}
|
|
1611
|
+
documentColumns() {
|
|
1612
|
+
const base = "id, title, slug, summary, language, topic_id, source_url, source_revision, source_updated_at, attribution_json";
|
|
1613
|
+
return this.schemaVersion >= 3 ? `${base}, ordinal, meta_json` : base;
|
|
1614
|
+
}
|
|
1615
|
+
documentOrder() {
|
|
1616
|
+
return this.schemaVersion >= 3 ? "ORDER BY (ordinal IS NULL), ordinal, slug, id" : "ORDER BY slug, id";
|
|
1617
|
+
}
|
|
1618
|
+
/**
|
|
1619
|
+
* Every topic with its direct count and a rollup over its subtree. On a
|
|
1620
|
+
* 0.5 catalog every document sits at a root, so both counts agree there.
|
|
1621
|
+
*/
|
|
1364
1622
|
topics() {
|
|
1365
1623
|
return this.router.db.prepare(
|
|
1366
|
-
|
|
1624
|
+
`WITH RECURSIVE sub(root, id, depth) AS (
|
|
1625
|
+
SELECT id, id, 0 FROM topics
|
|
1626
|
+
UNION ALL
|
|
1627
|
+
SELECT sub.root, t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1628
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1629
|
+
),
|
|
1630
|
+
rollup AS (
|
|
1631
|
+
SELECT sub.root AS id, SUM(t.document_count) AS total
|
|
1632
|
+
FROM sub JOIN topics t ON t.id = sub.id GROUP BY sub.root
|
|
1633
|
+
)
|
|
1634
|
+
SELECT t.id, t.parent_id, t.name, t.description, t.sort_key, t.document_count,
|
|
1635
|
+
rollup.total AS total_document_count
|
|
1636
|
+
FROM topics t JOIN rollup ON rollup.id = t.id
|
|
1637
|
+
ORDER BY t.sort_key, t.id`
|
|
1367
1638
|
).all().map((r) => ({
|
|
1368
1639
|
id: r.id,
|
|
1369
1640
|
parentId: r.parent_id,
|
|
1370
1641
|
name: r.name,
|
|
1371
1642
|
description: r.description,
|
|
1372
1643
|
sortKey: r.sort_key,
|
|
1373
|
-
documentCount: Number(r.document_count)
|
|
1644
|
+
documentCount: Number(r.document_count),
|
|
1645
|
+
totalDocumentCount: Number(r.total_document_count)
|
|
1374
1646
|
}));
|
|
1375
1647
|
}
|
|
1648
|
+
/**
|
|
1649
|
+
* A page of documents, by default including those filed under the topic's
|
|
1650
|
+
* descendants (`descendants: false` lists only the topic's own).
|
|
1651
|
+
*/
|
|
1376
1652
|
documentsPage(opts = {}) {
|
|
1377
1653
|
const limit = Math.min(200, Math.max(1, opts.limit ?? 50));
|
|
1378
1654
|
const offset = Math.max(0, opts.offset ?? 0);
|
|
1379
|
-
const
|
|
1380
|
-
|
|
1655
|
+
const descendants = opts.descendants ?? true;
|
|
1656
|
+
let scope = "";
|
|
1657
|
+
let where = "";
|
|
1658
|
+
const params = [];
|
|
1659
|
+
if (opts.topicId && descendants) {
|
|
1660
|
+
scope = `WITH RECURSIVE sub(id, depth) AS (
|
|
1661
|
+
SELECT ?, 0
|
|
1662
|
+
UNION ALL
|
|
1663
|
+
SELECT t.id, sub.depth + 1 FROM topics t JOIN sub ON t.parent_id = sub.id
|
|
1664
|
+
WHERE sub.depth < ${TOPIC_WALK_MAX_DEPTH}
|
|
1665
|
+
)`;
|
|
1666
|
+
where = "WHERE topic_id IN (SELECT id FROM sub)";
|
|
1667
|
+
params.push(opts.topicId);
|
|
1668
|
+
} else if (opts.topicId) {
|
|
1669
|
+
where = "WHERE topic_id = ?";
|
|
1670
|
+
params.push(opts.topicId);
|
|
1671
|
+
}
|
|
1381
1672
|
const total = Number(
|
|
1382
|
-
this.router.db.prepare(
|
|
1673
|
+
this.router.db.prepare(`${scope} SELECT COUNT(*) AS n FROM documents ${where}`).get(...params).n
|
|
1383
1674
|
);
|
|
1384
1675
|
const documents = this.router.db.prepare(
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
FROM documents ${where} ORDER BY slug, id LIMIT ? OFFSET ?`
|
|
1676
|
+
`${scope} SELECT ${this.documentColumns()}
|
|
1677
|
+
FROM documents ${where} ${this.documentOrder()} LIMIT ? OFFSET ?`
|
|
1388
1678
|
).all(...params, limit, offset).map(rowToDocumentMeta);
|
|
1389
1679
|
return { documents, total };
|
|
1390
1680
|
}
|
|
1391
1681
|
getDocument(id) {
|
|
1392
1682
|
const row = this.router.db.prepare(
|
|
1393
|
-
`SELECT
|
|
1394
|
-
source_updated_at, attribution_json, body_codec, body_blob
|
|
1683
|
+
`SELECT ${this.documentColumns()}, body_codec, body_blob
|
|
1395
1684
|
FROM documents WHERE id = ?`
|
|
1396
1685
|
).get(id);
|
|
1397
1686
|
if (!row) return null;
|
|
1687
|
+
return { ...rowToDocumentMeta(row), markdown: this.decodeBody(id, row) };
|
|
1688
|
+
}
|
|
1689
|
+
/** Documents whose `topic_id` names no declared topic (a validator check). */
|
|
1690
|
+
documentsWithUndeclaredTopic() {
|
|
1691
|
+
return Number(
|
|
1692
|
+
this.router.db.prepare(
|
|
1693
|
+
"SELECT COUNT(*) AS n FROM documents d LEFT JOIN topics t ON t.id = d.topic_id WHERE t.id IS NULL"
|
|
1694
|
+
).get().n
|
|
1695
|
+
);
|
|
1696
|
+
}
|
|
1697
|
+
/**
|
|
1698
|
+
* Validator support: every `meta_json` must be a JSON object within the
|
|
1699
|
+
* byte limit. Returns the offending ids (capped) rather than throwing.
|
|
1700
|
+
*/
|
|
1701
|
+
checkDocumentMeta(maxBytes, cap = 5) {
|
|
1702
|
+
const invalid = [];
|
|
1703
|
+
const oversize = [];
|
|
1704
|
+
if (this.schemaVersion < 3) return { invalid, oversize };
|
|
1705
|
+
const rows = this.router.db.prepare("SELECT id, meta_json FROM documents WHERE meta_json IS NOT NULL").iterate();
|
|
1706
|
+
for (const row of rows) {
|
|
1707
|
+
if (invalid.length >= cap && oversize.length >= cap) break;
|
|
1708
|
+
if (Buffer.byteLength(row.meta_json, "utf8") > maxBytes) {
|
|
1709
|
+
if (oversize.length < cap) oversize.push(row.id);
|
|
1710
|
+
continue;
|
|
1711
|
+
}
|
|
1712
|
+
if (parseJsonObject(row.meta_json) === null && invalid.length < cap) invalid.push(row.id);
|
|
1713
|
+
}
|
|
1714
|
+
return { invalid, oversize };
|
|
1715
|
+
}
|
|
1716
|
+
/** Validator support: decoded bodies, streamed one at a time. */
|
|
1717
|
+
*documentBodies() {
|
|
1718
|
+
const rows = this.router.db.prepare("SELECT id, body_codec, body_blob FROM documents ORDER BY id").iterate();
|
|
1719
|
+
for (const row of rows) {
|
|
1720
|
+
const id = row.id;
|
|
1721
|
+
yield { id, markdown: this.decodeBody(id, row) };
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
decodeBody(id, row) {
|
|
1398
1725
|
const blob = row.body_blob;
|
|
1399
1726
|
if (blob.byteLength > MAX_KNOWLEDGE_DOCUMENT_BYTES + 1024) {
|
|
1400
1727
|
throw new CatalogOpenError(`document body exceeds the stored-size limit: ${id}`, "corrupt");
|
|
@@ -1417,21 +1744,82 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1417
1744
|
} else {
|
|
1418
1745
|
throw new CatalogOpenError(`unknown document body codec for ${id}`, "corrupt");
|
|
1419
1746
|
}
|
|
1420
|
-
|
|
1421
|
-
return { ...rowToDocumentMeta(row), markdown };
|
|
1747
|
+
return body.toString("utf8");
|
|
1422
1748
|
}
|
|
1423
|
-
// ──
|
|
1424
|
-
/**
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1749
|
+
// ── assets ────────────────────────────────────────────────────────────────
|
|
1750
|
+
/**
|
|
1751
|
+
* The manifest's `files` entries under `assets/`, read lazily: the
|
|
1752
|
+
* declaration in the manifest is what authorizes serving a file, since
|
|
1753
|
+
* extraction reconciled and hashed every declared entry.
|
|
1754
|
+
*/
|
|
1755
|
+
assetIndex() {
|
|
1756
|
+
if (this.assetsByPath) return this.assetsByPath;
|
|
1757
|
+
const index = /* @__PURE__ */ new Map();
|
|
1758
|
+
let raw;
|
|
1428
1759
|
try {
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1760
|
+
raw = JSON.parse(readFileSync2(join2(this.rootDir, MANIFEST_PATH), "utf8"));
|
|
1761
|
+
} catch (error) {
|
|
1762
|
+
if (error.code === "ENOENT") {
|
|
1763
|
+
this.assetsByPath = index;
|
|
1764
|
+
return index;
|
|
1765
|
+
}
|
|
1766
|
+
throw new CatalogOpenError(
|
|
1767
|
+
`cannot read ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1768
|
+
"corrupt"
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
const files = raw?.files;
|
|
1772
|
+
if (Array.isArray(files)) {
|
|
1773
|
+
for (const file of files) {
|
|
1774
|
+
const path = file?.path;
|
|
1775
|
+
if (typeof path !== "string" || !isKnowledgeAssetPath(path)) continue;
|
|
1776
|
+
const contentType = assetContentType(path);
|
|
1777
|
+
if (!contentType || typeof file.sizeBytes !== "number" || typeof file.sha256 !== "string") {
|
|
1778
|
+
continue;
|
|
1779
|
+
}
|
|
1780
|
+
index.set(path, { path, contentType, sizeBytes: file.sizeBytes, sha256: file.sha256 });
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
this.assetsByPath = index;
|
|
1784
|
+
return index;
|
|
1785
|
+
}
|
|
1786
|
+
/** Declared assets, sorted by path. */
|
|
1787
|
+
assets() {
|
|
1788
|
+
return [...this.assetIndex().values()].sort((a, b) => a.path < b.path ? -1 : 1);
|
|
1789
|
+
}
|
|
1790
|
+
/** One declared asset's bytes, or null when the catalog declares no such asset. */
|
|
1791
|
+
readAsset(path) {
|
|
1792
|
+
const info = this.assetIndex().get(path);
|
|
1793
|
+
if (!info) return null;
|
|
1794
|
+
if (info.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES2) {
|
|
1795
|
+
throw new CatalogOpenError(`asset exceeds the size limit: ${path}`, "corrupt");
|
|
1796
|
+
}
|
|
1797
|
+
let bytes;
|
|
1798
|
+
try {
|
|
1799
|
+
bytes = readFileSync2(this.resolveCatalogPath(path));
|
|
1800
|
+
} catch (error) {
|
|
1801
|
+
throw new CatalogOpenError(
|
|
1802
|
+
`cannot read asset ${path}: ${error instanceof Error ? error.message : String(error)}`,
|
|
1803
|
+
"corrupt"
|
|
1804
|
+
);
|
|
1805
|
+
}
|
|
1806
|
+
if (bytes.byteLength !== info.sizeBytes) {
|
|
1807
|
+
throw new CatalogOpenError(`asset size differs from the manifest: ${path}`, "corrupt");
|
|
1808
|
+
}
|
|
1809
|
+
return { ...info, bytes: new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength) };
|
|
1810
|
+
}
|
|
1811
|
+
// ── search ────────────────────────────────────────────────────────────────
|
|
1812
|
+
/** Catalog-wide title/summary/alias FTS — always runs regardless of routing. */
|
|
1813
|
+
searchDocumentsFts(query, limit = 10) {
|
|
1814
|
+
const match = ftsQuery(query);
|
|
1815
|
+
if (!match) return [];
|
|
1816
|
+
try {
|
|
1817
|
+
return documentFtsTopIds(this.router.db, match, limit).map((documentId, i) => ({
|
|
1818
|
+
documentId,
|
|
1819
|
+
rank: i
|
|
1820
|
+
}));
|
|
1821
|
+
} catch {
|
|
1822
|
+
return [];
|
|
1435
1823
|
}
|
|
1436
1824
|
}
|
|
1437
1825
|
/**
|
|
@@ -1487,16 +1875,16 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1487
1875
|
* after global cross-catalog routing has already spent the S budget.
|
|
1488
1876
|
*/
|
|
1489
1877
|
searchShards(queryVector, shardIds, finalK) {
|
|
1490
|
-
const queryBits =
|
|
1878
|
+
const queryBits = quantizeBinary(queryVector);
|
|
1491
1879
|
const hits = [];
|
|
1492
1880
|
for (const shardId of shardIds) {
|
|
1493
1881
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1494
1882
|
if (!shard) continue;
|
|
1495
1883
|
const db = this.shardDb(shard);
|
|
1496
1884
|
const k = rerankK(finalK, shard.chunkCount);
|
|
1497
|
-
const candidates =
|
|
1498
|
-
|
|
1499
|
-
)
|
|
1885
|
+
const candidates = hammingTopK(this.shardBits(shard), queryBits, k).map((hit) => ({
|
|
1886
|
+
chunk_id: hit.chunkId
|
|
1887
|
+
}));
|
|
1500
1888
|
if (candidates.length === 0) continue;
|
|
1501
1889
|
const getInt8 = db.prepare("SELECT v FROM chunk_vectors_int8 WHERE chunk_id = ?");
|
|
1502
1890
|
const reranked = candidates.map((c) => {
|
|
@@ -1563,35 +1951,35 @@ var CatalogHandle = class _CatalogHandle {
|
|
|
1563
1951
|
return hits;
|
|
1564
1952
|
}
|
|
1565
1953
|
/**
|
|
1566
|
-
* Embedder-free integrity smoke
|
|
1567
|
-
*
|
|
1568
|
-
*
|
|
1954
|
+
* Embedder-free integrity smoke: chunk 1's own bit vector must be its own
|
|
1955
|
+
* nearest neighbor — proves the shard's vectors load and scan. Returns
|
|
1956
|
+
* false instead of throwing (callers report a reason).
|
|
1569
1957
|
*/
|
|
1570
1958
|
selfKnnSmoke(shardId) {
|
|
1571
1959
|
const shard = this.shards.find((s) => s.id === shardId);
|
|
1572
1960
|
if (!shard) return false;
|
|
1573
1961
|
try {
|
|
1574
|
-
const
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
const nearest =
|
|
1578
|
-
|
|
1579
|
-
).get(Buffer.from(first.embedding));
|
|
1580
|
-
return nearest !== void 0 && Number(nearest.chunk_id) === 1;
|
|
1962
|
+
const index = this.shardBits(shard);
|
|
1963
|
+
if (index.rows === 0) return false;
|
|
1964
|
+
const first = index.bits.subarray(0, index.bytesPerRow);
|
|
1965
|
+
const nearest = hammingTopK(index, first, 1)[0];
|
|
1966
|
+
return nearest !== void 0 && nearest.chunkId === 1 && nearest.distance === 0;
|
|
1581
1967
|
} catch {
|
|
1582
1968
|
return false;
|
|
1583
1969
|
}
|
|
1584
1970
|
}
|
|
1585
1971
|
};
|
|
1586
|
-
function
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
}
|
|
1972
|
+
function parseJsonObject(raw) {
|
|
1973
|
+
if (typeof raw !== "string") return null;
|
|
1974
|
+
try {
|
|
1975
|
+
const parsed = JSON.parse(raw);
|
|
1976
|
+
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
|
|
1977
|
+
} catch {
|
|
1978
|
+
return null;
|
|
1594
1979
|
}
|
|
1980
|
+
}
|
|
1981
|
+
function rowToDocumentMeta(row) {
|
|
1982
|
+
const ordinal = row.ordinal;
|
|
1595
1983
|
return {
|
|
1596
1984
|
id: row.id,
|
|
1597
1985
|
title: row.title,
|
|
@@ -1599,10 +1987,12 @@ function rowToDocumentMeta(row) {
|
|
|
1599
1987
|
summary: row.summary ?? null,
|
|
1600
1988
|
language: row.language,
|
|
1601
1989
|
topicId: row.topic_id,
|
|
1990
|
+
ordinal: typeof ordinal === "number" || typeof ordinal === "bigint" ? Number(ordinal) : null,
|
|
1602
1991
|
sourceUrl: row.source_url ?? null,
|
|
1603
1992
|
sourceRevision: row.source_revision ?? null,
|
|
1604
1993
|
sourceUpdatedAt: row.source_updated_at ?? null,
|
|
1605
|
-
attribution
|
|
1994
|
+
attribution: parseJsonObject(row.attribution_json),
|
|
1995
|
+
meta: parseJsonObject(row.meta_json)
|
|
1606
1996
|
};
|
|
1607
1997
|
}
|
|
1608
1998
|
function parseHeadingPath(raw) {
|
|
@@ -1618,7 +2008,17 @@ var ftsQuery = sanitizeFtsQuery;
|
|
|
1618
2008
|
// src/reader/validate.ts
|
|
1619
2009
|
import { readFile } from "fs/promises";
|
|
1620
2010
|
import { join as join3 } from "path";
|
|
1621
|
-
import {
|
|
2011
|
+
import {
|
|
2012
|
+
KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema3,
|
|
2013
|
+
MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES as MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2014
|
+
MAX_KNOWLEDGE_ASSET_BYTES as MAX_KNOWLEDGE_ASSET_BYTES3,
|
|
2015
|
+
MAX_KNOWLEDGE_ASSET_COUNT as MAX_KNOWLEDGE_ASSET_COUNT2,
|
|
2016
|
+
assetExtension as assetExtension2,
|
|
2017
|
+
assetKindForExtension as assetKindForExtension2,
|
|
2018
|
+
isKnowledgeAssetPath as isKnowledgeAssetPath2,
|
|
2019
|
+
sniffAssetType as sniffAssetType2,
|
|
2020
|
+
svgInertnessProblem as svgInertnessProblem2
|
|
2021
|
+
} from "@bendyline/gezk";
|
|
1622
2022
|
async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
1623
2023
|
const checks = [];
|
|
1624
2024
|
const check = (name, ok, detail) => {
|
|
@@ -1685,8 +2085,48 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1685
2085
|
handle.meta.embedding_profile_id === manifest.embedding.id,
|
|
1686
2086
|
`router meta profile ${handle.meta.embedding_profile_id}`
|
|
1687
2087
|
);
|
|
2088
|
+
check(
|
|
2089
|
+
"meta-format",
|
|
2090
|
+
handle.formatVersion === manifest.formatVersion && handle.schemaVersion === manifest.indexSchemaVersion,
|
|
2091
|
+
`router says format ${handle.formatVersion} / schema ${handle.schemaVersion}, manifest says ${manifest.formatVersion} / ${manifest.indexSchemaVersion}`
|
|
2092
|
+
);
|
|
1688
2093
|
const topics = handle.topics();
|
|
1689
2094
|
check("toc-present", topics.length >= 1, `${topics.length} topics`);
|
|
2095
|
+
check("topics-tree", ...topicTreeProblem(topics));
|
|
2096
|
+
const undeclared = handle.documentsWithUndeclaredTopic();
|
|
2097
|
+
check(
|
|
2098
|
+
"documents-topic-declared",
|
|
2099
|
+
undeclared === 0,
|
|
2100
|
+
`${undeclared} documents are filed under a topic the router does not declare`
|
|
2101
|
+
);
|
|
2102
|
+
const assetFiles = manifest.files.filter((f) => f.path.startsWith("assets/"));
|
|
2103
|
+
if (manifest.formatVersion === "0.5") {
|
|
2104
|
+
check(
|
|
2105
|
+
"assets-not-in-0.5",
|
|
2106
|
+
assetFiles.length === 0,
|
|
2107
|
+
`${assetFiles.length} assets/ entries in a 0.5 catalog`
|
|
2108
|
+
);
|
|
2109
|
+
} else {
|
|
2110
|
+
const badPaths = assetFiles.filter((f) => !isKnowledgeAssetPath2(f.path)).map((f) => f.path);
|
|
2111
|
+
check("assets-paths", badPaths.length === 0, `invalid asset paths: ${badPaths.join(", ")}`);
|
|
2112
|
+
const oversize = assetFiles.filter((f) => f.sizeBytes > MAX_KNOWLEDGE_ASSET_BYTES3);
|
|
2113
|
+
const totalBytes = assetFiles.reduce((sum, f) => sum + f.sizeBytes, 0);
|
|
2114
|
+
check(
|
|
2115
|
+
"assets-limits",
|
|
2116
|
+
assetFiles.length <= MAX_KNOWLEDGE_ASSET_COUNT2 && oversize.length === 0 && totalBytes <= MAX_KNOWLEDGE_ASSETS_TOTAL_BYTES2,
|
|
2117
|
+
`${assetFiles.length} assets, ${totalBytes} bytes, ${oversize.length} over the per-asset limit`
|
|
2118
|
+
);
|
|
2119
|
+
check(
|
|
2120
|
+
"counts-assets",
|
|
2121
|
+
(manifest.counts.assets ?? 0) === assetFiles.length,
|
|
2122
|
+
`manifest counts ${manifest.counts.assets ?? 0} assets, files declare ${assetFiles.length}`
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
2125
|
+
check(
|
|
2126
|
+
"license-notice",
|
|
2127
|
+
manifest.files.some((f) => f.path === manifest.license.noticePath),
|
|
2128
|
+
`manifest.files lacks the declared notice ${manifest.license.noticePath}`
|
|
2129
|
+
);
|
|
1690
2130
|
const topicDocSum = topics.reduce((sum, t) => sum + t.documentCount, 0);
|
|
1691
2131
|
check(
|
|
1692
2132
|
"counts-documents",
|
|
@@ -1737,7 +2177,7 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1737
2177
|
db.prepare("SELECT COUNT(*) AS n FROM chunks").get().n
|
|
1738
2178
|
);
|
|
1739
2179
|
const vecCount = Number(
|
|
1740
|
-
db.prepare("SELECT COUNT(*) AS n FROM
|
|
2180
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit").get().n
|
|
1741
2181
|
);
|
|
1742
2182
|
const int8Count = Number(
|
|
1743
2183
|
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8").get().n
|
|
@@ -1747,11 +2187,68 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1747
2187
|
chunkCount === shard.chunkCount && vecCount === chunkCount && int8Count === chunkCount,
|
|
1748
2188
|
`chunks ${chunkCount}, vec ${vecCount}, int8 ${int8Count}, shards row ${shard.chunkCount}`
|
|
1749
2189
|
);
|
|
2190
|
+
const dims = manifest.embedding.dimensions;
|
|
2191
|
+
const badBit = Number(
|
|
2192
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_bit WHERE length(v) != ?").get(Math.ceil(dims / 8)).n
|
|
2193
|
+
);
|
|
2194
|
+
const badInt8 = Number(
|
|
2195
|
+
db.prepare("SELECT COUNT(*) AS n FROM chunk_vectors_int8 WHERE length(v) != ?").get(dims).n
|
|
2196
|
+
);
|
|
2197
|
+
check(
|
|
2198
|
+
`vector-widths:${shard.path}`,
|
|
2199
|
+
badBit === 0 && badInt8 === 0,
|
|
2200
|
+
`${badBit} bit rows and ${badInt8} int8 rows have the wrong width for ${dims} dimensions`
|
|
2201
|
+
);
|
|
2202
|
+
const span = db.prepare("SELECT MIN(id) AS lo, MAX(id) AS hi FROM chunks").get();
|
|
2203
|
+
check(
|
|
2204
|
+
`chunk-ids-dense:${shard.path}`,
|
|
2205
|
+
chunkCount === 0 || Number(span.lo) === 1 && Number(span.hi) === chunkCount,
|
|
2206
|
+
`ids span ${String(span.lo)}..${String(span.hi)} for ${chunkCount} chunks`
|
|
2207
|
+
);
|
|
1750
2208
|
} finally {
|
|
1751
2209
|
conn.close();
|
|
1752
2210
|
}
|
|
1753
2211
|
check(`self-knn:${shard.id}`, handle.selfKnnSmoke(shard.id));
|
|
1754
2212
|
}
|
|
2213
|
+
if (handle.schemaVersion >= 3) {
|
|
2214
|
+
const meta = handle.checkDocumentMeta(MAX_KNOWLEDGE_DOCUMENT_META_BYTES);
|
|
2215
|
+
check(
|
|
2216
|
+
"document-meta-json",
|
|
2217
|
+
meta.invalid.length === 0 && meta.oversize.length === 0,
|
|
2218
|
+
`${meta.invalid.length > 0 ? `not a JSON object: ${meta.invalid.join(", ")}` : ""}${meta.oversize.length > 0 ? ` over ${MAX_KNOWLEDGE_DOCUMENT_META_BYTES} bytes: ${meta.oversize.join(", ")}` : ""}`.trim()
|
|
2219
|
+
);
|
|
2220
|
+
const declared = new Set(assetFiles.map((f) => f.path));
|
|
2221
|
+
for (const file of assetFiles) {
|
|
2222
|
+
const ext = assetExtension2(file.path);
|
|
2223
|
+
if (!ext) continue;
|
|
2224
|
+
const bytes = await readFile(join3(rootDir, file.path));
|
|
2225
|
+
const kind = sniffAssetType2(bytes);
|
|
2226
|
+
check(
|
|
2227
|
+
`asset-type:${file.path}`,
|
|
2228
|
+
kind === assetKindForExtension2(ext),
|
|
2229
|
+
`leading bytes say ${kind ?? "unknown"}, the extension says ${assetKindForExtension2(ext)}`
|
|
2230
|
+
);
|
|
2231
|
+
if (ext === "svg") {
|
|
2232
|
+
const problem = svgInertnessProblem2(bytes);
|
|
2233
|
+
check(`asset-svg-inert:${file.path}`, problem === null, problem ?? void 0);
|
|
2234
|
+
}
|
|
2235
|
+
}
|
|
2236
|
+
if (declared.size > 0) {
|
|
2237
|
+
const missing = [];
|
|
2238
|
+
for (const doc of handle.documentBodies()) {
|
|
2239
|
+
for (const target of assetReferences(doc.markdown)) {
|
|
2240
|
+
if (!declared.has(target)) missing.push(`${doc.id} \u2192 ${target}`);
|
|
2241
|
+
if (missing.length >= 5) break;
|
|
2242
|
+
}
|
|
2243
|
+
if (missing.length >= 5) break;
|
|
2244
|
+
}
|
|
2245
|
+
check(
|
|
2246
|
+
"document-asset-refs",
|
|
2247
|
+
missing.length === 0,
|
|
2248
|
+
`undeclared asset references: ${missing.join("; ")}`
|
|
2249
|
+
);
|
|
2250
|
+
}
|
|
2251
|
+
}
|
|
1755
2252
|
for (const smoke of manifest.smokeQueries ?? []) {
|
|
1756
2253
|
const hits = handle.searchDocumentsFts(smoke.query, SMOKE_QUERY_TOP_N).map((h) => h.documentId);
|
|
1757
2254
|
const missing = smoke.expectedDocumentIds.filter((id) => !hits.includes(id));
|
|
@@ -1767,6 +2264,528 @@ async function validateExtractedCatalog(rootDir, opts = {}) {
|
|
|
1767
2264
|
}
|
|
1768
2265
|
return report(manifest);
|
|
1769
2266
|
}
|
|
2267
|
+
function assetReferences(markdown) {
|
|
2268
|
+
const found = /* @__PURE__ */ new Set();
|
|
2269
|
+
for (const match of markdown.matchAll(/\]\(\s*<?(assets\/[^)\s>]+)/g)) {
|
|
2270
|
+
const target = match[1];
|
|
2271
|
+
if (target) found.add(target);
|
|
2272
|
+
}
|
|
2273
|
+
return [...found];
|
|
2274
|
+
}
|
|
2275
|
+
function topicTreeProblem(topics) {
|
|
2276
|
+
const byId = new Map(topics.map((t) => [t.id, t]));
|
|
2277
|
+
for (const topic of topics) {
|
|
2278
|
+
if (topic.parentId !== null && !byId.has(topic.parentId)) {
|
|
2279
|
+
return [false, `topic '${topic.id}' names an undeclared parent '${topic.parentId}'`];
|
|
2280
|
+
}
|
|
2281
|
+
}
|
|
2282
|
+
const depthOf = /* @__PURE__ */ new Map();
|
|
2283
|
+
for (const topic of topics) {
|
|
2284
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2285
|
+
let depth = 0;
|
|
2286
|
+
let current = topic;
|
|
2287
|
+
while (current) {
|
|
2288
|
+
if (seen.has(current.id)) return [false, `topic '${topic.id}' sits in a parent cycle`];
|
|
2289
|
+
seen.add(current.id);
|
|
2290
|
+
depth += 1;
|
|
2291
|
+
if (depth > MAX_KNOWLEDGE_TOPIC_DEPTH) {
|
|
2292
|
+
return [false, `topic '${topic.id}' is deeper than ${MAX_KNOWLEDGE_TOPIC_DEPTH}`];
|
|
2293
|
+
}
|
|
2294
|
+
current = current.parentId === null ? void 0 : byId.get(current.parentId);
|
|
2295
|
+
}
|
|
2296
|
+
depthOf.set(topic.id, depth);
|
|
2297
|
+
}
|
|
2298
|
+
return [true, `${topics.length} topics, max depth ${Math.max(0, ...depthOf.values())}`];
|
|
2299
|
+
}
|
|
2300
|
+
|
|
2301
|
+
// src/export/duckdb.ts
|
|
2302
|
+
import { spawn } from "child_process";
|
|
2303
|
+
import { access } from "fs/promises";
|
|
2304
|
+
var DuckdbError = class extends Error {
|
|
2305
|
+
constructor(message, stderr) {
|
|
2306
|
+
super(message);
|
|
2307
|
+
this.stderr = stderr;
|
|
2308
|
+
this.name = "DuckdbError";
|
|
2309
|
+
}
|
|
2310
|
+
stderr;
|
|
2311
|
+
};
|
|
2312
|
+
async function duckdbVersion(binaryPath) {
|
|
2313
|
+
const { stdout } = await runDuckdbProcess(binaryPath, ["--version"], "");
|
|
2314
|
+
const match = /v?(\d+\.\d+\.\d+)/.exec(stdout);
|
|
2315
|
+
if (!match?.[1])
|
|
2316
|
+
throw new DuckdbError(`could not read the DuckDB version from: ${stdout.trim()}`, "");
|
|
2317
|
+
return match[1];
|
|
2318
|
+
}
|
|
2319
|
+
async function assertDuckdbCli(cli) {
|
|
2320
|
+
await access(cli.binaryPath).catch(() => {
|
|
2321
|
+
throw new DuckdbError(`DuckDB CLI not found at ${cli.binaryPath}`, "");
|
|
2322
|
+
});
|
|
2323
|
+
const version = await duckdbVersion(cli.binaryPath);
|
|
2324
|
+
if (cli.expectedVersion && version !== cli.expectedVersion) {
|
|
2325
|
+
throw new DuckdbError(
|
|
2326
|
+
`DuckDB ${cli.expectedVersion} expected, but ${cli.binaryPath} is ${version}; Parquet output is only reproducible against the pinned release`,
|
|
2327
|
+
""
|
|
2328
|
+
);
|
|
2329
|
+
}
|
|
2330
|
+
return version;
|
|
2331
|
+
}
|
|
2332
|
+
async function runDuckdbScript(cli, sql) {
|
|
2333
|
+
const { stdout } = await runDuckdbProcess(cli.binaryPath, ["-batch", ":memory:"], sql);
|
|
2334
|
+
return stdout;
|
|
2335
|
+
}
|
|
2336
|
+
function runDuckdbProcess(binaryPath, args, stdin) {
|
|
2337
|
+
return new Promise((resolve5, reject) => {
|
|
2338
|
+
const child = spawn(binaryPath, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
2339
|
+
let stdout = "";
|
|
2340
|
+
let stderr = "";
|
|
2341
|
+
child.stdout.setEncoding("utf8");
|
|
2342
|
+
child.stderr.setEncoding("utf8");
|
|
2343
|
+
child.stdout.on("data", (chunk) => {
|
|
2344
|
+
stdout += chunk;
|
|
2345
|
+
});
|
|
2346
|
+
child.stderr.on("data", (chunk) => {
|
|
2347
|
+
stderr += chunk;
|
|
2348
|
+
});
|
|
2349
|
+
child.on(
|
|
2350
|
+
"error",
|
|
2351
|
+
(err) => reject(new DuckdbError(`could not start DuckDB: ${err.message}`, stderr))
|
|
2352
|
+
);
|
|
2353
|
+
child.on("close", (code) => {
|
|
2354
|
+
if (code === 0) resolve5({ stdout });
|
|
2355
|
+
else
|
|
2356
|
+
reject(
|
|
2357
|
+
new DuckdbError(
|
|
2358
|
+
`DuckDB exited with code ${code}: ${stderr.trim() || stdout.trim()}`,
|
|
2359
|
+
stderr
|
|
2360
|
+
)
|
|
2361
|
+
);
|
|
2362
|
+
});
|
|
2363
|
+
child.stdin.on("error", () => {
|
|
2364
|
+
});
|
|
2365
|
+
child.stdin.end(stdin);
|
|
2366
|
+
});
|
|
2367
|
+
}
|
|
2368
|
+
|
|
2369
|
+
// src/export/parquet.ts
|
|
2370
|
+
import { createHash as createHash3 } from "crypto";
|
|
2371
|
+
import { createWriteStream as createWriteStream3 } from "fs";
|
|
2372
|
+
import { mkdir as mkdir3, readFile as readFile2, rm, stat as stat2 } from "fs/promises";
|
|
2373
|
+
import { join as join4, resolve as resolve3 } from "path";
|
|
2374
|
+
import { brotliDecompressSync as brotliDecompressSync2 } from "zlib";
|
|
2375
|
+
import { KnowledgeCatalogManifestSchema as KnowledgeCatalogManifestSchema4 } from "@bendyline/gezk";
|
|
2376
|
+
var PARQUET_EXPORT_VERSION = 2;
|
|
2377
|
+
var DEFAULT_ROW_GROUP_SIZE = 32768;
|
|
2378
|
+
var NDJSON_MAX_OBJECT_BYTES = Math.max(64 * 1024 * 1024, MAX_KNOWLEDGE_DOCUMENT_BYTES * 2);
|
|
2379
|
+
var PARQUET_REPORT_PATH = "parquet-manifest.json";
|
|
2380
|
+
async function exportCatalogParquet(opts) {
|
|
2381
|
+
const duckdbVersion2 = await assertDuckdbCli(opts.duckdb);
|
|
2382
|
+
const outDir = resolve3(opts.outDir);
|
|
2383
|
+
await mkdir3(outDir, { recursive: true });
|
|
2384
|
+
const work = join4(outDir, `.export-${process.pid}-${Date.now().toString(36)}`);
|
|
2385
|
+
await mkdir3(work, { recursive: true });
|
|
2386
|
+
try {
|
|
2387
|
+
let rootDir;
|
|
2388
|
+
if ("archivePath" in opts.source) {
|
|
2389
|
+
opts.onProgress?.({ phase: "extract" });
|
|
2390
|
+
rootDir = join4(work, "catalog");
|
|
2391
|
+
await extractGezkVerified(opts.source.archivePath, rootDir);
|
|
2392
|
+
} else {
|
|
2393
|
+
rootDir = resolve3(opts.source.rootDir);
|
|
2394
|
+
}
|
|
2395
|
+
const manifest = KnowledgeCatalogManifestSchema4.parse(
|
|
2396
|
+
JSON.parse(await readFile2(join4(rootDir, MANIFEST_PATH), "utf8"))
|
|
2397
|
+
);
|
|
2398
|
+
const staging = join4(work, "ndjson");
|
|
2399
|
+
await mkdir3(staging, { recursive: true });
|
|
2400
|
+
const files = [];
|
|
2401
|
+
const metadata = kvMetadata(manifest);
|
|
2402
|
+
const rowGroupSize = opts.rowGroupSize ?? DEFAULT_ROW_GROUP_SIZE;
|
|
2403
|
+
const router = openCatalogDatabase(join4(rootDir, ROUTER_DB_PATH));
|
|
2404
|
+
try {
|
|
2405
|
+
const topicPaths = topicPathsOf(router);
|
|
2406
|
+
const topicRows = await stageTopics(router, join4(staging, "topics.ndjson"));
|
|
2407
|
+
opts.onProgress?.({ phase: "stage", table: "topics", rows: topicRows });
|
|
2408
|
+
files.push(
|
|
2409
|
+
await writeParquet(opts.duckdb, {
|
|
2410
|
+
outDir,
|
|
2411
|
+
file: "topics.parquet",
|
|
2412
|
+
table: "topics",
|
|
2413
|
+
rows: topicRows,
|
|
2414
|
+
sql: topicsCopy(
|
|
2415
|
+
join4(staging, "topics.ndjson"),
|
|
2416
|
+
join4(outDir, "topics.parquet"),
|
|
2417
|
+
rowGroupSize,
|
|
2418
|
+
metadata
|
|
2419
|
+
),
|
|
2420
|
+
onProgress: opts.onProgress
|
|
2421
|
+
})
|
|
2422
|
+
);
|
|
2423
|
+
for (const shard of manifest.router.shards) {
|
|
2424
|
+
const tag = String(shard.id).padStart(3, "0");
|
|
2425
|
+
const documentsNdjson = join4(staging, `documents-${tag}.ndjson`);
|
|
2426
|
+
const documentRows = await stageDocuments(router, shard.id, topicPaths, documentsNdjson);
|
|
2427
|
+
opts.onProgress?.({
|
|
2428
|
+
phase: "stage",
|
|
2429
|
+
table: "documents",
|
|
2430
|
+
shardId: shard.id,
|
|
2431
|
+
rows: documentRows
|
|
2432
|
+
});
|
|
2433
|
+
files.push(
|
|
2434
|
+
await writeParquet(opts.duckdb, {
|
|
2435
|
+
outDir,
|
|
2436
|
+
file: `documents-${tag}.parquet`,
|
|
2437
|
+
table: "documents",
|
|
2438
|
+
shardId: shard.id,
|
|
2439
|
+
rows: documentRows,
|
|
2440
|
+
sql: documentsCopy(
|
|
2441
|
+
documentsNdjson,
|
|
2442
|
+
join4(outDir, `documents-${tag}.parquet`),
|
|
2443
|
+
rowGroupSize,
|
|
2444
|
+
metadata
|
|
2445
|
+
),
|
|
2446
|
+
onProgress: opts.onProgress
|
|
2447
|
+
})
|
|
2448
|
+
);
|
|
2449
|
+
const shardDb = openCatalogDatabase(join4(rootDir, shard.path));
|
|
2450
|
+
const chunksNdjson = join4(staging, `chunks-${tag}.ndjson`);
|
|
2451
|
+
let chunkRows;
|
|
2452
|
+
try {
|
|
2453
|
+
chunkRows = await stageChunks(
|
|
2454
|
+
shardDb,
|
|
2455
|
+
shard.id,
|
|
2456
|
+
manifest.embedding.dimensions,
|
|
2457
|
+
chunksNdjson
|
|
2458
|
+
);
|
|
2459
|
+
} finally {
|
|
2460
|
+
shardDb.close();
|
|
2461
|
+
}
|
|
2462
|
+
opts.onProgress?.({ phase: "stage", table: "chunks", shardId: shard.id, rows: chunkRows });
|
|
2463
|
+
files.push(
|
|
2464
|
+
await writeParquet(opts.duckdb, {
|
|
2465
|
+
outDir,
|
|
2466
|
+
file: `chunks-${tag}.parquet`,
|
|
2467
|
+
table: "chunks",
|
|
2468
|
+
shardId: shard.id,
|
|
2469
|
+
rows: chunkRows,
|
|
2470
|
+
sql: chunksCopy(
|
|
2471
|
+
chunksNdjson,
|
|
2472
|
+
join4(outDir, `chunks-${tag}.parquet`),
|
|
2473
|
+
manifest.embedding.dimensions,
|
|
2474
|
+
rowGroupSize,
|
|
2475
|
+
metadata
|
|
2476
|
+
),
|
|
2477
|
+
onProgress: opts.onProgress
|
|
2478
|
+
})
|
|
2479
|
+
);
|
|
2480
|
+
}
|
|
2481
|
+
} finally {
|
|
2482
|
+
router.close();
|
|
2483
|
+
}
|
|
2484
|
+
const report = {
|
|
2485
|
+
exportVersion: PARQUET_EXPORT_VERSION,
|
|
2486
|
+
catalogId: manifest.id,
|
|
2487
|
+
version: manifest.version,
|
|
2488
|
+
publisherId: manifest.publisher.id,
|
|
2489
|
+
formatVersion: manifest.formatVersion,
|
|
2490
|
+
embeddingProfileId: manifest.embedding.id,
|
|
2491
|
+
dimensions: manifest.embedding.dimensions,
|
|
2492
|
+
files,
|
|
2493
|
+
duckdbVersion: duckdbVersion2,
|
|
2494
|
+
exporter: { name: KNOWLEDGE_TOOLCHAIN.name, version: KNOWLEDGE_TOOLCHAIN.version }
|
|
2495
|
+
};
|
|
2496
|
+
const { writeFile } = await import("fs/promises");
|
|
2497
|
+
await writeFile(
|
|
2498
|
+
join4(outDir, PARQUET_REPORT_PATH),
|
|
2499
|
+
`${JSON.stringify(report, null, 2)}
|
|
2500
|
+
`,
|
|
2501
|
+
"utf8"
|
|
2502
|
+
);
|
|
2503
|
+
return report;
|
|
2504
|
+
} finally {
|
|
2505
|
+
await rm(work, { recursive: true, force: true }).catch(() => {
|
|
2506
|
+
});
|
|
2507
|
+
}
|
|
2508
|
+
}
|
|
2509
|
+
function ndjsonWriter(path) {
|
|
2510
|
+
const stream = createWriteStream3(path, { encoding: "utf8" });
|
|
2511
|
+
return {
|
|
2512
|
+
write(row) {
|
|
2513
|
+
const line = `${JSON.stringify(row)}
|
|
2514
|
+
`;
|
|
2515
|
+
if (stream.write(line)) return Promise.resolve();
|
|
2516
|
+
return new Promise((resolveDrain) => stream.once("drain", () => resolveDrain()));
|
|
2517
|
+
},
|
|
2518
|
+
close() {
|
|
2519
|
+
return new Promise((resolveClose, reject) => {
|
|
2520
|
+
stream.on("error", reject);
|
|
2521
|
+
stream.end(() => resolveClose());
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
};
|
|
2525
|
+
}
|
|
2526
|
+
function topicPathsOf(router) {
|
|
2527
|
+
const parents = /* @__PURE__ */ new Map();
|
|
2528
|
+
for (const row of router.db.prepare("SELECT id, parent_id FROM topics").all()) {
|
|
2529
|
+
parents.set(row.id, row.parent_id);
|
|
2530
|
+
}
|
|
2531
|
+
const paths = /* @__PURE__ */ new Map();
|
|
2532
|
+
for (const id of parents.keys()) {
|
|
2533
|
+
const path = [];
|
|
2534
|
+
let cursor = id;
|
|
2535
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2536
|
+
while (cursor && !seen.has(cursor)) {
|
|
2537
|
+
seen.add(cursor);
|
|
2538
|
+
path.unshift(cursor);
|
|
2539
|
+
cursor = parents.get(cursor) ?? null;
|
|
2540
|
+
}
|
|
2541
|
+
paths.set(id, path);
|
|
2542
|
+
}
|
|
2543
|
+
return paths;
|
|
2544
|
+
}
|
|
2545
|
+
async function stageTopics(router, path) {
|
|
2546
|
+
const out = ndjsonWriter(path);
|
|
2547
|
+
let rows = 0;
|
|
2548
|
+
const statement = router.db.prepare(
|
|
2549
|
+
"SELECT id, parent_id, name, description, sort_key, document_count FROM topics ORDER BY sort_key, id"
|
|
2550
|
+
);
|
|
2551
|
+
for (const row of statement.iterate()) {
|
|
2552
|
+
await out.write({
|
|
2553
|
+
id: row.id,
|
|
2554
|
+
parent_id: row.parent_id ?? null,
|
|
2555
|
+
name: row.name,
|
|
2556
|
+
description: row.description ?? null,
|
|
2557
|
+
sort_key: row.sort_key,
|
|
2558
|
+
document_count: Number(row.document_count)
|
|
2559
|
+
});
|
|
2560
|
+
rows++;
|
|
2561
|
+
}
|
|
2562
|
+
await out.close();
|
|
2563
|
+
return rows;
|
|
2564
|
+
}
|
|
2565
|
+
async function stageDocuments(router, shardId, topicPaths, path) {
|
|
2566
|
+
const out = ndjsonWriter(path);
|
|
2567
|
+
let rows = 0;
|
|
2568
|
+
const aliasesOf = router.db.prepare(
|
|
2569
|
+
"SELECT alias FROM aliases WHERE document_id = ? ORDER BY alias"
|
|
2570
|
+
);
|
|
2571
|
+
const extra = router.schemaVersion >= 3 ? ", ordinal, meta_json" : ", NULL AS ordinal, NULL AS meta_json";
|
|
2572
|
+
const statement = router.db.prepare(
|
|
2573
|
+
`SELECT id, title, slug, summary, language, topic_id, shard_id, chunk_count,
|
|
2574
|
+
source_url, source_revision, source_updated_at, attribution_json, body_codec, body_blob${extra}
|
|
2575
|
+
FROM documents WHERE shard_id = ? ORDER BY id`
|
|
2576
|
+
);
|
|
2577
|
+
for (const row of statement.iterate(shardId)) {
|
|
2578
|
+
const aliases = aliasesOf.all(row.id).map(
|
|
2579
|
+
(a) => a.alias
|
|
2580
|
+
);
|
|
2581
|
+
await out.write({
|
|
2582
|
+
id: row.id,
|
|
2583
|
+
title: row.title,
|
|
2584
|
+
slug: row.slug,
|
|
2585
|
+
summary: row.summary ?? null,
|
|
2586
|
+
language: row.language,
|
|
2587
|
+
topic_id: row.topic_id,
|
|
2588
|
+
topic_path: topicPaths.get(row.topic_id) ?? [row.topic_id],
|
|
2589
|
+
shard_id: Number(row.shard_id),
|
|
2590
|
+
chunk_count: Number(row.chunk_count),
|
|
2591
|
+
markdown: decodeBody(row.body_codec, row.body_blob),
|
|
2592
|
+
source_url: row.source_url ?? null,
|
|
2593
|
+
source_revision: row.source_revision ?? null,
|
|
2594
|
+
source_updated_at: row.source_updated_at ?? null,
|
|
2595
|
+
attribution: row.attribution_json ?? null,
|
|
2596
|
+
ordinal: row.ordinal == null ? null : Number(row.ordinal),
|
|
2597
|
+
meta: row.meta_json ?? null,
|
|
2598
|
+
aliases
|
|
2599
|
+
});
|
|
2600
|
+
rows++;
|
|
2601
|
+
}
|
|
2602
|
+
await out.close();
|
|
2603
|
+
return rows;
|
|
2604
|
+
}
|
|
2605
|
+
function decodeBody(codec, blob) {
|
|
2606
|
+
const bytes = Buffer.from(blob);
|
|
2607
|
+
if (codec === "br") {
|
|
2608
|
+
return brotliDecompressSync2(bytes, { maxOutputLength: MAX_KNOWLEDGE_DOCUMENT_BYTES }).toString(
|
|
2609
|
+
"utf8"
|
|
2610
|
+
);
|
|
2611
|
+
}
|
|
2612
|
+
if (codec === "none") return bytes.toString("utf8");
|
|
2613
|
+
throw new Error(`unknown body codec ${codec}`);
|
|
2614
|
+
}
|
|
2615
|
+
async function stageChunks(shard, shardId, dimensions, path) {
|
|
2616
|
+
const out = ndjsonWriter(path);
|
|
2617
|
+
const bitBytes = Math.ceil(dimensions / 8);
|
|
2618
|
+
let rows = 0;
|
|
2619
|
+
const statement = shard.db.prepare(
|
|
2620
|
+
`SELECT c.id, c.chunk_uid, c.document_id, c.ordinal, c.title, c.heading_path, c.heading_text,
|
|
2621
|
+
c.text, c.token_count, c.content_hash, c.line_start, c.line_end,
|
|
2622
|
+
i.v AS int8, b.v AS bits
|
|
2623
|
+
FROM chunks c
|
|
2624
|
+
JOIN chunk_vectors_int8 i ON i.chunk_id = c.id
|
|
2625
|
+
JOIN chunk_vectors_bit b ON b.chunk_id = c.id
|
|
2626
|
+
ORDER BY c.id`
|
|
2627
|
+
);
|
|
2628
|
+
for (const row of statement.iterate()) {
|
|
2629
|
+
const int8 = Buffer.from(row.int8);
|
|
2630
|
+
const bits = Buffer.from(row.bits);
|
|
2631
|
+
if (int8.length !== dimensions || bits.length !== bitBytes) {
|
|
2632
|
+
throw new Error(
|
|
2633
|
+
`chunk ${row.chunk_uid}: vector widths ${int8.length}/${bits.length} do not match the ${dimensions}-dimension profile`
|
|
2634
|
+
);
|
|
2635
|
+
}
|
|
2636
|
+
await out.write({
|
|
2637
|
+
id: Number(row.id),
|
|
2638
|
+
chunk_uid: row.chunk_uid,
|
|
2639
|
+
document_id: row.document_id,
|
|
2640
|
+
ordinal: Number(row.ordinal),
|
|
2641
|
+
title: row.title,
|
|
2642
|
+
heading_path: parseJsonStrings(row.heading_path),
|
|
2643
|
+
heading_text: row.heading_text,
|
|
2644
|
+
text: row.text,
|
|
2645
|
+
token_count: Number(row.token_count),
|
|
2646
|
+
content_hash: row.content_hash,
|
|
2647
|
+
line_start: Number(row.line_start),
|
|
2648
|
+
line_end: Number(row.line_end),
|
|
2649
|
+
shard_id: shardId,
|
|
2650
|
+
embedding: Array.from(new Int8Array(int8.buffer, int8.byteOffset, int8.length)),
|
|
2651
|
+
embedding_bit: bits.toString("base64")
|
|
2652
|
+
});
|
|
2653
|
+
rows++;
|
|
2654
|
+
}
|
|
2655
|
+
await out.close();
|
|
2656
|
+
return rows;
|
|
2657
|
+
}
|
|
2658
|
+
function parseJsonStrings(raw) {
|
|
2659
|
+
try {
|
|
2660
|
+
const parsed = JSON.parse(raw);
|
|
2661
|
+
return Array.isArray(parsed) ? parsed.filter((p) => typeof p === "string") : [];
|
|
2662
|
+
} catch {
|
|
2663
|
+
return [];
|
|
2664
|
+
}
|
|
2665
|
+
}
|
|
2666
|
+
function sqlString(value) {
|
|
2667
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
2668
|
+
}
|
|
2669
|
+
function kvMetadata(manifest) {
|
|
2670
|
+
return {
|
|
2671
|
+
gezk_catalog_id: manifest.id,
|
|
2672
|
+
gezk_catalog_version: manifest.version,
|
|
2673
|
+
gezk_publisher_id: manifest.publisher.id,
|
|
2674
|
+
gezk_format_version: manifest.formatVersion,
|
|
2675
|
+
gezk_embedding_profile: manifest.embedding.id,
|
|
2676
|
+
gezk_embedding_model: manifest.embedding.model.repo,
|
|
2677
|
+
gezk_embedding_dimensions: String(manifest.embedding.dimensions),
|
|
2678
|
+
gezk_embedding_int8: "symmetric-linear scale 127: x = q / 127",
|
|
2679
|
+
gezk_embedding_bit: "sign bits, LSB-first, ceil(dimensions/8) bytes: bit = x > 0",
|
|
2680
|
+
gezk_parquet_export_version: String(PARQUET_EXPORT_VERSION),
|
|
2681
|
+
gezk_license: manifest.license.name
|
|
2682
|
+
};
|
|
2683
|
+
}
|
|
2684
|
+
function copyOptions(rowGroupSize, metadata) {
|
|
2685
|
+
const kv = Object.entries(metadata).map(([key, value]) => `${key}: ${sqlString(value)}`).join(", ");
|
|
2686
|
+
return `(FORMAT parquet, COMPRESSION zstd, COMPRESSION_LEVEL 3, ROW_GROUP_SIZE ${rowGroupSize}, KV_METADATA {${kv}})`;
|
|
2687
|
+
}
|
|
2688
|
+
var PRELUDE = "SET threads = 1;\nSET preserve_insertion_order = true;\n";
|
|
2689
|
+
function readNdjson(path, columns) {
|
|
2690
|
+
const spec = Object.entries(columns).map(([name, type]) => `${name}: ${sqlString(type)}`).join(", ");
|
|
2691
|
+
return `read_ndjson(${sqlString(path)}, columns = {${spec}}, maximum_object_size = ${NDJSON_MAX_OBJECT_BYTES})`;
|
|
2692
|
+
}
|
|
2693
|
+
function topicsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2694
|
+
const source = readNdjson(ndjson, {
|
|
2695
|
+
id: "VARCHAR",
|
|
2696
|
+
parent_id: "VARCHAR",
|
|
2697
|
+
name: "VARCHAR",
|
|
2698
|
+
description: "VARCHAR",
|
|
2699
|
+
sort_key: "VARCHAR",
|
|
2700
|
+
document_count: "BIGINT"
|
|
2701
|
+
});
|
|
2702
|
+
return `${PRELUDE}COPY (SELECT id, parent_id, name, description, sort_key, document_count FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2703
|
+
`;
|
|
2704
|
+
}
|
|
2705
|
+
function documentsCopy(ndjson, out, rowGroupSize, metadata) {
|
|
2706
|
+
const source = readNdjson(ndjson, {
|
|
2707
|
+
id: "VARCHAR",
|
|
2708
|
+
title: "VARCHAR",
|
|
2709
|
+
slug: "VARCHAR",
|
|
2710
|
+
summary: "VARCHAR",
|
|
2711
|
+
language: "VARCHAR",
|
|
2712
|
+
topic_id: "VARCHAR",
|
|
2713
|
+
topic_path: "VARCHAR[]",
|
|
2714
|
+
shard_id: "INTEGER",
|
|
2715
|
+
chunk_count: "INTEGER",
|
|
2716
|
+
markdown: "VARCHAR",
|
|
2717
|
+
source_url: "VARCHAR",
|
|
2718
|
+
source_revision: "VARCHAR",
|
|
2719
|
+
source_updated_at: "VARCHAR",
|
|
2720
|
+
attribution: "VARCHAR",
|
|
2721
|
+
ordinal: "INTEGER",
|
|
2722
|
+
meta: "VARCHAR",
|
|
2723
|
+
aliases: "VARCHAR[]"
|
|
2724
|
+
});
|
|
2725
|
+
return `${PRELUDE}COPY (SELECT id, title, slug, summary, language, topic_id, topic_path, shard_id, chunk_count, markdown, source_url, source_revision, source_updated_at, attribution, ordinal, meta, aliases FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2726
|
+
`;
|
|
2727
|
+
}
|
|
2728
|
+
function chunksCopy(ndjson, out, dimensions, rowGroupSize, metadata) {
|
|
2729
|
+
const source = readNdjson(ndjson, {
|
|
2730
|
+
id: "BIGINT",
|
|
2731
|
+
chunk_uid: "VARCHAR",
|
|
2732
|
+
document_id: "VARCHAR",
|
|
2733
|
+
ordinal: "INTEGER",
|
|
2734
|
+
title: "VARCHAR",
|
|
2735
|
+
heading_path: "VARCHAR[]",
|
|
2736
|
+
heading_text: "VARCHAR",
|
|
2737
|
+
text: "VARCHAR",
|
|
2738
|
+
token_count: "INTEGER",
|
|
2739
|
+
content_hash: "VARCHAR",
|
|
2740
|
+
line_start: "INTEGER",
|
|
2741
|
+
line_end: "INTEGER",
|
|
2742
|
+
shard_id: "INTEGER",
|
|
2743
|
+
embedding: "TINYINT[]",
|
|
2744
|
+
embedding_bit: "VARCHAR"
|
|
2745
|
+
});
|
|
2746
|
+
return `${PRELUDE}COPY (SELECT id, chunk_uid, document_id, ordinal, title, heading_path, heading_text, text, token_count, content_hash, line_start, line_end, shard_id, embedding::TINYINT[${dimensions}] AS embedding, from_base64(embedding_bit) AS embedding_bit FROM ${source}) TO ${sqlString(out)} ${copyOptions(rowGroupSize, metadata)};
|
|
2747
|
+
`;
|
|
2748
|
+
}
|
|
2749
|
+
async function writeParquet(cli, spec) {
|
|
2750
|
+
spec.onProgress?.({ phase: "write", file: spec.file });
|
|
2751
|
+
const target = join4(spec.outDir, spec.file);
|
|
2752
|
+
await rm(target, { force: true });
|
|
2753
|
+
await runDuckdbScript(cli, spec.sql);
|
|
2754
|
+
const bytes = await readFile2(target);
|
|
2755
|
+
return {
|
|
2756
|
+
path: spec.file,
|
|
2757
|
+
table: spec.table,
|
|
2758
|
+
...spec.shardId !== void 0 ? { shardId: spec.shardId } : {},
|
|
2759
|
+
rows: spec.rows,
|
|
2760
|
+
sizeBytes: (await stat2(target)).size,
|
|
2761
|
+
sha256: createHash3("sha256").update(bytes).digest("hex")
|
|
2762
|
+
};
|
|
2763
|
+
}
|
|
2764
|
+
async function findDuckdbBinary(env = process.env) {
|
|
2765
|
+
const configured = env.GEZEL_DUCKDB_BIN?.trim();
|
|
2766
|
+
if (configured) return configured;
|
|
2767
|
+
const { readdir: readdir2 } = await import("fs/promises");
|
|
2768
|
+
const { homedir } = await import("os");
|
|
2769
|
+
const name = process.platform === "win32" ? "duckdb.exe" : "duckdb";
|
|
2770
|
+
const homes = [env.GEZEL_HOME, join4(homedir(), ".gezel-dev"), join4(homedir(), ".gezel")].filter(
|
|
2771
|
+
(h) => Boolean(h)
|
|
2772
|
+
);
|
|
2773
|
+
for (const home of homes) {
|
|
2774
|
+
const engines = join4(home, "engines", "duckdb");
|
|
2775
|
+
const versions = (await readdir2(engines).catch(() => [])).sort().reverse();
|
|
2776
|
+
for (const version of versions) {
|
|
2777
|
+
const candidate = join4(engines, version, name);
|
|
2778
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
const pathDirs = (env.PATH ?? "").split(process.platform === "win32" ? ";" : ":");
|
|
2782
|
+
for (const dir of pathDirs) {
|
|
2783
|
+
if (!dir) continue;
|
|
2784
|
+
const candidate = join4(dir, name);
|
|
2785
|
+
if (await stat2(candidate).catch(() => null)) return candidate;
|
|
2786
|
+
}
|
|
2787
|
+
return null;
|
|
2788
|
+
}
|
|
1770
2789
|
|
|
1771
2790
|
// src/profiles/registry.ts
|
|
1772
2791
|
var SHARED_ENCODING = {
|
|
@@ -1774,50 +2793,152 @@ var SHARED_ENCODING = {
|
|
|
1774
2793
|
normalized: true,
|
|
1775
2794
|
dimensions: 384,
|
|
1776
2795
|
maxTokens: 512,
|
|
1777
|
-
vectorEncoding: "
|
|
2796
|
+
vectorEncoding: "bit+int8",
|
|
1778
2797
|
distance: { stage1: "hamming", stage2: "cosine" },
|
|
1779
2798
|
quantization: {
|
|
1780
2799
|
int8: { method: "symmetric-linear", scale: 127 },
|
|
1781
2800
|
binary: { method: "sign", threshold: 0, packing: "lsb-first" }
|
|
1782
2801
|
}
|
|
1783
2802
|
};
|
|
1784
|
-
var
|
|
1785
|
-
id: "
|
|
2803
|
+
var MULTILINGUAL_E5_SMALL_1 = {
|
|
2804
|
+
id: "multilingual-e5-small@1",
|
|
1786
2805
|
model: {
|
|
1787
2806
|
repo: "Xenova/multilingual-e5-small",
|
|
1788
|
-
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78"
|
|
2807
|
+
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
|
|
2808
|
+
onnxFile: "onnx/model.onnx",
|
|
2809
|
+
onnxDigest: "sha256:4aa845c27760e06e9a686b9d8b5d440eae4b6612cd09e5b522b716d3941f77ff"
|
|
2810
|
+
},
|
|
2811
|
+
tokenizer: {
|
|
2812
|
+
kind: "sentencepiece-xlmr",
|
|
2813
|
+
file: "tokenizer.json",
|
|
2814
|
+
digest: "sha256:0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39"
|
|
1789
2815
|
},
|
|
1790
|
-
tokenizer: { kind: "sentencepiece-xlmr" },
|
|
1791
2816
|
queryInstruction: "query: ",
|
|
1792
2817
|
passageInstruction: "passage: ",
|
|
1793
2818
|
...SHARED_ENCODING
|
|
1794
2819
|
};
|
|
1795
|
-
var
|
|
1796
|
-
id: "
|
|
2820
|
+
var BGE_SMALL_EN_V15_1 = {
|
|
2821
|
+
id: "bge-small-en-v1.5@1",
|
|
1797
2822
|
model: {
|
|
1798
2823
|
repo: "Xenova/bge-small-en-v1.5",
|
|
1799
|
-
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3"
|
|
2824
|
+
revision: "ea104dacec62c0de699686887e3f920caeb4f3e3",
|
|
2825
|
+
onnxFile: "onnx/model.onnx",
|
|
2826
|
+
onnxDigest: "sha256:828e1496d7fabb79cfa4dcd84fa38625c0d3d21da474a00f08db0f559940cf35"
|
|
2827
|
+
},
|
|
2828
|
+
tokenizer: {
|
|
2829
|
+
kind: "wordpiece-bert",
|
|
2830
|
+
file: "tokenizer.json",
|
|
2831
|
+
digest: "sha256:d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66"
|
|
1800
2832
|
},
|
|
1801
|
-
tokenizer: { kind: "wordpiece-bert" },
|
|
1802
2833
|
queryInstruction: "Represent this sentence for searching relevant passages: ",
|
|
1803
2834
|
passageInstruction: "",
|
|
1804
2835
|
...SHARED_ENCODING
|
|
1805
2836
|
};
|
|
1806
2837
|
var KNOWLEDGE_EMBEDDING_PROFILES = [
|
|
1807
|
-
|
|
1808
|
-
|
|
2838
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
2839
|
+
BGE_SMALL_EN_V15_1
|
|
1809
2840
|
];
|
|
1810
2841
|
function knowledgeEmbeddingProfile(id) {
|
|
1811
2842
|
return KNOWLEDGE_EMBEDDING_PROFILES.find((p) => p.id === id) ?? null;
|
|
1812
2843
|
}
|
|
1813
|
-
var
|
|
1814
|
-
id: "
|
|
2844
|
+
var MARKDOWN_CHUNKS_2 = {
|
|
2845
|
+
id: "markdown-chunks@2",
|
|
1815
2846
|
unit: "tokens",
|
|
1816
2847
|
tokenizer: "profile",
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
contextHeader: {
|
|
2848
|
+
target: 420,
|
|
2849
|
+
overlap: 64,
|
|
2850
|
+
contextHeader: { max: 64 }
|
|
1820
2851
|
};
|
|
2852
|
+
var GEZEL_MULTILINGUAL_E5_SMALL_1 = MULTILINGUAL_E5_SMALL_1;
|
|
2853
|
+
var GEZEL_BGE_SMALL_EN_V15_1 = BGE_SMALL_EN_V15_1;
|
|
2854
|
+
var GEZEL_MARKDOWN_CHUNKS_2 = MARKDOWN_CHUNKS_2;
|
|
2855
|
+
|
|
2856
|
+
// src/embedding/profile-embedder.ts
|
|
2857
|
+
import { embeddingProfileArtifacts as embeddingProfileArtifacts2 } from "@bendyline/gezk";
|
|
2858
|
+
|
|
2859
|
+
// src/embedding/artifact-verify.ts
|
|
2860
|
+
import { createHash as createHash4 } from "crypto";
|
|
2861
|
+
import { createReadStream } from "fs";
|
|
2862
|
+
import { stat as stat3 } from "fs/promises";
|
|
2863
|
+
import { join as join5 } from "path";
|
|
2864
|
+
import { embeddingProfileArtifacts } from "@bendyline/gezk";
|
|
2865
|
+
var EmbedderArtifactError = class extends Error {
|
|
2866
|
+
constructor(reason, profileId, checks) {
|
|
2867
|
+
super(describeFailure(reason, profileId, checks));
|
|
2868
|
+
this.reason = reason;
|
|
2869
|
+
this.profileId = profileId;
|
|
2870
|
+
this.checks = checks;
|
|
2871
|
+
this.name = "EmbedderArtifactError";
|
|
2872
|
+
}
|
|
2873
|
+
reason;
|
|
2874
|
+
profileId;
|
|
2875
|
+
checks;
|
|
2876
|
+
isActionable = true;
|
|
2877
|
+
};
|
|
2878
|
+
function describeFailure(reason, profileId, checks) {
|
|
2879
|
+
const detail = checks.filter((c) => c.actual !== c.expected).map(
|
|
2880
|
+
(c) => c.actual === null ? `${c.file} not found at ${c.path}` : `${c.file} is ${c.actual}, profile pins ${c.expected}`
|
|
2881
|
+
).join("; ");
|
|
2882
|
+
return reason === "missing" ? `cannot verify the model files for profile ${profileId}: ${detail}` : `the model files loaded for profile ${profileId} differ from the ones it pins: ${detail}`;
|
|
2883
|
+
}
|
|
2884
|
+
function transformersCachePath(cacheDir, repo, revision, file) {
|
|
2885
|
+
return revision === "main" ? join5(cacheDir, repo, file) : join5(cacheDir, repo, revision, file);
|
|
2886
|
+
}
|
|
2887
|
+
var digestMemo = /* @__PURE__ */ new Map();
|
|
2888
|
+
var DIGEST_MEMO_MAX = 32;
|
|
2889
|
+
async function fileDigest(path) {
|
|
2890
|
+
let size;
|
|
2891
|
+
let mtimeMs;
|
|
2892
|
+
try {
|
|
2893
|
+
const info = await stat3(path);
|
|
2894
|
+
if (!info.isFile()) return null;
|
|
2895
|
+
size = info.size;
|
|
2896
|
+
mtimeMs = info.mtimeMs;
|
|
2897
|
+
} catch {
|
|
2898
|
+
return null;
|
|
2899
|
+
}
|
|
2900
|
+
const key = `${path} ${size} ${mtimeMs}`;
|
|
2901
|
+
const memo = digestMemo.get(key);
|
|
2902
|
+
if (memo) return memo;
|
|
2903
|
+
const digest = await new Promise((resolve5, reject) => {
|
|
2904
|
+
const hash = createHash4("sha256");
|
|
2905
|
+
createReadStream(path).on("error", reject).on("data", (chunk) => hash.update(chunk)).on("end", () => resolve5(`sha256:${hash.digest("hex")}`));
|
|
2906
|
+
});
|
|
2907
|
+
if (digestMemo.size >= DIGEST_MEMO_MAX) {
|
|
2908
|
+
const oldest = digestMemo.keys().next().value;
|
|
2909
|
+
if (oldest !== void 0) digestMemo.delete(oldest);
|
|
2910
|
+
}
|
|
2911
|
+
digestMemo.set(key, digest);
|
|
2912
|
+
return digest;
|
|
2913
|
+
}
|
|
2914
|
+
async function verifyProfileArtifacts(profile, opts) {
|
|
2915
|
+
const artifacts = embeddingProfileArtifacts(profile);
|
|
2916
|
+
const revision = opts.revision ?? profile.model.revision;
|
|
2917
|
+
const pinned = [];
|
|
2918
|
+
if (artifacts.onnxDigest) {
|
|
2919
|
+
pinned.push({ role: "onnx", file: artifacts.onnxFile, expected: artifacts.onnxDigest });
|
|
2920
|
+
}
|
|
2921
|
+
if (artifacts.tokenizerDigest) {
|
|
2922
|
+
pinned.push({
|
|
2923
|
+
role: "tokenizer",
|
|
2924
|
+
file: artifacts.tokenizerFile,
|
|
2925
|
+
expected: artifacts.tokenizerDigest
|
|
2926
|
+
});
|
|
2927
|
+
}
|
|
2928
|
+
if (pinned.length === 0) return { status: "unpinned", checks: [] };
|
|
2929
|
+
const checks = [];
|
|
2930
|
+
for (const { role, file, expected } of pinned) {
|
|
2931
|
+
const path = transformersCachePath(opts.cacheDir, profile.model.repo, revision, file);
|
|
2932
|
+
checks.push({ role, file, path, expected, actual: await fileDigest(path) });
|
|
2933
|
+
}
|
|
2934
|
+
if (checks.some((c) => c.actual === null)) {
|
|
2935
|
+
throw new EmbedderArtifactError("missing", profile.id, checks);
|
|
2936
|
+
}
|
|
2937
|
+
if (checks.some((c) => c.actual !== c.expected)) {
|
|
2938
|
+
throw new EmbedderArtifactError("mismatch", profile.id, checks);
|
|
2939
|
+
}
|
|
2940
|
+
return { status: "verified", checks };
|
|
2941
|
+
}
|
|
1821
2942
|
|
|
1822
2943
|
// src/embedding/profile-embedder.ts
|
|
1823
2944
|
var MAX_BATCH = 8;
|
|
@@ -1829,6 +2950,42 @@ var EmbedderUnavailableError = class extends Error {
|
|
|
1829
2950
|
this.name = "EmbedderUnavailableError";
|
|
1830
2951
|
}
|
|
1831
2952
|
};
|
|
2953
|
+
var ONNX_DTYPE_BY_SUFFIX = [
|
|
2954
|
+
["_q4f16", "q4f16"],
|
|
2955
|
+
["_quantized", "q8"],
|
|
2956
|
+
["_uint8", "uint8"],
|
|
2957
|
+
["_int8", "int8"],
|
|
2958
|
+
["_bnb4", "bnb4"],
|
|
2959
|
+
["_fp16", "fp16"],
|
|
2960
|
+
["_q4", "q4"]
|
|
2961
|
+
];
|
|
2962
|
+
var POOLING_BY_PROFILE = {
|
|
2963
|
+
mean: "mean",
|
|
2964
|
+
cls: "cls"
|
|
2965
|
+
};
|
|
2966
|
+
function resolveTransformersModelOptions(profile) {
|
|
2967
|
+
const { onnxFile, tokenizerFile } = embeddingProfileArtifacts2(profile);
|
|
2968
|
+
if (tokenizerFile !== "tokenizer.json") {
|
|
2969
|
+
throw new EmbedderUnavailableError(
|
|
2970
|
+
`profile ${profile.id} pins tokenizer file ${tokenizerFile}; this runtime loads tokenizer.json only`
|
|
2971
|
+
);
|
|
2972
|
+
}
|
|
2973
|
+
if (!onnxFile.endsWith(".onnx")) {
|
|
2974
|
+
throw new EmbedderUnavailableError(
|
|
2975
|
+
`profile ${profile.id} pins ${onnxFile}; this runtime loads ONNX graphs only`
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
const slash = onnxFile.lastIndexOf("/");
|
|
2979
|
+
const subfolder = slash === -1 ? "" : onnxFile.slice(0, slash);
|
|
2980
|
+
const base = onnxFile.slice(slash + 1, -".onnx".length);
|
|
2981
|
+
const variant = ONNX_DTYPE_BY_SUFFIX.find(([suffix2]) => base.endsWith(suffix2));
|
|
2982
|
+
const [suffix, dtype] = variant ?? ["", "fp32"];
|
|
2983
|
+
const modelFileName = base.slice(0, base.length - suffix.length);
|
|
2984
|
+
if (modelFileName === "") {
|
|
2985
|
+
throw new EmbedderUnavailableError(`profile ${profile.id} pins an unnamed graph ${onnxFile}`);
|
|
2986
|
+
}
|
|
2987
|
+
return { revision: profile.model.revision, dtype, subfolder, model_file_name: modelFileName };
|
|
2988
|
+
}
|
|
1832
2989
|
async function loadTransformers() {
|
|
1833
2990
|
try {
|
|
1834
2991
|
const specifier = "@huggingface/transformers";
|
|
@@ -1844,33 +3001,52 @@ async function loadTransformers() {
|
|
|
1844
3001
|
}
|
|
1845
3002
|
}
|
|
1846
3003
|
async function createProfileEmbedder(profile, opts = {}) {
|
|
1847
|
-
const
|
|
3004
|
+
const pooling = POOLING_BY_PROFILE[profile.pooling];
|
|
3005
|
+
if (!pooling) {
|
|
3006
|
+
throw new EmbedderUnavailableError(
|
|
3007
|
+
`profile ${profile.id} uses ${profile.pooling} pooling, which this runtime does not implement`
|
|
3008
|
+
);
|
|
3009
|
+
}
|
|
3010
|
+
const modelOptions = resolveTransformersModelOptions(profile);
|
|
3011
|
+
const transformers = opts.transformers ?? await loadTransformers();
|
|
1848
3012
|
const cacheDir = opts.cacheDir ?? process.env.GEZEL_HF_CACHE_DIR;
|
|
1849
3013
|
if (cacheDir) {
|
|
1850
3014
|
transformers.env.cacheDir = cacheDir;
|
|
1851
3015
|
transformers.env.useFSCache = true;
|
|
1852
3016
|
transformers.env.allowRemoteModels = true;
|
|
1853
3017
|
}
|
|
1854
|
-
const modelOpts = {
|
|
1855
|
-
revision: profile.model.revision,
|
|
1856
|
-
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
1857
|
-
};
|
|
1858
3018
|
const [pipe, tokenizer] = await Promise.all([
|
|
1859
|
-
transformers.pipeline("feature-extraction", profile.model.repo,
|
|
1860
|
-
|
|
3019
|
+
transformers.pipeline("feature-extraction", profile.model.repo, {
|
|
3020
|
+
...modelOptions,
|
|
3021
|
+
...opts.sessionOptions ? { session_options: opts.sessionOptions } : {}
|
|
3022
|
+
}),
|
|
3023
|
+
transformers.AutoTokenizer.from_pretrained(profile.model.repo, {
|
|
3024
|
+
revision: modelOptions.revision
|
|
3025
|
+
})
|
|
1861
3026
|
]);
|
|
3027
|
+
let verification;
|
|
3028
|
+
try {
|
|
3029
|
+
verification = await verifyProfileArtifacts(profile, {
|
|
3030
|
+
cacheDir: cacheDir ?? transformers.env.cacheDir ?? ""
|
|
3031
|
+
});
|
|
3032
|
+
} catch (err) {
|
|
3033
|
+
await pipe.dispose?.().catch(() => {
|
|
3034
|
+
});
|
|
3035
|
+
throw err;
|
|
3036
|
+
}
|
|
1862
3037
|
const cap = (text) => text.length <= MAX_CHARS ? text : text.slice(0, MAX_CHARS);
|
|
1863
3038
|
const embed = async (texts) => {
|
|
1864
3039
|
const out = [];
|
|
1865
3040
|
for (let i = 0; i < texts.length; i += MAX_BATCH) {
|
|
1866
3041
|
const slice = texts.slice(i, i + MAX_BATCH).map(cap);
|
|
1867
|
-
const result = await pipe(slice, { pooling
|
|
3042
|
+
const result = await pipe(slice, { pooling, normalize: true });
|
|
1868
3043
|
out.push(...result.tolist());
|
|
1869
3044
|
}
|
|
1870
3045
|
return out;
|
|
1871
3046
|
};
|
|
1872
3047
|
return {
|
|
1873
3048
|
profile,
|
|
3049
|
+
verification,
|
|
1874
3050
|
embed,
|
|
1875
3051
|
embedQuery: async (text) => {
|
|
1876
3052
|
const [vector] = await embed([`${profile.queryInstruction}${text}`]);
|
|
@@ -1884,29 +3060,427 @@ async function createProfileEmbedder(profile, opts = {}) {
|
|
|
1884
3060
|
}
|
|
1885
3061
|
|
|
1886
3062
|
// src/markdown-adapter/load.ts
|
|
1887
|
-
import {
|
|
1888
|
-
import {
|
|
1889
|
-
import {
|
|
3063
|
+
import { existsSync } from "fs";
|
|
3064
|
+
import { readFile as readFile3, readdir } from "fs/promises";
|
|
3065
|
+
import { basename, dirname as dirname3, isAbsolute as isAbsolute3, join as join6, posix as posix2, relative as relative3, resolve as resolve4, sep } from "path";
|
|
3066
|
+
import {
|
|
3067
|
+
CatalogDocumentSchema,
|
|
3068
|
+
KnowledgeDocumentIdSchema,
|
|
3069
|
+
KnowledgeOrdinalSchema,
|
|
3070
|
+
assetExtension as assetExtension3,
|
|
3071
|
+
formatKnowledgeUri,
|
|
3072
|
+
topicSortKeyForOrder
|
|
3073
|
+
} from "@bendyline/gezk";
|
|
3074
|
+
|
|
3075
|
+
// src/markdown-adapter/frontmatter.ts
|
|
3076
|
+
import { parseDocument } from "yaml";
|
|
3077
|
+
var FRONTMATTER_MAX_BYTES = 64 * 1024;
|
|
3078
|
+
var OUTLINE_MAX_BYTES = 1024 * 1024;
|
|
3079
|
+
function parseYaml(text, opts) {
|
|
3080
|
+
const maxBytes = opts.maxBytes ?? FRONTMATTER_MAX_BYTES;
|
|
3081
|
+
if (Buffer.byteLength(text, "utf8") > maxBytes) {
|
|
3082
|
+
throw new Error(`${opts.what} exceeds ${maxBytes} bytes`);
|
|
3083
|
+
}
|
|
3084
|
+
const doc = parseDocument(text, {
|
|
3085
|
+
version: "1.2",
|
|
3086
|
+
schema: "core",
|
|
3087
|
+
uniqueKeys: true,
|
|
3088
|
+
logLevel: "silent"
|
|
3089
|
+
});
|
|
3090
|
+
const problem = doc.errors[0] ?? (opts.tolerateUnknownTags ? void 0 : doc.warnings[0]);
|
|
3091
|
+
if (problem) throw new Error(`${opts.what}: ${problem.message.split("\n")[0]}`);
|
|
3092
|
+
return doc.toJS({ maxAliasCount: opts.maxAliasCount ?? 0 });
|
|
3093
|
+
}
|
|
3094
|
+
var OPEN = /^---[ \t]*\r?\n/;
|
|
3095
|
+
var CLOSE = /^(?:---|\.\.\.)[ \t]*$/;
|
|
3096
|
+
function parseMarkdownFrontMatter(source) {
|
|
3097
|
+
const open = OPEN.exec(source);
|
|
3098
|
+
if (!open) return { data: {}, body: source };
|
|
3099
|
+
const lines = source.slice(open[0].length).split(/\r?\n/);
|
|
3100
|
+
const closeAt = lines.findIndex((line) => CLOSE.test(line));
|
|
3101
|
+
if (closeAt === -1) throw new Error("front matter is not closed with ---");
|
|
3102
|
+
const yamlText = lines.slice(0, closeAt).join("\n");
|
|
3103
|
+
const body = lines.slice(closeAt + 1).join("\n");
|
|
3104
|
+
const parsed = parseYaml(yamlText, { what: "front matter" });
|
|
3105
|
+
if (parsed === null || parsed === void 0) return { data: {}, body };
|
|
3106
|
+
if (typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
3107
|
+
throw new Error("front matter must be a mapping");
|
|
3108
|
+
}
|
|
3109
|
+
return { data: parsed, body };
|
|
3110
|
+
}
|
|
3111
|
+
|
|
3112
|
+
// src/markdown-adapter/outline.ts
|
|
3113
|
+
import { posix } from "path";
|
|
3114
|
+
var HAS_SCHEME = /^[a-z][a-z0-9+.-]*:/i;
|
|
3115
|
+
var MARKDOWN_EXT = /\.(md|markdown)$/i;
|
|
3116
|
+
function newOutlineTopic(name) {
|
|
3117
|
+
return { name, children: [], entries: [] };
|
|
3118
|
+
}
|
|
3119
|
+
function isRecord(value) {
|
|
3120
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3121
|
+
}
|
|
3122
|
+
function isExternal(target) {
|
|
3123
|
+
return HAS_SCHEME.test(target) || target.startsWith("//");
|
|
3124
|
+
}
|
|
3125
|
+
function safeDecode(value) {
|
|
3126
|
+
try {
|
|
3127
|
+
return decodeURIComponent(value);
|
|
3128
|
+
} catch {
|
|
3129
|
+
return value;
|
|
3130
|
+
}
|
|
3131
|
+
}
|
|
3132
|
+
function stem(file) {
|
|
3133
|
+
return posix.basename(file).replace(MARKDOWN_EXT, "");
|
|
3134
|
+
}
|
|
3135
|
+
function resolveOutlinePath(baseDir, target) {
|
|
3136
|
+
const cleaned = safeDecode(
|
|
3137
|
+
target.replace(/\\/g, "/").replace(/[#?].*$/, "").trim()
|
|
3138
|
+
);
|
|
3139
|
+
if (cleaned === "" || posix.isAbsolute(cleaned)) return null;
|
|
3140
|
+
const joined = posix.normalize(posix.join(baseDir, cleaned));
|
|
3141
|
+
if (joined === ".." || joined.startsWith("../")) return null;
|
|
3142
|
+
return joined === "." ? "" : joined;
|
|
3143
|
+
}
|
|
3144
|
+
function assignOutlineOrders(topic) {
|
|
3145
|
+
topic.entries.forEach((entry, i) => {
|
|
3146
|
+
if (entry.order === void 0) entry.order = i + 1;
|
|
3147
|
+
});
|
|
3148
|
+
topic.children.forEach((child, i) => {
|
|
3149
|
+
if (child.order === void 0) child.order = i + 1;
|
|
3150
|
+
assignOutlineOrders(child);
|
|
3151
|
+
});
|
|
3152
|
+
}
|
|
3153
|
+
function titleFromFolderName(name) {
|
|
3154
|
+
const words = name.replace(/[_-]+/g, " ").trim().split(/\s+/).filter(Boolean);
|
|
3155
|
+
if (words.length === 0) return name;
|
|
3156
|
+
return words.map((word) => /[A-Z]/.test(word) ? word : word.charAt(0).toUpperCase() + word.slice(1)).join(" ");
|
|
3157
|
+
}
|
|
3158
|
+
var HEADING2 = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
|
|
3159
|
+
var LIST_ITEM = /^([ \t]*)(?:[-*+]|\d+[.)])\s+(.+?)\s*$/;
|
|
3160
|
+
var ITEM_LINK = /^\[([^\]]*)\]\(\s*<?([^)>\s]*)>?(?:\s+"[^"]*")?\s*\)$/;
|
|
3161
|
+
function materializeSummaryTopic(slot) {
|
|
3162
|
+
if (slot.topic) return slot.topic;
|
|
3163
|
+
const topic = newOutlineTopic(slot.title || "Untitled");
|
|
3164
|
+
if (slot.entry) {
|
|
3165
|
+
const at = slot.container.entries.indexOf(slot.entry);
|
|
3166
|
+
if (at >= 0) slot.container.entries.splice(at, 1);
|
|
3167
|
+
topic.entries.push(slot.entry);
|
|
3168
|
+
}
|
|
3169
|
+
slot.container.children.push(topic);
|
|
3170
|
+
slot.topic = topic;
|
|
3171
|
+
return topic;
|
|
3172
|
+
}
|
|
3173
|
+
function parseGitbookSummary(text, summaryFile) {
|
|
3174
|
+
const dir = posix.dirname(summaryFile);
|
|
3175
|
+
const baseDir = dir === "." ? "" : dir;
|
|
3176
|
+
const outline = {
|
|
3177
|
+
format: "gitbook",
|
|
3178
|
+
root: newOutlineTopic(""),
|
|
3179
|
+
consumed: [summaryFile],
|
|
3180
|
+
warnings: []
|
|
3181
|
+
};
|
|
3182
|
+
let container = outline.root;
|
|
3183
|
+
const stack = [];
|
|
3184
|
+
let fence = null;
|
|
3185
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
3186
|
+
const line = rawLine.replace(/\t/g, " ");
|
|
3187
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
3188
|
+
if (fenceMatch) {
|
|
3189
|
+
const marker = fenceMatch[1][0];
|
|
3190
|
+
if (fence === null) fence = marker;
|
|
3191
|
+
else if (marker === fence) fence = null;
|
|
3192
|
+
continue;
|
|
3193
|
+
}
|
|
3194
|
+
if (fence !== null) continue;
|
|
3195
|
+
const heading = HEADING2.exec(line);
|
|
3196
|
+
if (heading) {
|
|
3197
|
+
if (heading[1].length === 1) continue;
|
|
3198
|
+
const part = newOutlineTopic(heading[2].replace(/[*_`]/g, "").trim() || "Part");
|
|
3199
|
+
outline.root.children.push(part);
|
|
3200
|
+
container = part;
|
|
3201
|
+
stack.length = 0;
|
|
3202
|
+
continue;
|
|
3203
|
+
}
|
|
3204
|
+
const item = LIST_ITEM.exec(line);
|
|
3205
|
+
if (!item) continue;
|
|
3206
|
+
const indent = item[1].length;
|
|
3207
|
+
while (stack.length > 0 && stack[stack.length - 1].indent >= indent) {
|
|
3208
|
+
stack.pop();
|
|
3209
|
+
}
|
|
3210
|
+
const parent = stack[stack.length - 1];
|
|
3211
|
+
const parentTopic = parent ? materializeSummaryTopic(parent) : container;
|
|
3212
|
+
const content = item[2].trim();
|
|
3213
|
+
const link = ITEM_LINK.exec(content);
|
|
3214
|
+
const title = (link ? link[1] : content).replace(/[*_`]/g, "").trim();
|
|
3215
|
+
const slot = { indent, title, container: parentTopic };
|
|
3216
|
+
const target = link?.[2] ?? "";
|
|
3217
|
+
if (link && target !== "" && !isExternal(target)) {
|
|
3218
|
+
const file = resolveOutlinePath(baseDir, target);
|
|
3219
|
+
if (file === null) {
|
|
3220
|
+
outline.warnings.push(`${summaryFile}: '${target}' leaves the catalog tree; skipped`);
|
|
3221
|
+
} else if (!MARKDOWN_EXT.test(file)) {
|
|
3222
|
+
outline.warnings.push(`${summaryFile}: '${target}' is not a Markdown page; skipped`);
|
|
3223
|
+
} else {
|
|
3224
|
+
slot.entry = { file, ...title ? { title } : {} };
|
|
3225
|
+
parentTopic.entries.push(slot.entry);
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
stack.push(slot);
|
|
3229
|
+
}
|
|
3230
|
+
assignOutlineOrders(outline.root);
|
|
3231
|
+
return outline;
|
|
3232
|
+
}
|
|
3233
|
+
function parseMkdocsNav(config, docsDir, configFile = "mkdocs.yml") {
|
|
3234
|
+
if (!isRecord(config)) throw new Error(`${configFile}: not a mapping`);
|
|
3235
|
+
const nav = config.nav;
|
|
3236
|
+
if (nav === void 0 || nav === null) return null;
|
|
3237
|
+
if (!Array.isArray(nav)) throw new Error(`${configFile}: 'nav' must be a list`);
|
|
3238
|
+
const outline = {
|
|
3239
|
+
format: "mkdocs",
|
|
3240
|
+
root: newOutlineTopic(""),
|
|
3241
|
+
consumed: [],
|
|
3242
|
+
warnings: []
|
|
3243
|
+
};
|
|
3244
|
+
const add = (container, target, title) => {
|
|
3245
|
+
if (isExternal(target)) return;
|
|
3246
|
+
const file = resolveOutlinePath(docsDir, target);
|
|
3247
|
+
if (file === null) {
|
|
3248
|
+
outline.warnings.push(
|
|
3249
|
+
`${configFile}: nav entry '${target}' leaves the docs directory; skipped`
|
|
3250
|
+
);
|
|
3251
|
+
return;
|
|
3252
|
+
}
|
|
3253
|
+
if (!MARKDOWN_EXT.test(file)) {
|
|
3254
|
+
outline.warnings.push(`${configFile}: nav entry '${target}' is not a Markdown page; skipped`);
|
|
3255
|
+
return;
|
|
3256
|
+
}
|
|
3257
|
+
container.entries.push({ file, ...title ? { title } : {} });
|
|
3258
|
+
};
|
|
3259
|
+
const walk = (items, container) => {
|
|
3260
|
+
for (const item of items) {
|
|
3261
|
+
if (typeof item === "string") {
|
|
3262
|
+
add(container, item);
|
|
3263
|
+
continue;
|
|
3264
|
+
}
|
|
3265
|
+
if (!isRecord(item)) {
|
|
3266
|
+
outline.warnings.push(`${configFile}: unrecognized nav entry ${JSON.stringify(item)}`);
|
|
3267
|
+
continue;
|
|
3268
|
+
}
|
|
3269
|
+
for (const [title, value] of Object.entries(item)) {
|
|
3270
|
+
if (typeof value === "string") {
|
|
3271
|
+
add(container, value, title.trim());
|
|
3272
|
+
} else if (Array.isArray(value)) {
|
|
3273
|
+
const section = newOutlineTopic(title.trim() || "Section");
|
|
3274
|
+
walk(value, section);
|
|
3275
|
+
container.children.push(section);
|
|
3276
|
+
} else {
|
|
3277
|
+
outline.warnings.push(
|
|
3278
|
+
`${configFile}: nav entry '${title}' is neither a page nor a section`
|
|
3279
|
+
);
|
|
3280
|
+
}
|
|
3281
|
+
}
|
|
3282
|
+
}
|
|
3283
|
+
};
|
|
3284
|
+
walk(nav, outline.root);
|
|
3285
|
+
assignOutlineOrders(outline.root);
|
|
3286
|
+
return outline;
|
|
3287
|
+
}
|
|
3288
|
+
function globToRegExp(pattern) {
|
|
3289
|
+
let source = "";
|
|
3290
|
+
for (let i = 0; i < pattern.length; i++) {
|
|
3291
|
+
const ch = pattern[i];
|
|
3292
|
+
if (ch === "*") {
|
|
3293
|
+
if (pattern[i + 1] === "*") {
|
|
3294
|
+
source += ".*";
|
|
3295
|
+
i++;
|
|
3296
|
+
} else source += "[^/]*";
|
|
3297
|
+
} else if (ch === "?") source += "[^/]";
|
|
3298
|
+
else source += ch.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
3299
|
+
}
|
|
3300
|
+
return new RegExp(`^${source}$`);
|
|
3301
|
+
}
|
|
3302
|
+
function parseJupyterBookToc(doc, tocDir, files, tocFile = "_toc.yml") {
|
|
3303
|
+
const outline = {
|
|
3304
|
+
format: "jupyter-book",
|
|
3305
|
+
root: newOutlineTopic(""),
|
|
3306
|
+
consumed: [],
|
|
3307
|
+
warnings: []
|
|
3308
|
+
};
|
|
3309
|
+
const known = new Set(files.map((f) => f.normalize("NFC")));
|
|
3310
|
+
const resolveFile = (spec) => {
|
|
3311
|
+
const candidates = MARKDOWN_EXT.test(spec) ? [spec] : [`${spec}.md`, `${spec}.markdown`];
|
|
3312
|
+
for (const candidate of candidates) {
|
|
3313
|
+
const path = resolveOutlinePath(tocDir, candidate);
|
|
3314
|
+
if (path !== null && known.has(path.normalize("NFC"))) return path;
|
|
3315
|
+
}
|
|
3316
|
+
return null;
|
|
3317
|
+
};
|
|
3318
|
+
const expandGlob = (pattern) => {
|
|
3319
|
+
const regex = globToRegExp(pattern.replace(/\\/g, "/"));
|
|
3320
|
+
const prefix = tocDir ? `${tocDir}/` : "";
|
|
3321
|
+
return files.filter((file) => {
|
|
3322
|
+
if (!file.startsWith(prefix)) return false;
|
|
3323
|
+
const rel = file.slice(prefix.length);
|
|
3324
|
+
return regex.test(rel) || regex.test(rel.replace(MARKDOWN_EXT, ""));
|
|
3325
|
+
}).sort();
|
|
3326
|
+
};
|
|
3327
|
+
const walkItems = (items, container, where) => {
|
|
3328
|
+
if (items === void 0 || items === null) return;
|
|
3329
|
+
if (!Array.isArray(items)) {
|
|
3330
|
+
outline.warnings.push(`${tocFile}: ${where} must be a list`);
|
|
3331
|
+
return;
|
|
3332
|
+
}
|
|
3333
|
+
for (const item of items) {
|
|
3334
|
+
if (!isRecord(item)) {
|
|
3335
|
+
outline.warnings.push(`${tocFile}: ${where} holds an unrecognized entry`);
|
|
3336
|
+
continue;
|
|
3337
|
+
}
|
|
3338
|
+
const title = typeof item.title === "string" ? item.title.trim() : void 0;
|
|
3339
|
+
if (typeof item.url === "string") continue;
|
|
3340
|
+
if (typeof item.glob === "string") {
|
|
3341
|
+
for (const file2 of expandGlob(item.glob)) container.entries.push({ file: file2 });
|
|
3342
|
+
continue;
|
|
3343
|
+
}
|
|
3344
|
+
if (typeof item.file !== "string") {
|
|
3345
|
+
outline.warnings.push(`${tocFile}: ${where} holds an entry without 'file'`);
|
|
3346
|
+
continue;
|
|
3347
|
+
}
|
|
3348
|
+
const file = resolveFile(item.file);
|
|
3349
|
+
if (file === null) {
|
|
3350
|
+
outline.warnings.push(
|
|
3351
|
+
`${tocFile}: '${item.file}' is not a Markdown file in the tree (notebooks are not supported); skipped`
|
|
3352
|
+
);
|
|
3353
|
+
continue;
|
|
3354
|
+
}
|
|
3355
|
+
const entry = { file, ...title ? { title } : {} };
|
|
3356
|
+
const children = item.sections ?? item.chapters ?? item.entries;
|
|
3357
|
+
if (Array.isArray(children) && children.length > 0) {
|
|
3358
|
+
const topic = newOutlineTopic(title ?? titleFromFolderName(stem(file)));
|
|
3359
|
+
if (!title) topic.titleFromFile = file;
|
|
3360
|
+
topic.entries.push(entry);
|
|
3361
|
+
walkItems(children, topic, `'${item.file}'`);
|
|
3362
|
+
container.children.push(topic);
|
|
3363
|
+
} else {
|
|
3364
|
+
container.entries.push(entry);
|
|
3365
|
+
}
|
|
3366
|
+
}
|
|
3367
|
+
};
|
|
3368
|
+
if (Array.isArray(doc)) {
|
|
3369
|
+
for (const item of doc) {
|
|
3370
|
+
if (isRecord(item) && typeof item.part === "string") {
|
|
3371
|
+
const part = newOutlineTopic(item.part.trim() || "Part");
|
|
3372
|
+
walkItems(item.chapters ?? item.sections, part, `part '${item.part}'`);
|
|
3373
|
+
outline.root.children.push(part);
|
|
3374
|
+
} else {
|
|
3375
|
+
walkItems([item], outline.root, "the list");
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
} else if (isRecord(doc)) {
|
|
3379
|
+
if (typeof doc.root === "string") {
|
|
3380
|
+
const file = resolveFile(doc.root);
|
|
3381
|
+
if (file === null) {
|
|
3382
|
+
outline.warnings.push(
|
|
3383
|
+
`${tocFile}: root '${doc.root}' is not a Markdown file in the tree; skipped`
|
|
3384
|
+
);
|
|
3385
|
+
} else {
|
|
3386
|
+
outline.root.entries.push({ file });
|
|
3387
|
+
}
|
|
3388
|
+
}
|
|
3389
|
+
if (Array.isArray(doc.parts)) {
|
|
3390
|
+
for (const part of doc.parts) {
|
|
3391
|
+
if (!isRecord(part)) continue;
|
|
3392
|
+
const caption = typeof part.caption === "string" ? part.caption.trim() : "";
|
|
3393
|
+
const topic = newOutlineTopic(caption || "Part");
|
|
3394
|
+
walkItems(part.chapters ?? part.sections, topic, `part '${topic.name}'`);
|
|
3395
|
+
outline.root.children.push(topic);
|
|
3396
|
+
}
|
|
3397
|
+
}
|
|
3398
|
+
walkItems(doc.chapters, outline.root, "chapters");
|
|
3399
|
+
walkItems(doc.sections, outline.root, "sections");
|
|
3400
|
+
} else {
|
|
3401
|
+
throw new Error(`${tocFile}: not a mapping or a list`);
|
|
3402
|
+
}
|
|
3403
|
+
assignOutlineOrders(outline.root);
|
|
3404
|
+
return outline;
|
|
3405
|
+
}
|
|
3406
|
+
|
|
3407
|
+
// src/markdown-adapter/load.ts
|
|
1890
3408
|
var ROOT_TOPIC_ID = "general";
|
|
1891
3409
|
var SUMMARY_MAX_CHARS = 280;
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
3410
|
+
var TOPIC_SIDECAR = "_topic.yaml";
|
|
3411
|
+
var HUGO_SECTION_PAGE = "_index";
|
|
3412
|
+
var SECTION_PAGE_ORDINAL = -2147483648;
|
|
3413
|
+
var RESERVED_KEYS = /* @__PURE__ */ new Set(["title", "summary", "aliases", "id", "order", "subcategory"]);
|
|
3414
|
+
function readFrontMatter(raw, file, hugo) {
|
|
3415
|
+
let data;
|
|
3416
|
+
let body;
|
|
3417
|
+
try {
|
|
3418
|
+
({ data, body } = parseMarkdownFrontMatter(raw));
|
|
3419
|
+
} catch (error) {
|
|
3420
|
+
throw new Error(`${file}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3421
|
+
}
|
|
3422
|
+
const out = { meta: {}, body };
|
|
3423
|
+
const str = (key) => {
|
|
3424
|
+
const value = data[key];
|
|
3425
|
+
if (value === void 0 || value === null) return void 0;
|
|
3426
|
+
if (typeof value === "number") return String(value);
|
|
3427
|
+
if (typeof value !== "string")
|
|
3428
|
+
throw new Error(`${file}: front matter '${key}' must be a string`);
|
|
3429
|
+
return value.trim() || void 0;
|
|
3430
|
+
};
|
|
3431
|
+
out.title = str("title");
|
|
3432
|
+
out.summary = str("summary");
|
|
3433
|
+
out.id = str("id");
|
|
3434
|
+
const aliases = data.aliases;
|
|
3435
|
+
if (typeof aliases === "string") {
|
|
3436
|
+
out.aliases = aliases.split(",").map((a) => a.trim()).filter(Boolean);
|
|
3437
|
+
} else if (Array.isArray(aliases)) {
|
|
3438
|
+
out.aliases = aliases.map((a) => String(a).trim()).filter(Boolean);
|
|
3439
|
+
} else if (aliases !== void 0 && aliases !== null) {
|
|
3440
|
+
throw new Error(`${file}: front matter 'aliases' must be a list or a comma-separated string`);
|
|
3441
|
+
}
|
|
3442
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
3443
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
3444
|
+
if (!order.success) throw new Error(`${file}: front matter 'order' must be an int32`);
|
|
3445
|
+
out.order = order.data;
|
|
3446
|
+
}
|
|
3447
|
+
const sub = data.subcategory;
|
|
3448
|
+
if (sub !== void 0 && sub !== null) {
|
|
3449
|
+
if (typeof sub !== "object" || Array.isArray(sub)) {
|
|
3450
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
3451
|
+
}
|
|
3452
|
+
const record = sub;
|
|
3453
|
+
if (typeof record.id !== "string" || !record.id.trim()) {
|
|
3454
|
+
throw new Error(`${file}: front matter 'subcategory' must be a mapping with an id`);
|
|
3455
|
+
}
|
|
3456
|
+
const subcategory = { id: record.id.trim() };
|
|
3457
|
+
if (record.title !== void 0 && record.title !== null) {
|
|
3458
|
+
if (typeof record.title !== "string") {
|
|
3459
|
+
throw new Error(`${file}: front matter 'subcategory.title' must be a string`);
|
|
3460
|
+
}
|
|
3461
|
+
subcategory.title = record.title.trim();
|
|
3462
|
+
}
|
|
3463
|
+
if (record.order !== void 0 && record.order !== null) {
|
|
3464
|
+
const order = KnowledgeOrdinalSchema.safeParse(record.order);
|
|
3465
|
+
if (!order.success)
|
|
3466
|
+
throw new Error(`${file}: front matter 'subcategory.order' must be an int32`);
|
|
3467
|
+
subcategory.order = order.data;
|
|
3468
|
+
}
|
|
3469
|
+
out.subcategory = subcategory;
|
|
3470
|
+
}
|
|
3471
|
+
if (hugo) {
|
|
3472
|
+
if (data.draft === true) out.unpublished = "draft";
|
|
3473
|
+
else if (data.headless === true) out.unpublished = "headless";
|
|
3474
|
+
if (data.weight !== void 0 && data.weight !== null) {
|
|
3475
|
+
const weight = KnowledgeOrdinalSchema.safeParse(data.weight);
|
|
3476
|
+
if (!weight.success) throw new Error(`${file}: front matter 'weight' must be an int32`);
|
|
3477
|
+
if (weight.data > 0) out.weight = weight.data;
|
|
1909
3478
|
}
|
|
3479
|
+
out.description = str("description");
|
|
3480
|
+
}
|
|
3481
|
+
for (const [key, value] of Object.entries(data)) {
|
|
3482
|
+
if (RESERVED_KEYS.has(key) || value === void 0) continue;
|
|
3483
|
+
out.meta[key] = value;
|
|
1910
3484
|
}
|
|
1911
3485
|
return out;
|
|
1912
3486
|
}
|
|
@@ -1940,7 +3514,7 @@ async function walkMarkdownFiles(root) {
|
|
|
1940
3514
|
);
|
|
1941
3515
|
for (const entry of entries) {
|
|
1942
3516
|
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
1943
|
-
const abs =
|
|
3517
|
+
const abs = join6(dir, entry.name);
|
|
1944
3518
|
if (entry.isDirectory()) await visit(abs);
|
|
1945
3519
|
else if (entry.isFile() && /\.(md|markdown)$/i.test(entry.name)) out.push(abs);
|
|
1946
3520
|
}
|
|
@@ -1948,17 +3522,233 @@ async function walkMarkdownFiles(root) {
|
|
|
1948
3522
|
await visit(root);
|
|
1949
3523
|
return out;
|
|
1950
3524
|
}
|
|
1951
|
-
|
|
3525
|
+
function isRecord2(value) {
|
|
3526
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
3527
|
+
}
|
|
3528
|
+
async function readTopicSidecar(dir) {
|
|
3529
|
+
const path = join6(dir, TOPIC_SIDECAR);
|
|
3530
|
+
if (!existsSync(path)) return void 0;
|
|
3531
|
+
const raw = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
3532
|
+
const data = parseYaml(raw, { what: path });
|
|
3533
|
+
if (data === null || data === void 0) return {};
|
|
3534
|
+
if (!isRecord2(data)) throw new Error(`${path}: must be a mapping`);
|
|
3535
|
+
const out = {};
|
|
3536
|
+
if (typeof data.name === "string" && data.name.trim()) out.name = data.name.trim();
|
|
3537
|
+
if (typeof data.description === "string" && data.description.trim()) {
|
|
3538
|
+
out.description = data.description.trim();
|
|
3539
|
+
}
|
|
3540
|
+
if (data.order !== void 0 && data.order !== null) {
|
|
3541
|
+
const order = KnowledgeOrdinalSchema.safeParse(data.order);
|
|
3542
|
+
if (!order.success) throw new Error(`${path}: 'order' must be an int32`);
|
|
3543
|
+
out.order = order.data;
|
|
3544
|
+
}
|
|
3545
|
+
return out;
|
|
3546
|
+
}
|
|
3547
|
+
function applyOverride(topic, override) {
|
|
3548
|
+
if (!topic || !override) return;
|
|
3549
|
+
if (override.name) topic.name = override.name;
|
|
3550
|
+
if (override.description) topic.description = override.description;
|
|
3551
|
+
if (override.order !== void 0) topic.sortKey = topicSortKeyForOrder(override.order);
|
|
3552
|
+
}
|
|
3553
|
+
function insideRoot(rootDir, abs) {
|
|
3554
|
+
const rel = relative3(rootDir, abs).split(sep).join("/");
|
|
3555
|
+
if (rel === ".." || rel.startsWith("../") || isAbsolute3(rel)) return null;
|
|
3556
|
+
return rel;
|
|
3557
|
+
}
|
|
3558
|
+
function firstExisting(candidates) {
|
|
3559
|
+
return candidates.find((candidate) => existsSync(candidate));
|
|
3560
|
+
}
|
|
3561
|
+
async function readMkdocsDocsDir(mkdocsPath) {
|
|
3562
|
+
const config = parseYaml((await readFile3(mkdocsPath, "utf8")).replace(/^\uFEFF/, ""), {
|
|
3563
|
+
what: basename(mkdocsPath),
|
|
3564
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
3565
|
+
tolerateUnknownTags: true,
|
|
3566
|
+
maxAliasCount: 100
|
|
3567
|
+
});
|
|
3568
|
+
const docsDir = isRecord2(config) && typeof config.docs_dir === "string" ? config.docs_dir : "docs";
|
|
3569
|
+
return resolve4(dirname3(mkdocsPath), docsDir);
|
|
3570
|
+
}
|
|
3571
|
+
async function detectTableOfContents(rootDir, projectDir = rootDir) {
|
|
3572
|
+
const summary = join6(rootDir, "SUMMARY.md");
|
|
3573
|
+
if (existsSync(summary)) return { format: "gitbook", path: summary };
|
|
3574
|
+
const jupyter = firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
3575
|
+
if (jupyter) return { format: "jupyter-book", path: jupyter };
|
|
3576
|
+
const mkdocs = firstExisting(
|
|
3577
|
+
[rootDir, projectDir, dirname3(rootDir)].flatMap((dir) => [
|
|
3578
|
+
join6(dir, "mkdocs.yml"),
|
|
3579
|
+
join6(dir, "mkdocs.yaml")
|
|
3580
|
+
])
|
|
3581
|
+
);
|
|
3582
|
+
if (mkdocs) return { format: "mkdocs", path: mkdocs };
|
|
1952
3583
|
const files = await walkMarkdownFiles(rootDir);
|
|
3584
|
+
if (files.some((file) => basename(file).replace(/\.(md|markdown)$/i, "") === HUGO_SECTION_PAGE)) {
|
|
3585
|
+
return { format: "hugo" };
|
|
3586
|
+
}
|
|
3587
|
+
return { format: "folders" };
|
|
3588
|
+
}
|
|
3589
|
+
async function readOutlineText(path) {
|
|
3590
|
+
const text = (await readFile3(path, "utf8")).replace(/^\uFEFF/, "");
|
|
3591
|
+
if (Buffer.byteLength(text, "utf8") > OUTLINE_MAX_BYTES) {
|
|
3592
|
+
throw new Error(`${basename(path)} exceeds ${OUTLINE_MAX_BYTES} bytes`);
|
|
3593
|
+
}
|
|
3594
|
+
return text;
|
|
3595
|
+
}
|
|
3596
|
+
async function loadOutline(rootDir, toc, files) {
|
|
3597
|
+
if (toc.format === "folders" || toc.format === "hugo") return { outline: null };
|
|
3598
|
+
const explicit = toc.path ? isAbsolute3(toc.path) ? toc.path : join6(rootDir, toc.path) : void 0;
|
|
3599
|
+
if (explicit && !existsSync(explicit)) {
|
|
3600
|
+
throw new Error(`${toc.format} table of contents not found: ${explicit}`);
|
|
3601
|
+
}
|
|
3602
|
+
if (toc.format === "gitbook") {
|
|
3603
|
+
const path2 = explicit ?? join6(rootDir, "SUMMARY.md");
|
|
3604
|
+
const rel = insideRoot(rootDir, path2);
|
|
3605
|
+
if (!existsSync(path2)) throw new Error(`gitbook table of contents not found: ${path2}`);
|
|
3606
|
+
if (rel === null)
|
|
3607
|
+
throw new Error(`${basename(path2)} must live inside the content root ${rootDir}`);
|
|
3608
|
+
return { outline: parseGitbookSummary(await readOutlineText(path2), rel), path: path2 };
|
|
3609
|
+
}
|
|
3610
|
+
if (toc.format === "mkdocs") {
|
|
3611
|
+
const path2 = explicit ?? firstExisting(
|
|
3612
|
+
[rootDir, dirname3(rootDir)].flatMap((dir) => [
|
|
3613
|
+
join6(dir, "mkdocs.yml"),
|
|
3614
|
+
join6(dir, "mkdocs.yaml")
|
|
3615
|
+
])
|
|
3616
|
+
);
|
|
3617
|
+
if (!path2) throw new Error(`mkdocs.yml not found in ${rootDir} or its parent`);
|
|
3618
|
+
const config = parseYaml(await readOutlineText(path2), {
|
|
3619
|
+
what: basename(path2),
|
|
3620
|
+
maxBytes: OUTLINE_MAX_BYTES,
|
|
3621
|
+
tolerateUnknownTags: true,
|
|
3622
|
+
maxAliasCount: 100
|
|
3623
|
+
});
|
|
3624
|
+
const docsDir = await readMkdocsDocsDir(path2);
|
|
3625
|
+
const docsRel = insideRoot(rootDir, docsDir);
|
|
3626
|
+
if (docsRel === null) {
|
|
3627
|
+
throw new Error(
|
|
3628
|
+
`${basename(path2)}: docs_dir ${docsDir} is outside the content root ${rootDir}`
|
|
3629
|
+
);
|
|
3630
|
+
}
|
|
3631
|
+
return { outline: parseMkdocsNav(config, docsRel, basename(path2)), path: path2 };
|
|
3632
|
+
}
|
|
3633
|
+
const path = explicit ?? firstExisting([join6(rootDir, "_toc.yml"), join6(rootDir, "_toc.yaml")]);
|
|
3634
|
+
if (!path) throw new Error(`_toc.yml not found in ${rootDir}`);
|
|
3635
|
+
const tocDir = insideRoot(rootDir, dirname3(path));
|
|
3636
|
+
if (tocDir === null)
|
|
3637
|
+
throw new Error(`${basename(path)} must live inside the content root ${rootDir}`);
|
|
3638
|
+
const doc = parseYaml(await readOutlineText(path), {
|
|
3639
|
+
what: basename(path),
|
|
3640
|
+
maxBytes: OUTLINE_MAX_BYTES
|
|
3641
|
+
});
|
|
3642
|
+
return { outline: parseJupyterBookToc(doc, tocDir, files, basename(path)), path };
|
|
3643
|
+
}
|
|
3644
|
+
var INLINE_LINK = /(!?)\[([^\]]*)\]\(\s*(<[^>]*>|[^)\s]+)((?:\s+"[^"]*")?)\s*\)/g;
|
|
3645
|
+
var HAS_SCHEME2 = /^[a-z][a-z0-9+.-]*:/i;
|
|
3646
|
+
function resolveInTree(ctx, target) {
|
|
3647
|
+
const resolved = posix2.normalize(posix2.join(ctx.docDir, target));
|
|
3648
|
+
if (resolved === ".." || resolved.startsWith("../") || posix2.isAbsolute(resolved)) return null;
|
|
3649
|
+
return resolved === "." ? "" : resolved;
|
|
3650
|
+
}
|
|
3651
|
+
function rewriteTarget(ctx, isImage, rawTarget) {
|
|
3652
|
+
const target = rawTarget.startsWith("<") ? rawTarget.slice(1, -1) : rawTarget;
|
|
3653
|
+
if (target === "" || HAS_SCHEME2.test(target) || target.startsWith("//") || target.startsWith("#") || target.startsWith("/")) {
|
|
3654
|
+
return null;
|
|
3655
|
+
}
|
|
3656
|
+
const hashAt = target.search(/[#?]/);
|
|
3657
|
+
const pathPart = hashAt === -1 ? target : target.slice(0, hashAt);
|
|
3658
|
+
const decoded = safeDecode2(pathPart);
|
|
3659
|
+
const resolved = resolveInTree(ctx, decoded);
|
|
3660
|
+
if (resolved === null) {
|
|
3661
|
+
ctx.warn(`${ctx.file}: link '${target}' leaves the catalog tree; left as is`);
|
|
3662
|
+
return null;
|
|
3663
|
+
}
|
|
3664
|
+
if (isImage) {
|
|
3665
|
+
if (!assetExtension3(resolved)) {
|
|
3666
|
+
ctx.warn(`${ctx.file}: image '${target}' is not a supported asset type; left as is`);
|
|
3667
|
+
return null;
|
|
3668
|
+
}
|
|
3669
|
+
const abs = join6(ctx.rootDir, ...resolved.split("/"));
|
|
3670
|
+
if (!existsSync(abs)) {
|
|
3671
|
+
const message = `${ctx.file}: image '${target}' does not exist in the catalog tree`;
|
|
3672
|
+
if (ctx.missingAssets === "error") throw new Error(message);
|
|
3673
|
+
ctx.warn(`${message}; left as is`);
|
|
3674
|
+
return null;
|
|
3675
|
+
}
|
|
3676
|
+
const archivePath = resolved.startsWith("assets/") ? resolved : `assets/${resolved}`;
|
|
3677
|
+
if (!ctx.assets.has(archivePath))
|
|
3678
|
+
ctx.assets.set(archivePath, { path: archivePath, absPath: abs });
|
|
3679
|
+
return archivePath;
|
|
3680
|
+
}
|
|
3681
|
+
if (!/\.(md|markdown)$/i.test(resolved)) return null;
|
|
3682
|
+
const documentId = ctx.idByRel.get(resolved.replace(/\.(md|markdown)$/i, "").normalize("NFC"));
|
|
3683
|
+
if (!documentId) {
|
|
3684
|
+
ctx.warn(`${ctx.file}: link '${target}' does not name a document in the catalog; left as is`);
|
|
3685
|
+
return null;
|
|
3686
|
+
}
|
|
3687
|
+
if (!ctx.uri) return null;
|
|
3688
|
+
return formatKnowledgeUri({
|
|
3689
|
+
publisherId: ctx.uri.publisherId,
|
|
3690
|
+
catalogId: ctx.uri.catalogId,
|
|
3691
|
+
documentId
|
|
3692
|
+
});
|
|
3693
|
+
}
|
|
3694
|
+
function safeDecode2(value) {
|
|
3695
|
+
try {
|
|
3696
|
+
return decodeURIComponent(value);
|
|
3697
|
+
} catch {
|
|
3698
|
+
return value;
|
|
3699
|
+
}
|
|
3700
|
+
}
|
|
3701
|
+
function rewriteLinks(ctx, markdown) {
|
|
3702
|
+
const lines = markdown.split("\n");
|
|
3703
|
+
let fence = null;
|
|
3704
|
+
return lines.map((line) => {
|
|
3705
|
+
const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
|
|
3706
|
+
if (fenceMatch) {
|
|
3707
|
+
const marker = fenceMatch[1];
|
|
3708
|
+
if (fence === null) fence = marker[0];
|
|
3709
|
+
else if (marker[0] === fence) fence = null;
|
|
3710
|
+
return line;
|
|
3711
|
+
}
|
|
3712
|
+
if (fence !== null) return line;
|
|
3713
|
+
return line.replace(
|
|
3714
|
+
INLINE_LINK,
|
|
3715
|
+
(whole, bang, text, target, title) => {
|
|
3716
|
+
const rewritten = rewriteTarget(ctx, bang === "!", target);
|
|
3717
|
+
return rewritten === null ? whole : `${bang}[${text}](${rewritten}${title})`;
|
|
3718
|
+
}
|
|
3719
|
+
);
|
|
3720
|
+
}).join("\n");
|
|
3721
|
+
}
|
|
3722
|
+
async function loadMarkdownCatalog(rootDir, opts) {
|
|
3723
|
+
const warn = opts.onWarning ?? (() => {
|
|
3724
|
+
});
|
|
3725
|
+
const toc = opts.toc ?? { format: "folders" };
|
|
3726
|
+
const hugo = toc.format === "hugo";
|
|
3727
|
+
const ignore = new Set(opts.ignore ?? []);
|
|
3728
|
+
const relOf = (abs) => relative3(rootDir, abs).split(sep).join("/");
|
|
3729
|
+
let files = (await walkMarkdownFiles(rootDir)).filter((abs) => !ignore.has(relOf(abs)));
|
|
3730
|
+
const { outline, path: outlinePath } = await loadOutline(rootDir, toc, files.map(relOf));
|
|
3731
|
+
for (const message of outline?.warnings ?? []) warn(message);
|
|
3732
|
+
if (outline) {
|
|
3733
|
+
const consumed = new Set(outline.consumed.map((file) => file.normalize("NFC")));
|
|
3734
|
+
files = files.filter((abs) => !consumed.has(relOf(abs).normalize("NFC")));
|
|
3735
|
+
}
|
|
1953
3736
|
if (files.length === 0) {
|
|
1954
3737
|
throw new Error(`no Markdown files found under ${rootDir}`);
|
|
1955
3738
|
}
|
|
1956
3739
|
const takenTopicIds = /* @__PURE__ */ new Set();
|
|
1957
3740
|
const topicIdByDir = /* @__PURE__ */ new Map();
|
|
1958
3741
|
const topics = [];
|
|
3742
|
+
const topicById = /* @__PURE__ */ new Map();
|
|
1959
3743
|
const rootTopicId = opts.rootTopicId ?? ROOT_TOPIC_ID;
|
|
1960
3744
|
let rootTopicUsed = false;
|
|
1961
|
-
|
|
3745
|
+
let rootOverride;
|
|
3746
|
+
const addTopic = (topic) => {
|
|
3747
|
+
topics.push(topic);
|
|
3748
|
+
topicById.set(topic.id, topic);
|
|
3749
|
+
return topic;
|
|
3750
|
+
};
|
|
3751
|
+
const ensureTopicChain = async (relDirPosix) => {
|
|
1962
3752
|
if (!relDirPosix) {
|
|
1963
3753
|
rootTopicUsed = true;
|
|
1964
3754
|
return [rootTopicId];
|
|
@@ -1972,32 +3762,141 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
1972
3762
|
if (!id) {
|
|
1973
3763
|
id = topicIdFor(segment, takenTopicIds);
|
|
1974
3764
|
topicIdByDir.set(prefix, id);
|
|
1975
|
-
|
|
3765
|
+
const topic = addTopic({
|
|
1976
3766
|
id,
|
|
1977
|
-
name: segment,
|
|
3767
|
+
name: titleFromFolderName(segment),
|
|
1978
3768
|
...path.length > 0 ? { parentId: path[path.length - 1] } : {}
|
|
1979
3769
|
});
|
|
3770
|
+
applyOverride(topic, await readTopicSidecar(join6(rootDir, ...prefix.split("/"))));
|
|
3771
|
+
applyOverride(topic, opts.topics?.[prefix]);
|
|
1980
3772
|
}
|
|
1981
3773
|
path.push(id);
|
|
1982
3774
|
}
|
|
1983
3775
|
return path;
|
|
1984
3776
|
};
|
|
1985
|
-
const
|
|
1986
|
-
|
|
3777
|
+
const placements = /* @__PURE__ */ new Map();
|
|
3778
|
+
if (outline) {
|
|
3779
|
+
const walk = (topic, chain) => {
|
|
3780
|
+
for (const entry of topic.entries) {
|
|
3781
|
+
const key = entry.file.normalize("NFC");
|
|
3782
|
+
if (placements.has(key)) {
|
|
3783
|
+
warn(
|
|
3784
|
+
`${entry.file}: listed more than once in the table of contents; the first place wins`
|
|
3785
|
+
);
|
|
3786
|
+
} else {
|
|
3787
|
+
placements.set(key, { chain, entry });
|
|
3788
|
+
}
|
|
3789
|
+
}
|
|
3790
|
+
for (const child of topic.children) walk(child, [...chain, child]);
|
|
3791
|
+
};
|
|
3792
|
+
walk(outline.root, []);
|
|
3793
|
+
}
|
|
3794
|
+
const outlineTopicIds = /* @__PURE__ */ new Map();
|
|
3795
|
+
const ensureOutlineChain = (chain) => {
|
|
3796
|
+
if (chain.length === 0) {
|
|
3797
|
+
rootTopicUsed = true;
|
|
3798
|
+
return [rootTopicId];
|
|
3799
|
+
}
|
|
3800
|
+
const ids = [];
|
|
3801
|
+
for (const node of chain) {
|
|
3802
|
+
let id = outlineTopicIds.get(node);
|
|
3803
|
+
if (!id) {
|
|
3804
|
+
id = topicIdFor(node.name, takenTopicIds);
|
|
3805
|
+
outlineTopicIds.set(node, id);
|
|
3806
|
+
const topic = {
|
|
3807
|
+
id,
|
|
3808
|
+
name: node.name,
|
|
3809
|
+
...ids.length > 0 ? { parentId: ids[ids.length - 1] } : {}
|
|
3810
|
+
};
|
|
3811
|
+
if (node.description) topic.description = node.description;
|
|
3812
|
+
if (node.order !== void 0) topic.sortKey = topicSortKeyForOrder(node.order);
|
|
3813
|
+
addTopic(topic);
|
|
3814
|
+
}
|
|
3815
|
+
ids.push(id);
|
|
3816
|
+
}
|
|
3817
|
+
return ids;
|
|
3818
|
+
};
|
|
3819
|
+
const shelves = /* @__PURE__ */ new Map();
|
|
3820
|
+
const ensureShelf = (parentId, sub, file) => {
|
|
3821
|
+
const key = `${parentId}/${sub.id}`;
|
|
3822
|
+
const existing = shelves.get(key);
|
|
3823
|
+
if (existing) {
|
|
3824
|
+
if (existing.title !== sub.title || existing.order !== sub.order) {
|
|
3825
|
+
throw new Error(
|
|
3826
|
+
`${file}: subcategory '${sub.id}' is declared with a different title or order in ${existing.file}`
|
|
3827
|
+
);
|
|
3828
|
+
}
|
|
3829
|
+
return existing.topicId;
|
|
3830
|
+
}
|
|
3831
|
+
const topicId = topicIdFor(`${parentId}-${sub.id}`, takenTopicIds);
|
|
3832
|
+
const topic = { id: topicId, name: sub.title ?? sub.id, parentId };
|
|
3833
|
+
if (sub.order !== void 0) topic.sortKey = topicSortKeyForOrder(sub.order);
|
|
3834
|
+
addTopic(topic);
|
|
3835
|
+
shelves.set(key, { topicId, title: sub.title, order: sub.order, file });
|
|
3836
|
+
return topicId;
|
|
3837
|
+
};
|
|
3838
|
+
const loaded = [];
|
|
3839
|
+
const fileById = /* @__PURE__ */ new Map();
|
|
3840
|
+
const idByRel = /* @__PURE__ */ new Map();
|
|
3841
|
+
const seenRel = /* @__PURE__ */ new Set();
|
|
1987
3842
|
for (const abs of files) {
|
|
1988
|
-
const rel =
|
|
3843
|
+
const rel = relOf(abs);
|
|
1989
3844
|
const relDir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
1990
|
-
const
|
|
1991
|
-
const
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
const
|
|
1999
|
-
|
|
2000
|
-
|
|
3845
|
+
const relNoExt = rel.replace(/\.(md|markdown)$/i, "").normalize("NFC");
|
|
3846
|
+
const raw = (await readFile3(abs, "utf8")).replace(/^\uFEFF/, "");
|
|
3847
|
+
const fm = readFrontMatter(raw, rel, hugo);
|
|
3848
|
+
if (fm.unpublished) {
|
|
3849
|
+
warn(`${rel}: skipped, ${fm.unpublished} page`);
|
|
3850
|
+
continue;
|
|
3851
|
+
}
|
|
3852
|
+
seenRel.add(rel.normalize("NFC"));
|
|
3853
|
+
const id = fm.id ?? relNoExt;
|
|
3854
|
+
const idCheck = KnowledgeDocumentIdSchema.safeParse(id);
|
|
3855
|
+
if (!idCheck.success) throw new Error(`${rel}: front matter 'id' is not a valid document id`);
|
|
3856
|
+
const previous = fileById.get(id);
|
|
3857
|
+
if (previous) throw new Error(`${rel}: document id '${id}' is already used by ${previous}`);
|
|
3858
|
+
fileById.set(id, rel);
|
|
3859
|
+
idByRel.set(relNoExt, id);
|
|
3860
|
+
const stem2 = rel.slice(rel.lastIndexOf("/") + 1).replace(/\.(md|markdown)$/i, "");
|
|
3861
|
+
let title = fm.title ?? firstHeading(fm.body) ?? stem2;
|
|
3862
|
+
let ordinal = fm.order;
|
|
3863
|
+
let topicPath;
|
|
3864
|
+
const placement = placements.get(rel.normalize("NFC"));
|
|
3865
|
+
if (placement) {
|
|
3866
|
+
topicPath = ensureOutlineChain(placement.chain);
|
|
3867
|
+
if (placement.entry.title) title = placement.entry.title;
|
|
3868
|
+
if (placement.entry.order !== void 0) ordinal = placement.entry.order;
|
|
3869
|
+
} else {
|
|
3870
|
+
if (outline)
|
|
3871
|
+
warn(`${rel}: not in the ${outline.format} table of contents; filed under its folder`);
|
|
3872
|
+
topicPath = await ensureTopicChain(relDir);
|
|
3873
|
+
if (hugo && ordinal === void 0) ordinal = fm.weight;
|
|
3874
|
+
}
|
|
3875
|
+
if (hugo && stem2 === HUGO_SECTION_PAGE) {
|
|
3876
|
+
const override = {
|
|
3877
|
+
...fm.title ? { name: fm.title } : {},
|
|
3878
|
+
...fm.description ?? fm.summary ? { description: fm.description ?? fm.summary } : {},
|
|
3879
|
+
...fm.weight !== void 0 ? { order: fm.weight } : {}
|
|
3880
|
+
};
|
|
3881
|
+
if (relDir === "") {
|
|
3882
|
+
rootOverride = override;
|
|
3883
|
+
} else {
|
|
3884
|
+
const leaf = topicById.get(topicPath[topicPath.length - 1]);
|
|
3885
|
+
applyOverride(leaf, override);
|
|
3886
|
+
applyOverride(leaf, opts.topics?.[relDir]);
|
|
3887
|
+
}
|
|
3888
|
+
if (!fm.body.trim()) continue;
|
|
3889
|
+
ordinal = SECTION_PAGE_ORDINAL;
|
|
3890
|
+
}
|
|
3891
|
+
if (fm.subcategory) {
|
|
3892
|
+
const parentId = topicPath[topicPath.length - 1];
|
|
3893
|
+
topicPath = [...topicPath, ensureShelf(parentId, fm.subcategory, rel)];
|
|
3894
|
+
}
|
|
3895
|
+
const summary = fm.summary ?? (hugo ? fm.description : void 0) ?? firstParagraph(fm.body);
|
|
3896
|
+
loaded.push({
|
|
3897
|
+
rel,
|
|
3898
|
+
file: rel,
|
|
3899
|
+
doc: {
|
|
2001
3900
|
id,
|
|
2002
3901
|
title,
|
|
2003
3902
|
slug: documentSlug(title),
|
|
@@ -2005,9 +3904,33 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2005
3904
|
language: opts.language,
|
|
2006
3905
|
topicPath,
|
|
2007
3906
|
markdown: fm.body,
|
|
2008
|
-
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {}
|
|
2009
|
-
|
|
3907
|
+
...fm.aliases && fm.aliases.length > 0 ? { aliases: fm.aliases } : {},
|
|
3908
|
+
...ordinal !== void 0 ? { ordinal } : {},
|
|
3909
|
+
...Object.keys(fm.meta).length > 0 ? { meta: fm.meta } : {}
|
|
3910
|
+
}
|
|
3911
|
+
});
|
|
3912
|
+
}
|
|
3913
|
+
if (outline) {
|
|
3914
|
+
const titleByRel = new Map(
|
|
3915
|
+
loaded.map((entry) => [entry.rel.normalize("NFC"), entry.doc.title])
|
|
2010
3916
|
);
|
|
3917
|
+
const nameFromPages = (topic) => {
|
|
3918
|
+
for (const child of topic.children) {
|
|
3919
|
+
const id = outlineTopicIds.get(child);
|
|
3920
|
+
const title = child.titleFromFile ? titleByRel.get(child.titleFromFile.normalize("NFC")) : void 0;
|
|
3921
|
+
const compiled = id ? topicById.get(id) : void 0;
|
|
3922
|
+
if (compiled && title) compiled.name = title;
|
|
3923
|
+
nameFromPages(child);
|
|
3924
|
+
}
|
|
3925
|
+
};
|
|
3926
|
+
nameFromPages(outline.root);
|
|
3927
|
+
for (const [file, placement] of placements) {
|
|
3928
|
+
if (!seenRel.has(file)) {
|
|
3929
|
+
warn(
|
|
3930
|
+
`${placement.entry.file}: named by the table of contents but not found among the Markdown files`
|
|
3931
|
+
);
|
|
3932
|
+
}
|
|
3933
|
+
}
|
|
2011
3934
|
}
|
|
2012
3935
|
if (rootTopicUsed) {
|
|
2013
3936
|
if (takenTopicIds.has(rootTopicId)) {
|
|
@@ -2015,137 +3938,54 @@ async function loadMarkdownCatalog(rootDir, opts) {
|
|
|
2015
3938
|
`root topic id '${rootTopicId}' collides with a folder-derived topic; pass rootTopicId`
|
|
2016
3939
|
);
|
|
2017
3940
|
}
|
|
2018
|
-
|
|
3941
|
+
const root = { id: rootTopicId, name: opts.rootTopicName ?? "General" };
|
|
3942
|
+
applyOverride(root, await readTopicSidecar(rootDir));
|
|
3943
|
+
applyOverride(root, rootOverride);
|
|
3944
|
+
applyOverride(root, opts.topics?.[""]);
|
|
3945
|
+
topics.unshift(root);
|
|
3946
|
+
topicById.set(root.id, root);
|
|
2019
3947
|
}
|
|
2020
|
-
|
|
3948
|
+
const assets = /* @__PURE__ */ new Map();
|
|
3949
|
+
const documents = [];
|
|
3950
|
+
for (const entry of loaded) {
|
|
3951
|
+
const ctx = {
|
|
3952
|
+
rootDir,
|
|
3953
|
+
docDir: entry.rel.includes("/") ? entry.rel.slice(0, entry.rel.lastIndexOf("/")) : "",
|
|
3954
|
+
file: entry.file,
|
|
3955
|
+
idByRel,
|
|
3956
|
+
uri: opts.uri,
|
|
3957
|
+
assets,
|
|
3958
|
+
missingAssets: opts.missingAssets ?? "error",
|
|
3959
|
+
warn
|
|
3960
|
+
};
|
|
3961
|
+
documents.push(
|
|
3962
|
+
CatalogDocumentSchema.parse({
|
|
3963
|
+
...entry.doc,
|
|
3964
|
+
markdown: rewriteLinks(ctx, entry.doc.markdown)
|
|
3965
|
+
})
|
|
3966
|
+
);
|
|
3967
|
+
}
|
|
3968
|
+
return {
|
|
3969
|
+
topics,
|
|
3970
|
+
documents,
|
|
3971
|
+
assets: [...assets.values()].sort((a, b) => a.path < b.path ? -1 : 1),
|
|
3972
|
+
toc: { format: toc.format, ...outlinePath ? { path: outlinePath } : {} }
|
|
3973
|
+
};
|
|
2021
3974
|
}
|
|
2022
3975
|
|
|
2023
3976
|
// src/registry-client/fetch.ts
|
|
2024
|
-
import {
|
|
2025
|
-
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezel";
|
|
3977
|
+
import { KnowledgeRegistryIndexSchema } from "@bendyline/gezk";
|
|
2026
3978
|
|
|
2027
3979
|
// src/signatures/signing.ts
|
|
2028
3980
|
import {
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
// src/signatures/jcs.ts
|
|
2038
|
-
function canonicalizeJson(value) {
|
|
2039
|
-
return serialize(value, "$");
|
|
2040
|
-
}
|
|
2041
|
-
function serialize(value, path) {
|
|
2042
|
-
if (value === null) return "null";
|
|
2043
|
-
switch (typeof value) {
|
|
2044
|
-
case "boolean":
|
|
2045
|
-
return value ? "true" : "false";
|
|
2046
|
-
case "number":
|
|
2047
|
-
if (!Number.isFinite(value)) throw new Error(`non-finite number at ${path}`);
|
|
2048
|
-
return JSON.stringify(value);
|
|
2049
|
-
case "string":
|
|
2050
|
-
return JSON.stringify(value);
|
|
2051
|
-
case "object":
|
|
2052
|
-
break;
|
|
2053
|
-
default:
|
|
2054
|
-
throw new Error(`value of type ${typeof value} at ${path} has no JSON identity`);
|
|
2055
|
-
}
|
|
2056
|
-
if (Array.isArray(value)) {
|
|
2057
|
-
return `[${value.map((item, i) => serialize(item ?? null, `${path}[${i}]`)).join(",")}]`;
|
|
2058
|
-
}
|
|
2059
|
-
const entries = Object.entries(value).filter(([, v]) => v !== void 0).sort(([a], [b]) => compareUtf16(a, b));
|
|
2060
|
-
return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${serialize(v, `${path}.${k}`)}`).join(",")}}`;
|
|
2061
|
-
}
|
|
2062
|
-
function compareUtf16(a, b) {
|
|
2063
|
-
const n = Math.min(a.length, b.length);
|
|
2064
|
-
for (let i = 0; i < n; i++) {
|
|
2065
|
-
const d = a.charCodeAt(i) - b.charCodeAt(i);
|
|
2066
|
-
if (d !== 0) return d;
|
|
2067
|
-
}
|
|
2068
|
-
return a.length - b.length;
|
|
2069
|
-
}
|
|
2070
|
-
|
|
2071
|
-
// src/signatures/signing.ts
|
|
2072
|
-
function knowledgeKeyId(publicKeyPem) {
|
|
2073
|
-
const der = createPublicKey(publicKeyPem).export({ type: "spki", format: "der" });
|
|
2074
|
-
return createHash5("sha256").update(der).digest("hex").slice(0, 16);
|
|
2075
|
-
}
|
|
2076
|
-
function generateKnowledgeSigningKeyPair() {
|
|
2077
|
-
const { publicKey, privateKey } = generateKeyPairSync("ed25519");
|
|
2078
|
-
const publicKeyPem = publicKey.export({ type: "spki", format: "pem" }).toString();
|
|
2079
|
-
return {
|
|
2080
|
-
publicKeyPem,
|
|
2081
|
-
privateKeyPem: privateKey.export({ type: "pkcs8", format: "pem" }).toString(),
|
|
2082
|
-
keyId: knowledgeKeyId(publicKeyPem)
|
|
2083
|
-
};
|
|
2084
|
-
}
|
|
2085
|
-
function manifestSigningPayload(manifest) {
|
|
2086
|
-
const { signature: _omitted, ...unsigned } = manifest;
|
|
2087
|
-
return Buffer.from(canonicalizeJson(unsigned), "utf8");
|
|
2088
|
-
}
|
|
2089
|
-
function signManifest(manifest, privateKeyPem) {
|
|
2090
|
-
const key = createPrivateKey(privateKeyPem);
|
|
2091
|
-
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2092
|
-
const value = cryptoSign(null, manifestSigningPayload(manifest), key).toString("base64");
|
|
2093
|
-
return {
|
|
2094
|
-
...manifest,
|
|
2095
|
-
signature: {
|
|
2096
|
-
algorithm: "ed25519",
|
|
2097
|
-
keyId: knowledgeKeyId(publicKeyPem),
|
|
2098
|
-
canonicalization: "rfc8785",
|
|
2099
|
-
value
|
|
2100
|
-
}
|
|
2101
|
-
};
|
|
2102
|
-
}
|
|
2103
|
-
function verifyManifestSignature(manifest, anchors) {
|
|
2104
|
-
return verifySignedDocument(manifest, anchors);
|
|
2105
|
-
}
|
|
2106
|
-
function signRegistryIndex(index, privateKeyPem) {
|
|
2107
|
-
const key = createPrivateKey(privateKeyPem);
|
|
2108
|
-
const publicKeyPem = createPublicKey(key).export({ type: "spki", format: "pem" }).toString();
|
|
2109
|
-
const { signature: _omitted, ...unsigned } = index;
|
|
2110
|
-
const value = cryptoSign(null, Buffer.from(canonicalizeJson(unsigned), "utf8"), key).toString(
|
|
2111
|
-
"base64"
|
|
2112
|
-
);
|
|
2113
|
-
return {
|
|
2114
|
-
...index,
|
|
2115
|
-
signature: {
|
|
2116
|
-
algorithm: "ed25519",
|
|
2117
|
-
keyId: knowledgeKeyId(publicKeyPem),
|
|
2118
|
-
canonicalization: "rfc8785",
|
|
2119
|
-
value
|
|
2120
|
-
}
|
|
2121
|
-
};
|
|
2122
|
-
}
|
|
2123
|
-
function verifyRegistryIndex(index, anchors) {
|
|
2124
|
-
return verifySignedDocument(index, anchors);
|
|
2125
|
-
}
|
|
2126
|
-
function verifySignedDocument(document, anchors) {
|
|
2127
|
-
const signature = document.signature;
|
|
2128
|
-
if (!signature) return { ok: false, reason: "unsigned" };
|
|
2129
|
-
const anchor = anchors.find((a) => a.keyId === signature.keyId);
|
|
2130
|
-
if (!anchor) {
|
|
2131
|
-
return { ok: false, reason: "unknown-key", detail: `no trust anchor for ${signature.keyId}` };
|
|
2132
|
-
}
|
|
2133
|
-
try {
|
|
2134
|
-
if (knowledgeKeyId(anchor.publicKeyPem) !== signature.keyId) {
|
|
2135
|
-
return { ok: false, reason: "unknown-key", detail: "anchor keyId does not match its key" };
|
|
2136
|
-
}
|
|
2137
|
-
const { signature: _omitted, ...unsigned } = document;
|
|
2138
|
-
const ok = cryptoVerify(
|
|
2139
|
-
null,
|
|
2140
|
-
Buffer.from(canonicalizeJson(unsigned), "utf8"),
|
|
2141
|
-
createPublicKey(anchor.publicKeyPem),
|
|
2142
|
-
Buffer.from(signature.value, "base64")
|
|
2143
|
-
);
|
|
2144
|
-
return ok ? { ok: true, keyId: signature.keyId } : { ok: false, reason: "bad-signature" };
|
|
2145
|
-
} catch (err) {
|
|
2146
|
-
return { ok: false, reason: "error", detail: err instanceof Error ? err.message : String(err) };
|
|
2147
|
-
}
|
|
2148
|
-
}
|
|
3981
|
+
generateKnowledgeSigningKeyPair,
|
|
3982
|
+
knowledgeKeyId,
|
|
3983
|
+
manifestSigningPayload,
|
|
3984
|
+
signManifest,
|
|
3985
|
+
signRegistryIndex,
|
|
3986
|
+
verifyManifestSignature,
|
|
3987
|
+
verifyRegistryIndex
|
|
3988
|
+
} from "@bendyline/gezk/node";
|
|
2149
3989
|
|
|
2150
3990
|
// src/registry-client/fetch.ts
|
|
2151
3991
|
var DEFAULT_MAX_REGISTRY_BYTES = 4 * 1024 * 1024;
|
|
@@ -2164,7 +4004,7 @@ async function fetchKnowledgeRegistry(url, opts) {
|
|
|
2164
4004
|
let response;
|
|
2165
4005
|
try {
|
|
2166
4006
|
response = await fetchImpl(url, {
|
|
2167
|
-
signal:
|
|
4007
|
+
signal: opts.signal ?? AbortSignal.timeout(opts.timeoutMs ?? DEFAULT_TIMEOUT_MS),
|
|
2168
4008
|
redirect: "follow",
|
|
2169
4009
|
headers: { accept: "application/json" }
|
|
2170
4010
|
});
|
|
@@ -2197,7 +4037,7 @@ async function fetchKnowledgeRegistry(url, opts) {
|
|
|
2197
4037
|
} catch (err) {
|
|
2198
4038
|
throw new KnowledgeRegistryFetchError(
|
|
2199
4039
|
"invalid",
|
|
2200
|
-
`registry is not a valid
|
|
4040
|
+
`registry is not a valid gezk-registry document: ${err instanceof Error ? err.message : String(err)}`
|
|
2201
4041
|
);
|
|
2202
4042
|
}
|
|
2203
4043
|
const verdict = verifyRegistryIndex(registry, opts.anchors);
|
|
@@ -2243,7 +4083,11 @@ function newerRegistryEntries(registry, installed) {
|
|
|
2243
4083
|
|
|
2244
4084
|
// src/signatures/anchors.ts
|
|
2245
4085
|
var BUILTIN_KNOWLEDGE_TRUST_ANCHORS = [];
|
|
4086
|
+
|
|
4087
|
+
// src/signatures/jcs.ts
|
|
4088
|
+
import { canonicalizeJson as canonicalizeJson2 } from "@bendyline/gezk";
|
|
2246
4089
|
export {
|
|
4090
|
+
BGE_SMALL_EN_V15_1,
|
|
2247
4091
|
BODY_CODEC_MIN_BYTES,
|
|
2248
4092
|
BUILTIN_KNOWLEDGE_TRUST_ANCHORS,
|
|
2249
4093
|
CENTROID_CHUNKS_PER,
|
|
@@ -2252,19 +4096,35 @@ export {
|
|
|
2252
4096
|
CENTROID_SAMPLE_MAX,
|
|
2253
4097
|
CatalogHandle,
|
|
2254
4098
|
CatalogOpenError,
|
|
4099
|
+
DuckdbError,
|
|
4100
|
+
EmbedderArtifactError,
|
|
2255
4101
|
EmbedderUnavailableError,
|
|
2256
4102
|
GEZEL_BGE_SMALL_EN_V15_1,
|
|
2257
4103
|
GEZEL_MARKDOWN_CHUNKS_2,
|
|
2258
4104
|
GEZEL_MULTILINGUAL_E5_SMALL_1,
|
|
2259
4105
|
GEZK_APPLICATION_ID,
|
|
2260
4106
|
GEZK_ARCHIVE_LIMITS,
|
|
4107
|
+
GEZK_FORMAT_GENERATIONS,
|
|
2261
4108
|
GEZK_FORMAT_VERSION,
|
|
2262
4109
|
GEZK_INDEX_SCHEMA_VERSION,
|
|
4110
|
+
GEZK_MIME_TYPE,
|
|
4111
|
+
GEZK_SUPPORTED_FORMAT_VERSIONS,
|
|
4112
|
+
GEZK_SUPPORTED_INDEX_SCHEMA_VERSIONS,
|
|
2263
4113
|
GezkArchiveError,
|
|
2264
4114
|
KNOWLEDGE_EMBEDDING_PROFILES,
|
|
4115
|
+
KNOWLEDGE_TOOLCHAIN,
|
|
2265
4116
|
KnowledgeRegistryFetchError,
|
|
4117
|
+
LICENSE_NOTICE_PATH,
|
|
2266
4118
|
MANIFEST_PATH,
|
|
4119
|
+
MARKDOWN_CHUNKS_2,
|
|
2267
4120
|
MAX_KNOWLEDGE_DOCUMENT_BYTES,
|
|
4121
|
+
MAX_KNOWLEDGE_DOCUMENT_META_BYTES,
|
|
4122
|
+
MAX_KNOWLEDGE_TOPIC_DEPTH,
|
|
4123
|
+
MIMETYPE_PATH,
|
|
4124
|
+
MULTILINGUAL_E5_SMALL_1,
|
|
4125
|
+
PARQUET_EXPORT_VERSION,
|
|
4126
|
+
PARQUET_REPORT_PATH,
|
|
4127
|
+
README_PATH,
|
|
2268
4128
|
RERANK_FINAL_K,
|
|
2269
4129
|
ROUTER_DB_PATH,
|
|
2270
4130
|
ROUTER_DDL,
|
|
@@ -2273,37 +4133,58 @@ export {
|
|
|
2273
4133
|
SHARD_DDL,
|
|
2274
4134
|
SHARD_MAX_CHUNKS,
|
|
2275
4135
|
SHARD_TARGET_CHUNKS,
|
|
4136
|
+
SOURCE_NOTICES_PATH,
|
|
2276
4137
|
ZIP_FIXED_MTIME,
|
|
2277
|
-
|
|
4138
|
+
assertDuckdbCli,
|
|
4139
|
+
assetReferences,
|
|
4140
|
+
assignOutlineOrders,
|
|
4141
|
+
canonicalizeJson2 as canonicalizeJson,
|
|
2278
4142
|
chunkContentHash,
|
|
2279
4143
|
chunkMarkdownProfile,
|
|
2280
4144
|
chunkUid,
|
|
2281
4145
|
compareCatalogVersions,
|
|
2282
4146
|
compileKnowledgeCatalog,
|
|
2283
4147
|
createProfileEmbedder,
|
|
4148
|
+
detectTableOfContents,
|
|
2284
4149
|
documentSlug,
|
|
4150
|
+
duckdbVersion,
|
|
4151
|
+
exportCatalogParquet,
|
|
2285
4152
|
extractGezkVerified,
|
|
2286
4153
|
fetchKnowledgeRegistry,
|
|
4154
|
+
findDuckdbBinary,
|
|
2287
4155
|
findRegistryEntry,
|
|
2288
4156
|
generateKnowledgeSigningKeyPair,
|
|
4157
|
+
hammingTopK,
|
|
2289
4158
|
inspectGezkArchive,
|
|
4159
|
+
isSupportedFormatVersion,
|
|
4160
|
+
isSupportedIndexSchemaVersion,
|
|
2290
4161
|
knowledgeEmbeddingProfile,
|
|
2291
4162
|
knowledgeKeyId,
|
|
2292
4163
|
l2Normalize,
|
|
2293
4164
|
loadMarkdownCatalog,
|
|
2294
4165
|
manifestSigningPayload,
|
|
4166
|
+
newOutlineTopic,
|
|
2295
4167
|
newerRegistryEntries,
|
|
2296
4168
|
openCatalogDatabase,
|
|
4169
|
+
parseGitbookSummary,
|
|
4170
|
+
parseJupyterBookToc,
|
|
4171
|
+
parseMkdocsNav,
|
|
2297
4172
|
quantizeBinary,
|
|
2298
4173
|
quantizeInt8,
|
|
2299
4174
|
readGezkManifest,
|
|
4175
|
+
readMkdocsDocsDir,
|
|
2300
4176
|
rerankK,
|
|
2301
4177
|
rerankScore,
|
|
4178
|
+
resolveOutlinePath,
|
|
4179
|
+
resolveTransformersModelOptions,
|
|
4180
|
+
runDuckdbScript,
|
|
2302
4181
|
signManifest,
|
|
2303
4182
|
signRegistryIndex,
|
|
4183
|
+
titleFromFolderName,
|
|
4184
|
+
transformersCachePath,
|
|
2304
4185
|
validateExtractedCatalog,
|
|
2305
|
-
vecChunksDdl,
|
|
2306
4186
|
verifyManifestSignature,
|
|
4187
|
+
verifyProfileArtifacts,
|
|
2307
4188
|
verifyRegistryIndex,
|
|
2308
4189
|
writeGezkArchive
|
|
2309
4190
|
};
|